diff options
| -rw-r--r-- | README.md | 44 | ||||
| -rw-r--r-- | _selfcheck.lua | 59 | ||||
| -rw-r--r-- | panto-web-scm-1.rockspec | 29 | ||||
| -rw-r--r-- | panto-web/init.lua | 530 |
4 files changed, 662 insertions, 0 deletions
diff --git a/README.md b/README.md new file mode 100644 index 0000000..ef4a1db --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# panto-web + +The `web` extension for [pantograph](https://github.com/travisp/pantograph)'s +`panto` CLI: keyless web access packaged as a luarocks rock. + +## Tools + +- **`web.fetch`** — fetches a public HTTP(S) URL with `curl`, follows up to + five public redirects, and converts HTML to compact readable text. Blocked + and JavaScript-only pages get one keyless retry through Jina Reader. +- **`web.search`** — searches through Exa's public MCP endpoint without + configuration or an API key, returning compact titles, URLs, and excerpts + for `web.fetch` to read. + +## Install + +Via the rock (once published): + +```toml +[extensions] +rocks = ["panto-web"] +``` + +Or as a local checkout: + +```toml +[extensions] +paths = ["/path/to/panto-web"] +``` + +The extension can be disabled with the usual policy globs, e.g. +`deny = ["web"]`. + +## Development + +Run the self-check (needs `panto` for its embedded Lua + luv): + +```sh +panto lua _selfcheck.lua +``` + +The source lives under `panto-web/` so `require` works identically installed +and from a checkout; `_selfcheck.lua` is `_`-prefixed so panto's directory +scan skips it. diff --git a/_selfcheck.lua b/_selfcheck.lua new file mode 100644 index 0000000..b7f35be --- /dev/null +++ b/_selfcheck.lua @@ -0,0 +1,59 @@ +-- Self-check for panto-web. Run with: +-- +-- panto lua panto-web/_selfcheck.lua + +local registered = {} +package.loaded["panto"] = { + ext = { + register_tool = function(tool) registered[#registered + 1] = tool end, + json = { + decode = function() return {} end, + encode = function() return "{}" end, + }, + }, +} + +local here = debug.getinfo(1, "S").source:match("^@(.*)/[^/]+$") or "." +package.path = here .. "/panto-web/?.lua;" .. here .. "/panto-web/?/init.lua;" .. package.path + +local web = require("init") +assert(web.name == "web", "entry name") + +local parsed = assert(web.parse_url("https://example.com:8443/docs?q=1")) +assert(parsed.host == "example.com" and parsed.port == 8443, "URL host and port") +local _, local_err = web.parse_url("http://user:pass@example.com/") +assert(local_err:find("credentials", 1, true), "URL credentials rejected") +assert(web.is_public_address("8.8.8.8"), "public IPv4 allowed") +assert(not web.is_public_address("127.0.0.1"), "loopback IPv4 blocked") +assert(not web.is_public_address("::1"), "loopback IPv6 blocked") +assert(not web.is_public_address("::127.0.0.1"), "IPv4-compatible loopback blocked") + +local page, title = web.html_to_text([[<html><head><title>A & B</title> +<style>hidden</style></head><body><h1>Heading</h1><p>Hello <world>.</p> +<script>also hidden</script><a href="https://example.com/x">Read</a></body></html>]]) +assert(title == "A & B", "HTML title decoded") +assert(page:find("Heading", 1, true) and page:find("Hello <world>.", 1, true), "HTML text retained") +assert(page:find("Read (https://example.com/x)", 1, true), "link destination retained") +assert(not page:find("hidden", 1, true), "non-content HTML removed") + +local search_text = web.format_search([[Title: First result +URL: https://example.com/one +Highlights: +Useful first excerpt. +... +--- + +Title: Second result +URL: https://example.com/two +Highlights: +Useful second excerpt.]], "example query") +assert(search_text:find("1. First result", 1, true), "first search result") +assert(search_text:find("URL: https://example.com/two", 1, true), "second search URL") + +web.activate() +assert(registered[1].name == "web.fetch", "fetch tool registered") +assert(registered[2].name == "web.search", "search tool registered") +assert(web.on_fetch({ url = "file:///etc/passwd" }):find("only http", 1, true), "fetch rejects file URLs") +assert(web.on_search({ query = "" }):find("non-empty", 1, true), "search rejects empty query") + +print("panto-web selfcheck: OK") diff --git a/panto-web-scm-1.rockspec b/panto-web-scm-1.rockspec new file mode 100644 index 0000000..5441c9d --- /dev/null +++ b/panto-web-scm-1.rockspec @@ -0,0 +1,29 @@ +rockspec_format = "3.0" +package = "panto-web" +version = "scm-1" + +source = { + -- Placeholder until the extension moves to its own repository. + url = "git+https://example.invalid/panto-web.git", +} + +description = { + summary = "Web fetch and search tools for pantograph", + detailed = [[ +Keyless web access for the panto CLI: web.fetch returns readable text from +public HTTP(S) URLs and web.search searches Exa's public MCP endpoint. +Load by adding "panto-web" to `extensions.rocks` in panto's config.toml. +]], + license = "MIT", +} + +dependencies = { + "lua >= 5.1", +} + +build = { + type = "builtin", + modules = { + ["panto-web"] = "panto-web/init.lua", + }, +} diff --git a/panto-web/init.lua b/panto-web/init.lua new file mode 100644 index 0000000..2e52b4d --- /dev/null +++ b/panto-web/init.lua @@ -0,0 +1,530 @@ +-- Keyless web access: readable page fetches and Exa's public MCP search. + +local uv = require("luv") +local panto = require("panto") + +local M = { name = "web" } + +local MAX_DOWNLOAD = 2 * 1024 * 1024 +local MAX_RESULT = 50 * 1024 +local META = "__PANTO_WEB_META__" + +local function await(label, arm) + local co = assert(coroutine.running(), label .. ": await must run inside a coroutine") + local remaining = 3 + arm(function() + remaining = remaining - 1 + if remaining == 0 then + local ok, err = coroutine.resume(co) + if not ok then error(err, 0) end + end + end) + coroutine.yield() +end + +local function curl(args) + local stdout = uv.new_pipe(false) + local stderr = uv.new_pipe(false) + local out, errors = {}, {} + local out_bytes, error_bytes = 0, 0 + local exit_code, process + local too_large = false + local signal + + process = uv.spawn("curl", { args = args, stdio = { nil, stdout, stderr } }, function(code) + exit_code = code + if process and not process:is_closing() then process:close() end + signal() + end) + if not process then + stdout:close() + stderr:close() + return nil, "curl is not installed or could not be started" + end + + await("curl", function(done) + signal = done + stdout:read_start(function(err, data) + if err then errors[#errors + 1] = tostring(err) end + if data then + local room = MAX_DOWNLOAD + 4096 - out_bytes + if room > 0 then + local chunk = data:sub(1, room) + out[#out + 1] = chunk + out_bytes = out_bytes + #chunk + end + if #data > room and not too_large then + too_large = true + if not process:is_closing() then process:kill("sigterm") end + end + else + done() + end + end) + stderr:read_start(function(err, data) + if err then errors[#errors + 1] = tostring(err) end + if data then + local room = 8192 - error_bytes + if room > 0 then + local chunk = data:sub(1, room) + errors[#errors + 1] = chunk + error_bytes = error_bytes + #chunk + end + else + done() + end + end) + end) + + if not stdout:is_closing() then stdout:close() end + if not stderr:is_closing() then stderr:close() end + if too_large then return nil, "response exceeded 2 MB" end + if exit_code ~= 0 then + local message = table.concat(errors):gsub("%s+$", "") + return nil, message ~= "" and message or "curl exited with code " .. tostring(exit_code) + end + return table.concat(out) +end + +local function curl_args() + return { + "--silent", "--show-error", "--compressed", + "--connect-timeout", "10", "--max-time", "30", + "--max-filesize", tostring(MAX_DOWNLOAD), + "--proto", "=http,https", "--proto-redir", "=http,https", + } +end + +local function add(args, ...) + for i = 1, select("#", ...) do args[#args + 1] = select(i, ...) end +end + +local function split_response(raw) + local at, from = nil, 1 + while true do + local found = raw:find(META, from, true) + if not found then break end + at, from = found, found + #META + end + if not at then return nil, "curl returned no response metadata" end + local body = raw:sub(1, at - 1):gsub("\n$", "") + local status, content_type, redirect = raw:sub(at + #META):match("^(%d%d%d)\t([^\t]*)\t(.-)%s*$") + if not status then return nil, "curl returned malformed response metadata" end + return { + body = body, + status = tonumber(status), + content_type = content_type, + redirect = redirect, + } +end + +local function parse_ipv4(address) + local parts = {} + for part in address:gmatch("[^.]+") do parts[#parts + 1] = tonumber(part) end + if #parts ~= 4 or address:find("[^%d%.]") then return nil end + for _, part in ipairs(parts) do + if not part or part < 0 or part > 255 or part % 1 ~= 0 then return nil end + end + return parts +end + +local function parse_ipv6(address) + local ipv4 = address:match("([^:]+%.[^:]+)$") + if ipv4 then + local octets = parse_ipv4(ipv4) + if not octets then return nil end + address = address:sub(1, #address - #ipv4) + .. string.format("%x:%x", octets[1] * 256 + octets[2], octets[3] * 256 + octets[4]) + end + local compression = address:find("::", 1, true) + if compression and address:find("::", compression + 2, true) then return nil end + local left, right = address:match("^(.-)::(.-)$") + if address:find("::", 1, true) and not left then return nil end + local groups = {} + local function append(side) + if side == "" then return true end + for part in side:gmatch("[^:]+") do + if not part:match("^[%da-fA-F]+$") or #part > 4 then return false end + groups[#groups + 1] = tonumber(part, 16) + end + return true + end + if left then + if not append(left) then return nil end + local left_count = #groups + local tail = {} + for part in right:gmatch("[^:]+") do + if not part:match("^[%da-fA-F]+$") or #part > 4 then return nil end + tail[#tail + 1] = tonumber(part, 16) + end + local missing = 8 - left_count - #tail + if missing < 1 then return nil end + for _ = 1, missing do groups[#groups + 1] = 0 end + for _, part in ipairs(tail) do groups[#groups + 1] = part end + elseif not append(address) or #groups ~= 8 then + return nil + end + return #groups == 8 and groups or nil +end + +function M.is_public_address(address) + local v4 = parse_ipv4(address) + if v4 then + local a, b, c = v4[1], v4[2], v4[3] + return not (a == 0 or a == 10 or a == 127 + or (a == 100 and b >= 64 and b <= 127) + or (a == 169 and b == 254) + or (a == 172 and b >= 16 and b <= 31) + or (a == 192 and b == 0 and c == 0) + or (a == 192 and b == 0 and c == 2) + or (a == 192 and b == 168) + or (a == 198 and (b == 18 or b == 19 or (b == 51 and c == 100))) + or (a == 203 and b == 0 and c == 113) + or a >= 224) + end + + local v6 = parse_ipv6(address) + if not v6 then return false end + if v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0 + and v6[5] == 0 and (v6[6] == 0 or v6[6] == 0xffff) then + return M.is_public_address(string.format("%d.%d.%d.%d", + v6[7] >> 8, v6[7] & 255, v6[8] >> 8, v6[8] & 255)) + end + local all_zero = true + for _, part in ipairs(v6) do if part ~= 0 then all_zero = false break end end + return not (all_zero + or (v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0 + and v6[5] == 0 and v6[6] == 0 and v6[7] == 0 and v6[8] == 1) + or (v6[1] & 0xfe00) == 0xfc00 + or (v6[1] & 0xffc0) == 0xfe80 + or (v6[1] & 0xff00) == 0xff00 + or (v6[1] == 0x2001 and v6[2] == 0x0db8)) +end + +function M.parse_url(url) + if type(url) ~= "string" or url == "" then return nil, "url must be a non-empty string" end + if #url > 4096 then return nil, "url is too long" end + if url:find("%c") then return nil, "url contains control characters" end + local scheme, authority = url:match("^([%a][%w+.-]*)://([^/?#]+)") + scheme = scheme and scheme:lower() + if scheme ~= "http" and scheme ~= "https" then return nil, "only http and https URLs are supported" end + if authority:find("@", 1, true) then return nil, "URLs with credentials are not supported" end + + local host, port + if authority:sub(1, 1) == "[" then + host, port = authority:match("^%[([^]]+)%]:?(%d*)$") + if not host or not parse_ipv6(host) then return nil, "invalid IPv6 URL host" end + else + host, port = authority:match("^(.-):(%d+)$") + if not host then host, port = authority, "" end + if host:find(":", 1, true) or not host:match("^[%w.-]+$") then + return nil, "invalid URL host" + end + end + port = port ~= "" and tonumber(port) or (scheme == "https" and 443 or 80) + if not port or port < 1 or port > 65535 then return nil, "invalid URL port" end + local normalized = host:lower():gsub("%.$", "") + if normalized == "" or normalized == "localhost" or normalized:match("%.localhost$") then + return nil, "internal host is not allowed" + end + return { host = normalized, resolve_host = host:lower(), port = port } +end + +local function resolve_url(url) + local parsed, err = M.parse_url(url) + if not parsed then return nil, err end + if parse_ipv4(parsed.host) or parse_ipv6(parsed.host) then + if not M.is_public_address(parsed.host) then return nil, "internal address is not allowed" end + parsed.address = parsed.host + return parsed + end + + local dns_err, addresses + local co = assert(coroutine.running(), "DNS lookup requires a coroutine") + uv.getaddrinfo(parsed.host, tostring(parsed.port), { socktype = "stream" }, function(e, found) + dns_err, addresses = e, found + local ok, resume_err = coroutine.resume(co) + if not ok then error(resume_err, 0) end + end) + coroutine.yield() + if dns_err or not addresses or #addresses == 0 then + return nil, "failed to resolve " .. parsed.host .. ": " .. tostring(dns_err or "no addresses") + end + for _, item in ipairs(addresses) do + local address = item.addr or item.address + if not address or not M.is_public_address(address) then + return nil, "internal address resolved for " .. parsed.host .. ": " .. tostring(address) + end + parsed.address = parsed.address or address + end + return parsed +end + +local redirect_status = { [301] = true, [302] = true, [303] = true, [307] = true, [308] = true } + +local function request_once(url, resolved) + local args = curl_args() + local address = resolved.address:find(":", 1, true) and "[" .. resolved.address .. "]" or resolved.address + add(args, + "--resolve", resolved.resolve_host .. ":" .. resolved.port .. ":" .. address, + "--user-agent", "panto-web/web", + "--header", "Accept: text/html,application/xhtml+xml,text/plain,application/json,application/xml;q=0.9,*/*;q=0.1", + "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}", + "--url", url) + local raw, err = curl(args) + if not raw then return nil, err end + return split_response(raw) +end + +local function fetch_http(url) + local current = url + for redirects = 0, 5 do + local resolved, err = resolve_url(current) + if not resolved then return nil, err end + local response + response, err = request_once(current, resolved) + if not response then return nil, err end + if redirect_status[response.status] and response.redirect ~= "" then + if redirects == 5 then return nil, "too many redirects" end + current = response.redirect + else + response.url = current + return response + end + end +end + +local entities = { + amp = "&", lt = "<", gt = ">", quot = '"', apos = "'", nbsp = " ", + ndash = "–", mdash = "—", hellip = "…", middot = "·", copy = "©", reg = "®", +} + +local function codepoint(n) + n = tonumber(n) + if not n or n < 1 or n > 0x10ffff or (n >= 0xd800 and n <= 0xdfff) then return "�" end + return utf8.char(n) +end + +function M.html_to_text(html) + local title = html:match("<[Tt][Ii][Tt][Ll][Ee][^>]*>(.-)</[Tt][Ii][Tt][Ll][Ee]%s*>") + local text = html + for _, tag in ipairs({ "script", "style", "noscript", "svg", "head" }) do + local letters = tag:gsub(".", function(c) return "[" .. c:lower() .. c:upper() .. "]" end) + text = text:gsub("<" .. letters .. "[^>]*>.-</" .. letters .. "%s*>", " ") + end + text = text:gsub("<!%-%-.-%-%->", " ") + text = text:gsub('<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*"([^"]+)"[^>]*>(.-)</[Aa]%s*>', + function(href, label) return label .. " (" .. href .. ")" end) + text = text:gsub("<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*'([^']+)'[^>]*>(.-)</[Aa]%s*>", + function(href, label) return label .. " (" .. href .. ")" end) + text = text:gsub("<[Bb][Rr]%s*/?%s*>", "\n") + text = text:gsub("<[Ll][Ii][^>]*>", "\n- ") + text = text:gsub("</[Pp]%s*>", "\n\n") + text = text:gsub("</[Dd][Ii][Vv]%s*>", "\n") + text = text:gsub("</[Hh][1-6]%s*>", "\n\n") + text = text:gsub("</[Tt][Rr]%s*>", "\n") + text = text:gsub("</[Tt][DdHh]%s*>", "\t") + text = text:gsub("<[^>]+>", " ") + text = text:gsub("&#[xX]([%da-fA-F]+);", function(n) return codepoint(tonumber(n, 16)) end) + text = text:gsub("&#(%d+);", codepoint) + text = text:gsub("&([%a]+);", function(name) return entities[name] or "&" .. name .. ";" end) + text = text:gsub("\r", ""):gsub("[\t\f\v ]+", " ") + text = text:gsub(" *\n *", "\n"):gsub("\n\n\n+", "\n\n") + text = text:match("^%s*(.-)%s*$") or "" + if title then + title = M.html_to_text(title) + if title == "" then title = nil end + end + return text, title +end + +local function trim_result(text) + if #text <= MAX_RESULT then return text end + return text:sub(1, MAX_RESULT) .. "\n\n[truncated after 50 KB]" +end + +local function is_html(content_type, body) + content_type = content_type:lower() + return content_type:find("text/html", 1, true) + or content_type:find("application/xhtml+xml", 1, true) + or body:match("^%s*<[!%a]") ~= nil +end + +local function needs_reader(response, text) + local lower = text:lower() + return response.status >= 400 or #text < 200 + or lower:find("please enable javascript", 1, true) + or lower:find("sorry, you have been blocked", 1, true) + or lower:find("please enable cookies", 1, true) + or lower:find("just a moment", 1, true) +end + +local function jina(url) + local response, err = fetch_http("https://r.jina.ai/" .. url) + if not response or response.status < 200 or response.status >= 300 then return nil, err end + local content = response.body:match("Markdown Content:%s*(.*)") + if not content or #content < 100 then return nil end + return content:match("^%s*(.-)%s*$") +end + +function M.on_fetch(input) + local url = type(input) == "table" and input.url or nil + local parsed, validation_error = M.parse_url(url) + if not parsed then return "Error: " .. validation_error end + local response, err = fetch_http(url) + if not response then return "Error fetching " .. url .. ": " .. err end + + local text, title, via_reader + if is_html(response.content_type, response.body) then + text, title = M.html_to_text(response.body) + if needs_reader(response, text) then + local readable = jina(response.url) + if readable then text, title, via_reader = readable, nil, true end + end + elseif response.content_type:lower():find("^text/") + or response.content_type:lower():find("json", 1, true) + or response.content_type:lower():find("xml", 1, true) then + text = response.body + else + return "Error: unsupported content type: " + .. (response.content_type ~= "" and response.content_type or "unknown") + end + + local lines = { "URL: " .. response.url } + if title then lines[#lines + 1] = "Title: " .. title:sub(1, 200) end + if via_reader then + lines[#lines + 1] = "Reader: Jina" + elseif response.status < 200 or response.status >= 300 then + lines[#lines + 1] = "HTTP: " .. response.status + end + lines[#lines + 1] = "" + lines[#lines + 1] = text + return trim_result(table.concat(lines, "\n")) +end + +local function register_fetch() + panto.ext.register_tool { + name = "web.fetch", + description = "Fetch an HTTP(S) URL and return readable text instead of raw HTML. " + .. "Follows validated public redirects and may retry blocked or JavaScript-only pages through Jina Reader.", + schema = { + type = "object", + properties = { + url = { type = "string", description = "Public HTTP(S) URL to fetch." }, + }, + required = { "url" }, + }, + handler = M.on_fetch, + } +end + +local function format_search(text, query) + local results, current, collecting = {}, nil, false + local function finish() + if not current or not current.url then return end + current.snippet = table.concat(current.parts, " "):gsub("%s+", " ") + :gsub("%.%.%.", ""):match("^%s*(.-)%s*$") or "" + if #current.snippet > 1000 then current.snippet = current.snippet:sub(1, 1000) .. "…" end + results[#results + 1] = current + end + for line in (text:gsub("\r", "") .. "\n"):gmatch("(.-)\n") do + local value = line:match("^Title:%s*(.*)$") + if value then + finish() + current, collecting = { title = value, parts = {} }, false + elseif current then + value = line:match("^URL:%s*(.*)$") + if value then + current.url = value + elseif line == "Highlights:" or line:match("^Text:%s*") then + collecting = true + local first = line:match("^Text:%s*(.+)$") + if first then current.parts[#current.parts + 1] = first end + elseif collecting and line ~= "..." and line ~= "---" then + current.parts[#current.parts + 1] = line + end + end + end + finish() + if #results == 0 then return trim_result(text) end + local out = { "Search results for: " .. query, "" } + for i, result in ipairs(results) do + out[#out + 1] = i .. ". " .. (result.title ~= "" and result.title or result.url) + out[#out + 1] = "URL: " .. result.url + if result.snippet ~= "" then out[#out + 1] = result.snippet end + out[#out + 1] = "" + end + return trim_result(table.concat(out, "\n"):gsub("%s+$", "")) +end + +function M.on_search(input) + local query = type(input) == "table" and input.query or nil + if type(query) ~= "string" or query:match("^%s*$") then return "Error: query must be a non-empty string" end + if #query > 2000 then return "Error: query is too long" end + local count = input.num_results or 5 + if type(count) ~= "number" or count % 1 ~= 0 or count < 1 or count > 10 then + return "Error: num_results must be an integer from 1 to 10" + end + + local args = curl_args() + add(args, + "--header", "Content-Type: application/json", + "--header", "Accept: application/json, text/event-stream", + "--header", "x-exa-source: panto-web", + "--data-binary", panto.ext.json.encode({ + jsonrpc = "2.0", id = 1, method = "tools/call", + params = { name = "web_search_exa", arguments = { query = query, numResults = count } }, + }), + "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}", + "--url", "https://mcp.exa.ai/mcp?tools=web_search_exa") + local raw, err = curl(args) + if not raw then return "Error searching the web: " .. err end + local response + response, err = split_response(raw) + if not response then return "Error searching the web: " .. err end + if response.status < 200 or response.status >= 300 then + return "Error searching the web: Exa MCP returned HTTP " .. response.status + end + + local rpc + for payload in response.body:gmatch("data:%s*([^\r\n]+)") do + local ok, value = pcall(panto.ext.json.decode, payload) + if ok and type(value) == "table" and (value.result or value.error) then rpc = value break end + end + if not rpc then + local ok, value = pcall(panto.ext.json.decode, response.body) + if ok then rpc = value end + end + if type(rpc) ~= "table" then return "Error searching the web: Exa MCP returned malformed JSON" end + if rpc.error then return "Error searching the web: " .. tostring(rpc.error.message or "Exa MCP error") end + local content = rpc.result and rpc.result.content + if rpc.result and rpc.result.isError then + return "Error searching the web: " .. tostring(content and content[1] and content[1].text or "Exa MCP error") + end + local text = content and content[1] and content[1].text + if type(text) ~= "string" or text == "" then return "No search results." end + return format_search(text, query) +end + +function M.activate() + register_fetch() + panto.ext.register_tool { + name = "web.search", + description = "Search the web without an API key through Exa's public MCP endpoint. " + .. "Returns titles, URLs, and short excerpts; use web.fetch to read a result.", + schema = { + type = "object", + properties = { + query = { type = "string", description = "Web search query." }, + num_results = { type = "integer", minimum = 1, maximum = 10, + description = "Number of results (default 5)." }, + }, + required = { "query" }, + }, + handler = M.on_search, + } +end + +M.format_search = format_search + +return M |