diff options
| -rw-r--r-- | README.md | 36 | ||||
| -rw-r--r-- | _selfcheck.lua | 93 | ||||
| -rw-r--r-- | panto-web-scm-1.rockspec | 8 | ||||
| -rw-r--r-- | panto-web/init.lua | 763 |
4 files changed, 415 insertions, 485 deletions
@@ -1,16 +1,35 @@ # panto-web The `web` extension for [pantograph](https://github.com/travisp/pantograph)'s -`panto` CLI: keyless web access packaged as a luarocks rock. +`panto` CLI, backed by a persistent [Lightpanda](https://lightpanda.io/) MCP +subprocess. ## Tools -- **`web.fetch`** — fetches a public HTTP(S) URL with `curl`, follows up to - five public redirects, and converts HTML to compact readable text. Blocked - and JavaScript-only pages get one keyless retry through Jina Reader. -- **`web.search`** — searches through Exa's public MCP endpoint without - configuration or an API key, returning compact titles, URLs, and excerpts - for `web.fetch` to read. +- **`web.fetch`** — renders a URL in Lightpanda and returns its DOM-derived + CommonMark, including JavaScript-rendered content. +- **`web.search`** — searches through Exa's keyless MCP HTTP endpoint by + default. If `EXA_API_KEY` is set, it uses Exa's REST Search API instead. + +Tool output is capped at 100 KB. + +## Requirements + +Install `lightpanda` and ensure it is on `PATH`. The extension does not install +or fall back to another browser. See [Lightpanda installation](https://lightpanda.io/docs/run-locally/installation/one-liner), +or install the nightly with Lightpanda's Homebrew tap: + +```sh +brew tap lightpanda-io/browser +brew install lightpanda +``` + +An `EXA_API_KEY` is optional. When set, `web.search` uses the authenticated +Exa REST API instead of the rate-limited keyless endpoint: + +```sh +export EXA_API_KEY=... +``` ## Install @@ -33,7 +52,8 @@ The extension can be disabled with the usual policy globs, e.g. ## Development -Run the self-check (needs `panto` for its embedded Lua + luv): +The self-check requires panto's embedded Lua and `luv`, but does not start +Lightpanda or access the network: ```sh panto lua _selfcheck.lua diff --git a/_selfcheck.lua b/_selfcheck.lua index b7f35be..7c779d0 100644 --- a/_selfcheck.lua +++ b/_selfcheck.lua @@ -1,15 +1,11 @@ --- Self-check for panto-web. Run with: --- --- panto lua panto-web/_selfcheck.lua +-- Self-check for panto-web. Run with: panto lua _selfcheck.lua local registered = {} +local json = require("panto").ext.json package.loaded["panto"] = { ext = { register_tool = function(tool) registered[#registered + 1] = tool end, - json = { - decode = function() return {} end, - encode = function() return "{}" end, - }, + json = json, }, } @@ -19,41 +15,60 @@ package.path = here .. "/panto-web/?.lua;" .. here .. "/panto-web/?/init.lua;" . local web = require("init") assert(web.name == "web", "entry name") -local parsed = assert(web.parse_url("https://example.com:8443/docs?q=1")) -assert(parsed.host == "example.com" and parsed.port == 8443, "URL host and port") -local _, local_err = web.parse_url("http://user:pass@example.com/") -assert(local_err:find("credentials", 1, true), "URL credentials rejected") -assert(web.is_public_address("8.8.8.8"), "public IPv4 allowed") -assert(not web.is_public_address("127.0.0.1"), "loopback IPv4 blocked") -assert(not web.is_public_address("::1"), "loopback IPv6 blocked") -assert(not web.is_public_address("::127.0.0.1"), "IPv4-compatible loopback blocked") - -local page, title = web.html_to_text([[<html><head><title>A & B</title> -<style>hidden</style></head><body><h1>Heading</h1><p>Hello <world>.</p> -<script>also hidden</script><a href="https://example.com/x">Read</a></body></html>]]) -assert(title == "A & B", "HTML title decoded") -assert(page:find("Heading", 1, true) and page:find("Hello <world>.", 1, true), "HTML text retained") -assert(page:find("Read (https://example.com/x)", 1, true), "link destination retained") -assert(not page:find("hidden", 1, true), "non-content HTML removed") - -local search_text = web.format_search([[Title: First result -URL: https://example.com/one -Highlights: -Useful first excerpt. -... ---- - -Title: Second result -URL: https://example.com/two -Highlights: -Useful second excerpt.]], "example query") -assert(search_text:find("1. First result", 1, true), "first search result") -assert(search_text:find("URL: https://example.com/two", 1, true), "second search URL") - web.activate() assert(registered[1].name == "web.fetch", "fetch tool registered") assert(registered[2].name == "web.search", "search tool registered") -assert(web.on_fetch({ url = "file:///etc/passwd" }):find("only http", 1, true), "fetch rejects file URLs") +assert(web.on_fetch({}):find("non-empty", 1, true), "fetch rejects missing URL") +assert(web.on_fetch({ url = "" }):find("non-empty", 1, true), "fetch rejects empty URL") assert(web.on_search({ query = "" }):find("non-empty", 1, true), "search rejects empty query") +assert(web.on_search({ query = "ok", num_results = 0 }):find("integer", 1, true), "search rejects count") + +local markdown = web.build_request(7, "tools/call", + web.tool_params("markdown", { url = "https://example.com" })) +assert(markdown.id == 7 and markdown.params.name == "markdown", "markdown MCP request") +local evaluate = web.build_request(8, "tools/call", + web.tool_params("evaluate", { script = "1 + 1" })) +assert(evaluate.id == 8 and evaluate.params.name == "evaluate", "evaluate MCP request") + +-- Replies must resume only the coroutine waiting for their id. +local resumed = {} +local fake = { pending = {}, buf = "" } +for _, id in ipairs({ 1, 2 }) do + local co = coroutine.create(function() + coroutine.yield() + resumed[#resumed + 1] = id + end) + assert(coroutine.resume(co)) + fake.pending[id] = { co = co } +end +assert(not web.feed(fake, json.encode({ jsonrpc = "2.0", id = 2, result = {} }) .. "\n")) +assert(#resumed == 1 and resumed[1] == 2 and fake.pending[1], "response routed by id") +assert(not web.feed(fake, json.encode({ jsonrpc = "2.0", id = 1, result = {} }) .. "\n")) +assert(#resumed == 2 and resumed[2] == 1, "second response routed by id") + +local body = json.decode(web.exa_body("Lightpanda", 3)) +assert(body.query == "Lightpanda" and body.numResults == 3 and body.type == "auto", "Exa request fields") +assert(body.contents.highlights == true, "Exa highlights requested") +local mcp_body = json.decode(web.exa_mcp_body("Lightpanda", 3)) +assert(mcp_body.params.name == "web_search_exa", "keyless Exa MCP tool") +assert(mcp_body.params.arguments.query == "Lightpanda" and mcp_body.params.arguments.numResults == 3, + "keyless Exa MCP arguments") +local mcp_payload = json.encode({ jsonrpc = "2.0", id = 1, result = { + content = { { type = "text", text = "Title: Result" } }, isError = false, +} }) +assert(web.exa_mcp_result("event: message\ndata: " .. mcp_payload .. "\n") == "Title: Result", + "keyless Exa SSE result") +local fake_key = "fake-secret-key" +assert(not web.exa_body("Lightpanda", 3):find(fake_key, 1, true), "key absent from Exa body") +assert(not web.scrub("failed with " .. fake_key, fake_key):find(fake_key, 1, true), "key redacted from output") +local oversized = web.truncate(string.rep("x", 101 * 1024)) +assert(#oversized < 101 * 1024 and oversized:find("truncated", 1, true), "output capped") + +local text = assert(web.result_text({ content = { + { type = "text", text = "first" }, { type = "text", text = "second" }, +} })) +assert(text == "first\nsecond", "MCP text extracted") +local _, tool_err = web.result_text({ isError = true, content = { { type = "text", text = "failed" } } }) +assert(tool_err == "failed", "MCP tool error extracted") print("panto-web selfcheck: OK") diff --git a/panto-web-scm-1.rockspec b/panto-web-scm-1.rockspec index 221b2cf..e98cc2d 100644 --- a/panto-web-scm-1.rockspec +++ b/panto-web-scm-1.rockspec @@ -8,10 +8,12 @@ source = { } description = { - summary = "Web fetch and search tools for pantograph", + summary = "Lightpanda-backed web fetch and Exa search tools for pantograph", detailed = [[ -Keyless web access for the panto CLI: web.fetch returns readable text from -public HTTP(S) URLs and web.search searches Exa's public MCP endpoint. +Web access for the panto CLI: web.fetch returns JavaScript-rendered Markdown +through a Lightpanda MCP subprocess, and web.search uses Exa's keyless MCP HTTP +endpoint or its REST API when EXA_API_KEY is configured. +Lightpanda must be on PATH. Load by adding "panto-web" to `extensions.rocks` in panto's config.toml. ]], license = "MIT", diff --git a/panto-web/init.lua b/panto-web/init.lua index 2e52b4d..637ee27 100644 --- a/panto-web/init.lua +++ b/panto-web/init.lua @@ -1,522 +1,417 @@ --- Keyless web access: readable page fetches and Exa's public MCP search. +-- Web tools backed by a Lightpanda MCP subprocess. +-- +-- web.fetch -> Lightpanda's `markdown` tool (renders JavaScript, returns CommonMark). +-- web.search -> Exa search, using its REST API when configured and keyless MCP otherwise. +-- +-- `lightpanda` must be on PATH; installing it is the user's responsibility. local uv = require("luv") local panto = require("panto") +local json = panto.ext.json local M = { name = "web" } -local MAX_DOWNLOAD = 2 * 1024 * 1024 -local MAX_RESULT = 50 * 1024 -local META = "__PANTO_WEB_META__" +-- Single documented cap so a huge page cannot flood the model's context. +local MAX_OUTPUT = 100 * 1024 -local function await(label, arm) - local co = assert(coroutine.running(), label .. ": await must run inside a coroutine") - local remaining = 3 - arm(function() - remaining = remaining - 1 - if remaining == 0 then - local ok, err = coroutine.resume(co) - if not ok then error(err, 0) end - end - end) - coroutine.yield() -end - -local function curl(args) - local stdout = uv.new_pipe(false) - local stderr = uv.new_pipe(false) - local out, errors = {}, {} - local out_bytes, error_bytes = 0, 0 - local exit_code, process - local too_large = false - local signal - - process = uv.spawn("curl", { args = args, stdio = { nil, stdout, stderr } }, function(code) - exit_code = code - if process and not process:is_closing() then process:close() end - signal() - end) - if not process then - stdout:close() - stderr:close() - return nil, "curl is not installed or could not be started" - end - - await("curl", function(done) - signal = done - stdout:read_start(function(err, data) - if err then errors[#errors + 1] = tostring(err) end - if data then - local room = MAX_DOWNLOAD + 4096 - out_bytes - if room > 0 then - local chunk = data:sub(1, room) - out[#out + 1] = chunk - out_bytes = out_bytes + #chunk - end - if #data > room and not too_large then - too_large = true - if not process:is_closing() then process:kill("sigterm") end - end - else - done() - end - end) - stderr:read_start(function(err, data) - if err then errors[#errors + 1] = tostring(err) end - if data then - local room = 8192 - error_bytes - if room > 0 then - local chunk = data:sub(1, room) - errors[#errors + 1] = chunk - error_bytes = error_bytes + #chunk - end - else - done() - end - end) - end) +local INSTALL_HINT = "lightpanda is not on PATH. Install it " + .. "(https://lightpanda.io/docs/run-locally/installation/one-liner or " + .. "`brew tap lightpanda-io/browser && brew install lightpanda`) " + .. "so `lightpanda mcp` can start." - if not stdout:is_closing() then stdout:close() end - if not stderr:is_closing() then stderr:close() end - if too_large then return nil, "response exceeded 2 MB" end - if exit_code ~= 0 then - local message = table.concat(errors):gsub("%s+$", "") - return nil, message ~= "" and message or "curl exited with code " .. tostring(exit_code) - end - return table.concat(out) +function M.truncate(text) + if #text <= MAX_OUTPUT then return text end + return text:sub(1, MAX_OUTPUT) .. "\n\n[truncated at 100 KB]" end -local function curl_args() - return { - "--silent", "--show-error", "--compressed", - "--connect-timeout", "10", "--max-time", "30", - "--max-filesize", tostring(MAX_DOWNLOAD), - "--proto", "=http,https", "--proto-redir", "=http,https", - } +-- Redact a secret from anything that may reach tool output. +function M.scrub(text, secret) + if type(text) ~= "string" or type(secret) ~= "string" or secret == "" then return text end + return (text:gsub((secret:gsub("%W", "%%%0")), "[redacted]")) end -local function add(args, ...) - for i = 1, select("#", ...) do args[#args + 1] = select(i, ...) end -end +-------------------------------------------------------------------------------- +-- Lightpanda MCP client (only the subset this extension needs) +-------------------------------------------------------------------------------- -local function split_response(raw) - local at, from = nil, 1 - while true do - local found = raw:find(META, from, true) - if not found then break end - at, from = found, found + #META - end - if not at then return nil, "curl returned no response metadata" end - local body = raw:sub(1, at - 1):gsub("\n$", "") - local status, content_type, redirect = raw:sub(at + #META):match("^(%d%d%d)\t([^\t]*)\t(.-)%s*$") - if not status then return nil, "curl returned malformed response metadata" end - return { - body = body, - status = tonumber(status), - content_type = content_type, - redirect = redirect, - } +-- One lazily started `lightpanda mcp` process per extension lifetime. Idle +-- handles are unreferenced so panto's `uv.run("default")` can return after a +-- tool batch while the browser session remains available for later calls. +local client + +function M.build_request(id, method, params) + return { jsonrpc = "2.0", id = id, method = method, params = params } end -local function parse_ipv4(address) - local parts = {} - for part in address:gmatch("[^.]+") do parts[#parts + 1] = tonumber(part) end - if #parts ~= 4 or address:find("[^%d%.]") then return nil end - for _, part in ipairs(parts) do - if not part or part < 0 or part > 255 or part % 1 ~= 0 then return nil end +-- Resolve a pending request exactly once, whatever the outcome. +local function settle(req, message, err) + if req.done then return end + req.done, req.message, req.err = true, message, err + if req.co and coroutine.status(req.co) == "suspended" then + local ok, resume_err = coroutine.resume(req.co) + if not ok then error(resume_err, 0) end end - return parts end -local function parse_ipv6(address) - local ipv4 = address:match("([^:]+%.[^:]+)$") - if ipv4 then - local octets = parse_ipv4(ipv4) - if not octets then return nil end - address = address:sub(1, #address - #ipv4) - .. string.format("%x:%x", octets[1] * 256 + octets[2], octets[3] * 256 + octets[4]) +local function teardown(c, reason) + if c.dead then return end + c.dead = true + if client == c then client = nil end + local pending = c.pending + c.pending = {} + for _, req in pairs(pending) do settle(req, nil, reason) end + local waiters = c.ready_waiters or {} + c.ready_waiters = {} + for _, req in ipairs(waiters) do settle(req, nil, reason) end + for _, pipe in ipairs({ c.stdin, c.stdout, c.stderr }) do + if pipe and not pipe:is_closing() then pipe:close() end end - local compression = address:find("::", 1, true) - if compression and address:find("::", compression + 2, true) then return nil end - local left, right = address:match("^(.-)::(.-)$") - if address:find("::", 1, true) and not left then return nil end - local groups = {} - local function append(side) - if side == "" then return true end - for part in side:gmatch("[^:]+") do - if not part:match("^[%da-fA-F]+$") or #part > 4 then return false end - groups[#groups + 1] = tonumber(part, 16) - end - return true + if c.proc and not c.proc:is_closing() then + if not c.exited then pcall(c.proc.kill, c.proc, "sigterm") end + c.proc:close() end - if left then - if not append(left) then return nil end - local left_count = #groups - local tail = {} - for part in right:gmatch("[^:]+") do - if not part:match("^[%da-fA-F]+$") or #part > 4 then return nil end - tail[#tail + 1] = tonumber(part, 16) - end - local missing = 8 - left_count - #tail - if missing < 1 then return nil end - for _ = 1, missing do groups[#groups + 1] = 0 end - for _, part in ipairs(tail) do groups[#groups + 1] = part end - elseif not append(address) or #groups ~= 8 then - return nil - end - return #groups == 8 and groups or nil end -function M.is_public_address(address) - local v4 = parse_ipv4(address) - if v4 then - local a, b, c = v4[1], v4[2], v4[3] - return not (a == 0 or a == 10 or a == 127 - or (a == 100 and b >= 64 and b <= 127) - or (a == 169 and b == 254) - or (a == 172 and b >= 16 and b <= 31) - or (a == 192 and b == 0 and c == 0) - or (a == 192 and b == 0 and c == 2) - or (a == 192 and b == 168) - or (a == 198 and (b == 18 or b == 19 or (b == 51 and c == 100))) - or (a == 203 and b == 0 and c == 113) - or a >= 224) +-- Consume newline-delimited stdout and route each reply to its waiting coroutine. +-- Returns an error string when the stream is unusable. +function M.feed(c, chunk) + c.buf = (c.buf or "") .. chunk + while true do + local line, rest = c.buf:match("^([^\n]*)\n(.*)$") + if not line then return nil end + c.buf = rest + if line:match("%S") then + local ok, message = pcall(json.decode, line) + if not ok or type(message) ~= "table" then + return "lightpanda mcp sent a malformed JSON-RPC message" + end + if message.jsonrpc ~= "2.0" then + return "lightpanda mcp sent a malformed JSON-RPC message" + end + local req = message.id ~= nil and c.pending[message.id] + if message.id ~= nil then + if message.result == nil and message.error == nil then + return "lightpanda mcp sent a malformed JSON-RPC response" + end + if req then + c.pending[message.id] = nil + settle(req, message) + end + elseif type(message.method) ~= "string" then + return "lightpanda mcp sent a malformed JSON-RPC message" + end + end end +end - local v6 = parse_ipv6(address) - if not v6 then return false end - if v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0 - and v6[5] == 0 and (v6[6] == 0 or v6[6] == 0xffff) then - return M.is_public_address(string.format("%d.%d.%d.%d", - v6[7] >> 8, v6[7] & 255, v6[8] >> 8, v6[8] & 255)) +local function write(c, payload) + local ok, req, immediate_err = pcall(c.stdin.write, c.stdin, payload, function(err) + if err then teardown(c, "lightpanda mcp write failed: " .. tostring(err)) end + end) + if not ok or not req then + teardown(c, "lightpanda mcp write failed: " .. tostring(immediate_err or req)) end - local all_zero = true - for _, part in ipairs(v6) do if part ~= 0 then all_zero = false break end end - return not (all_zero - or (v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0 - and v6[5] == 0 and v6[6] == 0 and v6[7] == 0 and v6[8] == 1) - or (v6[1] & 0xfe00) == 0xfc00 - or (v6[1] & 0xffc0) == 0xfe80 - or (v6[1] & 0xff00) == 0xff00 - or (v6[1] == 0x2001 and v6[2] == 0x0db8)) end -function M.parse_url(url) - if type(url) ~= "string" or url == "" then return nil, "url must be a non-empty string" end - if #url > 4096 then return nil, "url is too long" end - if url:find("%c") then return nil, "url contains control characters" end - local scheme, authority = url:match("^([%a][%w+.-]*)://([^/?#]+)") - scheme = scheme and scheme:lower() - if scheme ~= "http" and scheme ~= "https" then return nil, "only http and https URLs are supported" end - if authority:find("@", 1, true) then return nil, "URLs with credentials are not supported" end +local function request(c, method, params) + local co, is_main = coroutine.running() + if not co or is_main then return nil, "web tools must run inside a coroutine" end - local host, port - if authority:sub(1, 1) == "[" then - host, port = authority:match("^%[([^]]+)%]:?(%d*)$") - if not host or not parse_ipv6(host) then return nil, "invalid IPv6 URL host" end - else - host, port = authority:match("^(.-):(%d+)$") - if not host then host, port = authority, "" end - if host:find(":", 1, true) or not host:match("^[%w.-]+$") then - return nil, "invalid URL host" - end - end - port = port ~= "" and tonumber(port) or (scheme == "https" and 443 or 80) - if not port or port < 1 or port > 65535 then return nil, "invalid URL port" end - local normalized = host:lower():gsub("%.$", "") - if normalized == "" or normalized == "localhost" or normalized:match("%.localhost$") then - return nil, "internal host is not allowed" + c.next_id = c.next_id + 1 + local id = c.next_id + local req = { co = co } + c.pending[id] = req + write(c, json.encode(M.build_request(id, method, params)) .. "\n") + if not req.done then coroutine.yield() end + c.pending[id] = nil + + if not req.message then return nil, req.err or "lightpanda mcp did not respond" end + if req.message.error then + local rpc_error = req.message.error + return nil, type(rpc_error) == "table" + and tostring(rpc_error.message or "lightpanda mcp returned an error") + or "lightpanda mcp returned an error" end - return { host = normalized, resolve_host = host:lower(), port = port } + return req.message end -local function resolve_url(url) - local parsed, err = M.parse_url(url) - if not parsed then return nil, err end - if parse_ipv4(parsed.host) or parse_ipv6(parsed.host) then - if not M.is_public_address(parsed.host) then return nil, "internal address is not allowed" end - parsed.address = parsed.host - return parsed - end +local function start() + local c = { pending = {}, ready_waiters = {}, next_id = 0, buf = "" } + c.stdin, c.stdout, c.stderr = uv.new_pipe(false), uv.new_pipe(false), uv.new_pipe(false) - local dns_err, addresses - local co = assert(coroutine.running(), "DNS lookup requires a coroutine") - uv.getaddrinfo(parsed.host, tostring(parsed.port), { socktype = "stream" }, function(e, found) - dns_err, addresses = e, found - local ok, resume_err = coroutine.resume(co) - if not ok then error(resume_err, 0) end + local proc = uv.spawn("lightpanda", { + args = { "mcp" }, + stdio = { c.stdin, c.stdout, c.stderr }, + }, function(code) + c.exited = true + teardown(c, "lightpanda mcp exited (code " .. tostring(code) .. ")") end) - coroutine.yield() - if dns_err or not addresses or #addresses == 0 then - return nil, "failed to resolve " .. parsed.host .. ": " .. tostring(dns_err or "no addresses") + if not proc then + teardown(c, INSTALL_HINT) + return nil, INSTALL_HINT end - for _, item in ipairs(addresses) do - local address = item.addr or item.address - if not address or not M.is_public_address(address) then - return nil, "internal address resolved for " .. parsed.host .. ": " .. tostring(address) - end - parsed.address = parsed.address or address - end - return parsed -end + c.proc = proc + client = c -local redirect_status = { [301] = true, [302] = true, [303] = true, [307] = true, [308] = true } - -local function request_once(url, resolved) - local args = curl_args() - local address = resolved.address:find(":", 1, true) and "[" .. resolved.address .. "]" or resolved.address - add(args, - "--resolve", resolved.resolve_host .. ":" .. resolved.port .. ":" .. address, - "--user-agent", "panto-web/web", - "--header", "Accept: text/html,application/xhtml+xml,text/plain,application/json,application/xml;q=0.9,*/*;q=0.1", - "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}", - "--url", url) - local raw, err = curl(args) - if not raw then return nil, err end - return split_response(raw) -end + c.stdout:read_start(function(err, data) + if err then return teardown(c, "lightpanda mcp read error: " .. tostring(err)) end + if not data then return teardown(c, "lightpanda mcp closed its output") end + local feed_err = M.feed(c, data) + if feed_err then teardown(c, feed_err) end + end) + c.stderr:read_start(function() end) -local function fetch_http(url) - local current = url - for redirects = 0, 5 do - local resolved, err = resolve_url(current) - if not resolved then return nil, err end - local response - response, err = request_once(current, resolved) - if not response then return nil, err end - if redirect_status[response.status] and response.redirect ~= "" then - if redirects == 5 then return nil, "too many redirects" end - current = response.redirect - else - response.url = current - return response - end + local _, err = request(c, "initialize", { + protocolVersion = "2024-11-05", + capabilities = {}, + clientInfo = { name = "panto-web", version = "scm" }, + }) + if err then + teardown(c, err) + return nil, err end + write(c, json.encode(M.build_request(nil, "notifications/initialized", {})) .. "\n") + c.ready = true + for _, waiter in ipairs(c.ready_waiters) do settle(waiter, true) end + c.ready_waiters = {} + return c end -local entities = { - amp = "&", lt = "<", gt = ">", quot = '"', apos = "'", nbsp = " ", - ndash = "–", mdash = "—", hellip = "…", middot = "·", copy = "©", reg = "®", -} +local function ready_client() + if not client or client.dead then return start() end + if client.ready then return client end -local function codepoint(n) - n = tonumber(n) - if not n or n < 1 or n > 0x10ffff or (n >= 0xd800 and n <= 0xdfff) then return "�" end - return utf8.char(n) + local co, is_main = coroutine.running() + if not co or is_main then return nil, "web tools must run inside a coroutine" end + local c = client + local waiter = { co = co } + c.ready_waiters[#c.ready_waiters + 1] = waiter + coroutine.yield() + if waiter.err then return nil, waiter.err end + if c.dead or not c.ready then return nil, "lightpanda mcp failed to initialize" end + return c end -function M.html_to_text(html) - local title = html:match("<[Tt][Ii][Tt][Ll][Ee][^>]*>(.-)</[Tt][Ii][Tt][Ll][Ee]%s*>") - local text = html - for _, tag in ipairs({ "script", "style", "noscript", "svg", "head" }) do - local letters = tag:gsub(".", function(c) return "[" .. c:lower() .. c:upper() .. "]" end) - text = text:gsub("<" .. letters .. "[^>]*>.-</" .. letters .. "%s*>", " ") +-- Flatten an MCP tool result into text. +function M.result_text(result) + if type(result) ~= "table" then return nil, "lightpanda mcp returned no result" end + if result.content ~= nil and type(result.content) ~= "table" then + return nil, "lightpanda mcp returned malformed tool content" + end + local parts = {} + for _, item in ipairs(result.content or {}) do + if type(item) == "table" and type(item.text) == "string" then parts[#parts + 1] = item.text end end - text = text:gsub("<!%-%-.-%-%->", " ") - text = text:gsub('<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*"([^"]+)"[^>]*>(.-)</[Aa]%s*>', - function(href, label) return label .. " (" .. href .. ")" end) - text = text:gsub("<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*'([^']+)'[^>]*>(.-)</[Aa]%s*>", - function(href, label) return label .. " (" .. href .. ")" end) - text = text:gsub("<[Bb][Rr]%s*/?%s*>", "\n") - text = text:gsub("<[Ll][Ii][^>]*>", "\n- ") - text = text:gsub("</[Pp]%s*>", "\n\n") - text = text:gsub("</[Dd][Ii][Vv]%s*>", "\n") - text = text:gsub("</[Hh][1-6]%s*>", "\n\n") - text = text:gsub("</[Tt][Rr]%s*>", "\n") - text = text:gsub("</[Tt][DdHh]%s*>", "\t") - text = text:gsub("<[^>]+>", " ") - text = text:gsub("&#[xX]([%da-fA-F]+);", function(n) return codepoint(tonumber(n, 16)) end) - text = text:gsub("&#(%d+);", codepoint) - text = text:gsub("&([%a]+);", function(name) return entities[name] or "&" .. name .. ";" end) - text = text:gsub("\r", ""):gsub("[\t\f\v ]+", " ") - text = text:gsub(" *\n *", "\n"):gsub("\n\n\n+", "\n\n") - text = text:match("^%s*(.-)%s*$") or "" - if title then - title = M.html_to_text(title) - if title == "" then title = nil end + local text = table.concat(parts, "\n") + if result.isError then + return nil, text ~= "" and text or "lightpanda mcp reported a tool error" end - return text, title + return text end -local function trim_result(text) - if #text <= MAX_RESULT then return text end - return text:sub(1, MAX_RESULT) .. "\n\n[truncated after 50 KB]" +local function set_referenced(c, referenced) + local method = referenced and "ref" or "unref" + for _, handle in ipairs({ c.stdin, c.stdout, c.stderr, c.proc }) do + if handle and not handle:is_closing() then handle[method](handle) end + end end -local function is_html(content_type, body) - content_type = content_type:lower() - return content_type:find("text/html", 1, true) - or content_type:find("application/xhtml+xml", 1, true) - or body:match("^%s*<[!%a]") ~= nil +function M.tool_params(tool, arguments) + return { name = tool, arguments = arguments } end -local function needs_reader(response, text) - local lower = text:lower() - return response.status >= 400 or #text < 200 - or lower:find("please enable javascript", 1, true) - or lower:find("sorry, you have been blocked", 1, true) - or lower:find("please enable cookies", 1, true) - or lower:find("just a moment", 1, true) +local function call(tool, arguments) + local c, ready_err = ready_client() + if not c then return nil, ready_err end + set_referenced(c, true) + local message, err = request(c, "tools/call", M.tool_params(tool, arguments)) + if not message then + if not c.dead and next(c.pending) == nil then set_referenced(c, false) end + return nil, err + end + local text, result_err = M.result_text(message.result) + if not c.dead and next(c.pending) == nil then set_referenced(c, false) end + return text, result_err end -local function jina(url) - local response, err = fetch_http("https://r.jina.ai/" .. url) - if not response or response.status < 200 or response.status >= 300 then return nil, err end - local content = response.body:match("Markdown Content:%s*(.*)") - if not content or #content < 100 then return nil end - return content:match("^%s*(.-)%s*$") -end +-------------------------------------------------------------------------------- +-- Tools +-------------------------------------------------------------------------------- function M.on_fetch(input) local url = type(input) == "table" and input.url or nil - local parsed, validation_error = M.parse_url(url) - if not parsed then return "Error: " .. validation_error end - local response, err = fetch_http(url) - if not response then return "Error fetching " .. url .. ": " .. err end - - local text, title, via_reader - if is_html(response.content_type, response.body) then - text, title = M.html_to_text(response.body) - if needs_reader(response, text) then - local readable = jina(response.url) - if readable then text, title, via_reader = readable, nil, true end - end - elseif response.content_type:lower():find("^text/") - or response.content_type:lower():find("json", 1, true) - or response.content_type:lower():find("xml", 1, true) then - text = response.body - else - return "Error: unsupported content type: " - .. (response.content_type ~= "" and response.content_type or "unknown") + if type(url) ~= "string" or url:match("^%s*$") then + return "Error: url must be a non-empty string" end - local lines = { "URL: " .. response.url } - if title then lines[#lines + 1] = "Title: " .. title:sub(1, 200) end - if via_reader then - lines[#lines + 1] = "Reader: Jina" - elseif response.status < 200 or response.status >= 300 then - lines[#lines + 1] = "HTTP: " .. response.status - end - lines[#lines + 1] = "" - lines[#lines + 1] = text - return trim_result(table.concat(lines, "\n")) + local markdown, err = call("markdown", { url = url }) + if not markdown then return "Error fetching " .. url .. ": " .. err end + if markdown:match("^%s*$") then return "URL: " .. url .. "\n\n[no content]" end + return M.truncate("URL: " .. url .. "\n\n" .. (markdown:gsub("^%s+", ""))) end -local function register_fetch() - panto.ext.register_tool { - name = "web.fetch", - description = "Fetch an HTTP(S) URL and return readable text instead of raw HTML. " - .. "Follows validated public redirects and may retry blocked or JavaScript-only pages through Jina Reader.", - schema = { - type = "object", - properties = { - url = { type = "string", description = "Public HTTP(S) URL to fetch." }, - }, - required = { "url" }, +-- Exa request body; deliberately free of credentials. +function M.exa_body(query, count) + return json.encode({ + query = query, + numResults = count, + type = "auto", + contents = { highlights = true }, + }) +end + +-- Page-side transport: POST to Exa and hand back a status/body envelope. The key +-- arrives as a JSON literal over the MCP stdin pipe, never a shell argument. +function M.exa_script(body, key) + return ("const r = await fetch(%s, { method: 'POST', headers:" + .. " { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + %s }," + .. " body: %s }); return { status: r.status, body: await r.text() };") + :format(json.encode("https://api.exa.ai/search"), json.encode(key), json.encode(body)) +end + +function M.exa_mcp_body(query, count) + return json.encode({ + jsonrpc = "2.0", + id = 1, + method = "tools/call", + params = { + name = "web_search_exa", + arguments = { query = query, numResults = count }, }, - handler = M.on_fetch, - } + }) end -local function format_search(text, query) - local results, current, collecting = {}, nil, false - local function finish() - if not current or not current.url then return end - current.snippet = table.concat(current.parts, " "):gsub("%s+", " ") - :gsub("%.%.%.", ""):match("^%s*(.-)%s*$") or "" - if #current.snippet > 1000 then current.snippet = current.snippet:sub(1, 1000) .. "…" end - results[#results + 1] = current - end - for line in (text:gsub("\r", "") .. "\n"):gmatch("(.-)\n") do - local value = line:match("^Title:%s*(.*)$") - if value then - finish() - current, collecting = { title = value, parts = {} }, false - elseif current then - value = line:match("^URL:%s*(.*)$") - if value then - current.url = value - elseif line == "Highlights:" or line:match("^Text:%s*") then - collecting = true - local first = line:match("^Text:%s*(.+)$") - if first then current.parts[#current.parts + 1] = first end - elseif collecting and line ~= "..." and line ~= "---" then - current.parts[#current.parts + 1] = line - end +function M.exa_mcp_script(body) + return ("const r = await fetch(%s, { method: 'POST', headers:" + .. " { 'Content-Type': 'application/json', 'Accept': 'application/json, text/event-stream'," + .. " 'x-exa-source': 'panto-web' }, body: %s });" + .. " return { status: r.status, body: await r.text() };") + :format(json.encode("https://mcp.exa.ai/mcp?tools=web_search_exa"), json.encode(body)) +end + +function M.exa_mcp_result(body) + local rpc + for payload in body:gmatch("data:%s*([^\r\n]+)") do + local ok, candidate = pcall(json.decode, payload) + if ok and type(candidate) == "table" and (candidate.result or candidate.error) then + rpc = candidate + break end end - finish() - if #results == 0 then return trim_result(text) end - local out = { "Search results for: " .. query, "" } - for i, result in ipairs(results) do - out[#out + 1] = i .. ". " .. (result.title ~= "" and result.title or result.url) - out[#out + 1] = "URL: " .. result.url - if result.snippet ~= "" then out[#out + 1] = result.snippet end - out[#out + 1] = "" + if not rpc then + local ok, candidate = pcall(json.decode, body) + if ok and type(candidate) == "table" then rpc = candidate end end - return trim_result(table.concat(out, "\n"):gsub("%s+$", "")) + if not rpc then return nil, "Exa MCP returned malformed JSON" end + if rpc.error then + return nil, type(rpc.error) == "table" + and tostring(rpc.error.message or "Exa MCP error") or "Exa MCP error" + end + local result = rpc.result + if type(result) ~= "table" or type(result.content) ~= "table" then + return nil, "Exa MCP returned malformed tool content" + end + local parts = {} + for _, item in ipairs(result.content) do + if type(item) == "table" and type(item.text) == "string" then parts[#parts + 1] = item.text end + end + local text = table.concat(parts, "\n") + if result.isError then return nil, text ~= "" and text or "Exa MCP error" end + if text == "" then return nil, "Exa MCP returned empty content" end + return text +end + +local function decode_http_envelope(text) + local ok, envelope = pcall(json.decode, text) + if not ok or type(envelope) ~= "table" or type(envelope.status) ~= "number" + or type(envelope.body) ~= "string" then + return nil, "lightpanda returned an unexpected response" + end + return envelope end function M.on_search(input) local query = type(input) == "table" and input.query or nil - if type(query) ~= "string" or query:match("^%s*$") then return "Error: query must be a non-empty string" end + if type(query) ~= "string" or query:match("^%s*$") then + return "Error: query must be a non-empty string" + end if #query > 2000 then return "Error: query is too long" end - local count = input.num_results or 5 - if type(count) ~= "number" or count % 1 ~= 0 or count < 1 or count > 10 then - return "Error: num_results must be an integer from 1 to 10" + + local count = input.num_results + if count == nil then count = 5 end + if type(count) ~= "number" or count % 1 ~= 0 or count < 1 or count > 100 then + return "Error: num_results must be an integer from 1 to 100" end - local args = curl_args() - add(args, - "--header", "Content-Type: application/json", - "--header", "Accept: application/json, text/event-stream", - "--header", "x-exa-source: panto-web", - "--data-binary", panto.ext.json.encode({ - jsonrpc = "2.0", id = 1, method = "tools/call", - params = { name = "web_search_exa", arguments = { query = query, numResults = count } }, - }), - "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}", - "--url", "https://mcp.exa.ai/mcp?tools=web_search_exa") - local raw, err = curl(args) - if not raw then return "Error searching the web: " .. err end - local response - response, err = split_response(raw) - if not response then return "Error searching the web: " .. err end - if response.status < 200 or response.status >= 300 then - return "Error searching the web: Exa MCP returned HTTP " .. response.status + local key = os.getenv("EXA_API_KEY") + if type(key) ~= "string" or key == "" then + local text, err = call("evaluate", { + url = "https://example.com", + timeout = 60000, + script = M.exa_mcp_script(M.exa_mcp_body(query, count)), + }) + if not text then return "Error searching the web: " .. err end + local envelope, envelope_err = decode_http_envelope(text) + if not envelope then return "Error searching the web: " .. envelope_err end + if envelope.status < 200 or envelope.status >= 300 then + return "Error searching the web: Exa MCP returned HTTP " .. math.floor(envelope.status) + end + local result, result_err = M.exa_mcp_result(envelope.body) + if not result then return "Error searching the web: " .. result_err end + return M.truncate(result) end - local rpc - for payload in response.body:gmatch("data:%s*([^\r\n]+)") do - local ok, value = pcall(panto.ext.json.decode, payload) - if ok and type(value) == "table" and (value.result or value.error) then rpc = value break end + local text, err = call("evaluate", { + url = "https://example.com", + timeout = 60000, + script = M.exa_script(M.exa_body(query, count), key), + }) + if not text then + if err == INSTALL_HINT then return "Error searching the web: " .. err end + if err == "lightpanda mcp sent a malformed JSON-RPC message" + or err == "lightpanda mcp sent a malformed JSON-RPC response" + or err == "lightpanda mcp returned malformed tool content" then + return "Error searching the web: " .. err + end + -- Tool/server errors can echo the evaluated script, which contains the key. + return "Error searching the web: Lightpanda request failed" end - if not rpc then - local ok, value = pcall(panto.ext.json.decode, response.body) - if ok then rpc = value end + + local envelope, envelope_err = decode_http_envelope(text) + if not envelope then return "Error searching the web: " .. envelope_err end + if envelope.status < 200 or envelope.status >= 300 then + return "Error searching the web: Exa returned HTTP " .. math.floor(envelope.status) end - if type(rpc) ~= "table" then return "Error searching the web: Exa MCP returned malformed JSON" end - if rpc.error then return "Error searching the web: " .. tostring(rpc.error.message or "Exa MCP error") end - local content = rpc.result and rpc.result.content - if rpc.result and rpc.result.isError then - return "Error searching the web: " .. tostring(content and content[1] and content[1].text or "Exa MCP error") + local valid, response = pcall(json.decode, envelope.body) + if not valid or type(response) ~= "table" then + return "Error searching the web: Exa returned malformed JSON" end - local text = content and content[1] and content[1].text - if type(text) ~= "string" or text == "" then return "No search results." end - return format_search(text, query) + return M.truncate(M.scrub(envelope.body, key)) end function M.activate() - register_fetch() + panto.ext.register_tool { + name = "web.fetch", + description = "Fetch an HTTP(S) URL through the Lightpanda browser and return the page " + .. "as Markdown, including JavaScript-rendered content. Requires `lightpanda` on PATH.", + schema = { + type = "object", + properties = { + url = { type = "string", description = "HTTP(S) URL to fetch." }, + }, + required = { "url" }, + }, + handler = M.on_fetch, + } panto.ext.register_tool { name = "web.search", - description = "Search the web without an API key through Exa's public MCP endpoint. " - .. "Returns titles, URLs, and short excerpts; use web.fetch to read a result.", + description = "Search the web with Exa and return its results. Uses keyless Exa MCP by default; " + .. "EXA_API_KEY enables the REST API. Requires `lightpanda` on PATH.", schema = { type = "object", properties = { query = { type = "string", description = "Web search query." }, - num_results = { type = "integer", minimum = 1, maximum = 10, + num_results = { type = "integer", minimum = 1, maximum = 100, description = "Number of results (default 5)." }, }, required = { "query" }, @@ -525,6 +420,4 @@ function M.activate() } end -M.format_search = format_search - return M |