Initial commit: web.{fetch,search} tools

external `curl` for fetching, `exa` public service for search

1 files changed, 530 insertions, 0 deletions

diff --git a/panto-web/init.lua b/panto-web/init.lua
new file mode 100644
index 0000000..2e52b4d
--- /dev/null
+++ b/panto-web/init.lua
@@ -0,0 +1,530 @@
+-- Keyless web access: readable page fetches and Exa's public MCP search.
+
+local uv = require("luv")
+local panto = require("panto")
+
+local M = { name = "web" }
+
+local MAX_DOWNLOAD = 2 * 1024 * 1024
+local MAX_RESULT = 50 * 1024
+local META = "__PANTO_WEB_META__"
+
+local function await(label, arm)
+    local co = assert(coroutine.running(), label .. ": await must run inside a coroutine")
+    local remaining = 3
+    arm(function()
+        remaining = remaining - 1
+        if remaining == 0 then
+            local ok, err = coroutine.resume(co)
+            if not ok then error(err, 0) end
+        end
+    end)
+    coroutine.yield()
+end
+
+local function curl(args)
+    local stdout = uv.new_pipe(false)
+    local stderr = uv.new_pipe(false)
+    local out, errors = {}, {}
+    local out_bytes, error_bytes = 0, 0
+    local exit_code, process
+    local too_large = false
+    local signal
+
+    process = uv.spawn("curl", { args = args, stdio = { nil, stdout, stderr } }, function(code)
+        exit_code = code
+        if process and not process:is_closing() then process:close() end
+        signal()
+    end)
+    if not process then
+        stdout:close()
+        stderr:close()
+        return nil, "curl is not installed or could not be started"
+    end
+
+    await("curl", function(done)
+        signal = done
+        stdout:read_start(function(err, data)
+            if err then errors[#errors + 1] = tostring(err) end
+            if data then
+                local room = MAX_DOWNLOAD + 4096 - out_bytes
+                if room > 0 then
+                    local chunk = data:sub(1, room)
+                    out[#out + 1] = chunk
+                    out_bytes = out_bytes + #chunk
+                end
+                if #data > room and not too_large then
+                    too_large = true
+                    if not process:is_closing() then process:kill("sigterm") end
+                end
+            else
+                done()
+            end
+        end)
+        stderr:read_start(function(err, data)
+            if err then errors[#errors + 1] = tostring(err) end
+            if data then
+                local room = 8192 - error_bytes
+                if room > 0 then
+                    local chunk = data:sub(1, room)
+                    errors[#errors + 1] = chunk
+                    error_bytes = error_bytes + #chunk
+                end
+            else
+                done()
+            end
+        end)
+    end)
+
+    if not stdout:is_closing() then stdout:close() end
+    if not stderr:is_closing() then stderr:close() end
+    if too_large then return nil, "response exceeded 2 MB" end
+    if exit_code ~= 0 then
+        local message = table.concat(errors):gsub("%s+$", "")
+        return nil, message ~= "" and message or "curl exited with code " .. tostring(exit_code)
+    end
+    return table.concat(out)
+end
+
+local function curl_args()
+    return {
+        "--silent", "--show-error", "--compressed",
+        "--connect-timeout", "10", "--max-time", "30",
+        "--max-filesize", tostring(MAX_DOWNLOAD),
+        "--proto", "=http,https", "--proto-redir", "=http,https",
+    }
+end
+
+local function add(args, ...)
+    for i = 1, select("#", ...) do args[#args + 1] = select(i, ...) end
+end
+
+local function split_response(raw)
+    local at, from = nil, 1
+    while true do
+        local found = raw:find(META, from, true)
+        if not found then break end
+        at, from = found, found + #META
+    end
+    if not at then return nil, "curl returned no response metadata" end
+    local body = raw:sub(1, at - 1):gsub("\n$", "")
+    local status, content_type, redirect = raw:sub(at + #META):match("^(%d%d%d)\t([^\t]*)\t(.-)%s*$")
+    if not status then return nil, "curl returned malformed response metadata" end
+    return {
+        body = body,
+        status = tonumber(status),
+        content_type = content_type,
+        redirect = redirect,
+    }
+end
+
+local function parse_ipv4(address)
+    local parts = {}
+    for part in address:gmatch("[^.]+") do parts[#parts + 1] = tonumber(part) end
+    if #parts ~= 4 or address:find("[^%d%.]") then return nil end
+    for _, part in ipairs(parts) do
+        if not part or part < 0 or part > 255 or part % 1 ~= 0 then return nil end
+    end
+    return parts
+end
+
+local function parse_ipv6(address)
+    local ipv4 = address:match("([^:]+%.[^:]+)$")
+    if ipv4 then
+        local octets = parse_ipv4(ipv4)
+        if not octets then return nil end
+        address = address:sub(1, #address - #ipv4)
+            .. string.format("%x:%x", octets[1] * 256 + octets[2], octets[3] * 256 + octets[4])
+    end
+    local compression = address:find("::", 1, true)
+    if compression and address:find("::", compression + 2, true) then return nil end
+    local left, right = address:match("^(.-)::(.-)$")
+    if address:find("::", 1, true) and not left then return nil end
+    local groups = {}
+    local function append(side)
+        if side == "" then return true end
+        for part in side:gmatch("[^:]+") do
+            if not part:match("^[%da-fA-F]+$") or #part > 4 then return false end
+            groups[#groups + 1] = tonumber(part, 16)
+        end
+        return true
+    end
+    if left then
+        if not append(left) then return nil end
+        local left_count = #groups
+        local tail = {}
+        for part in right:gmatch("[^:]+") do
+            if not part:match("^[%da-fA-F]+$") or #part > 4 then return nil end
+            tail[#tail + 1] = tonumber(part, 16)
+        end
+        local missing = 8 - left_count - #tail
+        if missing < 1 then return nil end
+        for _ = 1, missing do groups[#groups + 1] = 0 end
+        for _, part in ipairs(tail) do groups[#groups + 1] = part end
+    elseif not append(address) or #groups ~= 8 then
+        return nil
+    end
+    return #groups == 8 and groups or nil
+end
+
+function M.is_public_address(address)
+    local v4 = parse_ipv4(address)
+    if v4 then
+        local a, b, c = v4[1], v4[2], v4[3]
+        return not (a == 0 or a == 10 or a == 127
+            or (a == 100 and b >= 64 and b <= 127)
+            or (a == 169 and b == 254)
+            or (a == 172 and b >= 16 and b <= 31)
+            or (a == 192 and b == 0 and c == 0)
+            or (a == 192 and b == 0 and c == 2)
+            or (a == 192 and b == 168)
+            or (a == 198 and (b == 18 or b == 19 or (b == 51 and c == 100)))
+            or (a == 203 and b == 0 and c == 113)
+            or a >= 224)
+    end
+
+    local v6 = parse_ipv6(address)
+    if not v6 then return false end
+    if v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0
+        and v6[5] == 0 and (v6[6] == 0 or v6[6] == 0xffff) then
+        return M.is_public_address(string.format("%d.%d.%d.%d",
+            v6[7] >> 8, v6[7] & 255, v6[8] >> 8, v6[8] & 255))
+    end
+    local all_zero = true
+    for _, part in ipairs(v6) do if part ~= 0 then all_zero = false break end end
+    return not (all_zero
+        or (v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0
+            and v6[5] == 0 and v6[6] == 0 and v6[7] == 0 and v6[8] == 1)
+        or (v6[1] & 0xfe00) == 0xfc00
+        or (v6[1] & 0xffc0) == 0xfe80
+        or (v6[1] & 0xff00) == 0xff00
+        or (v6[1] == 0x2001 and v6[2] == 0x0db8))
+end
+
+function M.parse_url(url)
+    if type(url) ~= "string" or url == "" then return nil, "url must be a non-empty string" end
+    if #url > 4096 then return nil, "url is too long" end
+    if url:find("%c") then return nil, "url contains control characters" end
+    local scheme, authority = url:match("^([%a][%w+.-]*)://([^/?#]+)")
+    scheme = scheme and scheme:lower()
+    if scheme ~= "http" and scheme ~= "https" then return nil, "only http and https URLs are supported" end
+    if authority:find("@", 1, true) then return nil, "URLs with credentials are not supported" end
+
+    local host, port
+    if authority:sub(1, 1) == "[" then
+        host, port = authority:match("^%[([^]]+)%]:?(%d*)$")
+        if not host or not parse_ipv6(host) then return nil, "invalid IPv6 URL host" end
+    else
+        host, port = authority:match("^(.-):(%d+)$")
+        if not host then host, port = authority, "" end
+        if host:find(":", 1, true) or not host:match("^[%w.-]+$") then
+            return nil, "invalid URL host"
+        end
+    end
+    port = port ~= "" and tonumber(port) or (scheme == "https" and 443 or 80)
+    if not port or port < 1 or port > 65535 then return nil, "invalid URL port" end
+    local normalized = host:lower():gsub("%.$", "")
+    if normalized == "" or normalized == "localhost" or normalized:match("%.localhost$") then
+        return nil, "internal host is not allowed"
+    end
+    return { host = normalized, resolve_host = host:lower(), port = port }
+end
+
+local function resolve_url(url)
+    local parsed, err = M.parse_url(url)
+    if not parsed then return nil, err end
+    if parse_ipv4(parsed.host) or parse_ipv6(parsed.host) then
+        if not M.is_public_address(parsed.host) then return nil, "internal address is not allowed" end
+        parsed.address = parsed.host
+        return parsed
+    end
+
+    local dns_err, addresses
+    local co = assert(coroutine.running(), "DNS lookup requires a coroutine")
+    uv.getaddrinfo(parsed.host, tostring(parsed.port), { socktype = "stream" }, function(e, found)
+        dns_err, addresses = e, found
+        local ok, resume_err = coroutine.resume(co)
+        if not ok then error(resume_err, 0) end
+    end)
+    coroutine.yield()
+    if dns_err or not addresses or #addresses == 0 then
+        return nil, "failed to resolve " .. parsed.host .. ": " .. tostring(dns_err or "no addresses")
+    end
+    for _, item in ipairs(addresses) do
+        local address = item.addr or item.address
+        if not address or not M.is_public_address(address) then
+            return nil, "internal address resolved for " .. parsed.host .. ": " .. tostring(address)
+        end
+        parsed.address = parsed.address or address
+    end
+    return parsed
+end
+
+local redirect_status = { [301] = true, [302] = true, [303] = true, [307] = true, [308] = true }
+
+local function request_once(url, resolved)
+    local args = curl_args()
+    local address = resolved.address:find(":", 1, true) and "[" .. resolved.address .. "]" or resolved.address
+    add(args,
+        "--resolve", resolved.resolve_host .. ":" .. resolved.port .. ":" .. address,
+        "--user-agent", "panto-web/web",
+        "--header", "Accept: text/html,application/xhtml+xml,text/plain,application/json,application/xml;q=0.9,*/*;q=0.1",
+        "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}",
+        "--url", url)
+    local raw, err = curl(args)
+    if not raw then return nil, err end
+    return split_response(raw)
+end
+
+local function fetch_http(url)
+    local current = url
+    for redirects = 0, 5 do
+        local resolved, err = resolve_url(current)
+        if not resolved then return nil, err end
+        local response
+        response, err = request_once(current, resolved)
+        if not response then return nil, err end
+        if redirect_status[response.status] and response.redirect ~= "" then
+            if redirects == 5 then return nil, "too many redirects" end
+            current = response.redirect
+        else
+            response.url = current
+            return response
+        end
+    end
+end
+
+local entities = {
+    amp = "&", lt = "<", gt = ">", quot = '"', apos = "'", nbsp = " ",
+    ndash = "–", mdash = "—", hellip = "…", middot = "·", copy = "©", reg = "®",
+}
+
+local function codepoint(n)
+    n = tonumber(n)
+    if not n or n < 1 or n > 0x10ffff or (n >= 0xd800 and n <= 0xdfff) then return "�" end
+    return utf8.char(n)
+end
+
+function M.html_to_text(html)
+    local title = html:match("<[Tt][Ii][Tt][Ll][Ee][^>]*>(.-)</[Tt][Ii][Tt][Ll][Ee]%s*>")
+    local text = html
+    for _, tag in ipairs({ "script", "style", "noscript", "svg", "head" }) do
+        local letters = tag:gsub(".", function(c) return "[" .. c:lower() .. c:upper() .. "]" end)
+        text = text:gsub("<" .. letters .. "[^>]*>.-</" .. letters .. "%s*>", " ")
+    end
+    text = text:gsub("<!%-%-.-%-%->", " ")
+    text = text:gsub('<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*"([^"]+)"[^>]*>(.-)</[Aa]%s*>',
+        function(href, label) return label .. " (" .. href .. ")" end)
+    text = text:gsub("<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*'([^']+)'[^>]*>(.-)</[Aa]%s*>",
+        function(href, label) return label .. " (" .. href .. ")" end)
+    text = text:gsub("<[Bb][Rr]%s*/?%s*>", "\n")
+    text = text:gsub("<[Ll][Ii][^>]*>", "\n- ")
+    text = text:gsub("</[Pp]%s*>", "\n\n")
+    text = text:gsub("</[Dd][Ii][Vv]%s*>", "\n")
+    text = text:gsub("</[Hh][1-6]%s*>", "\n\n")
+    text = text:gsub("</[Tt][Rr]%s*>", "\n")
+    text = text:gsub("</[Tt][DdHh]%s*>", "\t")
+    text = text:gsub("<[^>]+>", " ")
+    text = text:gsub("&#[xX]([%da-fA-F]+);", function(n) return codepoint(tonumber(n, 16)) end)
+    text = text:gsub("&#(%d+);", codepoint)
+    text = text:gsub("&([%a]+);", function(name) return entities[name] or "&" .. name .. ";" end)
+    text = text:gsub("\r", ""):gsub("[\t\f\v ]+", " ")
+    text = text:gsub(" *\n *", "\n"):gsub("\n\n\n+", "\n\n")
+    text = text:match("^%s*(.-)%s*$") or ""
+    if title then
+        title = M.html_to_text(title)
+        if title == "" then title = nil end
+    end
+    return text, title
+end
+
+local function trim_result(text)
+    if #text <= MAX_RESULT then return text end
+    return text:sub(1, MAX_RESULT) .. "\n\n[truncated after 50 KB]"
+end
+
+local function is_html(content_type, body)
+    content_type = content_type:lower()
+    return content_type:find("text/html", 1, true)
+        or content_type:find("application/xhtml+xml", 1, true)
+        or body:match("^%s*<[!%a]") ~= nil
+end
+
+local function needs_reader(response, text)
+    local lower = text:lower()
+    return response.status >= 400 or #text < 200
+        or lower:find("please enable javascript", 1, true)
+        or lower:find("sorry, you have been blocked", 1, true)
+        or lower:find("please enable cookies", 1, true)
+        or lower:find("just a moment", 1, true)
+end
+
+local function jina(url)
+    local response, err = fetch_http("https://r.jina.ai/" .. url)
+    if not response or response.status < 200 or response.status >= 300 then return nil, err end
+    local content = response.body:match("Markdown Content:%s*(.*)")
+    if not content or #content < 100 then return nil end
+    return content:match("^%s*(.-)%s*$")
+end
+
+function M.on_fetch(input)
+    local url = type(input) == "table" and input.url or nil
+    local parsed, validation_error = M.parse_url(url)
+    if not parsed then return "Error: " .. validation_error end
+    local response, err = fetch_http(url)
+    if not response then return "Error fetching " .. url .. ": " .. err end
+
+    local text, title, via_reader
+    if is_html(response.content_type, response.body) then
+        text, title = M.html_to_text(response.body)
+        if needs_reader(response, text) then
+            local readable = jina(response.url)
+            if readable then text, title, via_reader = readable, nil, true end
+        end
+    elseif response.content_type:lower():find("^text/")
+        or response.content_type:lower():find("json", 1, true)
+        or response.content_type:lower():find("xml", 1, true) then
+        text = response.body
+    else
+        return "Error: unsupported content type: "
+            .. (response.content_type ~= "" and response.content_type or "unknown")
+    end
+
+    local lines = { "URL: " .. response.url }
+    if title then lines[#lines + 1] = "Title: " .. title:sub(1, 200) end
+    if via_reader then
+        lines[#lines + 1] = "Reader: Jina"
+    elseif response.status < 200 or response.status >= 300 then
+        lines[#lines + 1] = "HTTP: " .. response.status
+    end
+    lines[#lines + 1] = ""
+    lines[#lines + 1] = text
+    return trim_result(table.concat(lines, "\n"))
+end
+
+local function register_fetch()
+    panto.ext.register_tool {
+        name = "web.fetch",
+        description = "Fetch an HTTP(S) URL and return readable text instead of raw HTML. "
+            .. "Follows validated public redirects and may retry blocked or JavaScript-only pages through Jina Reader.",
+        schema = {
+            type = "object",
+            properties = {
+                url = { type = "string", description = "Public HTTP(S) URL to fetch." },
+            },
+            required = { "url" },
+        },
+        handler = M.on_fetch,
+    }
+end
+
+local function format_search(text, query)
+    local results, current, collecting = {}, nil, false
+    local function finish()
+        if not current or not current.url then return end
+        current.snippet = table.concat(current.parts, " "):gsub("%s+", " ")
+            :gsub("%.%.%.", ""):match("^%s*(.-)%s*$") or ""
+        if #current.snippet > 1000 then current.snippet = current.snippet:sub(1, 1000) .. "…" end
+        results[#results + 1] = current
+    end
+    for line in (text:gsub("\r", "") .. "\n"):gmatch("(.-)\n") do
+        local value = line:match("^Title:%s*(.*)$")
+        if value then
+            finish()
+            current, collecting = { title = value, parts = {} }, false
+        elseif current then
+            value = line:match("^URL:%s*(.*)$")
+            if value then
+                current.url = value
+            elseif line == "Highlights:" or line:match("^Text:%s*") then
+                collecting = true
+                local first = line:match("^Text:%s*(.+)$")
+                if first then current.parts[#current.parts + 1] = first end
+            elseif collecting and line ~= "..." and line ~= "---" then
+                current.parts[#current.parts + 1] = line
+            end
+        end
+    end
+    finish()
+    if #results == 0 then return trim_result(text) end
+    local out = { "Search results for: " .. query, "" }
+    for i, result in ipairs(results) do
+        out[#out + 1] = i .. ". " .. (result.title ~= "" and result.title or result.url)
+        out[#out + 1] = "URL: " .. result.url
+        if result.snippet ~= "" then out[#out + 1] = result.snippet end
+        out[#out + 1] = ""
+    end
+    return trim_result(table.concat(out, "\n"):gsub("%s+$", ""))
+end
+
+function M.on_search(input)
+    local query = type(input) == "table" and input.query or nil
+    if type(query) ~= "string" or query:match("^%s*$") then return "Error: query must be a non-empty string" end
+    if #query > 2000 then return "Error: query is too long" end
+    local count = input.num_results or 5
+    if type(count) ~= "number" or count % 1 ~= 0 or count < 1 or count > 10 then
+        return "Error: num_results must be an integer from 1 to 10"
+    end
+
+    local args = curl_args()
+    add(args,
+        "--header", "Content-Type: application/json",
+        "--header", "Accept: application/json, text/event-stream",
+        "--header", "x-exa-source: panto-web",
+        "--data-binary", panto.ext.json.encode({
+            jsonrpc = "2.0", id = 1, method = "tools/call",
+            params = { name = "web_search_exa", arguments = { query = query, numResults = count } },
+        }),
+        "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}",
+        "--url", "https://mcp.exa.ai/mcp?tools=web_search_exa")
+    local raw, err = curl(args)
+    if not raw then return "Error searching the web: " .. err end
+    local response
+    response, err = split_response(raw)
+    if not response then return "Error searching the web: " .. err end
+    if response.status < 200 or response.status >= 300 then
+        return "Error searching the web: Exa MCP returned HTTP " .. response.status
+    end
+
+    local rpc
+    for payload in response.body:gmatch("data:%s*([^\r\n]+)") do
+        local ok, value = pcall(panto.ext.json.decode, payload)
+        if ok and type(value) == "table" and (value.result or value.error) then rpc = value break end
+    end
+    if not rpc then
+        local ok, value = pcall(panto.ext.json.decode, response.body)
+        if ok then rpc = value end
+    end
+    if type(rpc) ~= "table" then return "Error searching the web: Exa MCP returned malformed JSON" end
+    if rpc.error then return "Error searching the web: " .. tostring(rpc.error.message or "Exa MCP error") end
+    local content = rpc.result and rpc.result.content
+    if rpc.result and rpc.result.isError then
+        return "Error searching the web: " .. tostring(content and content[1] and content[1].text or "Exa MCP error")
+    end
+    local text = content and content[1] and content[1].text
+    if type(text) ~= "string" or text == "" then return "No search results." end
+    return format_search(text, query)
+end
+
+function M.activate()
+    register_fetch()
+    panto.ext.register_tool {
+        name = "web.search",
+        description = "Search the web without an API key through Exa's public MCP endpoint. "
+            .. "Returns titles, URLs, and short excerpts; use web.fetch to read a result.",
+        schema = {
+            type = "object",
+            properties = {
+                query = { type = "string", description = "Web search query." },
+                num_results = { type = "integer", minimum = 1, maximum = 10,
+                    description = "Number of results (default 5)." },
+            },
+            required = { "query" },
+        },
+        handler = M.on_search,
+    }
+end
+
+M.format_search = format_search
+
+return M