path: root/panto-web/init.lua

-- Keyless web access: readable page fetches and Exa's public MCP search.

local uv = require("luv")
local panto = require("panto")

local M = { name = "web" }

local MAX_DOWNLOAD = 2 * 1024 * 1024
local MAX_RESULT = 50 * 1024
local META = "__PANTO_WEB_META__"

local function await(label, arm)
    local co = assert(coroutine.running(), label .. ": await must run inside a coroutine")
    local remaining = 3
    arm(function()
        remaining = remaining - 1
        if remaining == 0 then
            local ok, err = coroutine.resume(co)
            if not ok then error(err, 0) end
        end
    end)
    coroutine.yield()
end

local function curl(args)
    local stdout = uv.new_pipe(false)
    local stderr = uv.new_pipe(false)
    local out, errors = {}, {}
    local out_bytes, error_bytes = 0, 0
    local exit_code, process
    local too_large = false
    local signal

    process = uv.spawn("curl", { args = args, stdio = { nil, stdout, stderr } }, function(code)
        exit_code = code
        if process and not process:is_closing() then process:close() end
        signal()
    end)
    if not process then
        stdout:close()
        stderr:close()
        return nil, "curl is not installed or could not be started"
    end

    await("curl", function(done)
        signal = done
        stdout:read_start(function(err, data)
            if err then errors[#errors + 1] = tostring(err) end
            if data then
                local room = MAX_DOWNLOAD + 4096 - out_bytes
                if room > 0 then
                    local chunk = data:sub(1, room)
                    out[#out + 1] = chunk
                    out_bytes = out_bytes + #chunk
                end
                if #data > room and not too_large then
                    too_large = true
                    if not process:is_closing() then process:kill("sigterm") end
                end
            else
                done()
            end
        end)
        stderr:read_start(function(err, data)
            if err then errors[#errors + 1] = tostring(err) end
            if data then
                local room = 8192 - error_bytes
                if room > 0 then
                    local chunk = data:sub(1, room)
                    errors[#errors + 1] = chunk
                    error_bytes = error_bytes + #chunk
                end
            else
                done()
            end
        end)
    end)

    if not stdout:is_closing() then stdout:close() end
    if not stderr:is_closing() then stderr:close() end
    if too_large then return nil, "response exceeded 2 MB" end
    if exit_code ~= 0 then
        local message = table.concat(errors):gsub("%s+$", "")
        return nil, message ~= "" and message or "curl exited with code " .. tostring(exit_code)
    end
    return table.concat(out)
end

local function curl_args()
    return {
        "--silent", "--show-error", "--compressed",
        "--connect-timeout", "10", "--max-time", "30",
        "--max-filesize", tostring(MAX_DOWNLOAD),
        "--proto", "=http,https", "--proto-redir", "=http,https",
    }
end

local function add(args, ...)
    for i = 1, select("#", ...) do args[#args + 1] = select(i, ...) end
end

local function split_response(raw)
    local at, from = nil, 1
    while true do
        local found = raw:find(META, from, true)
        if not found then break end
        at, from = found, found + #META
    end
    if not at then return nil, "curl returned no response metadata" end
    local body = raw:sub(1, at - 1):gsub("\n$", "")
    local status, content_type, redirect = raw:sub(at + #META):match("^(%d%d%d)\t([^\t]*)\t(.-)%s*$")
    if not status then return nil, "curl returned malformed response metadata" end
    return {
        body = body,
        status = tonumber(status),
        content_type = content_type,
        redirect = redirect,
    }
end

local function parse_ipv4(address)
    local parts = {}
    for part in address:gmatch("[^.]+") do parts[#parts + 1] = tonumber(part) end
    if #parts ~= 4 or address:find("[^%d%.]") then return nil end
    for _, part in ipairs(parts) do
        if not part or part < 0 or part > 255 or part % 1 ~= 0 then return nil end
    end
    return parts
end

local function parse_ipv6(address)
    local ipv4 = address:match("([^:]+%.[^:]+)$")
    if ipv4 then
        local octets = parse_ipv4(ipv4)
        if not octets then return nil end
        address = address:sub(1, #address - #ipv4)
            .. string.format("%x:%x", octets[1] * 256 + octets[2], octets[3] * 256 + octets[4])
    end
    local compression = address:find("::", 1, true)
    if compression and address:find("::", compression + 2, true) then return nil end
    local left, right = address:match("^(.-)::(.-)$")
    if address:find("::", 1, true) and not left then return nil end
    local groups = {}
    local function append(side)
        if side == "" then return true end
        for part in side:gmatch("[^:]+") do
            if not part:match("^[%da-fA-F]+$") or #part > 4 then return false end
            groups[#groups + 1] = tonumber(part, 16)
        end
        return true
    end
    if left then
        if not append(left) then return nil end
        local left_count = #groups
        local tail = {}
        for part in right:gmatch("[^:]+") do
            if not part:match("^[%da-fA-F]+$") or #part > 4 then return nil end
            tail[#tail + 1] = tonumber(part, 16)
        end
        local missing = 8 - left_count - #tail
        if missing < 1 then return nil end
        for _ = 1, missing do groups[#groups + 1] = 0 end
        for _, part in ipairs(tail) do groups[#groups + 1] = part end
    elseif not append(address) or #groups ~= 8 then
        return nil
    end
    return #groups == 8 and groups or nil
end

function M.is_public_address(address)
    local v4 = parse_ipv4(address)
    if v4 then
        local a, b, c = v4[1], v4[2], v4[3]
        return not (a == 0 or a == 10 or a == 127
            or (a == 100 and b >= 64 and b <= 127)
            or (a == 169 and b == 254)
            or (a == 172 and b >= 16 and b <= 31)
            or (a == 192 and b == 0 and c == 0)
            or (a == 192 and b == 0 and c == 2)
            or (a == 192 and b == 168)
            or (a == 198 and (b == 18 or b == 19 or (b == 51 and c == 100)))
            or (a == 203 and b == 0 and c == 113)
            or a >= 224)
    end

    local v6 = parse_ipv6(address)
    if not v6 then return false end
    if v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0
        and v6[5] == 0 and (v6[6] == 0 or v6[6] == 0xffff) then
        return M.is_public_address(string.format("%d.%d.%d.%d",
            v6[7] >> 8, v6[7] & 255, v6[8] >> 8, v6[8] & 255))
    end
    local all_zero = true
    for _, part in ipairs(v6) do if part ~= 0 then all_zero = false break end end
    return not (all_zero
        or (v6[1] == 0 and v6[2] == 0 and v6[3] == 0 and v6[4] == 0
            and v6[5] == 0 and v6[6] == 0 and v6[7] == 0 and v6[8] == 1)
        or (v6[1] & 0xfe00) == 0xfc00
        or (v6[1] & 0xffc0) == 0xfe80
        or (v6[1] & 0xff00) == 0xff00
        or (v6[1] == 0x2001 and v6[2] == 0x0db8))
end

function M.parse_url(url)
    if type(url) ~= "string" or url == "" then return nil, "url must be a non-empty string" end
    if #url > 4096 then return nil, "url is too long" end
    if url:find("%c") then return nil, "url contains control characters" end
    local scheme, authority = url:match("^([%a][%w+.-]*)://([^/?#]+)")
    scheme = scheme and scheme:lower()
    if scheme ~= "http" and scheme ~= "https" then return nil, "only http and https URLs are supported" end
    if authority:find("@", 1, true) then return nil, "URLs with credentials are not supported" end

    local host, port
    if authority:sub(1, 1) == "[" then
        host, port = authority:match("^%[([^]]+)%]:?(%d*)$")
        if not host or not parse_ipv6(host) then return nil, "invalid IPv6 URL host" end
    else
        host, port = authority:match("^(.-):(%d+)$")
        if not host then host, port = authority, "" end
        if host:find(":", 1, true) or not host:match("^[%w.-]+$") then
            return nil, "invalid URL host"
        end
    end
    port = port ~= "" and tonumber(port) or (scheme == "https" and 443 or 80)
    if not port or port < 1 or port > 65535 then return nil, "invalid URL port" end
    local normalized = host:lower():gsub("%.$", "")
    if normalized == "" or normalized == "localhost" or normalized:match("%.localhost$") then
        return nil, "internal host is not allowed"
    end
    return { host = normalized, resolve_host = host:lower(), port = port }
end

local function resolve_url(url)
    local parsed, err = M.parse_url(url)
    if not parsed then return nil, err end
    if parse_ipv4(parsed.host) or parse_ipv6(parsed.host) then
        if not M.is_public_address(parsed.host) then return nil, "internal address is not allowed" end
        parsed.address = parsed.host
        return parsed
    end

    local dns_err, addresses
    local co = assert(coroutine.running(), "DNS lookup requires a coroutine")
    uv.getaddrinfo(parsed.host, tostring(parsed.port), { socktype = "stream" }, function(e, found)
        dns_err, addresses = e, found
        local ok, resume_err = coroutine.resume(co)
        if not ok then error(resume_err, 0) end
    end)
    coroutine.yield()
    if dns_err or not addresses or #addresses == 0 then
        return nil, "failed to resolve " .. parsed.host .. ": " .. tostring(dns_err or "no addresses")
    end
    for _, item in ipairs(addresses) do
        local address = item.addr or item.address
        if not address or not M.is_public_address(address) then
            return nil, "internal address resolved for " .. parsed.host .. ": " .. tostring(address)
        end
        parsed.address = parsed.address or address
    end
    return parsed
end

local redirect_status = { [301] = true, [302] = true, [303] = true, [307] = true, [308] = true }

local function request_once(url, resolved)
    local args = curl_args()
    local address = resolved.address:find(":", 1, true) and "[" .. resolved.address .. "]" or resolved.address
    add(args,
        "--resolve", resolved.resolve_host .. ":" .. resolved.port .. ":" .. address,
        "--user-agent", "panto-web/web",
        "--header", "Accept: text/html,application/xhtml+xml,text/plain,application/json,application/xml;q=0.9,*/*;q=0.1",
        "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}",
        "--url", url)
    local raw, err = curl(args)
    if not raw then return nil, err end
    return split_response(raw)
end

local function fetch_http(url)
    local current = url
    for redirects = 0, 5 do
        local resolved, err = resolve_url(current)
        if not resolved then return nil, err end
        local response
        response, err = request_once(current, resolved)
        if not response then return nil, err end
        if redirect_status[response.status] and response.redirect ~= "" then
            if redirects == 5 then return nil, "too many redirects" end
            current = response.redirect
        else
            response.url = current
            return response
        end
    end
end

local entities = {
    amp = "&", lt = "<", gt = ">", quot = '"', apos = "'", nbsp = " ",
    ndash = "–", mdash = "—", hellip = "…", middot = "·", copy = "©", reg = "®",
}

local function codepoint(n)
    n = tonumber(n)
    if not n or n < 1 or n > 0x10ffff or (n >= 0xd800 and n <= 0xdfff) then return "�" end
    return utf8.char(n)
end

function M.html_to_text(html)
    local title = html:match("<[Tt][Ii][Tt][Ll][Ee][^>]*>(.-)</[Tt][Ii][Tt][Ll][Ee]%s*>")
    local text = html
    for _, tag in ipairs({ "script", "style", "noscript", "svg", "head" }) do
        local letters = tag:gsub(".", function(c) return "[" .. c:lower() .. c:upper() .. "]" end)
        text = text:gsub("<" .. letters .. "[^>]*>.-</" .. letters .. "%s*>", " ")
    end
    text = text:gsub("<!%-%-.-%-%->", " ")
    text = text:gsub('<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*"([^"]+)"[^>]*>(.-)</[Aa]%s*>',
        function(href, label) return label .. " (" .. href .. ")" end)
    text = text:gsub("<[Aa][^>]-[Hh][Rr][Ee][Ff]%s*=%s*'([^']+)'[^>]*>(.-)</[Aa]%s*>",
        function(href, label) return label .. " (" .. href .. ")" end)
    text = text:gsub("<[Bb][Rr]%s*/?%s*>", "\n")
    text = text:gsub("<[Ll][Ii][^>]*>", "\n- ")
    text = text:gsub("</[Pp]%s*>", "\n\n")
    text = text:gsub("</[Dd][Ii][Vv]%s*>", "\n")
    text = text:gsub("</[Hh][1-6]%s*>", "\n\n")
    text = text:gsub("</[Tt][Rr]%s*>", "\n")
    text = text:gsub("</[Tt][DdHh]%s*>", "\t")
    text = text:gsub("<[^>]+>", " ")
    text = text:gsub("&#[xX]([%da-fA-F]+);", function(n) return codepoint(tonumber(n, 16)) end)
    text = text:gsub("&#(%d+);", codepoint)
    text = text:gsub("&([%a]+);", function(name) return entities[name] or "&" .. name .. ";" end)
    text = text:gsub("\r", ""):gsub("[\t\f\v ]+", " ")
    text = text:gsub(" *\n *", "\n"):gsub("\n\n\n+", "\n\n")
    text = text:match("^%s*(.-)%s*$") or ""
    if title then
        title = M.html_to_text(title)
        if title == "" then title = nil end
    end
    return text, title
end

local function trim_result(text)
    if #text <= MAX_RESULT then return text end
    return text:sub(1, MAX_RESULT) .. "\n\n[truncated after 50 KB]"
end

local function is_html(content_type, body)
    content_type = content_type:lower()
    return content_type:find("text/html", 1, true)
        or content_type:find("application/xhtml+xml", 1, true)
        or body:match("^%s*<[!%a]") ~= nil
end

local function needs_reader(response, text)
    local lower = text:lower()
    return response.status >= 400 or #text < 200
        or lower:find("please enable javascript", 1, true)
        or lower:find("sorry, you have been blocked", 1, true)
        or lower:find("please enable cookies", 1, true)
        or lower:find("just a moment", 1, true)
end

local function jina(url)
    local response, err = fetch_http("https://r.jina.ai/" .. url)
    if not response or response.status < 200 or response.status >= 300 then return nil, err end
    local content = response.body:match("Markdown Content:%s*(.*)")
    if not content or #content < 100 then return nil end
    return content:match("^%s*(.-)%s*$")
end

function M.on_fetch(input)
    local url = type(input) == "table" and input.url or nil
    local parsed, validation_error = M.parse_url(url)
    if not parsed then return "Error: " .. validation_error end
    local response, err = fetch_http(url)
    if not response then return "Error fetching " .. url .. ": " .. err end

    local text, title, via_reader
    if is_html(response.content_type, response.body) then
        text, title = M.html_to_text(response.body)
        if needs_reader(response, text) then
            local readable = jina(response.url)
            if readable then text, title, via_reader = readable, nil, true end
        end
    elseif response.content_type:lower():find("^text/")
        or response.content_type:lower():find("json", 1, true)
        or response.content_type:lower():find("xml", 1, true) then
        text = response.body
    else
        return "Error: unsupported content type: "
            .. (response.content_type ~= "" and response.content_type or "unknown")
    end

    local lines = { "URL: " .. response.url }
    if title then lines[#lines + 1] = "Title: " .. title:sub(1, 200) end
    if via_reader then
        lines[#lines + 1] = "Reader: Jina"
    elseif response.status < 200 or response.status >= 300 then
        lines[#lines + 1] = "HTTP: " .. response.status
    end
    lines[#lines + 1] = ""
    lines[#lines + 1] = text
    return trim_result(table.concat(lines, "\n"))
end

local function register_fetch()
    panto.ext.register_tool {
        name = "web.fetch",
        description = "Fetch an HTTP(S) URL and return readable text instead of raw HTML. "
            .. "Follows validated public redirects and may retry blocked or JavaScript-only pages through Jina Reader.",
        schema = {
            type = "object",
            properties = {
                url = { type = "string", description = "Public HTTP(S) URL to fetch." },
            },
            required = { "url" },
        },
        handler = M.on_fetch,
    }
end

local function format_search(text, query)
    local results, current, collecting = {}, nil, false
    local function finish()
        if not current or not current.url then return end
        current.snippet = table.concat(current.parts, " "):gsub("%s+", " ")
            :gsub("%.%.%.", ""):match("^%s*(.-)%s*$") or ""
        if #current.snippet > 1000 then current.snippet = current.snippet:sub(1, 1000) .. "…" end
        results[#results + 1] = current
    end
    for line in (text:gsub("\r", "") .. "\n"):gmatch("(.-)\n") do
        local value = line:match("^Title:%s*(.*)$")
        if value then
            finish()
            current, collecting = { title = value, parts = {} }, false
        elseif current then
            value = line:match("^URL:%s*(.*)$")
            if value then
                current.url = value
            elseif line == "Highlights:" or line:match("^Text:%s*") then
                collecting = true
                local first = line:match("^Text:%s*(.+)$")
                if first then current.parts[#current.parts + 1] = first end
            elseif collecting and line ~= "..." and line ~= "---" then
                current.parts[#current.parts + 1] = line
            end
        end
    end
    finish()
    if #results == 0 then return trim_result(text) end
    local out = { "Search results for: " .. query, "" }
    for i, result in ipairs(results) do
        out[#out + 1] = i .. ". " .. (result.title ~= "" and result.title or result.url)
        out[#out + 1] = "URL: " .. result.url
        if result.snippet ~= "" then out[#out + 1] = result.snippet end
        out[#out + 1] = ""
    end
    return trim_result(table.concat(out, "\n"):gsub("%s+$", ""))
end

function M.on_search(input)
    local query = type(input) == "table" and input.query or nil
    if type(query) ~= "string" or query:match("^%s*$") then return "Error: query must be a non-empty string" end
    if #query > 2000 then return "Error: query is too long" end
    local count = input.num_results or 5
    if type(count) ~= "number" or count % 1 ~= 0 or count < 1 or count > 10 then
        return "Error: num_results must be an integer from 1 to 10"
    end

    local args = curl_args()
    add(args,
        "--header", "Content-Type: application/json",
        "--header", "Accept: application/json, text/event-stream",
        "--header", "x-exa-source: panto-web",
        "--data-binary", panto.ext.json.encode({
            jsonrpc = "2.0", id = 1, method = "tools/call",
            params = { name = "web_search_exa", arguments = { query = query, numResults = count } },
        }),
        "--write-out", "\n" .. META .. "%{http_code}\t%{content_type}\t%{redirect_url}",
        "--url", "https://mcp.exa.ai/mcp?tools=web_search_exa")
    local raw, err = curl(args)
    if not raw then return "Error searching the web: " .. err end
    local response
    response, err = split_response(raw)
    if not response then return "Error searching the web: " .. err end
    if response.status < 200 or response.status >= 300 then
        return "Error searching the web: Exa MCP returned HTTP " .. response.status
    end

    local rpc
    for payload in response.body:gmatch("data:%s*([^\r\n]+)") do
        local ok, value = pcall(panto.ext.json.decode, payload)
        if ok and type(value) == "table" and (value.result or value.error) then rpc = value break end
    end
    if not rpc then
        local ok, value = pcall(panto.ext.json.decode, response.body)
        if ok then rpc = value end
    end
    if type(rpc) ~= "table" then return "Error searching the web: Exa MCP returned malformed JSON" end
    if rpc.error then return "Error searching the web: " .. tostring(rpc.error.message or "Exa MCP error") end
    local content = rpc.result and rpc.result.content
    if rpc.result and rpc.result.isError then
        return "Error searching the web: " .. tostring(content and content[1] and content[1].text or "Exa MCP error")
    end
    local text = content and content[1] and content[1].text
    if type(text) ~= "string" or text == "" then return "No search results." end
    return format_search(text, query)
end

function M.activate()
    register_fetch()
    panto.ext.register_tool {
        name = "web.search",
        description = "Search the web without an API key through Exa's public MCP endpoint. "
            .. "Returns titles, URLs, and short excerpts; use web.fetch to read a result.",
        schema = {
            type = "object",
            properties = {
                query = { type = "string", description = "Web search query." },
                num_results = { type = "integer", minimum = 1, maximum = 10,
                    description = "Number of results (default 5)." },
            },
            required = { "query" },
        },
        handler = M.on_search,
    }
end

M.format_search = format_search

return M