path: root/spec/test_luatool.lua

-- subagents/luatool.lua: sandboxing plus the session-scoped asynchronous
-- workflow API exposed to model-authored Lua.

local fake = require("spec.fake_ext")
local jobs = require("subagents.jobs")
local luatool = require("subagents.luatool")
local progress = require("subagents.progress")
local workflow = require("subagents.workflow")
local uv = require("luv")

local function has(text, needle)
    assert(type(text) == "string", "expected a string, got " .. type(text))
    assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text))
end

local function profile_set()
    local set = { list = {}, by_name = {}, warnings = {} }
    for _, name in ipairs({ "alpha", "beta" }) do
        local profile = { name = name, description = name, body = "You are " .. name .. ".\n" }
        set.list[#set.list + 1] = profile
        set.by_name[name] = profile
    end
    return set
end

local function pump(id)
    for _ = 1, 2000 do
        uv.run("nowait")
        local record = workflow.workflows[id]
        if record and record.status ~= "running" then
            uv.run("nowait") -- deliver the deferred primary wake
            return record
        end
        uv.sleep(1)
    end
    error("workflow did not settle: " .. tostring(id), 0)
end

local function with_host(fn)
    local handle = fake.install()
    local ok, err = pcall(fn, handle, profile_set())
    pcall(jobs.close_all)
    handle.restore()
    if not ok then error(err, 0) end
end

return {
    { "the guest environment has no filesystem, process, or module access", function()
        local env = luatool.build_env()
        for _, name in ipairs({
            "os", "io", "debug", "package", "require", "load", "loadstring", "dofile",
            "loadfile", "coroutine", "setmetatable", "getmetatable", "rawset", "rawget",
            "collectgarbage", "arg", "pcall", "xpcall",
        }) do
            assert(env[name] == nil, "the guest can reach " .. name)
        end
        assert(env._G == env)
        assert(type(env.subagents.workflow) == "function")
        assert(type(env.subagents.workflows) == "table")
        assert(env.string.dump == nil and env.string ~= string)
        assert(env.print() == nil)
    end },

    { "the guest json codec round-trips and reports bad input instead of raising", function()
        local env = luatool.build_env()
        local encoded = env.json.encode({ prs = { 124, 125 } })
        has(encoded, "124")
        local decoded = env.json.decode(encoded)
        assert(type(decoded) == "table" and decoded.prs[2] == 125)
        -- The sandbox has no pcall, so a raising decoder would kill a workflow
        -- over one malformed agent line.
        local value, message = env.json.decode("RESULT=not json")
        assert(value == nil and type(message) == "string")
        assert(env.json.decode(nil) == nil)
    end },

    { "the string metatable stays reachable and harmless", function()
        local env = luatool.build_env()
        assert(type(("").dump) == "function")
        assert(env.load == nil and env.loadstring == nil)
    end },

    { "source is required and ordinary source values return directly", function()
        with_host(function(_, profiles)
            assert(luatool.handle({ source = "return 42" }, profiles) == "42")
            has(luatool.handle({}, profiles), "Error: source is required")
            has(luatool.handle({ source = "return (" }, profiles), "Error: source did not compile")
            has(luatool.handle({ source = "error('nope')" }, profiles), "Error: source failed to run")
        end)
    end },

    { "a workflow returns an id immediately then records named results", function()
        with_host(function(handle, profiles)
            handle.queue_for("alpha", { id = "0198-a", output = "alpha output" })
            handle.queue_for("beta", { id = "0198-b", output = "beta output" })
            local id = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    local a = ctx:agent{name="research", agent="alpha", prompt="research"}
                    local b = ctx:agent{name="review", agent="beta", prompt="review"}
                    local results = ctx:await({a, b}, "all")
                    return results[1].output .. " + " .. results[2].output
                end)
            ]] }, profiles, { tool_call_id = "lua-call" })

            has(id, "workflow-")
            assert(workflow.workflows[id].status == "running")
            assert(#handle.spawns == 0, "the callback starts after the tool returns")

            local record = pump(id)
            assert(record.status == "completed", tostring(record.error))
            assert(record.result == "alpha output + beta output", tostring(record.result))
            assert(#record.agents == 2)
            assert(record.agents.research.output == "alpha output")
            assert(record.agents[2].name == "review")
            assert(record.agents.review.status == "completed")
            assert(#handle.submissions == 1, "completion wakes the primary once")
            has(handle.submissions[1], id)
            assert(table.concat(handle.emitted, ",") ==
                "background_work_start,background_work_end,agent_submission",
                "the primary wake runs after the completion callback unwinds")
        end)
    end },

    { "a background workflow keeps its completed tool entry pinned", function()
        with_host(function(handle, profiles)
            progress.reset()
            progress.begin_live_turn()
            local pins
            progress.claim({
                id = "pinned-workflow-call",
                tool_name = "subagents.lua",
                set_component = function()
                    pins = {}
                    return {
                        alive = function() return true end,
                        invalidate = function() end,
                        set_pinned = function(_, value) pins[#pins + 1] = value end,
                    }
                end,
            })
            handle.queue_for("alpha", { output = "finished" })
            local id = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    return ctx:agent{name="work", agent="alpha", prompt="go"}:await().output
                end)
            ]] }, profiles, { tool_call_id = "pinned-workflow-call" })
            progress.settle({ id = "pinned-workflow-call" })
            assert(#pins == 1 and pins[1] == true, "the outer result must not unpin the running workflow")

            local record = pump(id)
            assert(record.status == "completed", tostring(record.error))
            assert(#pins == 2 and pins[2] == false, "workflow completion restores transcript order")
            progress.reset()
        end)
    end },

    { "a later Lua call inspects workflows and records are read-only", function()
        with_host(function(_, profiles)
            local id = luatool.handle({ source = [[
                return subagents.workflow(function(ctx) return "done" end)
            ]] }, profiles)
            pump(id)
            local query = string.format(
                "local w=subagents.workflows[%q]; return w.status .. '|' .. w.result", id)
            assert(luatool.handle({ source = query }, profiles) == "completed|done")

            local mutation = string.format(
                "subagents.workflows[%q].status='forged'; return 'bad'", id)
            has(luatool.handle({ source = mutation }, profiles), "read-only")
            assert(workflow.workflows[id].status == "completed")
        end)
    end },

    { "completed agent output is inspectable while its workflow still runs", function()
        with_host(function(handle, profiles)
            handle.queue_for("alpha", { output = "early" })
            handle.queue_for("beta", { output = "late", settle = 100000 })
            local id = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    local first = ctx:agent{name="first", agent="alpha", prompt="first"}:await()
                    local second = ctx:agent{name="second", agent="beta", prompt="second"}:await()
                    return first.output .. second.output
                end)
            ]] }, profiles)
            for _ = 1, 100 do
                uv.run("nowait")
                local w = workflow.workflows[id]
                if w.agents.first and w.agents.first.status == "completed" and w.agents.second then break end
            end
            local w = workflow.workflows[id]
            assert(w.status == "running")
            assert(w.agents.first.output == "early")
            assert(w.agents.second.status == "running")
            local query = string.format(
                "local w=subagents.workflows[%q]; return w.agents.first.output .. '|' .. w.status", id)
            assert(luatool.handle({ source = query }, profiles) == "early|running")

            local iterated = {}
            for name, agent in pairs(w.agents) do iterated[#iterated + 1] = name .. ":" .. agent.status end
            assert(iterated[1] == "first:completed" and iterated[2] == "second:running")
            workflow.cancel_all(true)
            jobs.cancel_all()
            pump(id)
        end)
    end },

    { "public workflow records cancel agents or every running agent", function()
        with_host(function(handle, profiles)
            handle.ext.workflows = workflow.workflows
            handle.queue_for("alpha", { settle = 100000 })
            handle.queue_for("beta", { settle = 100000 })
            local id = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    local alpha = ctx:agent{name="alpha", agent="alpha", prompt="alpha"}
                    local beta = ctx:agent{name="beta", agent="beta", prompt="beta"}
                    ctx:await({alpha, beta}, "all")
                    return "completed"
                end)
            ]] }, profiles)

            local record = handle.ext.workflows[id]
            for _ = 1, 100 do
                uv.run("nowait")
                if record.agents.alpha and record.agents.beta then break end
                uv.sleep(1)
            end
            assert(type(record.cancel) == "function", "workflow exposes cancel")
            assert(type(record.agents.alpha.cancel) == "function", "agent exposes cancel")
            assert(record.agents.alpha:cancel(), "an in-flight agent accepts cancellation")

            for _ = 1, 100 do
                uv.run("nowait")
                if record.agents.alpha.status == "cancelled" then break end
                uv.sleep(1)
            end
            assert(record.agents.alpha.status == "cancelled")
            assert(record.agents.beta.status == "running")
            assert(record:cancel(), "an in-flight workflow accepts cancellation")

            record = pump(id)
            assert(record.status == "cancelled", tostring(record.error))
            assert(record.agents.beta.status == "cancelled")
            assert(record:cancel() == false, "terminal workflows ignore cancellation")
        end)
    end },

    { "agent names are required and unique within a workflow", function()
        with_host(function(handle, profiles)
            local id = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    ctx:agent{name="same", agent="alpha", prompt="one"}
                    ctx:agent{name="same", agent="beta", prompt="two"}
                    return "unreachable"
                end)
            ]] }, profiles)
            local record = pump(id)
            assert(record.status == "failed")
            has(record.error, "duplicate workflow agent name 'same'")
            assert(#handle.spawns == 1, "the duplicate is rejected before spawning")

            local missing = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    ctx:agent{agent="alpha", prompt="one"}
                    return "unreachable"
                end)
            ]] }, profiles)
            has(pump(missing).error, "requires a non-empty unique `name`")
        end)
    end },

    { "callback errors and non-string returns fail the workflow", function()
        with_host(function(_, profiles)
            local bad_type = luatool.handle({ source = [[
                return subagents.workflow(function(ctx) return {"no"} end)
            ]] }, profiles)
            local record = pump(bad_type)
            assert(record.status == "failed")
            has(record.error, "must return a string")

            local raised = luatool.handle({ source = [[
                return subagents.workflow(function(ctx) error("boom") end)
            ]] }, profiles)
            has(pump(raised).error, "boom")
        end)
    end },

    { "inline profiles are scoped to workflows started by one call", function()
        with_host(function(handle, profiles)
            handle.queue_for("local", { output = "local output" })
            local source = [[
                return subagents.workflow(function(ctx)
                    local result = ctx:agent{name="work", agent="local", prompt="inspect"}:await()
                    return result.output
                end)
            ]]
            local id = luatool.handle({
                source = source,
                agents = { { name = "local", system_prompt = "Be local." } },
            }, profiles)
            assert(pump(id).result == "local output")
            assert(handle.spawns[1].system_messages[2].text == "Be local.")

            local missing = luatool.handle({ source = source }, profiles)
            has(pump(missing).error, "unknown agent 'local'")
            assert(#handle.spawns == 1)
        end)
    end },

    { "the instruction and job budgets apply to background workflows", function()
        with_host(function(handle, profiles)
            local runaway = luatool.handle({ source = [[
                return subagents.workflow(function(ctx)
                    local n=0; while true do n=n+1 end
                end)
            ]] }, profiles)
            has(pump(runaway).error, "instruction budget exceeded")

            local source = string.format([[
                return subagents.workflow(function(ctx)
                    for index=1,%d do
                        ctx:agent{name="job-"..index, agent="alpha", prompt="spam"}
                    end
                    return "unreachable"
                end)
            ]], luatool.max_jobs + 1)
            local capped = luatool.handle({ source = source }, profiles)
            has(pump(capped).error, "job limit exceeded")
            assert(#handle.runs == luatool.max_jobs)
        end)
    end },
}