diff options
Diffstat (limited to 'spec/test_luatool.lua')
| -rw-r--r-- | spec/test_luatool.lua | 188 |
1 files changed, 188 insertions, 0 deletions
diff --git a/spec/test_luatool.lua b/spec/test_luatool.lua new file mode 100644 index 0000000..83b4984 --- /dev/null +++ b/spec/test_luatool.lua @@ -0,0 +1,188 @@ +-- subagents/luatool.lua: the restricted environment, the instruction budget, +-- and one real fan-out through the fake host. +-- +-- The sandbox cases check the environment the guest actually gets rather than +-- only the errors an escape attempt produces, because a missing global is the +-- whole mechanism. One documented gap is asserted as a gap: the real string +-- metatable is reachable from any literal, so `("").dump` exists. Without +-- `load` there is no way to run bytecode, so it stays noise rather than an +-- escape — the assertion is here so a future change to that reasoning is +-- deliberate. + +local fake = require("spec.fake_ext") +local luatool = require("subagents.luatool") + +local function has(text, needle) + assert(type(text) == "string", "expected a string, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function profile_set() + local set = { list = {}, by_name = {}, warnings = {} } + for _, name in ipairs({ "alpha", "beta" }) do + local profile = { name = name, description = name, body = "You are " .. name .. ".\n" } + set.list[#set.list + 1] = profile + set.by_name[name] = profile + end + return set +end + +local function with_host(fn) + local handle = fake.install() + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +return { + { "the guest environment has no filesystem, process, or module access", function() + local env = luatool.build_env() + for _, name in ipairs({ + "os", "io", "debug", "package", "require", "load", "loadstring", "dofile", + "loadfile", "coroutine", "setmetatable", "getmetatable", "rawset", "rawget", + "collectgarbage", "arg", "pcall", "xpcall", + }) do + assert(env[name] == nil, "the guest can reach " .. name) + end + assert(env._G == env, "_G must point at the restricted table") + assert(type(env.subagents.workflow) == "function", "the workflow constructor is the whole API") + assert(env.string.dump == nil, "string.dump is removed from the guest copy") + assert(env.string ~= string, "the guest gets a copy it may safely mutate") + assert(env.print() == nil, "print is a no-op") + end }, + + { "the string metatable stays reachable, and stays harmless", function() + local env = luatool.build_env() + -- Documented gap: ("").dump resolves through the real string metatable. + assert(type(("").dump) == "function", "the gap this note describes has moved") + assert(env.load == nil and env.loadstring == nil, + "bytecode is only dangerous with a loader, and there is none") + end }, + + { "an escape attempt inside the guest fails at the call", function() + with_host(function(handle, profiles) + local source = [[ + return subagents.workflow(function(ctx, input) + return { status = "completed", output = require("os").time() } + end) + ]] + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "Error:") + has(text, "nil value") + assert(#handle.spawns == 0, "the guest started no children") + end) + end }, + + { "source that does not return a workflow is refused", function() + with_host(function(handle, profiles) + has(luatool.handle({ prompt = "x", source = "return 42" }, profiles), + "Error: source must return subagents.workflow(function(ctx, input) ... end)") + has(luatool.handle({ prompt = "x", source = "return (" }, profiles), + "Error: source did not compile") + has(luatool.handle({ prompt = "x", source = "error('nope')" }, profiles), + "Error: source failed to run") + end) + end }, + + { "prompt and source are both required", function() + with_host(function(handle, profiles) + has(luatool.handle({ source = "return 1" }, profiles), "Error: prompt is required") + has(luatool.handle({ prompt = "x" }, profiles), "Error: source is required") + has(luatool.handle({ prompt = "", source = "return 1" }, profiles), "Error: prompt is required") + end) + end }, + + { "a runaway guest is stopped by the instruction budget", function() + with_host(function(handle, profiles) + local source = [[ + return subagents.workflow(function(ctx, input) + local n = 0 + while true do n = n + 1 end + end) + ]] + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "Error:") + has(text, "instruction budget exceeded") + end) + end }, + + { "the guest cannot catch the budget error and spin again", function() + with_host(function(handle, profiles) + -- Bounded so a regression fails this case instead of hanging it: with + -- pcall back in the environment the guest would burn three budgets and + -- then report success. + local source = [[ + return subagents.workflow(function(ctx, input) + for _ = 1, 3 do + pcall(function() while true do end end) + end + return { status = "completed", output = "outlived the budget" } + end) + ]] + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "Error:") + has(text, "pcall") + end) + end }, + + { "a fan-out runs end to end and renders one block per child", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { id = "0198-a", output = "alpha says hi" }) + handle.queue_for("beta", { id = "0198-b", output = "beta says hi" }) + + local source = [[ + return subagents.workflow(function(ctx, input) + local jobs = {} + for _, name in ipairs({ "alpha", "beta" }) do + jobs[#jobs + 1] = ctx:agent({ agent = name, prompt = "handle " .. input }) + end + return ctx:await(jobs, "all") + end) + ]] + local text = luatool.handle({ prompt = "the task", source = source }, profiles) + + assert(#handle.spawns == 2, "one spawn per ctx:agent") + assert(handle.spawns[1].prompt == "handle the task", tostring(handle.spawns[1].prompt)) + assert(handle.spawns[1].label == "alpha") + has(text, "id: 0198-a") + has(text, "alpha says hi") + has(text, "id: 0198-b") + has(text, "beta says hi") + assert(select(2, text:gsub("status: completed", "")) == 2, "expected two rendered blocks") + end) + end }, + + { "the job budget applies to a generated workflow", function() + with_host(function(handle, profiles) + local source = string.format([[ + return subagents.workflow(function(ctx, input) + for index = 1, %d do + ctx:agent({ agent = "alpha", prompt = "spam " .. index }) + end + end) + ]], luatool.max_jobs + 1) + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "job limit exceeded") + assert(#handle.runs == luatool.max_jobs, "the cap is enforced at the host boundary") + end) + end }, + + { "the guest cannot raise its own job cap through ctx", function() + with_host(function(handle, profiles) + local source = string.format([[ + return subagents.workflow(function(ctx, input) + ctx.max_jobs = nil + ctx.job_count = 0 + for index = 1, %d do + ctx:agent({ agent = "alpha", prompt = "spam " .. index }) + end + end) + ]], luatool.max_jobs + 1) + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "job limit exceeded") + assert(#handle.runs == luatool.max_jobs, "the cap is private state, not a ctx field") + end) + end }, +} |