diff options
Diffstat (limited to 'spec')
| -rw-r--r-- | spec/fake_ext.lua | 709 | ||||
| -rw-r--r-- | spec/run.lua | 81 | ||||
| -rw-r--r-- | spec/test_frontmatter.lua | 88 | ||||
| -rw-r--r-- | spec/test_init.lua | 180 | ||||
| -rw-r--r-- | spec/test_jobs.lua | 244 | ||||
| -rw-r--r-- | spec/test_luatool.lua | 188 | ||||
| -rw-r--r-- | spec/test_models.lua | 158 | ||||
| -rw-r--r-- | spec/test_profiles.lua | 138 | ||||
| -rw-r--r-- | spec/test_run.lua | 332 | ||||
| -rw-r--r-- | spec/test_toml_workflows.lua | 404 | ||||
| -rw-r--r-- | spec/test_workflow.lua | 299 |
11 files changed, 2821 insertions, 0 deletions
diff --git a/spec/fake_ext.lua b/spec/fake_ext.lua new file mode 100644 index 0000000..5ec4c06 --- /dev/null +++ b/spec/fake_ext.lua @@ -0,0 +1,709 @@ +-- A scriptable stand-in for the `panto` module the extension talks to. +-- +-- Every module reaches the host through `require("panto")` at call time, never +-- as a load-time alias, so a spec can install this fake after the modules are +-- already loaded. `install()` writes `package.loaded.panto`; `handle.restore()` +-- removes it again. +-- +-- The surface mirrors the real one, covering what the extension actually calls: +-- +-- panto.ext.session_info() -> { session_id, session_dir, model, reasoning } +-- panto.ext.resolve_model(q) -> cfg | nil, "resolve_model: ..." +-- panto.ext.models(query) -> catalog answer +-- panto.ext.agent -- the primary, borrowed: :tools(), :conversation() +-- panto.ext.on(name, handler) -- lifecycle subscriptions +-- panto.ext.register_tool / register_command +-- panto.ext.json -- panto's JSON codec (dkjson stands in here) +-- panto.agent{ config =, store =, session_id =, conversation = } -> agent +-- panto.file_system_jsonl_store{ dir = } / panto.null_store() +-- agent:conversation() / :session_id() / :tools() / :set_tools(decls) / :run_async(opts) +-- store:resolve(id) / :load(id) +-- conv:messages() / :message_metadata(i) / :add_system_message(text, { metadata = }) +-- job:next_event() / :result() / :request_cancel() / :close() +-- +-- Scripting. Every child takes its outcome from a queue: `handle.queue(outcome)` +-- is matched first-in-first-out, `handle.queue_for(label, outcome)` is matched +-- against the profile name the child was seeded with (its manifest metadata; a +-- resumed child carries no fresh manifest, so it always draws from the FIFO +-- queue). An outcome describes the settled turn — `{ status =, output =, +-- error =, structured_json =, resumable =, id =, settle = }`. A child with +-- nothing queued completes with a generated id and a generic output, which +-- keeps tests that only care about what was asked for short. +-- +-- Settling. There is no event loop here, so a fake job settles on the Nth call +-- to `job:result()`: `settle = N`, lowest first, which is the same ordering key +-- the previous fake used for "first" awaits. `uv.pipe()` fails in this harness +-- (see the stub below), so the job machinery takes its fd-less path and drains +-- its jobs in place; that is what turns those polls into progress and keeps the +-- specs plain assert scripts. `request_cancel` settles the next poll as +-- cancelled, exactly like the binding's pump. +-- +-- Everything the host was asked to do is recorded on the returned handle: +-- `spawns` (one record per child agent, in creation order, carrying the +-- resolve_model arguments, the store directory, the seeded system messages, the +-- tool declarations it was given and the run_async options), `runs` (run_async +-- options only, so a gated child is visibly not started yet), `jobs`, +-- `resolves`, `tools`, `commands`, `models_queries`, `subscriptions`, `mkdirs`, +-- `max_live` and `polls`. Assertions read those tables directly. + +local M = {} + +-- --------------------------------------------------------------------------- +-- luv stand-in, installed once when this module loads +-- --------------------------------------------------------------------------- + +-- The harness has no event loop, so it has no pipes: `uv.pipe()` fails and the +-- job machinery falls back to draining in place, which is the same path a host +-- without luv takes. `fs_mkdir` is recorded rather than performed, so a spec can +-- assert which directories a child store would need without touching the disk. +-- Everything else delegates to the real luv, so the filesystem cases are +-- unaffected. This has to happen at load time, not inside install(), because a +-- module that resolves luv once at load would otherwise capture the real one. +local made_dirs = {} +do + local ok, real = pcall(require, "luv") + if ok and type(real) == "table" then + package.loaded.luv = setmetatable({ + pipe = function() + return nil, "the spec harness provides no pipes" + end, + fs_mkdir = function(path) + made_dirs[#made_dirs + 1] = path + return true + end, + }, { __index = real }) + end +end + +-- A job polled this many times without settling means the drain loop is +-- spinning; failing beats hanging the whole suite. +local POLL_LIMIT = 5000 + +local DEFAULT_SESSION = { + session_id = "0198-primary", + session_dir = "/tmp/panto-spec-sessions/--Users-travis-Code-panto-subagents--", + model = "anthropic:sonnet", + reasoning = "medium", +} + +local DEFAULT_OVERVIEW = { + model = "anthropic:sonnet", + reasoning = "medium", + providers = { + { name = "anthropic", style = "messages", models = 7 }, + { name = "openai", style = "responses", models = 12 }, + }, +} + +-- Stands in for the lightuserdata re-registration tag a real decl carries. The +-- identity is what matters: a child must be given the primary's own tags. +M.SOURCE = setmetatable({}, { __name = "fake.tool_source" }) + +-- The primary's tools: two a child inherits, and the four it must not. +-- agent:tools() sorts by name, so this is the order the rock sees them in. +local DEFAULT_TOOL_NAMES = { + "bash", + "read_file", + "subagents.lua", + "subagents.models", + "subagents.run", + "subagents.workflow", +} + +local function default_tools() + local decls = {} + for index, name in ipairs(DEFAULT_TOOL_NAMES) do + decls[index] = { + name = name, + description = name .. " does a thing", + schema = { type = "object", properties = {} }, + _source = M.SOURCE, + } + end + return decls +end + +-- panto installs its own JSON codec as `panto.ext.json`; dkjson is the closest +-- stand-in available to a bare `lua` process. Absent, the field is simply +-- missing, exactly as it would be on a host too old to provide it. +local function json_codec() + local ok, dkjson = pcall(require, "dkjson") + if not ok or type(dkjson) ~= "table" then + return nil + end + return { + encode = function(value) + return dkjson.encode(value) + end, + decode = function(text) + local value, _, err = dkjson.decode(text) + if err then + error(err, 0) + end + return value + end, + } +end + +local function copy(source, fallback) + local out = {} + for key, value in pairs(source or fallback or {}) do + out[key] = value + end + return out +end + +-- --------------------------------------------------------------------------- +-- Conversations +-- --------------------------------------------------------------------------- + +local conv_mt = {} +conv_mt.__index = conv_mt +conv_mt.__name = "fake.conversation" + +-- scripted = array of { role =, text = | blocks =, metadata = }. `record`, when +-- given, is the child record seeded system messages are mirrored onto. +local function new_conversation(scripted, record) + local messages = {} + for index, message in ipairs(scripted or {}) do + messages[index] = { + role = message.role or "user", + blocks = message.blocks or { { type = "text", text = message.text or "" } }, + metadata = message.metadata, + } + end + return setmetatable({ _messages = messages, _record = record }, conv_mt) +end + +-- Faithful to the binding: metadata is not part of a message table, so the only +-- way to reach it is message_metadata(i). +function conv_mt:messages() + local out = {} + for index, message in ipairs(self._messages) do + local blocks = {} + for block_index, block in ipairs(message.blocks) do + blocks[block_index] = copy(block) + end + out[index] = { role = message.role, blocks = blocks } + end + return out +end + +function conv_mt:message_metadata(index) + local message = self._messages[tonumber(index) or 0] + if message == nil then + return nil + end + return message.metadata +end + +function conv_mt:add_system_message(text, opts) + if type(text) ~= "string" then + error("panto: add_system_message expects a string", 2) + end + local metadata = opts and opts.metadata + if metadata ~= nil and type(metadata) ~= "table" then + error("panto: metadata must be a table", 2) + end + self._messages[#self._messages + 1] = { + role = "system", + blocks = { { type = "text", text = text } }, + metadata = metadata, + } + local record = self._record + if record then + record.system_messages[#record.system_messages + 1] = { text = text, metadata = metadata } + local manifest = metadata and metadata.subagents + if type(manifest) == "table" and type(manifest.agent) == "string" then + record.label = manifest.agent + end + end +end + +-- --------------------------------------------------------------------------- +-- Session stores +-- --------------------------------------------------------------------------- + +local store_mt = {} +store_mt.__index = store_mt +store_mt.__name = "fake.store" + +function store_mt:resolve(id) + local session = self._harness.sessions[id] + if session == nil then + return nil + end + return { id = id, message_count = #session, api_style = "messages" } +end + +function store_mt:load(id) + local session = self._harness.sessions[id] + if session == nil then + return nil + end + return new_conversation(session) +end + +-- --------------------------------------------------------------------------- +-- Jobs +-- --------------------------------------------------------------------------- + +local job_mt = {} +job_mt.__index = job_mt +job_mt.__name = "fake.job" + +local function new_job(cfg) + return setmetatable({ + _settle_after = math.max(1, math.floor(tonumber(cfg.settle) or 1)), + _events = cfg.events or {}, + _finish = cfg.finish, + _harness = cfg.harness, + _polls = 0, + _settled = false, + _result = nil, + _cancel_requested = false, + _closed = false, + }, job_mt) +end + +function job_mt:next_event() + if self._closed or #self._events == 0 then + return nil + end + return table.remove(self._events, 1) +end + +function job_mt:result() + if self._closed then + -- close() frees the settled result in the binding, so a caller that + -- reads it back afterwards sees exactly this. + return nil + end + self._polls = self._polls + 1 + if self._polls > POLL_LIMIT then + error(string.format( + "fake job: polled %d times without settling; the drain loop is making no progress on a job with no wake pipe", + POLL_LIMIT), 0) + end + if self._harness then + self._harness.polls = self._harness.polls + 1 + end + if self._settled then + return self._result + end + if self._cancel_requested then + self._result = { status = "cancelled", error = "cancelled" } + elseif self._polls >= self._settle_after then + self._result = self._finish(self) + else + return nil + end + self._settled = true + if self._harness then + self._harness.live = self._harness.live - 1 + end + return self._result +end + +function job_mt:request_cancel() + self._cancel_requested = true +end + +function job_mt:close() + if self._closed then + return + end + self._closed = true + self._result = nil + self._events = {} +end + +-- job(spec) -> a standalone fake job, for specs that drive the job machinery +-- directly rather than through a child. spec = { result =, events =, settle = }. +function M.job(spec) + spec = spec or {} + return new_job({ + settle = spec.settle, + events = spec.events, + finish = function() + return spec.result or { status = "completed", text = "done" } + end, + }) +end + +-- --------------------------------------------------------------------------- +-- Outcomes: which scripted turn a child gets, and what it settles to +-- --------------------------------------------------------------------------- + +local function next_outcome(h, label) + local bucket = label and h.labelled[label] + if bucket and #bucket > 0 then + return table.remove(bucket, 1) + end + if #h.queued > 0 then + return table.remove(h.queued, 1) + end + return {} +end + +-- Bound as late as possible: the profile label only exists once the child has +-- been seeded with its manifest system message. +local function outcome_for(h, record) + if record.outcome == nil then + record.outcome = next_outcome(h, record.label) + end + return record.outcome +end + +local function child_id(h, record) + if record.session_id then + return record.session_id + end + if record.id == nil then + record.id = outcome_for(h, record).id or ("child-" .. record.index) + end + return record.id +end + +-- The settled turn, in the shape agent:run_async reports it. +local function settled_result(h, record) + local outcome = outcome_for(h, record) + local status = outcome.status or "completed" + local result = { status = status } + local one_shot = record.run and record.run.dispatch_tools == false + if status == "completed" then + if one_shot then + result.text = outcome.output + if outcome.structured_json then + local decl = record.tool_decls and record.tool_decls[1] + result.tool_calls = { { + id = "call-" .. record.index, + name = outcome.tool_name or (decl and decl.name) or "emit_result", + input = outcome.structured_json, + } } + end + else + result.text = outcome.output or ("output of " .. tostring(record.label or child_id(h, record))) + end + else + result.error = outcome.error or ("the child " .. status) + end + + -- The durable file: a child that never reached its first assistant message + -- has none, which is what makes it unresumable. + local id = child_id(h, record) + if outcome.resumable == false then + h.sessions[id] = nil + else + h.sessions[id] = h.sessions[id] or {} + end + return result +end + +-- --------------------------------------------------------------------------- +-- Agents +-- --------------------------------------------------------------------------- + +local agent_mt = {} +agent_mt.__index = agent_mt +agent_mt.__name = "fake.agent" + +function agent_mt:conversation() + return self._conv +end + +function agent_mt:session_id() + return child_id(self._harness, self._record) +end + +function agent_mt:tools() + local out = {} + for index, decl in ipairs(self._record.tool_decls or {}) do + out[index] = decl + end + return out +end + +function agent_mt:set_tools(decls) + if self._borrowed then + error("panto: set_tools is not supported on a borrowed agent", 2) + end + if type(decls) ~= "table" then + error("panto: set_tools expects an array of declarations", 2) + end + local names = {} + for index, decl in ipairs(decls) do + if type(decl) ~= "table" or type(decl.name) ~= "string" then + error("panto: tool declaration " .. index .. " has no name", 2) + end + names[index] = decl.name + end + self._record.tool_decls = decls + self._record.tools = names +end + +function agent_mt:run_async(options) + if self._borrowed then + return nil, "run_async: not supported on a borrowed agent" + end + if type(options) ~= "table" then + error("panto: run_async expects a table", 2) + end + if options.prompt == nil and options.blocks == nil then + return nil, "run_async: pass a prompt or blocks" + end + if options.metadata ~= nil and type(options.metadata) ~= "table" then + return nil, "run_async: metadata must be a table" + end + + local h, record = self._harness, self._record + record.run = { + prompt = options.prompt, + blocks = options.blocks, + metadata = options.metadata, + dispatch_tools = options.dispatch_tools, + wake_fd = options.wake_fd, + } + record.prompt = options.prompt + h.runs[#h.runs + 1] = record.run + + local job = new_job({ + settle = outcome_for(h, record).settle, + events = outcome_for(h, record).events, + harness = h, + finish = function() + return settled_result(h, record) + end, + }) + h.jobs[#h.jobs + 1] = job + record.job = job + h.live = h.live + 1 + if h.live > h.max_live then + h.max_live = h.live + end + return job +end + +-- --------------------------------------------------------------------------- +-- install +-- --------------------------------------------------------------------------- + +-- install(opts) -> handle +-- +-- opts = { +-- session = { session_id =, session_dir =, model =, reasoning = }, +-- models_response = table | function(query), +-- primary_tools = array of decl tables (defaults to DEFAULT_TOOL_NAMES), +-- primary_messages = array of { role =, text = } for the primary conversation, +-- sessions = { [id] = array of scripted messages } already on disk, +-- unknown_models = { ["provider:model"] = true } resolve_model rejects, +-- } +function M.install(opts) + opts = opts or {} + + for index = #made_dirs, 1, -1 do + made_dirs[index] = nil + end + + -- A partial `session` overrides only the fields it names. + local session = copy(DEFAULT_SESSION) + for key, value in pairs(opts.session or {}) do + session[key] = value + end + + local handle = { + session = session, + models_response = opts.models_response or DEFAULT_OVERVIEW, + primary_tools = opts.primary_tools or default_tools(), + sessions = opts.sessions or {}, + unknown_models = opts.unknown_models or {}, + spawns = {}, + runs = {}, + jobs = {}, + resolves = {}, + stores = {}, + tools = {}, + tools_by_name = {}, + commands = {}, + commands_by_name = {}, + models_queries = {}, + subscriptions = {}, + on_by_name = {}, + mkdirs = made_dirs, + queued = {}, + labelled = {}, + polls = 0, + live = 0, + max_live = 0, + } + + function handle.queue(outcome) + handle.queued[#handle.queued + 1] = outcome or {} + end + + function handle.queue_for(label, outcome) + local bucket = handle.labelled[label] + if bucket == nil then + bucket = {} + handle.labelled[label] = bucket + end + bucket[#bucket + 1] = outcome or {} + end + + -- A child session that already exists on disk, for the resume cases. + function handle.add_session(id, messages) + handle.sessions[id] = messages or {} + end + + function handle.made_dir(path) + for _, made in ipairs(handle.mkdirs) do + if made == path then + return true + end + end + return false + end + + -- resolve_model hands back an opaque config; this is how a spec gets from + -- the config a child was built with back to the query that produced it. + local resolved_from = setmetatable({}, { __mode = "k" }) + + local ext = {} + + function ext.session_info() + return copy(handle.session) + end + + function ext.resolve_model(query) + if type(query) ~= "table" then + return nil, "resolve_model: expected a table of arguments" + end + local request = { + model = query.model, + reasoning = query.reasoning, + tool_choice = query.tool_choice, + } + handle.resolves[#handle.resolves + 1] = request + if type(query.model) ~= "string" or query.model == "" then + return nil, "resolve_model: model must be a 'provider:model' string" + end + if handle.unknown_models[query.model] then + return nil, string.format("resolve_model: unknown model '%s'", query.model) + end + local cfg = { + model = query.model, + reasoning = query.reasoning, + style = "messages", + wire_model = query.model:match(":(.+)$") or query.model, + } + resolved_from[cfg] = request + return cfg + end + + function ext.models(query) + handle.models_queries[#handle.models_queries + 1] = query or {} + local response = handle.models_response + if type(response) == "function" then + return response(query or {}) + end + return response + end + + function ext.on(name, fn) + handle.subscriptions[#handle.subscriptions + 1] = { name = name, fn = fn } + handle.on_by_name[name] = fn + end + + -- Fire a subscribed lifecycle handler, the way the host would. + function handle.emit(name, event) + for _, subscription in ipairs(handle.subscriptions) do + if subscription.name == name then + subscription.fn(event) + end + end + end + + function ext.register_tool(tool) + handle.tools[#handle.tools + 1] = tool + handle.tools_by_name[tool.name] = tool + end + + function ext.register_command(command) + handle.commands[#handle.commands + 1] = command + handle.commands_by_name[command.name] = command + end + + ext.json = json_codec() + + -- The primary agent, borrowed: readable tools and conversation, nothing else. + ext.agent = setmetatable({ + _borrowed = true, + _harness = handle, + _record = { index = 0, tool_decls = handle.primary_tools, system_messages = {}, tools = {} }, + _conv = new_conversation(opts.primary_messages), + }, agent_mt) + + local function new_store(kind, arg) + local dir = nil + if type(arg) == "table" then + dir = arg.dir + elseif type(arg) == "string" then + dir = arg + end + if kind == "fs" and type(dir) ~= "string" then + error("panto.file_system_jsonl_store: missing dir", 2) + end + handle.stores[#handle.stores + 1] = dir + return setmetatable({ dir = dir, kind = kind, _harness = handle }, store_mt) + end + + local function new_agent(options) + if type(options) ~= "table" then + error("panto.agent expects a table", 2) + end + local record = { + index = #handle.spawns + 1, + config = options.config, + store_dir = type(options.store) == "table" and options.store.dir or nil, + session_id = options.session_id, + resumed = options.session_id ~= nil, + system_messages = {}, + tools = {}, + resolve = resolved_from[options.config], + } + if record.resolve then + record.model = record.resolve.model + record.reasoning = record.resolve.reasoning + record.tool_choice = record.resolve.tool_choice + end + handle.spawns[record.index] = record + + local conv = options.conversation + if conv == nil then + conv = new_conversation(nil, record) + else + conv._record = record + end + return setmetatable({ _harness = handle, _record = record, _conv = conv }, agent_mt) + end + + handle.ext = ext + package.loaded.panto = { + ext = ext, + agent = new_agent, + file_system_jsonl_store = function(arg) + return new_store("fs", arg) + end, + null_store = function() + return new_store("null", nil) + end, + } + + function handle.restore() + package.loaded.panto = nil + end + + return handle +end + +return M diff --git a/spec/run.lua b/spec/run.lua new file mode 100644 index 0000000..ee9f432 --- /dev/null +++ b/spec/run.lua @@ -0,0 +1,81 @@ +-- The spec runner: `lua spec/run.lua` from anywhere in the repo. +-- +-- Expected environment. Plain Lua 5.4 with the repo root on package.path, which +-- this file arranges from `arg[0]`, plus the rocks the extension depends on +-- (lyaml, toml2lua, luv) and dkjson for the specs' JSON decoding. `mise run +-- check` puts the ./.rocks tree on LUA_PATH/LUA_CPATH first and is the intended +-- entry point; a bare `lua spec/run.lua` also works if those rocks are on the +-- default path. `panto lua spec/run.lua` works too — panto's own rocks tree +-- already carries luv. +-- +-- Missing optional rocks do not fail the run. A test that needs one requires it +-- lazily and returns `"skip", reason`; the runner prints a SKIP line, counts it, +-- and still exits 0. Only a real assertion failure or an unexpected error exits +-- 1. `mise run deps` installs everything, so a local run exercises all of it. +-- +-- Test files. Every spec/test_*.lua returns an ordered array of { name, fn }. +-- `fn` asserts and returns nothing to pass, or returns "skip", reason. Ordering +-- is the array's, so a file reads top to bottom. + +local script = (arg and arg[0]) or "spec/run.lua" +local spec_dir = script:match("^(.*)/[^/]+$") or "." +local root = spec_dir:match("^(.*)/[^/]+$") or "." + +package.path = root .. "/?.lua;" .. root .. "/?/init.lua;" .. package.path + +local function test_files() + local found = {} + local pipe = io.popen("ls '" .. spec_dir .. "'/test_*.lua 2>/dev/null") + if not pipe then + return found + end + for line in pipe:lines() do + found[#found + 1] = line + end + pipe:close() + table.sort(found) + return found +end + +local function traceback(err) + return debug.traceback(tostring(err), 2) +end + +local passed, skipped, failed = 0, 0, 0 + +local function record(label, ok, first, second) + if not ok then + failed = failed + 1 + print("FAIL " .. label) + print(first) + elseif first == "skip" then + skipped = skipped + 1 + print("SKIP " .. label .. " — " .. tostring(second)) + else + passed = passed + 1 + print("ok " .. label) + end +end + +for _, file in ipairs(test_files()) do + local name = file:match("[^/]+$") + local chunk, load_err = loadfile(file) + if not chunk then + record(name, false, "could not load: " .. tostring(load_err)) + else + local loaded, cases = xpcall(chunk, traceback) + if not loaded then + record(name, false, cases) + elseif type(cases) ~= "table" then + record(name, false, "expected an array of { name, fn }, got " .. type(cases)) + else + for _, case in ipairs(cases) do + local ok, first, second = xpcall(case[2], traceback) + record(name .. ": " .. tostring(case[1]), ok, first, second) + end + end + end +end + +print(string.format("\n%d passed, %d skipped, %d failed", passed, skipped, failed)) +os.exit(failed == 0 and 0 or 1) diff --git a/spec/test_frontmatter.lua b/spec/test_frontmatter.lua new file mode 100644 index 0000000..3f0c4c5 --- /dev/null +++ b/spec/test_frontmatter.lua @@ -0,0 +1,88 @@ +-- subagents/frontmatter.lua: splitting a profile into YAML header and body. +-- +-- The body is the part a broken header must never cost the user, so every +-- degraded case is checked for "body preserved verbatim" as well as for the +-- warning. Cases that actually parse YAML need lyaml and skip without it. + +local frontmatter = require("subagents.frontmatter") + +local function lyaml_or_skip() + return pcall(require, "lyaml") +end + +return { + { "fenced header parses and keeps the body verbatim", function() + if not lyaml_or_skip() then + return "skip", "lyaml is not installed" + end + local body = "You are a reviewer.\n\n indented line \n" + local data, parsed_body, warning = frontmatter.parse( + "---\nname: reviewer\ndescription: Reviews changes\nmodel: anthropic:sonnet\n---\n" .. body) + assert(warning == nil, "unexpected warning: " .. tostring(warning)) + assert(type(data) == "table", "expected a mapping") + assert(data.name == "reviewer", tostring(data.name)) + assert(data.description == "Reviews changes") + assert(data.model == "anthropic:sonnet", tostring(data.model)) + assert(parsed_body == body, string.format("body was rewritten: %q", parsed_body)) + end }, + + { "CRLF fences are tolerated", function() + if not lyaml_or_skip() then + return "skip", "lyaml is not installed" + end + local data, body, warning = frontmatter.parse("---\r\nname: crlf\r\n---\r\nbody\r\n") + assert(warning == nil, tostring(warning)) + assert(type(data) == "table" and data.name == "crlf", "header did not parse") + assert(body == "body\r\n", string.format("%q", body)) + end }, + + { "no fence means the whole file is the body", function() + local text = "You are a reviewer.\n\n---\n\nNot a header.\n" + local data, body, warning = frontmatter.parse(text) + assert(data == nil, "expected no metadata") + assert(body == text, "body was rewritten") + assert(warning == nil, tostring(warning)) + end }, + + { "an unterminated fence is treated as prose", function() + local text = "---\nname: never closed\n\nstill prose\n" + local data, body, warning = frontmatter.parse(text) + assert(data == nil, "expected no metadata") + assert(body == text, "body was rewritten") + assert(warning == nil, "a lone rule is not an error") + end }, + + { "an empty fenced block is an empty mapping", function() + local data, body, warning = frontmatter.parse("---\n---\nbody\n") + assert(type(data) == "table" and next(data) == nil, "expected an empty mapping") + assert(body == "body\n", string.format("%q", body)) + assert(warning == nil, tostring(warning)) + end }, + + { "broken YAML warns and keeps the body", function() + if not lyaml_or_skip() then + return "skip", "lyaml is not installed" + end + local data, body, warning = frontmatter.parse("---\na: [unclosed\n---\nbody\n") + assert(data == nil, "a broken header must not produce metadata") + assert(body == "body\n", string.format("%q", body)) + assert(type(warning) == "string" and warning:find("did not parse", 1, true), + "expected a parse warning, got " .. tostring(warning)) + end }, + + { "a non-mapping document warns and keeps the body", function() + if not lyaml_or_skip() then + return "skip", "lyaml is not installed" + end + local data, body, warning = frontmatter.parse("---\njust a string\n---\nbody\n") + assert(data == nil, "a scalar header must not produce metadata") + assert(body == "body\n", string.format("%q", body)) + assert(type(warning) == "string" and warning:find("not a mapping", 1, true), + "expected a mapping warning, got " .. tostring(warning)) + end }, + + { "empty input is empty output", function() + local data, body, warning = frontmatter.parse("") + assert(data == nil and body == "" and warning == nil) + end }, +} diff --git a/spec/test_init.lua b/spec/test_init.lua new file mode 100644 index 0000000..800dcaa --- /dev/null +++ b/spec/test_init.lua @@ -0,0 +1,180 @@ +-- init.lua: the extension entry point pantograph evaluates and activates. +-- +-- Activation is the whole contract with the host: the wrong shape, a missing +-- tool, a description that does not name the discovered profiles, or a missing +-- lifecycle subscription is invisible until a user notices the tools are gone or +-- a cancelled turn leaves children running. Discovery is pointed at a temporary +-- config layer, so the assertions do not depend on this machine's ~/.config; the +-- first case skips when luv or lyaml is missing because the profile it looks for +-- could not be read without them. + +local fake = require("spec.fake_ext") +local jobs = require("subagents.jobs") +local paths = require("subagents.paths") + +local entry = require("init") + +local function has(text, needle) + assert(type(text) == "string", "expected a string, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +-- Activate against the fake host with discovery pointed at nothing, for the +-- cases that care about registration rather than profiles. Profile discovery is +-- cached in subagents.spawn, so a case that ran earlier may have filled it. +local function activate_bare(fn, opts) + local original = paths.config_roots + paths.config_roots = function() + return {} + end + local handle = fake.install(opts) + local ok, err = pcall(function() + if opts and opts.before then + opts.before(handle) + end + entry.activate() + end) + paths.config_roots = original + local result = table.pack(pcall(fn, handle, ok, err)) + handle.restore() + if not result[1] then + error(result[2], 0) + end +end + +return { + { "the entry is the extension shape pantograph expects", function() + assert(entry.name == "subagents", tostring(entry.name)) + assert(type(entry.activate) == "function", "activate must be a function") + end }, + + { "activation registers the four tools and names the discovered profiles", function() + local ok_uv, uv = pcall(require, "luv") + if not ok_uv then + return "skip", "luv is not installed" + end + if not pcall(require, "lyaml") then + return "skip", "lyaml is not installed" + end + + local tmp = assert(uv.fs_mkdtemp("/tmp/panto-subagents-init-XXXXXX")) + assert(os.execute("mkdir -p " .. tmp .. "/agents")) + local file = assert(io.open(tmp .. "/agents/reviewer.md", "w")) + file:write("---\ndescription: Reviews changes\n---\nYou are a reviewer.\n") + file:close() + + local original_roots = paths.config_roots + paths.config_roots = function(kind) + return { tmp .. "/" .. kind } + end + local handle = fake.install() + + local ok, err = pcall(entry.activate) + + paths.config_roots = original_roots + handle.restore() + os.execute("rm -rf " .. tmp) + assert(ok, tostring(err)) + + for _, name in ipairs({ "subagents.run", "subagents.models", "subagents.lua", "subagents.workflow" }) do + assert(handle.tools_by_name[name], "missing tool " .. name) + end + assert(#handle.tools == 4, "expected exactly four tools, saw " .. #handle.tools) + + local run_tool = handle.tools_by_name["subagents.run"] + has(run_tool.description, "reviewer — Reviews changes") + has(run_tool.description, "exactly one of `agent`") + has(run_tool.description, "subagents.models") + has(run_tool.description, "one tool batch") + assert(run_tool.schema.required[1] == "prompt", "prompt is the only required field") + assert(run_tool.schema.properties.agent and run_tool.schema.properties.id) + assert(type(run_tool.handler) == "function") + + assert(handle.tools_by_name["subagents.lua"].schema.properties.source, "the lua tool takes source") + assert(handle.tools_by_name["subagents.workflow"].schema.properties.steps.items.required, + "the workflow tool describes its step shape") + end }, + + { "an interrupted turn cancels every live child, and its end closes them", function() + activate_bare(function(handle, ok, err) + assert(ok, tostring(err)) + assert(type(handle.on_by_name["turn_interrupt"]) == "function", + "an interrupted turn must be able to cancel its children") + assert(type(handle.on_by_name["turn_end"]) == "function", + "a finished turn must be able to close its children") + + -- A child that would not settle on its own, so the lifecycle is the + -- only thing that can end it. close_all runs whatever happens, or a + -- leaked handle would count against the next case's gate. + local job = fake.job({ settle = 99 }) + local started = assert(jobs.start({ label = "alpha", build = function() + return job + end })) + local checked, failure = pcall(function() + assert(started:result() == nil, "the child is still running") + handle.emit("turn_interrupt", { phase = "interrupt" }) + assert(job._cancel_requested, "an interrupted turn asks its children to stop") + handle.emit("turn_end", { phase = "end", reason = "interrupted" }) + assert(job._closed, "the end of the turn joins every child") + end) + jobs.close_all() + assert(checked, failure) + end) + end }, + + { "only the subagents tool calls claim a progress component", function() + activate_bare(function(handle, ok, err) + assert(ok, tostring(err)) + assert(type(handle.on_by_name["tool_call_complete"]) == "function", + "children report progress through the tool call that started them") + + local claimed + handle.emit("tool_call_complete", { + id = "call-1", + tool_name = "subagents.run", + set_component = function(_, component) + claimed = component + return { invalidate = function() end, alive = function() + return true + end } + end, + }) + assert(type(claimed) == "table" and type(claimed.render) == "function", + "the entry is given a component that renders the cards") + + local foreign + handle.emit("tool_call_complete", { + id = "call-2", + tool_name = "read_file", + set_component = function(_, component) + foreign = component + end, + }) + assert(foreign == nil, "another tool's entry is left alone") + end) + end }, + + { "a host without resolve_model fails activation loudly", function() + activate_bare(function(handle, ok, err) + assert(not ok, "activation must not silently register unusable tools") + has(tostring(err), "too old") + assert(#handle.tools == 0, "nothing is registered by a failed activation") + end, { + before = function(handle) + handle.ext.resolve_model = nil + end, + }) + end }, + + { "a host without panto.agent fails activation loudly", function() + activate_bare(function(handle, ok, err) + assert(not ok, "a host that cannot build a child agent is too old") + has(tostring(err), "too old") + assert(#handle.tools == 0, "nothing is registered by a failed activation") + end, { + before = function() + package.loaded.panto.agent = nil + end, + }) + end }, +} diff --git a/spec/test_jobs.lua b/spec/test_jobs.lua new file mode 100644 index 0000000..e6ce097 --- /dev/null +++ b/spec/test_jobs.lua @@ -0,0 +1,244 @@ +-- subagents/jobs.lua: the concurrency gate, the queue, cancellation, and the +-- await contract every caller (run.lua, workflow.lua) is written against. +-- +-- These cases drive the job machinery directly with hand-made fake jobs rather +-- than through a child, so a failure here points at the gate and not at spawn +-- policy. The fake jobs have no wake pipe (the harness has none), so awaiting +-- drains them in place; `settle = N` means "settles on the Nth poll", which is +-- how the ordering cases stay deterministic without an event loop. + +local fake = require("spec.fake_ext") +local jobs = require("subagents.jobs") + +-- The host may report one result or an array of them; both are normalized here +-- exactly as workflow.lua normalizes them. +local function as_array(value) + if type(value) ~= "table" then + return {} + end + if value.status ~= nil then + return { value } + end + return value +end + +-- Every case leaves the module clean: close_all drops whatever is still live. +local function with_jobs(fn, max_concurrent) + local original = jobs.MAX_CONCURRENT + if max_concurrent then + jobs.MAX_CONCURRENT = max_concurrent + end + local ok, err = pcall(fn) + pcall(jobs.close_all) + jobs.MAX_CONCURRENT = original + if not ok then + error(err, 0) + end +end + +-- A starter that records which builds actually ran, and the jobs they made. +local function starter() + local built, made = {}, {} + local function start(name, spec) + spec = spec or {} + return jobs.start({ + label = name, + id = spec.id, + one_shot = spec.one_shot, + on_event = spec.on_event, + build = function() + built[#built + 1] = name + if spec.build_error then + return nil, spec.build_error + end + local job = fake.job({ + settle = spec.settle, + events = spec.events, + result = spec.result or { status = "completed", text = name }, + }) + made[name] = job + return job + end, + }) + end + return start, built, made +end + +local function contains(list, value) + for _, entry in ipairs(list) do + if entry == value then + return true + end + end + return false +end + +return { + { "a started job settles through await", function() + with_jobs(function() + local start = starter() + local handle = assert(start("alpha")) + assert(handle:result() == nil, "a job that has not settled has no result") + + local results = jobs.await({ handle }, "all") + assert(#results == 1, "one handle, one result") + assert(results[1].status == "completed", tostring(results[1].status)) + assert(results[1].text == "alpha", tostring(results[1].text)) + assert(handle:result().text == "alpha", "the settled result stays on the handle") + end) + end }, + + { "await all returns results in input order, not settle order", function() + with_jobs(function() + local start = starter() + local handles = { + assert(start("alpha", { settle = 3 })), + assert(start("beta", { settle = 1 })), + assert(start("gamma", { settle = 2 })), + } + local results = jobs.await(handles, "all") + assert(#results == 3, "expected three results") + assert(results[1].text == "alpha", tostring(results[1].text)) + assert(results[2].text == "beta", tostring(results[2].text)) + assert(results[3].text == "gamma", tostring(results[3].text)) + end) + end }, + + { "await first returns the earliest settler and the remaining handles", function() + with_jobs(function() + local start = starter() + local handles = { + assert(start("alpha", { settle = 3 })), + assert(start("beta", { settle = 1 })), + assert(start("gamma", { settle = 2 })), + } + local seen = {} + while #handles > 0 do + local results, remaining = jobs.await(handles, "first") + results = as_array(results) + assert(#results >= 1, "an await that returns must settle something") + for _, result in ipairs(results) do + seen[#seen + 1] = result.text + end + assert(type(remaining) == "table", "first mode reports what is still running") + assert(#remaining < #handles, "every await makes progress") + handles = remaining + end + assert(seen[1] == "beta", "the lowest settle key comes back first: " .. table.concat(seen, ",")) + assert(contains(seen, "gamma") and contains(seen, "alpha"), table.concat(seen, ",")) + assert(#seen == 3, table.concat(seen, ",")) + end) + end }, + + { "the gate runs four at a time and queues the rest", function() + with_jobs(function() + local start, built = starter() + local handles = {} + for _, name in ipairs({ "a", "b", "c", "d", "e", "f" }) do + handles[#handles + 1] = assert(start(name)) + end + assert(#built == 4, "the gate holds at four running, saw " .. #built) + assert(handles[5]:result() == nil, "a queued child has not settled") + + local results = jobs.await(handles, "all") + assert(#built == 6, "the queue drains as slots free up, saw " .. #built) + assert(#results == 6, "every child reports") + assert(results[5].text == "e" and results[6].text == "f", "queued children keep their place") + end) + end }, + + { "a child cancelled while queued never starts", function() + with_jobs(function() + local start, built = starter() + local handles = {} + for _, name in ipairs({ "a", "b", "c", "d", "e" }) do + handles[#handles + 1] = assert(start(name)) + end + handles[5]:cancel() + + local result = handles[5]:result() + assert(type(result) == "table", "a cancelled queued child settles immediately") + assert(result.status == "cancelled", tostring(result.status)) + assert(result.error == "cancelled before the child started", tostring(result.error)) + assert(#built == 4, "the queued child was never built") + assert(not contains(built, "e"), "the queued child was never built") + + jobs.await({ handles[1], handles[2], handles[3], handles[4] }, "all") + assert(#built == 4, "a cancelled child does not start when a slot frees up") + end) + end }, + + { "cancelling a running child asks its job to cancel", function() + with_jobs(function() + local start, _, made = starter() + local handle = assert(start("alpha", { settle = 5 })) + handle:cancel() + assert(made.alpha._cancel_requested, "the job was asked to cancel") + + local results = jobs.await({ handle }, "all") + assert(results[1].status == "cancelled", tostring(results[1].status)) + end) + end }, + + { "events reach on_event before the job settles", function() + with_jobs(function() + local seen = {} + local start = starter() + local handle = assert(start("alpha", { + settle = 2, + events = { + { type = "content_delta", text = "half " }, + { type = "content_delta", text = "a thought" }, + }, + on_event = function(event) + seen[#seen + 1] = event.text + end, + })) + jobs.await({ handle }, "all") + assert(table.concat(seen) == "half a thought", "events arrive in order: " .. table.concat(seen, "|")) + end) + end }, + + { "cancel_all stops the running children and drops the queued ones", function() + with_jobs(function() + local start, built, made = starter() + local handles = {} + for _, name in ipairs({ "a", "b", "c", "d", "e" }) do + handles[#handles + 1] = assert(start(name, { settle = 5 })) + end + jobs.cancel_all() + + for _, name in ipairs({ "a", "b", "c", "d" }) do + assert(made[name]._cancel_requested, "running child " .. name .. " was not cancelled") + end + assert(#built == 4, "cancel_all must not start the queued child") + assert(handles[5]:result().status == "cancelled", "the queued child settles cancelled") + + local results = jobs.await(handles, "all") + for index, result in ipairs(results) do + assert(result.status == "cancelled", "child " .. index .. " is " .. tostring(result.status)) + end + end) + end }, + + { "close_all closes every job it started", function() + with_jobs(function() + local start, _, made = starter() + local handle = assert(start("alpha")) + jobs.await({ handle }, "all") + assert(not made.alpha._closed, "awaiting does not close a job") + + jobs.close_all() + assert(made.alpha._closed, "turn end closes the job") + end) + end }, + + { "a build failure is a nil return, never an exception", function() + with_jobs(function() + local start = starter() + local handle, err = start("alpha", { build_error = "the host refused" }) + assert(handle == nil, "a failed build starts nothing") + assert(tostring(err):find("the host refused", 1, true), tostring(err)) + end) + end }, +} diff --git a/spec/test_luatool.lua b/spec/test_luatool.lua new file mode 100644 index 0000000..83b4984 --- /dev/null +++ b/spec/test_luatool.lua @@ -0,0 +1,188 @@ +-- subagents/luatool.lua: the restricted environment, the instruction budget, +-- and one real fan-out through the fake host. +-- +-- The sandbox cases check the environment the guest actually gets rather than +-- only the errors an escape attempt produces, because a missing global is the +-- whole mechanism. One documented gap is asserted as a gap: the real string +-- metatable is reachable from any literal, so `("").dump` exists. Without +-- `load` there is no way to run bytecode, so it stays noise rather than an +-- escape — the assertion is here so a future change to that reasoning is +-- deliberate. + +local fake = require("spec.fake_ext") +local luatool = require("subagents.luatool") + +local function has(text, needle) + assert(type(text) == "string", "expected a string, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function profile_set() + local set = { list = {}, by_name = {}, warnings = {} } + for _, name in ipairs({ "alpha", "beta" }) do + local profile = { name = name, description = name, body = "You are " .. name .. ".\n" } + set.list[#set.list + 1] = profile + set.by_name[name] = profile + end + return set +end + +local function with_host(fn) + local handle = fake.install() + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +return { + { "the guest environment has no filesystem, process, or module access", function() + local env = luatool.build_env() + for _, name in ipairs({ + "os", "io", "debug", "package", "require", "load", "loadstring", "dofile", + "loadfile", "coroutine", "setmetatable", "getmetatable", "rawset", "rawget", + "collectgarbage", "arg", "pcall", "xpcall", + }) do + assert(env[name] == nil, "the guest can reach " .. name) + end + assert(env._G == env, "_G must point at the restricted table") + assert(type(env.subagents.workflow) == "function", "the workflow constructor is the whole API") + assert(env.string.dump == nil, "string.dump is removed from the guest copy") + assert(env.string ~= string, "the guest gets a copy it may safely mutate") + assert(env.print() == nil, "print is a no-op") + end }, + + { "the string metatable stays reachable, and stays harmless", function() + local env = luatool.build_env() + -- Documented gap: ("").dump resolves through the real string metatable. + assert(type(("").dump) == "function", "the gap this note describes has moved") + assert(env.load == nil and env.loadstring == nil, + "bytecode is only dangerous with a loader, and there is none") + end }, + + { "an escape attempt inside the guest fails at the call", function() + with_host(function(handle, profiles) + local source = [[ + return subagents.workflow(function(ctx, input) + return { status = "completed", output = require("os").time() } + end) + ]] + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "Error:") + has(text, "nil value") + assert(#handle.spawns == 0, "the guest started no children") + end) + end }, + + { "source that does not return a workflow is refused", function() + with_host(function(handle, profiles) + has(luatool.handle({ prompt = "x", source = "return 42" }, profiles), + "Error: source must return subagents.workflow(function(ctx, input) ... end)") + has(luatool.handle({ prompt = "x", source = "return (" }, profiles), + "Error: source did not compile") + has(luatool.handle({ prompt = "x", source = "error('nope')" }, profiles), + "Error: source failed to run") + end) + end }, + + { "prompt and source are both required", function() + with_host(function(handle, profiles) + has(luatool.handle({ source = "return 1" }, profiles), "Error: prompt is required") + has(luatool.handle({ prompt = "x" }, profiles), "Error: source is required") + has(luatool.handle({ prompt = "", source = "return 1" }, profiles), "Error: prompt is required") + end) + end }, + + { "a runaway guest is stopped by the instruction budget", function() + with_host(function(handle, profiles) + local source = [[ + return subagents.workflow(function(ctx, input) + local n = 0 + while true do n = n + 1 end + end) + ]] + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "Error:") + has(text, "instruction budget exceeded") + end) + end }, + + { "the guest cannot catch the budget error and spin again", function() + with_host(function(handle, profiles) + -- Bounded so a regression fails this case instead of hanging it: with + -- pcall back in the environment the guest would burn three budgets and + -- then report success. + local source = [[ + return subagents.workflow(function(ctx, input) + for _ = 1, 3 do + pcall(function() while true do end end) + end + return { status = "completed", output = "outlived the budget" } + end) + ]] + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "Error:") + has(text, "pcall") + end) + end }, + + { "a fan-out runs end to end and renders one block per child", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { id = "0198-a", output = "alpha says hi" }) + handle.queue_for("beta", { id = "0198-b", output = "beta says hi" }) + + local source = [[ + return subagents.workflow(function(ctx, input) + local jobs = {} + for _, name in ipairs({ "alpha", "beta" }) do + jobs[#jobs + 1] = ctx:agent({ agent = name, prompt = "handle " .. input }) + end + return ctx:await(jobs, "all") + end) + ]] + local text = luatool.handle({ prompt = "the task", source = source }, profiles) + + assert(#handle.spawns == 2, "one spawn per ctx:agent") + assert(handle.spawns[1].prompt == "handle the task", tostring(handle.spawns[1].prompt)) + assert(handle.spawns[1].label == "alpha") + has(text, "id: 0198-a") + has(text, "alpha says hi") + has(text, "id: 0198-b") + has(text, "beta says hi") + assert(select(2, text:gsub("status: completed", "")) == 2, "expected two rendered blocks") + end) + end }, + + { "the job budget applies to a generated workflow", function() + with_host(function(handle, profiles) + local source = string.format([[ + return subagents.workflow(function(ctx, input) + for index = 1, %d do + ctx:agent({ agent = "alpha", prompt = "spam " .. index }) + end + end) + ]], luatool.max_jobs + 1) + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "job limit exceeded") + assert(#handle.runs == luatool.max_jobs, "the cap is enforced at the host boundary") + end) + end }, + + { "the guest cannot raise its own job cap through ctx", function() + with_host(function(handle, profiles) + local source = string.format([[ + return subagents.workflow(function(ctx, input) + ctx.max_jobs = nil + ctx.job_count = 0 + for index = 1, %d do + ctx:agent({ agent = "alpha", prompt = "spam " .. index }) + end + end) + ]], luatool.max_jobs + 1) + local text = luatool.handle({ prompt = "x", source = source }, profiles) + has(text, "job limit exceeded") + assert(#handle.runs == luatool.max_jobs, "the cap is private state, not a ctx field") + end) + end }, +} diff --git a/spec/test_models.lua b/spec/test_models.lua new file mode 100644 index 0000000..ae01396 --- /dev/null +++ b/spec/test_models.lua @@ -0,0 +1,158 @@ +-- subagents/models.lua: the four catalog query forms and the profile join. +-- +-- The fake host answers by query shape, so each case checks both what the tool +-- asked the host for (`handle.models_queries`) and how it rendered the answer. + +local fake = require("spec.fake_ext") +local models = require("subagents.models") + +local function has(text, needle) + assert(type(text) == "string", "expected a string result, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function catalog(query) + if query.model then + if query.model == "anthropic:sonnet" then + return { + found = true, + ref = "anthropic:sonnet", + wire_model = "claude-sonnet-4-6", + reasoning_default = "medium", + reasoning_levels = { "low", "medium", "high" }, + context_window = 200000, + max_tokens = 64000, + } + end + return { found = false } + end + if query.provider or query.query then + return { + matches = { + { ref = "anthropic:sonnet", wire_model = "claude-sonnet-4-6", reasoning_default = "medium" }, + { ref = "anthropic:opus", wire_model = "claude-opus-4-6" }, + }, + truncated = true, + } + end + return { + model = "anthropic:sonnet", + reasoning = "medium", + providers = { { name = "anthropic", style = "messages", models = 7 } }, + } +end + +local function profile_set() + local reviewer = { + name = "reviewer", + description = "Reviews changes", + model = "anthropic:sonnet", + reasoning = "high", + body = "b", + } + local scout = { name = "scout", description = "", body = "b" } + return { + list = { reviewer, scout }, + by_name = { reviewer = reviewer, scout = scout }, + warnings = {}, + } +end + +local function with_host(fn) + local handle = fake.install({ models_response = catalog }) + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +return { + { "no arguments reports the inherited model and provider counts", function() + with_host(function(handle, profiles) + local text = models.handle({}, profiles) + has(text, "inherited model: anthropic:sonnet") + has(text, "inherited reasoning: medium") + has(text, "anthropic (messages, 7 models)") + assert(next(handle.models_queries[1]) == nil, "the overview query carries no fields") + end) + end }, + + { "an exact model lookup reports its reasoning levels", function() + with_host(function(handle, profiles) + local text = models.handle({ model = "anthropic:sonnet" }, profiles) + has(text, "model: anthropic:sonnet") + has(text, "wire model: claude-sonnet-4-6") + has(text, "default reasoning: medium") + has(text, "reasoning levels: low, medium, high") + has(text, "context window: 200000") + assert(handle.models_queries[1].model == "anthropic:sonnet") + end) + end }, + + { "an unknown model says so and suggests a search", function() + with_host(function(handle, profiles) + local text = models.handle({ model = "acme:turbo" }, profiles) + has(text, "No configured model matches 'acme:turbo'") + has(text, "subagents.models") + end) + end }, + + { "a search reports matches and says when more exist", function() + with_host(function(handle, profiles) + local text = models.handle({ query = "son" }, profiles) + has(text, "2 match(es):") + has(text, "anthropic:sonnet — wire claude-sonnet-4-6, default reasoning medium") + has(text, "more exist — refine the query") + assert(handle.models_queries[1].limit == 10, "the default limit is 10") + end) + end }, + + { "limit is clamped to 1..50", function() + with_host(function(handle, profiles) + models.handle({ query = "son", limit = 500 }, profiles) + assert(handle.models_queries[1].limit == 50, tostring(handle.models_queries[1].limit)) + models.handle({ provider = "anthropic", limit = 0 }, profiles) + assert(handle.models_queries[2].limit == 1, tostring(handle.models_queries[2].limit)) + models.handle({ query = "son", limit = 7.6 }, profiles) + assert(handle.models_queries[3].limit == 7, "a fractional limit is floored") + end) + end }, + + { "an agent with a model reports that model", function() + with_host(function(handle, profiles) + local text = models.handle({ agent = "reviewer" }, profiles) + has(text, "agent: reviewer") + has(text, "description: Reviews changes") + has(text, "wire model: claude-sonnet-4-6") + has(text, "profile reasoning: high") + assert(handle.models_queries[1].model == "anthropic:sonnet", "the profile model is looked up") + end) + end }, + + { "an agent without a model says it inherits", function() + with_host(function(handle, profiles) + local text = models.handle({ agent = "scout" }, profiles) + has(text, "agent: scout") + has(text, "model: inherits the primary model") + has(text, "inherited model: anthropic:sonnet") + assert(next(handle.models_queries[1]) == nil, "the inherited case asks for the overview") + end) + end }, + + { "an unknown agent names the known profiles", function() + with_host(function(handle, profiles) + local text = models.handle({ agent = "ghost" }, profiles) + has(text, "Error: unknown agent 'ghost'") + has(text, "reviewer, scout") + end) + end }, + + { "a non-string field is refused", function() + with_host(function(handle, profiles) + has(models.handle({ query = 12 }, profiles), "Error: `query` must be a non-empty string") + has(models.handle({ model = "" }, profiles), "Error: `model` must be a non-empty string") + assert(#handle.models_queries == 0, "a refused call never reaches the host") + end) + end }, +} diff --git a/spec/test_profiles.lua b/spec/test_profiles.lua new file mode 100644 index 0000000..3a052b9 --- /dev/null +++ b/spec/test_profiles.lua @@ -0,0 +1,138 @@ +-- subagents/profiles.lua: two-layer recursive discovery over real directories. +-- +-- Discovery is exercised against temporary directories rather than the machine's +-- real config, by passing explicit roots to `discover` — the same seam the +-- extension uses for the user and project layers, in the same order. +-- +-- Needs luv (the recursive walk) and lyaml (the frontmatter); without either the +-- whole file skips rather than asserting on a degraded parse. + +local profiles = require("subagents.profiles") + +local function write(path, text) + assert(os.execute("mkdir -p " .. (path:match("^(.*)/[^/]+$") or "."))) + local file = assert(io.open(path, "w")) + file:write(text) + file:close() +end + +-- Build both layers once; every case reads the same discovery result. +local function discover_fixture() + local ok_uv, uv = pcall(require, "luv") + if not ok_uv then + return nil, "luv is not installed" + end + if not pcall(require, "lyaml") then + return nil, "lyaml is not installed" + end + + local tmp = assert(uv.fs_mkdtemp("/tmp/panto-subagents-profiles-XXXXXX")) + local user = tmp .. "/user/agents" + local project = tmp .. "/project/agents" + + write(user .. "/reviewer.md", table.concat({ + "---", + "description: Reviews changes", + "model: anthropic:sonnet", + "reasoning: high", + "---", + "You are a reviewer.", + "", + }, "\n")) + write(user .. "/nested/deeper/planner.md", "You plan.\n") + write(user .. "/shared.md", "---\nname: shared\ndescription: from the user layer\n---\nuser body\n") + write(user .. "/renamed.md", "---\nname: from-frontmatter\n---\nbody\n") + write(project .. "/shared.md", "---\nname: shared\ndescription: from the project layer\n---\nproject body\n") + write(project .. "/foreign.md", "---\ndescription: written for another harness\nmodel: claude-sonnet-4\n---\nbody\n") + + local found = profiles.discover({ user, project }) + os.execute("rm -rf " .. tmp) + return found +end + +local fixture, skip_reason = discover_fixture() + +local function fixture_or_skip() + if not fixture then + return nil, skip_reason + end + return fixture +end + +return { + { "frontmatter fields land on the profile", function() + local found, reason = fixture_or_skip() + if not found then + return "skip", reason + end + local reviewer = found.by_name.reviewer + assert(reviewer, "reviewer was not discovered") + assert(reviewer.name == "reviewer", "name should default to the file stem") + assert(reviewer.description == "Reviews changes", tostring(reviewer.description)) + assert(reviewer.model == "anthropic:sonnet", tostring(reviewer.model)) + assert(reviewer.reasoning == "high", tostring(reviewer.reasoning)) + assert(reviewer.body:find("You are a reviewer.", 1, true), "body was lost") + end }, + + { "discovery recurses and defaults the name to the stem", function() + local found, reason = fixture_or_skip() + if not found then + return "skip", reason + end + local planner = found.by_name.planner + assert(planner, "a nested profile was not discovered") + assert(planner.description == "", "a bodyless header means an empty description") + assert(planner.body == "You plan.\n", string.format("%q", planner.body)) + assert(found.by_name["from-frontmatter"], "`name` should override the stem") + assert(found.by_name.renamed == nil, "the stem must not survive an explicit name") + end }, + + { "the project layer shadows the user layer by name", function() + local found, reason = fixture_or_skip() + if not found then + return "skip", reason + end + local shared = found.by_name.shared + assert(shared, "shared was not discovered") + assert(shared.description == "from the project layer", tostring(shared.description)) + assert(shared.body == "project body\n", string.format("%q", shared.body)) + local count = 0 + for _, profile in ipairs(found.list) do + if profile.name == "shared" then + count = count + 1 + end + end + assert(count == 1, "a shadowed profile must appear once, saw " .. count) + end }, + + { "a foreign model spelling warns and inherits instead", function() + local found, reason = fixture_or_skip() + if not found then + return "skip", reason + end + local foreign = found.by_name.foreign + assert(foreign, "foreign was not discovered") + assert(foreign.model == nil, "an unparseable model must be dropped") + assert(foreign.description == "written for another harness", "the rest of the header must survive") + local warned = false + for _, warning in ipairs(found.warnings) do + if warning:find("ignoring model 'claude-sonnet-4'", 1, true) then + warned = true + end + end + assert(warned, "expected a warning naming the ignored model, got: " .. + table.concat(found.warnings, " | ")) + end }, + + { "the list is sorted by name", function() + local found, reason = fixture_or_skip() + if not found then + return "skip", reason + end + assert(#found.list >= 5, "expected every profile in the list, saw " .. #found.list) + for index = 2, #found.list do + assert(found.list[index - 1].name < found.list[index].name, + "list is not sorted at " .. index) + end + end }, +} diff --git a/spec/test_run.lua b/spec/test_run.lua new file mode 100644 index 0000000..706faff --- /dev/null +++ b/spec/test_run.lua @@ -0,0 +1,332 @@ +-- subagents/run.lua and subagents/spawn.lua: what a child is built out of, and +-- what the model is told afterwards. +-- +-- Every case installs a fresh fake host, so `handle.spawns` holds exactly the +-- children this case produced: one record per child agent, carrying the +-- resolve_model arguments, the store directory, the system messages it was +-- seeded with, the tool declarations it was given and the run_async options. +-- Profile sets are built inline rather than discovered: precedence, not +-- discovery, is what these cases are about, and an explicit set also keeps the +-- machine's real ~/.config out of the run. + +local fake = require("spec.fake_ext") +local run = require("subagents.run") +local spawn = require("subagents.spawn") + +local function has(text, needle) + assert(type(text) == "string", "expected a string result, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function profile_set() + local reviewer = { + name = "reviewer", + description = "Reviews changes", + model = "anthropic:sonnet", + reasoning = "high", + body = "You are a reviewer.\n", + } + local scout = { name = "scout", description = "", body = "" } + return { + list = { reviewer, scout }, + by_name = { reviewer = reviewer, scout = scout }, + warnings = {}, + } +end + +-- with_host(fn, opts): install the fake, run fn(handle, profiles), restore. +local function with_host(fn, opts) + local handle = fake.install(opts) + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +-- The child conversation a resumed reviewer loads: the manifest on its profile +-- system message, and the model/reasoning it last ran with on its last turn. +local function stored_reviewer() + return { + { role = "system", text = spawn.CHILD_ROLE }, + { + role = "system", + text = "You are a reviewer.\n", + metadata = { subagents = { owner = "0198-primary", agent = "reviewer" } }, + }, + { + role = "user", + text = "the first turn", + metadata = { subagents = { model = "openai:gpt-5.6", reasoning = "xhigh" } }, + }, + { role = "assistant", text = "first answer" }, + } +end + +return { + { "neither agent nor id is refused before anything is spawned", function() + with_host(function(handle, profiles) + local text = run.handle({ prompt = "do the thing" }, profiles) + has(text, "Error:") + has(text, "exactly one of `agent`") + assert(#handle.spawns == 0, "nothing may be spawned by a rejected call") + assert(not text:find("id:", 1, true), "a pre-allocation failure has no id") + end) + end }, + + { "both agent and id is refused", function() + with_host(function(handle, profiles) + local text = run.handle({ agent = "reviewer", id = "0198-x", prompt = "go" }, profiles) + has(text, "Error:") + has(text, "not both") + assert(#handle.spawns == 0) + end) + end }, + + { "an empty or missing prompt is refused", function() + with_host(function(handle, profiles) + has(run.handle({ agent = "reviewer", prompt = "" }, profiles), "`prompt` must be a non-empty string") + has(run.handle({ agent = "reviewer", prompt = " " }, profiles), "`prompt` must be a non-empty string") + has(run.handle({ agent = "reviewer" }, profiles), "`prompt` must be a non-empty string") + assert(#handle.spawns == 0) + end) + end }, + + { "an unknown agent names the known profiles", function() + with_host(function(handle, profiles) + local text = run.handle({ agent = "ghost", prompt = "go" }, profiles) + has(text, "unknown agent 'ghost'") + has(text, "reviewer, scout") + assert(#handle.spawns == 0) + end) + end }, + + { "a new child is seeded with the child-role and profile system messages", function() + with_host(function(handle, profiles) + run.handle({ agent = "reviewer", prompt = "Review the auth change." }, profiles) + assert(#handle.spawns == 1, "expected exactly one child") + local child = handle.spawns[1] + + assert(child.store_dir == handle.session.session_dir .. "/subagents/" .. handle.session.session_id, + "wrong child store dir: " .. tostring(child.store_dir)) + assert(child.session_id == nil, "a new child must not name a session") + assert(child.prompt == "Review the auth change.", tostring(child.prompt)) + + local messages = child.system_messages + assert(type(messages) == "table" and #messages == 2, "expected role + profile messages, saw " .. #messages) + assert(messages[1].text == spawn.CHILD_ROLE, "the first message is the fixed child role") + assert(messages[1].metadata == nil, "the role message carries no manifest") + assert(messages[2].text == "You are a reviewer.\n", "the profile body is the second message") + local manifest = messages[2].metadata.subagents + assert(manifest.owner == handle.session.session_id, tostring(manifest.owner)) + assert(manifest.agent == "reviewer", tostring(manifest.agent)) + end) + end }, + + { "the primary's system context comes first, then the role, then the profile", function() + with_host(function(handle, profiles) + run.handle({ agent = "reviewer", prompt = "go" }, profiles) + local messages = handle.spawns[1].system_messages + assert(#messages == 4, "expected two copied messages plus role and profile, saw " .. #messages) + assert(messages[1].text == "Project context.", tostring(messages[1].text)) + assert(messages[2].text == "House style.", tostring(messages[2].text)) + assert(messages[3].text == spawn.CHILD_ROLE, "the child role follows the primary's context") + assert(messages[4].text == "You are a reviewer.\n", "the profile body is last") + assert(messages[1].metadata == nil, "copied context carries no manifest") + end, { + primary_messages = { + { role = "system", text = "Project context." }, + { role = "user", text = "the parent dialogue is never copied" }, + { role = "assistant", text = "nor this" }, + { role = "system", text = "House style." }, + }, + }) + end }, + + { "the child store directory is created before the store is opened", function() + with_host(function(handle, profiles) + run.handle({ agent = "reviewer", prompt = "go" }, profiles) + local dir = handle.session.session_dir .. "/subagents/" .. handle.session.session_id + assert(handle.made_dir(dir), "the child catalog is created, not assumed: " .. + table.concat(handle.mkdirs, ", ")) + assert(handle.stores[1] == dir, "the store opens on that directory: " .. tostring(handle.stores[1])) + end) + end }, + + { "a child inherits the primary's tools except the subagents ones", function() + with_host(function(handle, profiles) + run.handle({ agent = "reviewer", prompt = "go" }, profiles) + local child = handle.spawns[1] + assert(table.concat(child.tools, ",") == "bash,read_file", + "expected only the non-subagents tools, saw " .. table.concat(child.tools, ",")) + for _, decl in ipairs(child.tool_decls) do + assert(decl._source == fake.SOURCE, "the re-registration tag must be passed through untouched") + end + end) + end }, + + { "the resolved model and reasoning are recorded on the turn", function() + with_host(function(handle, profiles) + run.handle({ agent = "reviewer", prompt = "go" }, profiles) + local metadata = handle.spawns[1].run.metadata + assert(type(metadata) == "table" and type(metadata.subagents) == "table", + "every child turn records what it ran on") + assert(metadata.subagents.model == "anthropic:sonnet", tostring(metadata.subagents.model)) + assert(metadata.subagents.reasoning == "high", tostring(metadata.subagents.reasoning)) + assert(handle.spawns[1].run.dispatch_tools ~= false, "an ordinary child dispatches its tools") + end) + end }, + + { "the child-role instruction states the contract the design fixes", function() + local role = spawn.CHILD_ROLE + has(role, "You are a subagent") + has(role, "self-contained report") + has(role, "verbatim") + has(role, "cannot ask the user questions") + end }, + + { "model and reasoning resolve tool over profile over inherited", function() + with_host(function(handle, profiles) + run.handle({ agent = "reviewer", prompt = "a" }, profiles) + assert(handle.spawns[1].model == "anthropic:sonnet", "the profile model applies") + assert(handle.spawns[1].reasoning == "high", "the profile reasoning applies") + + run.handle({ agent = "reviewer", prompt = "b", model = "openai:gpt-5.6", reasoning = "xhigh" }, profiles) + assert(handle.spawns[2].model == "openai:gpt-5.6", "the call overrides the profile") + assert(handle.spawns[2].reasoning == "xhigh", "the call overrides the profile") + + run.handle({ agent = "reviewer", prompt = "d", reasoning = "low" }, profiles) + assert(handle.spawns[3].model == "anthropic:sonnet", "model and reasoning resolve independently") + assert(handle.spawns[3].reasoning == "low") + end) + end }, + + { "a profile that names neither inherits the primary's pair", function() + with_host(function(handle, profiles) + run.handle({ agent = "scout", prompt = "look around" }, profiles) + assert(handle.spawns[1].model == "openai:gpt-5.6", tostring(handle.spawns[1].model)) + assert(handle.spawns[1].reasoning == "low", tostring(handle.spawns[1].reasoning)) + end, { session = { model = "openai:gpt-5.6", reasoning = "low" } }) + end }, + + { "a profile with an empty body contributes no system message", function() + with_host(function(handle, profiles) + run.handle({ agent = "scout", prompt = "look around" }, profiles) + local messages = handle.spawns[1].system_messages + assert(#messages == 1, "expected only the child-role message, saw " .. #messages) + assert(messages[1].text == spawn.CHILD_ROLE) + end) + end }, + + { "a resume loads the stored conversation and seeds nothing", function() + with_host(function(handle, profiles) + handle.add_session("0198-child", stored_reviewer()) + handle.queue({ output = "second turn" }) + + local text = run.handle({ id = "0198-child", prompt = "now the tests" }, profiles) + local child = handle.spawns[1] + assert(child.session_id == "0198-child", tostring(child.session_id)) + assert(child.resumed, "the agent is built on the resolved session") + assert(#child.system_messages == 0, "the stored conversation is canonical on resume") + assert(child.store_dir == handle.session.session_dir .. "/subagents/" .. handle.session.session_id) + has(text, "id: 0198-child") + has(text, "second turn") + end) + end }, + + { "a resumed child keeps the model and reasoning of its last turn", function() + with_host(function(handle, profiles) + handle.add_session("0198-child", stored_reviewer()) + handle.queue({}) + run.handle({ id = "0198-child", prompt = "carry on" }, profiles) + assert(handle.spawns[1].model == "openai:gpt-5.6", tostring(handle.spawns[1].model)) + assert(handle.spawns[1].reasoning == "xhigh", tostring(handle.spawns[1].reasoning)) + end) + end }, + + { "a per-turn override beats the stored default and only where it is given", function() + with_host(function(handle, profiles) + handle.add_session("0198-child", stored_reviewer()) + handle.queue({}) + run.handle({ id = "0198-child", prompt = "carry on", reasoning = "high" }, profiles) + assert(handle.spawns[1].reasoning == "high", "the call overrides the stored default") + assert(handle.spawns[1].model == "openai:gpt-5.6", "the model keeps its last effective value") + end) + end }, + + { "a resumed child takes its agent name from the manifest", function() + with_host(function(handle, profiles) + handle.add_session("0198-child", stored_reviewer()) + handle.queue({ output = "done" }) + local text = run.handle({ id = "0198-child", prompt = "carry on" }, profiles) + has(text, "agent: reviewer") + end) + end }, + + { "an id this session never started is refused before anything is built", function() + with_host(function(handle, profiles) + local text = run.handle({ id = "0198-nope", prompt = "carry on" }, profiles) + has(text, "Error:") + has(text, "unknown subagent id '0198-nope'") + assert(#handle.spawns == 0, "no child is built for an id that does not resolve") + assert(#handle.runs == 0, "and no turn is started") + end) + end }, + + { "a completed result renders every field", function() + with_host(function(handle, profiles) + handle.queue({ id = "0198-abc", status = "completed", output = "Found two issues." }) + local text = run.handle({ agent = "reviewer", prompt = "review" }, profiles) + assert(text == table.concat({ + "id: 0198-abc", + "agent: reviewer", + "status: completed", + "resumable: true", + "--- output ---", + "Found two issues.", + }, "\n"), "unexpected block:\n" .. text) + end) + end }, + + { "a failed first turn reports the error and is not resumable", function() + with_host(function(handle, profiles) + handle.queue({ id = "0198-def", status = "failed", error = "provider refused", resumable = false }) + local text = run.handle({ agent = "reviewer", prompt = "review" }, profiles) + has(text, "status: failed") + has(text, "resumable: false") + has(text, "provider refused") + end) + end }, + + { "resumability is read back from the store after the turn settles", function() + with_host(function(handle, profiles) + handle.queue({ id = "0198-ghi", output = "wrote something" }) + has(run.handle({ agent = "reviewer", prompt = "review" }, profiles), "resumable: true") + assert(handle.sessions["0198-ghi"], "a durable child leaves a session behind") + + handle.queue({ id = "0198-jkl", output = "died young", resumable = false }) + has(run.handle({ agent = "reviewer", prompt = "review" }, profiles), "resumable: false") + assert(handle.sessions["0198-jkl"] == nil, "no file, no continuation") + end) + end }, + + { "a cancelled child settles as cancelled", function() + with_host(function(handle, profiles) + handle.queue({ id = "0198-ghi", status = "cancelled", error = "cancelled by the user" }) + local text = run.handle({ agent = "reviewer", prompt = "review" }, profiles) + has(text, "status: cancelled") + has(text, "cancelled by the user") + end) + end }, + + { "a model the host cannot resolve is reported and starts nothing", function() + with_host(function(handle, profiles) + local text = run.handle({ agent = "reviewer", prompt = "review", model = "openai:ghost" }, profiles) + assert(text == "Error: resolve_model: unknown model 'openai:ghost'", text) + assert(#handle.resolves == 1, "the host decides what a model reference means, not the rock") + assert(handle.resolves[1].model == "openai:ghost", tostring(handle.resolves[1].model)) + assert(#handle.spawns == 0, "the child is never built") + assert(#handle.runs == 0, "a rejected child is never started") + end, { unknown_models = { ["openai:ghost"] = true } }) + end }, +} diff --git a/spec/test_toml_workflows.lua b/spec/test_toml_workflows.lua new file mode 100644 index 0000000..8ae2cac --- /dev/null +++ b/spec/test_toml_workflows.lua @@ -0,0 +1,404 @@ +-- subagents/toml_workflows.lua: validation, dependency prompt assembly, branch +-- failure, terminal ordering, discovery, and the generated commands. +-- +-- Validation and execution run off definitions built in Lua, so they exercise +-- the DAG rules without needing the TOML rock. The parse, discovery, and command +-- cases do need toml2lua (and luv, for the recursive walk) and skip without them. +-- +-- Discovery is pointed at temporary directories by replacing paths.config_roots +-- for the duration of the case: the two layers, in the same order the extension +-- uses, without reading the machine's real ~/.config. + +local fake = require("spec.fake_ext") +local paths = require("subagents.paths") +local toml_workflows = require("subagents.toml_workflows") + +local function has(text, needle) + assert(type(text) == "string", "expected a string, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function profile_set() + local set = { list = {}, by_name = {}, warnings = {} } + for _, name in ipairs({ "alpha", "beta", "gamma" }) do + local profile = { name = name, description = name, body = "You are " .. name .. ".\n" } + set.list[#set.list + 1] = profile + set.by_name[name] = profile + end + return set +end + +local function with_host(fn) + local handle = fake.install() + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +local function write(path, text) + assert(os.execute("mkdir -p " .. (path:match("^(.*)/[^/]+$") or "."))) + local file = assert(io.open(path, "w")) + file:write(text) + file:close() +end + +-- Point discovery at two temporary layers for the duration of `fn`. +local function with_layers(files, fn) + local ok_uv, uv = pcall(require, "luv") + if not ok_uv then + return "skip", "luv is not installed" + end + if not pcall(require, "toml") then + return "skip", "toml2lua is not installed" + end + + local tmp = assert(uv.fs_mkdtemp("/tmp/panto-subagents-workflows-XXXXXX")) + for name, text in pairs(files) do + write(tmp .. "/" .. name, text) + end + + local original = paths.config_roots + paths.config_roots = function(kind) + return { tmp .. "/user/" .. kind, tmp .. "/project/" .. kind } + end + local ok, err = pcall(fn, tmp) + paths.config_roots = original + os.execute("rm -rf " .. tmp) + if not ok then + error(err, 0) + end +end + +local BRANCH_DEF = { + name = "branch", + steps = { + { id = "a", agent = "alpha", prompt = "Inspect." }, + { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } }, + { id = "c", agent = "gamma", prompt = "Unrelated." }, + }, +} + +return { + { "a valid definition normalizes and marks its terminal steps", function() + local def = assert(toml_workflows.validate(BRANCH_DEF, "fallback")) + assert(def.name == "branch") + assert(#def.steps == 3) + assert(def.terminal.b and def.terminal.c, "b and c are nobody's dependency") + assert(def.terminal.a == nil, "a is depended on, so it is not terminal") + assert(def.by_id.b.needs[1] == "a") + end }, + + { "the name falls back to the file stem", function() + local def = assert(toml_workflows.validate({ steps = BRANCH_DEF.steps }, "from-stem")) + assert(def.name == "from-stem", tostring(def.name)) + end }, + + { "structural mistakes are refused with a readable reason", function() + local function bad(def, needle) + local ok, err = toml_workflows.validate(def, "w") + assert(ok == nil, "expected a rejection for " .. needle) + has(err, needle) + end + + bad({ steps = {} }, "has no `steps` array") + bad({ steps = "nope" }, "has no `steps` array") + bad({ steps = { { id = "a", agent = "alpha" } } }, "`prompt` is required") + bad({ steps = { { id = "a", prompt = "p" } } }, "`agent` is required") + bad({ steps = { { agent = "alpha", prompt = "p" } } }, "`id` is required") + bad({ steps = { + { id = "a", agent = "alpha", prompt = "p" }, + { id = "a", agent = "beta", prompt = "q" }, + } }, "duplicate step id 'a'") + bad({ steps = { + { id = "a", agent = "alpha", prompt = "p", needs = { "ghost" } }, + } }, "unknown dependency 'ghost'") + bad({ steps = { + { id = "a", agent = "alpha", prompt = "p", needs = { "b" } }, + { id = "b", agent = "beta", prompt = "q", needs = { "a" } }, + } }, "dependency cycle") + end }, + + { "a step's prompt carries the input and each dependency in needs order", function() + local step = { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a", "z" } } + local settled = { + a = { status = "completed", output = "A output" }, + z = { status = "failed", error = "boom" }, + } + local prompt = toml_workflows.step_prompt(step, "the workflow input", settled) + assert(prompt == table.concat({ + "Summarize.", + "", + "## Workflow input", + "", + "the workflow input", + "", + "## Output of a", + "", + "A output", + "", + "## Output of z", + "", + "[failed: boom]", + }, "\n"), string.format("unexpected prompt:\n%s", prompt)) + end }, + + { "a dependent receives its dependency's output and runs after it", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate({ + name = "chain", + steps = { + { id = "a", agent = "alpha", prompt = "Inspect." }, + { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } }, + }, + }, "chain")) + handle.queue_for("alpha", { output = "A output" }) + handle.queue_for("beta", { output = "B output" }) + + local results = toml_workflows.run(def, "the input", profiles) + assert(#handle.spawns == 2, "both steps ran") + assert(handle.spawns[1].label == "alpha", "the root starts first") + has(handle.spawns[2].prompt, "## Workflow input\n\nthe input") + has(handle.spawns[2].prompt, "## Output of a\n\nA output") + assert(#results == 1 and results[1].id == "b", "only the terminal step is returned") + assert(results[1].output == "B output") + end) + end }, + + { "a failed step skips its dependents while other branches finish", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate(BRANCH_DEF, "branch")) + handle.queue_for("alpha", { status = "failed", error = "alpha died", settle = 1 }) + handle.queue_for("gamma", { output = "gamma output", settle = 2 }) + + local results = toml_workflows.run(def, "input", profiles) + + assert(#handle.spawns == 2, "the skipped step never spawns") + for _, spec in ipairs(handle.spawns) do + assert(spec.label ~= "beta", "beta depends on a failure and must not run") + end + assert(#results == 2, "both terminal steps are reported") + assert(results[1].id == "b" and results[1].status == "skipped", "terminals keep declaration order") + has(results[1].error, "a dependency did not complete") + assert(results[2].id == "c" and results[2].status == "completed") + assert(results[2].output == "gamma output") + end) + end }, + + { "independent roots start together", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate({ + name = "fan", + steps = { + { id = "a", agent = "alpha", prompt = "A" }, + { id = "c", agent = "gamma", prompt = "C" }, + { id = "b", agent = "beta", prompt = "B", needs = { "a", "c" } }, + }, + }, "fan")) + handle.queue_for("alpha", { output = "A out", settle = 2 }) + handle.queue_for("gamma", { output = "C out", settle = 1 }) + handle.queue_for("beta", { output = "B out" }) + + local results = toml_workflows.run(def, "input", profiles) + assert(handle.max_live == 2, "both roots were in flight at once, peaked at " .. handle.max_live) + has(handle.spawns[3].prompt, "## Output of a\n\nA out") + has(handle.spawns[3].prompt, "## Output of c\n\nC out") + assert(#results == 1 and results[1].id == "b" and results[1].status == "completed") + end) + end }, + + { "TOML parses into the same definition shape", function() + if not pcall(require, "toml") then + return "skip", "toml2lua is not installed" + end + local def, err = toml_workflows.parse(table.concat({ + 'name = "review-chain"', + 'description = "Two angles, then a synthesis."', + '', + '[[steps]]', + 'id = "correctness"', + 'agent = "alpha"', + 'prompt = "Review for correctness."', + '', + '[[steps]]', + 'id = "synthesis"', + 'agent = "beta"', + 'prompt = "Synthesize."', + 'reasoning = "high"', + 'needs = ["correctness"]', + }, "\n"), "stem") + assert(def, tostring(err)) + assert(def.name == "review-chain") + assert(def.description == "Two angles, then a synthesis.") + assert(#def.steps == 2) + assert(def.steps[2].reasoning == "high") + assert(def.terminal.synthesis and def.terminal.correctness == nil) + + local bad, bad_err = toml_workflows.parse("name = = broken", "stem") + assert(bad == nil, "malformed TOML must not parse") + has(bad_err, "invalid TOML") + end }, + + { "discovery shadows by name and registers one command per valid workflow", function() + local valid = table.concat({ + 'name = "review-chain"', + 'description = "Inspect a change from two angles."', + '[[steps]]', + 'id = "one"', + 'agent = "alpha"', + 'prompt = "Look."', + }, "\n") + local user_shadowed = table.concat({ + 'name = "shadowed"', + 'description = "the user layer"', + '[[steps]]', + 'id = "one"', + 'agent = "alpha"', + 'prompt = "user"', + }, "\n") + local project_shadowed = table.concat({ + 'name = "shadowed"', + 'description = "the project layer"', + '[[steps]]', + 'id = "one"', + 'agent = "beta"', + 'prompt = "project"', + }, "\n") + local broken = table.concat({ + '[[steps]]', + 'id = "dup"', + 'agent = "alpha"', + 'prompt = "one"', + '[[steps]]', + 'id = "dup"', + 'agent = "alpha"', + 'prompt = "two"', + }, "\n") + + return with_layers({ + ["user/workflows/review-chain.toml"] = valid, + ["user/workflows/nested/shadowed.toml"] = user_shadowed, + ["project/workflows/shadowed.toml"] = project_shadowed, + ["project/workflows/broken.toml"] = broken, + }, function() + with_host(function(handle, profiles) + local found = toml_workflows.discover_and_register(profiles) + + assert(handle.commands_by_name["workflow:review-chain"], "a valid workflow registers a command") + assert(handle.commands_by_name["workflow:review-chain"].description == + "Inspect a change from two angles.", "the description comes from the file") + assert(handle.commands_by_name["workflow:shadowed"].description == "the project layer", + "the project layer shadows the user layer") + assert(handle.commands_by_name["workflow:broken"] == nil, "an invalid file registers nothing") + assert(#handle.commands == 2, "exactly two commands, saw " .. #handle.commands) + + local warned = false + for _, warning in ipairs(found.warnings) do + if warning:find("duplicate step id 'dup'", 1, true) then + warned = true + end + end + assert(warned, "the invalid file's error is kept: " .. table.concat(found.warnings, " | ")) + + -- The command tail becomes the workflow input. + handle.queue_for("alpha", { output = "looked" }) + local text = handle.commands_by_name["workflow:review-chain"].handler("check the parser") + has(text, "step: one") + has(text, "status: completed") + has(text, "looked") + has(handle.spawns[1].prompt, "## Workflow input\n\ncheck the parser") + + -- And the tool can run the same workflow, or explain a broken one. + has(toml_workflows.handle({ name = "broken", prompt = "x" }, profiles), + "failed to load") + has(toml_workflows.handle({ name = "ghost", prompt = "x" }, profiles), + "unknown workflow 'ghost'") + end) + end) + end }, + + { "a broken project workflow shadows the user one under its declared name", function() + local user_valid = table.concat({ + 'name = "dup"', + 'description = "the user layer"', + '[[steps]]', + 'id = "one"', + 'agent = "alpha"', + 'prompt = "user"', + }, "\n") + -- Declares the same name from a differently-named file, and is invalid. + local project_broken = table.concat({ + 'name = "dup"', + '[[steps]]', + 'id = "same"', + 'agent = "alpha"', + 'prompt = "one"', + '[[steps]]', + 'id = "same"', + 'agent = "alpha"', + 'prompt = "two"', + }, "\n") + + return with_layers({ + ["user/workflows/dup.toml"] = user_valid, + ["project/workflows/whatever.toml"] = project_broken, + }, function() + with_host(function(handle, profiles) + local found = toml_workflows.discover_and_register(profiles) + + local entry = found.by_name["dup"] + assert(entry, "the project error must be indexed under the name it declares") + assert(entry.definition == nil, "the shadowed user definition must not survive") + has(entry.error, "duplicate step id 'same'") + assert(#handle.commands == 0, "a shadowed-out workflow registers no command") + has(toml_workflows.handle({ name = "dup", prompt = "x" }, profiles), "failed to load") + end) + end) + end }, + + { "a command whose workflow cannot run reports the error, not a stack trace", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate({ + name = "ghosted", + steps = { { id = "one", agent = "nobody", prompt = "Do it." } }, + }, "ghosted")) + local ok, err = pcall(toml_workflows.run, def, "input", profiles) + assert(not ok, "an unknown agent stops the workflow") + has(tostring(err), "unknown agent 'nobody'") + + has(toml_workflows.handle({ + prompt = "x", + steps = { { id = "one", agent = "nobody", prompt = "Do it." } }, + }, profiles), "Error:") + assert(#handle.spawns == 0) + end) + end }, + + { "the tool takes exactly one of name and steps", function() + with_host(function(handle, profiles) + has(toml_workflows.handle({ prompt = "x" }, profiles), "pass exactly one of `name`") + has(toml_workflows.handle({ prompt = "x", name = "a", steps = {} }, profiles), "not both") + has(toml_workflows.handle({ name = "a" }, profiles), "prompt is required") + assert(#handle.spawns == 0) + end) + end }, + + { "the tool runs a transient definition", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { output = "transient output" }) + local text = toml_workflows.handle({ + prompt = "the input", + steps = { { id = "only", agent = "alpha", prompt = "Do it." } }, + }, profiles) + has(text, "step: only") + has(text, "transient output") + has(handle.spawns[1].prompt, "Do it.\n\n## Workflow input\n\nthe input") + + has(toml_workflows.handle({ + prompt = "x", + steps = { { id = "only", agent = "alpha", prompt = "p", needs = { "ghost" } } }, + }, profiles), "unknown dependency 'ghost'") + end) + end }, +} diff --git a/spec/test_workflow.lua b/spec/test_workflow.lua new file mode 100644 index 0000000..c5ba06c --- /dev/null +++ b/spec/test_workflow.lua @@ -0,0 +1,299 @@ +-- subagents/workflow.lua: the callback API's awaiting, failure-as-value rule, +-- structured workers, the concurrency gate, and the job budget. +-- +-- The fake host settles jobs in the order each outcome's `settle` key asks for, +-- which is what makes the ordering cases meaningful: input order and settle +-- order are deliberately different everywhere below. + +local fake = require("spec.fake_ext") +local workflow = require("subagents.workflow") + +local function has(text, needle) + assert(type(text) == "string", "expected a string, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function profile_set() + local set = { list = {}, by_name = {}, warnings = {} } + for _, name in ipairs({ "alpha", "beta", "gamma" }) do + local profile = { name = name, description = name, body = "You are " .. name .. ".\n" } + set.list[#set.list + 1] = profile + set.by_name[name] = profile + end + return set +end + +local function with_host(fn, opts) + local handle = fake.install(opts) + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +local function three_children(ctx) + local handles = {} + for index, name in ipairs({ "alpha", "beta", "gamma" }) do + handles[index] = ctx:agent({ agent = name, prompt = "work on " .. name }) + end + return handles +end + +local function json_available() + return pcall(require, "dkjson") +end + +local ITEMS_SCHEMA = { + type = "object", + required = { "items" }, + properties = { + items = { type = "array", items = { type = "string" } }, + }, +} + +local GHOST = { unknown_models = { ["openai:ghost"] = true } } + +return { + { "await all returns results in input order, not settle order", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { output = "A", settle = 3 }) + handle.queue_for("beta", { output = "B", settle = 1 }) + handle.queue_for("gamma", { output = "C", settle = 2 }) + + local results = workflow.execute(workflow.workflow(function(ctx) + return ctx:await(three_children(ctx), "all") + end), "input", { profiles = profiles }) + + assert(#results == 3, "expected three results") + assert(results[1].output == "A", tostring(results[1].output)) + assert(results[2].output == "B", tostring(results[2].output)) + assert(results[3].output == "C", tostring(results[3].output)) + assert(#handle.spawns == 3, "one child per ctx:agent") + assert(handle.spawns[1].prompt == "work on alpha") + end) + end }, + + { "await first returns the earliest settler and the remaining handles", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { output = "A", settle = 3 }) + handle.queue_for("beta", { output = "B", settle = 1 }) + handle.queue_for("gamma", { output = "C", settle = 2 }) + + local order = workflow.execute(workflow.workflow(function(ctx) + local handles = three_children(ctx) + local seen = {} + while #handles > 0 do + local result, remaining = ctx:await(handles, "first") + seen[#seen + 1] = result.output + assert(#remaining == #handles - 1, "one handle settles per await") + handles = remaining + end + return seen + end), "input", { profiles = profiles }) + + assert(table.concat(order, ",") == "B,C,A", table.concat(order, ",")) + end) + end }, + + { "handle:await settles one child", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { output = "just me" }) + local result = workflow.execute(workflow.workflow(function(ctx) + return ctx:agent({ agent = "alpha", prompt = "go" }):await() + end), "input", { profiles = profiles }) + assert(result.status == "completed", tostring(result.status)) + assert(result.output == "just me", tostring(result.output)) + end) + end }, + + { "the gate keeps four children in flight and queues the rest", function() + with_host(function(handle, profiles) + local results = workflow.execute(workflow.workflow(function(ctx) + local handles = {} + for index = 1, 5 do + handles[index] = ctx:agent({ agent = "alpha", prompt = "task " .. index }) + end + assert(#handle.runs == 4, "four turns run at once, saw " .. #handle.runs) + return ctx:await(handles, "all") + end), "input", { profiles = profiles }) + + assert(#results == 5, "every child reports") + assert(#handle.runs == 5, "the queued child starts once a slot frees up") + assert(handle.max_live == 4, "never more than four at once, peaked at " .. handle.max_live) + end) + end }, + + { "a rejected child is a failed result, not an error", function() + with_host(function(handle, profiles) + handle.queue_for("beta", { output = "B" }) + + local results = workflow.execute(workflow.workflow(function(ctx) + local first = ctx:agent({ agent = "alpha", model = "openai:ghost", prompt = "a" }) + local second = ctx:agent({ agent = "beta", prompt = "b" }) + return ctx:await({ first, second }, "all") + end), "input", { profiles = profiles }) + + assert(results[1].status == "failed", tostring(results[1].status)) + has(results[1].error, "unknown model 'openai:ghost'") + assert(results[1].resumable == false, "a child that never allocated is not resumable") + assert(results[2].status == "completed", "a sibling failure must not disturb this one") + assert(#handle.runs == 1, "only the sibling was ever started") + end, GHOST) + end }, + + { "a failed child is reported as a value", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { status = "failed", error = "provider refused", resumable = false }) + local result = workflow.execute(workflow.workflow(function(ctx) + return ctx:agent({ agent = "alpha", prompt = "a" }):await() + end), "input", { profiles = profiles }) + assert(result.status == "failed") + assert(result.error == "provider refused", tostring(result.error)) + end) + end }, + + { "an id with a turn already in flight is refused", function() + with_host(function(handle, profiles) + handle.add_session("0198-child", { + { role = "system", text = "You are alpha.\n", + metadata = { subagents = { owner = "0198-primary", agent = "alpha" } } }, + }) + handle.queue({ output = "the first turn wins" }) + + local results = workflow.execute(workflow.workflow(function(ctx) + local first = ctx:agent({ id = "0198-child", prompt = "a" }) + local second = ctx:agent({ id = "0198-child", prompt = "b" }) + return ctx:await({ first, second }, "all") + end), "input", { profiles = profiles }) + + assert(results[1].status == "completed", tostring(results[1].error)) + assert(results[2].status == "failed", "one turn per child at a time") + has(results[2].error, "already has a turn in flight") + assert(#handle.runs == 1, "the second call never starts a turn") + end) + end }, + + { "a structured worker decodes and validates its output", function() + if not json_available() then + return "skip", "dkjson is not installed" + end + with_host(function(handle, profiles) + handle.queue_for("alpha", { structured_json = '{"items":["x","y"]}' }) + local result = workflow.execute(workflow.workflow(function(ctx) + return ctx:agent({ + agent = "alpha", + prompt = "split it", + output = { description = "Return the work items.", schema = ITEMS_SCHEMA }, + }):await() + end), "input", { profiles = profiles }) + + assert(result.status == "completed", tostring(result.error)) + assert(type(result.output) == "table", "structured output decodes to a table") + assert(result.output.items[1] == "x" and result.output.items[2] == "y") + assert(result.resumable == false, "a one-shot child is not resumable") + assert(result.id == nil, "and has no durable session to name") + assert(#handle.stores == 0, "a structured worker never touches the child catalog") + + local child = handle.spawns[1] + assert(type(child.tool_choice) == "table" and child.tool_choice.name == "emit_result", + "the output tool is the only choice the child has") + assert(child.run.dispatch_tools == false, "a one-shot child never dispatches a tool call") + assert(#child.tool_decls == 1, "the output tool replaces the inherited set, saw " .. + table.concat(child.tools, ",")) + + local decl = child.tool_decls[1] + assert(decl.name == "emit_result", "the synthetic tool is named for the host") + assert(decl.description == "Return the work items.") + assert(decl.schema == ITEMS_SCHEMA, "the schema is passed through untouched") + assert(decl._source == nil and decl._ctx == nil and decl._vt == nil, + "the output tool is declaration-only: nothing may dispatch it") + end) + end }, + + { "structured output that violates the schema fails the result", function() + if not json_available() then + return "skip", "dkjson is not installed" + end + with_host(function(handle, profiles) + handle.queue_for("alpha", { structured_json = '{"nope":1}' }) + local result = workflow.execute(workflow.workflow(function(ctx) + return ctx:agent({ + agent = "alpha", + prompt = "split it", + output = { schema = ITEMS_SCHEMA }, + }):await() + end), "input", { profiles = profiles }) + + assert(result.status == "failed", "a schema violation is never a success") + has(result.error, "structured output failed validation") + assert(result.output == nil, "invalid output is not handed to the caller") + end) + end }, + + { "a structured worker that answers in prose fails", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { output = "prose, not a tool call" }) + local result = workflow.execute(workflow.workflow(function(ctx) + return ctx:agent({ + agent = "alpha", + prompt = "split it", + output = { schema = ITEMS_SCHEMA }, + }):await() + end), "input", { profiles = profiles }) + assert(result.status == "failed", tostring(result.status)) + has(result.error, "did not call the required 'emit_result' output tool") + end) + end }, + + { "an already settled handle is served without polling again", function() + with_host(function(handle, profiles) + handle.queue_for("beta", { output = "B" }) + handle.queue_for("gamma", { output = "C" }) + + local first_status = workflow.execute(workflow.workflow(function(ctx) + local rejected = ctx:agent({ agent = "alpha", model = "openai:ghost", prompt = "a" }) + local live = { rejected, ctx:agent({ agent = "beta", prompt = "b" }) } + local result, remaining = ctx:await(live, "first") + assert(#remaining == 1, "the pending sibling stays outstanding") + assert(handle.polls == 0, "a cached result must not reach the job machinery") + return result.status + end), "input", { profiles = profiles }) + + assert(first_status == "failed", tostring(first_status)) + assert(#handle.runs == 1, "only the live sibling was ever started") + end, GHOST) + end }, + + { "max_jobs caps how many children one workflow starts", function() + with_host(function(handle, profiles) + local ok, err = pcall(workflow.execute, workflow.workflow(function(ctx) + three_children(ctx) + end), "input", { profiles = profiles, max_jobs = 2 }) + assert(not ok, "the third ctx:agent must be refused") + has(tostring(err), "job limit exceeded") + assert(#handle.runs == 2, "the capped call never reaches the host") + end) + end }, + + { "handles the callback never awaited are settled before returning", function() + with_host(function(handle, profiles) + workflow.execute(workflow.workflow(function(ctx) + ctx:agent({ agent = "alpha", prompt = "orphan" }) + return "done" + end), "input", { profiles = profiles }) + assert(#handle.jobs == 1, "one child ran") + assert(handle.jobs[1]._settled, "no child is left running") + end) + end }, + + { "an unknown agent inside a workflow is a workflow error", function() + with_host(function(handle, profiles) + local ok, err = pcall(workflow.execute, workflow.workflow(function(ctx) + ctx:agent({ agent = "ghost", prompt = "go" }) + end), "input", { profiles = profiles }) + assert(not ok, "an unknown profile is a programmer error, not a child failure") + has(tostring(err), "unknown agent 'ghost'") + end) + end }, +} |