diff options
Diffstat (limited to 'spec/test_toml_workflows.lua')
| -rw-r--r-- | spec/test_toml_workflows.lua | 404 |
1 files changed, 404 insertions, 0 deletions
diff --git a/spec/test_toml_workflows.lua b/spec/test_toml_workflows.lua new file mode 100644 index 0000000..8ae2cac --- /dev/null +++ b/spec/test_toml_workflows.lua @@ -0,0 +1,404 @@ +-- subagents/toml_workflows.lua: validation, dependency prompt assembly, branch +-- failure, terminal ordering, discovery, and the generated commands. +-- +-- Validation and execution run off definitions built in Lua, so they exercise +-- the DAG rules without needing the TOML rock. The parse, discovery, and command +-- cases do need toml2lua (and luv, for the recursive walk) and skip without them. +-- +-- Discovery is pointed at temporary directories by replacing paths.config_roots +-- for the duration of the case: the two layers, in the same order the extension +-- uses, without reading the machine's real ~/.config. + +local fake = require("spec.fake_ext") +local paths = require("subagents.paths") +local toml_workflows = require("subagents.toml_workflows") + +local function has(text, needle) + assert(type(text) == "string", "expected a string, got " .. type(text)) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text)) +end + +local function profile_set() + local set = { list = {}, by_name = {}, warnings = {} } + for _, name in ipairs({ "alpha", "beta", "gamma" }) do + local profile = { name = name, description = name, body = "You are " .. name .. ".\n" } + set.list[#set.list + 1] = profile + set.by_name[name] = profile + end + return set +end + +local function with_host(fn) + local handle = fake.install() + local ok, err = pcall(fn, handle, profile_set()) + handle.restore() + if not ok then + error(err, 0) + end +end + +local function write(path, text) + assert(os.execute("mkdir -p " .. (path:match("^(.*)/[^/]+$") or "."))) + local file = assert(io.open(path, "w")) + file:write(text) + file:close() +end + +-- Point discovery at two temporary layers for the duration of `fn`. +local function with_layers(files, fn) + local ok_uv, uv = pcall(require, "luv") + if not ok_uv then + return "skip", "luv is not installed" + end + if not pcall(require, "toml") then + return "skip", "toml2lua is not installed" + end + + local tmp = assert(uv.fs_mkdtemp("/tmp/panto-subagents-workflows-XXXXXX")) + for name, text in pairs(files) do + write(tmp .. "/" .. name, text) + end + + local original = paths.config_roots + paths.config_roots = function(kind) + return { tmp .. "/user/" .. kind, tmp .. "/project/" .. kind } + end + local ok, err = pcall(fn, tmp) + paths.config_roots = original + os.execute("rm -rf " .. tmp) + if not ok then + error(err, 0) + end +end + +local BRANCH_DEF = { + name = "branch", + steps = { + { id = "a", agent = "alpha", prompt = "Inspect." }, + { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } }, + { id = "c", agent = "gamma", prompt = "Unrelated." }, + }, +} + +return { + { "a valid definition normalizes and marks its terminal steps", function() + local def = assert(toml_workflows.validate(BRANCH_DEF, "fallback")) + assert(def.name == "branch") + assert(#def.steps == 3) + assert(def.terminal.b and def.terminal.c, "b and c are nobody's dependency") + assert(def.terminal.a == nil, "a is depended on, so it is not terminal") + assert(def.by_id.b.needs[1] == "a") + end }, + + { "the name falls back to the file stem", function() + local def = assert(toml_workflows.validate({ steps = BRANCH_DEF.steps }, "from-stem")) + assert(def.name == "from-stem", tostring(def.name)) + end }, + + { "structural mistakes are refused with a readable reason", function() + local function bad(def, needle) + local ok, err = toml_workflows.validate(def, "w") + assert(ok == nil, "expected a rejection for " .. needle) + has(err, needle) + end + + bad({ steps = {} }, "has no `steps` array") + bad({ steps = "nope" }, "has no `steps` array") + bad({ steps = { { id = "a", agent = "alpha" } } }, "`prompt` is required") + bad({ steps = { { id = "a", prompt = "p" } } }, "`agent` is required") + bad({ steps = { { agent = "alpha", prompt = "p" } } }, "`id` is required") + bad({ steps = { + { id = "a", agent = "alpha", prompt = "p" }, + { id = "a", agent = "beta", prompt = "q" }, + } }, "duplicate step id 'a'") + bad({ steps = { + { id = "a", agent = "alpha", prompt = "p", needs = { "ghost" } }, + } }, "unknown dependency 'ghost'") + bad({ steps = { + { id = "a", agent = "alpha", prompt = "p", needs = { "b" } }, + { id = "b", agent = "beta", prompt = "q", needs = { "a" } }, + } }, "dependency cycle") + end }, + + { "a step's prompt carries the input and each dependency in needs order", function() + local step = { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a", "z" } } + local settled = { + a = { status = "completed", output = "A output" }, + z = { status = "failed", error = "boom" }, + } + local prompt = toml_workflows.step_prompt(step, "the workflow input", settled) + assert(prompt == table.concat({ + "Summarize.", + "", + "## Workflow input", + "", + "the workflow input", + "", + "## Output of a", + "", + "A output", + "", + "## Output of z", + "", + "[failed: boom]", + }, "\n"), string.format("unexpected prompt:\n%s", prompt)) + end }, + + { "a dependent receives its dependency's output and runs after it", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate({ + name = "chain", + steps = { + { id = "a", agent = "alpha", prompt = "Inspect." }, + { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } }, + }, + }, "chain")) + handle.queue_for("alpha", { output = "A output" }) + handle.queue_for("beta", { output = "B output" }) + + local results = toml_workflows.run(def, "the input", profiles) + assert(#handle.spawns == 2, "both steps ran") + assert(handle.spawns[1].label == "alpha", "the root starts first") + has(handle.spawns[2].prompt, "## Workflow input\n\nthe input") + has(handle.spawns[2].prompt, "## Output of a\n\nA output") + assert(#results == 1 and results[1].id == "b", "only the terminal step is returned") + assert(results[1].output == "B output") + end) + end }, + + { "a failed step skips its dependents while other branches finish", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate(BRANCH_DEF, "branch")) + handle.queue_for("alpha", { status = "failed", error = "alpha died", settle = 1 }) + handle.queue_for("gamma", { output = "gamma output", settle = 2 }) + + local results = toml_workflows.run(def, "input", profiles) + + assert(#handle.spawns == 2, "the skipped step never spawns") + for _, spec in ipairs(handle.spawns) do + assert(spec.label ~= "beta", "beta depends on a failure and must not run") + end + assert(#results == 2, "both terminal steps are reported") + assert(results[1].id == "b" and results[1].status == "skipped", "terminals keep declaration order") + has(results[1].error, "a dependency did not complete") + assert(results[2].id == "c" and results[2].status == "completed") + assert(results[2].output == "gamma output") + end) + end }, + + { "independent roots start together", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate({ + name = "fan", + steps = { + { id = "a", agent = "alpha", prompt = "A" }, + { id = "c", agent = "gamma", prompt = "C" }, + { id = "b", agent = "beta", prompt = "B", needs = { "a", "c" } }, + }, + }, "fan")) + handle.queue_for("alpha", { output = "A out", settle = 2 }) + handle.queue_for("gamma", { output = "C out", settle = 1 }) + handle.queue_for("beta", { output = "B out" }) + + local results = toml_workflows.run(def, "input", profiles) + assert(handle.max_live == 2, "both roots were in flight at once, peaked at " .. handle.max_live) + has(handle.spawns[3].prompt, "## Output of a\n\nA out") + has(handle.spawns[3].prompt, "## Output of c\n\nC out") + assert(#results == 1 and results[1].id == "b" and results[1].status == "completed") + end) + end }, + + { "TOML parses into the same definition shape", function() + if not pcall(require, "toml") then + return "skip", "toml2lua is not installed" + end + local def, err = toml_workflows.parse(table.concat({ + 'name = "review-chain"', + 'description = "Two angles, then a synthesis."', + '', + '[[steps]]', + 'id = "correctness"', + 'agent = "alpha"', + 'prompt = "Review for correctness."', + '', + '[[steps]]', + 'id = "synthesis"', + 'agent = "beta"', + 'prompt = "Synthesize."', + 'reasoning = "high"', + 'needs = ["correctness"]', + }, "\n"), "stem") + assert(def, tostring(err)) + assert(def.name == "review-chain") + assert(def.description == "Two angles, then a synthesis.") + assert(#def.steps == 2) + assert(def.steps[2].reasoning == "high") + assert(def.terminal.synthesis and def.terminal.correctness == nil) + + local bad, bad_err = toml_workflows.parse("name = = broken", "stem") + assert(bad == nil, "malformed TOML must not parse") + has(bad_err, "invalid TOML") + end }, + + { "discovery shadows by name and registers one command per valid workflow", function() + local valid = table.concat({ + 'name = "review-chain"', + 'description = "Inspect a change from two angles."', + '[[steps]]', + 'id = "one"', + 'agent = "alpha"', + 'prompt = "Look."', + }, "\n") + local user_shadowed = table.concat({ + 'name = "shadowed"', + 'description = "the user layer"', + '[[steps]]', + 'id = "one"', + 'agent = "alpha"', + 'prompt = "user"', + }, "\n") + local project_shadowed = table.concat({ + 'name = "shadowed"', + 'description = "the project layer"', + '[[steps]]', + 'id = "one"', + 'agent = "beta"', + 'prompt = "project"', + }, "\n") + local broken = table.concat({ + '[[steps]]', + 'id = "dup"', + 'agent = "alpha"', + 'prompt = "one"', + '[[steps]]', + 'id = "dup"', + 'agent = "alpha"', + 'prompt = "two"', + }, "\n") + + return with_layers({ + ["user/workflows/review-chain.toml"] = valid, + ["user/workflows/nested/shadowed.toml"] = user_shadowed, + ["project/workflows/shadowed.toml"] = project_shadowed, + ["project/workflows/broken.toml"] = broken, + }, function() + with_host(function(handle, profiles) + local found = toml_workflows.discover_and_register(profiles) + + assert(handle.commands_by_name["workflow:review-chain"], "a valid workflow registers a command") + assert(handle.commands_by_name["workflow:review-chain"].description == + "Inspect a change from two angles.", "the description comes from the file") + assert(handle.commands_by_name["workflow:shadowed"].description == "the project layer", + "the project layer shadows the user layer") + assert(handle.commands_by_name["workflow:broken"] == nil, "an invalid file registers nothing") + assert(#handle.commands == 2, "exactly two commands, saw " .. #handle.commands) + + local warned = false + for _, warning in ipairs(found.warnings) do + if warning:find("duplicate step id 'dup'", 1, true) then + warned = true + end + end + assert(warned, "the invalid file's error is kept: " .. table.concat(found.warnings, " | ")) + + -- The command tail becomes the workflow input. + handle.queue_for("alpha", { output = "looked" }) + local text = handle.commands_by_name["workflow:review-chain"].handler("check the parser") + has(text, "step: one") + has(text, "status: completed") + has(text, "looked") + has(handle.spawns[1].prompt, "## Workflow input\n\ncheck the parser") + + -- And the tool can run the same workflow, or explain a broken one. + has(toml_workflows.handle({ name = "broken", prompt = "x" }, profiles), + "failed to load") + has(toml_workflows.handle({ name = "ghost", prompt = "x" }, profiles), + "unknown workflow 'ghost'") + end) + end) + end }, + + { "a broken project workflow shadows the user one under its declared name", function() + local user_valid = table.concat({ + 'name = "dup"', + 'description = "the user layer"', + '[[steps]]', + 'id = "one"', + 'agent = "alpha"', + 'prompt = "user"', + }, "\n") + -- Declares the same name from a differently-named file, and is invalid. + local project_broken = table.concat({ + 'name = "dup"', + '[[steps]]', + 'id = "same"', + 'agent = "alpha"', + 'prompt = "one"', + '[[steps]]', + 'id = "same"', + 'agent = "alpha"', + 'prompt = "two"', + }, "\n") + + return with_layers({ + ["user/workflows/dup.toml"] = user_valid, + ["project/workflows/whatever.toml"] = project_broken, + }, function() + with_host(function(handle, profiles) + local found = toml_workflows.discover_and_register(profiles) + + local entry = found.by_name["dup"] + assert(entry, "the project error must be indexed under the name it declares") + assert(entry.definition == nil, "the shadowed user definition must not survive") + has(entry.error, "duplicate step id 'same'") + assert(#handle.commands == 0, "a shadowed-out workflow registers no command") + has(toml_workflows.handle({ name = "dup", prompt = "x" }, profiles), "failed to load") + end) + end) + end }, + + { "a command whose workflow cannot run reports the error, not a stack trace", function() + with_host(function(handle, profiles) + local def = assert(toml_workflows.validate({ + name = "ghosted", + steps = { { id = "one", agent = "nobody", prompt = "Do it." } }, + }, "ghosted")) + local ok, err = pcall(toml_workflows.run, def, "input", profiles) + assert(not ok, "an unknown agent stops the workflow") + has(tostring(err), "unknown agent 'nobody'") + + has(toml_workflows.handle({ + prompt = "x", + steps = { { id = "one", agent = "nobody", prompt = "Do it." } }, + }, profiles), "Error:") + assert(#handle.spawns == 0) + end) + end }, + + { "the tool takes exactly one of name and steps", function() + with_host(function(handle, profiles) + has(toml_workflows.handle({ prompt = "x" }, profiles), "pass exactly one of `name`") + has(toml_workflows.handle({ prompt = "x", name = "a", steps = {} }, profiles), "not both") + has(toml_workflows.handle({ name = "a" }, profiles), "prompt is required") + assert(#handle.spawns == 0) + end) + end }, + + { "the tool runs a transient definition", function() + with_host(function(handle, profiles) + handle.queue_for("alpha", { output = "transient output" }) + local text = toml_workflows.handle({ + prompt = "the input", + steps = { { id = "only", agent = "alpha", prompt = "Do it." } }, + }, profiles) + has(text, "step: only") + has(text, "transient output") + has(handle.spawns[1].prompt, "Do it.\n\n## Workflow input\n\nthe input") + + has(toml_workflows.handle({ + prompt = "x", + steps = { { id = "only", agent = "alpha", prompt = "p", needs = { "ghost" } } }, + }, profiles), "unknown dependency 'ghost'") + end) + end }, +} |