path: root/spec/test_toml_workflows.lua

-- subagents/toml_workflows.lua: validation, dependency prompt assembly, branch
-- failure, terminal ordering, discovery, and the generated commands.
--
-- Validation and execution run off definitions built in Lua, so they exercise
-- the DAG rules without needing the TOML rock. The parse, discovery, and command
-- cases do need toml2lua (and luv, for the recursive walk) and skip without them.
--
-- Discovery is pointed at temporary directories by replacing paths.config_roots
-- for the duration of the case: the two layers, in the same order the extension
-- uses, without reading the machine's real ~/.config.

local fake = require("spec.fake_ext")
local paths = require("subagents.paths")
local progress = require("subagents.progress")
local toml_workflows = require("subagents.toml_workflows")

local function has(text, needle)
    assert(type(text) == "string", "expected a string, got " .. type(text))
    assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. tostring(text))
end

local function profile_set()
    local set = { list = {}, by_name = {}, warnings = {} }
    for _, name in ipairs({ "alpha", "beta", "gamma" }) do
        local profile = { name = name, description = name, body = "You are " .. name .. ".\n" }
        set.list[#set.list + 1] = profile
        set.by_name[name] = profile
    end
    return set
end

local function with_host(fn)
    local handle = fake.install()
    local ok, err = pcall(fn, handle, profile_set())
    handle.restore()
    if not ok then
        error(err, 0)
    end
end

local function write(path, text)
    assert(os.execute("mkdir -p " .. (path:match("^(.*)/[^/]+$") or ".")))
    local file = assert(io.open(path, "w"))
    file:write(text)
    file:close()
end

-- Point discovery at two temporary layers for the duration of `fn`.
local function with_layers(files, fn)
    local ok_uv, uv = pcall(require, "luv")
    if not ok_uv then
        return "skip", "luv is not installed"
    end
    if not pcall(require, "toml") then
        return "skip", "toml2lua is not installed"
    end

    local tmp = assert(uv.fs_mkdtemp("/tmp/panto-subagents-workflows-XXXXXX"))
    for name, text in pairs(files) do
        write(tmp .. "/" .. name, text)
    end

    local original = paths.config_roots
    paths.config_roots = function(kind)
        return { tmp .. "/user/" .. kind, tmp .. "/project/" .. kind }
    end
    local ok, err = pcall(fn, tmp)
    paths.config_roots = original
    os.execute("rm -rf " .. tmp)
    if not ok then
        error(err, 0)
    end
end

local BRANCH_DEF = {
    name = "branch",
    steps = {
        { id = "a", agent = "alpha", prompt = "Inspect." },
        { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } },
        { id = "c", agent = "gamma", prompt = "Unrelated." },
    },
}

return {
    { "a valid definition normalizes and marks its terminal steps", function()
        local def = assert(toml_workflows.validate(BRANCH_DEF, "fallback"))
        assert(def.name == "branch")
        assert(#def.steps == 3)
        assert(def.terminal.b and def.terminal.c, "b and c are nobody's dependency")
        assert(def.terminal.a == nil, "a is depended on, so it is not terminal")
        assert(def.by_id.b.needs[1] == "a")
    end },

    { "`output` chooses the reported steps, in its own order", function()
        local def = assert(toml_workflows.validate({
            name = "branch",
            steps = BRANCH_DEF.steps,
            output = { "c", "a" },
        }, "fallback"))
        assert(def.report[1] == "c" and def.report[2] == "a" and #def.report == 2,
            "the file's order wins: " .. table.concat(def.report, ","))

        local default = assert(toml_workflows.validate(BRANCH_DEF, "fallback"))
        assert(default.report[1] == "b" and default.report[2] == "c" and #default.report == 2,
            "without `output`, terminal steps in declaration order")
    end },

    { "the name falls back to the file stem", function()
        local def = assert(toml_workflows.validate({ steps = BRANCH_DEF.steps }, "from-stem"))
        assert(def.name == "from-stem", tostring(def.name))
    end },

    { "structural mistakes are refused with a readable reason", function()
        local function bad(def, needle)
            local ok, err = toml_workflows.validate(def, "w")
            assert(ok == nil, "expected a rejection for " .. needle)
            has(err, needle)
        end

        bad({ steps = {} }, "has no `steps` array")
        bad({ steps = "nope" }, "has no `steps` array")
        bad({ steps = { { id = "a", agent = "alpha" } } }, "`prompt` is required")
        bad({ steps = { { id = "a", prompt = "p" } } }, "`agent` is required")
        bad({ steps = { { agent = "alpha", prompt = "p" } } }, "`id` is required")
        bad({ steps = BRANCH_DEF.steps, output = {} }, "`output` must be a non-empty array")
        bad({ steps = BRANCH_DEF.steps, output = "b" }, "`output` must be a non-empty array")
        bad({ steps = BRANCH_DEF.steps, output = { "ghost" } }, "`output` names unknown step 'ghost'")
        bad({ steps = BRANCH_DEF.steps, output = { "b", "b" } }, "`output` names 'b' twice")
        bad({ steps = {
            { id = "a", agent = "alpha", prompt = "p" },
            { id = "a", agent = "beta", prompt = "q" },
        } }, "duplicate step id 'a'")
        bad({ steps = {
            { id = "a", agent = "alpha", prompt = "p", needs = { "ghost" } },
        } }, "unknown dependency 'ghost'")
        bad({ steps = {
            { id = "a", agent = "alpha", prompt = "p", needs = { "b" } },
            { id = "b", agent = "beta", prompt = "q", needs = { "a" } },
        } }, "dependency cycle")
    end },

    { "a step's prompt carries the input and each dependency in needs order", function()
        local step = { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a", "z" } }
        local settled = {
            a = { status = "completed", output = "A output" },
            z = { status = "failed", error = "boom" },
        }
        local prompt = toml_workflows.step_prompt(step, "the workflow input", settled)
        assert(prompt == table.concat({
            "Summarize.",
            "",
            "## Workflow input",
            "",
            "the workflow input",
            "",
            "## Output of a",
            "",
            "A output",
            "",
            "## Output of z",
            "",
            "[failed: boom]",
        }, "\n"), string.format("unexpected prompt:\n%s", prompt))

        -- An unparameterized workflow gets no input heading at all.
        local bare = toml_workflows.step_prompt(
            { id = "a", agent = "alpha", prompt = "Do it.", needs = {} }, "", {})
        assert(bare == "Do it.", string.format("unexpected bare prompt:\n%s", bare))
    end },

    { "a dependent receives its dependency's output and runs after it", function()
        with_host(function(handle, profiles)
            local def = assert(toml_workflows.validate({
                name = "chain",
                steps = {
                    { id = "a", agent = "alpha", prompt = "Inspect." },
                    { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } },
                },
            }, "chain"))
            handle.queue_for("alpha", { output = "A output" })
            handle.queue_for("beta", { output = "B output" })

            local results = toml_workflows.run(def, "the input", profiles)
            assert(#handle.spawns == 2, "both steps ran")
            assert(handle.spawns[1].label == "alpha", "the root starts first")
            has(handle.spawns[2].prompt, "## Workflow input\n\nthe input")
            has(handle.spawns[2].prompt, "## Output of a\n\nA output")
            assert(#results == 1 and results[1].id == "b", "only the terminal step is returned")
            assert(results[1].output == "B output")
        end)
    end },

    { "`output` reports an intermediate step instead of the terminal one", function()
        with_host(function(handle, profiles)
            local def = assert(toml_workflows.validate({
                name = "chain",
                steps = {
                    { id = "a", agent = "alpha", prompt = "Inspect." },
                    { id = "b", agent = "beta", prompt = "Summarize.", needs = { "a" } },
                },
                output = { "a" },
            }, "chain"))
            handle.queue_for("alpha", { output = "A output" })
            handle.queue_for("beta", { output = "B output" })

            local results = toml_workflows.run(def, "the input", profiles)
            assert(#handle.spawns == 2, "both steps still run")
            assert(#results == 1 and results[1].id == "a", "only the named step is reported")
            assert(results[1].output == "A output")
        end)
    end },

    { "a failed step skips its dependents while other branches finish", function()
        with_host(function(handle, profiles)
            local def = assert(toml_workflows.validate(BRANCH_DEF, "branch"))
            handle.queue_for("alpha", { status = "failed", error = "alpha died", settle = 1 })
            handle.queue_for("gamma", { output = "gamma output", settle = 2 })

            local results = toml_workflows.run(def, "input", profiles)

            assert(#handle.spawns == 2, "the skipped step never spawns")
            for _, spec in ipairs(handle.spawns) do
                assert(spec.label ~= "beta", "beta depends on a failure and must not run")
            end
            assert(#results == 2, "both terminal steps are reported")
            assert(results[1].id == "b" and results[1].status == "skipped", "terminals keep declaration order")
            has(results[1].error, "a dependency did not complete")
            assert(results[2].id == "c" and results[2].status == "completed")
            assert(results[2].output == "gamma output")
        end)
    end },

    { "independent roots start together", function()
        with_host(function(handle, profiles)
            local def = assert(toml_workflows.validate({
                name = "fan",
                steps = {
                    { id = "a", agent = "alpha", prompt = "A" },
                    { id = "c", agent = "gamma", prompt = "C" },
                    { id = "b", agent = "beta", prompt = "B", needs = { "a", "c" } },
                },
            }, "fan"))
            handle.queue_for("alpha", { output = "A out", settle = 2 })
            handle.queue_for("gamma", { output = "C out", settle = 1 })
            handle.queue_for("beta", { output = "B out" })

            local results = toml_workflows.run(def, "input", profiles)
            assert(handle.max_live == 2, "both roots were in flight at once, peaked at " .. handle.max_live)
            has(handle.spawns[3].prompt, "## Output of a\n\nA out")
            has(handle.spawns[3].prompt, "## Output of c\n\nC out")
            assert(#results == 1 and results[1].id == "b" and results[1].status == "completed")
        end)
    end },

    { "TOML parses into the same definition shape", function()
        if not pcall(require, "toml") then
            return "skip", "toml2lua is not installed"
        end
        local def, err = toml_workflows.parse(table.concat({
            'name = "review-chain"',
            'description = "Two angles, then a synthesis."',
            '',
            '[[steps]]',
            'id = "correctness"',
            'agent = "alpha"',
            'prompt = "Review for correctness."',
            '',
            '[[steps]]',
            'id = "synthesis"',
            'agent = "beta"',
            'prompt = "Synthesize."',
            'reasoning = "high"',
            'needs = ["correctness"]',
        }, "\n"), "stem")
        assert(def, tostring(err))
        assert(def.name == "review-chain")
        assert(def.description == "Two angles, then a synthesis.")
        assert(#def.steps == 2)
        assert(def.steps[2].reasoning == "high")
        assert(def.terminal.synthesis and def.terminal.correctness == nil)

        local bad, bad_err = toml_workflows.parse("name = = broken", "stem")
        assert(bad == nil, "malformed TOML must not parse")
        has(bad_err, "invalid TOML")
    end },

    { "discovery shadows by name and registers one command per valid workflow", function()
        local valid = table.concat({
            'name = "review-chain"',
            'description = "Inspect a change from two angles."',
            '[[steps]]',
            'id = "one"',
            'agent = "alpha"',
            'prompt = "Look."',
        }, "\n")
        local user_shadowed = table.concat({
            'name = "shadowed"',
            'description = "the user layer"',
            '[[steps]]',
            'id = "one"',
            'agent = "alpha"',
            'prompt = "user"',
        }, "\n")
        local project_shadowed = table.concat({
            'name = "shadowed"',
            'description = "the project layer"',
            '[[steps]]',
            'id = "one"',
            'agent = "beta"',
            'prompt = "project"',
        }, "\n")
        local broken = table.concat({
            '[[steps]]',
            'id = "dup"',
            'agent = "alpha"',
            'prompt = "one"',
            '[[steps]]',
            'id = "dup"',
            'agent = "alpha"',
            'prompt = "two"',
        }, "\n")

        return with_layers({
            ["user/workflows/review-chain.toml"] = valid,
            ["user/workflows/nested/shadowed.toml"] = user_shadowed,
            ["project/workflows/shadowed.toml"] = project_shadowed,
            ["project/workflows/broken.toml"] = broken,
        }, function()
            with_host(function(handle, profiles)
                local found = toml_workflows.discover_and_register(profiles)

                assert(handle.commands_by_name["workflow:review-chain"], "a valid workflow registers a command")
                assert(handle.commands_by_name["workflow:review-chain"].description ==
                    "Inspect a change from two angles.", "the description comes from the file")
                assert(handle.commands_by_name["workflow:shadowed"].description == "the project layer",
                    "the project layer shadows the user layer")
                assert(handle.commands_by_name["workflow:broken"] == nil, "an invalid file registers nothing")
                assert(#handle.commands == 2, "exactly two commands, saw " .. #handle.commands)

                local warned = false
                for _, warning in ipairs(found.warnings) do
                    if warning:find("duplicate step id 'dup'", 1, true) then
                        warned = true
                    end
                end
                assert(warned, "the invalid file's error is kept: " .. table.concat(found.warnings, " | "))

                -- The command tail becomes the workflow input.
                handle.queue_for("alpha", { output = "looked" })
                local text = handle.commands_by_name["workflow:review-chain"].handler("check the parser")
                has(text, "step: one")
                has(text, "status: completed")
                has(text, "looked")
                has(handle.spawns[1].prompt, "## Workflow input\n\ncheck the parser")

                -- And the tool can run the same workflow, or explain a broken one.
                has(toml_workflows.handle({ name = "broken", prompt = "x" }, profiles),
                    "failed to load")
                has(toml_workflows.handle({ name = "ghost", prompt = "x" }, profiles),
                    "unknown workflow 'ghost'")
            end)
        end)
    end },

    { "a workflow naming an undiscovered agent is rejected at discovery", function()
        return with_layers({
            ["project/workflows/ghosted.toml"] = table.concat({
                '[[steps]]',
                'id = "one"',
                'agent = "nobody"',
                'prompt = "Do it."',
            }, "\n"),
        }, function()
            with_host(function(handle, profiles)
                local found = toml_workflows.discover_and_register(profiles)
                local entry = found.by_name["ghosted"]
                assert(entry and entry.definition == nil, "the definition must not survive")
                has(entry.error, "unknown agent 'nobody'")
                assert(#handle.commands == 0, "no command is registered for it")
                has(table.concat(found.warnings, " | "), "unknown agent 'nobody'")
                has(toml_workflows.handle({ name = "ghosted", prompt = "x" }, profiles),
                    "unknown agent 'nobody'")
                assert(#handle.spawns == 0, "nothing ran before the failure")
            end)
        end)
    end },

    { "a broken project workflow shadows the user one under its declared name", function()
        local user_valid = table.concat({
            'name = "dup"',
            'description = "the user layer"',
            '[[steps]]',
            'id = "one"',
            'agent = "alpha"',
            'prompt = "user"',
        }, "\n")
        -- Declares the same name from a differently-named file, and is invalid.
        local project_broken = table.concat({
            'name = "dup"',
            '[[steps]]',
            'id = "same"',
            'agent = "alpha"',
            'prompt = "one"',
            '[[steps]]',
            'id = "same"',
            'agent = "alpha"',
            'prompt = "two"',
        }, "\n")

        return with_layers({
            ["user/workflows/dup.toml"] = user_valid,
            ["project/workflows/whatever.toml"] = project_broken,
        }, function()
            with_host(function(handle, profiles)
                local found = toml_workflows.discover_and_register(profiles)

                local entry = found.by_name["dup"]
                assert(entry, "the project error must be indexed under the name it declares")
                assert(entry.definition == nil, "the shadowed user definition must not survive")
                has(entry.error, "duplicate step id 'same'")
                assert(#handle.commands == 0, "a shadowed-out workflow registers no command")
                has(toml_workflows.handle({ name = "dup", prompt = "x" }, profiles), "failed to load")
            end)
        end)
    end },

    { "a command whose workflow cannot run reports the error, not a stack trace", function()
        with_host(function(handle, profiles)
            local def = assert(toml_workflows.validate({
                name = "ghosted",
                steps = { { id = "one", agent = "nobody", prompt = "Do it." } },
            }, "ghosted"))
            local ok, err = pcall(toml_workflows.run, def, "input", profiles)
            assert(not ok, "an unknown agent stops the workflow")
            has(tostring(err), "unknown agent 'nobody'")
            assert(#handle.spawns == 0)
        end)
    end },

    { "the tool requires both a name and a prompt", function()
        with_host(function(handle, profiles)
            has(toml_workflows.handle({ prompt = "x" }, profiles), "`name` is required")
            has(toml_workflows.handle({ name = "a" }, profiles), "prompt is required")
            assert(#handle.spawns == 0)
        end)
    end },

    { "the tool runs a named workflow and presents each child prompt only when expanded", function()
        return with_layers({
            ["project/workflows/solo.toml"] = table.concat({
                'name = "solo"',
                '[[steps]]',
                'id = "only"',
                'agent = "alpha"',
                'prompt = "Do it."',
            }, "\n"),
        }, function()
            with_host(function(handle, profiles)
                toml_workflows.discover_and_register(profiles)
                handle.queue_for("alpha", { output = "named output" })
                progress.reset()
                local component
                progress.claim({
                    id = "workflow-call",
                    tool_name = "subagents.workflow",
                    collapsed = true,
                    set_component = function(_, value)
                        component = value
                        return {
                            invalidate = function() end,
                            alive = function() return true end,
                            set_pinned = function() end,
                        }
                    end,
                })
                progress.bind({ tool_call_id = "workflow-call" })
                local text = toml_workflows.handle({ name = "solo", prompt = "the input" }, profiles)
                has(text, "step: only")
                has(text, "named output")
                has(handle.spawns[1].prompt, "Do it.\n\n## Workflow input\n\nthe input")
                local compact = table.concat(component:render(100), "\n")
                assert(not compact:find("Do it.", 1, true), compact)
                assert(not compact:find("the input", 1, true), compact)
                progress.collapse({ collapsed = false })
                local expanded = table.concat(component:render(100), "\n")
                has(expanded, "prompt: Do it.")
                has(expanded, "## Workflow input")
                has(expanded, "the input")
                assert(not expanded:find("system prompt:", 1, true), expanded)
                progress.reset()
            end)
        end)
    end },
}