Add background Lua workflows, inline child prompts, and layered config

subagents.lua now starts a workflow on its own coroutine and returns a
session-scoped id immediately, so fan-out continues while the primary keeps
working; completion wakes the primary, and later calls read immutable records
from subagents.workflows. Every ctx:agent takes a workflow-unique name so those
records are addressable. A session_start guidance message tells the primary when
to reach for run vs lua.

Children no longer inherit the primary's system context: a child starts from the
fixed child-role instruction plus its profile, and subagents.run/ctx:agent
accept an inline system_prompt instead of a profile. The now-redundant `agent`
form of subagents.models is gone.

Concurrency defaults to five and is configurable through [subagents]
max_concurrent in any layered config.toml; turn boundaries reap only settled
jobs so background workflows survive, while interrupt and session end cancel.

Config roots come from panto.ext.dirs.layers rather than a hand-rolled XDG
lookup, which picks up the base and git-ignored local layers for both agents/
and workflows/.

TOML workflows tighten up: the subagents.workflow tool takes a discovered name
only (inline `steps` duplicated subagents.lua at less power), an optional
top-level `output` array chooses the reported steps and their order instead of
the terminal set, a step with no workflow input gets no empty input heading, and
a workflow naming an undiscovered agent is rejected at discovery rather than
part-way through a run.

1 files changed, 122 insertions, 43 deletions

diff --git a/init.lua b/init.lua
index a104324..2ec5f8c 100644
--- a/init.lua
+++ b/init.lua
@@ -1,9 +1,10 @@
 -- panto-subagents: the `subagents` extension entry point.
 --
 -- Activation happens once per panto session (a new session, `/new`, or
--- `/resume` rebuilds the whole Lua state and runs this again). It discovers
--- the agent profiles once, registers the four model-facing tools, and
--- registers a `/workflow:<name>` command for every valid TOML workflow.
+-- `/resume` rebuilds the whole Lua state and runs this again). It reads the
+-- layered `[subagents]` settings, discovers the agent profiles once, registers
+-- the four model-facing tools, and registers a `/workflow:<name>` command for
+-- every valid TOML workflow.
 --
 -- Discovery order is deterministic and side-effect free apart from
 -- registration, because pantograph evaluates every candidate extension file
@@ -17,10 +18,11 @@
 -- of the run description — the one place the user and the model both see the
 -- profile list.
 --
--- Activation also subscribes to the turn lifecycle: an interrupted turn asks
--- every live child to stop, and the end of a turn joins them and clears the
--- coroutine bindings. Settled progress cards remain attached to their
--- transcript entries until the host destroys those components.
+-- Activation also subscribes to the session lifecycle: interruption cancels
+-- live work, ordinary turn boundaries reap only settled jobs so background
+-- workflows survive, and session teardown cancels everything. Settled progress
+-- cards remain attached to their transcript entries until the host destroys
+-- those components.
 --
 -- If the host predates the model-resolution seam, activation fails loudly
 -- instead of registering tools that cannot work.
@@ -28,12 +30,15 @@
 local jobs = require("subagents.jobs")
 local luatool = require("subagents.luatool")
 local models = require("subagents.models")
+local paths = require("subagents.paths")
 local progress = require("subagents.progress")
 local run = require("subagents.run")
 local spawn = require("subagents.spawn")
 local toml_workflows = require("subagents.toml_workflows")
+local workflow_api = require("subagents.workflow")
 
 local MAX_SHOWN_WARNINGS = 5
+local DEFAULT_MAX_CONCURRENT = 5
 local DIM, RESET = "\27[2m", "\27[0m"
 
 local function host()
@@ -102,7 +107,7 @@ end
 
 local function run_description(profiles)
     local lines = {
-        "Delegate a task to a subagent and wait for its report. Start a new child from an agent profile, or continue a child you started earlier in this session.",
+        "Delegate a task to a subagent and wait for its report. Start a new child from an agent profile or an inline system prompt, or continue a child you started earlier in this session.",
         "",
         "Agent profiles:",
     }
@@ -120,7 +125,7 @@ local function run_description(profiles)
 
     lines[#lines + 1] = ""
     lines[#lines + 1] = "Rules:"
-    lines[#lines + 1] = "- Pass exactly one of `agent` (start a new child from that profile) or `id` (continue a child from an earlier result). `prompt` is always required and must be non-empty."
+    lines[#lines + 1] = "- Pass exactly one of `agent` (start from a profile), `system_prompt` (start without a profile), or `id` (continue a child). `prompt` is always required and must be non-empty."
     lines[#lines + 1] = "- Omit `model` and `reasoning` normally; a child inherits yours. Call subagents.models before choosing an unfamiliar model or reasoning level."
     lines[#lines + 1] = "- A child shares your workspace and tools but not your conversation, and cannot ask the user questions. Put every piece of task context it needs into `prompt`."
     lines[#lines + 1] = "- To delegate in parallel, emit several subagents.run calls in one tool batch; they run concurrently and one failure does not discard the others."
@@ -141,12 +146,99 @@ local function run_description(profiles)
     return table.concat(lines, "\n")
 end
 
+local SUBAGENTS_GUIDANCE = [=[
+## Subagents
+
+Use `subagents.run` for one delegation, or emit several `subagents.run` calls in one tool batch for simple independent parallel work. Use `subagents.lua` for dynamic fan-out, branching, staged waves, or work that should continue while you interact with the user.
+
+`subagents.lua` takes Lua `source` and optional inline `agents`; it has no shared prompt. Put the complete, specific context each worker needs in that worker's `prompt`. Start a background workflow and save its returned id:
+
+```lua
+return subagents.workflow(function(ctx)
+  local a = ctx:agent{name="api", system_prompt="Research APIs.", prompt="Inspect the API surface."}
+  local b = ctx:agent{name="tests", system_prompt="Research tests.", prompt="Inspect test coverage."}
+  local research = ctx:await({a, b}, "all")
+
+  local implementation = ctx:agent{
+    name="implement",
+    system_prompt="Implement focused changes.",
+    prompt="Implement using this research:\n" .. research[1].output .. "\n" .. research[2].output,
+  }:await()
+
+  local reviewers = {
+    ctx:agent{name="correctness", system_prompt="Review correctness.", prompt=implementation.output},
+    ctx:agent{name="simplicity", system_prompt="Review for simplicity.", prompt=implementation.output},
+  }
+  local reviews = ctx:await(reviewers, "all")
+  return reviews[1].output .. "\n\n" .. reviews[2].output
+end)
+```
+
+Every `ctx:agent` needs a non-empty `name` unique within that workflow. A workflow callback must return a string. Inspect a running or finished workflow with a later `subagents.lua` call, for example:
+
+```lua
+local w = subagents.workflows["workflow-1"]
+return w.status .. (w.result and ("\n" .. w.result) or "")
+```
+
+Workflow fields are `id`, `status`, `result`, `error`, and `agents`. Agents are available by name or iteration and expose `name`, `status`, `output`, `error`, and `id`. Records are read-only. Workflow completion wakes you with its id; use another `subagents.lua` call to retrieve whichever outputs you need.
+]=]
+
+local function install_guidance(ext)
+    subscribe(ext, "session_start", function()
+        local primary = ext.agent
+        if primary and type(primary.add_system_message) == "function" then
+            primary:add_system_message(SUBAGENTS_GUIDANCE)
+        end
+    end)
+end
+
+local function configured_max_concurrent(ext)
+    local value = DEFAULT_MAX_CONCURRENT
+    local layers = ext.dirs and ext.dirs.layers
+    if type(layers) ~= "table" then
+        return value
+    end
+
+    local ok_toml, toml = pcall(require, "toml")
+    if not ok_toml or type(toml) ~= "table" or type(toml.parse) ~= "function" then
+        error("panto-subagents: the 'toml2lua' rock is required to read configuration")
+    end
+
+    for _, layer in ipairs(layers) do
+        if type(layer) == "table" and type(layer.dir) == "string" then
+            local path = layer.dir .. "/config.toml"
+            local text = paths.read_file(path)
+            if text then
+                local parsed_ok, parsed = pcall(toml.parse, text, { strict = true })
+                if not parsed_ok or type(parsed) ~= "table" then
+                    error("panto-subagents: could not parse " .. path)
+                end
+                local section = parsed.subagents
+                if section ~= nil and type(section) ~= "table" then
+                    error("panto-subagents: [subagents] must be a table in " .. path)
+                end
+                local configured = section and section.max_concurrent
+                if configured ~= nil then
+                    if type(configured) ~= "number" or configured < 1 or configured ~= math.floor(configured) then
+                        error("panto-subagents: subagents.max_concurrent must be a positive integer in " .. path)
+                    end
+                    value = configured
+                end
+            end
+        end
+    end
+    return value
+end
+
 local function activate()
     local ext = host()
     if type(ext.resolve_model) ~= "function" or type(require("panto").agent) ~= "function" then
         error("panto-subagents: this pantograph is too old for subagents (panto.ext.resolve_model is missing)")
     end
 
+    jobs.MAX_CONCURRENT = configured_max_concurrent(ext)
+
     -- Discover through spawn so the workflow lanes, which resolve profiles
     -- lazily, share the exact set this tool description advertises.
     local profiles = spawn.profiles()
@@ -161,9 +253,16 @@ local function activate()
         progress.begin_live_turn()
     end)
     subscribe(ext, "turn_interrupt", function()
+        workflow_api.cancel_all(false)
         jobs.cancel_all()
     end)
     subscribe(ext, "turn_end", function()
+        jobs.reap()
+        progress.reset()
+    end)
+    subscribe(ext, "session_end", function()
+        workflow_api.cancel_all(true)
+        jobs.cancel_all()
         jobs.close_all()
         progress.reset()
     end)
@@ -184,10 +283,11 @@ local function activate()
         schema = {
             type = "object",
             properties = {
-                agent = { type = "string", description = "Profile name for a new child. Mutually exclusive with `id`." },
-                id = { type = "string", description = "Id of a child started earlier in this session, to continue it. Mutually exclusive with `agent`." },
+                agent = { type = "string", description = "Profile name for a new child. Mutually exclusive with `system_prompt` and `id`." },
+                system_prompt = { type = "string", description = "System prompt for a new child without a profile. Mutually exclusive with `agent` and `id`." },
+                id = { type = "string", description = "Id of a child started earlier in this session, to continue it. Mutually exclusive with `agent` and `system_prompt`." },
                 prompt = { type = "string", description = "The complete task for the child. It sees none of this conversation." },
-                model = { type = "string", description = "Optional `provider:model` override. Omit to inherit." },
+                model = { type = "string", description = "Optional `provider:model` override. Omit to inherit; query available options with `subagents.models`." },
                 reasoning = { type = "string", description = "Optional reasoning level override. Omit to inherit." },
             },
             required = { "prompt" },
@@ -202,7 +302,7 @@ local function activate()
 
     ext.register_tool {
         name = "subagents.models",
-        description = "Query the configured model catalog: no arguments for the inherited model plus provider counts, `model` for an exact lookup including valid reasoning levels, `provider`/`query` for a bounded search, or `agent` for what a profile will actually run on.",
+        description = "Query the configured model catalog: no arguments for the inherited model plus provider counts, `model` for an exact lookup including valid reasoning levels, or `provider`/`query` for a bounded search.",
         schema = {
             type = "object",
             properties = {
@@ -210,22 +310,20 @@ local function activate()
                 query = { type = "string", description = "Substring to search model names for." },
                 limit = { type = "integer", description = "Maximum matches to return (1-50, default 10).", minimum = 1, maximum = 50 },
                 model = { type = "string", description = "Exact `provider:model` to look up." },
-                agent = { type = "string", description = "Agent profile whose effective model to report." },
             },
         },
         handler = function(input)
-            return models.handle(input, profiles)
+            return models.handle(input)
         end,
     }
 
     ext.register_tool {
         name = "subagents.lua",
-        description = "Run a one-off Lua workflow that fans several subagents out and combines their results. `source` must return subagents.workflow(function(ctx, input) ... end) and runs in a restricted environment with no filesystem, process, or module access. Optional `agents` define profiles available only to this workflow.",
+        description = "Run sandboxed Lua that starts background subagent workflows or inspects existing ones. `source` can call subagents.workflow(function(ctx) ... end), which returns a workflow id immediately, and can read immutable records from subagents.workflows. Optional `agents` define profiles available only to workflows started by this call.",
         schema = {
             type = "object",
             properties = {
-                prompt = { type = "string", description = "The workflow input, passed to the callback as its second argument." },
-                source = { type = "string", description = "Lua source returning subagents.workflow(function(ctx, input) ... end)." },
+                source = { type = "string", description = "Lua source that starts a workflow or inspects subagents.workflows." },
                 agents = {
                     type = "array",
                     description = "Agent profiles available only to this workflow. Inline profiles shadow discovered profiles with the same name.",
@@ -240,11 +338,11 @@ local function activate()
                     },
                 },
             },
-            required = { "prompt", "source" },
+            required = { "source" },
         },
         handler = function(input, context)
             progress.bind(context)
-            local result = luatool.handle(input, profiles)
+            local result = luatool.handle(input, profiles, context)
             progress.settle(context)
             return result
         end,
@@ -252,34 +350,14 @@ local function activate()
 
     ext.register_tool {
         name = "subagents.workflow",
-        description = "Run a fixed dependency graph of subagents: `name` runs a discovered TOML workflow, or `steps` defines one inline. Every step receives `prompt` as the workflow input, and a step with `needs` also receives those steps' outputs.",
+        description = "Run a discovered TOML workflow by `name`: a fixed dependency graph of subagents. Every step receives `prompt` as the workflow input, and a step with `needs` also receives those steps' outputs. For dynamic graphs, use `subagents.lua`.",
         schema = {
             type = "object",
             properties = {
-                name = { type = "string", description = "Name of a discovered workflow. Mutually exclusive with `steps`." },
+                name = { type = "string", description = "Name of a discovered workflow." },
                 prompt = { type = "string", description = "The workflow input, given to every step." },
-                steps = {
-                    type = "array",
-                    description = "Inline workflow definition. Mutually exclusive with `name`.",
-                    items = {
-                        type = "object",
-                        properties = {
-                            id = { type = "string", description = "Unique step id." },
-                            agent = { type = "string", description = "Agent profile to run this step." },
-                            prompt = { type = "string", description = "Step instruction." },
-                            model = { type = "string", description = "Optional `provider:model` override." },
-                            reasoning = { type = "string", description = "Optional reasoning level override." },
-                            needs = {
-                                type = "array",
-                                description = "Ids of steps whose output this step receives.",
-                                items = { type = "string" },
-                            },
-                        },
-                        required = { "id", "agent", "prompt" },
-                    },
-                },
             },
-            required = { "prompt" },
+            required = { "name", "prompt" },
         },
         handler = function(input, context)
             progress.bind(context)
@@ -290,6 +368,7 @@ local function activate()
     }
 
     local workflows = toml_workflows.discover_and_register(profiles)
+    install_guidance(ext)
     install_header(ext, profiles, workflows)
 end