From cc69a2e431779528578211a5cb3385bb23e076bf Mon Sep 17 00:00:00 2001 From: t Date: Tue, 18 Aug 2026 14:18:14 -0600 Subject: Persist and replay subagent progress across sessions Record durable child-turn metadata, restore bounded progress cards during startup replay, and keep settled cards attached to transcript entries. Add workflow-local Lua profiles and expose discovered agents and workflows in the session header. --- DESIGN.md | 655 ----------------------------------- README.md | 8 +- REFACTOR.md | 196 ----------- e2e/run.lua | 34 +- init.lua | 89 ++++- spec/fake_ext.lua | 82 ++++- spec/test_init.lua | 57 ++- spec/test_luatool.lua | 69 ++++ spec/test_progress.lua | 322 +++++++++++++++++ spec/test_progress_replay.lua | 316 +++++++++++++++++ spec/test_run.lua | 48 +++ spec/test_toml_workflows.lua | 29 +- subagents/jobs.lua | 3 +- subagents/luatool.lua | 74 +++- subagents/progress.lua | 788 +++++++++++++++++++++++++++++++++--------- subagents/spawn.lua | 59 +++- 16 files changed, 1792 insertions(+), 1037 deletions(-) delete mode 100644 DESIGN.md delete mode 100644 REFACTOR.md create mode 100644 spec/test_progress.lua create mode 100644 spec/test_progress_replay.lua diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 1042885..0000000 --- a/DESIGN.md +++ /dev/null @@ -1,655 +0,0 @@ -# panto-subagents design - -## Product shape - -`panto-subagents` is a separate rock exposing one extension named -`subagents`. Its model-facing tools use the `subagents.*` namespace. It does -not shell out to `panto -p`; every child is an ordinary in-process -libpantograph `Agent` driven by pantograph. - -The primary owns delegation. Children share its workspace, but not its -dialogue, and cannot call `subagents.*`. Recursive delegation, worktree -management, and merge orchestration are outside this design. - -## v0 tools - -### `subagents.run` - -One tool call starts or resumes one conversational child: - -```lua --- Start a new child. -{ - agent = "reviewer", - prompt = "Review the authentication changes.", - model = "anthropic:sonnet", -- optional; always provider:model - reasoning = "high", -- optional -} - --- Resume an existing child owned by this primary session. -{ - id = "0198...", - prompt = "Now focus on the missing tests.", - model = "openai:gpt-5.6", -- optional per-turn change - reasoning = "xhigh", -- optional per-turn change -} -``` - -`prompt` must be non-empty. Exactly one of `agent` and `id` is required: -`agent` selects the profile for a new child; `id` identifies a persisted child -conversation. An ID cannot be resumed by a different primary session, and -only one turn may be active for an ID. - -Once a valid new child has been allocated, every settled result includes its -ID: - -```lua -{ - id = "0198...", - agent = "reviewer", - status = "completed", -- completed, failed, or cancelled - output = "...", -- present on success - error = nil, -- present on failure - resumable = true, -} -``` - -Validation failures that happen before a child is allocated have no ID. -Under the existing filesystem-store flush discipline, a new child that fails -before producing its first assistant message has no durable file; such a -result reports `resumable = false` rather than promising continuation. - -Parallel delegation uses ordinary model tool batching. The primary emits -several `subagents.run` calls in one batch; pantograph starts them concurrently -under one session-wide bound, initially four. A failure in one call does not -discard successful siblings. There is no second `tasks` array, task ID, or -extension-specific batch report format. - -### `subagents.models` - -The model catalog is too large to place in `subagents.run`'s description or -JSON schema. A small query tool exposes it on demand: - -```lua -subagents.models {} --- Current inherited model/reasoning plus configured provider names and counts. - -subagents.models { - provider = "anthropic", - query = "sonnet", - limit = 10, -} - -subagents.models { model = "anthropic:sonnet" } -subagents.models { agent = "reviewer" } -``` - -An exact model or agent lookup returns its effective model, default reasoning, -and the reasoning levels known to be valid for it. A search returns only the -requested bounded matches and says when more exist. The tool reuses -pantograph's model registry and provider-specific reasoning-option logic, -including effort choices reported dynamically by Lua protocols. - -The `subagents.run` description tells the primary to omit overrides normally -and call `subagents.models` before choosing an unfamiliar model or reasoning -level. Runtime validation remains authoritative when catalog capability data -is incomplete. - -## Agent profiles - -Profiles are Markdown files with YAML frontmatter, compatible at the common -format boundary used by other agent harnesses: - -```markdown ---- -name: reviewer -description: Reviews changes for correctness and missing tests -model: anthropic:sonnet -reasoning: high ---- - -You are a focused code reviewer. - -Report only concrete, evidence-backed findings. -``` - -They are discovered recursively from two layers, lowest precedence first: - -1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/agents/**/*.md` -2. `/.panto/agents/**/*.md` - -The project definition shadows a user definition with the same resolved name. -`name` is optional and defaults to the extension-less filename. -`description` is included with the discovered names in `subagents.run`'s tool -description so the primary can choose a profile without loading its full -prompt. - -The portable contract is `name`, `description`, and the Markdown body as the -profile's system prompt. Panto additionally recognizes `model` only in full -`provider:model` form and recognizes its existing reasoning labels. Unknown -frontmatter is ignored. A foreign model spelling that is not valid Panto -syntax is ignored with a warning so the prompt remains usable and inherits -the primary model. - -For a new child, model and reasoning are resolved independently: - -```text -model = tool.model ?? profile.model ?? primary.model -reasoning = tool.reasoning ?? profile.reasoning ?? primary.reasoning -``` - -Pantograph resolves the selected model against the configured provider and -model registries, applies the reasoning level, and fails before inference if -the known combination is invalid. The Lua extension never receives provider -credentials. - -The resolved model reference and reasoning label are recorded on each child -turn. On continuation, omitted overrides keep the child's last effective -values; an explicit `model` or `reasoning` changes that value for the new turn -and for later continuations. - -## Conversation construction - -A new child receives a fresh `Conversation` containing: - -1. the primary's effective system and project context; -2. a fixed child-role instruction; -3. the selected profile's Markdown body as a separate system message; and -4. the tool's `prompt` as its only initial user message. - -The parent dialogue is not copied. The primary must put task-specific context -in the user prompt. - -The profile system message carries opaque, non-model-visible message metadata -recording the immutable subagent manifest: its owning primary session ID and -profile name. Each user message carries the full model reference and reasoning -label effective for that turn. This avoids a mutable sidecar manifest: resume -reads identity from the first system message and defaults from the latest user -message. The existing filesystem-store load path must preserve per-message -metadata when rebuilding a `Conversation`. - -The stored conversation is canonical on resume. Profile edits affect new -children, not an existing child ID. - -Every child sees the parent's normal registered tools except `subagents.*`. -Tools still run through the shared extension runtime. Children share the -workspace, so concurrent writers can race or conflict; the primary must -delegate independent work. - -## Durable child sessions - -Subagent conversations are ordinary `FileSystemJSONLStore` sessions, not an -in-memory `id -> Conversation` map. The public subagent ID is the actual Panto -session UUID and the JSONL filename stem. - -Each primary session owns a separate child catalog below its already-resolved -per-cwd session directory: - -```text -/ -├── .jsonl -└── subagents/ - └── / - ├── .jsonl - └── .jsonl -``` - -This layout automatically follows `PANTO_SESSION_DIR` and the normal XDG -fallback. The main store's `list` and `resolve` operations examine only direct -`.jsonl` children, so nested subagent files never appear in `panto sessions` -or the `/resume` picker. - -The primary session ID is the ownership boundary. After panto restarts and -the user resumes a primary conversation, extension activation derives the -same child-catalog path from that primary ID. A referenced child ID therefore -resolves again. A different primary uses a different directory and cannot -resolve it. There is no global child lookup and no cross-primary continuation. - -Starting and resuming use the ordinary session lifecycle: - -```text -new: - session = child_store.create() - agent = Agent.init(session, no prior conversation) - seed system/profile messages - run prompt - -resume: - session = child_store.resolve(id) - conversation = session.load() - agent = Agent.init(session, conversation) - run prompt -``` - -The Agent appends through its `Session` handle, and the filesystem store syncs -completed entries normally. The Agent may be destroyed after a turn; a later -turn reconstructs the conversation from JSONL. Only active job handles, -progress, and cancellation state remain in memory. - -## Foreground progress and cancellation - -Silent foreground waiting is not acceptable. Each child job has its ID and -profile before inference starts and emits tagged progress events to the TUI. -The user can see the child's assistant stream, tool calls, and tool results -while the primary model remains blocked on the tool call. Concurrent children -remain visually distinguishable. - -These nested events are presentation data, not partial tool results: they are -not inserted into the primary model's conversation. `subagents.run` returns -only the settled result, while the full child conversation is persisted under -its ID for inspection and continuation. - -Escape cancels every still-running child belonging to the current foreground -tool operation. Completed siblings remain successful. Cancellation targets -the child streams and the protocol streams those turns opened, then lets -every affected tool call settle coherently as `cancelled` rather than -abandoning the parent's batch. - -## Generic pantograph child-job seam - -libpantograph remains unaware of Lua, extensions, and subagents. Its `Agent` -and `SessionStore` abstractions already provide the conversation and -persistence machinery, and no subagent scheduler or tree policy lives -anywhere in Zig. The seam above it has three layers: a Lua binding -(libpanto-lua) that adds generic job, metadata, and tool-control primitives -usable by any embedder; a thin surface in pantograph itself (`panto.ext`) for -the few things that need the running process or its credentials; and this -rock, which is Lua policy written entirely on those two layers. Nothing -subagent-specific exists below the rock. - -The binding's async job is the generic replacement for a hand-rolled worker -thread: - -```lua -local job = agent:run_async { - prompt = "...", -- or blocks, mirroring agent:run - metadata = { ... }, -- optional user-message metadata, a JSON - -- object (array-shaped tables are refused) - dispatch_tools = false, -- optional, default true: false ends the - -- turn at the first assistant response - -- instead of dispatching its tool calls - wake_fd = fd, -- optional: one byte is written on every - -- event and on settle -} - -job:next_event() -- -> event table | nil (non-blocking; owned copy) -job:result() -- -> settled result table | nil while running -job:request_cancel() -job:close() -- joins the pump and frees the settled result; a - -- caller must read and cache job:result() first -``` - -A pump thread owned by the job, not by the caller, drives the blocking turn -to completion, copies each event off the stream before the next pull frees -it, and buffers it under a mutex. The `wake_fd` contract is deliberately -loop-agnostic — a luv consumer arms `uv.new_poll(fd)`, any other reactor -selects on it — so no loop dependency reaches into the binding. Tool dispatch -inside the turn is unchanged libpantograph behavior: when the job's agent -carries tools registered through pantograph's Lua tool source (the primary's -own extension tools, copied over via `agent:set_tools`), `invoke_batch` posts -to the Lua owner thread through the same multiplexed executor a primary's -batch uses (below), bound to that job's agent as its lane. `dispatch_tools = -false` is the generic knob that makes a one-shot structured worker possible -without any host involvement: the settled result carries the assistant's -tool calls (id, name, and raw input JSON) unresolved, and no second model -round runs. - -Two more binding primitives complete the generic surface. Message metadata — -`conv:message_metadata(i)` / `conv:set_message_metadata(i, tbl)`, plus an -optional `metadata` argument on `add_system_message`/`add_user_message` — -reads and writes the same JSON-object metadata the disk format already -round-trips, on owned and borrowed conversations alike; the rock uses it for -both the subagent manifest and the per-turn model/reasoning stamp, with no -separate sidecar record. Tool control — `agent:tools()` returns a mutable -copy of an agent's declarations, `agent:set_tools(decls)` replaces them, and -`agent:set_config` additionally accepts `tool_choice` to force a named tool — -lets the rock copy the primary's tools onto a child and drop `subagents.*`, -or hand a child exactly one synthetic, call-refusing declaration for a -one-shot worker's output tool. Dispatch is name-keyed on the shared runtime -tool source, so a copied declaration reaches the same handler regardless of -which agent it is registered on. - -pantograph adds only what requires the running process or its credentials. -`panto.ext.resolve_model { model = "provider:alias", reasoning = label, -tool_choice = ... }` returns an **opaque config userdata** accepted by -`panto.agent { config = ... }` and `agent:set_config`; resolution reuses the -existing provider/model registries and reasoning pipeline, fails before -inference, and the userdata exposes only `model`/`reasoning`/`style`/ -`wire_model` — credentials never reach Lua. This is the one piece of the seam -that could not be a binding API. It, `panto.ext.session_info()`, and -`panto.ext.models` all read from one feature-neutral, read-only session -record: the provider and model registries, the primary's already-credentialed -live provider config, the current model label, registered extension -protocols, and the session's id and directory. `panto.ext.on("turn_start" | -"turn_interrupt" | "turn_end", fn)` fires around every turn on the primary -session — interrupt before the pump parks on Escape or Ctrl+C, end on every -exit path with a reason — generic and observe-only; the rock is one -subscriber among any number that could exist, cancelling every live child job -on interrupt and closing them on end. `event:set_component(tbl)` returns a -handle (`h:invalidate()`, `h:alive()`) so a component that mutates after it is -set — a progress card gaining a line — can ask for a repaint instead of -rendering once and going stale. - -Everything else — the concurrency gate and queue, child store layout and -directory creation on top of `SessionStore`, manifest and resume-default -extraction from message metadata, tool filtering, one-shot worker -construction, cancellation propagation, and progress rendering — is Lua -policy in the rock, built entirely on the primitives above. A different -extension could build an unrelated concurrent-job feature on exactly the same -seam. - -## Shared Lua runtime executor - -All agents in one primary session share its Lua state, activated extensions, -and libuv loop. Children do not create Lua states, reload rocks, or -reactivate extensions. - -The runtime multiplexes tool batches rather than tracking one global -in-flight batch — the primary's and one per in-flight child agent can be live -at once, all executing on the single Lua owner thread: - -1. Parent tool handlers start child jobs and yield. -2. Child workers drive ordinary libpantograph streams. -3. Built-in providers run directly on those workers. -4. A child Lua tool call is posted to a thread-safe runtime queue, waking the - shared loop through a pantograph-owned `uv_async_t`. A Lua protocol call - made on behalf of a child travels the same queue. -5. The runtime executes a tool callback as a coroutine on the sole Lua owner - thread, where async extension code may yield normally. A protocol callback - is posted the same way but runs to completion under `pcall` on that - thread: it cannot yield. -6. Completion returns to the waiting child worker, while unrelated batches - keep making progress. - -Tool batches carry explicit identity, so result recording always addresses -the right batch rather than one global slot. Each batch is bound to the agent -that raised it, and `panto.ext.agent` resolves to that Agent while the -batch's coroutine runs. Protocol sources are not bound this way — one -registered protocol serves every agent in the session, and its streams keep -the turns apart. Module globals remain shared intentionally. A -child's tool declarations are exactly whatever its own `agent:set_tools` call -put there — there is no host-side filtered view to keep in sync. - -## Extension-provided protocol concurrency - -A registered protocol is process-wide, not per-child: the primary and every -child job reach it through the one `ProtocolSource` the session installed. -What keeps concurrent turns apart is the stream, not a per-job lane — each -`open` returns its own. This remains a pantograph implementation detail -rather than a libpantograph or workflow concept. - -Each call to an extension-provided protocol's `open` creates an independently -managed stream, conceptually: - -```lua -panto.ext.register_protocol { - name = "external-runtime", - open = function(request) - local transport = open_transport(request) - return { - next = function() return transport:next() end, - cancel_turn = function() transport:cancel_turn() end, - close = function() transport:close() end, - } - end, - close = function() -- runtime-wide teardown - end, -} -``` - -Mutable turn state belongs to the returned stream or its closure, never a -module-global "current session." Multiple streams from one registered -protocol may be open concurrently and interleave between calls. A protocol -body — `open`, or a stream's `next`, `cancel_turn`, or `close` — runs to -completion on the loop thread under `pcall`: it cannot yield, and it must not -block, because that thread is the one every other stream and tool batch needs -back. Awaiting belongs in tool handlers, which do run as coroutines and may -park on luv work; the shared pantograph scheduler remains the sole event-loop -owner. - -Cancellation and compaction are scoped to the stream, not the registration. -Abandoning a turn calls that stream's own `cancel_turn`; a protocol that -defines none falls back to the registration-level `cancel_turn`, which is -process-wide by construction. A child compacting its own conversation does -not reset protocol sessions the primary and its siblings are streaming on; -only the session's own teardown runs a protocol's registration-level -`close`. - -Every child turn, including a resumed child, receives a fresh protocol stream. -The loaded Panto JSONL conversation is canonical and is present in the open -request; extension protocols cannot depend on private transport history for -continuation. A protocol may serialize the structured messages directly. If -its transport only accepts a textual bootstrap, it can use the existing -generic un-summarized transcript serializer, taking care not to duplicate -system context supplied separately in the request. - -## Lua workflow API - -Workflows build on the same Agent jobs after v0. The canonical Lua API is -callback- and await-based so a workflow can branch and fan out from structured -child output: - -```lua -return subagents.workflow(function(ctx, input) - local split = ctx:agent { - agent = "splitter", - prompt = input, - output = { - description = "Return the independent work items.", - schema = { - type = "object", - required = { "items" }, - properties = { - items = { - type = "array", - items = { type = "string" }, - }, - }, - }, - }, - }:await() - - local jobs = {} - for _, item in ipairs(split.output.items) do - jobs[#jobs + 1] = ctx:agent { - agent = "worker", - prompt = "Process this item:\n" .. item, - } - end - - local results = ctx:await(jobs, "all") - return results -end) -``` - -`ctx:agent` starts immediately and returns a job handle. `handle:await()` -waits for one. `ctx:await(handles, "all")` returns settled results in input -order. `ctx:await(handles, "first")` returns the first settled result and the -remaining handles. Child failures are result values so the callback can -decide whether to continue, retry differently, or return an error. All jobs -share pantograph's global concurrency bound. - -When `output` is present, the child is a one-shot structured worker: - -- it receives only one synthetic output tool; -- tool choice is forced to that tool; -- its first model response must call the tool exactly once; -- the tool input is validated against `output.schema`; -- no tool result is sent back and no second model round runs; and -- `result.output` is the decoded tool input, not parsed assistant prose. - -One-shot structured workers are ephemeral and not resumable conversations. -They exist to make reliable reducers, classifiers, planners, and dynamic -fan-out inputs without inventing another structured-output protocol. - -The module is available directly to human-authored Panto extensions, which -can register their own commands with `panto.ext.register_command`. - -## Model-authored transient workflows - -A later model-facing tool executes a transient Lua workflow without writing a -definition to disk: - -```lua -subagents.lua { - prompt = "Review the changed subsystems.", - source = "...Lua returning subagents.workflow(function(ctx, input) ... end)...", -} -``` - -The source runs as a coroutine in Panto's existing Lua state, but under a -restricted `_ENV` containing the workflow API and safe standard libraries. -It cannot access `os`, `io`, `debug`, `package`, `require`, raw Panto runtime -objects, or extension globals. An instruction budget between yields and a -bounded number of jobs prevent generated Lua from wedging the owner thread or -creating unbounded inference work. Human-authored extensions are trusted Lua -and use the module directly without this restricted environment. - -No separate workflow-authoring skill ships initially. The tool schema and -description are sufficient until actual traces show that longer guidance is -needed. - -## TOML workflows and slash commands - -Persistent TOML workflows are the smaller, fully pre-specified DAG surface. -They are discovered recursively from: - -1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/workflows/**/*.toml` -2. `/.panto/workflows/**/*.toml` - -Project workflows shadow user workflows by resolved name. A workflow may -define independent roots, fixed dependencies, and per-step call overrides: - -```toml -name = "review-chain" -description = "Inspect a change from two angles, then synthesize." - -[[steps]] -id = "correctness" -agent = "reviewer" -prompt = "Review for correctness and regressions." - -[[steps]] -id = "tests" -agent = "reviewer" -prompt = "Review test coverage and validation gaps." -reasoning = "high" - -[[steps]] -id = "synthesis" -agent = "reviewer" -prompt = "Synthesize the findings and remove duplicates." -needs = ["correctness", "tests"] -``` - -The parser validates duplicate IDs, unknown dependencies, and cycles before -starting work, then lowers the definition onto the same Lua job/await -primitives. Every step receives the original workflow input. A dependent also -receives its dependencies' outputs, clearly labeled and ordered by `needs`. -All ready steps start concurrently. A failed node skips its dependents while -unrelated branches finish. Terminal-node results and failures are returned in -declaration order. - -Every discovered workflow registers `/workflow:`. The command tail—the -equivalent of `$ARGUMENTS` in prompt commands—becomes the user prompt supplied -to the DAG's head nodes. Human-authored Lua workflows register any desired -slash command themselves. - -The model-facing `subagents.workflow` tool runs either a discovered workflow -by name or a transient static step definition. Output-dependent branches and -dynamic fan-out remain Lua features; TOML does not grow into a programming -language. - -## Deferred model-facing background control - -Foreground tool batching and workflow-internal awaits cover the initial -parallelism requirements. Background control can be added without changing -the job model: - -```lua -subagents.run { - agent = "researcher", - prompt = "Investigate this in the background.", - await = false, -} --- { id = "0198...", status = "running" } - -subagents.await { - ids = { "0198...", "0199..." }, - mode = "first", -- or "all" -} -``` - -`first` means the first settled result, successful or not; `all` waits for -every named child. IDs are explicit so a primary never waits on unrelated -work. Results are not consumed by awaiting them. Adding this surface also -requires background result retention, targeted cancellation/status, and TUI -lifecycle behavior, so it is not part of v0. - -## Validation - -Pantograph checks must prove: - -- independent child Agents make concurrent progress under the global bound; -- Lua tool batches from multiple children cannot corrupt results or enter the - Lua state concurrently; -- built-in and Lua protocol children can run together; -- lane-local `panto.ext.agent` resolves to the correct child; -- progress events remain tagged to the correct child; -- cancellation settles every affected stream and tool batch coherently; -- one failed child does not cancel successful siblings; and -- child tool declarations omit `subagents.*` while shared extension - activation happens exactly once. - -Session checks must prove: - -- successful child turns persist as normal JSONL sessions; -- the child ID is the persisted session ID; -- restarting and resuming a primary can continue its referenced child; -- another primary cannot resolve that ID; -- nested child files never appear in the primary session picker or listing; -- the subagent manifest and full conversation round-trip through persistence; - and -- incomplete first turns do not falsely report durable resumability. - -Extension checks must cover profile discovery and shadowing, portable -frontmatter parsing, independent model/reasoning precedence, catalog query -bounding, new-versus-resume validation, and tool-batch concurrency. - -Workflow checks, when those phases land, must cover callback awaiting, -structured-tool validation, output-driven fan-out, DAG validation, dependency -prompt assembly, branch-local failure, deterministic terminal ordering, and -generated slash-command registration. - -Extension-protocol checks must run multiple mocked streams concurrently and -prove that startup, events, tool calls, cancellation, transcript bootstrap, -and closure stay scoped to the stream that opened them — including that -cancelling one turn leaves its siblings' streams untouched. - -## Delivery order - -1. Add generic concurrent Agent jobs, global bounding, multiplexed Lua - dispatch, per-stream protocol concurrency, nested progress, and foreground - cancellation. -2. Add layered Markdown profiles, model/reasoning resolution, bounded catalog - queries, durable tree-scoped child stores, continuation, and - `subagents.run`/`subagents.models`. -3. Add canonical-conversation bootstrap support for extension protocols whose - transports start without history. -4. Add the callback-based Lua workflow API and one-shot structured workers. -5. Add the restricted model-authored Lua workflow tool. -6. Add TOML DAG loading, `subagents.workflow`, and generated - `/workflow:` commands. -7. Add model-facing background start/await/status/cancellation only when the - foreground behavior is proven. - -## Not in v0 - -- cross-primary subagent lookup or continuation; -- recursive subagents; -- background jobs or model-facing await/status tools; -- worktree creation, merge orchestration, or write-conflict protection; -- durable workflow-run state or artifact management; -- agent/profile CRUD tools; -- a workflow-authoring skill; or -- Lua/TOML condition syntax beyond the callback and fixed-DAG surfaces above. diff --git a/README.md b/README.md index 713ea60..73a3146 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,11 @@ outputs. The Lua workflow API handles dynamic branching and fan-out. It can await one job or a group, and supports one-turn workers whose validated tool input is -their structured result. Panto extensions can use this API directly, while -`subagents.lua` runs model-authored one-off workflows in a restricted Lua -environment. +their structured result. `subagents.lua` may also define workflow-local agent +profiles inline, so a primary can create specialized workers without writing +profile files or restarting Panto; those profiles disappear when the tool call +ends. Panto extensions can use this API directly, while `subagents.lua` runs +model-authored one-off workflows in a restricted Lua environment. ## Development diff --git a/REFACTOR.md b/REFACTOR.md deleted file mode 100644 index 3eec504..0000000 --- a/REFACTOR.md +++ /dev/null @@ -1,196 +0,0 @@ -# panto-subagents refactor: extension-API-first - -## Goal - -The v0 implementation works end to end, but pantograph absorbed the feature: -`subagent_host.zig`, `subagent_runtime.zig`, and `tui_subagents.zig` are a -subagent backend in core, and the rock is mostly validation and formatting -glue. Per the "Extension features: two responses" rubric in pantograph's -AGENTS.md, this refactor takes response 2 retroactively: the missing *general* -capabilities are added to libpanto-lua and `panto.ext`, subagent policy moves -into this rock, and the feature-shaped core surface is deleted. - -Definition of done, in one line: `rg -i subagent pantograph/src` returns -nothing, and every behavior in DESIGN.md still holds through the real binary. - -The product surface (DESIGN.md) is unchanged. This is a re-division of the -implementation, not a redesign. DESIGN.md's "Generic pantograph child-job -seam" section is superseded by this document and should be rewritten to match -when this lands. - -## What exists today (all currently uncommitted working-tree state) - -Generic and staying (invisible runtime correctness, landed with v0): - -- The multiplexed Lua executor: tool batches keyed by id instead of a single - `current_batch`, cross-thread posted jobs onto the Lua owner thread - (`lua_executor.zig`, `lua_runtime.zig`). -- Lane identity: `panto.ext.agent` resolves through `coroutine.running()`; - per-open protocol stream state and stream-level `cancel_turn` - (`lua_provider.zig`). -- libpantograph: message metadata survives the JSONL load path; - `UserMessage.metadata` exists on `Agent.run`. -- `panto.ext.models` (bounded catalog queries) and `panto.ext.session_info()`. - -Feature-shaped and going: `subagent_host.zig` (vtable), `subagent_runtime.zig` -(spawn transaction, worker threads, `Gate` concurrency bound, metadata -stamping, manifest extraction, resume defaults, one-shot workers, progress -queue), `tui_subagents.zig` (hardcoded progress panel), and the -`spawn_agent`/`await_jobs` thunks in `lua_bridge.zig`. - -## New libpanto-lua APIs - -These are library bindings, useful to any embedder of libpanto-lua; nothing -about them is subagent-specific. - -### Async agent jobs - -```lua -local job = agent:run_async { - prompt = "...", -- or blocks, mirroring agent:run - metadata = { ... }, -- optional user-message metadata (JSON-encoded) - dispatch_tools = false, -- optional: end the turn at the first - -- assistant response instead of dispatching - -- its tool calls; they are reported settled - wake_fd = fd, -- optional: a byte is written on every event - -- arrival and on settle -} - -job:next_event() -- -> event table | nil (non-blocking; owned copy) -job:result() -- -> settled result | nil while running -job:request_cancel() -job:close() -- join + release; safe after settle or cancel -``` - -- Implemented with a binding-internal pump thread per job: it drives the - blocking `stream.next()`, copies each event payload (payloads die at the - next pull), buffers under a mutex, and writes one byte to `wake_fd`. -- The fd contract is deliberately loop-agnostic: a luv consumer arms - `uv.new_poll(fd)`; a Go or C embedder selects on it. No loop dependency - enters libpanto-lua. -- Tool dispatch inside the turn is unchanged libpantograph behavior; when the - agent's tools include panto's Lua tool source, its `invoke_batch` posts to - the Lua owner thread through the existing executor — that path already - exists and does not change. -- `dispatch_tools = false` is the generic knob that makes structured one-shot - workers a pure consumer feature: the settled result carries the assistant's - tool calls (name + input JSON) unresolved, and no second model round runs. - -### Conversation message metadata - -```lua -conv:message_metadata(i) -- -> table | nil (decoded JSON) -conv:set_message_metadata(i, tbl) -- JSON-encodes; nil clears -``` - -Plus optional `metadata` on `conv:add_system_message`/`add_user_message`. -The disk format already round-trips metadata; this is access only. - -### Agent tool control - -```lua -agent:tools() -- -> array of decl tables { name, description, schema }, - -- a mutable copy -agent:set_tools(decls) -- replace this agent's tool list with these decls; - -- handlers resolve by name through the owning source -``` - -Copying between agents works because dispatch is name-keyed on the shared -runtime. `agent:set_config` additionally accepts `tool_choice` (force a named -tool). This retires the Zig-side `toolSourceExcluding`. - -## New/changed pantograph surfaces - -- `panto.ext.resolve_model { model = "provider:alias", reasoning = label }` - → an **opaque config userdata** accepted by `panto.agent { config = ... }` - and `agent:set_config`. Resolution and validation reuse the existing - registry/reasoning pipeline and fail before inference; credentials are - embedded host-side and never readable from Lua. This is the one place the - credential boundary requires a host API rather than a binding API. -- `panto.ext.models` and `panto.ext.session_info()` remain as-is (already - generic). -- `spawn_agent`, `await_jobs`, the job-handle userdata, and the - `SubagentHost` vtable are deleted once the rock is ported. Awaiting becomes - ordinary luv: the rock arms `uv.new_poll` on each job's `wake_fd` and its - tool-handler coroutine yields/resumes through the standard async-tool - contract. -- Progress rendering: `tui_subagents.zig` is deleted. The rock pulls its own - job events and surfaces activity through the existing extension event and - component machinery (`panto.ext.on`/`emit` + component swap). If that - machinery proves insufficient for a live multi-card panel, the fallback is - one small *generic* host API (a "live status card" keyed by an id — second - consumer: any long-running extension activity), to be co-designed before - building, per AGENTS.md. - -## Policy migration map (the closure check) - -Every deleted Zig behavior must have a Lua expression using only the APIs -above. This table is the completeness test for the API set: - -| Behavior (today in Zig) | New home in the rock | -|------------------------------------------|----------------------| -| Child store layout `/subagents/` | string assembly (already rock-side) | -| Store create/resolve, ownership boundary | `panto.file_system_jsonl_store(dir)` + `store:resolve/load` | -| Manifest on profile system message | `conv:set_message_metadata` at seeding | -| Per-turn model/reasoning stamp | `run_async { metadata = ... }` | -| Resume defaults from last user message | `store:load` + `conv:message_metadata` scan | -| Primary system context for new children | copy system messages from `panto.ext.agent:conversation()` | -| Concurrency bound (4) + queue + cancel-while-queued | rock-side gate: start ≤4 jobs, queue the rest, drop queued on cancel (~15 lines of Lua) | -| `subagents.*` exclusion (no recursion) | `agent:set_tools(filtered)` from the primary's `agent:tools()` | -| One-shot structured workers | new agent + single synthetic tool + `tool_choice` + `dispatch_tools = false` + `panto.null_store()`; validate with the existing jsonschema path | -| `resumable` determination | fresh `store:resolve(id)` after failure | -| Result shaping (id/agent/status/output) | already rock-side; loses the host `JobResult` intermediary | -| Progress cards | rock consumes `job:next_event()`; renders via event/component machinery | -| Escape cancels all children | rock tracks its live jobs; a host cancellation event (or the turn-teardown hook that fires it) calls `job:request_cancel()` on each — verify the existing bus exposes interrupt; if not, add a generic turn-lifecycle event | - -Two rows need small verifications during implementation (marked above): -turn-lifecycle/interrupt visibility to extensions, and whether the component -machinery can host the live panel. Both resolutions must stay feature-neutral. - -## Deletions - -pantograph: `subagent_host.zig`, `subagent_runtime.zig`, `tui_subagents.zig`, -the spawn/await thunks and job userdata in `lua_bridge.zig`, the subagent -wiring in `main.zig`/`tui_app.zig`, and `subagent_integration_tests.zig` in -its current form (see validation). The executor, lane machinery, and -`lua_provider` lane calls remain as internal runtime correctness — reviewed -afterward for members only the deleted code used. - -## Validation - -- The v0 end-to-end proof must survive: a scripted extension-protocol fixture - driving the real binary with the real rock — parallel `subagents.run` + - unknown-agent error in one batch, child resume with history, workflow - diamond, sandbox fan-out, catalog round-trip, child JSONL - isolation/manifest/metadata asserted from bytes on disk. (Reconstruct the - harness from this list; treat it as the primary gate.) -- Rock suite (`mise run check`) extended to cover the migrated policy: - gating/bound behavior, resume-default extraction, one-shot capture via - unresolved tool calls, tool filtering, manifest seeding. -- pantograph integration tests rewritten against the *generic* surfaces - (async jobs driving concurrent agents, batch isolation, lane agent - resolution, cancellation) with no subagent vocabulary. -- `rg -i subagent /Users/travis/Code/pantograph/src` → empty. -- All suites green: `mise exec -- zig build test`, `mise run check`. - -## Delivery order - -1. libpanto-lua: `run_async` jobs + fd contract + `dispatch_tools` knob, with - binding-level tests (fixture provider, no panto involvement). -2. libpanto-lua: message-metadata access; agent tool get/set + `tool_choice`. -3. pantograph: `resolve_model` opaque config; verify interrupt/lifecycle - event visibility (add the generic event if missing). -4. Rock: port spawn/await/bound/policy onto the new APIs behind the same tool - surfaces; port progress to event/component rendering. -5. Delete the pantograph subagent files and thunks; rewrite integration tests - generically; run the full validation list. -6. Update DESIGN.md's seam sections to describe the final division. - -## Interop with the panto-on-libuv project (separate; order-independent) - -The `wake_fd` contract works identically in both worlds: today the rock arms -it with `uv.new_poll` on luv's loop; after the libuv unification it is the -same call on the shared process loop. If the libuv project lands first, the -executor internals this refactor relies on (posted jobs onto the Lua owner) -will already use `uv_async` instead of the pipe waker — no change to anything -specified here. Neither project should block on the other. diff --git a/e2e/run.lua b/e2e/run.lua index c240813..56deda6 100644 --- a/e2e/run.lua +++ b/e2e/run.lua @@ -409,6 +409,14 @@ local function meta(needle) return (needle:gsub('"', '\\"')) end +local function json_string(text) + return '"' .. tostring(text) + :gsub('\\', '\\\\') + :gsub('"', '\\"') + :gsub('\r', '\\r') + :gsub('\n', '\\n') .. '"' +end + local function jsonl_ids(dir) local ids = {} for _, name in ipairs(list_dir(dir)) do @@ -630,6 +638,8 @@ scenario("resume", function() check_contains("the resumed child kept its id", second, "id: " .. child_id) local text = read_file(children .. "/" .. child_id .. ".jsonl") or "" + check_contains("the resumed child records its owning outer tool call", text, + meta('"tool_call_id"')) local turns = 0 for _, line in ipairs(lines_of(text)) do if line:find(meta('"subagents"'), 1, true) and line:find(meta('"model":'), 1, true) then @@ -705,7 +715,29 @@ scenario("sandbox-fanout", function() #jsonl_ids(assert(child_dir(install, primary_id))), 2) end) --- 5. The catalog tool, in all four shapes, against a provider whose reasoning +-- 5. An inline profile in the dynamic workflow gets a durable marker and +-- body, so restart replay can show the requested system prompt without +-- exposing discovered profile prompts. +scenario("inline-replay-manifest", function() + local install = make_install("inline-replay-manifest") + local source = "return subagents.workflow(function(ctx, input) return ctx:agent({ agent = 'inline', prompt = 'say INLINE-CHILD' }):await() end)" + local script = string.format( + 'tool subagents.lua {"prompt":"INLINE-INPUT","source":%s,"agents":[{"name":"inline","system_prompt":"INLINE-SYSTEM"}]}', + json_string(source)) + local out = turn(install, script) + check_contains("the inline workflow child completed", out, "INLINE-CHILD") + + local dir = assert(session_dir(install)) + local primary_id = jsonl_ids(dir)[1] + local children = assert(child_dir(install, primary_id)) + local ids = jsonl_ids(children) + check_equal("one inline workflow child persisted", #ids, 1) + local text = read_file(children .. "/" .. ids[1] .. ".jsonl") or "" + check_contains("the inline manifest is marked", text, meta('"inline":true')) + check_contains("the inline system prompt remains durable", text, "INLINE-SYSTEM") +end) + +-- 6. The catalog tool, in all four shapes, against a provider whose reasoning -- levels come from the protocol rather than models.toml. scenario("catalog", function() local install = make_install("catalog") diff --git a/init.lua b/init.lua index 5f28308..2e4cab1 100644 --- a/init.lua +++ b/init.lua @@ -18,9 +18,9 @@ -- profile list. -- -- Activation also subscribes to the turn lifecycle: an interrupted turn asks --- every live child to stop, and the end of a turn joins them and drops the --- progress cards, which are presentation state that must not survive the turn --- that produced it. +-- every live child to stop, and the end of a turn joins them and clears the +-- coroutine bindings. Settled progress cards remain attached to their +-- transcript entries until the host destroys those components. -- -- If the host predates the model-resolution seam, activation fails loudly -- instead of registering tools that cannot work. @@ -34,6 +34,7 @@ local spawn = require("subagents.spawn") local toml_workflows = require("subagents.toml_workflows") local MAX_SHOWN_WARNINGS = 5 +local DIM, RESET = "\27[2m", "\27[0m" local function host() return require("panto").ext @@ -47,6 +48,58 @@ local function subscribe(ext, name, handler) end end +local function wrap_plain(text, width) + width = math.max(1, width or 1) + local lines, from = {}, 1 + while #text - from + 1 > width do + local window = text:sub(from, from + width - 1) + local cut = window:match("^.*() ") or width + lines[#lines + 1] = text:sub(from, from + cut - 1):gsub("%s+$", "") + from = from + cut + while from <= #text and text:sub(from, from) == " " do + from = from + 1 + end + end + lines[#lines + 1] = text:sub(from) + return lines +end + +local function install_header(ext, profiles, workflows) + local profile_names, workflow_names = {}, {} + for _, profile in ipairs(profiles.list) do + profile_names[#profile_names + 1] = profile.name + end + for _, workflow in ipairs(workflows.list) do + if workflow.definition then workflow_names[#workflow_names + 1] = workflow.name end + end + table.sort(profile_names) + table.sort(workflow_names) + + subscribe(ext, "session_start", function(event) + local inner = event:get_component() + event:set_component({ + render = function(_, width) + local lines = inner:render(width) + local extras = {} + for _, inventory in ipairs({ + { "subagents", profile_names }, + { "workflows", workflow_names }, + }) do + local value = #inventory[2] == 0 and "(none)" or table.concat(inventory[2], ", ") + for _, line in ipairs(wrap_plain(" " .. inventory[1] .. ": " .. value, width)) do + extras[#extras + 1] = DIM .. line .. RESET + end + end + local at = (#lines > 0 and lines[#lines] == "") and #lines or (#lines + 1) + for index = #extras, 1, -1 do + table.insert(lines, at, extras[index]) + end + return lines + end, + }) + end) +end + local function run_description(profiles) local lines = { "Delegate a task to a subagent and wait for its report. Start a new child from an agent profile, or continue a child you started earlier in this session.", @@ -100,6 +153,12 @@ local function activate() -- Escape reaches the children through the turn, not through a tool: the -- primary is parked inside a tool call when they are running. + -- Startup session replay fires tool lifecycle events before the first + -- live turn. After that boundary, progress.claim must stay presentation- + -- only and never rescan the child catalog for ordinary calls. + subscribe(ext, "turn_start", function() + progress.begin_live_turn() + end) subscribe(ext, "turn_interrupt", function() jobs.cancel_all() end) @@ -111,6 +170,12 @@ local function activate() subscribe(ext, "tool_call_complete", function(event) progress.claim(event) end) + subscribe(ext, "tool_result", function(event) + progress.settle(event) + end) + subscribe(ext, "tool_collapse", function(event) + progress.collapse(event) + end) ext.register_tool { name = "subagents.run", @@ -152,12 +217,25 @@ local function activate() ext.register_tool { name = "subagents.lua", - description = "Run a one-off Lua workflow that fans several subagents out and combines their results. `source` must return subagents.workflow(function(ctx, input) ... end) and runs in a restricted environment with no filesystem, process, or module access.", + description = "Run a one-off Lua workflow that fans several subagents out and combines their results. `source` must return subagents.workflow(function(ctx, input) ... end) and runs in a restricted environment with no filesystem, process, or module access. Optional `agents` define profiles available only to this workflow.", schema = { type = "object", properties = { prompt = { type = "string", description = "The workflow input, passed to the callback as its second argument." }, source = { type = "string", description = "Lua source returning subagents.workflow(function(ctx, input) ... end)." }, + agents = { + type = "array", + description = "Agent profiles available only to this workflow. Inline profiles shadow discovered profiles with the same name.", + items = { + type = "object", + properties = { + name = { type = "string", description = "Profile name used by ctx:agent." }, + description = { type = "string", description = "Optional human-readable purpose." }, + system_prompt = { type = "string", description = "System prompt for this profile." }, + }, + required = { "name", "system_prompt" }, + }, + }, }, required = { "prompt", "source" }, }, @@ -204,7 +282,8 @@ local function activate() end, } - toml_workflows.discover_and_register(profiles) + local workflows = toml_workflows.discover_and_register(profiles) + install_header(ext, profiles, workflows) end return { diff --git a/spec/fake_ext.lua b/spec/fake_ext.lua index dfdfdb3..5207e03 100644 --- a/spec/fake_ext.lua +++ b/spec/fake_ext.lua @@ -17,7 +17,8 @@ -- panto.agent{ config =, store =, session_id =, conversation = } -> agent -- panto.file_system_jsonl_store{ dir = } / panto.null_store() -- agent:conversation() / :session_id() / :tools() / :set_tools(decls) / :run_async(opts) --- store:resolve(id) / :load(id) +-- store:list() / :list_bounded(opts) / :resolve(id) / :load(id) / +-- :load_messages(id, opts) -- conv:messages() / :message_metadata(i) / :add_system_message(text, { metadata = }) -- job:next_event() / :result() / :request_cancel() / :close() -- @@ -189,6 +190,10 @@ function conv_mt:messages() return out end +function conv_mt:len() + return #self._messages +end + function conv_mt:message_metadata(index) local message = self._messages[tonumber(index) or 0] if message == nil then @@ -239,7 +244,75 @@ function store_mt:resolve(id) return { id = id, message_count = #session, api_style = "messages" } end +function store_mt:list() + if self._harness.reject_unbounded_replay then error("unbounded list was used", 2) end + local ids = {} + for id in pairs(self._harness.sessions) do + ids[#ids + 1] = id + end + table.sort(ids) + local out = {} + for _, id in ipairs(ids) do + out[#out + 1] = { + id = id, + created = "", + modified = "", + message_count = #self._harness.sessions[id], + model = "", + } + end + return out +end + +function store_mt:list_bounded(opts) + local limit = type(opts) == "table" and math.max(0, math.floor(tonumber(opts.limit) or 0)) or 0 + self._harness.bounded_list_calls = self._harness.bounded_list_calls + 1 + local ids = {} + for id in pairs(self._harness.sessions) do ids[#ids + 1] = id end + table.sort(ids) + local out = {} + for index = 1, math.min(limit, #ids) do + local id = ids[index] + out[#out + 1] = { + id = id, + created = "", + modified = "", + message_count = #self._harness.sessions[id], + model = "", + } + end + return out +end + +local function metadata_has_key(metadata, key) + return type(metadata) == "table" and metadata[key] ~= nil +end + +function store_mt:load_messages(id, opts) + self._harness.bounded_load_calls = self._harness.bounded_load_calls + 1 + local source = self._harness.sessions[id] + if type(source) ~= "table" then return nil end + opts = type(opts) == "table" and opts or {} + local selected = {} + for index, message in ipairs(source) do + local role_ok = opts.role == nil or message.role == opts.role + local metadata_ok = opts.metadata_key == nil or metadata_has_key(message.metadata, opts.metadata_key) + if role_ok and metadata_ok then selected[#selected + 1] = index end + end + local indices = {} + if opts.from_end then + local first = math.max(1, #selected - (tonumber(opts.limit) or 0) + 1) + for index = first, #selected do indices[#indices + 1] = selected[index] end + else + for index = 1, math.min(#selected, tonumber(opts.limit) or 0) do indices[#indices + 1] = selected[index] end + end + local messages = {} + for _, index in ipairs(indices) do messages[#messages + 1] = source[index] end + return new_conversation(messages) +end + function store_mt:load(id) + if self._harness.reject_unbounded_replay then error("unbounded load was used", 2) end local session = self._harness.sessions[id] if session == nil then return nil @@ -415,6 +488,11 @@ function agent_mt:conversation() return self._conv end +function agent_mt:set_message_metadata(index, metadata) + self._record.final_metadata = metadata + return true +end + function agent_mt:session_id() return child_id(self._harness, self._record) end @@ -538,6 +616,8 @@ function M.install(opts) polls = 0, live = 0, max_live = 0, + bounded_list_calls = 0, + bounded_load_calls = 0, } function handle.queue(outcome) diff --git a/spec/test_init.lua b/spec/test_init.lua index 338b7a7..2d5e7d1 100644 --- a/spec/test_init.lua +++ b/spec/test_init.lua @@ -58,10 +58,16 @@ return { end local tmp = assert(uv.fs_mkdtemp("/tmp/panto-subagents-init-XXXXXX")) - assert(os.execute("mkdir -p " .. tmp .. "/agents")) + assert(os.execute("mkdir -p " .. tmp .. "/agents " .. tmp .. "/workflows")) local file = assert(io.open(tmp .. "/agents/reviewer.md", "w")) file:write("---\ndescription: Reviews changes\n---\nYou are a reviewer.\n") file:close() + file = assert(io.open(tmp .. "/workflows/review-chain.toml", "w")) + file:write('name = "review-chain"\n[[steps]]\nid = "review"\nagent = "reviewer"\nprompt = "Review it."\n') + file:close() + file = assert(io.open(tmp .. "/workflows/unusable.toml", "w")) + file:write('name = "unusable"\nsteps = []\n') + file:close() local original_roots = paths.config_roots paths.config_roots = function(kind) @@ -70,6 +76,17 @@ return { local handle = fake.install() local ok, err = pcall(entry.activate) + local header + if ok then + handle.emit("session_start", { + get_component = function() + return { render = function() return { "Panto", "" } end } + end, + set_component = function(_, component) + header = component + end, + }) + end paths.config_roots = original_roots handle.restore() @@ -91,8 +108,28 @@ return { assert(type(run_tool.handler) == "function") assert(handle.tools_by_name["subagents.lua"].schema.properties.source, "the lua tool takes source") + local inline_agents = handle.tools_by_name["subagents.lua"].schema.properties.agents + assert(inline_agents and inline_agents.items.required, + "the lua tool describes workflow-local agent profiles") + assert(inline_agents.items.properties.system_prompt, + "an inline profile carries its system prompt") assert(handle.tools_by_name["subagents.workflow"].schema.properties.steps.items.required, "the workflow tool describes its step shape") + + assert(type(header) == "table" and type(header.render) == "function", + "activation wraps the session header") + local rendered = header:render(18) + local plain = table.concat(rendered, "\n"):gsub("\27%[[%d;]*m", "") + has(plain, "Panto") + has(plain, "subagents:") + has(plain, "reviewer") + has(plain, "workflows:") + has(plain, "review-chain") + assert(not plain:find("unusable", 1, true), "invalid workflows stay out of the usable inventory") + assert(handle.commands_by_name["workflow:unusable"] == nil, + "the header inventory matches command registration") + assert(rendered[#rendered] == "", "annotations stay before the trailing blank") + assert(#rendered > 4, "inventories wrap at the component width") end }, { "an interrupted turn cancels every live child, and its end closes them", function() @@ -100,6 +137,8 @@ return { assert(ok, tostring(err)) assert(type(handle.on_by_name["turn_interrupt"]) == "function", "an interrupted turn must be able to cancel its children") + assert(type(handle.on_by_name["turn_start"]) == "function", + "startup replay must end before the first live turn") assert(type(handle.on_by_name["turn_end"]) == "function", "a finished turn must be able to close its children") @@ -132,19 +171,27 @@ return { assert(type(handle.on_by_name["tool_call_complete"]) == "function", "children report progress through the tool call that started them") - local claimed + assert(type(handle.on_by_name["tool_result"]) == "function", + "the result restores the component's transcript position") + + local claimed, pins = nil, {} handle.emit("tool_call_complete", { id = "call-1", tool_name = "subagents.run", set_component = function(_, component) claimed = component - return { invalidate = function() end, alive = function() - return true - end } + return { + invalidate = function() end, + alive = function() return true end, + set_pinned = function(_, value) pins[#pins + 1] = value end, + } end, }) assert(type(claimed) == "table" and type(claimed.render) == "function", "the entry is given a component that renders the cards") + assert(pins[1] == true, "the progress component pins after claim") + handle.emit("tool_result", { id = "call-1", tool_name = "subagents.run" }) + assert(pins[2] == false, "the matching result unpins it") local foreign handle.emit("tool_call_complete", { diff --git a/spec/test_luatool.lua b/spec/test_luatool.lua index 83b4984..65723db 100644 --- a/spec/test_luatool.lua +++ b/spec/test_luatool.lua @@ -11,6 +11,7 @@ local fake = require("spec.fake_ext") local luatool = require("subagents.luatool") +local progress = require("subagents.progress") local function has(text, needle) assert(type(text) == "string", "expected a string, got " .. type(text)) @@ -127,6 +128,74 @@ return { end) end }, + { "inline agent profiles are scoped to one workflow invocation", function() + with_host(function(handle, profiles) + handle.queue_for("local-reviewer", { id = "0198-local", output = "reviewed" }) + local source = [[ + return subagents.workflow(function(ctx, input) + return ctx:agent({ agent = "local-reviewer", prompt = input }):await() + end) + ]] + progress.reset() + local component + progress.claim({ + id = "lua-call", + tool_name = "subagents.lua", + collapsed = true, + set_component = function(_, value) + component = value + return { + invalidate = function() end, + alive = function() return true end, + set_pinned = function() end, + } + end, + }) + progress.bind({ tool_call_id = "lua-call" }) + local text = luatool.handle({ + prompt = "inspect this", + source = source, + agents = { + { + name = "local-reviewer", + description = "One-off reviewer", + system_prompt = "Review only the requested change.", + }, + }, + }, profiles) + + has(text, "reviewed") + local compact = table.concat(component:render(100), "\n") + assert(not compact:find("inspect this", 1, true), compact) + assert(not compact:find("Review only the requested change.", 1, true), compact) + progress.collapse({ collapsed = false }) + local expanded = table.concat(component:render(100), "\n") + has(expanded, "system prompt: Review only the requested change.") + has(expanded, "prompt: inspect this") + assert(#handle.spawns == 1, "the inline profile started one child") + assert(handle.spawns[1].label == "local-reviewer") + local seeded = handle.spawns[1].system_messages + assert(seeded[#seeded].text == "Review only the requested change.") + + local missing = luatool.handle({ prompt = "again", source = source }, profiles) + has(missing, "unknown agent 'local-reviewer'") + assert(#handle.spawns == 1, "the inline profile did not leak into the next workflow") + progress.reset() + end) + end }, + + { "invalid inline profiles fail before running guest source", function() + with_host(function(handle, profiles) + local text = luatool.handle({ + prompt = "x", + source = "error('guest source should not run')", + agents = { { name = "local", system_prompt = "" } }, + }, profiles) + has(text, "agents[1].system_prompt must be a non-empty string") + assert(#handle.spawns == 0) + end) + end }, + { "a fan-out runs end to end and renders one block per child", function() with_host(function(handle, profiles) handle.queue_for("alpha", { id = "0198-a", output = "alpha says hi" }) diff --git a/spec/test_progress.lua b/spec/test_progress.lua new file mode 100644 index 0000000..06509bf --- /dev/null +++ b/spec/test_progress.lua @@ -0,0 +1,322 @@ +local progress = require("subagents.progress") + +local function board(id, collapsed) + local component + progress.claim({ + id = id, + tool_name = "subagents.run", + collapsed = collapsed, + set_component = function(_, value) + component = value + return { invalidate = function() end, alive = function() return true end } + end, + }) + progress.bind({ tool_call_id = id }) + return component +end + +local function plain(lines) + return table.concat(lines, "\n"):gsub("\27%[[%d;]*m", "") +end + +return { + { "concurrent boards pin independently until their matching result", function() + progress.reset() + local pins = { a = {}, b = {} } + local function claim(id) + progress.claim({ + id = id, + tool_name = "subagents.workflow", + set_component = function() + return { + alive = function() return true end, + invalidate = function() end, + set_pinned = function(_, value) + pins[id][#pins[id] + 1] = value + end, + } + end, + }) + end + claim("a") + claim("b") + assert(pins.a[1] == true and pins.b[1] == true, "each invocation pins its own handle") + progress.settle({ id = "a" }) + assert(pins.a[2] == false, "the matching result restores transcript order") + assert(#pins.b == 1, "another in-flight workflow remains pinned") + progress.settle({ id = "missing" }) + assert(#pins.b == 1, "an unrelated or stale result is inert") + progress.reset() + assert(pins.b[2] == false, "reset safely releases unresolved boards") + end }, + + { "a pinned board leaves one blank line before the waiting indicator", function() + progress.reset() + local component, pins + progress.claim({ + id = "spacing-call", + tool_name = "subagents.workflow", + collapsed = true, + set_component = function(_, value) + component = value + pins = {} + return { + alive = function() return true end, + invalidate = function() end, + set_pinned = function(_, value) pins[#pins + 1] = value end, + } + end, + }) + progress.bind({ tool_call_id = "spacing-call" }) + local card = progress.card("worker", "child") + card:event({ type = "block_start", block_type = "text", index = 1 }) + card:event({ type = "content_delta", index = 1, delta = "final line" }) + + local compact = component:render(80) + assert(pins[1] == true, "the board pins before it renders") + assert(compact[#compact] == "", "pinned compact boards end with a blank line") + assert(compact[#compact - 1]:find("final line", 1, true), "the gap follows the final compact line") + + progress.collapse({ collapsed = false }) + local expanded = component:render(80) + assert(expanded[#expanded] == "", "pinned expanded boards end with a blank line") + assert(expanded[#expanded - 1]:find("final line", 1, true), "the gap follows the final expanded line") + + progress.settle({ id = "spacing-call" }) + local historical = component:render(80) + assert(historical[#historical] ~= "", "settled boards do not keep the spacing line") + assert(pins[2] == false, "settling unpins the board") + progress.reset() + end }, + + { "failed pin transitions do not change rendered spacing state", function() + progress.reset() + local component, pins, fail_unpin = nil, {}, true + progress.claim({ + id = "spacing-lifecycle", + tool_name = "subagents.run", + set_component = function(_, value) + component = value + return { + alive = function() return true end, + invalidate = function() end, + set_pinned = function(_, value) + pins[#pins + 1] = value + if value == false and fail_unpin then error("cannot unpin") end + end, + } + end, + }) + progress.bind({ tool_call_id = "spacing-lifecycle" }) + local card = progress.card("worker", "child") + card:event({ type = "block_start", block_type = "text", index = 1 }) + card:event({ type = "content_delta", index = 1, delta = "final line" }) + local active = component:render(80) + assert(active[#active] == "", "a successful pin enables the gap") + + progress.settle({ id = "spacing-lifecycle" }) + local still_pinned = component:render(80) + assert(still_pinned[#still_pinned] == "", "a failed unpin keeps the board active") + fail_unpin = false + progress.settle({ id = "spacing-lifecycle" }) + local unpinned = component:render(80) + assert(unpinned[#unpinned] ~= "", "a successful unpin removes the gap") + assert(#pins == 3 and pins[1] == true and pins[2] == false and pins[3] == false, + "unpin retries only after the failed lifecycle transition") + progress.reset() + end }, + + { "concurrent UUIDv7 cards across boards show distinguishable id prefixes", function() + progress.reset() + local first_board = board("call-1", true) + local first = "0198aaaa-aaaa-7aaa-8aaa-aaaaaaaaaaaa" + progress.card("worker", first) + local initial_line = first_board:render(120)[2] + assert(initial_line:find("0198aaaa…", 1, true), initial_line) + + local second_board = board("call-2", true) + local second = "0198aaaa-aaaa-7aab-8aaa-aaaaaaaaaaaa" + progress.card("worker", second) + + local first_line = first_board:render(120)[2] + local second_line = second_board:render(120)[2] + assert(first_line ~= second_line, "distinct session ids must not render identical card headers") + assert(first_line:find("0198aaaa%-aaaa%-7aaa"), first_line) + assert(second_line:find("0198aaaa%-aaaa%-7aab"), second_line) + progress.reset() + end }, + + { "Ctrl+O state expands and recollapses every existing board", function() + progress.reset() + local first = board("call-a", true) + local a = progress.card("alpha", "a") + for i = 1, 8 do + a:event({ type = "block_start", block_type = "text", index = i }) + a:event({ type = "content_delta", index = i, delta = "line-" .. i .. "\n" }) + a:event({ type = "block_complete", block_type = "text", index = i, text = "line-" .. i }) + end + local second = board("call-b", true) + local b = progress.card("beta", "b") + for i = 1, 8 do b:event({ type = "content_delta", index = 99, delta = "ignored" }) end + for i = 1, 8 do + b:event({ type = "block_start", block_type = "text", index = i }) + b:event({ type = "content_delta", index = i, delta = "beta-" .. i .. "\n" }) + end + + assert(#first:render(80) == 6, "blank + header + at most four recent lines") + assert(#second:render(80) == 6, "each board is independently collapsed") + progress.collapse({ collapsed = false }) + assert(#first:render(80) == 10, "expanded board should retain all eight lines") + assert(#second:render(80) == 10, "global state should expand concurrent boards") + progress.collapse({ collapsed = true }) + assert(#first:render(80) == 6 and #second:render(80) == 6) + progress.reset() + end }, + + { "settled historical boards keep responding to Ctrl+O until their handles die", function() + progress.reset() + local alive = { a = true, b = true } + local invalidations = { a = 0, b = 0 } + local components = {} + for _, id in ipairs({ "a", "b" }) do + progress.claim({ + id = id, + tool_name = "subagents.run", + collapsed = true, + set_component = function(_, component) + components[id] = component + return { + alive = function() return alive[id] end, + invalidate = function() invalidations[id] = invalidations[id] + 1 end, + set_pinned = function() end, + } + end, + }) + progress.bind({ tool_call_id = id }) + local card = progress.card(id, id) + for i = 1, 6 do card:event({ type = "block_start", block_type = "text", index = i }) + card:event({ type = "content_delta", index = i, delta = id .. i .. "\n" }) end + progress.settle({ id = id }) + end + + progress.reset() -- turn_end clears coroutine bindings, not historical components + progress.collapse({ collapsed = false }) + assert(#components.a:render(80) == 8 and #components.b:render(80) == 8, + "settled concurrent boards should both expand after turn_end") + alive.a = false + local a_invalidations = invalidations.a + progress.collapse({ collapsed = true }) + assert(invalidations.a == a_invalidations, "dead historical handles should be pruned") + assert(#components.b:render(80) == 6, "a live concurrent board should remain isolated") + alive.b = false + progress.reset() + end }, + + { "prompts are presentation-only in expanded mode and bounded", function() + progress.reset() + local component = board("call-prompts", true) + local card = progress.card("inline", "child", nil, { + prompt = "inspect this\nthen summarize", + system_prompt = "INLINE-SYSTEM " .. string.rep("x", 64 * 1024), + }) + card:event({ type = "block_start", block_type = "text", index = 1 }) + card:event({ type = "content_delta", index = 1, delta = "assistant line\n" }) + + local compact = plain(component:render(100)) + assert(compact:find("assistant line", 1, true), compact) + assert(not compact:find("inspect this", 1, true), compact) + assert(not compact:find("INLINE-SYSTEM", 1, true), compact) + + progress.collapse({ collapsed = false }) + local expanded = plain(component:render(100)) + assert(expanded:find("system prompt: INLINE-SYSTEM", 1, true), expanded) + assert(expanded:find("prompt: inspect this", 1, true), expanded) + assert(expanded:find("then summarize", 1, true), expanded) + assert(#card.system_prompt == 32 * 1024, "presentation metadata must be bounded") + assert(#card.history == 1, "presentation metadata must not enter accumulated output history") + progress.reset() + end }, + + { "expanded history includes streamed assistant text and available tool call fields", function() + progress.reset() + local component = board("call-tools", false) + local card = progress.card("worker", "child") + card:event({ type = "block_start", block_type = "text", index = 0 }) + card:event({ type = "content_delta", index = 0, delta = "assistant output" }) + card:event({ type = "block_complete", block_type = "text", index = 0, text = "assistant output" }) + card:event({ type = "block_start", block_type = "tool_use", index = 1 }) + card:event({ type = "tool_details", index = 1, id = "tool-7", name = "std.echo" }) + card:event({ type = "block_complete", block_type = "tool_use", index = 1, + id = "tool-7", name = "std.echo", text = '{"text":"hello"}' }) + card:event({ type = "tool_dispatch_result", tool_results = { + { tool_use_id = "tool-7", output = "hello\nworld", is_error = false }, + { tool_use_id = "tool-8", output = "boom", is_error = true }, + } }) + + local text = plain(component:render(80)) + assert(text:find("assistant output", 1, true), text) + assert(text:find("std.echo [tool-7]", 1, true), text) + assert(text:find('input: {"text":"hello"}', 1, true), text) + assert(text:find("result [tool-7]: hello", 1, true), text) + assert(text:find(" world", 1, true), text) + assert(text:find("error [tool-8]: boom", 1, true), text) + progress.reset() + end }, + + { "tool names use internal dotted spelling in progress labels", function() + progress.reset() + local component = board("call-tool-names", false) + local card = progress.card("worker", "child") + card:event({ type = "tool_details", index = 1, id = "wire", name = "std__shell" }) + card:event({ type = "tool_details", index = 2, id = "internal", name = "std.read" }) + card:event({ type = "block_complete", block_type = "tool_use", index = 3, + id = "complete", name = "web__fetch" }) + + local text = plain(component:render(80)) + assert(text:find("std.shell [wire]", 1, true), text) + assert(text:find("std.read [internal]", 1, true), text) + assert(text:find("web.fetch [complete]", 1, true), text) + assert(not text:find("std__shell", 1, true), text) + progress.reset() + end }, + + { "pathological single-line payloads are bounded before history ingestion", function() + progress.reset() + local component = board("call-huge", false) + local card = progress.card("worker", "huge") + local huge = "head\0" .. string.rep("x", 2 * 1024 * 1024) + card:event({ type = "block_start", block_type = "text", index = 1 }) + card:event({ type = "content_delta", index = 1, delta = huge }) + card:event({ type = "tool_dispatch_result", tool_results = { + { tool_use_id = "huge-tool", output = huge }, + } }) + + assert(#card.history == 2) + assert(#card.history[1] <= 4096 and #card.history[2] <= 4096, + "assistant and tool lines must be bounded at ingestion") + local text = plain(component:render(120)) + assert(text:find("head x", 1, true), "ordinary control sanitization should be preserved") + progress.reset() + end }, + + { "rendering sanitizes controls, stays width-bound, and caps retained history", function() + progress.reset() + local component = board("call-bounds", false) + local card = progress.card("bad\27label", "child") + for i = 1, 600 do + card:event({ type = "block_start", block_type = "text", index = i }) + card:event({ type = "content_delta", index = i, + delta = string.format("history-%03d-abcdefghijklmnopqrstuvwxyz\n", i) }) + end + local lines = component:render(16) + local text = plain(lines) + assert(not text:find("\27", 1, true), "control bytes must not reach the renderer") + assert(not text:find("history%-001"), "old history should be evicted") + assert(text:find("history%-600"), "recent history should remain") + assert(#lines <= 4098, "rendering itself must remain bounded") + for _, line in ipairs(lines) do + assert(utf8.len(line) <= 16, "line exceeded component width: " .. line) + end + progress.reset() + end }, +} diff --git a/spec/test_progress_replay.lua b/spec/test_progress_replay.lua new file mode 100644 index 0000000..aba8e33 --- /dev/null +++ b/spec/test_progress_replay.lua @@ -0,0 +1,316 @@ +local fake = require("spec.fake_ext") +local progress = require("subagents.progress") +local run = require("subagents.run") +local luatool = require("subagents.luatool") +local toml_workflows = require("subagents.toml_workflows") + +local function plain(lines) + return table.concat(lines, "\n"):gsub("\27%[[%d;]*m", "") +end + +local function has(text, needle) + assert(text:find(needle, 1, true), "expected to find " .. needle .. " in:\n" .. text) +end + +local function profile_set() + local alpha = { name = "alpha", description = "alpha", body = "ALPHA\n" } + local beta = { name = "beta", description = "beta", body = "BETA\n" } + return { + list = { alpha, beta }, + by_name = { alpha = alpha, beta = beta }, + warnings = {}, + } +end + +local function message(role, text, metadata) + return { role = role, text = text, metadata = metadata } +end + +local function tool_use(id, name, input) + return { + type = "tool_use", id = id, name = name, input = input, + } +end + +local function tool_result(id, output, is_error) + return { + type = "tool_result", tool_use_id = id, is_error = is_error == true, + parts = { { text = output } }, + } +end + +local function claim(handle, id, tool_name) + local component + local alive = true + progress.claim({ + id = id, + tool_name = tool_name or "subagents.run", + collapsed = true, + set_component = function(_, value) + component = value + return { + render = value.render, + invalidate = function() end, + alive = function() return alive end, + set_pinned = function() end, + } + end, + }) + assert(component ~= nil, "the production claim path installed a component") + return component, function() alive = false end +end + +local function with_host(fn, opts) + local handle = fake.install(opts) + local ok, err = pcall(fn, handle) + progress.reset() + handle.restore() + if not ok then error(err, 0) end +end + +return { + { "replay reconstructs a direct child turn, tools, result, model, and status", function() + with_host(function(handle) + handle.add_session("child-direct", { + message("system", "You are a subagent."), + message("system", "Discovered profile.", { + subagents = { owner = "0198-primary", agent = "reviewer" }, + }), + message("user", "inspect the change", { + subagents = { tool_call_id = "outer-direct", model = "openai:test", reasoning = "high" }, + }), + { role = "assistant", blocks = { + { type = "text", text = "checking" }, + tool_use("tool-1", "std__read", '{"path":"auth.lua"}'), + } }, + { role = "user", blocks = { tool_result("tool-1", "file contents") } }, + { role = "assistant", blocks = { { type = "text", text = "found the issue" } } }, + }) + local component, kill = claim(handle, "outer-direct") + progress.collapse({ collapsed = false }) + local text = plain(component:render(120)) + has(text, "✔ reviewer child-direct") + has(text, "completed") + has(text, "↳ openai:test") + has(text, "prompt: inspect the change") + has(text, "checking") + has(text, "std.read [tool-1]") + has(text, 'input: {"path":"auth.lua"}') + has(text, "result [tool-1]: file contents") + has(text, "found the issue") + kill() + end) + end }, + + { "replay isolates dynamic and fixed workflow cards and only inline manifests show system prompts", function() + with_host(function(handle) + handle.add_session("child-dynamic", { + message("system", "You are a subagent."), + message("system", "INLINE-WORKFLOW-SYSTEM", { + subagents = { owner = "0198-primary", agent = "inline", inline = true }, + }), + message("user", "dynamic step", { + subagents = { tool_call_id = "outer-dynamic", model = "fixture:one" }, + }), + message("assistant", "dynamic answer"), + }) + handle.add_session("child-fixed", { + message("system", "You are a subagent."), + message("system", "DISCOVERED-SYSTEM", { + subagents = { owner = "0198-primary", agent = "alpha" }, + }), + message("user", "fixed step", { + subagents = { tool_call_id = "outer-fixed", model = "fixture:two" }, + }), + message("assistant", "fixed answer"), + }) + + local dynamic, kill_dynamic = claim(handle, "outer-dynamic", "subagents.lua") + local fixed, kill_fixed = claim(handle, "outer-fixed", "subagents.workflow") + progress.collapse({ collapsed = false }) + local dynamic_text = plain(dynamic:render(120)) + local fixed_text = plain(fixed:render(120)) + has(dynamic_text, "✔ inline child-dynamic") + has(dynamic_text, "system prompt: INLINE-WORKFLOW-SYSTEM") + has(dynamic_text, "prompt: dynamic step") + has(dynamic_text, "dynamic answer") + assert(not dynamic_text:find("fixed answer", 1, true), dynamic_text) + has(fixed_text, "✔ alpha child-fixed") + has(fixed_text, "prompt: fixed step") + has(fixed_text, "fixed answer") + assert(not fixed_text:find("system prompt:", 1, true), fixed_text) + kill_dynamic() + kill_fixed() + end) + end }, + + { "one resumed child can contribute separate turns to separate outer calls", function() + with_host(function(handle) + handle.add_session("child-shared", { + message("system", "role"), + message("system", "Alpha", { subagents = { owner = "0198-primary", agent = "alpha" } }), + message("user", "first turn", { subagents = { tool_call_id = "outer-a", model = "model-a" } }), + message("assistant", "first answer"), + message("user", "second turn", { subagents = { tool_call_id = "outer-b", model = "model-b" } }), + message("assistant", "second answer"), + message("user", "third turn", { subagents = { tool_call_id = "outer-a", model = "model-c" } }), + message("assistant", "third answer"), + }) + + local a, kill_a = claim(handle, "outer-a") + local b, kill_b = claim(handle, "outer-b") + progress.collapse({ collapsed = false }) + local a_text = plain(a:render(120)) + local b_text = plain(b:render(120)) + has(a_text, "first answer") + has(a_text, "third answer") + assert(not a_text:find("second answer", 1, true), a_text) + has(b_text, "second answer") + assert(not b_text:find("first answer", 1, true), b_text) + assert(not b_text:find("third answer", 1, true), b_text) + local _, shared_cards = a_text:gsub("child%-shared", "") + assert(shared_cards == 2, "both turns owned by outer-a are separate cards:\n" .. a_text) + kill_a() + kill_b() + end) + end }, + + { "replay preserves interleaved spawn order across child sessions", function() + with_host(function(handle) + handle.add_session("child-a", { + message("system", "role"), + message("system", "Alpha", { subagents = { owner = "0198-primary", agent = "alpha" } }), + message("user", "A1", { subagents = { tool_call_id = "outer-order", sequence = 1, model = "m" } }), + message("assistant", "answer A1"), + message("user", "A2", { subagents = { tool_call_id = "outer-order", sequence = 3, model = "m" } }), + message("assistant", "answer A2"), + }) + handle.add_session("child-b", { + message("system", "role"), + message("system", "Beta", { subagents = { owner = "0198-primary", agent = "beta" } }), + message("user", "B1", { subagents = { tool_call_id = "outer-order", sequence = 2, model = "m" } }), + message("assistant", "answer B1"), + }) + local component, kill = claim(handle, "outer-order") + progress.collapse({ collapsed = false }) + local text = plain(component:render(120)) + local a1 = assert(text:find("answer A1", 1, true)) + local b1 = assert(text:find("answer B1", 1, true)) + local a2 = assert(text:find("answer A2", 1, true)) + assert(a1 < b1 and b1 < a2, text) + kill() + end) + end }, + + { "replay keeps cancelled and non-text completed turns terminal", function() + with_host(function(handle) + handle.add_session("child-cancel", { + message("system", "role"), + message("system", "Alpha", { subagents = { owner = "0198-primary", agent = "alpha" } }), + message("user", "cancelled prompt", { subagents = { + tool_call_id = "outer-status", sequence = 1, status = "cancelled", + } }), + }) + handle.add_session("child-thinking", { + message("system", "role"), + message("system", "Beta", { subagents = { owner = "0198-primary", agent = "beta" } }), + message("user", "thinking prompt", { subagents = { + tool_call_id = "outer-status", sequence = 2, status = "completed", + } }), + { role = "assistant", blocks = { { type = "thinking", text = "private reasoning" } } }, + }) + local component, kill = claim(handle, "outer-status") + progress.collapse({ collapsed = false }) + local text = plain(component:render(120)) + has(text, "⊘ alpha child-cancel") + has(text, "✔ beta child-thinking") + assert(not text:find("✖", 1, true), text) + kill() + end) + end }, + + { "replay finds a manifest after long primary system context through the bounded API", function() + with_host(function(handle) + handle.reject_unbounded_replay = true + local messages = {} + for index = 1, 80 do messages[#messages + 1] = message("system", "context " .. index) end + messages[#messages + 1] = message("system", "INLINE-LONG", { + subagents = { owner = "0198-primary", agent = "long", inline = true }, + }) + messages[#messages + 1] = message("user", "long prompt", { + subagents = { tool_call_id = "outer-long", sequence = 1 }, + }) + messages[#messages + 1] = message("assistant", "long answer") + handle.add_session("child-long", messages) + local component, kill = claim(handle, "outer-long") + progress.collapse({ collapsed = false }) + local text = plain(component:render(120)) + has(text, "✔ long child-long") + has(text, "system prompt: INLINE-LONG") + assert(handle.bounded_list_calls == 1, "replay must use the bounded catalog API") + assert(handle.bounded_load_calls == 2, "replay must use bounded tail + manifest reads") + kill() + end) + end }, + + { "durable turns carry the owning outer call and inline manifests are marked", function() + with_host(function(handle) + local profiles = profile_set() + local component, kill = claim(handle, "outer-run") + progress.bind({ tool_call_id = "outer-run" }) + handle.queue_for("alpha", { output = "done" }) + run.handle({ agent = "alpha", prompt = "direct" }, profiles) + assert(handle.runs[1].metadata.subagents.tool_call_id == "outer-run", + "direct child metadata names its outer tool call") + assert(handle.runs[1].metadata.subagents.sequence == 1, + "direct child metadata records its card sequence") + assert(handle.runs[1].metadata.subagents.status == "completed", + "settled status is retained for durable presentation") + + handle.queue_for("alpha", { output = "fixed done" }) + toml_workflows.handle({ + prompt = "fixed", + steps = { { id = "step", agent = "alpha", prompt = "run fixed" } }, + }, profiles) + assert(handle.runs[2].metadata.subagents.tool_call_id == "outer-run", + "fixed workflow child keeps the same outer owner") + assert(handle.runs[2].metadata.subagents.sequence == 2, + "fixed workflow child follows spawn order") + + handle.queue_for("inline", { output = "inline done" }) + local source = [[return subagents.workflow(function(ctx, input) + return ctx:agent({ agent = "inline", prompt = input }):await() + end)]] + luatool.handle({ + prompt = "dynamic", + source = source, + agents = { { name = "inline", system_prompt = "INLINE" } }, + }, profiles) + local child = handle.spawns[3] + local manifest = child.system_messages[2].metadata.subagents + assert(manifest.inline == true, "inline workflow manifest is explicitly marked") + assert(handle.runs[3].metadata.subagents.tool_call_id == "outer-run", + "inline workflow child keeps the same outer owner") + assert(handle.runs[3].metadata.subagents.sequence == 3, + "inline workflow child follows spawn order") + assert(handle.runs[3].metadata.subagents.status == "completed", + "inline completion is durably annotated") + kill() + progress.reset() + _ = component + end) + end }, + + { "missing and malformed child logs degrade to an empty or partial board", function() + with_host(function(handle) + handle.add_session("malformed", "not a message array") + handle.add_session("valid", { + message("system", "role"), + message("user", "no ownership", { subagents = { model = "model" } }), + }) + local component, kill = claim(handle, "outer-missing") + assert(#component:render(100) == 0, "unowned or malformed logs do not invent cards") + kill() + end, { sessions = {} }) + end }, +} diff --git a/spec/test_run.lua b/spec/test_run.lua index e4bb038..57c6e47 100644 --- a/spec/test_run.lua +++ b/spec/test_run.lua @@ -10,6 +10,7 @@ -- machine's real ~/.config out of the run. local fake = require("spec.fake_ext") +local progress = require("subagents.progress") local run = require("subagents.run") local spawn = require("subagents.spawn") @@ -123,6 +124,53 @@ return { end) end }, + { "subagents.run presents its child prompt only when expanded", function() + with_host(function(handle, profiles) + handle.queue_for("reviewer", { events = { + { type = "block_start", block_type = "text", index = 0 }, + { type = "content_delta", index = 0, delta = "checking auth\n" }, + { type = "block_start", block_type = "tool_use", index = 1 }, + { type = "tool_details", index = 1, id = "tool-1", name = "std__read" }, + { type = "block_complete", block_type = "tool_use", index = 1, + id = "tool-1", name = "std__read", text = '{"path":"auth.lua"}' }, + { type = "tool_dispatch_result", tool_results = { + { tool_use_id = "tool-1", output = "file contents", is_error = false }, + } }, + } }) + progress.reset() + local component + progress.claim({ + id = "run-call", + tool_name = "subagents.run", + collapsed = true, + set_component = function(_, value) + component = value + return { + invalidate = function() end, + alive = function() return true end, + set_pinned = function() end, + } + end, + }) + progress.bind({ tool_call_id = "run-call" }) + + run.handle({ agent = "reviewer", prompt = "Review the auth change." }, profiles) + local compact = table.concat(component:render(100), "\n") + assert(not compact:find("Review the auth change.", 1, true), compact) + + progress.collapse({ collapsed = false }) + local expanded = table.concat(component:render(100), "\n") + has(expanded, "prompt: Review the auth change.") + has(expanded, "checking auth") + has(expanded, "std.read [tool-1]") + has(expanded, 'input: {"path":"auth.lua"}') + has(expanded, "result [tool-1]: file contents") + assert(not expanded:find("system prompt:", 1, true), + "a discovered profile system prompt is not presentation metadata:\n" .. expanded) + progress.reset() + end) + end }, + { "the primary's system context comes first, then the role, then the profile", function() with_host(function(handle, profiles) run.handle({ agent = "reviewer", prompt = "go" }, profiles) diff --git a/spec/test_toml_workflows.lua b/spec/test_toml_workflows.lua index 8ae2cac..4755b8e 100644 --- a/spec/test_toml_workflows.lua +++ b/spec/test_toml_workflows.lua @@ -11,6 +11,7 @@ local fake = require("spec.fake_ext") local paths = require("subagents.paths") +local progress = require("subagents.progress") local toml_workflows = require("subagents.toml_workflows") local function has(text, needle) @@ -384,9 +385,25 @@ return { end) end }, - { "the tool runs a transient definition", function() + { "the tool runs a transient definition and presents each child prompt only when expanded", function() with_host(function(handle, profiles) handle.queue_for("alpha", { output = "transient output" }) + progress.reset() + local component + progress.claim({ + id = "workflow-call", + tool_name = "subagents.workflow", + collapsed = true, + set_component = function(_, value) + component = value + return { + invalidate = function() end, + alive = function() return true end, + set_pinned = function() end, + } + end, + }) + progress.bind({ tool_call_id = "workflow-call" }) local text = toml_workflows.handle({ prompt = "the input", steps = { { id = "only", agent = "alpha", prompt = "Do it." } }, @@ -394,11 +411,21 @@ return { has(text, "step: only") has(text, "transient output") has(handle.spawns[1].prompt, "Do it.\n\n## Workflow input\n\nthe input") + local compact = table.concat(component:render(100), "\n") + assert(not compact:find("Do it.", 1, true), compact) + assert(not compact:find("the input", 1, true), compact) + progress.collapse({ collapsed = false }) + local expanded = table.concat(component:render(100), "\n") + has(expanded, "prompt: Do it.") + has(expanded, "## Workflow input") + has(expanded, "the input") + assert(not expanded:find("system prompt:", 1, true), expanded) has(toml_workflows.handle({ prompt = "x", steps = { { id = "only", agent = "alpha", prompt = "p", needs = { "ghost" } } }, }, profiles), "unknown dependency 'ghost'") + progress.reset() end) end }, } diff --git a/subagents/jobs.lua b/subagents/jobs.lua index 3f9a8af..a774b27 100644 --- a/subagents/jobs.lua +++ b/subagents/jobs.lua @@ -28,7 +28,8 @@ -- `job:close()` frees it. Jobs are otherwise left open until close_all() ends -- the turn, so a result can still be read after its coroutine resumed. -- --- `job:close()` also JOINS the pump, and a pump parked in a tool batch is +-- A settled consumer may close a job immediately after caching its result; +-- otherwise close_all() handles it. `job:close()` also JOINS the pump, and a pump parked in a tool batch is -- waiting on the owner thread to come back to the uv loop — the very thread -- every entry point here runs on. Closing an unsettled job would therefore -- deadlock, so nothing does: only settle() closes a job, once its pump has diff --git a/subagents/luatool.lua b/subagents/luatool.lua index 8aaa5c9..53966aa 100644 --- a/subagents/luatool.lua +++ b/subagents/luatool.lua @@ -4,7 +4,8 @@ -- workflow without writing a definition to disk. The source must evaluate to -- `subagents.workflow(function(ctx, input) ... end)`; the tool then executes it -- with the tool's `prompt` as the workflow input and formats the terminal --- results for the calling model. +-- results for the calling model. Optional inline agent profiles are overlaid +-- for this execution only; they are never persisted or added to discovery. -- -- The source is loaded in text mode only (`load(source, chunkname, "t", env)`) -- against a restricted `_ENV`. That environment holds a safe slice of the @@ -46,6 +47,7 @@ local workflow = require("subagents.workflow") local run = require("subagents.run") +local spawn = require("subagents.spawn") local MAX_JOBS = 32 local INSTRUCTION_BUDGET = 10000000 @@ -91,6 +93,69 @@ end M.build_env = build_env +-- Overlay workflow-local profiles without mutating the activation-time set. +-- Inline names intentionally win, matching normal layered profile precedence +-- while keeping their lifetime bounded to this one execute() call. +local function workflow_profiles(inline, discovered) + if inline == nil then + return discovered, nil + end + if type(inline) ~= "table" then + return nil, "agents must be an array" + end + + local count = 0 + for key in pairs(inline) do + if type(key) ~= "number" then + return nil, "agents must be an array" + end + count = count + 1 + end + if count ~= #inline then + return nil, "agents must be a dense array" + end + + local base = spawn.profiles(discovered) + local by_name = {} + for name, profile in pairs(base.by_name or {}) do + by_name[name] = profile + end + local seen = {} + for index, raw in ipairs(inline) do + if type(raw) ~= "table" then + return nil, string.format("agents[%d] must be an object", index) + end + local name = raw.name + if type(name) ~= "string" or name:match("^%s*$") then + return nil, string.format("agents[%d].name must be a non-empty string", index) + end + if seen[name] then + return nil, string.format("duplicate inline agent '%s'", name) + end + local system_prompt = raw.system_prompt + if type(system_prompt) ~= "string" or system_prompt:match("^%s*$") then + return nil, string.format("agents[%d].system_prompt must be a non-empty string", index) + end + if raw.description ~= nil and type(raw.description) ~= "string" then + return nil, string.format("agents[%d].description must be a string when given", index) + end + seen[name] = true + by_name[name] = { + name = name, + description = raw.description or "", + body = system_prompt, + layer = "workflow", + } + end + + local list = {} + for _, profile in pairs(by_name) do + list[#list + 1] = profile + end + table.sort(list, function(a, b) return a.name < b.name end) + return { list = list, by_name = by_name, warnings = base.warnings or {} }, nil +end + local function budget_hook() error("instruction budget exceeded", 0) end @@ -151,6 +216,11 @@ function M.handle(input, profiles) return "Error: source is required and must be a non-empty string" end + local profiles_for_run, profiles_err = workflow_profiles(input.agents, profiles) + if not profiles_for_run then + return "Error: " .. profiles_err + end + local chunk, load_err = load(input.source, CHUNK_NAME, "t", build_env()) if not chunk then return "Error: source did not compile: " .. tostring(load_err) @@ -167,7 +237,7 @@ function M.handle(input, profiles) local armed = nil local ran_ok, result = pcall(workflow.execute, built, input.prompt, { max_jobs = MAX_JOBS, - profiles = profiles, + profiles = profiles_for_run, on_resume = function(co) armed = co debug.sethook(co, budget_hook, "", INSTRUCTION_BUDGET) diff --git a/subagents/progress.lua b/subagents/progress.lua index 087a3c3..fb95115 100644 --- a/subagents/progress.lua +++ b/subagents/progress.lua @@ -1,32 +1,27 @@ --- Live progress cards for in-flight children. --- --- While the primary model is blocked on one of the subagents.* tools, its --- children are streaming. This folds each child's event stream into one small --- card and renders the set as the component of the tool-call entry that --- started them, so several concurrent children stay visually distinct and the --- cards disappear with the entry they belong to. The events are presentation --- data only: nothing here ever reaches a conversation. --- --- Linking a card to the right entry. The host fires `tool_call_complete` with --- the tool-use id just before dispatching that call, and hands the same id to --- the handler as `context.tool_call_id`; `claim` records the board under the --- id, `bind` ties the running handler coroutine to it, and `card` finds the --- board of whichever coroutine is asking. A workflow callback runs on its --- handler's own coroutine, so a card raised deep inside one still lands on the --- entry the model can see. --- --- Everything degrades to a no-op card: in print mode, in a test, or under a --- host with no component machinery there is no board to attach to, and a child --- still runs exactly the same. --- --- Rendering is deliberately small — a four-line ring per card, truncated to the --- terminal width — because several children run at once and the panel must not --- push the transcript off the screen. - -local MAX_CARD_LINES = 4 -local MAX_LINE_BYTES = 1024 +-- Expandable progress boards for in-flight children, with a bounded replay +-- path that rebuilds durable child turns when a primary session is reopened. + +local paths = require("subagents.paths") + +local COLLAPSED_LINES = 4 +local MAX_HISTORY_LINES = 512 +local MAX_HISTORY_BYTES = 128 * 1024 +local MAX_LINE_BYTES = 4096 +local MAX_PRESENTATION_BYTES = 32 * 1024 +local MAX_PRESENTATION_LINES = 128 +local MAX_RENDER_LINES = 4096 +local MAX_REPLAY_SESSIONS = 1024 +local MAX_REPLAY_MESSAGES = 8192 +local MAX_REPLAY_CARDS = 512 +local MAX_REPLAY_BLOCKS = 4096 +local MAX_REPLAY_BLOCK_BYTES = 64 * 1024 +local MAX_REPLAY_MESSAGE_BYTES = 256 * 1024 local BODY_INDENT = " " -local TOOL_PREFIX = "subagents." +local PROGRESS_TOOLS = { + ["subagents.run"] = true, + ["subagents.lua"] = true, + ["subagents.workflow"] = true, +} local GLYPHS = { running = "◷", @@ -36,99 +31,162 @@ local GLYPHS = { } local M = {} - --- Boards keyed by tool-call id, and the board bound to each handler coroutine. --- Weak keys on the second so a finished handler's binding disappears with it. local boards = {} local bound = setmetatable({}, { __mode = "k" }) +local tools_collapsed = true +local sid_width = 8 +local replay_board +local replaying = true --- --------------------------------------------------------------------------- --- Text handling --- --------------------------------------------------------------------------- - --- Control bytes would move the cursor or confuse the renderer's width --- accounting, so they become spaces. UTF-8 continuation bytes are >= 0x80 and --- pass through untouched. local function sanitize(text) - return (text:gsub("[%z\1-\31\127]", " ")) + return (tostring(text or ""):gsub("[%z\1-\31\127]", " ")) end -local function truncate(text, width) - if width == nil or width < 1 then - return text - end - local length = utf8.len(text) - if length == nil or length <= width then - return text - end - return text:sub(1, (utf8.offset(text, width + 1) or (#text + 1)) - 1) +local function sanitize_multiline(text) + return (tostring(text or ""):gsub("[%z\1-\9\11-\31\127]", " ")) +end + +local function display_tool_name(name) + return type(name) == "string" and name:gsub("__", ".") or "tool" end --- --------------------------------------------------------------------------- --- Cards --- --------------------------------------------------------------------------- +local function prefix(text, width) + if width < 1 then return "" end + local count, stop = 0, 0 + local ok = pcall(function() + for pos in utf8.codes(text) do + if count == width then break end + stop = pos + count = count + 1 + end + if count > 0 then stop = (utf8.offset(text, count + 1) or (#text + 1)) - 1 end + end) + if not ok then return text:sub(1, width) end + return text:sub(1, stop) +end + +local function wrap(text, width, emit) + width = math.max(1, width) + text = sanitize(text) + if text == "" then emit(""); return end + local rest = text + while rest ~= "" do + local piece = prefix(rest, width) + if piece == "" then piece = rest:sub(1, 1) end + emit(piece) + rest = rest:sub(#piece + 1) + end +end local card_mt = {} card_mt.__index = card_mt local function repaint(card) local board = card.board - if board and board.handle then - pcall(board.handle.invalidate, board.handle) + if board and board.handle then pcall(board.handle.invalidate, board.handle) end +end + +local function trim(card) + while #card.history > MAX_HISTORY_LINES or card.history_bytes > MAX_HISTORY_BYTES do + local removed = table.remove(card.history, 1) + card.history_bytes = card.history_bytes - #removed end end local function adopt(card, line) - card.lines[#card.lines + 1] = line - while #card.lines > MAX_CARD_LINES do - table.remove(card.lines, 1) - end + line = sanitize(line):sub(1, MAX_LINE_BYTES) + card.history[#card.history + 1] = line + card.history_bytes = card.history_bytes + #line + trim(card) end --- A discrete, already-complete line: a tool marker, a model label, an error. -local function marker(card, prefix, text) +local function marker(card, text) card.partial = false - if text == nil or text == "" then - return - end - adopt(card, prefix .. sanitize(text:sub(1, MAX_LINE_BYTES))) + if text ~= nil and text ~= "" then adopt(card, text) end end --- Fold a text delta in, breaking it on newlines. A chunk with no newline leaves --- the tail line open so the next delta continues it. local function append_text(card, text) - local rest = text - while rest ~= "" do - local newline = rest:find("\n", 1, true) - local chunk = newline and rest:sub(1, newline - 1) or rest - if card.partial and #card.lines > 0 then - local index = #card.lines - local grown = card.lines[index] .. sanitize(chunk) - card.lines[index] = grown:sub(1, MAX_LINE_BYTES) - elseif chunk ~= "" then - adopt(card, sanitize(chunk:sub(1, MAX_LINE_BYTES))) + text = tostring(text or "") + local from = 1 + while from <= #text do + local newline = text:find("\n", from, true) + local last = newline and newline - 1 or #text + if card.partial and #card.history > 0 then + local index = #card.history + local old = card.history[index] + local room = MAX_LINE_BYTES - #old + local chunk = room > 0 and text:sub(from, math.min(last, from + room - 1)) or "" + local grown = old .. sanitize(chunk) + card.history[index] = grown + card.history_bytes = card.history_bytes + #grown - #old + trim(card) + else + adopt(card, text:sub(from, math.min(last, from + MAX_LINE_BYTES - 1))) end card.partial = newline == nil - rest = newline and rest:sub(newline + 1) or "" + if not newline then break end + from = newline + 1 end end --- One run_async event. `content_delta` carries only an index, so the block type --- comes from the `block_start` that opened it — the same flag the v0 pump kept. -function card_mt:event(event) - if type(event) ~= "table" then - return +local function append_labeled(card, label, text) + text = tostring(text or "") + local continuation = string.rep(" ", #label) + local from, first = 1, true + while true do + local newline = text:find("\n", from, true) + local last = newline and newline - 1 or #text + local prefix = first and label or continuation + local room = math.max(0, MAX_LINE_BYTES - #prefix) + local line = room > 0 and text:sub(from, math.min(last, from + room - 1)) or "" + adopt(card, prefix .. line) + first = false + if not newline then break end + from = newline + 1 end - local kind = event.type +end + +function card_mt:event(event) + if type(event) ~= "table" then return end + local kind, index = event.type, event.index if kind == "block_start" then - self.text_block = event.block_type == "text" + self.blocks[index] = event.block_type + self.block_had_delta[index] = false elseif kind == "content_delta" then - if self.text_block and type(event.delta) == "string" then + if self.blocks[index] == "text" and type(event.delta) == "string" then append_text(self, event.delta) + self.block_had_delta[index] = true end elseif kind == "tool_details" then - marker(self, "⚒ ", event.name) - elseif kind == "tool_dispatch_complete" or kind == "message_complete" then + self.tools[index] = { id = event.id, name = event.name } + local detail = display_tool_name(event.name) + if event.id and event.id ~= "" then detail = detail .. " [" .. event.id .. "]" end + marker(self, "⚒ " .. detail) + elseif kind == "block_complete" then + if event.block_type == "text" and not self.block_had_delta[index] and type(event.text) == "string" then + append_text(self, event.text) + elseif event.block_type == "tool_use" then + if not self.tools[index] then + local detail = display_tool_name(event.name) + if event.id and event.id ~= "" then detail = detail .. " [" .. event.id .. "]" end + marker(self, "⚒ " .. detail) + end + if type(event.text) == "string" then append_labeled(self, "input: ", event.text) end + end + self.blocks[index], self.block_had_delta[index] = nil, nil + self.partial = false + elseif kind == "tool_dispatch_result" then + for _, result in ipairs(type(event.tool_results) == "table" and event.tool_results or {}) do + if type(result) == "table" then + local label = result.is_error and "error" or "result" + if type(result.tool_use_id) == "string" and result.tool_use_id ~= "" then + label = label .. " [" .. result.tool_use_id .. "]" + end + append_labeled(self, label .. ": ", result.output) + end + end + self.partial = false + elseif kind == "message_complete" or kind == "tool_dispatch_complete" then self.partial = false else return @@ -136,123 +194,533 @@ function card_mt:event(event) repaint(self) end --- The child settled. Only a failure or a cancellation gets a closing line; a --- completed child's report is the tool result the model already sees. function card_mt:done(status, message) self.status = GLYPHS[status] and status or "failed" - if self.status ~= "completed" then - marker(self, "", message) - else - self.partial = false - end + if self.status ~= "completed" then marker(self, message) else self.partial = false end repaint(self) end -local NOOP = setmetatable({ lines = {} }, { - __index = { - event = function() end, - done = function() end, - }, -}) +local NOOP = setmetatable({ history = {} }, { __index = { event = function() end, done = function() end } }) + +local function register_sid(sid) + if sid == "" then return end + for _, board in pairs(boards) do + for _, card in ipairs(board.cards) do + local other = card.sid + if other ~= "" and other ~= sid then + local limit, common = math.min(#sid, #other), 0 + while common < limit and sid:byte(common + 1) == other:byte(common + 1) do + common = common + 1 + end + sid_width = math.max(sid_width, math.min(common + 1, math.max(#sid, #other))) + end + end + end +end --- --------------------------------------------------------------------------- --- Boards (one per tool-call entry) --- --------------------------------------------------------------------------- +local function body_lines(card, width, collapsed) + local out, total = {}, 0 + local function add(line) + total = total + 1 + if collapsed and #out == COLLAPSED_LINES then table.remove(out, 1) end + if #out < MAX_RENDER_LINES then out[#out + 1] = BODY_INDENT .. line end + end + if not collapsed then + local metadata_truncated = false + local function add_metadata(label, text) + local continuation = string.rep(" ", #label) + local from, first, lines = 1, true, 0 + local function add_bounded(line) + if lines < MAX_PRESENTATION_LINES then + lines = lines + 1 + add(line) + else + metadata_truncated = true + end + end + while true do + local newline = text:find("\n", from, true) + local last = newline and newline - 1 or #text + wrap((first and label or continuation) .. text:sub(from, last), width, add_bounded) + first = false + if not newline then break end + from = newline + 1 + end + end + if card.system_prompt then add_metadata("system prompt: ", card.system_prompt) end + if card.prompt then add_metadata("prompt: ", card.prompt) end + if metadata_truncated then add("… prompt display truncated") end + end + for _, line in ipairs(card.history) do wrap(line, width, add) end + if not collapsed and total > #out then + out[#out] = BODY_INDENT .. "… retained history exceeds render limit" + end + return out +end local function render(board, width) local out = {} - if #board.cards == 0 or (width or 0) <= 4 then - return out - end + width = math.floor(tonumber(width) or 0) + if #board.cards == 0 or width <= #BODY_INDENT then return out end out[#out + 1] = "" for _, card in ipairs(board.cards) do - local sid = card.sid ~= "" and (" " .. card.sid) or "" - out[#out + 1] = truncate(string.format("%s %s%s — %s", - GLYPHS[card.status] or GLYPHS.running, card.label, sid, card.status), width) - for _, line in ipairs(card.lines) do - out[#out + 1] = BODY_INDENT .. truncate(line, width - #BODY_INDENT) - end + local sid = card.sid ~= "" and (" " .. card.sid:sub(1, sid_width) .. + (sid_width < #card.sid and "…" or "")) or "" + out[#out + 1] = prefix(sanitize(string.format("%s %s%s — %s", + GLYPHS[card.status] or GLYPHS.running, card.label, sid, card.status)), width) + local lines = body_lines(card, width - #BODY_INDENT, board.collapsed) + for _, line in ipairs(lines) do out[#out + 1] = line end end + if board.pinned then out[#out + 1] = "" end return out end +local function set_board_pinned(board, pinned) + if board.pinned == pinned or not board.handle then return end + local ok = pcall(function() board.handle:set_pinned(pinned) end) + if ok then + board.pinned = pinned + pcall(board.handle.invalidate, board.handle) + end +end + +local function prune_boards() + for key, board in pairs(boards) do + if board.handle and board.handle.alive then + local ok, alive = pcall(board.handle.alive, board.handle) + if ok and not alive then boards[key] = nil end + end + end +end + local function new_board() - local board = { cards = {}, handle = nil } - board.component = { - render = function(_, width) - return render(board, width) - end, - } + local board = { cards = {}, handle = nil, collapsed = tools_collapsed, next_sequence = 0, pinned = false } + board.component = { render = function(_, width) return render(board, width) end } return board end --- --------------------------------------------------------------------------- --- Host wiring --- --------------------------------------------------------------------------- - --- `tool_call_complete` for one of our tools: claim that entry's component so --- the cards the handler is about to raise have somewhere to render. function M.claim(event) - -- The event is host userdata; an unknown field answers nil rather than - -- raising, and activation already refused a host too old to have these. + prune_boards() local name = event.tool_name - if type(name) ~= "string" or name:sub(1, #TOOL_PREFIX) ~= TOOL_PREFIX then - return - end - if type(event.set_component) ~= "function" then - return - end - - local key = event.id or name - local board = new_board() + if not PROGRESS_TOOLS[name] then return end + if type(event.set_component) ~= "function" then return end + if type(event.collapsed) == "boolean" then tools_collapsed = event.collapsed end + local key, board = event.id or event.tool_call_id or name, new_board() local attached, handle = pcall(event.set_component, event, board.component) - if not attached then - return - end + if not attached then return end board.handle = handle + local prior = boards[key] + if prior then set_board_pinned(prior, false) end boards[key] = board + set_board_pinned(board, true) + -- During startup/restart the child catalog already contains the durable + -- turns for this outer call. A live call normally finds nothing here; the + -- ownership stamp makes that distinction without parsing outer results. + if replaying then + replay_board(board, type(key) == "string" and key or nil) + end +end + +-- A result settles the outer tool invocation. Keep the board attached to its +-- transcript entry, but return it to that entry's original ordering. +function M.settle(event) + local key = type(event) == "table" and (event.id or event.tool_call_id) or nil + local board = key and boards[key] + if not board then return end + set_board_pinned(board, false) +end + +function M.collapse(event) + if type(event.collapsed) ~= "boolean" then return end + prune_boards() + tools_collapsed = event.collapsed + for _, board in pairs(boards) do + board.collapsed = tools_collapsed + if board.handle then pcall(board.handle.invalidate, board.handle) end + end end --- Tie the running handler coroutine to the entry it was dispatched for. function M.bind(context) local key = type(context) == "table" and context.tool_call_id or nil - if key == nil then - return - end - bound[coroutine.running()] = key + if type(key) ~= "string" or key == "" then key = nil end + local co = coroutine.running() + if co then bound[co] = key end end --- card(label, id, detail) -> card --- --- The card of the entry whose handler is running, or an inert one when there is --- no such entry (print mode, a plain script, a host without components). -function M.card(label, id, detail) - local key = bound[coroutine.running()] - local board = key and boards[key] - if board == nil or (board.handle and board.handle.alive and not board.handle:alive()) then - return NOOP - end +-- The outer tool call is durable child-turn metadata, not model-visible output. +-- Expose it to the spawn seam without making the board state global: workflow +-- callbacks run on the same coroutine as their owning tool handler. +function M.tool_call_id() + local co = coroutine.running() + return co and bound[co] or nil +end +local function new_card(board, label, id, detail, presentation, sequence) + if board == nil then return NOOP end + local sid = type(id) == "string" and sanitize(id):sub(1, MAX_LINE_BYTES) or "" + register_sid(sid) + if type(sequence) ~= "number" or sequence < 1 or sequence % 1 ~= 0 then + board.next_sequence = board.next_sequence + 1 + sequence = board.next_sequence + else + board.next_sequence = math.max(board.next_sequence, sequence) + end local card = setmetatable({ board = board, - label = tostring(label or "subagent"), - sid = type(id) == "string" and id:sub(1, 8) or "", + label = sanitize(label or "subagent"):sub(1, MAX_LINE_BYTES), + sid = sid, + sequence = sequence, status = "running", - lines = {}, + history = {}, + history_bytes = 0, partial = false, - text_block = false, + blocks = {}, + block_had_delta = {}, + tools = {}, + prompt = type(presentation) == "table" and type(presentation.prompt) == "string" and + sanitize_multiline(presentation.prompt):sub(1, MAX_PRESENTATION_BYTES) or nil, + system_prompt = type(presentation) == "table" and type(presentation.system_prompt) == "string" and + sanitize_multiline(presentation.system_prompt):sub(1, MAX_PRESENTATION_BYTES) or nil, }, card_mt) board.cards[#board.cards + 1] = card - marker(card, "↳ ", detail) + if detail then marker(card, "↳ " .. sanitize(detail):sub(1, MAX_LINE_BYTES)) end repaint(card) return card end --- End of turn: the entries are gone, and the primary's own tool results are the --- durable record from here on. +function M.card(label, id, detail, presentation) + local key = bound[coroutine.running()] + local board = key and boards[key] + if board == nil or (board.handle and board.handle.alive and not board.handle:alive()) then return NOOP end + return new_card(board, label, id, detail, presentation) +end + +local function nonempty(value) + return type(value) == "string" and value ~= "" and value or nil +end + +-- Keep replay work finite even when a damaged child file contains pathological +-- payloads. The card itself has stricter history limits; this cap bounds the +-- temporary strings made while translating stored blocks into progress events. +local function bounded_text(value, limit) + if type(value) ~= "string" then return "" end + return #value > limit and value:sub(1, limit) or value +end + +local function text_blocks(blocks, limit) + if type(blocks) ~= "table" then return "" end + local pieces, bytes, seen = {}, 0, 0 + for _, block in ipairs(blocks) do + seen = seen + 1 + if seen > MAX_REPLAY_BLOCKS then break end + if type(block) == "table" and (block.type == "text" or block.type == "system") and type(block.text) == "string" then + if #pieces > 0 and bytes < limit then + pieces[#pieces + 1] = "\n" + bytes = bytes + 1 + end + local room = limit - bytes + if room <= 0 then break end + local text = block.text:sub(1, room) + pieces[#pieces + 1] = text + bytes = bytes + #text + if #text < #block.text then break end + end + end + return table.concat(pieces) +end + +local function blocks_of(message) + return type(message) == "table" and type(message.blocks) == "table" and message.blocks or {} +end + +local function message_text(message) + return type(message) == "table" and text_blocks(blocks_of(message), MAX_REPLAY_BLOCK_BYTES) or "" +end + +local function metadata_for(conv, index) + if conv == nil or type(conv.message_metadata) ~= "function" then return nil end + local ok, metadata = pcall(conv.message_metadata, conv, index) + if not ok or type(metadata) ~= "table" or type(metadata.subagents) ~= "table" then + return nil + end + return metadata.subagents +end + +local function owner_of(metadata) + if type(metadata) ~= "table" then return nil end + return nonempty(metadata.tool_call_id) +end + +local function child_manifest(conv, messages) + for index = 1, #messages do + local message = messages[index] + if type(message) == "table" and message.role == "system" then + local metadata = metadata_for(conv, index) + local agent = metadata and nonempty(metadata.agent) + if agent then + local system_prompt + if metadata.inline == true then + system_prompt = nonempty(message_text(message)) + end + return agent, system_prompt + end + end + end + return nil, nil +end + +local function result_text(block) + if type(block) ~= "table" then return "" end + local parts = block.parts or block.content + if type(parts) ~= "table" then return "" end + local pieces, bytes, seen = {}, 0, 0 + for _, part in ipairs(parts) do + seen = seen + 1 + if seen > MAX_REPLAY_BLOCKS then break end + if type(part) == "table" and type(part.text) == "string" then + local room = MAX_REPLAY_BLOCK_BYTES - bytes + if room <= 0 then break end + local text = part.text:sub(1, room) + pieces[#pieces + 1] = text + bytes = bytes + #text + if #text < #part.text then break end + end + end + return table.concat(pieces, "\n") +end + +local function is_prompt_message(message) + return type(message) == "table" and message.role == "user" and message_text(message) ~= "" +end + +-- Translate one stored child turn through the same card event path used by a +-- live stream. No outer tool result is involved: all displayed content comes +-- from this child's durable conversation. +local function replay_turn(board, conv, messages, start, finish, info, agent, system_prompt, sequence) + local metadata = metadata_for(conv, start) + local prompt = message_text(messages[start]) + if prompt == "" then return end + + local id = nonempty(info and info.id) + local model = nonempty(metadata and metadata.model) or nonempty(info and info.model) + local card = new_card(board, agent or "subagent", id, model, { + prompt = prompt, + system_prompt = system_prompt, + }, sequence) + local pending, terminal_assistant = {}, false + local block_index = 0 + + for index = start + 1, finish - 1 do + local message = messages[index] + if type(message) == "table" and message.role == "assistant" then + local has_tool = false + local has_response = false + local seen_blocks = 0 + for _, block in ipairs(blocks_of(message)) do + seen_blocks = seen_blocks + 1 + if seen_blocks > MAX_REPLAY_BLOCKS then break end + if type(block) == "table" and block.type == "tool_use" then + has_tool = true + local tool_id = bounded_text(block.id, MAX_REPLAY_BLOCK_BYTES) + local tool_name = bounded_text(block.name, MAX_REPLAY_BLOCK_BYTES) + local input = bounded_text(block.input, MAX_REPLAY_BLOCK_BYTES) + if tool_id ~= "" then pending[tool_id] = true end + card:event({ type = "block_complete", block_type = "tool_use", + index = block_index, id = tool_id, name = tool_name, text = input }) + block_index = block_index + 1 + else + -- Thinking-only and other non-text assistant messages are + -- still real completed responses; presentation text is + -- optional and must not decide terminal status. + has_response = true + if type(block) == "table" and block.type == "text" then + local text = bounded_text(block.text, MAX_REPLAY_BLOCK_BYTES) + if text ~= "" then + card:event({ type = "block_complete", block_type = "text", + index = block_index, text = text }) + end + block_index = block_index + 1 + end + end + end + -- A response containing a tool call is not the settled assistant + -- answer; a later assistant message after its results is. + terminal_assistant = has_response and not has_tool + elseif type(message) == "table" and message.role == "user" then + local results = {} + local seen_blocks = 0 + for _, block in ipairs(blocks_of(message)) do + seen_blocks = seen_blocks + 1 + if seen_blocks > MAX_REPLAY_BLOCKS then break end + if type(block) == "table" and block.type == "tool_result" then + local tool_id = bounded_text(block.tool_use_id, MAX_REPLAY_BLOCK_BYTES) + if tool_id ~= "" then pending[tool_id] = nil end + results[#results + 1] = { + tool_use_id = tool_id, + output = result_text(block), + is_error = block.is_error == true, + } + end + end + if #results > 0 then + card:event({ type = "tool_dispatch_result", tool_results = results }) + end + end + end + + local status = metadata and nonempty(metadata.status) + if status ~= "completed" and status ~= "failed" and status ~= "cancelled" then + status = nil + end + if status == nil then + local settled = terminal_assistant + if settled then + for _ in pairs(pending) do + settled = false + break + end + end + status = settled and "completed" or "failed" + end + card:done(status, status == "failed" and "child turn has no settled assistant response" or nil) + card.replay_sequence = sequence + return card +end + +local function replay_sequence(metadata) + local sequence = type(metadata) == "table" and tonumber(metadata.sequence) or nil + if sequence == nil or sequence < 1 or sequence % 1 ~= 0 then return nil end + return sequence +end + +replay_board = function(board, tool_call_id) + if board == nil or type(tool_call_id) ~= "string" or tool_call_id == "" then return end + + local path_ok, store_dir = pcall(paths.child_store_dir) + if not path_ok or type(store_dir) ~= "string" or store_dir == "" then return end + local panto_ok, panto = pcall(require, "panto") + if not panto_ok or type(panto) ~= "table" or type(panto.file_system_jsonl_store) ~= "function" then return end + local store_ok, store = pcall(panto.file_system_jsonl_store, { dir = store_dir }) + if not store_ok or store == nil or type(store.list_bounded) ~= "function" or + type(store.load_messages) ~= "function" then return end + + -- The store owns both bounds. In particular, this never asks the binding + -- for an unbounded catalog or conversation and trims only after a backend + -- has already applied the requested limit. + local listed, infos = pcall(store.list_bounded, store, { limit = MAX_REPLAY_SESSIONS }) + if not listed or type(infos) ~= "table" then return end + local sessions = {} + for _, info in ipairs(infos) do + if type(info) == "table" and nonempty(info.id) then sessions[#sessions + 1] = info end + end + table.sort(sessions, function(a, b) + local ac, bc = nonempty(a.created), nonempty(b.created) + if ac and bc and ac ~= bc then return ac < bc end + return a.id < b.id + end) + + -- Translate each bounded session into bounded card history immediately; + -- retaining whole conversations for all sessions would merely move the + -- unbounded replay footprint from the store to Lua while sorting. + local scratch = new_board() + local cards = {} + local ordinal = 0 + for _, info in ipairs(sessions) do + if #cards >= MAX_REPLAY_CARDS then break end + local loaded, conv = pcall(store.load_messages, store, info.id, { + limit = MAX_REPLAY_MESSAGES, + from_end = true, + max_bytes = MAX_REPLAY_MESSAGE_BYTES, + }) + if loaded and conv ~= nil and type(conv.messages) == "function" then + local messages_ok, messages = pcall(conv.messages, conv) + if messages_ok and type(messages) == "table" and #messages > 0 then + local turns = {} + local index = 1 + while index <= #messages do + if is_prompt_message(messages[index]) then + local finish = index + 1 + while finish <= #messages and not is_prompt_message(messages[finish]) do + finish = finish + 1 + end + local metadata = metadata_for(conv, index) + if owner_of(metadata) == tool_call_id then + turns[#turns + 1] = { + start = index, + finish = finish, + metadata = metadata, + sequence = replay_sequence(metadata), + } + end + index = finish + else + index = index + 1 + end + end + + if #turns > 0 then + local manifest_ok, manifest_conv = pcall(store.load_messages, store, info.id, { + limit = 1, + role = "system", + metadata_key = "subagents", + max_bytes = MAX_REPLAY_MESSAGE_BYTES, + }) + local agent, system_prompt + if manifest_ok and manifest_conv ~= nil and type(manifest_conv.messages) == "function" then + local got_messages, manifest_messages = pcall(manifest_conv.messages, manifest_conv) + if got_messages and type(manifest_messages) == "table" then + local got_manifest + got_manifest, agent, system_prompt = pcall(child_manifest, manifest_conv, manifest_messages) + if not got_manifest then agent, system_prompt = nil, nil end + end + end + for _, turn in ipairs(turns) do + if #cards >= MAX_REPLAY_CARDS then break end + ordinal = ordinal + 1 + local card_ok, card = pcall(replay_turn, scratch, conv, messages, + turn.start, turn.finish, info, agent, system_prompt, turn.sequence) + if card_ok and card ~= nil then + card.replay_ordinal = ordinal + cards[#cards + 1] = card + end + end + end + end + end + end + + table.sort(cards, function(a, b) + if a.replay_sequence and b.replay_sequence and a.replay_sequence ~= b.replay_sequence then + return a.replay_sequence < b.replay_sequence + end + if (a.replay_sequence ~= nil) ~= (b.replay_sequence ~= nil) then + return a.replay_sequence ~= nil + end + return a.replay_ordinal < b.replay_ordinal + end) + for _, card in ipairs(cards) do + if #board.cards >= MAX_REPLAY_CARDS then break end + card.board = board + board.cards[#board.cards + 1] = card + register_sid(card.sid) + repaint(card) + end +end + +-- The host fires this before the first live model turn, after startup +-- conversation replay has finished. Keeping the mode explicit avoids scanning +-- child files for every ordinary foreground call. +function M.begin_live_turn() + replaying = false +end + function M.reset() - boards = {} + for _, board in pairs(boards) do + set_board_pinned(board, false) + end + prune_boards() bound = setmetatable({}, { __mode = "k" }) end return M + diff --git a/subagents/spawn.lua b/subagents/spawn.lua index b355d9a..caf65aa 100644 --- a/subagents/spawn.lua +++ b/subagents/spawn.lua @@ -18,9 +18,13 @@ -- context, then the fixed child-role instruction, then — when the profile has -- a body — the profile prompt as a further system message. That profile -- message carries the immutable manifest metadata (owning primary session id + --- profile name), which is how a resumed child re-identifies itself. The parent --- dialogue is never copied, which is why the child-role text tells the child --- its final message is the whole of what the delegator sees. +-- profile name); workflow-local profiles also carry an inline marker so the +-- progress replay path can expose only prompts the caller explicitly supplied. +-- The per-turn user metadata records the effective model/reasoning, the +-- per-outer-call card sequence, terminal presentation status, and — when the +-- spawn happened inside a bound extension tool — that outer tool call id. +-- The parent dialogue is never copied, which is why the child-role text tells +-- the child its final message is the whole of what the delegator sees. -- -- Edge cases: a profile with an empty body contributes no system message and -- therefore no manifest, so a resumed child started from a body-less profile @@ -206,12 +210,17 @@ function M.build_spec(input, profiles) local system_messages = { { text = M.CHILD_ROLE } } if profile.body and profile.body:match("%S") then + local manifest = { owner = info_or_err.session_id, agent = profile.name } + if profile.layer == "workflow" then manifest.inline = true end system_messages[#system_messages + 1] = { text = profile.body, - metadata = { subagents = { owner = info_or_err.session_id, agent = profile.name } }, + metadata = { subagents = manifest }, } end spec.system_messages = system_messages + if profile.layer == "workflow" then + spec.presentation_system_prompt = profile.body + end return spec end @@ -447,10 +456,31 @@ function M.spawn(spec) local named, session_id = try(agent.session_id, agent) local id = (not one_shot) and named and nonempty(session_id) or nil - local card = progress.card(spec.label or "subagent", id, model_label) + local turn_index + local got_conv, child_conv = try(agent.conversation, agent) + if got_conv and child_conv and type(child_conv.len) == "function" then + local got_len, length = try(child_conv.len, child_conv) + if got_len and type(length) == "number" then turn_index = length + 1 end + end + + local card = progress.card(spec.label or "subagent", id, model_label, { + prompt = spec.prompt, + system_prompt = spec.presentation_system_prompt, + }) + + local owner_tool_call_id = progress.tool_call_id() + local turn_metadata = { + subagents = { model = model_label, reasoning = reasoning_label }, + } + local sequence = card.sequence + if type(sequence) == "number" then turn_metadata.subagents.sequence = sequence end + if type(owner_tool_call_id) == "string" and owner_tool_call_id ~= "" then + turn_metadata.subagents.tool_call_id = owner_tool_call_id + end -- The settled run_async result becomes the result table every caller -- already reads: run.lua's block and the workflow API's shape_result. + local child_handle local function shape(raw) raw = type(raw) == "table" and raw or {} local result = { @@ -483,16 +513,30 @@ function M.spawn(spec) local ok, found = try(store.resolve, store, id) result.resumable = ok and found ~= nil end + turn_metadata.subagents.status = result.status + -- `agent:set_message_metadata` is deliberately refused while a pump + -- is live. The result has already been copied out, so joining here is + -- safe and releases the agent's mutation guard before the annotation. + if child_handle and child_handle.job then + pcall(child_handle.job.close, child_handle.job) + end + if turn_index and type(agent.set_message_metadata) == "function" then + -- The binding's annotation seam updates the already-written user + -- record, so cancellation has a durable presentation status even + -- though the stream correctly rolls its assistant messages back. + pcall(agent.set_message_metadata, agent, turn_index, turn_metadata) + end card:done(result.status, result.error) return result end - local handle, start_err = jobs.start { + local start_err + child_handle, start_err = jobs.start { id = id, build = function(wake_fd) local job, err = agent:run_async { prompt = spec.prompt, - metadata = { subagents = { model = model_label, reasoning = reasoning_label } }, + metadata = turn_metadata, dispatch_tools = not one_shot, wake_fd = wake_fd, } @@ -506,6 +550,7 @@ function M.spawn(spec) end, shape = shape, } + local handle = child_handle if not handle then card:done("failed", start_err) return nil, start_err -- cgit v1.3