diff options
| author | t <t@tjp.lol> | 2026-06-01 13:47:35 -0600 |
|---|---|---|
| committer | t <t@tjp.lol> | 2026-06-01 17:03:05 -0600 |
| commit | b6afac69b586dd9d68cccf686f3220581848c78b (patch) | |
| tree | 31808c44cf80c17c5bffd5139453a6358d6291b8 | |
| parent | f4c72eef847b093212c4dd6136cd7e0b1478afe2 (diff) | |
rework lua tool scheduler
| -rw-r--r-- | AGENTS.md | 2 | ||||
| -rw-r--r-- | agent/tools/edit.lua | 100 | ||||
| -rw-r--r-- | agent/tools/read.lua | 104 | ||||
| -rw-r--r-- | agent/tools/shell.lua | 179 | ||||
| -rw-r--r-- | agent/tools/write.lua | 65 | ||||
| -rw-r--r-- | build.zig | 4 | ||||
| -rw-r--r-- | src/lua_runtime.zig | 349 |
7 files changed, 582 insertions, 221 deletions
@@ -1,3 +1,5 @@ # pantograph Acronyms in CamelCase are fully capitalized: `APIStyle`, `HTTPClient`, `IDLookup`. Field names are snake_case: `api_style`, `base_url`, `model`. + +DO NOT read `mise.local.toml`, that contains API keys which we don't want exposed to LLM providers. diff --git a/agent/tools/edit.lua b/agent/tools/edit.lua index 7ad39c2..ec96aa6 100644 --- a/agent/tools/edit.lua +++ b/agent/tools/edit.lua @@ -10,6 +10,82 @@ -- All matches are evaluated against the original file, not -- progressively against the result of earlier entries. +local uv = require("luv") + +-- --------------------------------------------------------------------------- +-- Coroutine-synchronous libuv wrappers. panto runs each tool handler in +-- its own coroutine and drives a single `uv.run()` to completion; a +-- handler may only block by yielding on a pending libuv op whose +-- callback resumes it exactly once. `await` encapsulates that: arm one +-- op, yield, return the callback's `(err, value)` results. +-- --------------------------------------------------------------------------- +local function await(arm) + local co = assert(coroutine.running(), + "edit: await must run inside a tool handler coroutine") + local res, fired = nil, false + arm(function(...) + assert(not fired, "edit: libuv callback fired twice") + fired = true + res = table.pack(...) + local ok, err = coroutine.resume(co) + if not ok then error(err, 0) end + end) + coroutine.yield() + return table.unpack(res, 1, res.n) +end + +local function fs_open(path, flags, mode) + return await(function(resolve) uv.fs_open(path, flags, mode, resolve) end) +end +local function fs_fstat(fd) + return await(function(resolve) uv.fs_fstat(fd, resolve) end) +end +local function fs_read(fd, size, offset) + return await(function(resolve) uv.fs_read(fd, size, offset, resolve) end) +end +local function fs_write(fd, data, offset) + return await(function(resolve) uv.fs_write(fd, data, offset, resolve) end) +end +local function fs_close(fd) + return await(function(resolve) uv.fs_close(fd, resolve) end) +end + +-- Read an entire file into a string. Returns (content, nil) or +-- (nil, err). EOF on `fs_read` is the empty string, not nil. +local function read_all(path) + local err, fd = fs_open(path, "r", 0) + if not fd then return nil, err end + local serr, st = fs_fstat(fd) + if not st then fs_close(fd); return nil, serr end + local parts, offset = {}, 0 + while true do + local rerr, data = fs_read(fd, 64 * 1024, offset) + if rerr then fs_close(fd); return nil, rerr end + if data == nil or #data == 0 then break end + parts[#parts + 1] = data + offset = offset + #data + end + fs_close(fd) + return table.concat(parts), nil +end + +-- Overwrite a file with `content` (O_WRONLY|O_CREAT|O_TRUNC, 0644). +-- Returns (true, nil) or (nil, err). +local function write_all(path, content) + local err, fd = fs_open(path, "w", tonumber("644", 8)) + if not fd then return nil, err end + -- A single `fs_write` may short-write; loop until all bytes land. + local offset = 0 + while offset < #content do + local werr, n = fs_write(fd, content:sub(offset + 1), offset) + if not n then fs_close(fd); return nil, werr end + offset = offset + n + end + -- Empty content still needs the truncate that O_TRUNC gave us. + fs_close(fd) + return true, nil +end + return { name = "std.edit", description = "Apply exact-text replacements to a file. Each entry's `old` must match exactly once in the original file (not progressively). Edits whose ranges overlap, match zero times, or match multiple times reject the entire call — no partial edits, file untouched. Keep `old` minimal but unique; widen with surrounding context if needed.", @@ -45,14 +121,9 @@ return { end -- Load the file. - local f, open_err = io.open(path, "rb") - if not f then - return "Error: " .. (open_err or ("could not open " .. path)) - end - local original = f:read("a") - f:close() + local original, read_err = read_all(path) if not original then - return "Error: failed to read " .. path + return "Error: " .. (read_err or ("could not read " .. path)) end -- Validate every entry first, collecting the byte-range each @@ -170,19 +241,10 @@ return { local new_content = table.concat(out) -- Write back. - local wf, werr = io.open(path, "wb") - if not wf then - return "Error: file read OK but failed to reopen for write: " .. - (werr or "?") - end - local ok, write_err = wf:write(new_content) + local ok, write_err = write_all(path, new_content) if not ok then - wf:close() - return "Error: write failed: " .. tostring(write_err) - end - local close_ok, close_err = wf:close() - if not close_ok then - return "Error: close failed: " .. tostring(close_err) + return "Error: file read OK but write failed: " .. + tostring(write_err) end return string.format( diff --git a/agent/tools/read.lua b/agent/tools/read.lua index d345f99..f232890 100644 --- a/agent/tools/read.lua +++ b/agent/tools/read.lua @@ -19,18 +19,98 @@ local MAX_BYTES = 50 * 1024 -- 50 KB local MAX_LINES = 2000 local READ_CHUNK_SIZE = MAX_BYTES +-- --------------------------------------------------------------------------- +-- Coroutine-synchronous libuv wrappers. +-- +-- panto runs every tool handler inside its own Lua coroutine and drives +-- a single `uv.run()` to completion. The contract for an async handler +-- is: the ONLY way it may block is to yield on a pending libuv request +-- whose callback resumes it exactly once. These helpers encapsulate +-- that pattern — each fires one async libuv op, yields, and returns the +-- callback's results once resumed. They look synchronous to the caller +-- but cooperate with sibling tool calls sharing the event loop. +-- +-- INVARIANTS each wrapper upholds (and every async tool must too): +-- * exactly one libuv op armed per yield; +-- * the callback resumes the coroutine exactly once; +-- * `coroutine.resume` errors are re-raised so a crash in the resumed +-- handler isn't silently swallowed inside the uv callback. +-- --------------------------------------------------------------------------- + +-- Run a single callback-style libuv fs operation and block the +-- coroutine until its callback fires. `arm` receives a `resolve` +-- function and must start exactly one libuv op that calls it. +local function await(arm) + local co = assert(coroutine.running(), + "read: await must run inside a tool handler coroutine") + local res, fired = nil, false + arm(function(...) + assert(not fired, "read: libuv callback fired twice") + fired = true + res = table.pack(...) + local ok, err = coroutine.resume(co) + if not ok then error(err, 0) end + end) + coroutine.yield() + return table.unpack(res, 1, res.n) +end + +local function fs_stat(path) + return await(function(resolve) uv.fs_stat(path, resolve) end) +end + +local function fs_open(path, flags, mode) + return await(function(resolve) uv.fs_open(path, flags, mode, resolve) end) +end + +local function fs_read(fd, size, offset) + return await(function(resolve) uv.fs_read(fd, size, offset, resolve) end) +end + +local function fs_close(fd) + return await(function(resolve) uv.fs_close(fd, resolve) end) +end + +-- `uv.fs_opendir` takes the entries-per-readdir batch hint as its +-- SECOND positional arg, before the callback. We pass it explicitly so +-- each `fs_readdir` returns up to `n` entries per round-trip. +local READDIR_BATCH = 256 +local function fs_opendir(path) + return await(function(resolve) uv.fs_opendir(path, resolve, READDIR_BATCH) end) +end +local function fs_readdir(dir) + return await(function(resolve) uv.fs_readdir(dir, resolve) end) +end +local function fs_closedir(dir) + return await(function(resolve) uv.fs_closedir(dir, resolve) end) +end + local function scandir_sorted(path) - local req, err = uv.fs_scandir(path) - if not req then + -- Use the streaming opendir/readdir/closedir API rather than + -- `fs_scandir` + `fs_scandir_next`: the latter's iterator advances + -- with synchronous `readdir(3)` syscalls, which would block the + -- shared event loop on large directories. opendir/readdir route + -- every batch through libuv's thread pool, keeping us cooperative. + local err, dir = fs_opendir(path) + if not dir then return nil, err end local entries = {} while true do - local name, typ = uv.fs_scandir_next(req) - if not name then break end - entries[#entries + 1] = { name = name, typ = typ } + local rerr, batch = fs_readdir(dir) + if rerr then + fs_closedir(dir) + return nil, rerr + end + -- A nil/empty batch signals end of the directory stream. + if batch == nil or #batch == 0 then break end + for _, e in ipairs(batch) do + -- async readdir yields { name = ..., type = ... }. + entries[#entries + 1] = { name = e.name, typ = e.type } + end end + fs_closedir(dir) table.sort(entries, function(a, b) return a.name < b.name @@ -136,7 +216,7 @@ return { return string.format("Error: limit (%d) must be >= 1.", limit) end - local stat, stat_err = uv.fs_stat(path) + local stat_err, stat = fs_stat(path) if not stat then return "Error: " .. (stat_err or ("could not stat " .. path)) end @@ -144,7 +224,7 @@ return { return render_directory_listing(path, offset, limit) end - local fd, open_err = uv.fs_open(path, "r", 0) + local open_err, fd = fs_open(path, "r", 0) if not fd then return "Error: " .. (open_err or ("could not open " .. path)) end @@ -190,12 +270,16 @@ return { end while not done do - local data, err = uv.fs_read(fd, READ_CHUNK_SIZE, file_offset) + local err, data = fs_read(fd, READ_CHUNK_SIZE, file_offset) if err then read_err = err break end - if data == nil then + -- luv's `fs_read` signals EOF by returning the empty + -- string, NOT nil (nil only accompanies an error). Treat + -- both nil and "" as EOF; otherwise the loop re-reads at + -- the same offset forever, pinning a core at 100%. + if data == nil or #data == 0 then if #carry > 0 then process_line(carry) carry = "" @@ -226,7 +310,7 @@ return { end end - uv.fs_close(fd) + fs_close(fd) if read_err then return "Error: read failed: " .. tostring(read_err) diff --git a/agent/tools/shell.lua b/agent/tools/shell.lua index a9e9de3..815a8b9 100644 --- a/agent/tools/shell.lua +++ b/agent/tools/shell.lua @@ -33,6 +33,42 @@ local DEFAULT_TIMEOUT_S = 300 -- 5 minutes local SIGKILL_GRACE_MS = 250 -- --------------------------------------------------------------------------- +-- Coroutine-synchronous coordination. +-- +-- panto runs every tool handler in its own coroutine and drives a +-- single `uv.run()` to completion; the only legal way for a handler to +-- block is to yield on pending libuv work whose callback(s) resume it. +-- The shell tool waits on THREE independent events before it can build +-- its result: stdout EOF, stderr EOF, and process exit. `await_n` +-- captures that "resume once, after N signals" pattern. +-- +-- `arm(signal)` is called immediately with a `signal` function; it must +-- arrange for `signal` to be called exactly `n` times across its libuv +-- callbacks. After the n-th signal, the coroutine is resumed exactly +-- once. `signal` is safe to call from any libuv callback. +local function await_n(n, arm) + local co = assert(coroutine.running(), + "shell: await_n must run inside a tool handler coroutine") + local remaining, resumed = n, false + local function signal() + if remaining <= 0 then return end + remaining = remaining - 1 + if remaining > 0 or resumed then return end + resumed = true + local ok, err = coroutine.resume(co) + if not ok then + -- This runs inside a libuv callback, not the handler's + -- own frame; re-raising here would unwind the event loop. + -- Log instead; the scheduler surfaces the unfinished + -- coroutine as a tool error on reap. + io.stderr:write("shell tool: resume failed: " .. tostring(err) .. "\n") + end + end + arm(signal) + coroutine.yield() +end + +-- --------------------------------------------------------------------------- -- Spill-file path resolution. We mirror Zig's `panto_home.resolveHome`: -- PANTO_HOME wins, then XDG_DATA_HOME/panto, then HOME/.local/share/panto. -- The shell-output subdirectory is created lazily. @@ -117,8 +153,7 @@ return { local timeout_s = input.timeout or DEFAULT_TIMEOUT_S local timeout_ms = timeout_s * 1000 - local co = coroutine.running() - if not co then + if not coroutine.running() then -- Should never happen — panto always runs handlers in a -- coroutine — but be defensive rather than silently hang. return "Error: shell tool requires a coroutine scheduler " .. @@ -148,6 +183,14 @@ return { local spill_error = nil -- if opening/writing the spill file fails local total_written_to_spill = 0 + -- NOTE: the spill helpers below use SYNCHRONOUS `uv.fs_*` + -- calls on purpose. They run inside `append_chunk`, which runs + -- inside the stdout/stderr read callbacks — i.e. in event-loop + -- context, NOT in the handler coroutine's own frame. You cannot + -- `coroutine.yield` (and thus cannot `await`) from a libuv + -- callback, so these must be the blocking variants. The brief + -- per-write stall is acceptable; spilling only happens once + -- output exceeds the in-memory cap. local function open_spill_and_seed() -- Opens the spill file and writes every chunk currently -- live in the FIFO. Called exactly once, the moment we @@ -241,15 +284,10 @@ return { total_bytes = total_bytes + #data end - -- Coordination state. We resume the coroutine exactly once, - -- after `pending_events` hits zero: stdout EOF, stderr EOF, - -- on_exit fired = 3 events. - local pending_events = 3 - local resumed = false + -- Per-call outcome state, filled by the libuv callbacks below. local exit_code = nil local exit_signal = nil local timed_out = false - local spawn_error = nil local process_handle = nil local timeout_timer = nil local kill_timer = nil @@ -264,97 +302,90 @@ return { return table.concat(out) end - local function try_resume() - if pending_events > 0 then return end - if resumed then return end - resumed = true - -- Cancel any still-pending timers. They may have been - -- nil'd out already inside their own callback. - if timeout_timer and not timeout_timer:is_closing() then - timeout_timer:stop(); timeout_timer:close() - end - if kill_timer and not kill_timer:is_closing() then - kill_timer:stop(); kill_timer:close() - end - local ok, err = coroutine.resume(co) - if not ok then - -- The handler shouldn't be raising here; this is the - -- resume scheduler, not user code. Log via stderr; - -- the parent will surface the failure as a tool error. - io.stderr:write("shell tool: resume failed: " .. tostring(err) .. "\n") - end - end - - local function on_read(err, data) - if err then - -- Treat read errors as EOF — luv signals EOF by - -- calling with `data == nil`, so an actual `err` means - -- something genuinely broke. Record it once. - append_chunk("[read error: " .. tostring(err) .. "]\n") - pending_events = pending_events - 1 - try_resume() - return - end - if data == nil then - -- EOF on this pipe. - pending_events = pending_events - 1 - try_resume() - return - end - append_chunk(data) - end - + -- Spawn first so a spawn failure can return *before* we commit + -- to awaiting (a failed spawn fires no events to resume us). local stdio = { nil, stdout_pipe, stderr_pipe } local opts = { args = { "-c", command }, stdio = stdio } if cwd then opts.cwd = cwd end - local handle, pid_or_err, name = uv.spawn("/bin/sh", opts, function(code, signal) + local on_exit_signal -- set by await_n before any event fires + local handle, pid_or_err, name = uv.spawn("/bin/sh", opts, function(code, sig) exit_code = code - exit_signal = signal - pending_events = pending_events - 1 - -- The on-exit callback is responsible for closing the - -- process handle, per luv convention. + exit_signal = sig + -- The on-exit callback owns closing the process handle. if process_handle and not process_handle:is_closing() then process_handle:close() end - try_resume() + on_exit_signal() end) if not handle then - -- Spawn failed before we ever started — `pid_or_err` is - -- the error message. Clean up pipes and return. - spawn_error = tostring(pid_or_err) .. + -- Spawn failed before any event could fire. Clean up the + -- pipes and return without awaiting. + local spawn_error = tostring(pid_or_err) .. (name and (" (" .. name .. ")") or "") stdout_pipe:close(); stderr_pipe:close() return "Error: failed to spawn shell: " .. spawn_error end process_handle = handle - stdout_pipe:read_start(on_read) - stderr_pipe:read_start(on_read) + -- `await_n(3, ...)` resumes this coroutine exactly once, after + -- all three of {stdout EOF, stderr EOF, process exit} signal. + await_n(3, function(signal) + -- The exit callback was armed above (before this awaiter + -- existed); wire its signal now. + on_exit_signal = signal - -- Timeout timer. If it fires, we send SIGTERM and arm a - -- second short timer to follow up with SIGKILL. - timeout_timer = uv.new_timer() - timeout_timer:start(timeout_ms, 0, function() - timed_out = true - if process_handle and not process_handle:is_closing() then - process_handle:kill("sigterm") + -- stdout/stderr each signal once, on EOF or read error. + -- luv delivers stream EOF as a nil `data` with no error. + local function on_read(err, data) + if err then + append_chunk("[read error: " .. tostring(err) .. "]\n") + signal() + elseif data == nil then + signal() + else + append_chunk(data) + end end - timeout_timer:stop(); timeout_timer:close(); timeout_timer = nil - kill_timer = uv.new_timer() - kill_timer:start(SIGKILL_GRACE_MS, 0, function() + stdout_pipe:read_start(on_read) + stderr_pipe:read_start(on_read) + + -- Timeout timer. If it fires, send SIGTERM and arm a short + -- follow-up timer for SIGKILL. These timers do NOT signal; + -- they cause the child to exit, which fires the exit/EOF + -- callbacks that do. We close them here so they stop + -- holding the event loop open. + timeout_timer = uv.new_timer() + timeout_timer:start(timeout_ms, 0, function() + timed_out = true if process_handle and not process_handle:is_closing() then - process_handle:kill("sigkill") + process_handle:kill("sigterm") end - kill_timer:stop(); kill_timer:close(); kill_timer = nil + timeout_timer:stop(); timeout_timer:close(); timeout_timer = nil + kill_timer = uv.new_timer() + kill_timer:start(SIGKILL_GRACE_MS, 0, function() + if process_handle and not process_handle:is_closing() then + process_handle:kill("sigkill") + end + kill_timer:stop(); kill_timer:close(); kill_timer = nil + end) end) end) - -- Wait for stdout EOF, stderr EOF, and on_exit. - coroutine.yield() + if spawn_error then + return "Error: failed to spawn shell: " .. spawn_error + end - -- Close pipes (idempotent if already closing). + -- Resumed: all three events fired. Cancel any still-pending + -- timers (the command may have finished before the timeout) so + -- they don't keep the loop alive, then close pipes/spill. + if timeout_timer and not timeout_timer:is_closing() then + timeout_timer:stop(); timeout_timer:close() + end + if kill_timer and not kill_timer:is_closing() then + kill_timer:stop(); kill_timer:close() + end if not stdout_pipe:is_closing() then stdout_pipe:close() end if not stderr_pipe:is_closing() then stderr_pipe:close() end if spill_handle then uv.fs_close(spill_handle) end diff --git a/agent/tools/write.lua b/agent/tools/write.lua index 01bf3b2..42cb85a 100644 --- a/agent/tools/write.lua +++ b/agent/tools/write.lua @@ -4,8 +4,46 @@ local uv = require("luv") +-- --------------------------------------------------------------------------- +-- Coroutine-synchronous libuv wrappers. panto runs each tool handler in +-- its own coroutine and drives a single `uv.run()` to completion; a +-- handler may only block by yielding on a pending libuv op whose +-- callback resumes it exactly once. `await` encapsulates that. +-- --------------------------------------------------------------------------- +local function await(arm) + local co = assert(coroutine.running(), + "write: await must run inside a tool handler coroutine") + local res, fired = nil, false + arm(function(...) + assert(not fired, "write: libuv callback fired twice") + fired = true + res = table.pack(...) + local ok, err = coroutine.resume(co) + if not ok then error(err, 0) end + end) + coroutine.yield() + return table.unpack(res, 1, res.n) +end + +-- Async `fs_mkdir`. luv's async callback is `(err, success)`, and on +-- EEXIST it reports the error via a string code in `err`; we detect +-- that textually since the async form doesn't surface the errno name +-- separately the way the sync form's third return does. +local function fs_mkdir(path, mode) + return await(function(resolve) uv.fs_mkdir(path, mode, resolve) end) +end +local function fs_open(path, flags, mode) + return await(function(resolve) uv.fs_open(path, flags, mode, resolve) end) +end +local function fs_write(fd, data, offset) + return await(function(resolve) uv.fs_write(fd, data, offset, resolve) end) +end +local function fs_close(fd) + return await(function(resolve) uv.fs_close(fd, resolve) end) +end + -- mkdir -p equivalent: walk the path components and create each. --- Ignores EEXIST so the call is idempotent. +-- Ignores "already exists" so the call is idempotent. local function mkdir_p(path) if path == "" or path == "." or path == "/" then return true end local parts = {} @@ -15,8 +53,8 @@ local function mkdir_p(path) local cur = (path:sub(1, 1) == "/") and "" or "." for _, piece in ipairs(parts) do cur = cur .. "/" .. piece - local ok, err, name = uv.fs_mkdir(cur, tonumber("755", 8)) - if not ok and name ~= "EEXIST" then + local err = fs_mkdir(cur, tonumber("755", 8)) + if err and not tostring(err):find("EEXIST") then return nil, err end end @@ -60,19 +98,24 @@ return { end end - local f, open_err = io.open(path, "wb") - if not f then + local open_err, fd = fs_open(path, "w", tonumber("644", 8)) + if not fd then return "Error: " .. (open_err or ("could not open " .. path .. " for writing")) end - local ok, write_err = f:write(content) - if not ok then - f:close() - return "Error: write failed: " .. tostring(write_err) + -- A single `fs_write` may short-write; loop until all bytes land. + local offset = 0 + while offset < #content do + local werr, n = fs_write(fd, content:sub(offset + 1), offset) + if not n then + fs_close(fd) + return "Error: write failed: " .. tostring(werr) + end + offset = offset + n end - local close_ok, close_err = f:close() - if not close_ok then + local close_err = fs_close(fd) + if close_err then return "Error: close failed: " .. tostring(close_err) end @@ -52,6 +52,10 @@ pub fn build(b: *std.Build) void { versions_mod.addOption([]const u8, "lua_version", lua_version); versions_mod.addOption([]const u8, "lua_short_version", lua_short_version); versions_mod.addOption([]const u8, "luarocks_version", luarocks_version); + // Absolute repo root, for tests that need to locate source-tree + // assets (e.g. `agent/tools/*.lua`) regardless of the test + // runner's cwd. + versions_mod.addOption([]const u8, "repo_root", b.build_root.path orelse "."); // CLI executable const exe_mod = b.createModule(.{ diff --git a/src/lua_runtime.zig b/src/lua_runtime.zig index fed56a8..2cf3cf9 100644 --- a/src/lua_runtime.zig +++ b/src/lua_runtime.zig @@ -63,15 +63,10 @@ pub const LuaRuntime = struct { /// `panto._record_result`. Allocated once at `create`; reused for /// every call. `0` until `installScheduler` runs. wrapper_ref: c_int = 0, - /// Registry ref to `require("luv").run`, the function we call to - /// tick libuv between coroutine resumes. `0` until + /// Registry ref to `require("luv").run`, the function the scheduler + /// calls to drive libuv to completion. `0` until /// `installScheduler` runs. uv_run_ref: c_int = 0, - /// Registry ref to `require("luv").loop_alive`, used by the - /// scheduler to detect handler coroutines that yielded without - /// arming any libuv work (a deadlock we surface rather than hang - /// on). `0` until `installScheduler` runs. - uv_loop_alive_ref: c_int = 0, /// Pointer to the in-flight batch, valid only for the duration of /// one `invoke_batch` call. The `panto._record_result` C function @@ -134,9 +129,6 @@ pub const LuaRuntime = struct { if (self.uv_run_ref != 0) { c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, self.uv_run_ref); } - if (self.uv_loop_alive_ref != 0) { - c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, self.uv_loop_alive_ref); - } c.lua_close(self.L); @@ -489,7 +481,12 @@ fn runBatch( if (r != 0) c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, r); }; - var pending: usize = 0; + // Step 1: start every call's coroutine. A *synchronous* handler + // (one that never yields) runs to completion right here and + // records its result via the wrapper; `still_pending` is false. + // An *async* handler arms one or more libuv operations and yields + // — that armed work is what keeps the event loop alive in step 2. + var any_pending = false; for (calls, 0..) |call, i| { const handler_ref = self.handlers.get(call.tool_name) orelse { // Synthesize a recorded "err" result; don't even bother @@ -503,74 +500,60 @@ fn runBatch( }; const t = try startCoroutine(self, i, handler_ref, call.input, allocator); thread_refs[i] = t.thread_ref; - if (t.still_pending) pending += 1; + if (t.still_pending) any_pending = true; } - while (pending > 0) { - try driveUvOnce(self); - // Reap any coroutines that terminated during the tick. - var reaped: usize = 0; - for (thread_refs, 0..) |tref, i| { - if (tref == 0) continue; - // Push the thread, check status, pop. - _ = c.lua_rawgeti(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); - const co: *c.lua_State = @ptrCast(c.lua_tothread(self.L, -1).?); - const status = c.lua_status(co); - c.lua_settop(self.L, c.lua_gettop(self.L) - 1); - if (status != c.LUA_YIELD) { - // Terminated (LUA_OK or error). The wrapper should - // have called `_record_result` already; if not, synthesize. - if (!slots[i].recorded) { - slots[i] = .{ - .recorded = true, - .ok = false, - .err_msg = try allocator.dupe( - u8, - "panto: handler terminated without recording a result", - ), - }; - } - c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); - thread_refs[i] = 0; - reaped += 1; - } - } - if (reaped > 0) { - if (reaped > pending) { - // Defensive: keep the counter sane. - pending = 0; - } else { - pending -= reaped; - } - continue; - } - if (!try loopAlive(self)) { - // No libuv handles are pending, but we still have alive - // coroutines. They yielded without arranging to be woken. - // Mark them as failed and break. - // - // Note: we ask `uv.loop_alive` rather than relying on the - // return value of `uv.run("once")`, which is a boolean - // signalling whether `uv.stop()` was called — not a count - // of active handles. Conflating the two used to break - // multi-tick tools (e.g. `bash`) on their second tick. + // Step 2: drive the event loop to completion. The contract for + // async handlers is that the *only* way a handler coroutine may + // block is by yielding on a pending libuv operation whose callback + // will eventually resume it. Under that invariant, `uv.run` + // returns exactly when no active handles remain — which means + // every coroutine has either finished or violated the contract + // (yielded without keeping libuv work alive). Either way there is + // nothing left to wake, so a single run-to-completion pass is all + // we need; the per-tick reap/`loop_alive` dance the scheduler used + // to do is unnecessary. + if (any_pending) { + try driveUvToCompletion(self); + } - for (thread_refs, 0..) |tref, i| { - if (tref == 0) continue; - slots[i] = .{ - .recorded = true, - .ok = false, - .err_msg = try allocator.dupe( - u8, - "panto: handler yielded but no libuv handle is pending; " ++ - "did you forget to await with luv?", - ), - }; - c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); - thread_refs[i] = 0; - } - break; + // Step 3: reap. Every coroutine should now be terminated. Any that + // is still suspended (`LUA_YIELD`) violated the contract — it + // yielded without arming libuv work that would resume it (or armed + // work, then tore it down without resolving). Surface that as a + // per-call error rather than hanging the process. + for (thread_refs, 0..) |tref, i| { + if (tref == 0) continue; + _ = c.lua_rawgeti(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + const co: *c.lua_State = @ptrCast(c.lua_tothread(self.L, -1).?); + const status = c.lua_status(co); + c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (status == c.LUA_YIELD) { + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe( + u8, + "panto: handler is still suspended after the event loop " ++ + "drained; it yielded without a pending libuv operation " ++ + "to resume it (did a uv callback fail to resume the " ++ + "coroutine, or was its handle closed without resolving?)", + ), + }; + } else if (!slots[i].recorded) { + // Terminated cleanly or with an error, but the wrapper + // never recorded — should not happen, but don't drop it. + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe( + u8, + "panto: handler terminated without recording a result", + ), + }; } + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + thread_refs[i] = 0; } // Translate slots into the libpanto-shaped results. @@ -673,13 +656,15 @@ fn startCoroutine( }; } -/// Call `uv.run("once")`. The return value (a boolean meaning "was -/// `uv.stop()` called with handles still alive?") is not useful to us -/// — to decide whether to keep ticking we call `loopAlive` separately. -fn driveUvOnce(self: *LuaRuntime) !void { +/// Call `uv.run("default")`, which runs the event loop until there are +/// no active handles or requests left. Under the async-tool contract +/// (a handler may only block by yielding on a pending libuv operation +/// whose callback resumes it), this returns exactly when every handler +/// coroutine has finished. The boolean return value is discarded. +fn driveUvToCompletion(self: *LuaRuntime) !void { const L = self.L; _ = c.lua_rawgeti(L, lua_bridge.LUA_REGISTRYINDEX, @intCast(self.uv_run_ref)); - _ = c.lua_pushlstring(L, "once", 4); + _ = c.lua_pushlstring(L, "default", 7); if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { logTopAsError(L, "panto-lua: uv.run failed"); c.lua_settop(L, c.lua_gettop(L) - 1); @@ -689,23 +674,6 @@ fn driveUvOnce(self: *LuaRuntime) !void { c.lua_settop(L, c.lua_gettop(L) - 1); } -/// Call `uv.loop_alive()`. Returns true iff libuv has any referenced -/// active handles, requests, or closing handles. We use this — not -/// the return value of `uv.run` — to detect coroutines that yielded -/// without arranging to be woken. -fn loopAlive(self: *LuaRuntime) !bool { - const L = self.L; - _ = c.lua_rawgeti(L, lua_bridge.LUA_REGISTRYINDEX, @intCast(self.uv_loop_alive_ref)); - if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { - logTopAsError(L, "panto-lua: uv.loop_alive failed"); - c.lua_settop(L, c.lua_gettop(L) - 1); - return error.UvRunFailed; - } - const alive = c.lua_toboolean(L, -1) != 0; - c.lua_settop(L, c.lua_gettop(L) - 1); - return alive; -} - /// Pre-scheduler fallback (used in unit tests and during early /// startup before `installScheduler` has run). fn runLegacySync( @@ -895,32 +863,29 @@ fn installWrapperClosure(self: *LuaRuntime) !void { self.wrapper_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); } -/// Cache `require("luv").run` and `require("luv").loop_alive` in the -/// registry so the scheduler can invoke them cheaply per tick. +/// Cache `require("luv").run` in the registry so the scheduler can +/// invoke it cheaply without re-resolving `require` each batch. fn cacheUvRun(self: *LuaRuntime) !void { const L = self.L; const snippet = \\local uv = require("luv") - \\return uv.run, uv.loop_alive + \\return uv.run ; if (c.luaL_loadstring(L, snippet) != 0) { logTopAsError(L, "panto-lua: failed to compile luv lookup"); c.lua_settop(L, c.lua_gettop(L) - 1); return RuntimeError.LuaInitFailed; } - if (c.lua_pcallk(L, 0, 2, 0, 0, null) != 0) { + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { logTopAsError(L, "panto-lua: require('luv') failed (was the bootstrap successful?)"); c.lua_settop(L, c.lua_gettop(L) - 1); return RuntimeError.LuaInitFailed; } - // Stack: [..., uv.run, uv.loop_alive]. luaL_ref pops the top. - if (c.lua_type(L, -1) != lua_bridge.T_FUNCTION or - c.lua_type(L, -2) != lua_bridge.T_FUNCTION) - { - c.lua_settop(L, c.lua_gettop(L) - 2); + // Stack: [..., uv.run]. luaL_ref pops the top. + if (c.lua_type(L, -1) != lua_bridge.T_FUNCTION) { + c.lua_settop(L, c.lua_gettop(L) - 1); return RuntimeError.LuaInitFailed; } - self.uv_loop_alive_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); self.uv_run_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); } @@ -1328,6 +1293,176 @@ test "scheduler: yielding handler is resumed by libuv" { try testing.expectEqualStrings("awake", results[1].ok); } +// Contract-violation guard: a handler that yields WITHOUT arming any +// libuv work can never be resumed. Under the run-to-completion model, +// `uv.run` drains immediately (nothing to wait on) and the scheduler +// must surface the still-suspended coroutine as a per-call error +// rather than hanging the process. A sibling well-behaved call in the +// same batch must still succeed. +test "scheduler: handler that yields without arming libuv work is surfaced as an error" { + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + const luarocks_rt = bootstrapRealRuntime(rt) catch |e| return e; + defer luarocks_rt.deinit(); + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + const source = + \\panto.register_tool { + \\ name = "abandon", description = "yields into the void", + \\ schema = { type = "object" }, + \\ handler = function() coroutine.yield(); return "never" end, + \\} + \\panto.register_tool { + \\ name = "fine", description = "sync ok", + \\ schema = { type = "object" }, + \\ handler = function() return "ok" end, + \\} + ; + const path = try writeTempScript(tmp.dir, "abandon.lua", source); + defer testing.allocator.free(path); + try rt.loadExtension(path, null); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{ + .{ .tool_name = "abandon", .input = "{}" }, + .{ .tool_name = "fine", .input = "{}" }, + }; + var results: [2]panto.ToolCallResult = .{ + .{ .err = error.SourceDroppedCall }, + .{ .err = error.SourceDroppedCall }, + }; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer for (results) |r| switch (r) { + .ok => |b| testing.allocator.free(b), + .err => {}, + }; + + try testing.expect(std.mem.startsWith(u8, results[0].ok, "panto-lua: tool 'abandon' failed:")); + try testing.expect(std.mem.indexOf(u8, results[0].ok, "still suspended") != null); + try testing.expectEqualStrings("ok", results[1].ok); +} + +/// Absolute path to the repo's `agent/tools` directory, derived from +/// the build-time repo root (the test runner's cwd is unreliable). +/// Caller owns the returned slice. Skips the test if the directory +/// isn't present. +fn findAgentToolsDir() ![]const u8 { + const versions = @import("versions"); + const tools = try std.fs.path.join(testing.allocator, &.{ versions.repo_root, "agent", "tools" }); + errdefer testing.allocator.free(tools); + const probe = try std.fs.path.join(testing.allocator, &.{ tools, "shell.lua" }); + defer testing.allocator.free(probe); + std.Io.Dir.cwd().access(testing.io, probe, .{}) catch { + testing.allocator.free(tools); + return error.SkipZigTest; + }; + return tools; +} + +fn bootstrapRealRuntime(rt: *LuaRuntime) !*@import("luarocks_runtime.zig").LuarocksRuntime { + const home_z = std.c.getenv("PANTO_HOME") orelse return error.SkipZigTest; + const panto_home_env = std.mem.sliceTo(home_z, 0); + const manifest = @import("manifest.zig"); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const so_path = try std.fmt.bufPrint( + &path_buf, + "{s}/rocks/lua-{s}/lib/lua/{s}/luv.so", + .{ panto_home_env, manifest.lua_version, manifest.lua_short_version }, + ); + std.Io.Dir.cwd().access(testing.io, so_path, .{}) catch return error.SkipZigTest; + + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("PANTO_HOME", panto_home_env); + const luarocks_runtime = @import("luarocks_runtime.zig"); + const luarocks_rt = try luarocks_runtime.bootstrap( + testing.allocator, + testing.io, + &env, + rt.L, + "/usr/bin/true", + ); + try rt.installScheduler(); + return luarocks_rt; +} + +// Reproduction: two REAL `std.shell` calls in one batch. shell.lua +// uses spawn + two pipes + a timeout timer (3+ libuv events per call) +// and resumes its own coroutine from a libuv callback. This exercises +// the exact C-resume topology that the synthetic `timer_say` test does +// not (spawn, dual pipes, spill-file fs calls). Skipped without luv. +test "scheduler: two real std.shell calls in one batch do not deadlock" { + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + const luarocks_rt = bootstrapRealRuntime(rt) catch |e| return e; + defer luarocks_rt.deinit(); + + const tools_dir = try findAgentToolsDir(); + defer testing.allocator.free(tools_dir); + const shell_path = try std.fs.path.join(testing.allocator, &.{ tools_dir, "shell.lua" }); + defer testing.allocator.free(shell_path); + try rt.loadTool(shell_path, tools_dir); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{ + .{ .tool_name = "std.shell", .input = "{\"command\":\"echo first; sleep 0.05; echo first-done\"}" }, + .{ .tool_name = "std.shell", .input = "{\"command\":\"echo second; sleep 0.05; echo second-done\"}" }, + }; + var results: [2]panto.ToolCallResult = .{ + .{ .err = error.SourceDroppedCall }, + .{ .err = error.SourceDroppedCall }, + }; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer for (results) |r| switch (r) { + .ok => |b| testing.allocator.free(b), + .err => {}, + }; + + try testing.expect(std.mem.indexOf(u8, results[0].ok, "first-done") != null); + try testing.expect(std.mem.indexOf(u8, results[1].ok, "second-done") != null); +} + +// Reproduction: two REAL `std.read` calls in one batch. read.lua uses +// only synchronous `uv.fs_*` calls and never yields, so both should +// complete during dispatch without entering the drive loop. +test "scheduler: two real std.read calls in one batch do not deadlock" { + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + const luarocks_rt = bootstrapRealRuntime(rt) catch |e| return e; + defer luarocks_rt.deinit(); + + const tools_dir = try findAgentToolsDir(); + defer testing.allocator.free(tools_dir); + const read_path = try std.fs.path.join(testing.allocator, &.{ tools_dir, "read.lua" }); + defer testing.allocator.free(read_path); + try rt.loadTool(read_path, tools_dir); + + const in0 = try std.fmt.allocPrint(testing.allocator, "{{\"path\":\"{s}/shell.lua\"}}", .{tools_dir}); + defer testing.allocator.free(in0); + const in1 = try std.fmt.allocPrint(testing.allocator, "{{\"path\":\"{s}/read.lua\"}}", .{tools_dir}); + defer testing.allocator.free(in1); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{ + .{ .tool_name = "std.read", .input = in0 }, + .{ .tool_name = "std.read", .input = in1 }, + }; + var results: [2]panto.ToolCallResult = .{ + .{ .err = error.SourceDroppedCall }, + .{ .err = error.SourceDroppedCall }, + }; + + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer for (results) |r| switch (r) { + .ok => |b| testing.allocator.free(b), + .err => {}, + }; + + try testing.expect(results[0].ok.len > 0); + try testing.expect(results[1].ok.len > 0); +} + test "loadExtension: duplicate tool name from a second extension errors" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); |
