diff options
| author | t <t@tjp.lol> | 2026-06-10 10:06:07 -0600 |
|---|---|---|
| committer | t <t@tjp.lol> | 2026-06-10 12:17:27 -0600 |
| commit | bb85e867bc938138f29fb45230169af1ba60761c (patch) | |
| tree | 5ad1cf50274c61ccfffab7048b4a4e28003d82b5 | |
| parent | d26890bbc52e9f0050db40f208763a7f2b714ddd (diff) | |
Add addUserText helper; update all call sites
Conversation.addUserMessage now takes a []ContentBlock (symmetric with
addAssistantMessage). Introduce a thin addUserText wrapper in agent.zig
for the plain-text case and update every call site in agent.zig and
anthropic_messages_json.zig accordingly.
| -rw-r--r-- | build.zig | 53 | ||||
| -rw-r--r-- | build.zig.zon | 3 | ||||
| -rw-r--r-- | build/gen_panto_so_embed.zig | 42 | ||||
| -rw-r--r-- | docs/step19-cli-panto-module-plan.md | 144 | ||||
| -rw-r--r-- | libpanto-lua/build.zig | 143 | ||||
| -rw-r--r-- | libpanto-lua/build.zig.zon | 26 | ||||
| -rw-r--r-- | libpanto-lua/libpanto-lua-0.0.0-1.rockspec | 63 | ||||
| -rw-r--r-- | libpanto-lua/src/module.zig | 2275 | ||||
| -rw-r--r-- | libpanto/src/agent.zig | 35 | ||||
| -rw-r--r-- | libpanto/src/anthropic_messages_json.zig | 36 | ||||
| -rw-r--r-- | libpanto/src/conversation.zig | 45 | ||||
| -rw-r--r-- | libpanto/src/file_system_jsonl_store.zig | 11 | ||||
| -rw-r--r-- | libpanto/src/openai_chat_json.zig | 30 | ||||
| -rw-r--r-- | libpanto/src/provider_anthropic_messages.zig | 32 | ||||
| -rw-r--r-- | libpanto/src/provider_openai_chat.zig | 30 | ||||
| -rw-r--r-- | libpanto/src/public.zig | 5 | ||||
| -rw-r--r-- | src/lua_bridge.zig | 163 | ||||
| -rw-r--r-- | src/lua_event_bridge.zig | 20 | ||||
| -rw-r--r-- | src/lua_runtime.zig | 83 | ||||
| -rw-r--r-- | src/luarocks_runtime.zig | 40 | ||||
| -rw-r--r-- | src/main.zig | 7 | ||||
| -rw-r--r-- | src/system_prompt.zig | 15 |
22 files changed, 3168 insertions, 133 deletions
@@ -13,6 +13,18 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); + // The native Lua C-module (`libpanto-lua`). We embed its compiled + // `panto.so` into the panto binary and stage it onto the embedded + // VM's `package.cpath` at bootstrap, so `require('panto')` resolves + // to the native agent/stream module on a cold, network-less machine + // (the guaranteed initial module load — see + // docs/step19-cli-panto-module-plan.md). + const panto_lua_dep = b.dependency("panto_lua", .{ + .target = target, + .optimize = optimize, + }); + const panto_so_embed_path = generatePantoSoEmbed(b, panto_lua_dep); + // TOML parser (used by the CLI for ~/.config/panto/models.toml). // We disable the upstream's optional thread-pool dep — we only need // sequential parsing of a small config file. @@ -76,6 +88,9 @@ pub fn build(b: *std.Build) void { exe_mod.addImport("embedded_agent", b.createModule(.{ .root_source_file = agent_embed_path, })); + exe_mod.addImport("embedded_panto_so", b.createModule(.{ + .root_source_file = panto_so_embed_path, + })); // Link Lua. We can't use a static library here because the // standard archive-linking behavior drops object files whose // symbols nothing in our code references — e.g. `lua_settable`, @@ -136,6 +151,9 @@ pub fn build(b: *std.Build) void { test_mod.addImport("embedded_agent", b.createModule(.{ .root_source_file = agent_embed_path, })); + test_mod.addImport("embedded_panto_so", b.createModule(.{ + .root_source_file = panto_so_embed_path, + })); addLuaSources(test_mod, lua_src, lua_anchor_path); test_mod.addIncludePath(lua_src.path("src")); test_mod.linkLibrary(lua_repl); @@ -421,6 +439,41 @@ fn generateLuaHeadersEmbed( return out_zig; } +/// Codegen step: embed the compiled `libpanto-lua` `panto.so` into a Zig +/// module the bootstrap stages onto the embedded VM's `package.cpath`. +/// +/// We address the artifact via `dep.artifact("panto")` (the `addLibrary` +/// Compile step named `panto` in `libpanto-lua/build.zig`) and embed its +/// emitted binary by bytes — name-agnostic, so the upstream `libpanto.so`/ +/// `libpanto.dylib` filename is irrelevant; we always stage it as +/// `panto.so`. The `.so` is copied next to the generated module so +/// `@embedFile` can resolve it. +fn generatePantoSoEmbed( + b: *std.Build, + panto_lua_dep: *std.Build.Dependency, +) std.Build.LazyPath { + const so_path = panto_lua_dep.artifact("panto").getEmittedBin(); + + const tool = b.addExecutable(.{ + .name = "gen-panto-so-embed", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_panto_so_embed.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), + }); + + const run = b.addRunArtifact(tool); + // The generated module `@embedFile`s exactly this name. + run.addArg("panto.so"); + const gen_file = run.addOutputFileArg("embedded_panto_so.zig"); + + const wf = b.addWriteFiles(); + const out_zig = wf.addCopyFile(gen_file, "embedded_panto_so.zig"); + _ = wf.addCopyFile(so_path, "panto.so"); + return out_zig; +} + // Lua 5.4.x core VM + standard library. Excludes lua.c (interpreter entry // point) and luac.c (compiler entry point). const lua_files = [_][]const u8{ diff --git a/build.zig.zon b/build.zig.zon index 19feb70..6a04444 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -6,6 +6,9 @@ .panto = .{ .path = "libpanto", }, + .panto_lua = .{ + .path = "libpanto-lua", + }, .lua_src = .{ .url = "https://www.lua.org/ftp/lua-5.4.7.tar.gz", .hash = "N-V-__8AAIMvFABt-Qcpk24RD10ldEN743D8Q2e19Er8x3dJ", diff --git a/build/gen_panto_so_embed.zig b/build/gen_panto_so_embed.zig new file mode 100644 index 0000000..37b8329 --- /dev/null +++ b/build/gen_panto_so_embed.zig @@ -0,0 +1,42 @@ +//! Build-time codegen: emit a Zig module exposing the compiled +//! `libpanto-lua` shared object (`panto.so`) as embedded bytes. Bootstrap +//! writes these to `<tree>/lib/lua/<short>/panto.so` on first run so the +//! embedded VM's `require('panto')` resolves to the native module on a +//! cold machine with no network — the guaranteed initial module load. +//! +//! Invoked from `build.zig`: +//! +//! gen-panto-so-embed <embed-name> <out-file> +//! +//! `<embed-name>` is the filename the generated module `@embedFile`s; the +//! actual `.so` is materialized alongside the generated file under that +//! name via `addCopyFile` so `@embedFile` can reach it. + +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); + const embed_name = args.next() orelse return error.MissingEmbedName; + const out_path = args.next() orelse return error.MissingOutPath; + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.print( + \\//! Auto-generated. Do not edit. See build/gen_panto_so_embed.zig. + \\ + \\/// The compiled `libpanto-lua` shared object, embedded so the + \\/// bootstrap can stage it onto the embedded VM's `package.cpath`. + \\pub const bytes: []const u8 = @embedFile("{s}"); + \\ + , .{embed_name}); + + try w.flush(); +} diff --git a/docs/step19-cli-panto-module-plan.md b/docs/step19-cli-panto-module-plan.md new file mode 100644 index 0000000..f162403 --- /dev/null +++ b/docs/step19-cli-panto-module-plan.md @@ -0,0 +1,144 @@ +# Step 19 plan: CLI consumes `libpanto-lua` as a Lua module (Option A) + +Goal: inside the panto CLI's embedded Lua VM, `require('panto')` returns the +**native** `libpanto-lua` agent/stream table **plus** `panto.ext`. The CLI +obtains that table only through Lua's normal module machinery (`require`) — it +never calls `luaopen_panto` directly and knows nothing of the module's +internals. The single invariant relied upon: `luaopen_panto` returns a fresh +table (already true). + +## Mechanism + +1. **Embed `panto.so` in the CLI binary and stage it at bootstrap** into the + versioned rocks tree's C-module dir (`<tree>/lib/lua/5.4/panto.so`), which + is already on `package.cpath` (see `configurePackagePaths`). The staged + `.so` leaves `lua_*` undefined and resolves them against the host binary at + `dlopen` time — identical to how `luv.so` already works (`exe.rdynamic = + true`). + +2. **After luarocks bootstrap configures cpath**, the CLI: + ```lua + local panto = require('panto') -- finds staged panto.so + panto.ext = <ext table built in Zig> -- augment host's fresh copy + package.preload['panto'] = function() return panto end -- any later require gets ext too + ``` + `require` caches the augmented table in `package.loaded['panto']`, and the + preload entry (searcher slot 1) makes any *future* `require('panto')` — e.g. + from an extension — return the same augmented table without re-hitting cpath. + +## File-by-file changes + +### `build.zig` (CLI) +- Add `libpanto-lua` as a path dependency (`build.zig.zon`). +- Get its compiled shared object via `dep.artifact("panto").getEmittedBin()`. +- New codegen step `gen_panto_so_embed.zig`: emit a Zig module that + `@embedFile`s the `.so` bytes (binary embed is fine). Mirror + `generateLuaHeadersEmbed`. Materialize the `.so` next to the generated file + via `addCopyFile` so `@embedFile` can reach it. +- Add the embed module import to `exe_mod` and `test_mod` as + `embedded_panto_so`. + +### new `src/embedded_panto_so` import +- A single `pub const bytes: []const u8 = @embedFile("panto.so");` style module + (generated). Name: `embedded_panto_so`. + +### `src/luarocks_runtime.zig` +- New step `stagePantoModule`: write `embedded_panto_so.bytes` to + `<lib_lua_dir>/panto.so` via `writeIfDifferent`. Call it from `bootstrap` + after the tree dirs exist (alongside `stageLuaHeaders`). +- `lib_lua_dir` must exist (create if missing) before writing. + +### `src/lua_bridge.zig` +- **Stop building the native `panto` table and stop installing the preload + loader.** `install()` becomes "create the registrations tables + build the + `ext` table, stash `ext` in a registry slot" — it no longer fabricates a + `panto` table or touches `package.preload`. +- Keep all `ext` thunks (`register_tool`/`register_command`/`on`/`emit`) and + the registrations-table machinery unchanged. +- `pushPantoTable` is repurposed (or replaced by `pushExtTable`): internal Zig + code that today adds members to `panto` (`installEmit`, `_record_result`, + wrapper closure) must target the **augmented native** table after it is + obtained. Decision below. + +### `src/lua_runtime.zig` +- `create()` no longer relies on a Zig-built `panto` table. Split startup: + - `create()` opens the state, `openlibs`, builds the `ext` table + regs + tables (via the slimmed `lua_bridge.install`), creates the EventBridge. + **Defer** `installEmit` until the native table exists. + - New `installPantoModule(self)`: run the `require('panto')` + attach-`ext` + + set-`preload` sequence in Zig (a `luaL_dostring`-style snippet plus stack + surgery), then stash the augmented table in the registry slot so + `pushPantoTable` keeps working, then `installEmit`. +- `installScheduler` already runs after bootstrap; `installPantoModule` runs + between bootstrap and `installScheduler` (it needs cpath configured; the + scheduler's `_record_result`/wrapper need the table present). + +### `src/main.zig` +- After `luarocks_runtime.bootstrap(...)` returns (cpath configured, `panto.so` + staged) and before `rt.installScheduler()`, call `try rt.installPantoModule()`. + +## Sequencing (the hazard) + +Today, ordering is: `create` (builds table + preload + installEmit) → +bootstrap → installScheduler (adds `_record_result`, wrapper, caches uv.run). + +New ordering: +1. `create` — state, libs, `ext` + regs tables, EventBridge. **No table, no + preload, no emit yet.** +2. `bootstrap` — stages `panto.so`, configures cpath/path, installs luv. +3. `installPantoModule` — `require('panto')`, attach `ext`, set preload, stash + in registry slot, `installEmit`. +4. `installScheduler` — `_record_result` + wrapper closure (both reach the now- + augmented native table via the registry slot) + cache uv.run. +5. extension loading — `require('panto')` returns the augmented table. + +Everything that previously assumed "the `panto` table exists from `create`" +moves to step 3. The only consumers are `installEmit` and the scheduler, both +already post-bootstrap-or-movable. + +## Open decision: registry slot semantics + +`lua_bridge.panto_table_key` currently holds the Zig-built table. Two choices: + +- **(chosen)** Keep the slot; in `installPantoModule`, after building the + augmented table, `lua_rawsetp` it into `panto_table_key`. `pushPantoTable` + is unchanged and now yields the native+ext table. `installEmit` / + `_record_result` / wrapper all keep using `pushPantoTable`. Minimal churn. +- Alternative: split into `pushExtTable` (for `ext` members) vs `pushPantoTable` + (whole module). More precise but more edits. Not needed — `ext` lives on the + module table, so one slot pointing at the module suffices. + +## Tests to update / add + +- `lua_bridge.zig` tests assume `install` builds a `panto` table reachable via + `require('panto')`/`pushPantoTable`. These must change: either keep a tiny + test-only table builder, or rewrite the tests to drive `ext` directly. Since + `ext` is what those tests actually exercise (register_tool/command/on), point + them at the `ext` table via a test helper that fabricates a minimal module + table `{ ext = <ext> }` and stashes it — preserving coverage without the + native `.so`. +- `lua_runtime.zig` tests call `LuaRuntime.create` and `loadExtension` without + bootstrap, so `require('panto')` would fail there (no staged `.so`, no + preload). Add a test-only `installTestPantoTable(self)` that fabricates the + `{ ext = <ext> }` module table and registers the preload loader, mirroring + what the old `create` did — so the runtime tests keep passing without a real + native module. `writeTempScript` already prepends `require('panto')`. +- New integration check (best-effort, may be manual): in a built CLI, + `require('panto')` yields a table with both `agent` (native) and `ext`. + +## Risks + +- **Binary size**: embedding `panto.so` (~5.6 MB debug) bloats the CLI binary. + ReleaseFast/strip will shrink it; acceptable (luv and the agent tree are also + embedded). Confirm size is tolerable; consider stripping the `.so` in the + module's ReleaseSafe/Fast build. +- **Two Lua builds must be ABI-identical**: the CLI's statically-linked Lua + (5.4.7) and the headers `panto.so` was compiled against (5.4.7, same pin). + Both come from the same `lua_src` tarball hash — consistent by construction. +- **`dep.artifact("panto")`**: `libpanto-lua/build.zig` currently emits the + `.so` through `addLibrary` + a manual install-file rename. `artifact("panto")` + returns the `addLibrary` Compile step (named `panto`), whose `getEmittedBin()` + is `libpanto.so`/`libpanto.dylib`. Embedding by bytes is name-agnostic; we + stage it as `panto.so`. Verify the dependency exposes the artifact (it must be + `b.installArtifact`'d or at least addressable; may need a small tweak to + `libpanto-lua/build.zig` to expose it cleanly to dependents). diff --git a/libpanto-lua/build.zig b/libpanto-lua/build.zig new file mode 100644 index 0000000..8bfaf88 --- /dev/null +++ b/libpanto-lua/build.zig @@ -0,0 +1,143 @@ +const std = @import("std"); + +/// `libpanto-lua` — a native Lua 5.4 C-module implemented in pure Zig. +/// +/// Emits a loadable `panto.so` (no `lib` prefix) exporting `luaopen_panto`, +/// discovered on `package.cpath` and loaded by `require('panto')`. The +/// module `@cImport`s the Lua 5.4 headers and calls the Zig `libpanto` API +/// directly — no C translation units in this package, no dependency on +/// `libpanto-c`. +/// +/// Targets Lua 5.4 only (see `docs/libpanto-bindings.md`). A Lua C-module +/// does not link the Lua library: the host interpreter supplies every +/// `lua_*` symbol at load time, so we link only against the headers and +/// resolve the symbols via dynamic lookup at runtime. +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const panto_dep = b.dependency("panto", .{ + .target = target, + .optimize = optimize, + }); + const lua_src = b.dependency("lua_src", .{}); + + const mod = b.createModule(.{ + .root_source_file = b.path("src/module.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + mod.addImport("panto", panto_dep.module("panto")); + // Lua headers for `@cImport`. Headers only — see the note above. + mod.addIncludePath(lua_src.path("src")); + + const lib = b.addLibrary(.{ + .name = "panto", + .root_module = mod, + .linkage = .dynamic, + }); + // Lua modules export a fixed-name init function; keep it in the + // dynamic symbol table so the host's `require` can find it. + lib.rdynamic = true; + // The `lua_*` / `luaL_*` symbols are provided by the host + // interpreter at `dlopen` time, not by this module. Allow them to be + // left undefined at link time and resolved dynamically when loaded. + allowUndefinedHostSymbols(lib, target); + + // Register the Compile step as a named artifact so dependents (the + // panto CLI build) can address it via `dep.artifact("panto")` to embed + // the compiled `.so`. This installs `libpanto.so`/`libpanto.dylib` + // under the default `lib/` name; the bare-name staging below produces + // the `panto.so` a Lua `require` needs. + b.installArtifact(lib); + + // A Lua C-module must be named exactly `panto.so` (no `lib` prefix, + // no version suffix) to be found as `require('panto')` on `cpath`. + // `addLibrary` produces `libpanto.so` (or `.dylib`); install it under + // the bare module name into `lib/`. + const install_so = b.addInstallFileWithDir( + lib.getEmittedBin(), + .lib, + "panto.so", + ); + b.getInstallStep().dependOn(&install_so.step); + + // Unit tests. The test binary is an ordinary executable that links a + // real Lua so the C symbols resolve; we compile the Lua sources into + // it directly (see `addLuaForTests`). + const test_mod = b.createModule(.{ + .root_source_file = b.path("src/module.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + test_mod.addImport("panto", panto_dep.module("panto")); + test_mod.addIncludePath(lua_src.path("src")); + addLuaForTests(test_mod, lua_src); + + const unit_tests = b.addTest(.{ + .name = "panto-lua-tests", + .root_module = test_mod, + }); + const run_unit_tests = b.addRunArtifact(unit_tests); + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&run_unit_tests.step); +} + +/// On macOS the linker rejects undefined symbols by default; a Lua +/// C-module relies on the host interpreter to provide every `lua_*` +/// symbol at `dlopen` time. Pass the flag that defers resolution to load +/// time. On ELF (Linux/BSD) shared objects already allow undefined +/// symbols resolved by the loader, so nothing is needed. +fn allowUndefinedHostSymbols( + lib: *std.Build.Step.Compile, + target: std.Build.ResolvedTarget, +) void { + switch (target.result.os.tag) { + .macos => { + lib.linker_allow_shlib_undefined = true; + lib.root_module.addCMacro("LUA_USE_MACOSX", ""); + }, + .linux => lib.root_module.addCMacro("LUA_USE_LINUX", ""), + .freebsd, .netbsd, .openbsd => lib.root_module.addCMacro("LUA_USE_POSIX", ""), + else => {}, + } +} + +/// The Lua source files needed to compile a self-contained Lua into the +/// test binary so the `lua_*` symbols resolve (the standalone `.so` +/// borrows them from a host instead). This is the same `core + lib + aux` +/// set the CLI compiles, minus `lua.c`/`luac.c` (the standalone front +/// ends, which carry their own `main`). +const lua_files = [_][]const u8{ + // core + "lapi.c", "lcode.c", "lctype.c", "ldebug.c", "ldo.c", + "ldump.c", "lfunc.c", "lgc.c", "llex.c", "lmem.c", + "lobject.c", "lopcodes.c", "lparser.c", "lstate.c", "lstring.c", + "ltable.c", "ltm.c", "lundump.c", "lvm.c", "lzio.c", + // lib + "lauxlib.c", "lbaselib.c", "lcorolib.c", "ldblib.c", "liolib.c", + "lmathlib.c", "loadlib.c", "loslib.c", "lstrlib.c", "ltablib.c", + "lutf8lib.c", "linit.c", +}; + +fn addLuaForTests(mod: *std.Build.Module, lua_src: *std.Build.Dependency) void { + const cflags = [_][]const u8{ + "-std=gnu99", + "-Wall", + "-Wextra", + "-Wno-unused-parameter", + }; + mod.addCSourceFiles(.{ + .root = lua_src.path("src"), + .files = &lua_files, + .flags = &cflags, + }); + switch (mod.resolved_target.?.result.os.tag) { + .macos => mod.addCMacro("LUA_USE_MACOSX", ""), + .linux => mod.addCMacro("LUA_USE_LINUX", ""), + .freebsd, .netbsd, .openbsd => mod.addCMacro("LUA_USE_POSIX", ""), + else => {}, + } +} diff --git a/libpanto-lua/build.zig.zon b/libpanto-lua/build.zig.zon new file mode 100644 index 0000000..d685c13 --- /dev/null +++ b/libpanto-lua/build.zig.zon @@ -0,0 +1,26 @@ +.{ + .fingerprint = 0xe0ed1e60c285f54f, + .name = .panto_lua, + .version = "0.0.0", + .dependencies = .{ + .panto = .{ + .path = "../libpanto", + }, + // Lua 5.4 source — used only for its headers (`lua.h`, + // `lauxlib.h`, `lualib.h`) at `@cImport` time. A Lua C-module + // does NOT link the Lua library: the host interpreter provides + // every `lua_*`/`luaL_*` symbol at `dlopen` time, resolved via + // dynamic lookup. We pin the same tarball the CLI uses so the + // standalone module and the embedded VM build against identical + // 5.4 headers. + .lua_src = .{ + .url = "https://www.lua.org/ftp/lua-5.4.7.tar.gz", + .hash = "N-V-__8AAIMvFABt-Qcpk24RD10ldEN743D8Q2e19Er8x3dJ", + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + }, +} diff --git a/libpanto-lua/libpanto-lua-0.0.0-1.rockspec b/libpanto-lua/libpanto-lua-0.0.0-1.rockspec new file mode 100644 index 0000000..c14353f --- /dev/null +++ b/libpanto-lua/libpanto-lua-0.0.0-1.rockspec @@ -0,0 +1,63 @@ +-- LuaRocks rockspec for `libpanto-lua`: the native Lua 5.4 binding for +-- libpanto, built in pure Zig. +-- +-- Distribution model (see docs/libpanto-bindings.md): this is the *source* +-- rock — universal, buildable on any OS/arch that has a Zig toolchain. +-- LuaRocks prefers a matching pre-built `.PLATFORM.rock` when one is +-- published and falls back to building this source rock otherwise (the +-- pip "wheel-or-sdist" model). Pre-built binary rocks per platform are a +-- later step; no rockspec change is needed to add them. +-- +-- Build: `build.type = "command"` shells out to `zig build` (so the build +-- dependency is Zig, declared informally below since LuaRocks can't fetch +-- a compiler), then stages the emitted `panto.so` into `$(LIBDIR)` as the +-- bare module name `panto` — discovered by `require('panto')` on cpath. + +rockspec_format = "3.0" +package = "libpanto-lua" +version = "0.0.0-1" + +source = { + url = "git+https://github.com/earendil-works/pantograph.git", + -- The module lives in the `libpanto-lua/` subdirectory of the repo. + dir = "pantograph/libpanto-lua", +} + +description = { + summary = "Native Lua 5.4 bindings for libpanto (pure Zig).", + detailed = [[ + A loadable Lua 5.4 C-module exposing libpanto's agent + pull-stream + API. require('panto') yields panto.agent{...}; an Agent runs turns + that stream Events, registers Lua tool handlers (driven by libuv via + luv), swaps provider config, manages the system prompt, and compacts. + Implemented entirely in Zig (no C translation units); the .so is + emitted by `zig build`. + ]], + homepage = "https://github.com/earendil-works/pantograph", + license = "MIT", + labels = { "ai", "llm", "agent", "libpanto" }, +} + +-- Lua 5.4 only (Lua has no stable ABI across minor versions; a module +-- built for 5.4 will not load in 5.3 or 5.5). `luv` is required for tool +-- dispatch: tool handlers run as coroutines driven by `uv.run`. +dependencies = { + "lua >= 5.4, < 5.5", + "luv", +} + +-- Zig is required at build time. LuaRocks has no notion of a Zig toolchain +-- dependency, so this is enforced by the build_command failing loudly if +-- `zig` is absent rather than by a declared dependency. + +build = { + type = "command", + -- Build the shared object. `zig build` reads ./build.zig and emits + -- zig-out/lib/panto.so (see build.zig: the install step renames the + -- artifact to the bare `panto.so` a Lua C-module requires). + build_command = "zig build -Doptimize=ReleaseFast", + -- Stage the emitted module into LuaRocks' library dir under the bare + -- name `panto` so `require('panto')` resolves it on cpath. $(LIBDIR) + -- is substituted by LuaRocks at install time. + install_command = "cp zig-out/lib/panto.so $(LIBDIR)/panto.so", +} diff --git a/libpanto-lua/src/module.zig b/libpanto-lua/src/module.zig new file mode 100644 index 0000000..7f7f838 --- /dev/null +++ b/libpanto-lua/src/module.zig @@ -0,0 +1,2275 @@ +//! `libpanto-lua` — a native Lua 5.4 C-module for `libpanto`, in pure Zig. +//! +//! This file `@cImport`s the Lua 5.4 C headers and builds the module table +//! plus two `luaL_newmetatable` userdata types — `Agent` and `Stream` — +//! against the translated C types, calling the **Zig** `libpanto` API +//! directly (no `libpanto-c` dependency, no C translation units). The +//! `build.zig` emits a loadable `panto.so` exporting `luaopen_panto`, +//! discovered on `package.cpath` and loaded by `require('panto')`. +//! +//! The unifying streaming contract (see `docs/libpanto-bindings.md`) maps +//! onto Lua exactly like Python — Lua is a pull-iterator language too: +//! +//! | layer | progress / terminal | exhausted | failure | +//! | ----- | --------------------- | ------------- | ----------------------- | +//! | Lua | event table (yielded) | iterator ends | error(...) / pcall false | +//! +//! - `Event` -> push the event as a Lua table and return it. +//! - `null` -> return `nil`, ending the `for` loop. +//! - `error.X` -> `lua_error` with a mapped message, catchable via `pcall`. +//! +//! ## The surface +//! +//! local panto = require('panto') +//! local agent = panto.agent { +//! api_style = "openai_chat", -- or "anthropic_messages" +//! api_key = "...", +//! base_url = "https://...", +//! model = "...", +//! max_tokens = 64000, -- optional +//! reasoning = "medium", -- optional (openai_chat) +//! thinking = "enabled", -- optional (anthropic_messages) +//! -- ...see config.zig parsing below for the complete set... +//! } +//! +//! agent:register_tool { -- give the model a tool +//! name = "...", description = "...", +//! schema = { type = "object", ... }, +//! handler = function(input) ... return "result" end, +//! } +//! +//! agent:set_config { ... } -- swap provider/model/policy +//! agent:add_system_message("...") -- append to the system prompt +//! agent:set_system_prompt("...") -- replace it +//! agent:compact() -- summarize older turns +//! print(agent:session_id()) +//! +//! for ev in agent:run("hello"):events() do +//! if ev.type == "content_delta" then io.write(ev.delta) end +//! end +//! +//! ## Tools, coroutines, and luv +//! +//! Tool handlers run as Lua coroutines driven by libuv (via the `luv` +//! rock, a hard dependency of this package). The contract mirrors the CLI +//! runtime exactly: a handler may block **only** by yielding on a pending +//! libuv operation whose callback resumes it, so a single +//! `uv.run("default")` pass guarantees every handler has finished. A +//! purely synchronous handler runs to completion on its first resume and +//! needs no event loop. +//! +//! libpanto delivers every Lua-tool call for a turn in one `invoke_batch` +//! on a single thread (the source-grouped dispatch contract); that thread +//! is *not* the host thread, but the host thread is parked inside +//! `Stream.next()` for the whole batch, so only one thread ever touches +//! the `lua_State` at a time. We start one coroutine per call and drive +//! `uv.run` to completion. +//! +//! ## Lua-version pinning +//! +//! Lua has no stable ABI; this module is built against 5.4 headers and the +//! `luaL_checkversion` call in `luaopen_panto` makes a wrong-version load +//! fail loud. 5.3/5.2/5.1(LuaJIT)/5.5 are out of scope for v1. + +const std = @import("std"); +const panto = @import("panto"); + +pub const c = @cImport({ + @cInclude("lua.h"); + @cInclude("lauxlib.h"); + @cInclude("lualib.h"); +}); + +// translate_c surfaces Lua's `#define`d type tags as inline functions that +// aren't usable as comptime constants in switch prongs; mirror the +// canonical 5.4 values as clean `c_int`s. +const T_NIL: c_int = 0; +const T_BOOLEAN: c_int = 1; +const T_NUMBER: c_int = 3; +const T_STRING: c_int = 4; +const T_TABLE: c_int = 5; +const T_FUNCTION: c_int = 6; + +// LUAI_MAXSTACK defaults to 1_000_000 on 64-bit, so the registry pseudo- +// index is -1_001_000 (matches lua.h). Parity with `src/lua_bridge.zig`. +const LUA_REGISTRYINDEX: c_int = -1001000; + +const c_allocator = std.heap.c_allocator; + +const SOURCE_NAME = "panto-lua"; + +// =========================================================================== +// Process-global I/O context +// =========================================================================== +// +// `libpanto` needs a process-global HTTP client (`panto.init`) and a +// `std.Io` to drive blocking provider reads. A standalone Lua interpreter +// has no `std.process.Init`, so we own a `std.Io.Threaded` here and +// initialize it lazily on first `panto.agent{}`, tearing it down when the +// last Agent is collected. Single-interpreter assumption: a standalone +// `require('panto')` runs in one `lua_State` and the embedded CLI VM is +// single-threaded; we do not guard the refcount with an atomic. + +const IoContext = struct { + threaded: std.Io.Threaded, + io: std.Io, + agent_refs: usize = 0, + initialized: bool = false, +}; + +var io_ctx: IoContext = .{ .threaded = undefined, .io = undefined }; + +fn retainIo() void { + if (!io_ctx.initialized) { + io_ctx.threaded = .init(c_allocator, .{}); + io_ctx.io = io_ctx.threaded.io(); + panto.init(c_allocator, io_ctx.io); + io_ctx.initialized = true; + } + io_ctx.agent_refs += 1; +} + +fn releaseIo() void { + if (io_ctx.agent_refs == 0) return; + io_ctx.agent_refs -= 1; + if (io_ctx.agent_refs == 0 and io_ctx.initialized) { + panto.deinit(); + io_ctx.threaded.deinit(); + io_ctx.initialized = false; + } +} + +// =========================================================================== +// Userdata payloads +// =========================================================================== + +const AGENT_MT = "panto.Agent"; +const STREAM_MT = "panto.Stream"; +const CONV_MT = "panto.Conversation"; + +/// Boxed `*Agent` userdata. +/// +/// `libpanto` borrows `*const Config`, so we own the live `Config` and +/// every string it points at, kept alive for the agent's lifetime. A +/// `set_config` swap pins a *new* config without freeing the old one +/// eagerly: an in-flight turn (or a concurrent tool worker) may still be +/// reading the old snapshot, so old configs and their strings accumulate +/// in `string_arena`/`old_configs` and are freed together at agent +/// teardown. (Bounded by the number of `set_config` calls — a tiny, +/// deliberate retain-until-deinit.) +/// +/// A `NullStore` is embedded so a standalone consumer needs no session +/// wiring; the store is borrowed by the agent and must outlive it, so it +/// lives in the same box. +/// +/// The Lua tool source is created lazily on the first `register_tool` and +/// registered with the agent then. +const AgentBox = struct { + agent: *panto.Agent, + /// The live config the agent reads. Heap-pinned so `set_config` can + /// swap the pointee the agent holds without moving this box. + config: *panto.Config, + /// Superseded configs, freed at teardown (see the struct doc). + old_configs: std.ArrayList(*panto.Config), + store_impl: panto.NullStore, + /// Arena owning every config string (api_key/base_url/model/prompt…) + /// across the live config and all superseded ones. Freed at teardown. + string_arena: std.heap.ArenaAllocator, + /// The lazily-created Lua tool source (null until first register_tool). + tools: ?*LuaToolSource, + closed: bool = false, + + fn deinit(self: *AgentBox, L: *c.lua_State) void { + if (self.closed) return; + self.closed = true; + // Tear down the agent first (it borrows config + store + source). + self.agent.deinit(); + if (self.tools) |ts| ts.deinit(L); + // Free the live config + every superseded one. Their string bytes + // live in `string_arena`, freed in one shot below. + c_allocator.destroy(self.config); + for (self.old_configs.items) |cfg| c_allocator.destroy(cfg); + self.old_configs.deinit(c_allocator); + self.string_arena.deinit(); + releaseIo(); + } +}; + +/// Boxed `*Stream` userdata. Holds a Lua reference to the owning Agent +/// userdata so the Agent cannot be GC'd while a live Stream still borrows +/// its conversation + config. +const StreamBox = struct { + stream: *panto.Stream, + agent_ref: c_int, + closed: bool = false, + + fn deinit(self: *StreamBox, L: *c.lua_State) void { + if (self.closed) return; + self.closed = true; + self.stream.deinit(); + if (self.agent_ref != c.LUA_NOREF) { + c.luaL_unref(L, LUA_REGISTRYINDEX, self.agent_ref); + self.agent_ref = c.LUA_NOREF; + } + } +}; + +/// Boxed `Conversation` userdata. The same userdata type serves three +/// ownership states, because `libpanto`'s `Conversation` is both a +/// standalone buildable value and the agent's live transparent field: +/// +/// - `.owned`: built by `panto.conversation()`; this box owns `inner` +/// and frees it at `__gc`. Can be handed to `panto.agent{conversation=}`, +/// which moves ownership to the agent and flips this box to `.adopted`. +/// - `.adopted`: ownership moved into an `Agent`; the box is inert — +/// `__gc` frees nothing and methods raise. (The agent now owns it.) +/// - `.borrowed`: a live view returned by `agent:conversation()`. It +/// points at the agent's `*Agent.conversation` field; `__gc` frees +/// nothing. A Lua ref pins the owning agent so the view can't dangle. +/// +/// `view` is the pointer methods operate through: for `.owned` it is +/// `&self.inner`; for `.borrowed` it is `&agent.conversation`. +const ConvBox = struct { + state: enum { owned, adopted, borrowed }, + /// Storage for an owned conversation. Unused (undefined) when borrowed. + inner: panto.Conversation, + /// The conversation methods act on. Always valid while usable. + view: *panto.Conversation, + /// For `.borrowed`: a ref pinning the owning Agent userdata. NOREF + /// otherwise. + agent_ref: c_int = c.LUA_NOREF, + + fn deinit(self: *ConvBox, L: *c.lua_State) void { + switch (self.state) { + .owned => self.inner.deinit(), + .adopted => {}, // the agent owns and frees it + .borrowed => { + if (self.agent_ref != c.LUA_NOREF) { + c.luaL_unref(L, LUA_REGISTRYINDEX, self.agent_ref); + self.agent_ref = c.LUA_NOREF; + } + }, + } + } +}; + +// =========================================================================== +// Module entry point +// =========================================================================== + +/// `luaopen_panto` — the fixed-name init function the Lua loader calls. +/// Builds and returns a **fresh** module table on every call (standard +/// C-module behavior; never a process-shared singleton — the CLI relies on +/// this to safely attach its `ext` field to its own copy). +pub export fn luaopen_panto(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + // Fail loud if loaded into a wrong-version host (the wrong-ABI guard). + c.luaL_checkversion(L); + + registerMetatables(L); + + c.lua_createtable(L, 0, 3); + c.lua_pushcclosure(L, agentNew, 0); + c.lua_setfield(L, -2, "agent"); + c.lua_pushcclosure(L, conversationNew, 0); + c.lua_setfield(L, -2, "conversation"); + _ = c.lua_pushstring(L, "libpanto-lua 0.0.0 (Lua 5.4)"); + c.lua_setfield(L, -2, "_VERSION"); + return 1; +} + +fn registerMetatables(L: *c.lua_State) void { + if (c.luaL_newmetatable(L, AGENT_MT) != 0) { + c.lua_createtable(L, 0, 6); // methods + setMethod(L, "run", agentRun); + setMethod(L, "register_tool", agentRegisterTool); + setMethod(L, "set_config", agentSetConfig); + setMethod(L, "add_system_message", agentAddSystemMessage); + setMethod(L, "set_system_prompt", agentSetSystemPrompt); + setMethod(L, "compact", agentCompact); + setMethod(L, "session_id", agentSessionId); + setMethod(L, "conversation", agentConversation); + // Internal/unstable: drive a tool batch directly, bypassing a live + // provider, so the luv-coroutine dispatch path is testable without + // a real turn. Not part of the public surface. + setMethod(L, "_dispatch_tools", agentDispatchTools); + c.lua_setfield(L, -2, "__index"); + setMethod(L, "__gc", agentGc); + } + c.lua_settop(L, c.lua_gettop(L) - 1); + + if (c.luaL_newmetatable(L, STREAM_MT) != 0) { + c.lua_createtable(L, 0, 2); // methods + setMethod(L, "next", streamNext); + setMethod(L, "events", streamEvents); + c.lua_setfield(L, -2, "__index"); + setMethod(L, "__gc", streamGc); + } + c.lua_settop(L, c.lua_gettop(L) - 1); + + if (c.luaL_newmetatable(L, CONV_MT) != 0) { + c.lua_createtable(L, 0, 9); // methods + setMethod(L, "add_system_message", convAddSystemMessage); + setMethod(L, "replace_system_message", convReplaceSystemMessage); + setMethod(L, "add_user_message", convAddUserMessage); + setMethod(L, "add_user_blocks", convAddUserBlocks); + setMethod(L, "add_assistant_message", convAddAssistantMessage); + setMethod(L, "add_compaction_summary", convAddCompactionSummary); + setMethod(L, "messages", convMessages); + setMethod(L, "len", convLen); + c.lua_setfield(L, -2, "__index"); + setMethod(L, "__len", convLen); + setMethod(L, "__gc", convGc); + } + c.lua_settop(L, c.lua_gettop(L) - 1); +} + +fn setMethod(L: *c.lua_State, name: [:0]const u8, f: c.lua_CFunction) void { + c.lua_pushcclosure(L, f, 0); + c.lua_setfield(L, -2, name.ptr); +} + +// =========================================================================== +// panto.agent { ... } -> Agent userdata +// =========================================================================== + +fn agentNew(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + c.luaL_checktype(L, 1, T_TABLE); + + const ud = c.lua_newuserdatauv(L, @sizeOf(AgentBox), 0) orelse + return luaErr(L, "panto.agent: out of memory"); + const box: *AgentBox = @ptrCast(@alignCast(ud)); + box.* = .{ + .agent = undefined, + .config = undefined, + .old_configs = .empty, + .store_impl = undefined, + .string_arena = std.heap.ArenaAllocator.init(c_allocator), + .tools = null, + .closed = false, + }; + + buildAgent(L, box) catch |err| { + box.string_arena.deinit(); + box.old_configs.deinit(c_allocator); + return luaErr(L, configErrorMessage(err, "panto.agent")); + }; + + _ = c.luaL_setmetatable(L, AGENT_MT); + return 1; +} + +fn buildAgent(L: *c.lua_State, box: *AgentBox) ConfigError!void { + const cfg = try c_allocator.create(panto.Config); + errdefer c_allocator.destroy(cfg); + cfg.* = try parseConfig(L, 1, box.string_arena.allocator()); + + // Optional `conversation = <panto.conversation()>`: adopt it for + // resumption. The userdata must be an `.owned` Conversation; on + // success its inner is moved into the agent and the box flips to + // `.adopted`. We resolve the ConvBox pointer here but only consume it + // *after* `Agent.init` succeeds, so a failed init leaves the caller's + // conversation owned and intact. + var adopt_box: ?*ConvBox = null; + { + const t = c.lua_getfield(L, 1, "conversation"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t != T_NIL) { + const ud = c.luaL_testudata(L, -1, CONV_MT) orelse return error.BadField; + const cb: *ConvBox = @ptrCast(@alignCast(ud)); + if (cb.state != .owned) return error.ConversationNotOwned; + adopt_box = cb; + } + } + + retainIo(); + errdefer releaseIo(); + + box.config = cfg; + box.store_impl = panto.NullStore.init(c_allocator); + box.agent = panto.Agent.init( + c_allocator, + io_ctx.io, + box.config, + box.store_impl.store().create(), + if (adopt_box) |cb| cb.inner else null, + ) catch return error.AgentInit; + + // Ownership of the inner conversation moved into the agent; neuter the + // source box so its `__gc` won't double-free. + if (adopt_box) |cb| { + cb.state = .adopted; + cb.view = &box.agent.conversation; + } +} + +/// `agent:set_config{...}` — swap the active provider/model/policy. Takes +/// effect at the next turn boundary (libpanto re-reads the snapshot each +/// turn). The new config is parsed with the *same* parser as construction, +/// so the two argument shapes are identical by construction. The old +/// config is retained (not freed) until agent teardown, since an in-flight +/// turn may still read it. +fn agentSetConfig(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + c.luaL_checktype(L, 2, T_TABLE); + + const new_cfg = c_allocator.create(panto.Config) catch + return luaErr(L, "panto: out of memory"); + new_cfg.* = parseConfig(L, 2, box.string_arena.allocator()) catch |err| { + c_allocator.destroy(new_cfg); + return luaErr(L, configErrorMessage(err, "set_config")); + }; + + // Retain the prior config so an in-flight turn keeps a valid snapshot. + box.old_configs.append(c_allocator, box.config) catch { + c_allocator.destroy(new_cfg); + return luaErr(L, "panto: out of memory"); + }; + box.config = new_cfg; + box.agent.setConfig(new_cfg); + return 0; +} + +fn agentAddSystemMessage(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + box.agent.addSystemMessage(ptr[0..len]) catch + return luaErr(L, "panto: failed to add system message"); + return 0; +} + +fn agentSetSystemPrompt(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + box.agent.setSystemPrompt(ptr[0..len]) catch + return luaErr(L, "panto: failed to set system prompt"); + return 0; +} + +/// `agent:compact([override_prompt[, extra_instructions]])` -> table. +/// Returns `{ compacted=, kept_turns=, summarized_messages= }`. Both +/// arguments are optional strings; nil falls back to the configured +/// compaction prompt. Errors (e.g. no compaction prompt configured) +/// surface as a Lua error. +fn agentCompact(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + + const override = optArgString(L, 2); + const extra = optArgString(L, 3); + + const res = box.agent.compact(override, extra) catch |err| { + return switch (err) { + error.NoCompactionPrompt => luaErr(L, "panto: no compaction prompt configured (set compaction.prompt in the config)"), + else => luaErr(L, "panto: compaction failed"), + }; + }; + + c.lua_createtable(L, 0, 3); + c.lua_pushboolean(L, if (res.compacted) 1 else 0); + c.lua_setfield(L, -2, "compacted"); + setIntField(L, "kept_turns", @intCast(res.kept_turns)); + setIntField(L, "summarized_messages", @intCast(res.summarized_messages)); + return 1; +} + +fn agentSessionId(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + const id = box.agent.sessionId(); + _ = c.lua_pushlstring(L, id.ptr, id.len); + return 1; +} + +/// `agent:conversation()` -> a **borrowed** Conversation view of the +/// agent's live `conversation` field. Mutations and reads go straight +/// through to the agent's conversation (the transparent-field semantics of +/// the Zig API). The view pins the agent (a Lua ref) so it cannot dangle, +/// and frees nothing at `__gc`. +fn agentConversation(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + + const ud = c.lua_newuserdatauv(L, @sizeOf(ConvBox), 0) orelse + return luaErr(L, "panto: out of memory"); + const cb: *ConvBox = @ptrCast(@alignCast(ud)); + cb.* = .{ + .state = .borrowed, + .inner = undefined, + .view = &box.agent.conversation, + .agent_ref = c.LUA_NOREF, + }; + _ = c.luaL_setmetatable(L, CONV_MT); + + // Pin the agent so the borrowed view can't outlive it. + c.lua_pushvalue(L, 1); + cb.agent_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + return 1; +} + +fn agentRun(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + + const stream = box.agent.run(.{ .text = ptr[0..len] }) catch + return luaErr(L, "panto: failed to start turn"); + + const ud = c.lua_newuserdatauv(L, @sizeOf(StreamBox), 0) orelse { + stream.deinit(); + return luaErr(L, "panto: out of memory"); + }; + const sbox: *StreamBox = @ptrCast(@alignCast(ud)); + sbox.* = .{ .stream = stream, .agent_ref = c.LUA_NOREF, .closed = false }; + _ = c.luaL_setmetatable(L, STREAM_MT); + + // Pin the agent so it outlives the stream. + c.lua_pushvalue(L, 1); + sbox.agent_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + return 1; +} + +fn agentGc(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + box.deinit(L); + return 0; +} + +// =========================================================================== +// panto.conversation() -> Conversation userdata +// =========================================================================== + +/// Build a fresh, standalone (`.owned`) `Conversation`. Hand it to +/// `panto.agent { conversation = <conv> }` to resume against a +/// pre-built history; ownership transfers to the agent at that point. +fn conversationNew(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const ud = c.lua_newuserdatauv(L, @sizeOf(ConvBox), 0) orelse + return luaErr(L, "panto.conversation: out of memory"); + const cb: *ConvBox = @ptrCast(@alignCast(ud)); + cb.* = .{ + .state = .owned, + .inner = panto.Conversation.init(c_allocator), + .view = undefined, + .agent_ref = c.LUA_NOREF, + }; + cb.view = &cb.inner; + _ = c.luaL_setmetatable(L, CONV_MT); + return 1; +} + +fn checkConv(L: *c.lua_State, idx: c_int) *ConvBox { + return @ptrCast(@alignCast(c.luaL_checkudata(L, idx, CONV_MT))); +} + +/// Resolve the usable `*Conversation` for a method, or raise if the box was +/// adopted (the agent owns it now — operate through `agent:conversation()`). +fn convView(L: *c.lua_State, cb: *ConvBox) ?*panto.Conversation { + if (cb.state == .adopted) { + _ = luaErr(L, "panto: this conversation was adopted by an agent; use agent:conversation() to operate on it"); + return null; + } + return cb.view; +} + +fn convAddSystemMessage(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + conv.addSystemMessage(ptr[0..len]) catch return luaErr(L, "panto: out of memory"); + return 0; +} + +fn convReplaceSystemMessage(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + conv.replaceSystemMessage(ptr[0..len]) catch return luaErr(L, "panto: out of memory"); + return 0; +} + +/// `conv:add_user_message(text)` — the ergonomic single-text case. The +/// general block-slice form is `conv:add_user_blocks{...}`. +fn convAddUserMessage(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + const tb = panto.textualBlockFromSlice(conv.allocator, ptr[0..len]) catch + return luaErr(L, "panto: out of memory"); + var block: panto.ContentBlock = .{ .Text = tb }; + conv.addUserMessage(&.{block}) catch { + block.deinit(conv.allocator); + return luaErr(L, "panto: out of memory"); + }; + return 0; +} + +fn convAddCompactionSummary(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + conv.addCompactionSummary(ptr[0..len]) catch return luaErr(L, "panto: out of memory"); + return 0; +} + +/// `conv:add_user_blocks{ <block>, ... }` — the general user-message +/// builder, symmetric with `add_assistant_message`. Each block is a table +/// `{ type = "text"|"tool_result", ... }`. Lets a Lua app reconstruct full +/// user turns from serialized state: plain/queued text plus tool results. +fn convAddUserBlocks(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + c.luaL_checktype(L, 2, T_TABLE); + addMessageFromBlocks(L, conv, 2, .user) catch |e| return luaErr(L, blockErrorMessage(e)); + return 0; +} + +/// `conv:add_assistant_message{ blocks = { <block>, ... }, usage = {...} }` +/// — reconstruct an assistant turn: `text`, `thinking`, and `tool_use` +/// blocks, plus optional provider `usage`. Together with `add_user_blocks` +/// this makes a serialize→rebuild cycle lossless for tool-using turns. +fn convAddAssistantMessage(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + c.luaL_checktype(L, 2, T_TABLE); + + // Accept either `{ blocks = {...}, usage = ? }` or a bare block array. + const bt = c.lua_getfield(L, 2, "blocks"); + const blocks_idx: c_int = if (bt == T_TABLE) c.lua_gettop(L) else blk: { + c.lua_settop(L, c.lua_gettop(L) - 1); + break :blk 2; + }; + const has_wrapper = bt == T_TABLE; + defer if (has_wrapper) c.lua_settop(L, c.lua_gettop(L) - 1); + + const usage = parseUsage(L, 2) catch |e| return luaErr(L, blockErrorMessage(e)); + addAssistantFromBlocks(L, conv, blocks_idx, usage) catch |e| return luaErr(L, blockErrorMessage(e)); + return 0; +} + +const BlockError = error{ BadBlock, UnknownBlockType, OutOfMemory }; + +fn blockErrorMessage(e: BlockError) [:0]const u8 { + return switch (e) { + error.BadBlock => "panto: malformed content block (check field names/types)", + error.UnknownBlockType => "panto: unknown block type (want text/thinking/tool_use/tool_result)", + error.OutOfMemory => "panto: out of memory", + }; +} + +/// Build a `[]ContentBlock` from the Lua array at `arr_idx`, then hand it +/// to the conversation (user or compaction-style role via addUserMessage). +fn addMessageFromBlocks(L: *c.lua_State, conv: *panto.Conversation, arr_idx: c_int, comptime role: enum { user }) BlockError!void { + _ = role; + var blocks: std.ArrayList(panto.ContentBlock) = .empty; + defer blocks.deinit(c_allocator); + errdefer for (blocks.items) |*b| b.deinit(conv.allocator); + + try collectBlocks(L, conv.allocator, arr_idx, &blocks); + conv.addUserMessage(blocks.items) catch return error.OutOfMemory; + // Ownership transferred; clear so errdefer/deinit don't double-free. + blocks.clearRetainingCapacity(); +} + +fn addAssistantFromBlocks(L: *c.lua_State, conv: *panto.Conversation, arr_idx: c_int, usage: ?panto.Usage) BlockError!void { + var blocks: std.ArrayList(panto.ContentBlock) = .empty; + defer blocks.deinit(c_allocator); + errdefer for (blocks.items) |*b| b.deinit(conv.allocator); + + try collectBlocks(L, conv.allocator, arr_idx, &blocks); + conv.addAssistantMessage(blocks.items, usage) catch return error.OutOfMemory; + blocks.clearRetainingCapacity(); +} + +/// Walk the Lua array at `arr_idx`, append one `ContentBlock` per entry to +/// `out`. Bytes are owned by `alloc` (the conversation's allocator). On +/// error the caller frees whatever was appended. +fn collectBlocks(L: *c.lua_State, alloc: std.mem.Allocator, arr_idx: c_int, out: *std.ArrayList(panto.ContentBlock)) BlockError!void { + const abs = c.lua_absindex(L, arr_idx); + const n: usize = @intCast(c.lua_rawlen(L, abs)); + var i: usize = 1; + while (i <= n) : (i += 1) { + _ = c.lua_rawgeti(L, abs, @intCast(i)); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (c.lua_type(L, -1) != T_TABLE) return error.BadBlock; + const block = try buildContentBlock(L, alloc, c.lua_gettop(L)); + out.append(c_allocator, block) catch { + var b = block; + b.deinit(alloc); + return error.OutOfMemory; + }; + } +} + +/// Construct a single `ContentBlock` from the table at `idx`. Every owned +/// byte is allocated with `alloc` so the conversation can free it. +fn buildContentBlock(L: *c.lua_State, alloc: std.mem.Allocator, idx: c_int) BlockError!panto.ContentBlock { + const type_str = blockField(L, idx, "type") orelse return error.BadBlock; + + if (std.mem.eql(u8, type_str, "text")) { + const text = blockField(L, idx, "text") orelse return error.BadBlock; + const tb = panto.textualBlockFromSlice(alloc, text) catch return error.OutOfMemory; + return .{ .Text = tb }; + } else if (std.mem.eql(u8, type_str, "thinking")) { + const text = blockField(L, idx, "text") orelse ""; + var tb = panto.textualBlockFromSlice(alloc, text) catch return error.OutOfMemory; + errdefer tb.deinit(alloc); + const sig: ?[]const u8 = if (blockField(L, idx, "signature")) |s| + (alloc.dupe(u8, s) catch return error.OutOfMemory) + else + null; + return .{ .Thinking = .{ .text = tb, .signature = sig } }; + } else if (std.mem.eql(u8, type_str, "tool_use")) { + const id = blockField(L, idx, "id") orelse return error.BadBlock; + const name = blockField(L, idx, "name") orelse return error.BadBlock; + const input = blockField(L, idx, "input") orelse ""; + const id_owned = alloc.dupe(u8, id) catch return error.OutOfMemory; + errdefer alloc.free(id_owned); + const name_owned = alloc.dupe(u8, name) catch return error.OutOfMemory; + errdefer alloc.free(name_owned); + const input_tb = panto.textualBlockFromSlice(alloc, input) catch return error.OutOfMemory; + return .{ .ToolUse = .{ .id = id_owned, .name = name_owned, .input = input_tb } }; + } else if (std.mem.eql(u8, type_str, "tool_result")) { + return buildToolResultBlock(L, alloc, idx); + } + return error.UnknownBlockType; +} + +fn buildToolResultBlock(L: *c.lua_State, alloc: std.mem.Allocator, idx: c_int) BlockError!panto.ContentBlock { + const tuid = blockField(L, idx, "tool_use_id") orelse return error.BadBlock; + const tuid_owned = alloc.dupe(u8, tuid) catch return error.OutOfMemory; + errdefer alloc.free(tuid_owned); + + const is_error: bool = blk: { + const t = c.lua_getfield(L, idx, "is_error"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + break :blk (t == T_BOOLEAN and c.lua_toboolean(L, -1) != 0); + }; + + var parts: std.ArrayList(panto.ResultPartStored) = .empty; + errdefer { + for (parts.items) |*p| p.deinit(alloc); + parts.deinit(alloc); + } + + const pt = c.lua_getfield(L, idx, "parts"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (pt == T_TABLE) { + const pn: usize = @intCast(c.lua_rawlen(L, -1)); + var i: usize = 1; + while (i <= pn) : (i += 1) { + _ = c.lua_rawgeti(L, -1, @intCast(i)); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (c.lua_type(L, -1) != T_TABLE) return error.BadBlock; + const part = try buildStoredPart(L, alloc, c.lua_gettop(L)); + parts.append(alloc, part) catch { + var p = part; + p.deinit(alloc); + return error.OutOfMemory; + }; + } + } else if (pt != T_NIL) { + return error.BadBlock; + } + + return .{ .ToolResult = .{ .tool_use_id = tuid_owned, .parts = parts, .is_error = is_error } }; +} + +/// A stored result part: `{ text = "..." }` or `{ media_type = "...", +/// data = "<base64>" }`. Media `data` is the already-encoded bytes (this +/// is resumption of *stored* state, not raw tool output). +fn buildStoredPart(L: *c.lua_State, alloc: std.mem.Allocator, idx: c_int) BlockError!panto.ResultPartStored { + if (blockField(L, idx, "text")) |t| { + const tb = panto.textualBlockFromSlice(alloc, t) catch return error.OutOfMemory; + return .{ .text = tb }; + } + const media_type = blockField(L, idx, "media_type") orelse return error.BadBlock; + const data = blockField(L, idx, "data") orelse return error.BadBlock; + const mt_owned = alloc.dupe(u8, media_type) catch return error.OutOfMemory; + errdefer alloc.free(mt_owned); + const data_tb = panto.textualBlockFromSlice(alloc, data) catch return error.OutOfMemory; + return .{ .media = .{ .media_type = mt_owned, .data = data_tb } }; +} + +/// Read string field `name` from the table at `idx`, returning a borrowed +/// slice valid until the field value is popped. We pop immediately (Lua +/// keeps the string alive as long as it's referenced by the table), so the +/// returned slice stays valid for the synchronous dupe that follows. +fn blockField(L: *c.lua_State, idx: c_int, name: [*:0]const u8) ?[]const u8 { + const t = c.lua_getfield(L, idx, name); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t != T_STRING) return null; + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len); + if (ptr == null) return null; + return ptr[0..len]; +} + +fn parseUsage(L: *c.lua_State, tbl_idx: c_int) BlockError!?panto.Usage { + const t = c.lua_getfield(L, tbl_idx, "usage"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return null; + if (t != T_TABLE) return error.BadBlock; + const sub = c.lua_gettop(L); + return panto.Usage{ + .input = usageField(L, sub, "input"), + .output = usageField(L, sub, "output"), + .cache_read = usageField(L, sub, "cache_read"), + .cache_write = usageField(L, sub, "cache_write"), + .reasoning = usageField(L, sub, "reasoning"), + }; +} + +fn usageField(L: *c.lua_State, tbl_idx: c_int, name: [*:0]const u8) u64 { + const t = c.lua_getfield(L, tbl_idx, name); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t != T_NUMBER) return 0; + const v = c.lua_tointegerx(L, -1, null); + return if (v < 0) 0 else @intCast(v); +} + +fn convLen(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + c.lua_pushinteger(L, @intCast(conv.messages.items.len)); + return 1; +} + +/// `conv:messages()` -> array of `{ role=, blocks={ {type=, ...} } }`. +/// A read-only inspection snapshot: bytes are copied into Lua, so the +/// returned tables are independent of later conversation mutation. +fn convMessages(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + const conv = convView(L, cb) orelse return 0; + + const msgs = conv.messages.items; + c.lua_createtable(L, @intCast(msgs.len), 0); + for (msgs, 0..) |msg, mi| { + c.lua_createtable(L, 0, 3); + setStringField(L, "role", roleName(msg.role)); + if (msg.usage) |u| pushUsage(L, u); // emits a `usage` field + // blocks = { ... } — faithful per-block detail for lossless + // round-trip: each block table is shaped to be fed straight back + // into add_user_blocks / add_assistant_message. + c.lua_createtable(L, @intCast(msg.content.items.len), 0); + for (msg.content.items, 0..) |block, bi| { + pushInspectBlock(L, block); + c.lua_rawseti(L, -2, @intCast(bi + 1)); + } + c.lua_setfield(L, -2, "blocks"); + c.lua_rawseti(L, -2, @intCast(mi + 1)); + } + return 1; +} + +/// Push a fully-detailed block table (leaves it on the stack top). The +/// shape mirrors exactly what `add_user_blocks`/`add_assistant_message` +/// accept, so `conv:messages()` output can be replayed verbatim. The block +/// `type` strings here match `buildContentBlock`'s expectations. +fn pushInspectBlock(L: *c.lua_State, block: panto.ContentBlock) void { + c.lua_createtable(L, 0, 4); + switch (block) { + .Text => |t| { + setStringField(L, "type", "text"); + setStringField(L, "text", t.items); + }, + .Thinking => |t| { + setStringField(L, "type", "thinking"); + setStringField(L, "text", t.text.items); + if (t.signature) |s| setStringField(L, "signature", s); + }, + .ToolUse => |tu| { + setStringField(L, "type", "tool_use"); + setStringField(L, "id", tu.id); + setStringField(L, "name", tu.name); + setStringField(L, "input", tu.input.items); + }, + .ToolResult => |tr| { + setStringField(L, "type", "tool_result"); + setStringField(L, "tool_use_id", tr.tool_use_id); + c.lua_pushboolean(L, if (tr.is_error) 1 else 0); + c.lua_setfield(L, -2, "is_error"); + c.lua_createtable(L, @intCast(tr.parts.items.len), 0); + for (tr.parts.items, 0..) |part, pi| { + c.lua_createtable(L, 0, 2); + switch (part) { + .text => |t| setStringField(L, "text", t.items), + .media => |m| { + setStringField(L, "media_type", m.media_type); + setStringField(L, "data", m.data.items); + }, + } + c.lua_rawseti(L, -2, @intCast(pi + 1)); + } + c.lua_setfield(L, -2, "parts"); + }, + .System => |s| { + setStringField(L, "type", "system"); + setStringField(L, "text", s.text.items); + setStringField(L, "mode", switch (s.mode) { + .append => "append", + .replace => "replace", + }); + }, + .CompactionSummary => |s| { + setStringField(L, "type", "compaction_summary"); + setStringField(L, "text", s.text.items); + }, + } +} + +fn convGc(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const cb = checkConv(L, 1); + cb.deinit(L); + return 0; +} + +// =========================================================================== +// Config parsing (shared by panto.agent{} and agent:set_config{}) +// =========================================================================== + +const ConfigError = error{ + MissingField, + BadField, + UnknownApiStyle, + BadEnum, + ConversationNotOwned, + OutOfMemory, + AgentInit, +}; + +fn configErrorMessage(err: ConfigError, comptime ctx: [:0]const u8) [:0]const u8 { + return switch (err) { + error.MissingField => ctx ++ ": missing required field (need api_style, api_key, base_url, model)", + error.BadField => ctx ++ ": a field has the wrong type", + error.UnknownApiStyle => ctx ++ ": api_style must be 'openai_chat' or 'anthropic_messages'", + error.BadEnum => ctx ++ ": an enum field has an unrecognized value", + error.ConversationNotOwned => ctx ++ ": the `conversation` was already adopted by an agent or is a borrowed view; build a fresh one with panto.conversation()", + error.OutOfMemory => ctx ++ ": out of memory", + error.AgentInit => ctx ++ ": failed to initialize agent", + }; +} + +/// Parse a full `panto.Config` from the named-args table at `tbl_idx`. +/// Every string is duped into `arena` (owned by the AgentBox for the +/// config's lifetime). This is the single source of truth for both +/// construction and `set_config`, so the two argument shapes never drift. +/// +/// Provider fields (all required): `api_style`, `api_key`, `base_url`, +/// `model`. Common optional: `max_tokens`. +/// openai_chat optional: `reasoning`. +/// anthropic_messages optional: `api_version`, `thinking`, `effort`, +/// `thinking_budget_tokens`, `thinking_interleaved`. +/// Optional `compaction = { keep_verbatim=, prompt= }`. +/// Optional `retry = { max_attempts=, initial_delay_ms=, max_delay_ms=, +/// multiplier=, jitter= }`. +fn parseConfig(L: *c.lua_State, tbl_idx: c_int, arena: std.mem.Allocator) ConfigError!panto.Config { + const api_style = try requiredString(L, tbl_idx, "api_style"); + const api_key = try dupArena(arena, try requiredString(L, tbl_idx, "api_key")); + const base_url = try dupArena(arena, try requiredString(L, tbl_idx, "base_url")); + const model = try dupArena(arena, try requiredString(L, tbl_idx, "model")); + const max_tokens = try optU32(L, tbl_idx, "max_tokens"); + + var provider: panto.ProviderConfig = undefined; + if (std.mem.eql(u8, api_style, "openai_chat")) { + var p: panto.OpenAIChatConfig = .{ .api_key = api_key, .base_url = base_url, .model = model }; + if (max_tokens) |mt| p.max_tokens = mt; + if (try optString(L, tbl_idx, "reasoning")) |r| + p.reasoning = parseEnum(panto.ReasoningEffort, r) orelse return error.BadEnum; + provider = .{ .openai_chat = p }; + } else if (std.mem.eql(u8, api_style, "anthropic_messages")) { + var p: panto.AnthropicMessagesConfig = .{ .api_key = api_key, .base_url = base_url, .model = model }; + if (max_tokens) |mt| p.max_tokens = mt; + if (try optString(L, tbl_idx, "api_version")) |v| p.api_version = try dupArena(arena, v); + if (try optString(L, tbl_idx, "thinking")) |t| + p.thinking = parseEnum(panto.Thinking, t) orelse return error.BadEnum; + if (try optString(L, tbl_idx, "effort")) |e| + p.effort = parseEnum(panto.Effort, e) orelse return error.BadEnum; + if (try optU32(L, tbl_idx, "thinking_budget_tokens")) |b| p.thinking_budget_tokens = b; + if (try optBool(L, tbl_idx, "thinking_interleaved")) |i| p.thinking_interleaved = i; + provider = .{ .anthropic_messages = p }; + } else { + return error.UnknownApiStyle; + } + + var config: panto.Config = .{ .provider = provider }; + try parseCompaction(L, tbl_idx, arena, &config.compaction); + try parseRetry(L, tbl_idx, &config.retry); + return config; +} + +fn parseCompaction( + L: *c.lua_State, + tbl_idx: c_int, + arena: std.mem.Allocator, + out: *panto.CompactionConfig, +) ConfigError!void { + const t = c.lua_getfield(L, tbl_idx, "compaction"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return; + if (t != T_TABLE) return error.BadField; + const sub = c.lua_gettop(L); + if (try optU32(L, sub, "keep_verbatim")) |k| out.keep_verbatim = k; + if (try optString(L, sub, "prompt")) |p| out.compaction_prompt = try dupArena(arena, p); + // Note: a nested compaction `model` override is intentionally omitted + // for v1 — it would require recursively parsing a second provider + // block; the active model is used for compaction unless added later. +} + +fn parseRetry(L: *c.lua_State, tbl_idx: c_int, out: *panto.RetryConfig) ConfigError!void { + const t = c.lua_getfield(L, tbl_idx, "retry"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return; + if (t != T_TABLE) return error.BadField; + const sub = c.lua_gettop(L); + if (try optU32(L, sub, "max_attempts")) |v| out.max_attempts = v; + if (try optU64(L, sub, "initial_delay_ms")) |v| out.initial_delay_ms = v; + if (try optU64(L, sub, "max_delay_ms")) |v| out.max_delay_ms = v; + if (try optNumber(L, sub, "multiplier")) |v| out.multiplier = v; + if (try optBool(L, sub, "jitter")) |v| out.jitter = v; +} + +fn parseEnum(comptime E: type, s: []const u8) ?E { + inline for (@typeInfo(E).@"enum".fields) |f| { + if (std.mem.eql(u8, s, f.name)) return @enumFromInt(f.value); + } + return null; +} + +fn dupArena(arena: std.mem.Allocator, s: []const u8) ConfigError![]const u8 { + return arena.dupe(u8, s) catch error.OutOfMemory; +} + +// =========================================================================== +// Lua tool source: luv-driven cooperative coroutines +// =========================================================================== +// +// Ported from the CLI's `src/lua_runtime.zig` scheduler, operating on the +// host `lua_State` passed to `luaopen_panto`. Each registered tool stores +// its handler in the Lua registry (`luaL_ref`). On `invoke_batch` we run +// each call as a coroutine through a wrapper that `pcall`s the handler and +// reports the result via a registered `_record_result` C closure, then +// drive `uv.run("default")` to completion. Synchronous handlers finish on +// their first resume; async handlers yield on a libuv op and are resumed +// by its callback. + +/// One coroutine's outcome, written by `recordResult` and read after the +/// event loop drains. +const Slot = struct { + recorded: bool = false, + ok: bool = false, + value: ?panto.ResultParts = null, + err_msg: ?[]u8 = null, +}; + +const BatchState = struct { + allocator: std.mem.Allocator, + slots: []Slot, +}; + +const LuaToolSource = struct { + L: *c.lua_State, + /// Tool decls handed to libpanto (borrowed strings live in `arena`). + decls: std.ArrayList(panto.ToolDecl), + /// name -> handler registry ref. + handlers: std.StringHashMap(c_int), + /// Owns every decl string + handler-name key. + arena: std.heap.ArenaAllocator, + /// Wrapper closure ref: pcall(handler,input) -> _record_result(...). + wrapper_ref: c_int = c.LUA_NOREF, + /// Cached `require("luv").run`. + uv_run_ref: c_int = c.LUA_NOREF, + /// In-flight batch, valid only during one `invoke_batch`. + current_batch: ?*BatchState = null, + /// True once the source has been handed to the agent. + registered: bool = false, + + fn create(L: *c.lua_State) !*LuaToolSource { + const self = try c_allocator.create(LuaToolSource); + self.* = .{ + .L = L, + .decls = .empty, + .handlers = std.StringHashMap(c_int).init(c_allocator), + .arena = std.heap.ArenaAllocator.init(c_allocator), + }; + return self; + } + + fn deinit(self: *LuaToolSource, L: *c.lua_State) void { + var it = self.handlers.iterator(); + while (it.next()) |e| c.luaL_unref(L, LUA_REGISTRYINDEX, e.value_ptr.*); + self.handlers.deinit(); + if (self.wrapper_ref != c.LUA_NOREF) c.luaL_unref(L, LUA_REGISTRYINDEX, self.wrapper_ref); + if (self.uv_run_ref != c.LUA_NOREF) c.luaL_unref(L, LUA_REGISTRYINDEX, self.uv_run_ref); + self.decls.deinit(c_allocator); + self.arena.deinit(); + c_allocator.destroy(self); + } + + fn toolSource(self: *LuaToolSource) panto.ToolSource { + return .{ + .name = SOURCE_NAME, + .tools = self.decls.items, + .ctx = self, + .vtable = &source_vtable, + }; + } + + /// Lazily build the wrapper closure + cache `uv.run`. Requires `luv` + /// to be `require`-able in the host (a hard package dependency). + fn ensureScheduler(self: *LuaToolSource) !void { + if (self.wrapper_ref != c.LUA_NOREF) return; + const L = self.L; + + // Register `_record_result` as a C closure carrying `self`, and + // build the wrapper that closes over it. We keep them in a private + // table referenced only by the wrapper, so we never touch the + // module table (which is the host's, possibly augmented, copy). + const snippet = + \\local record = ... + \\return function(idx, handler, input) + \\ local ok, val = pcall(handler, input) + \\ record(idx, ok, val) + \\end + ; + if (c.luaL_loadstring(L, snippet) != 0) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuaInitFailed; + } + c.lua_pushlightuserdata(L, @ptrCast(self)); + c.lua_pushcclosure(L, recordResultC, 1); // the `record` arg + if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuaInitFailed; + } + self.wrapper_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + + // Cache uv.run. + const uv_snippet = + \\return require("luv").run + ; + if (c.luaL_loadstring(L, uv_snippet) != 0) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuaInitFailed; + } + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuvMissing; + } + if (c.lua_type(L, -1) != T_FUNCTION) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuvMissing; + } + self.uv_run_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + } +}; + +const source_vtable: panto.ToolSource.VTable = .{ + .invoke_batch = invokeBatch, + .deinit = deinitSrc, +}; + +/// The source's `ctx` is owned by the AgentBox (freed in `AgentBox.deinit` +/// after `agent.deinit`), so libpanto's source teardown is a no-op. +fn deinitSrc(_: *anyopaque, _: std.mem.Allocator) void {} + +/// `agent:register_tool { name=, description=, schema=, handler= }`. +/// Validates the named-args table, serializes the schema to JSON, refs the +/// handler, and appends a decl. The first call creates the source and +/// hands it to the agent; later calls extend the source (visible at the +/// next turn boundary, per libpanto's registry semantics). +fn agentRegisterTool(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + c.luaL_checktype(L, 2, T_TABLE); + + if (box.tools == null) { + box.tools = LuaToolSource.create(L) catch + return luaErr(L, "panto: out of memory"); + } + const ts = box.tools.?; + + // Make sure luv + the wrapper are ready before we accept any tool, so + // a missing `luv` fails at registration (clear) not mid-turn. + ts.ensureScheduler() catch |err| { + return switch (err) { + error.LuvMissing => luaErr(L, "panto: require('luv') failed — libpanto-lua needs the 'luv' rock for tool dispatch"), + else => luaErr(L, "panto: failed to initialize the tool scheduler"), + }; + }; + + addTool(L, ts, 2) catch |err| { + return luaErr(L, configErrorMessage2(err)); + }; + + // Register the source with the agent on first tool. + if (!ts.registered) { + box.agent.registerToolSource(ts.toolSource()) catch + return luaErr(L, "panto: failed to register tool source"); + ts.registered = true; + } + return 0; +} + +const ToolError = error{ MissingField, BadField, OutOfMemory }; + +fn configErrorMessage2(err: ToolError) [:0]const u8 { + return switch (err) { + error.MissingField => "register_tool: need name (string), description (string), schema (table), handler (function)", + error.BadField => "register_tool: a field has the wrong type", + error.OutOfMemory => "register_tool: out of memory", + }; +} + +fn addTool(L: *c.lua_State, ts: *LuaToolSource, tbl_idx: c_int) ToolError!void { + const arena = ts.arena.allocator(); + + const name = try arenaString(L, tbl_idx, "name", arena); + const description = try arenaString(L, tbl_idx, "description", arena); + + // schema (table) -> JSON. + const sty = c.lua_getfield(L, tbl_idx, "schema"); + if (sty != T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.BadField; + } + const schema_json = serializeSchema(L, c.lua_gettop(L), arena) catch { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.OutOfMemory; + }; + c.lua_settop(L, c.lua_gettop(L) - 1); // pop schema + + // handler (function) -> registry ref. luaL_ref pops it. + const hty = c.lua_getfield(L, tbl_idx, "handler"); + if (hty != T_FUNCTION) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.BadField; + } + const handler_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + + ts.handlers.put(name, handler_ref) catch { + c.luaL_unref(L, LUA_REGISTRYINDEX, handler_ref); + return error.OutOfMemory; + }; + ts.decls.append(c_allocator, .{ + .name = name, + .description = description, + .schema_json = schema_json, + }) catch { + _ = ts.handlers.remove(name); + c.luaL_unref(L, LUA_REGISTRYINDEX, handler_ref); + return error.OutOfMemory; + }; +} + +fn arenaString(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8, arena: std.mem.Allocator) ToolError![]const u8 { + const t = c.lua_getfield(L, tbl_idx, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return error.MissingField; + if (t != T_STRING) return error.BadField; + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len); + if (ptr == null) return error.BadField; + return arena.dupe(u8, ptr[0..len]) catch error.OutOfMemory; +} + +/// Internal test hook: `agent:_dispatch_tools({ {name=,input=}, ... })` +/// runs the calls through the real `invoke_batch` (coroutines + luv) and +/// returns an array of `{ ok=bool, text=string }`. Lets us validate the +/// dispatch path without a live provider emitting tool calls. +fn agentDispatchTools(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const box = checkAgent(L, 1); + if (box.closed) return luaErr(L, "panto: agent is closed"); + c.luaL_checktype(L, 2, T_TABLE); + const ts = box.tools orelse return luaErr(L, "panto: no tools registered"); + + const n: usize = @intCast(c.lua_rawlen(L, 2)); + var arena_state = std.heap.ArenaAllocator.init(c_allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const calls = arena.alloc(panto.ToolCall, n) catch return luaErr(L, "panto: oom"); + var i: usize = 0; + while (i < n) : (i += 1) { + _ = c.lua_rawgeti(L, 2, @intCast(i + 1)); + const name = (optTableString(L, -1, "name") catch null) orelse + return luaErr(L, "_dispatch_tools: each call needs name + input strings"); + const name_owned = arena.dupe(u8, name) catch return luaErr(L, "panto: oom"); + c.lua_settop(L, c.lua_gettop(L) - 1); // pop name value + const input = (optTableString(L, -1, "input") catch null) orelse "{}"; + const input_owned = arena.dupe(u8, input) catch return luaErr(L, "panto: oom"); + c.lua_settop(L, c.lua_gettop(L) - 1); // pop input value + c.lua_settop(L, c.lua_gettop(L) - 1); // pop call table + calls[i] = .{ .tool_name = name_owned, .input = input_owned }; + } + + const results = arena.alloc(panto.ToolCallResult, n) catch return luaErr(L, "panto: oom"); + invokeBatch(ts, calls, results, arena) catch return luaErr(L, "panto: dispatch failed"); + + c.lua_createtable(L, @intCast(n), 0); + for (results, 0..) |r, idx| { + c.lua_createtable(L, 0, 2); + switch (r) { + .ok => |parts| { + c.lua_pushboolean(L, 1); + c.lua_setfield(L, -2, "ok"); + const text: []const u8 = if (parts.items.len > 0 and parts.items[0] == .text) + parts.items[0].text + else + ""; + setStringField(L, "text", text); + parts.deinit(arena); + }, + .err => { + c.lua_pushboolean(L, 0); + c.lua_setfield(L, -2, "ok"); + }, + } + c.lua_rawseti(L, -2, @intCast(idx + 1)); + } + return 1; +} + +fn invokeBatch( + ctx: *anyopaque, + calls: []const panto.ToolCall, + results: []panto.ToolCallResult, + allocator: std.mem.Allocator, +) anyerror!void { + const self: *LuaToolSource = @ptrCast(@alignCast(ctx)); + const L = self.L; + + var slots = try allocator.alloc(Slot, calls.len); + defer allocator.free(slots); + for (slots) |*s| s.* = .{}; + + var batch: BatchState = .{ .allocator = allocator, .slots = slots }; + self.current_batch = &batch; + defer self.current_batch = null; + + var thread_refs = try allocator.alloc(c_int, calls.len); + defer allocator.free(thread_refs); + @memset(thread_refs, 0); + defer for (thread_refs) |r| { + if (r != 0) c.luaL_unref(L, LUA_REGISTRYINDEX, r); + }; + + // Step 1: start every call's coroutine. Sync handlers complete here. + var any_pending = false; + for (calls, 0..) |call, i| { + const handler_ref = self.handlers.get(call.tool_name) orelse { + slots[i] = .{ .recorded = true, .ok = false, .err_msg = try allocator.dupe(u8, "unknown tool name") }; + continue; + }; + const started = startCoroutine(self, i, handler_ref, call.input, allocator) catch { + slots[i] = .{ .recorded = true, .ok = false, .err_msg = try allocator.dupe(u8, "failed to start handler coroutine") }; + continue; + }; + thread_refs[i] = started.thread_ref; + if (started.still_pending) any_pending = true; + } + + // Step 2: drive libuv to completion (wakes any yielded coroutine). + if (any_pending) try driveUvToCompletion(self); + + // Step 3: reap. A still-suspended coroutine violated the contract. + for (thread_refs, 0..) |tref, i| { + if (tref == 0) continue; + _ = c.lua_rawgeti(L, LUA_REGISTRYINDEX, tref); + const co: *c.lua_State = @ptrCast(c.lua_tothread(L, -1).?); + const status = c.lua_status(co); + c.lua_settop(L, c.lua_gettop(L) - 1); + if (status == c.LUA_YIELD and !slots[i].recorded) { + slots[i] = .{ .recorded = true, .ok = false, .err_msg = try allocator.dupe( + u8, + "handler still suspended after the event loop drained (yielded without a pending libuv op)", + ) }; + } + c.luaL_unref(L, LUA_REGISTRYINDEX, tref); + thread_refs[i] = 0; + } + + // Step 4: translate slots into libpanto results. A Lua-level failure + // is surfaced to the model as a textual `.ok` result rather than a + // `.err` (which libpanto treats as turn-aborting) — same policy as the + // CLI runtime. + for (slots, 0..) |slot, i| { + if (slot.ok and slot.value != null) { + results[i] = .{ .ok = slot.value.? }; + if (slot.err_msg) |m| allocator.free(m); + } else { + if (slot.value) |v| v.deinit(allocator); + results[i] = .{ .ok = try formatToolError( + allocator, + calls[i].tool_name, + slot.err_msg orelse "(no message)", + ) }; + if (slot.err_msg) |m| allocator.free(m); + } + } +} + +fn formatToolError(allocator: std.mem.Allocator, tool_name: []const u8, message: []const u8) !panto.ResultParts { + const text = try std.fmt.allocPrint(allocator, "panto-lua: tool '{s}' failed: {s}", .{ tool_name, message }); + return panto.ResultParts.fromTextOwned(allocator, text); +} + +fn startCoroutine( + self: *LuaToolSource, + idx: usize, + handler_ref: c_int, + input: []const u8, + allocator: std.mem.Allocator, +) !struct { thread_ref: c_int, still_pending: bool } { + const L = self.L; + const co = c.lua_newthread(L) orelse return error.LuaInitFailed; + const thread_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + + _ = c.lua_rawgeti(co, LUA_REGISTRYINDEX, @intCast(self.wrapper_ref)); + c.lua_pushinteger(co, @intCast(idx)); + _ = c.lua_rawgeti(co, LUA_REGISTRYINDEX, @intCast(handler_ref)); + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + try pushJsonAsLua(co, arena_state.allocator(), input); + + var nres: c_int = 0; + const status = c.lua_resume(co, L, 3, &nres); + return .{ .thread_ref = thread_ref, .still_pending = status == c.LUA_YIELD }; +} + +fn driveUvToCompletion(self: *LuaToolSource) !void { + const L = self.L; + _ = c.lua_rawgeti(L, LUA_REGISTRYINDEX, @intCast(self.uv_run_ref)); + _ = c.lua_pushlstring(L, "default", 7); + if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.UvRunFailed; + } + c.lua_settop(L, c.lua_gettop(L) - 1); // discard the bool return +} + +/// `_record_result(idx, ok, value)` — carries `*LuaToolSource` as upvalue +/// 1, writes into the in-flight batch's slot. +fn recordResultC(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const self_ptr = c.lua_touserdata(L, c.lua_upvalueindex(1)) orelse return 0; + const self: *LuaToolSource = @ptrCast(@alignCast(self_ptr)); + const batch = self.current_batch orelse return 0; + + const idx: usize = @intCast(c.lua_tointegerx(L, 1, null)); + const ok = c.lua_toboolean(L, 2) != 0; + if (idx >= batch.slots.len) return 0; + + if (ok) { + const value = readHandlerResult(L, 3, batch.allocator) catch |e| { + const msg = std.fmt.allocPrint(batch.allocator, "failed to serialize handler result: {s}", .{@errorName(e)}) catch null; + batch.slots[idx] = .{ .recorded = true, .ok = false, .err_msg = msg }; + return 0; + }; + batch.slots[idx] = .{ .recorded = true, .ok = true, .value = value }; + } else { + var len: usize = 0; + const ptr = c.luaL_tolstring(L, 3, &len); + const owned = if (ptr != null) batch.allocator.dupe(u8, ptr[0..len]) catch null else null; + c.lua_settop(L, c.lua_gettop(L) - 1); // pop tolstring's string + batch.slots[idx] = .{ .recorded = true, .ok = false, .err_msg = owned }; + } + return 0; +} + +// =========================================================================== +// Stream methods +// =========================================================================== + +fn streamNext(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sbox = checkStream(L, 1); + if (sbox.closed) return luaErr(L, "panto: stream is closed"); + + const maybe_event = sbox.stream.next() catch + return luaErr(L, "panto: stream failed"); + if (maybe_event) |ev| { + pushEvent(L, ev) catch return luaErr(L, "panto: out of memory building event"); + return 1; + } + c.lua_pushnil(L); + return 1; +} + +/// `stream:events()` -> (iterator, stream, nil): the generic-`for` triple, +/// so `for ev in stream:events() do ... end` drives `next()`. +fn streamEvents(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + _ = checkStream(L, 1); + c.lua_pushcclosure(L, streamIterStep, 0); + c.lua_pushvalue(L, 1); + c.lua_pushnil(L); + return 3; +} + +fn streamIterStep(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sbox = checkStream(L, 1); + if (sbox.closed) { + c.lua_pushnil(L); + return 1; + } + const maybe_event = sbox.stream.next() catch + return luaErr(L, "panto: stream failed"); + if (maybe_event) |ev| { + pushEvent(L, ev) catch return luaErr(L, "panto: out of memory building event"); + return 1; + } + c.lua_pushnil(L); + return 1; +} + +fn streamGc(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sbox = checkStream(L, 1); + sbox.deinit(L); + return 0; +} + +// =========================================================================== +// Event -> Lua table marshalling +// =========================================================================== + +fn pushEvent(L: *c.lua_State, ev: panto.Event) !void { + c.lua_createtable(L, 0, 4); + switch (ev) { + .message_start => |role| { + setStringField(L, "type", "message_start"); + setStringField(L, "role", roleName(role)); + }, + .block_start => |bs| { + setStringField(L, "type", "block_start"); + setStringField(L, "block_type", blockTypeName(bs.block_type)); + setIntField(L, "index", @intCast(bs.index)); + }, + .tool_details => |td| { + setStringField(L, "type", "tool_details"); + setIntField(L, "index", @intCast(td.index)); + setStringField(L, "id", td.id); + setStringField(L, "name", td.name); + }, + .content_delta => |cd| { + setStringField(L, "type", "content_delta"); + setIntField(L, "index", @intCast(cd.index)); + setStringField(L, "delta", cd.delta); + }, + .block_complete => |bc| { + setStringField(L, "type", "block_complete"); + setIntField(L, "index", @intCast(bc.index)); + setStringField(L, "block_type", contentBlockTypeName(bc.block)); + pushBlockText(L, bc.block); + }, + .message_complete => |mc| { + setStringField(L, "type", "message_complete"); + setStringField(L, "role", roleName(mc.message.role)); + if (mc.usage) |u| pushUsage(L, u); + }, + .provider_retry => |pr| { + setStringField(L, "type", "provider_retry"); + setIntField(L, "attempt", @intCast(pr.attempt)); + setIntField(L, "max_attempts", @intCast(pr.max_attempts)); + setIntField(L, "delay_ms", @intCast(pr.delay_ms)); + }, + .tool_dispatch_start => |tds| { + setStringField(L, "type", "tool_dispatch_start"); + setIntField(L, "count", @intCast(tds.count)); + }, + .tool_dispatch_complete => setStringField(L, "type", "tool_dispatch_complete"), + .turn_complete => setStringField(L, "type", "turn_complete"), + } +} + +fn pushBlockText(L: *c.lua_State, block: panto.ContentBlock) void { + switch (block) { + .Text => |t| setStringField(L, "text", t.items), + .Thinking => |t| setStringField(L, "text", t.text.items), + .ToolUse => |tu| { + setStringField(L, "text", tu.input.items); + setStringField(L, "id", tu.id); + setStringField(L, "name", tu.name); + }, + .ToolResult => {}, + .System => |s| setStringField(L, "text", s.text.items), + .CompactionSummary => |s| setStringField(L, "text", s.text.items), + } +} + +fn pushUsage(L: *c.lua_State, u: panto.Usage) void { + c.lua_createtable(L, 0, 5); + setIntField(L, "input", @intCast(u.input)); + setIntField(L, "output", @intCast(u.output)); + setIntField(L, "cache_read", @intCast(u.cache_read)); + setIntField(L, "cache_write", @intCast(u.cache_write)); + setIntField(L, "reasoning", @intCast(u.reasoning)); + c.lua_setfield(L, -2, "usage"); +} + +fn roleName(role: panto.MessageRole) []const u8 { + return switch (role) { + .system => "system", + .user => "user", + .assistant => "assistant", + }; +} + +fn blockTypeName(t: panto.ContentBlockType) []const u8 { + return switch (t) { + .Text => "text", + .Thinking => "thinking", + .ToolUse => "tool_use", + .ToolResult => "tool_result", + }; +} + +fn contentBlockTypeName(block: panto.ContentBlock) []const u8 { + return switch (block) { + .Text => "text", + .Thinking => "thinking", + .ToolUse => "tool_use", + .ToolResult => "tool_result", + .System => "system", + .CompactionSummary => "compaction_summary", + }; +} + +// =========================================================================== +// JSON <-> Lua (ported from src/lua_bridge.zig; standalone copy so this +// package has no cross-package dependency on the CLI) +// =========================================================================== + +const JsonError = error{ OutOfMemory, BadHandlerReturn, InputNotJsonObject }; + +/// Parse JSON bytes (must be a top-level object — tool input convention) +/// and push the value onto `L`. +fn pushJsonAsLua(L: *c.lua_State, arena: std.mem.Allocator, input: []const u8) !void { + var parsed = std.json.parseFromSlice(std.json.Value, arena, input, .{}) catch + return JsonError.InputNotJsonObject; + defer parsed.deinit(); + if (parsed.value != .object) return JsonError.InputNotJsonObject; + try pushJsonValue(L, parsed.value); +} + +fn pushJsonValue(L: *c.lua_State, v: std.json.Value) !void { + switch (v) { + .null => c.lua_pushnil(L), + .bool => |b| c.lua_pushboolean(L, if (b) 1 else 0), + .integer => |i| c.lua_pushinteger(L, @intCast(i)), + .float => |f| c.lua_pushnumber(L, f), + .number_string => |s| { + if (std.fmt.parseInt(c.lua_Integer, s, 10)) |i| { + c.lua_pushinteger(L, i); + } else |_| { + const f = try std.fmt.parseFloat(c.lua_Number, s); + c.lua_pushnumber(L, f); + } + }, + .string => |s| _ = c.lua_pushlstring(L, s.ptr, s.len), + .array => |arr| { + c.lua_createtable(L, @intCast(arr.items.len), 0); + for (arr.items, 0..) |item, i| { + try pushJsonValue(L, item); + c.lua_rawseti(L, -2, @intCast(i + 1)); + } + }, + .object => |obj| { + c.lua_createtable(L, 0, @intCast(obj.count())); + var it = obj.iterator(); + while (it.next()) |kv| { + const key = kv.key_ptr.*; + _ = c.lua_pushlstring(L, key.ptr, key.len); + try pushJsonValue(L, kv.value_ptr.*); + c.lua_rawset(L, -3); + } + }, + } +} + +/// Read a handler's return at `idx` into owned `ResultParts`. A plain +/// string -> one text part; a table `{ text=, attachments={{media_type=, +/// data=}} }` -> optional text + one media part per attachment. +fn readHandlerResult(L: *c.lua_State, idx: c_int, allocator: std.mem.Allocator) !panto.ResultParts { + const ty = c.lua_type(L, idx); + if (ty == T_STRING) { + var len: usize = 0; + const ptr = c.lua_tolstring(L, idx, &len); + if (ptr == null) return JsonError.BadHandlerReturn; + return panto.ResultParts.fromText(allocator, ptr[0..len]); + } + if (ty != T_TABLE) { + // Coerce other scalars (numbers/bools) via tostring for ergonomics. + var len: usize = 0; + const ptr = c.luaL_tolstring(L, idx, &len); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (ptr == null) return JsonError.BadHandlerReturn; + return panto.ResultParts.fromText(allocator, ptr[0..len]); + } + return readHandlerResultTable(L, idx, allocator); +} + +fn readHandlerResultTable(L: *c.lua_State, idx: c_int, allocator: std.mem.Allocator) !panto.ResultParts { + var parts: std.ArrayList(panto.ResultPart) = .empty; + errdefer { + for (parts.items) |p| p.deinit(allocator); + parts.deinit(allocator); + } + const abs = c.lua_absindex(L, idx); + + // Optional `text`. + if (try optTableString(L, abs, "text")) |s| { + const owned = try allocator.dupe(u8, s); + c.lua_settop(L, c.lua_gettop(L) - 1); + errdefer allocator.free(owned); + try parts.append(allocator, .{ .text = owned }); + } + + // Optional `attachments`. + const at = c.lua_getfield(L, abs, "attachments"); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (at == T_TABLE) { + const n: usize = @intCast(c.lua_rawlen(L, -1)); + var i: usize = 1; + while (i <= n) : (i += 1) { + _ = c.lua_rawgeti(L, -1, @intCast(i)); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (c.lua_type(L, -1) != T_TABLE) return JsonError.BadHandlerReturn; + + const media_type: ?[]const u8 = if (try optTableString(L, -1, "media_type")) |mt| blk: { + const owned = try allocator.dupe(u8, mt); + c.lua_settop(L, c.lua_gettop(L) - 1); + break :blk owned; + } else null; + errdefer if (media_type) |mt| allocator.free(mt); + + const data_slice = (try optTableString(L, -1, "data")) orelse return JsonError.BadHandlerReturn; + const data = try allocator.dupe(u8, data_slice); + c.lua_settop(L, c.lua_gettop(L) - 1); + errdefer allocator.free(data); + + try parts.append(allocator, .{ .media = .{ .media_type = media_type, .data = data } }); + } + } else if (at != T_NIL) { + return JsonError.BadHandlerReturn; + } + + const items = try parts.toOwnedSlice(allocator); + return .{ .items = items }; +} + +/// Read string field `name` from the table at `tbl_idx`. Leaves the value +/// on the stack (caller pops after copying); null if absent. +fn optTableString(L: *c.lua_State, tbl_idx: c_int, name: [*:0]const u8) !?[]const u8 { + const t = c.lua_getfield(L, tbl_idx, name); + if (t == T_NIL) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return null; + } + if (t != T_STRING) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return JsonError.BadHandlerReturn; + } + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len); + if (ptr == null) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return JsonError.BadHandlerReturn; + } + return ptr[0..len]; +} + +/// Serialize the Lua table at `idx` to a JSON string (arena-owned). +fn serializeSchema(L: *c.lua_State, idx: c_int, arena: std.mem.Allocator) ![]const u8 { + var aw: std.Io.Writer.Allocating = .init(arena); + var s: std.json.Stringify = .{ .writer = &aw.writer }; + try writeLuaValueAsJson(L, idx, &s); + return aw.written(); +} + +const SchemaJsonError = error{ UnsupportedLuaType, UnsupportedLuaKey, WriteFailed }; + +fn writeLuaValueAsJson(L: *c.lua_State, idx: c_int, w: *std.json.Stringify) SchemaJsonError!void { + switch (c.lua_type(L, idx)) { + T_NIL => w.write(null) catch return error.WriteFailed, + T_BOOLEAN => w.write(c.lua_toboolean(L, idx) != 0) catch return error.WriteFailed, + T_NUMBER => { + var isnum: c_int = 0; + const as_int = c.lua_tointegerx(L, idx, &isnum); + if (isnum != 0) + w.write(as_int) catch return error.WriteFailed + else + w.write(c.lua_tonumberx(L, idx, null)) catch return error.WriteFailed; + }, + T_STRING => { + var len: usize = 0; + const ptr = c.lua_tolstring(L, idx, &len); + w.write(ptr[0..len]) catch return error.WriteFailed; + }, + T_TABLE => try writeLuaTableAsJson(L, idx, w), + else => return error.UnsupportedLuaType, + } +} + +fn writeLuaTableAsJson(L: *c.lua_State, idx_in: c_int, w: *std.json.Stringify) SchemaJsonError!void { + const abs = c.lua_absindex(L, idx_in); + const len = c.lua_rawlen(L, abs); + if (len > 0) { + w.beginArray() catch return error.WriteFailed; + var i: c.lua_Integer = 1; + while (i <= @as(c.lua_Integer, @intCast(len))) : (i += 1) { + _ = c.lua_rawgeti(L, abs, i); + try writeLuaValueAsJson(L, -1, w); + c.lua_settop(L, c.lua_gettop(L) - 1); + } + w.endArray() catch return error.WriteFailed; + return; + } + w.beginObject() catch return error.WriteFailed; + c.lua_pushnil(L); + while (c.lua_next(L, abs) != 0) { + if (c.lua_type(L, -2) != T_STRING) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.UnsupportedLuaKey; + } + var klen: usize = 0; + const kptr = c.lua_tolstring(L, -2, &klen); + w.objectField(kptr[0..klen]) catch return error.WriteFailed; + try writeLuaValueAsJson(L, -1, w); + c.lua_settop(L, c.lua_gettop(L) - 1); + } + w.endObject() catch return error.WriteFailed; +} + +// =========================================================================== +// Stack / argument helpers +// =========================================================================== + +fn checkAgent(L: *c.lua_State, idx: c_int) *AgentBox { + return @ptrCast(@alignCast(c.luaL_checkudata(L, idx, AGENT_MT))); +} + +fn checkStream(L: *c.lua_State, idx: c_int) *StreamBox { + return @ptrCast(@alignCast(c.luaL_checkudata(L, idx, STREAM_MT))); +} + +/// Raise a Lua error with a static message. Never returns (longjmp), typed +/// `c_int` so callers can `return` it. +fn luaErr(L: *c.lua_State, msg: [:0]const u8) c_int { + _ = c.luaL_error(L, "%s", msg.ptr); + return 0; +} + +fn requiredString(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError![]const u8 { + const t = c.lua_getfield(L, tbl_idx, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return error.MissingField; + if (t != T_STRING) return error.BadField; + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len); + if (ptr == null) return error.BadField; + return ptr[0..len]; +} + +fn optString(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError!?[]const u8 { + const t = c.lua_getfield(L, tbl_idx, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return null; + if (t != T_STRING) return error.BadField; + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len); + if (ptr == null) return error.BadField; + return ptr[0..len]; +} + +/// An optional positional string argument (for compact's override/extra). +fn optArgString(L: *c.lua_State, idx: c_int) ?[]const u8 { + if (c.lua_type(L, idx) != T_STRING) return null; + var len: usize = 0; + const ptr = c.lua_tolstring(L, idx, &len); + if (ptr == null) return null; + return ptr[0..len]; +} + +fn optU32(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError!?u32 { + const v = try optInteger(L, tbl_idx, name) orelse return null; + if (v < 0 or v > std.math.maxInt(u32)) return error.BadField; + return @intCast(v); +} + +fn optU64(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError!?u64 { + const v = try optInteger(L, tbl_idx, name) orelse return null; + if (v < 0) return error.BadField; + return @intCast(v); +} + +fn optInteger(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError!?c.lua_Integer { + const t = c.lua_getfield(L, tbl_idx, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return null; + if (t != T_NUMBER) return error.BadField; + var isnum: c_int = 0; + const v = c.lua_tointegerx(L, -1, &isnum); + if (isnum == 0) return error.BadField; + return v; +} + +fn optNumber(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError!?f64 { + const t = c.lua_getfield(L, tbl_idx, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return null; + if (t != T_NUMBER) return error.BadField; + return c.lua_tonumberx(L, -1, null); +} + +fn optBool(L: *c.lua_State, tbl_idx: c_int, name: [:0]const u8) ConfigError!?bool { + const t = c.lua_getfield(L, tbl_idx, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return null; + if (t != T_BOOLEAN) return error.BadField; + return c.lua_toboolean(L, -1) != 0; +} + +fn setStringField(L: *c.lua_State, key: [:0]const u8, value: []const u8) void { + _ = c.lua_pushlstring(L, value.ptr, value.len); + c.lua_setfield(L, -2, key.ptr); +} + +fn setIntField(L: *c.lua_State, key: [:0]const u8, value: c.lua_Integer) void { + c.lua_pushinteger(L, value); + c.lua_setfield(L, -2, key.ptr); +} + +// =========================================================================== +// Tests +// =========================================================================== + +const testing = std.testing; + +fn newTestState() !*c.lua_State { + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + c.luaL_openlibs(L); + _ = luaopen_panto(L); + c.lua_setglobal(L, "panto"); + return L; +} + +fn runScript(L: *c.lua_State, src: [:0]const u8) !void { + if (c.luaL_loadstring(L, src.ptr) != 0 or c.lua_pcallk(L, 0, c.LUA_MULTRET, 0, 0, null) != 0) { + var len: usize = 0; + const msg = c.lua_tolstring(L, -1, &len); + if (msg != null) std.debug.print("lua error: {s}\n", .{msg[0..len]}); + return error.LuaScriptFailed; + } +} + +test "luaopen_panto returns a module table with agent + _VERSION" { + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + c.luaL_openlibs(L); + + try testing.expectEqual(@as(c_int, 1), luaopen_panto(L)); + try testing.expectEqual(@as(c_int, T_TABLE), c.lua_type(L, -1)); + _ = c.lua_getfield(L, -1, "agent"); + try testing.expectEqual(@as(c_int, T_FUNCTION), c.lua_type(L, -1)); + c.lua_settop(L, c.lua_gettop(L) - 1); + _ = c.lua_getfield(L, -1, "_VERSION"); + try testing.expectEqual(@as(c_int, T_STRING), c.lua_type(L, -1)); +} + +test "luaopen_panto returns a FRESH table each call (not a shared singleton)" { + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + c.luaL_openlibs(L); + + _ = luaopen_panto(L); + _ = luaopen_panto(L); + _ = c.lua_pushstring(L, "marker"); + c.lua_setfield(L, -2, "_probe"); + _ = c.lua_getfield(L, -2, "_probe"); + try testing.expectEqual(@as(c_int, T_NIL), c.lua_type(L, -1)); +} + +test "agent: missing required fields raises (caught by pcall)" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local ok, err = pcall(function() return panto.agent { api_style = "openai_chat" } end) + \\assert(ok == false and type(err) == "string", "expected failure") + \\assert(err:find("panto.agent"), "expected contextual message") + ); +} + +test "agent: unknown api_style raises" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local ok = pcall(function() + \\ return panto.agent { api_style="nope", api_key="k", base_url="u", model="m" } + \\end) + \\assert(ok == false, "expected unknown api_style to fail") + ); +} + +test "agent: bad enum value raises" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local ok = pcall(function() + \\ return panto.agent { + \\ api_style="openai_chat", api_key="k", base_url="u", model="m", + \\ reasoning="banana", + \\ } + \\end) + \\assert(ok == false, "expected bad enum to fail") + ); +} + +test "agent: full config surface parses (anthropic + compaction + retry)" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local a = panto.agent { + \\ api_style = "anthropic_messages", + \\ api_key = "k", base_url = "http://127.0.0.1:1", model = "m", + \\ max_tokens = 1000, + \\ thinking = "adaptive", effort = "high", + \\ thinking_budget_tokens = 2048, thinking_interleaved = true, + \\ api_version = "2023-06-01", + \\ compaction = { keep_verbatim = 5000, prompt = "summarize" }, + \\ retry = { max_attempts = 2, initial_delay_ms = 100, max_delay_ms = 500, multiplier = 1.5, jitter = false }, + \\} + \\assert(type(a) == "userdata") + \\assert(type(a.session_id) == "function") + \\assert(type(a:session_id()) == "string") + ); +} + +test "agent: methods exist and stream surface works" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local a = panto.agent { api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m" } + \\for _, m in ipairs({"run","register_tool","set_config","add_system_message","set_system_prompt","compact","session_id"}) do + \\ assert(type(a[m]) == "function", "missing method "..m) + \\end + \\local s = a:run("hi") + \\assert(type(s) == "userdata") + \\local f, st, ctrl = s:events() + \\assert(type(f) == "function" and st == s and ctrl == nil) + ); +} + +test "agent: set_config swaps without error; system prompt setters work" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local a = panto.agent { api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m" } + \\a:add_system_message("be terse") + \\a:set_system_prompt("you are a test") + \\a:set_config { api_style="anthropic_messages", api_key="k2", base_url="http://127.0.0.1:2", model="m2" } + \\-- another swap, to exercise old-config retention + \\a:set_config { api_style="openai_chat", api_key="k3", base_url="http://127.0.0.1:3", model="m3" } + ); +} + +test "register_tool without luv fails loud at registration" { + // The test binary links Lua but NOT luv, so require('luv') fails. We + // assert that surfaces as a clear error at register time, not later. + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local a = panto.agent { api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m" } + \\local ok, err = pcall(function() + \\ a:register_tool { + \\ name="echo", description="echoes", schema={ type="object" }, + \\ handler=function(input) return "ok" end, + \\ } + \\end) + \\assert(ok == false, "expected register_tool to fail without luv") + \\assert(err:find("luv"), "error should mention luv: "..tostring(err)) + ); +} + +test "conversation: build standalone, inspect via messages()" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local conv = panto.conversation() + \\assert(type(conv) == "userdata") + \\conv:add_system_message("be terse") + \\conv:add_user_message("hello") + \\conv:add_user_message("again") + \\assert(conv:len() == 3, "expected 3 messages, got "..conv:len()) + \\assert(#conv == 3, "__len should work too") + \\local m = conv:messages() + \\assert(m[1].role == "system" and m[1].blocks[1].type == "system") + \\assert(m[1].blocks[1].text == "be terse") + \\assert(m[2].role == "user" and m[2].blocks[1].text == "hello") + \\assert(m[3].blocks[1].text == "again") + ); + try runScript(L, "collectgarbage('collect')"); // owned conv freed cleanly +} + +test "conversation: full-fidelity tool turn round-trips through messages()" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local conv = panto.conversation() + \\conv:add_user_message("use a tool") + \\conv:add_assistant_message { + \\ blocks = { + \\ { type="thinking", text="hmm", signature="sig" }, + \\ { type="text", text="calling" }, + \\ { type="tool_use", id="c1", name="echo", input='{"m":"x"}' }, + \\ }, + \\ usage = { input=10, output=20, reasoning=5 }, + \\} + \\conv:add_user_blocks { + \\ { type="tool_result", tool_use_id="c1", is_error=false, + \\ parts = { { text="echoed" }, { media_type="image/png", data="AAAA" } } }, + \\ { type="text", text="queued note" }, + \\} + \\assert(conv:len() == 3) + \\local m = conv:messages() + \\-- assistant message: usage + three typed blocks + \\assert(m[2].role == "assistant") + \\assert(m[2].usage.input == 10 and m[2].usage.reasoning == 5) + \\assert(m[2].blocks[1].type == "thinking" and m[2].blocks[1].signature == "sig") + \\assert(m[2].blocks[3].type == "tool_use" and m[2].blocks[3].id == "c1") + \\assert(m[2].blocks[3].input == '{"m":"x"}') + \\-- user tool-result message: result block + queued text + \\assert(m[3].blocks[1].type == "tool_result" and m[3].blocks[1].tool_use_id == "c1") + \\assert(#m[3].blocks[1].parts == 2) + \\assert(m[3].blocks[1].parts[1].text == "echoed") + \\assert(m[3].blocks[1].parts[2].media_type == "image/png" and m[3].blocks[1].parts[2].data == "AAAA") + \\assert(m[3].blocks[2].type == "text" and m[3].blocks[2].text == "queued note") + \\-- lossless replay into a fresh conversation + \\local c2 = panto.conversation() + \\for _, msg in ipairs(m) do + \\ if msg.role == "assistant" then c2:add_assistant_message { blocks = msg.blocks, usage = msg.usage } + \\ elseif msg.role == "user" then c2:add_user_blocks(msg.blocks) end + \\end + \\assert(c2:len() == conv:len(), "round-trip length mismatch") + \\local m2 = c2:messages() + \\assert(m2[2].blocks[3].id == "c1" and m2[3].blocks[1].parts[2].data == "AAAA") + ); + try runScript(L, "collectgarbage('collect')"); +} + +test "conversation: malformed block raises" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local conv = panto.conversation() + \\local ok = pcall(function() conv:add_user_blocks { { type="tool_use" } } end) -- missing id/name + \\assert(ok == false, "tool_use without id/name should raise") + \\local ok2 = pcall(function() conv:add_user_blocks { { type="frobnicate" } } end) + \\assert(ok2 == false, "unknown block type should raise") + ); +} + +test "conversation: adopt into agent for resumption, then operate via agent:conversation()" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local conv = panto.conversation() + \\conv:add_system_message("resumed prompt") + \\conv:add_user_message("earlier question") + \\local a = panto.agent { + \\ api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m", + \\ conversation = conv, + \\} + \\-- The standalone handle is now adopted; operating on it must raise. + \\local ok = pcall(function() conv:add_user_message("nope") end) + \\assert(ok == false, "adopted conversation should reject mutation") + \\-- The live view reflects the adopted history. + \\local live = a:conversation() + \\assert(live:len() == 2, "expected 2 adopted messages, got "..live:len()) + \\live:add_system_message("added live") + \\assert(live:len() == 3) + \\assert(a:conversation():messages()[3].blocks[1].text == "added live") + ); + try runScript(L, "collectgarbage('collect')"); +} + +test "conversation: reusing an adopted conversation in a second agent is rejected" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local conv = panto.conversation() + \\conv:add_user_message("x") + \\local a1 = panto.agent { api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m", conversation = conv } + \\local ok = pcall(function() + \\ return panto.agent { api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m", conversation = conv } + \\end) + \\assert(ok == false, "an already-adopted conversation must not be adopted again") + ); +} + +test "stream: a turn against an unreachable host surfaces as a Lua error" { + const L = try newTestState(); + defer c.lua_close(L); + try runScript(L, + \\local a = panto.agent { api_style="openai_chat", api_key="k", base_url="http://127.0.0.1:1", model="m" } + \\local s = a:run("hi") + \\local ok, err = pcall(function() for ev in s:events() do end end) + \\assert(ok == false and type(err) == "string", "unreachable host should raise") + ); +} + +test "json round-trip helpers: schema serialize + input parse" { + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + c.luaL_openlibs(L); + + // serializeSchema on a table. + try runScript(L, "schema = { type = \"object\", properties = { x = { type = \"number\" } } }"); + _ = c.lua_getglobal(L, "schema"); + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const json = try serializeSchema(L, -1, arena.allocator()); + try testing.expect(std.mem.indexOf(u8, json, "\"type\"") != null); + try testing.expect(std.mem.indexOf(u8, json, "\"object\"") != null); + c.lua_settop(L, c.lua_gettop(L) - 1); + + // pushJsonAsLua + readHandlerResult round trip. + try pushJsonAsLua(L, arena.allocator(), "{\"msg\":\"hello\"}"); + _ = c.lua_getfield(L, -1, "msg"); + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len); + try testing.expectEqualStrings("hello", ptr[0..len]); +} diff --git a/libpanto/src/agent.zig b/libpanto/src/agent.zig index 378205a..0252af6 100644 --- a/libpanto/src/agent.zig +++ b/libpanto/src/agent.zig @@ -55,6 +55,17 @@ pub const Config = config_mod.Config; /// usage per message, used for retention sizing). pub const conversation_Usage = @import("session.zig").Usage; +/// Append a single-text user message. `Conversation.addUserMessage` now +/// takes a block slice (symmetric with `addAssistantMessage`); this wraps +/// the common plain-text case used by the agent's interactive turn and the +/// compaction prompt. +fn addUserText(conv: *conversation.Conversation, text: []const u8) !void { + const tb = try conversation.textualBlockFromSlice(conv.allocator, text); + var block: conversation.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + /// Deep-copy a message (role + all content blocks) into fresh owned /// allocations. Used when rebuilding the conversation after compaction. fn cloneMessage(alloc: Allocator, msg: conversation.Message) !conversation.Message { @@ -373,7 +384,7 @@ pub const Agent = struct { // Append + persist the user prompt up front (the dangling-prompt // recovery guarantee). const user_start = self.conversation.messages.items.len; - try self.conversation.addUserMessage(message.text); + try addUserText(&self.conversation, message.text); try turn_persist.persistTurn( self._allocator, &self._session, @@ -816,7 +827,7 @@ pub const Agent = struct { var conv = conversation.Conversation.init(alloc); defer conv.deinit(); try conv.addSystemMessage(system_prompt); - try conv.addUserMessage(body); + try addUserText(&conv, body); // Drive one provider response to completion, ignoring every event. // Compaction doesn't need incremental output — the assembled message @@ -2681,11 +2692,11 @@ test "compact: summarizes prefix, keeps suffix, system survives" { const conv = &agent.conversation; try conv.addSystemMessage("you are helpful"); - try conv.addUserMessage("first question here with several words"); + try addUserText(conv, "first question here with several words"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "first answer with several words") }, }, null); - try conv.addUserMessage("second recent question"); + try addUserText(conv, "second recent question"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "second recent answer") }, }, null); @@ -2746,21 +2757,21 @@ test "compact: restated suffix usage reconstructs a fresh cumulative chain" { // Prefix turn (will be summarized). Cumulative footprint = 500+40+10+50 // = 600. Its real usage (with cache buckets) is irrelevant // post-compaction. - try conv.addUserMessage("first question here with several words"); + try addUserText(conv, "first question here with several words"); try conv.addAssistantMessage( &.{.{ .Text = try conversation.textualBlockFromSlice(allocator, "first answer with words") }}, .{ .input = 500, .output = 50, .cache_read = 40, .cache_write = 10 }, ); // Kept turn 1: cumulative 608 (delta 8 over prefix). Reasoning is a // subset of output. - try conv.addUserMessage("kept question"); // 2 words => ceil(2*1.3)=3 tokens + try addUserText(conv, "kept question"); // 2 words => ceil(2*1.3)=3 tokens try conv.addAssistantMessage( &.{.{ .Text = try conversation.textualBlockFromSlice(allocator, "kept answer") }}, .{ .input = 600, .output = 8, .reasoning = 5 }, ); // Kept turn 2: cumulative 614 (delta 6). Cache buckets present to prove // they collapse into `input` on restatement. - try conv.addUserMessage("two more words here"); // 4 words => ceil(4*1.3)=6 + try addUserText(conv, "two more words here"); // 4 words => ceil(4*1.3)=6 try conv.addAssistantMessage( &.{.{ .Text = try conversation.textualBlockFromSlice(allocator, "final answer") }}, .{ .input = 600, .output = 4, .cache_read = 8, .cache_write = 2 }, @@ -2818,7 +2829,7 @@ test "compact: no-op when conversation already fits the budget" { const conv = &agent.conversation; try conv.addSystemMessage("sys"); - try conv.addUserMessage("hi"); + try addUserText(conv, "hi"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "hello") }, }, null); @@ -2855,11 +2866,11 @@ test "compact: extra instructions are appended to the system prompt" { agent._open_stream_fn = stub.install(); const conv = &agent.conversation; - try conv.addUserMessage("question one two three"); + try addUserText(conv, "question one two three"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "answer one two three") }, }, null); - try conv.addUserMessage("question two"); + try addUserText(conv, "question two"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "answer two") }, }, null); @@ -2897,7 +2908,7 @@ test "runStep: auto-compacts on context overflow and retries once" { const conv = &agent.conversation; try conv.addSystemMessage("you are helpful"); - try conv.addUserMessage("first question with several words here"); + try addUserText(conv, "first question with several words here"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "first answer with several words") }, }, null); @@ -3277,7 +3288,7 @@ test "runStep: context-overflow compaction fires a compaction retry notification const conv = &agent.conversation; try conv.addSystemMessage("you are helpful"); - try conv.addUserMessage("first question with several words here"); + try addUserText(conv, "first question with several words here"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "first answer with several words") }, }, null); diff --git a/libpanto/src/anthropic_messages_json.zig b/libpanto/src/anthropic_messages_json.zig index 78f4b5f..3ad6069 100644 --- a/libpanto/src/anthropic_messages_json.zig +++ b/libpanto/src/anthropic_messages_json.zig @@ -606,6 +606,16 @@ fn testConfig(model: []const u8) config_mod.AnthropicMessagesConfig { }; } +/// Test helper: append a single-text user message. `addUserMessage` now +/// takes a block slice (symmetric with `addAssistantMessage`); this wraps +/// the common plain-text case the tests below use. +fn addUserText(conv: *conversation.Conversation, text: []const u8) !void { + const tb = try conversation.textualBlockFromSlice(conv.allocator, text); + var block: conversation.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "serializeRequest - system extracted into top-level field" { const allocator = testing.allocator; @@ -613,7 +623,7 @@ test "serializeRequest - system extracted into top-level field" { defer conv.deinit(); try conv.addSystemMessage("You are helpful."); - try conv.addUserMessage("Hello!"); + try addUserText(&conv, "Hello!"); const cfg = testConfig("claude-sonnet-4-20250514"); var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); @@ -650,7 +660,7 @@ test "serializeRequest - multiple system messages joined with horizontal rule" { try conv.addSystemMessage("Be terse."); try conv.addSystemMessage("Be accurate."); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); const cfg = testConfig("claude-x"); var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); @@ -676,7 +686,7 @@ test "serializeRequest - replace-mode system block wipes prior system text" { try conv.addSystemMessage("original append"); try conv.replaceSystemMessage("fresh seed"); try conv.addSystemMessage("fresh append"); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); const cfg = testConfig("claude-x"); var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); @@ -700,7 +710,7 @@ test "serializeRequest - trailing newlines stripped before the rule join" { try conv.addSystemMessage("line one\n\n"); try conv.addSystemMessage("line two\n"); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); const cfg = testConfig("claude-x"); var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); @@ -721,7 +731,7 @@ test "serializeRequest - no system messages omits the system field" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); const cfg = testConfig("claude-x"); var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); @@ -966,7 +976,7 @@ test "serializeRequest - emits tools array when registry non-empty" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("call something"); + try addUserText(&conv, "call something"); var tools = tool_registry_mod.ToolRegistry.init(allocator); defer tools.deinit(); @@ -1173,7 +1183,7 @@ test "serializeRequest - thinking disabled omits thinking and effort fields" { const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); cfg.thinking = .disabled; @@ -1193,7 +1203,7 @@ test "serializeRequest - thinking enabled explicit budget" { const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); // max_tokens = 1024 cfg.thinking = .enabled; @@ -1217,7 +1227,7 @@ test "serializeRequest - thinking enabled null budget falls back to max_tokens - const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); // max_tokens = 1024 cfg.thinking = .enabled; @@ -1237,7 +1247,7 @@ test "serializeRequest - thinking enabled budget clamped when >= max_tokens" { const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); // max_tokens = 1024 cfg.thinking = .enabled; @@ -1258,7 +1268,7 @@ test "serializeRequest - thinking adaptive default effort" { const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); cfg.thinking = .adaptive; @@ -1290,7 +1300,7 @@ test "serializeRequest - thinking adaptive explicit effort levels" { for (efforts) |entry| { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); cfg.thinking = .adaptive; @@ -1310,7 +1320,7 @@ test "serializeRequest - thinking adaptive ignores budget_tokens" { const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var cfg = testConfig("claude-x"); cfg.thinking = .adaptive; diff --git a/libpanto/src/conversation.zig b/libpanto/src/conversation.zig index a436378..1942a0c 100644 --- a/libpanto/src/conversation.zig +++ b/libpanto/src/conversation.zig @@ -265,10 +265,24 @@ pub const Conversation = struct { }); } - pub fn addUserMessage(self: *Conversation, text: []const u8) !void { - const tb = try textualBlockFromSlice(self.allocator, text); + /// Append a `user`-role message from a slice of content blocks. + /// Symmetric with `addAssistantMessage`: ownership of the blocks (and + /// every byte they reference) transfers to the conversation; the caller + /// must not deinit them after this call. This is the general user-side + /// builder — a user turn may carry plain text, multiple text blocks + /// (e.g. messages queued while the agent was mid-turn), one or more + /// `.ToolResult` blocks, or any mix. For the common single-text case, + /// build a one-element `.{ .Text = ... }` slice (see the `addUserText` + /// test helpers across the codebase). + /// + /// Blocks must use this conversation's allocator for any owned bytes, + /// since the conversation will free them with that allocator. + pub fn addUserMessage(self: *Conversation, blocks: []const ContentBlock) !void { var content: std.ArrayList(ContentBlock) = .empty; - try content.append(self.allocator, .{ .Text = tb }); + try content.ensureTotalCapacity(self.allocator, blocks.len); + for (blocks) |block| { + content.appendAssumeCapacity(block); + } try self.messages.append(self.allocator, .{ .role = .user, .content = content, @@ -401,6 +415,15 @@ pub fn activeMessageWindow(messages: []const Message) []const Message { return messages; } +/// Test helper: append a single-text user message. `addUserMessage` takes +/// a block slice; this wraps the plain-text case the tests below use. +fn addUserText(conv: *Conversation, text: []const u8) !void { + const tb = try textualBlockFromSlice(conv.allocator, text); + var block: ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "Conversation - add messages and verify content" { const allocator = std.testing.allocator; @@ -408,7 +431,7 @@ test "Conversation - add messages and verify content" { defer conv.deinit(); try conv.addSystemMessage("You are a helpful assistant."); - try conv.addUserMessage("Hello!"); + try addUserText(&conv, "Hello!"); try conv.addAssistantMessage(&.{ .{ .Text = try textualBlockFromSlice(allocator, "Hi there!") }, }, null); @@ -448,7 +471,7 @@ test "Conversation - deinit frees without leaks" { var conv = Conversation.init(allocator); try conv.addSystemMessage("system"); - try conv.addUserMessage("user message"); + try addUserText(&conv, "user message"); try conv.addAssistantMessage(&.{ .{ .Text = try textualBlockFromSlice(allocator, "response") }, }, null); @@ -494,7 +517,7 @@ test "effectiveSystemBlocks - append accumulates in order" { try conv.addSystemMessage("a"); try conv.addSystemMessage("b"); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); try conv.addSystemMessage("c"); var blocks = try effectiveSystemBlocks(allocator, conv.messages.items); @@ -548,7 +571,7 @@ test "addCompactionSummary - sits alone in a user message" { var conv = Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); try conv.addCompactionSummary("summary of earlier history"); const m = conv.messages.items[1]; @@ -567,7 +590,7 @@ test "latestCompactionIndex - null when never compacted" { defer conv.deinit(); try conv.addSystemMessage("sys"); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); try conv.addAssistantMessage(&.{ .{ .Text = try textualBlockFromSlice(allocator, "hello") }, }, null); @@ -582,11 +605,11 @@ test "latestCompactionIndex - returns the latest summary anchor" { defer conv.deinit(); try conv.addSystemMessage("sys"); - try conv.addUserMessage("old"); + try addUserText(&conv, "old"); try conv.addCompactionSummary("S1"); - try conv.addUserMessage("mid"); + try addUserText(&conv, "mid"); try conv.addCompactionSummary("S2"); // index 4 - try conv.addUserMessage("recent"); + try addUserText(&conv, "recent"); try std.testing.expectEqual(@as(?usize, 4), latestCompactionIndex(conv.messages.items)); } diff --git a/libpanto/src/file_system_jsonl_store.zig b/libpanto/src/file_system_jsonl_store.zig index c77d7d2..979b35e 100644 --- a/libpanto/src/file_system_jsonl_store.zig +++ b/libpanto/src/file_system_jsonl_store.zig @@ -1435,6 +1435,15 @@ fn anStamp() session_mod.WireStamp { return .{ .api_style = .anthropic_messages, .base_url = "https://api.anthropic.com", .model = "claude-sonnet-4-20250514" }; } +/// Test helper: append a single-text user message (see the `addUserMessage` +/// signature change to a block slice). +fn addUserText(conv: *conversation_mod.Conversation, text: []const u8) !void { + const tb = try conversation_mod.textualBlockFromSlice(conv.allocator, text); + var block: conversation_mod.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "newUuidV7: produces 36-char hyphenated string with version 7" { const io = testing.io; const id = try newUuidV7(testing.allocator, io); @@ -2058,7 +2067,7 @@ test "FileSystemJSONLStore catalog: create → append → load round-trips" { // messages owned here). var conv = conversation_mod.Conversation.init(testing.allocator); defer conv.deinit(); - try conv.addUserMessage("ping"); + try addUserText(&conv, "ping"); try conv.addAssistantMessage(&.{}, null); const id: session_store_mod.WireIdentity = .{ .api_style = .openai_chat, .base_url = "u", .model = "m" }; diff --git a/libpanto/src/openai_chat_json.zig b/libpanto/src/openai_chat_json.zig index b30b424..8af939f 100644 --- a/libpanto/src/openai_chat_json.zig +++ b/libpanto/src/openai_chat_json.zig @@ -535,6 +535,16 @@ fn emptyTools() tool_registry_mod.ToolRegistry { return tool_registry_mod.ToolRegistry.init(testing.allocator); } +/// Test helper: append a single-text user message. `addUserMessage` now +/// takes a block slice (symmetric with `addAssistantMessage`); this wraps +/// the common plain-text case the tests below use. +fn addUserText(conv: *conversation.Conversation, text: []const u8) !void { + const tb = try conversation.textualBlockFromSlice(conv.allocator, text); + var block: conversation.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "serializeRequest - system + user" { const allocator = testing.allocator; @@ -542,7 +552,7 @@ test "serializeRequest - system + user" { defer conv.deinit(); try conv.addSystemMessage("You are helpful."); - try conv.addUserMessage("Hello!"); + try addUserText(&conv, "Hello!"); const cfg = testConfig("gpt-4o"); var tools = emptyTools(); @@ -576,9 +586,9 @@ test "serializeRequest - multiple system blocks hoisted as separate leading mess defer conv.deinit(); try conv.addSystemMessage("seed"); - try conv.addUserMessage("Hello!"); + try addUserText(&conv, "Hello!"); try conv.addSystemMessage("mid-conversation append"); - try conv.addUserMessage("again"); + try addUserText(&conv, "again"); const cfg = testConfig("gpt-4o"); var tools = emptyTools(); @@ -611,7 +621,7 @@ test "serializeRequest - replace-mode system block wipes prior system messages" try conv.addSystemMessage("original"); try conv.replaceSystemMessage("fresh seed"); try conv.addSystemMessage("fresh append"); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); const cfg = testConfig("gpt-4o"); var tools = emptyTools(); @@ -663,7 +673,7 @@ test "serializeRequest - reasoning effort level included when set" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); var cfg = testConfig("gpt-4o"); cfg.reasoning = .high; @@ -687,7 +697,7 @@ test "serializeRequest - reasoning .off sends \"none\"" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("Hi"); + try addUserText(&conv, "Hi"); var cfg = testConfig("gpt-4o"); cfg.reasoning = .off; @@ -798,7 +808,7 @@ test "serializeRequest - emits tools array when registry non-empty" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("call something"); + try addUserText(&conv, "call something"); var tools = emptyTools(); defer tools.deinit(); @@ -833,7 +843,7 @@ test "serializeRequest - dotted tool name is wire-encoded with __" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("go"); + try addUserText(&conv, "go"); var tools = emptyTools(); defer tools.deinit(); @@ -1050,14 +1060,14 @@ test "serializeRequest - compaction summary trims superseded prefix" { // System prompt survives compaction. try conv.addSystemMessage("you are helpful"); // Old prefix that must be dropped. - try conv.addUserMessage("ancient question"); + try addUserText(&conv, "ancient question"); try conv.addAssistantMessage(&.{ .{ .Text = try conversation.textualBlockFromSlice(allocator, "ancient answer") }, }, null); // Compaction summary resets conversation context. try conv.addCompactionSummary("summary of the ancient exchange"); // Kept-verbatim suffix replayed after the summary. - try conv.addUserMessage("recent question"); + try addUserText(&conv, "recent question"); var tools = emptyTools(); defer tools.deinit(); diff --git a/libpanto/src/provider_anthropic_messages.zig b/libpanto/src/provider_anthropic_messages.zig index 8e41695..37ce818 100644 --- a/libpanto/src/provider_anthropic_messages.zig +++ b/libpanto/src/provider_anthropic_messages.zig @@ -707,12 +707,22 @@ fn runStreamedTurn( } } +/// Test helper: append a single-text user message. `addUserMessage` now +/// takes a block slice (symmetric with `addAssistantMessage`); this wraps +/// the common plain-text case the tests below use. +fn addUserText(conv: *conversation.Conversation, text: []const u8) !void { + const tb = try conversation.textualBlockFromSlice(conv.allocator, text); + var block: conversation.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "streams a text-only turn end-to-end" { const allocator = testing.allocator; var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hello"); + try addUserText(&conv, "hello"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -762,7 +772,7 @@ test "anthropic: captures usage from message_start and message_delta on message_ var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -805,7 +815,7 @@ test "anthropic: message_complete carries null usage when wire omits it" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -842,7 +852,7 @@ test "captures thinking signature for round-trip" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("solve"); + try addUserText(&conv, "solve"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -888,7 +898,7 @@ test "signature-only thinking block (display omitted)" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -925,7 +935,7 @@ test "ping and unknown events are ignored" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -962,7 +972,7 @@ test "tool_use blocks are captured with id, name, and assembled input" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("use a tool"); + try addUserText(&conv, "use a tool"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -1009,7 +1019,7 @@ test "inbound wire tool name is decoded to dotted form" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("use a tool"); + try addUserText(&conv, "use a tool"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -1036,7 +1046,7 @@ test "error event propagates as Zig error" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var queue = EventQueue.init(allocator); defer queue.deinit(); @@ -1069,7 +1079,7 @@ test "two streamed turns persist assistant replies in the conversation" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); try conv.addSystemMessage("Be brief."); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder.init(allocator); defer rec.deinit(); @@ -1088,7 +1098,7 @@ test "two streamed turns persist assistant replies in the conversation" { }; try runStreamedTurn(allocator, &conv, &rec, &turn1); - try conv.addUserMessage("what did you say?"); + try addUserText(&conv, "what did you say?"); const turn2 = [_][]const u8{ \\{"type":"message_start","message":{"role":"assistant"}} diff --git a/libpanto/src/provider_openai_chat.zig b/libpanto/src/provider_openai_chat.zig index ed646e8..8319d9d 100644 --- a/libpanto/src/provider_openai_chat.zig +++ b/libpanto/src/provider_openai_chat.zig @@ -794,6 +794,16 @@ const EventRecorder = struct { } }; +/// Test helper: append a single-text user message. `addUserMessage` now +/// takes a block slice (symmetric with `addAssistantMessage`); this wraps +/// the common plain-text case the tests below use. +fn addUserText(conv: *conversation.Conversation, text: []const u8) !void { + const tb = try conversation.textualBlockFromSlice(conv.allocator, text); + var block: conversation.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "two streamed turns persist assistant replies in the conversation" { // Regression test for the bug where `finish_reason` arrived before // `[DONE]` and `finalize` early-returned without appending the assistant @@ -805,7 +815,7 @@ test "two streamed turns persist assistant replies in the conversation" { defer conv.deinit(); try conv.addSystemMessage("You are a helpful assistant."); - try conv.addUserMessage("hello!"); + try addUserText(&conv, "hello!"); const turn1 = [_][]const u8{ \\{"choices":[{"delta":{"role":"assistant"}}]} @@ -828,7 +838,7 @@ test "two streamed turns persist assistant replies in the conversation" { ); // Second user turn: the assistant must still see its prior response. - try conv.addUserMessage("how did you respond to my greeting just now?"); + try addUserText(&conv, "how did you respond to my greeting just now?"); const turn2 = [_][]const u8{ \\{"choices":[{"delta":{"role":"assistant"}}]} @@ -855,7 +865,7 @@ test "openai_chat: terminating usage chunk lands on message_complete with split var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder{ .allocator = allocator }; defer rec.deinit(); @@ -888,7 +898,7 @@ test "openai_chat: omitted stream usage yields null on message_complete" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var rec = EventRecorder{ .allocator = allocator }; defer rec.deinit(); @@ -921,7 +931,7 @@ test "fragmented tool_call id and name are reassembled" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("call something"); + try addUserText(&conv, "call something"); const events = [_][]const u8{ \\{"choices":[{"delta":{"role":"assistant"}}]} @@ -956,7 +966,7 @@ test "inbound wire tool name is decoded to dotted form (even split across __)" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("read a file"); + try addUserText(&conv, "read a file"); const events = [_][]const u8{ \\{"choices":[{"delta":{"role":"assistant"}}]} @@ -991,7 +1001,7 @@ test "parallel tool_calls emit one complete start/delta/complete cycle per block var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("ping four hosts"); + try addUserText(&conv, "ping four hosts"); var rec: EventRecorder = .{ .allocator = allocator }; defer rec.deinit(); @@ -1057,7 +1067,7 @@ test "non-contiguous tool_call deltas: re-emission of a closed index is dropped" var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("go"); + try addUserText(&conv, "go"); var rec: EventRecorder = .{ .allocator = allocator }; defer rec.deinit(); @@ -1105,7 +1115,7 @@ test "onToolDetails fires after id+name complete, even mid-arg-stream" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("go"); + try addUserText(&conv, "go"); var rec: EventRecorder = .{ .allocator = allocator }; defer rec.deinit(); @@ -1171,7 +1181,7 @@ test "tool_call with no arguments still finalizes a well-formed ToolUse" { var conv = conversation.Conversation.init(allocator); defer conv.deinit(); - try conv.addUserMessage("ring it"); + try addUserText(&conv, "ring it"); const events = [_][]const u8{ \\{"choices":[{"delta":{"role":"assistant"}}]} diff --git a/libpanto/src/public.zig b/libpanto/src/public.zig index f9c2830..277b4ea 100644 --- a/libpanto/src/public.zig +++ b/libpanto/src/public.zig @@ -78,6 +78,11 @@ pub const ContentBlock = conversation_mod.ContentBlock; pub const Message = conversation_mod.Message; pub const MessageRole = conversation_mod.MessageRole; pub const TextualBlock = conversation_mod.TextualBlock; +/// Build a `TextualBlock` (the streaming text buffer used inside content +/// blocks) from a slice, copying the bytes with `alloc`. The canonical way +/// for binding code to construct `.Text`/`.Thinking`/tool-result text when +/// assembling `ContentBlock`s by hand (e.g. for conversation resumption). +pub const textualBlockFromSlice = conversation_mod.textualBlockFromSlice; pub const ThinkingBlock = conversation_mod.ThinkingBlock; pub const ToolUseBlock = conversation_mod.ToolUseBlock; pub const ToolResultBlock = conversation_mod.ToolResultBlock; diff --git a/src/lua_bridge.zig b/src/lua_bridge.zig index 9ee5089..6395293 100644 --- a/src/lua_bridge.zig +++ b/src/lua_bridge.zig @@ -102,12 +102,24 @@ pub var command_registrations_key: u8 = 0; pub var on_registrations_key: u8 = 0; /// The key under which we stash the canonical `panto` module table in -/// `LUA_REGISTRYINDEX`. The `package.preload['panto']` loader returns this -/// table, and internal Zig setup (`installEmit`, the runtime's +/// `LUA_REGISTRYINDEX`. Internal Zig setup (`installEmit`, the runtime's /// `_record_result` / wrapper / register_tool paths) fetches it via /// `pushPantoTable` instead of a global lookup — `panto` is not a global. +/// +/// This slot is populated by the runtime's `installPantoModule`, which +/// obtains the **native** `panto` table via `require('panto')` (resolving +/// the staged `panto.so` on `cpath`) and attaches `ext` to it. Until then +/// the slot is empty and `pushPantoTable` pushes `nil`; nothing fetches it +/// before module load. pub var panto_table_key: u8 = 0; +/// The key under which we stash the `panto.ext` extension subtable in +/// `LUA_REGISTRYINDEX`. Built by `install` (before any native module +/// exists) so the host can attach it to the native `panto` table once +/// `require('panto')` resolves. Distinct from `panto_table_key` because +/// `ext` is constructed first and outlives the module-load step. +pub var ext_table_key: u8 = 0; + /// A single declared tool, as harvested from a script's top-level call to /// `panto.ext.register_tool`. All slices reference Lua-owned strings on the /// state's stack/registry; copy them before closing the state. @@ -137,41 +149,68 @@ pub const OnRegistration = struct { // Public bridge API // --------------------------------------------------------------------------- -/// Push the canonical `panto` table (built by `install`) onto the stack -/// from its registry slot. Internal Zig setup uses this instead of a -/// global lookup, since `panto` is not a global — it is delivered to Lua -/// only through `require('panto')`. +/// Push the canonical `panto` module table onto the stack from its +/// registry slot. The slot is populated by the runtime's +/// `installPantoModule` (which `require`s the native module and attaches +/// `ext`); before that runs the slot is empty and this pushes `nil`. +/// Internal Zig setup uses this instead of a global lookup, since `panto` +/// is not a global — it is delivered to Lua only through `require('panto')`. pub fn pushPantoTable(L: *c.lua_State) void { _ = c.lua_rawgetp(L, LUA_REGISTRYINDEX, &panto_table_key); } -/// The `package.preload['panto']` loader. Returns the canonical `panto` -/// table so `require('panto')` yields it (and `panto.ext`) without a -/// global. Standard preload-loader contract: one return value. +/// Push the `panto.ext` extension subtable from its registry slot. Built +/// by `install`, so it is available before the native module loads (the +/// host attaches it to the native table in `installPantoModule`). +pub fn pushExtTable(L: *c.lua_State) void { + _ = c.lua_rawgetp(L, LUA_REGISTRYINDEX, &ext_table_key); +} + +/// Stash a `panto` module table (on the stack top) into `panto_table_key`. +/// Called by `installPantoModule` after it builds the native+`ext` table. +/// Does NOT pop — leaves the table on the stack for the caller. +pub fn setPantoTable(L: *c.lua_State) void { + c.lua_pushvalue(L, -1); // dup for the registry slot + c.lua_rawsetp(L, LUA_REGISTRYINDEX, &panto_table_key); +} + +/// Register a `package.preload['panto']` loader returning the `panto` +/// module table currently on the stack top. Does NOT pop the table — +/// leaves it in place for the caller. The loader returns the table stashed +/// under `panto_table_key`, so it must be called together with +/// `setPantoTable` (the runtime does both in `installPantoModule`). +pub fn installPantoPreload(L: *c.lua_State) void { + _ = c.lua_getglobal(L, "package"); + _ = c.lua_getfield(L, -1, "preload"); + c.lua_pushcclosure(L, pantoPreloadThunk, 0); + c.lua_setfield(L, -2, "panto"); // preload.panto = thunk + c.lua_settop(L, c.lua_gettop(L) - 2); // pop preload + package +} + +/// The `package.preload['panto']` loader: returns the augmented `panto` +/// table from `panto_table_key`. Standard preload-loader contract: one +/// return value. Any `require('panto')` after `installPantoModule` — from +/// an extension or otherwise — routes here (preload is searcher slot 1). fn pantoPreloadThunk(L_opt: ?*c.lua_State) callconv(.c) c_int { const L = L_opt.?; pushPantoTable(L); return 1; } -/// Install the `panto` table (with its `panto.ext` extension subtable) -/// into the given state, reachable from Lua via `require('panto')`. +/// Build the `panto.ext` extension subtable and the registry tables that +/// hold harvested tool/command/event-handler registrations. /// -/// There is no `panto` global. The table is stashed in the registry -/// (`panto_table_key`) and a loader is registered into -/// `package.preload['panto']`, so any `require('panto')` — from an -/// extension or otherwise — returns it. Internal Zig code reaches the -/// same table via `pushPantoTable`. +/// This does **not** build the `panto` module table and does **not** touch +/// `package.preload` — the `panto` table comes from the native +/// `libpanto-lua` module via `require('panto')`, wired by the runtime's +/// `installPantoModule` once the staged `panto.so` is on `cpath`. Here we +/// only construct `ext` (stashed under `ext_table_key`) so the host can +/// attach it to that native table later. /// /// All extension-authoring APIs (`register_tool`, `register_command`, -/// `on`, `emit`) live under `panto.ext`; the bare `panto` table is -/// reserved for the future libpanto API. `emit` is installed here as a +/// `on`, `emit`) live under `panto.ext`. `emit` is installed here as a /// safe no-op default; the long-lived runtime overrides it with a closure -/// carrying its context via `installEmit` so a Lua `emit` can drive the -/// native `EventBus`. -/// -/// Also creates the registry tables that hold harvested tool, command, and -/// event-handler registrations. +/// carrying its context via `installEmit` once the module table exists. pub fn install(L: *c.lua_State) void { // Create the registrations table: an array of records, each shaped // { name=, description=, schema_json=, handler= }. @@ -187,8 +226,7 @@ pub fn install(L: *c.lua_State) void { c.lua_createtable(L, 0, 0); c.lua_rawsetp(L, LUA_REGISTRYINDEX, &on_registrations_key); - // Build the `panto` global as { ext = { ... } }. - c.lua_createtable(L, 0, 1); // panto + // Build the `panto.ext` subtable and stash it for later attachment. c.lua_createtable(L, 0, 4); // panto.ext c.lua_pushcclosure(L, registerToolThunk, 0); c.lua_setfield(L, -2, "register_tool"); @@ -199,27 +237,8 @@ pub fn install(L: *c.lua_State) void { // Default `emit`: a no-op until the runtime installs the real one. c.lua_pushcclosure(L, emitNoopThunk, 0); c.lua_setfield(L, -2, "emit"); - // panto.ext = <ext table> - c.lua_setfield(L, -2, "ext"); - - // Stash the `panto` table in the registry (keeps internal Zig access - // working without a global), then register the preload loader so - // `require('panto')` returns it. The table is still on the stack top. - c.lua_pushvalue(L, -1); // dup `panto` for the registry slot - c.lua_rawsetp(L, LUA_REGISTRYINDEX, &panto_table_key); - // Stack: ..., panto. Register the preload loader, consuming `panto`. - installPreloadLoader(L); -} - -/// Register `pantoPreloadThunk` into `package.preload['panto']`. Consumes -/// the `panto` table currently on the stack top (pops it). -fn installPreloadLoader(L: *c.lua_State) void { - c.lua_settop(L, c.lua_gettop(L) - 1); // pop the leftover `panto` - _ = c.lua_getglobal(L, "package"); - _ = c.lua_getfield(L, -1, "preload"); - c.lua_pushcclosure(L, pantoPreloadThunk, 0); - c.lua_setfield(L, -2, "panto"); - c.lua_settop(L, c.lua_gettop(L) - 2); // pop preload + package + // Stash `ext` under its registry key, consuming it. + c.lua_rawsetp(L, LUA_REGISTRYINDEX, &ext_table_key); } /// Override `panto.ext.emit` with a closure that carries `ctx` as a @@ -235,12 +254,11 @@ pub fn installEmit( ctx: *anyopaque, emit_fn: *const fn (L_opt: ?*c.lua_State) callconv(.c) c_int, ) void { - pushPantoTable(L); - _ = c.lua_getfield(L, -1, "ext"); + pushExtTable(L); c.lua_pushlightuserdata(L, ctx); c.lua_pushcclosure(L, emit_fn, 1); c.lua_setfield(L, -2, "emit"); - c.lua_settop(L, c.lua_gettop(L) - 2); // pop ext + panto + c.lua_settop(L, c.lua_gettop(L) - 1); // pop ext } /// The default `panto.ext.emit`: a no-op. Replaced by `installEmit` once @@ -562,6 +580,24 @@ fn readHandlerResultTable( return .{ .items = items }; } +/// Test-only: fabricate a minimal `panto` module table `{ ext = <ext> }` +/// and wire `require('panto')` to it, mirroring what the runtime's +/// `installPantoModule` does in production (minus the native agent/stream +/// surface, which needs a staged `panto.so` unavailable in unit tests). +/// +/// Production code never calls this — it `require`s the real native +/// module. Unit tests that load extension scripts (which `require('panto')` +/// for `panto.ext.*`) call `install(L)` then this, so those scripts +/// resolve without a `.so` on disk. +pub fn installTestPantoTable(L: *c.lua_State) void { + c.lua_createtable(L, 0, 1); // panto + pushExtTable(L); + c.lua_setfield(L, -2, "ext"); // panto.ext = ext + installPantoPreload(L); // preload.panto -> panto_table_key + setPantoTable(L); // stash; leaves panto on stack + c.lua_settop(L, c.lua_gettop(L) - 1); // pop panto +} + // --------------------------------------------------------------------------- // Lua-callable C functions // --------------------------------------------------------------------------- @@ -885,24 +921,38 @@ fn readStringField( // Tests // --------------------------------------------------------------------------- -test "install creates panto.ext table with register_tool/register_command/on/emit" { +test "install creates the ext table with register_tool/register_command/on/emit" { const L = c.luaL_newstate() orelse return error.LuaInitFailed; defer c.lua_close(L); c.luaL_openlibs(L); install(L); - // No `panto` global: it is delivered via `require('panto')`. + // No `panto` global, and `install` alone does NOT build the module + // table — the native module supplies it via `require('panto')`, and + // the host attaches `ext` in `installPantoModule`. So before that + // wiring the module slot is empty. try std.testing.expectEqual(@as(c_int, T_NIL), c.lua_getglobal(L, "panto")); c.lua_settop(L, c.lua_gettop(L) - 1); pushPantoTable(L); - try std.testing.expectEqual(@as(c_int, T_TABLE), c.lua_type(L, -1)); - _ = c.lua_getfield(L, -1, "ext"); + try std.testing.expectEqual(@as(c_int, T_NIL), c.lua_type(L, -1)); + c.lua_settop(L, c.lua_gettop(L) - 1); + + // The `ext` table exists and carries the four authoring functions. + pushExtTable(L); try std.testing.expectEqual(@as(c_int, T_TABLE), c.lua_type(L, -1)); inline for (.{ "register_tool", "register_command", "on", "emit" }) |field| { _ = c.lua_getfield(L, -1, field); try std.testing.expectEqual(@as(c_int, T_FUNCTION), c.lua_type(L, -1)); c.lua_settop(L, c.lua_gettop(L) - 1); } + + // After the test wiring, `require('panto')` resolves to a `{ ext }` + // module table. + installTestPantoTable(L); + pushPantoTable(L); + try std.testing.expectEqual(@as(c_int, T_TABLE), c.lua_type(L, -1)); + _ = c.lua_getfield(L, -1, "ext"); + try std.testing.expectEqual(@as(c_int, T_TABLE), c.lua_type(L, -1)); } test "panto.ext.on records event registrations in order" { @@ -910,6 +960,7 @@ test "panto.ext.on records event registrations in order" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\local panto = require("panto") @@ -959,6 +1010,7 @@ test "register_command records name and description" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\local panto = require("panto") @@ -989,6 +1041,7 @@ test "register_command rejects a non-function handler" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\local panto = require("panto") @@ -1010,6 +1063,7 @@ test "register_tool records name, description, schema_json" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\local panto = require("panto") @@ -1044,6 +1098,7 @@ test "handler invocation: input parsed, result captured" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\local panto = require("panto") @@ -1080,6 +1135,7 @@ test "readHandlerResult: table with text and attachments" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\return { @@ -1109,6 +1165,7 @@ test "handler crash: error message surfaces via xpcall traceback hook" { defer c.lua_close(L); c.luaL_openlibs(L); install(L); + installTestPantoTable(L); const script = \\local panto = require("panto") diff --git a/src/lua_event_bridge.zig b/src/lua_event_bridge.zig index 1fbc82a..f6d67e0 100644 --- a/src/lua_event_bridge.zig +++ b/src/lua_event_bridge.zig @@ -738,6 +738,7 @@ test "Lua on-handler fires through the native bus into Lua" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -770,6 +771,7 @@ test "Lua-defined component renders lines through the bridged vtable" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -807,6 +809,7 @@ test "bridged component render truncates to the width contract" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -835,6 +838,7 @@ test "bridged component render error yields a safe fallback line, not a crash" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -862,6 +866,7 @@ test "wrap pattern: Lua reads the native default, wraps it, and renders through defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -933,6 +938,7 @@ test "skills-style claim-by-name (§7.5) works end-to-end through the Lua bridge defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1008,6 +1014,7 @@ test "native-passthrough get/set round-trip: setComponent(getComponent()) keeps defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1056,6 +1063,7 @@ test "Lua emit drives the native bus (custom event reaches a native handler)" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1093,6 +1101,7 @@ test "bridged render: a long error on a NARROW width still satisfies the width c defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1125,6 +1134,7 @@ test "bridged render: non-array / nil / non-string returns each yield a safe fal defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1162,6 +1172,7 @@ test "bridged render: empty array renders zero lines (no fallback)" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1189,6 +1200,7 @@ test "bridged render: UTF-8 line truncates on codepoint boundaries to the width" defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1222,6 +1234,7 @@ test "bridged firstLineChanged is cache-derived: append stays near the tail" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1270,6 +1283,7 @@ test "bridged firstLineChanged: a mid-line replace reports the changed line, shr defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1317,6 +1331,7 @@ test "bridged handleInput round-trips: a Lua method mutates state the next rende defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1363,6 +1378,7 @@ test "bridged component: invalidate frees the ref+cache eagerly; teardown is lea defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); // frees all tracked BridgedComponents @@ -1396,6 +1412,7 @@ test "lifecycle payload fields marshal to Lua: tool id/delta/input/output, think defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1453,6 +1470,7 @@ test "lifecycle handlers fire for every new event name" { defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1506,6 +1524,7 @@ test "claim-by-name at tool_details swaps over the start default; releasing the defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); @@ -1613,6 +1632,7 @@ test "intra-emit clobber: two handlers each mint a Lua component in ONE emit; bo defer c.lua_close(L); c.luaL_openlibs(L); lua_bridge.install(L); + lua_bridge.installTestPantoTable(L); var bridge = EventBridge.init(testing.allocator, L); defer bridge.deinit(); diff --git a/src/lua_runtime.zig b/src/lua_runtime.zig index e26d017..2574fae 100644 --- a/src/lua_runtime.zig +++ b/src/lua_runtime.zig @@ -130,6 +130,15 @@ pub const LuaRuntime = struct { // Install the real `panto.ext.emit` now that the bridge exists, so // a Lua `emit(name, data)` reaches the native bus once attached. lua_bridge.installEmit(L, @ptrCast(eb), lua_event_bridge.emitThunk); + + // In production, `require('panto')` is wired to the native module + // by `installPantoModule` (after bootstrap stages `panto.so`). + // Unit tests have no staged `.so`, and load extension scripts that + // `require('panto')` for `panto.ext.*`; give them a fabricated + // `{ ext = ... }` module table so those `require`s resolve. + if (@import("builtin").is_test) { + lua_bridge.installTestPantoTable(L); + } return self; } @@ -139,6 +148,62 @@ pub const LuaRuntime = struct { return self.event_bridge; } + /// Wire `require('panto')` to the native `libpanto-lua` module plus the + /// CLI's `ext` subtable. Run **after** luarocks bootstrap has staged + /// `panto.so` and configured `package.cpath`, and **before** + /// `installScheduler` (whose `_record_result` / wrapper closure live on + /// the module table). + /// + /// The sequence (all in Zig, via the C API, so the CLI never calls + /// `luaopen_panto` itself — it only speaks `require`): + /// + /// 1. `require('panto')` resolves the staged `panto.so` on `cpath`, + /// running its `luaopen_panto`, which returns a **fresh** table + /// (the agent/stream surface). + /// 2. Attach the `ext` subtable (built by `lua_bridge.install`) onto + /// that fresh native table — mutating the host's own copy, which + /// is safe precisely because the table is fresh, not shared. + /// 3. Install a `package.preload['panto']` loader returning that same + /// augmented table, so any *later* `require('panto')` (e.g. from + /// an extension) gets `ext` too. `require` already cached the + /// table in `package.loaded` from step 1; the preload entry wins + /// over `cpath` for any cache miss. + /// 4. Stash the augmented table under `panto_table_key` so internal + /// Zig (`pushPantoTable`, the scheduler) reaches it. + pub fn installPantoModule(self: *LuaRuntime) !void { + const L = self.L; + + // 1. require('panto') -> native agent/stream table on the stack. + _ = c.lua_getglobal(L, "require"); + _ = c.lua_pushlstring(L, "panto", 5); + if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: require('panto') failed (was panto.so staged on cpath?)"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + // Stack: ..., panto (native, fresh) + + // 2. panto.ext = <ext table>. + lua_bridge.pushExtTable(L); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 2); + return RuntimeError.LuaInitFailed; + } + c.lua_setfield(L, -2, "ext"); // panto.ext = ext; pops ext + // Stack: ..., panto (augmented) + + // 3. Set package.preload['panto'] to return this augmented table. + lua_bridge.installPantoPreload(L); // leaves `panto` on the stack + + // 4. Stash in the registry slot for internal Zig access. + lua_bridge.setPantoTable(L); // dups + stores; leaves `panto` + c.lua_settop(L, c.lua_gettop(L) - 1); // pop the leftover `panto` + } + /// Install the libuv-driven coroutine scheduler: /// - Register `panto._record_result` (C function with `self` as /// light-userdata upvalue) so the wrapper closure can hand @@ -148,7 +213,9 @@ pub const LuaRuntime = struct { /// - Cache `require("luv").run` for fast per-tick access. /// /// Must be called after luarocks bootstrap has installed luv, - /// otherwise the `require("luv")` step will fail. + /// otherwise the `require("luv")` step will fail. Also after + /// `installPantoModule`, since the scheduler writes onto the module + /// table. pub fn installScheduler(self: *LuaRuntime) !void { try installRecordResult(self); try installWrapperClosure(self); @@ -261,15 +328,15 @@ pub const LuaRuntime = struct { // // Push `panto.ext.register_tool` *first*, then load+run the chunk // so its return value naturally lands above it; calling pcall then - // consumes both in the right order. + // consumes both in the right order. We reach `register_tool` + // through the `ext` table directly (it exists from `install`, + // independent of whether the native module table has been wired). const L = self.L; - lua_bridge.pushPantoTable(L); - _ = c.lua_getfield(L, -1, "ext"); + lua_bridge.pushExtTable(L); _ = c.lua_getfield(L, -1, "register_tool"); - // Stack: ..., panto, ext, register_tool. Collapse to just - // register_tool by overwriting the lowest of the three. - c.lua_copy(L, -1, -3); // overwrite `panto` with `register_tool` - c.lua_settop(L, c.lua_gettop(L) - 2); // pop ext + duplicate + // Stack: ..., ext, register_tool. Collapse to just register_tool. + c.lua_copy(L, -1, -2); // overwrite `ext` with `register_tool` + c.lua_settop(L, c.lua_gettop(L) - 1); // pop duplicate // Stack: ..., register_tool if (c.luaL_loadfilex(L, path_z.ptr, null) != 0) { diff --git a/src/luarocks_runtime.zig b/src/luarocks_runtime.zig index e652096..7f01c19 100644 --- a/src/luarocks_runtime.zig +++ b/src/luarocks_runtime.zig @@ -38,6 +38,7 @@ const panto_home = @import("panto_home.zig"); const embedded_luarocks = @import("embedded_luarocks"); const embedded_lua_headers = @import("embedded_lua_headers"); const embedded_agent = @import("embedded_agent"); +const embedded_panto_so = @import("embedded_panto_so"); const lua_bridge = @import("lua_bridge.zig"); const c = lua_bridge.c; @@ -137,6 +138,7 @@ pub fn bootstrap( try panto_home.ensureDirsExist(layout, io); try stageLuaHeaders(allocator, io, layout); try stageAgentTree(allocator, io, layout); + try stagePantoModule(allocator, io, layout); try stageDefaultConfig(allocator, io, layout); const lua_wrapper_path = try writeLuaWrapper(allocator, io, layout, panto_executable_path); defer allocator.free(lua_wrapper_path); @@ -203,6 +205,33 @@ fn stageLuaHeaders( } // --------------------------------------------------------------------------- +// Stage the embedded native `panto.so` +// --------------------------------------------------------------------------- + +/// Write the binary-embedded `libpanto-lua` shared object to +/// `<tree>/lib/lua/<short>/panto.so`, which is already on `package.cpath` +/// (see `configurePackagePaths`). This guarantees the embedded VM's +/// `require('panto')` resolves to the native agent/stream module on a +/// cold, network-less machine — the module is *ours* and version-locked +/// to this binary's Lua, so unlike luv it cannot be fetched from luarocks. +/// +/// The `.so` leaves `lua_*` undefined and resolves them against the host +/// binary at `dlopen` time, identical to how a luarocks-installed C rock +/// (e.g. luv) works — `exe.rdynamic = true` keeps those symbols findable. +/// +/// Uses the same `writeIfDifferent` discipline as `stageLuaHeaders` so an +/// unchanged `.so` keeps its on-disk mtime across reruns. +fn stagePantoModule( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, +) !void { + var dir = try Io.Dir.cwd().openDir(io, layout.lib_lua_dir, .{}); + defer dir.close(io); + try writeIfDifferent(allocator, io, dir, "panto.so", embedded_panto_so.bytes); +} + +// --------------------------------------------------------------------------- // Stage the embedded `agent/` tree // --------------------------------------------------------------------------- @@ -311,11 +340,20 @@ fn writeIfDifferent( name: []const u8, contents: []const u8, ) !void { - if (dir.readFileAlloc(io, name, allocator, .limited(1 << 22))) |existing| { + // Size the read limit to the new content (plus a small margin) so it + // never truncates the comparison — staged files range from tiny + // headers to a multi-megabyte `panto.so`. A larger on-disk file can't + // match `contents` anyway, so capping at `contents.len + 1` is enough + // to detect inequality without reading unbounded bytes. + const limit = contents.len + 1; + if (dir.readFileAlloc(io, name, allocator, .limited(limit))) |existing| { defer allocator.free(existing); if (std.mem.eql(u8, existing, contents)) return; } else |err| switch (err) { error.FileNotFound => {}, + // A file larger than our limit definitionally differs from + // `contents`; fall through and rewrite. + error.StreamTooLong => {}, else => return err, } try dir.writeFile(io, .{ .sub_path = name, .data = contents }); diff --git a/src/main.zig b/src/main.zig index 34a687a..06fd29e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -355,6 +355,13 @@ pub fn main(init: std.process.Init) !void { ); } + // Bootstrap staged `panto.so` onto the embedded VM's cpath and + // configured `package.path`/`cpath`; now wire `require('panto')` to + // the native module + the CLI's `ext` subtable. Must run before + // `installScheduler` (which writes onto the module table) and before + // any extension loads (which `require('panto')`). + try rt.installPantoModule(); + // luv is installed (or already present) at this point; wire the // libuv-driven coroutine scheduler before any extensions get a // chance to register tools that might want to yield. diff --git a/src/system_prompt.zig b/src/system_prompt.zig index 6676442..9dce3e5 100644 --- a/src/system_prompt.zig +++ b/src/system_prompt.zig @@ -330,6 +330,15 @@ fn projectLayerDir(arena: Allocator, io: Io) ![]u8 { const testing = std.testing; +/// Test helper: append a single-text user message (the `addUserMessage` +/// signature now takes a block slice). +fn addUserText(conv: *panto.Conversation, text: []const u8) !void { + const tb = try panto.textualBlockFromSlice(conv.allocator, text); + var block: panto.ContentBlock = .{ .Text = tb }; + errdefer block.deinit(conv.allocator); + try conv.addUserMessage(&.{block}); +} + test "blocksEqual - length and content" { try testing.expect(blocksEqual(&.{ "a", "b" }, &.{ "a", "b" })); try testing.expect(!blocksEqual(&.{ "a", "b" }, &.{ "a", "c" })); @@ -344,7 +353,7 @@ test "effectiveConfigWindow - no replace anchors to leading system blocks" { try conv.addSystemMessage("seed"); try conv.addSystemMessage("append-1"); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); @@ -361,10 +370,10 @@ test "effectiveConfigWindow - anchors to last replace block" { defer conv.deinit(); try conv.addSystemMessage("old seed"); - try conv.addUserMessage("hi"); + try addUserText(&conv, "hi"); try conv.replaceSystemMessage("new seed"); try conv.addSystemMessage("new append"); - try conv.addUserMessage("again"); + try addUserText(&conv, "again"); var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); |
