From 576891dc2ec4d917932a4c396471d4bbbad90c8e Mon Sep 17 00:00:00 2001 From: T Date: Wed, 27 May 2026 08:04:04 -0600 Subject: Finish the hard parts of the lua makeover - bundle luarocks source in the panto binary - bootstrap process (intended for first `panto` run): - make ~/.local/share/panto/... - write out luarocks sources into it - run luarocks to install luv - new `panto bootstrap` command just runs the bootstrap - `panto bootstrap --force` removes everything and re-bootstraps - new `panto lua` command just runs panto's embedded lua --- build.zig | 251 +++++++++++-- build.zig.zon | 4 + build/gen_lua_anchor.zig | 144 ++++++++ build/gen_lua_headers_embed.zig | 68 ++++ build/gen_luarocks_embed.zig | 157 +++++++++ build/panto_lua_repl.c | 49 +++ docs/archive/LUA_MAKEOVER.md | 594 +++++++++++++++++++++++++++++++ docs/archive/ideas.md | 38 ++ docs/archive/phase-1.md | 449 +++++++++++++++++++++++ docs/archive/phase-2.md | 157 +++++++++ docs/archive/phase-3.md | 391 ++++++++++++++++++++ docs/phase-1.md | 449 ----------------------- docs/phase-2.md | 157 --------- docs/phase-3.md | 391 -------------------- ideas.md | 38 -- src/lua_runtime.zig | 522 +++++++++++++++++++++++++-- src/luarocks_runtime.zig | 763 ++++++++++++++++++++++++++++++++++++++++ src/main.zig | 46 +++ src/manifest.zig | 43 +++ src/panto_home.zig | 206 +++++++++++ src/self_exe.zig | 90 +++++ src/subcommand.zig | 231 ++++++++++++ 22 files changed, 4146 insertions(+), 1092 deletions(-) create mode 100644 build/gen_lua_anchor.zig create mode 100644 build/gen_lua_headers_embed.zig create mode 100644 build/gen_luarocks_embed.zig create mode 100644 build/panto_lua_repl.c create mode 100644 docs/archive/LUA_MAKEOVER.md create mode 100644 docs/archive/ideas.md create mode 100644 docs/archive/phase-1.md create mode 100644 docs/archive/phase-2.md create mode 100644 docs/archive/phase-3.md delete mode 100644 docs/phase-1.md delete mode 100644 docs/phase-2.md delete mode 100644 docs/phase-3.md delete mode 100644 ideas.md create mode 100644 src/luarocks_runtime.zig create mode 100644 src/manifest.zig create mode 100644 src/panto_home.zig create mode 100644 src/self_exe.zig create mode 100644 src/subcommand.zig diff --git a/build.zig b/build.zig index e675859..5bf5d44 100644 --- a/build.zig +++ b/build.zig @@ -1,5 +1,9 @@ const std = @import("std"); +const lua_version = "5.4.7"; +const lua_short_version = "5.4"; +const luarocks_version = "3.13.0"; + pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); @@ -9,10 +13,31 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); - // Fetch upstream Lua 5.4.7 (source-only tarball from lua.org). - // Reproducibility comes from the content-addressed hash in build.zig.zon. + // Fetch upstream Lua source (used both for our static library and + // staged at runtime as the `include/` headers under $PANTO_HOME). + // Reproducibility comes from the content-addressed hash in + // build.zig.zon. const lua_src = b.dependency("lua_src", .{}); - const lua = buildLua(b, target, optimize, lua_src); + const lua_anchor_path = generateLuaAnchor(b, lua_src); + + // Fetch luarocks source. We don't compile any of it — it's pure + // Lua. The codegen step below produces a `embedded_luarocks.zig` + // module that `@embedFile`s every script so the runtime can serve + // them as `require("luarocks.*")` results without disk I/O. + const luarocks_src = b.dependency("luarocks_src", .{}); + const luarocks_embed_path = generateLuarocksEmbed(b, luarocks_src); + + // Same trick for the Lua headers: bundle them so first-run + // bootstrap can stage them under $PANTO_HOME/rocks/lua-X.Y.Z/include/ + // for luarocks to find when compiling C rocks. + const lua_headers_embed_path = generateLuaHeadersEmbed(b, lua_src); + + // Constants module: makes Zig know the Lua and luarocks versions + // without duplicating literals everywhere. + const versions_mod = b.addOptions(); + versions_mod.addOption([]const u8, "lua_version", lua_version); + versions_mod.addOption([]const u8, "lua_short_version", lua_short_version); + versions_mod.addOption([]const u8, "luarocks_version", luarocks_version); // CLI executable const exe_mod = b.createModule(.{ @@ -22,13 +47,42 @@ pub fn build(b: *std.Build) void { .link_libc = true, }); exe_mod.addImport("panto", panto_lib_dep.module("panto")); - exe_mod.linkLibrary(lua); + exe_mod.addImport("versions", versions_mod.createModule()); + exe_mod.addImport("embedded_luarocks", b.createModule(.{ + .root_source_file = luarocks_embed_path, + })); + exe_mod.addImport("embedded_lua_headers", b.createModule(.{ + .root_source_file = lua_headers_embed_path, + })); + // Link Lua. We can't use a static library here because the + // standard archive-linking behavior drops object files whose + // symbols nothing in our code references — e.g. `lua_settable`, + // which luv.so needs at runtime via `dlopen` but neither panto + // itself nor the embedded lua.c repl ever call directly. Instead, + // compile the Lua sources directly *into* the executable's own + // module so every object lands in the binary. `rdynamic = true` + // below ensures the symbols are exported into the dynamic symbol + // table for `dlopen`'d rocks to resolve. + addLuaSources(exe_mod, lua_src, lua_anchor_path); exe_mod.addIncludePath(lua_src.path("src")); + // Compile lua.c (the upstream standalone interpreter, ~600 lines) + // as a separate static library so we can route the `panto lua` + // subcommand into its main() without it conflicting with our own + // main(). + const lua_repl = buildLuaRepl(b, target, optimize, lua_src); + exe_mod.linkLibrary(lua_repl); + const exe = b.addExecutable(.{ .name = "panto", .root_module = exe_mod, }); + // C rocks (luv.so and anything else loaded via dlopen) need + // Lua's C API symbols to be findable in the running process's + // symbol table. Without `-rdynamic`, the linker strips them as + // unused-from-outside; `dlopen` then fails at load time with + // "symbol not found in flat namespace '_lua_settable'" etc. + exe.rdynamic = true; b.installArtifact(exe); @@ -41,7 +95,7 @@ pub fn build(b: *std.Build) void { const run_step = b.step("run", "Run the app"); run_step.dependOn(&run_cmd.step); - // CLI unit tests (minimal — most tests live in libpanto) + // CLI unit tests const test_mod = b.createModule(.{ .root_source_file = b.path("src/main.zig"), .target = target, @@ -49,8 +103,16 @@ pub fn build(b: *std.Build) void { .link_libc = true, }); test_mod.addImport("panto", panto_lib_dep.module("panto")); - test_mod.linkLibrary(lua); + test_mod.addImport("versions", versions_mod.createModule()); + test_mod.addImport("embedded_luarocks", b.createModule(.{ + .root_source_file = luarocks_embed_path, + })); + test_mod.addImport("embedded_lua_headers", b.createModule(.{ + .root_source_file = lua_headers_embed_path, + })); + addLuaSources(test_mod, lua_src, lua_anchor_path); test_mod.addIncludePath(lua_src.path("src")); + test_mod.linkLibrary(lua_repl); const unit_tests = b.addTest(.{ .name = "panto-tests", @@ -66,16 +128,20 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&lib_test_step.step); } -/// Compile Lua 5.4.7 as a static library against the upstream source tarball. -/// Excludes `lua.c` (interpreter REPL) and `luac.c` (bytecode compiler) — we -/// only want the embedded library. -fn buildLua( +/// Compile a thin wrapper around the upstream `lua.c` standalone +/// interpreter that exposes `pmain` for panto's `lua` subcommand to +/// invoke against a pre-configured `lua_State`. +/// +/// We rename `main` to `lua_unused_main` so the symbol doesn't collide +/// with panto's `main` (the wrapper never references it; the linker +/// just needs a place for it to land). +fn buildLuaRepl( b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, lua_src: *std.Build.Dependency, ) *std.Build.Step.Compile { - const lua_mod = b.createModule(.{ + const mod = b.createModule(.{ .target = target, .optimize = optimize, .link_libc = true, @@ -84,33 +150,176 @@ fn buildLua( const cflags = [_][]const u8{ "-std=gnu99", "-Wall", - "-Wextra", "-Wno-unused-parameter", + "-Wno-unused-function", + // Rename upstream `main` so it doesn't collide with panto's. + // We never call it; the wrapper invokes `pmain` directly. + "-Dmain=lua_unused_main", }; - lua_mod.addCSourceFiles(.{ + mod.addCSourceFile(.{ + .file = b.path("build/panto_lua_repl.c"), + .flags = &cflags, + }); + // The wrapper `#include`s lua.c by name; we add lua_src/src to + // the search path so `#include "lua.c"` resolves. + mod.addIncludePath(lua_src.path("src")); + addLuaPlatformMacros(mod, target); + + return b.addLibrary(.{ + .name = "lua-repl", + .linkage = .static, + .root_module = mod, + }); +} + +/// Add all the Lua 5.4 core + standard-library C sources directly to +/// `mod`. Compared to building a separate static library and linking, +/// this guarantees the linker doesn't drop "unused" objects — we need +/// every Lua API symbol present in the final binary so C rocks loaded +/// via `dlopen` can resolve them against the host process's symbol +/// table. +fn addLuaSources( + mod: *std.Build.Module, + lua_src: *std.Build.Dependency, + lua_anchor_path: std.Build.LazyPath, +) void { + const cflags = [_][]const u8{ + "-std=gnu99", + "-Wall", + "-Wextra", + "-Wno-unused-parameter", + }; + mod.addCSourceFiles(.{ .root = lua_src.path("src"), .files = &lua_files, .flags = &cflags, }); + // Tiny extra TU (codegen'd at build time): takes the address of + // every Lua API function into a `__attribute__((used))` array. + // This forces the linker to keep every API function in the binary + // even under ReleaseFast's LTO and gc-sections, regardless of + // whether our own code references them. C rocks loaded via + // `dlopen` (luv.so etc.) need every public Lua symbol resolvable + // in the host process's symbol table; combined with + // `rdynamic = true` on the executable, this gets them all into + // the dynamic symbol table. + mod.addCSourceFile(.{ + .file = lua_anchor_path, + .flags = &cflags, + }); + addLuaPlatformMacros(mod, mod.resolved_target.?); +} +fn addLuaPlatformMacros( + mod: *std.Build.Module, + target: std.Build.ResolvedTarget, +) void { // Tell Lua which platform features are available. The macros change // which `dlopen`/readline/etc. paths Lua compiles in. switch (target.result.os.tag) { - .macos => lua_mod.addCMacro("LUA_USE_MACOSX", ""), - .linux => lua_mod.addCMacro("LUA_USE_LINUX", ""), - .freebsd, .netbsd, .openbsd => lua_mod.addCMacro("LUA_USE_POSIX", ""), + .macos => mod.addCMacro("LUA_USE_MACOSX", ""), + .linux => mod.addCMacro("LUA_USE_LINUX", ""), + .freebsd, .netbsd, .openbsd => mod.addCMacro("LUA_USE_POSIX", ""), else => {}, } +} - return b.addLibrary(.{ - .name = "lua", - .linkage = .static, - .root_module = lua_mod, +/// Codegen step: walk the luarocks tarball, emit a Zig module that +/// exposes every Lua source as a `@embedFile`d entry plus a top-level +/// table mapping `require` paths (e.g. `"luarocks.core.cfg"`) to those +/// contents. The runtime installs this table into `package.preload` so +/// `require("luarocks.*")` resolves without touching disk. +/// +/// The generator emits `@embedFile("luarocks/<...>.lua")`-style paths. +/// We then materialize the luarocks source tree alongside the +/// generated Zig file via `addCopyDirectory`, so the embedded paths +/// resolve against the same root the module compiles from. (`@embedFile` +/// only allows reading files inside the module's import-path tree.) +/// +/// Two outputs are present in the resulting module: +/// - `pub const files: []const Entry` — { require_name, contents } +/// - `pub const luarocks_main` / `pub const luarocks_admin_main` +/// for the two `src/bin/` driver scripts. +fn generateLuaAnchor( + b: *std.Build, + lua_src: *std.Build.Dependency, +) std.Build.LazyPath { + const tool = b.addExecutable(.{ + .name = "gen-lua-anchor", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_lua_anchor.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), + }); + + const run = b.addRunArtifact(tool); + run.addDirectoryArg(lua_src.path("src")); + return run.addOutputFileArg("lua_anchor.c"); +} + +fn generateLuarocksEmbed( + b: *std.Build, + luarocks_src: *std.Build.Dependency, +) std.Build.LazyPath { + const tool = b.addExecutable(.{ + .name = "gen-luarocks-embed", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_luarocks_embed.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); + + const run = b.addRunArtifact(tool); + // Pass a marker arg the tool will treat as the "embed-relative" + // subpath — every `@embedFile` reference uses this prefix so + // they resolve against the same directory the generated zig file + // lives in (the one we set up next via addWriteFiles). + run.addArg("luarocks_src"); + run.addDirectoryArg(luarocks_src.path("src")); + const gen_file = run.addOutputFileArg("embedded_luarocks.zig"); + + // Stage the generated zig file next to a copy of the luarocks src + // tree, so `@embedFile("luarocks_src/...")` resolves. + const wf = b.addWriteFiles(); + const out_zig = wf.addCopyFile(gen_file, "embedded_luarocks.zig"); + _ = wf.addCopyDirectory(luarocks_src.path("src"), "luarocks_src", .{}); + return out_zig; +} + +/// Codegen step: emit a Zig module that exposes every Lua public header +/// from the Lua source tarball as embedded bytes. The bootstrap stages +/// these under `$PANTO_HOME/rocks/lua-X.Y.Z/include/` so luarocks can +/// compile C rocks against them. +fn generateLuaHeadersEmbed( + b: *std.Build, + lua_src: *std.Build.Dependency, +) std.Build.LazyPath { + const tool = b.addExecutable(.{ + .name = "gen-lua-headers-embed", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_lua_headers_embed.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), + }); + + const run = b.addRunArtifact(tool); + // Marker subpath: each emitted `@embedFile` reference is + // `lua_src/
`, and we stage the headers under that name. + run.addArg("lua_src"); + run.addDirectoryArg(lua_src.path("src")); + const gen_file = run.addOutputFileArg("embedded_lua_headers.zig"); + + const wf = b.addWriteFiles(); + const out_zig = wf.addCopyFile(gen_file, "embedded_lua_headers.zig"); + _ = wf.addCopyDirectory(lua_src.path("src"), "lua_src", .{}); + return out_zig; } -// Lua 5.4.7 core VM + standard library. Excludes lua.c (interpreter entry +// Lua 5.4.x core VM + standard library. Excludes lua.c (interpreter entry // point) and luac.c (compiler entry point). const lua_files = [_][]const u8{ // Core VM diff --git a/build.zig.zon b/build.zig.zon index 82794fc..f7eb08b 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -10,6 +10,10 @@ .url = "https://www.lua.org/ftp/lua-5.4.7.tar.gz", .hash = "N-V-__8AAIMvFABt-Qcpk24RD10ldEN743D8Q2e19Er8x3dJ", }, + .luarocks_src = .{ + .url = "https://luarocks.github.io/luarocks/releases/luarocks-3.13.0.tar.gz", + .hash = "N-V-__8AADZhHwD_K21NLLYGySn7TnLhpLyP8ca2JnXSLgbp", + }, }, .paths = .{ "build.zig", diff --git a/build/gen_lua_anchor.zig b/build/gen_lua_anchor.zig new file mode 100644 index 0000000..1143c2c --- /dev/null +++ b/build/gen_lua_anchor.zig @@ -0,0 +1,144 @@ +//! Build-time codegen: scan Lua's `lua.h` and `lauxlib.h` for every +//! `LUA_API` / `LUALIB_API` function declaration, and emit a C source +//! file that takes the address of each into a single `__attribute__ +//! ((used))` array. +//! +//! Result: the linker can't drop those functions even under LTO + +//! gc-sections, because they're reachable from the surviving array. +//! Combined with `rdynamic = true` on the executable, every Lua API +//! function ends up in the dynamic symbol table for C rocks (luv.so +//! etc.) to resolve at `dlopen` time. +//! +//! Invoked from `build.zig`: +//! +//! gen-lua-anchor + +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); + const src_dir = args.next() orelse return error.MissingSrcDir; + const out_path = args.next() orelse return error.MissingOutPath; + + var names: std.array_list.Managed([]const u8) = .init(arena); + try collectFrom(arena, io, src_dir, "lua.h", &names); + try collectFrom(arena, io, src_dir, "lauxlib.h", &names); + + // Deduplicate via a string set. + var seen: std.StringHashMapUnmanaged(void) = .empty; + try seen.ensureTotalCapacity(arena, @intCast(names.items.len * 2)); + var unique: std.array_list.Managed([]const u8) = .init(arena); + for (names.items) |n| { + const gop = seen.getOrPutAssumeCapacity(n); + if (!gop.found_existing) try unique.append(n); + } + std.mem.sort([]const u8, unique.items, {}, lessThanString); + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.writeAll( + \\/* Auto-generated by build/gen_lua_anchor.zig. Do not edit. + \\ * + \\ * Holds a reference to every Lua public API function so the + \\ * linker can't drop them under LTO + gc-sections. C rocks + \\ * loaded at runtime (luv.so etc.) resolve these symbols + \\ * against the host process's symbol table; combined with + \\ * rdynamic on the executable, this keeps the whole API + \\ * surface exported. + \\ */ + \\ + \\#include "lua.h" + \\#include "lauxlib.h" + \\ + \\__attribute__((used, visibility("default"))) + \\const void *const _panto_lua_api_anchor[] = { + \\ + ); + for (unique.items) |n| { + try w.print(" (const void *)&{s},\n", .{n}); + } + try w.writeAll("};\n"); + try w.flush(); +} + +fn lessThanString(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); +} + +fn collectFrom( + arena: std.mem.Allocator, + io: std.Io, + src_dir: []const u8, + filename: []const u8, + out: *std.array_list.Managed([]const u8), +) !void { + const path = try std.fs.path.join(arena, &.{ src_dir, filename }); + const contents = try std.Io.Dir.cwd().readFileAlloc(io, path, arena, .unlimited); + + // Walk line by line. When we hit a line starting with `LUA_API`, + // accumulate continuation lines until we see `);`. From the joined + // string, extract the function name \u2014 either parenthesized + // (`(lua_foo)`) or bare (`lua_foo(...)`). + var line_it = std.mem.splitScalar(u8, contents, '\n'); + while (line_it.next()) |first_line| { + const trimmed = std.mem.trimStart(u8, first_line, " \t"); + if (!std.mem.startsWith(u8, trimmed, "LUA_API") and + !std.mem.startsWith(u8, trimmed, "LUALIB_API")) continue; + + var joined: std.array_list.Managed(u8) = .init(arena); + try joined.appendSlice(first_line); + while (std.mem.indexOf(u8, joined.items, ");") == null) { + const next_line = line_it.next() orelse break; + try joined.append(' '); + try joined.appendSlice(next_line); + } + + if (extractName(joined.items)) |name| { + try out.append(try arena.dupe(u8, name)); + } + } +} + +/// Return the function-name portion of a Lua API declaration. Handles +/// both `LUA_API (lua_foo) (...)` and `LUA_API lua_foo(...)`. +fn extractName(decl: []const u8) ?[]const u8 { + // Find a `(name)` token first. + var i: usize = 0; + while (i + 1 < decl.len) : (i += 1) { + if (decl[i] != '(') continue; + const end = std.mem.indexOfScalar(u8, decl[i + 1 ..], ')') orelse return null; + const candidate = decl[i + 1 .. i + 1 + end]; + if (isApiName(candidate)) return candidate; + // Skip this candidate; try further on. + } + // Fall back: bare-name pattern (no parens around the name). + var j: usize = 0; + while (j < decl.len) : (j += 1) { + if (j + 4 > decl.len) break; + if (!std.mem.eql(u8, decl[j .. j + 4], "lua_") and + !std.mem.eql(u8, decl[j .. j + 5], "luaL_")) continue; + var end = j; + while (end < decl.len and (std.ascii.isAlphanumeric(decl[end]) or decl[end] == '_')) end += 1; + return decl[j..end]; + } + return null; +} + +fn isApiName(s: []const u8) bool { + if (s.len < 4) return false; + const ok_prefix = std.mem.startsWith(u8, s, "lua_") or std.mem.startsWith(u8, s, "luaL_"); + if (!ok_prefix) return false; + for (s) |ch| { + if (!std.ascii.isAlphanumeric(ch) and ch != '_') return false; + } + return true; +} diff --git a/build/gen_lua_headers_embed.zig b/build/gen_lua_headers_embed.zig new file mode 100644 index 0000000..1ac4004 --- /dev/null +++ b/build/gen_lua_headers_embed.zig @@ -0,0 +1,68 @@ +//! Build-time codegen: emit a Zig module exposing every Lua public +//! header from the Lua source tarball as embedded bytes. Bootstrap +//! writes these to `$PANTO_HOME/rocks/lua-X.Y.Z/include/` on first run +//! so luarocks can compile C rocks against them. +//! +//! Invoked from `build.zig`: +//! +//! gen-lua-headers-embed +//! +//! `` is the `src/` directory of the Lua tarball. + +const std = @import("std"); + +const headers = [_][]const u8{ + "lua.h", + "luaconf.h", + "lualib.h", + "lauxlib.h", + // hpp is the C++-friendly include shim shipped upstream; some C + // rocks reference it. + "lua.hpp", +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); + const embed_prefix = args.next() orelse return error.MissingEmbedPrefix; + const src_dir = args.next() orelse return error.MissingSrcDir; + const out_path = args.next() orelse return error.MissingOutPath; + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.writeAll( + \\//! Auto-generated. Do not edit. See build/gen_lua_headers_embed.zig. + \\ + \\pub const Entry = struct { + \\ name: []const u8, + \\ contents: []const u8, + \\}; + \\ + \\pub const files: []const Entry = &.{ + \\ + ); + + for (headers) |name| { + const abs = try std.fs.path.join(arena, &.{ src_dir, name }); + // Verify the file exists; emit nothing for missing files (lua.hpp + // exists in current Lua releases but we don't want to hard-fail + // if the tarball ever drops it). + std.Io.Dir.cwd().access(io, abs, .{}) catch continue; + const embed_path = try std.fs.path.join(arena, &.{ embed_prefix, name }); + try w.print( + " .{{ .name = \"{s}\", .contents = @embedFile(\"{s}\") }},\n", + .{ name, embed_path }, + ); + } + + try w.writeAll("};\n"); + try w.flush(); +} diff --git a/build/gen_luarocks_embed.zig b/build/gen_luarocks_embed.zig new file mode 100644 index 0000000..3f85177 --- /dev/null +++ b/build/gen_luarocks_embed.zig @@ -0,0 +1,157 @@ +//! Build-time codegen: walk the luarocks `src/` tree and emit a Zig +//! module that exposes every Lua file as an `@embedFile` entry, with a +//! comptime table mapping `require`-style module names to contents. +//! +//! Invoked from `build.zig`: +//! +//! gen-luarocks-embed +//! +//! `` is `luarocks_src/src` (contains `luarocks/`, `compat53/`, +//! and `bin/`). `` is the Zig source to write. +//! +//! The generated file looks like: +//! +//! pub const Entry = struct { name: []const u8, contents: []const u8 }; +//! pub const files: []const Entry = &.{ +//! .{ .name = "luarocks.cmd", .contents = @embedFile("...") }, +//! ... +//! }; +//! pub const luarocks_main: []const u8 = @embedFile("..."); +//! pub const luarocks_admin_main: []const u8 = @embedFile("..."); +//! +//! The `@embedFile` paths are absolute paths into the Zig cache. That's +//! safe because the codegen step depends on the luarocks dependency, so +//! the cache entry exists by the time the generated source is compiled. + +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); // skip program name + const embed_prefix = args.next() orelse return error.MissingEmbedPrefix; + const src_dir_arg = args.next() orelse return error.MissingSrcDir; + const out_path_arg = args.next() orelse return error.MissingOutPath; + + // Walk src/luarocks and src/compat53 collecting *.lua files; pick + // up src/bin/luarocks and src/bin/luarocks-admin separately. + var entries: std.array_list.Managed(Entry) = .init(arena); + try collect(arena, io, src_dir_arg, embed_prefix, "luarocks", &entries); + try collect(arena, io, src_dir_arg, embed_prefix, "compat53", &entries); + + // Sort for deterministic output. + std.mem.sort(Entry, entries.items, {}, Entry.lessThan); + + const bin_luarocks = try std.fs.path.join(arena, &.{ embed_prefix, "bin/luarocks" }); + const bin_luarocks_admin = try std.fs.path.join(arena, &.{ embed_prefix, "bin/luarocks-admin" }); + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path_arg, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.writeAll( + \\//! Auto-generated. Do not edit. See build/gen_luarocks_embed.zig. + \\ + \\pub const Entry = struct { + \\ /// Lua `require` path, e.g. "luarocks.core.cfg". + \\ name: []const u8, + \\ /// File contents (Lua source). + \\ contents: []const u8, + \\}; + \\ + \\pub const files: []const Entry = &.{ + \\ + ); + + for (entries.items) |e| { + try w.print( + " .{{ .name = \"{s}\", .contents = @embedFile(\"{s}\") }},\n", + .{ e.require_name, e.embed_path }, + ); + } + + try w.writeAll("};\n\n"); + try w.print( + "pub const luarocks_main: []const u8 = @embedFile(\"{s}\");\n", + .{bin_luarocks}, + ); + try w.print( + "pub const luarocks_admin_main: []const u8 = @embedFile(\"{s}\");\n", + .{bin_luarocks_admin}, + ); + + try w.flush(); +} + +const Entry = struct { + require_name: []const u8, + embed_path: []const u8, + + fn lessThan(_: void, a: Entry, b: Entry) bool { + return std.mem.lessThan(u8, a.require_name, b.require_name); + } +}; + +/// Walk `/` recursively, collecting every `.lua` file. The +/// `require` name is computed as the path relative to `` with `/` +/// replaced by `.` and `.lua` stripped, then `/init.lua` collapsed to +/// the parent directory. +fn collect( + arena: std.mem.Allocator, + io: std.Io, + root_abs: []const u8, + embed_prefix: []const u8, + subdir: []const u8, + out: *std.array_list.Managed(Entry), +) !void { + const start_abs = try std.fs.path.join(arena, &.{ root_abs, subdir }); + var dir = std.Io.Dir.cwd().openDir(io, start_abs, .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => return, + else => return err, + }; + defer dir.close(io); + + var walker = try dir.walk(arena); + defer walker.deinit(); + + while (try walker.next(io)) |entry| { + if (entry.kind != .file) continue; + if (!std.mem.endsWith(u8, entry.basename, ".lua")) continue; + + // `embed_path` lives under the staged tree (`/...`) + // so `@embedFile` can resolve it relative to the generated zig. + const embed_path = try std.fs.path.join( + arena, + &.{ embed_prefix, subdir, entry.path }, + ); + const rel = try std.mem.concat(arena, u8, &.{ subdir, "/", entry.path }); + + // Compute `require` name: drop ".lua", convert `/` to `.`. + // + // We deliberately do NOT collapse `/init.lua` into the parent + // module name. Lua's stock path searcher checks both + // `foo.lua` and `foo/init.lua` when you `require("foo")`, so + // the on-disk layout would resolve fine without our help — + // and luarocks's own tree contains both `luarocks/cmd.lua` + // (the package) and `luarocks/cmd/init.lua` (the subcommand + // module named `luarocks.cmd.init`). Collapsing would alias + // them onto one name. The embedded searcher serves whichever + // name appears literally in the source tree; the standard + // searcher behavior emerges from cmd.lua being a sibling. + const stem = rel[0 .. rel.len - ".lua".len]; + const require_name = try arena.alloc(u8, stem.len); + for (stem, 0..) |ch, i| { + require_name[i] = if (ch == '/') '.' else ch; + } + + try out.append(.{ + .require_name = require_name, + .embed_path = embed_path, + }); + } +} diff --git a/build/panto_lua_repl.c b/build/panto_lua_repl.c new file mode 100644 index 0000000..ce99cb1 --- /dev/null +++ b/build/panto_lua_repl.c @@ -0,0 +1,49 @@ +/* + * panto's hook into the upstream `lua.c` standalone interpreter. + * + * We need to call into `pmain` (the body of the standalone REPL/driver) + * with a pre-configured `lua_State` — specifically, one where panto's + * embedded-luarocks searcher and configured `package.path` have already + * been installed. Upstream `pmain` is `static`, so we can't link to it + * from Zig directly. + * + * Trick: include `lua.c` verbatim with `static` redefined to empty, so + * every static gets external linkage. We rename `main` to a stub via + * macro so it doesn't conflict with panto's own `main`. Then we expose + * a single function, `panto_lua_pmain`, that runs `pmain` against a + * caller-provided state. + * + * `lua.c` also defines its own signal handlers via `setsignal` / + * `laction`. We leave those alone — they're useful in `panto lua` too, + * giving Ctrl-C the same behavior it has in upstream `lua`. + */ + +/* Remove `static` everywhere lua.c uses it so symbols are externally + * accessible. We undef `static` back to default for any system headers + * that follow, in case they care. */ +#define static +#include "lua.c" +#undef static + +/* Forward declarations of the (now non-static) symbols we use. */ +extern int pmain(lua_State *L); + +/* + * Trampoline: push (argc, argv) onto `L` and run `pmain` in protected + * mode, mirroring `lua.c::main`'s call pattern. Returns the exit code + * appropriate for `lua` (0 on success, 1 on failure). + * + * The caller is responsible for state lifecycle (lua_close, etc.) — + * we don't open or close `L` ourselves. This is what lets panto's + * subcommand install the embedded-luarocks searcher into `L` *before* + * calling us. + */ +int panto_lua_pmain(lua_State *L, int argc, char **argv) { + lua_pushcfunction(L, &pmain); + lua_pushinteger(L, argc); + lua_pushlightuserdata(L, argv); + int status = lua_pcall(L, 2, 1, 0); + int result = lua_toboolean(L, -1); + report(L, status); + return (result && status == LUA_OK) ? 0 : 1; +} diff --git a/docs/archive/LUA_MAKEOVER.md b/docs/archive/LUA_MAKEOVER.md new file mode 100644 index 0000000..f701a29 --- /dev/null +++ b/docs/archive/LUA_MAKEOVER.md @@ -0,0 +1,594 @@ +# Lua Runtime Makeover + +Side project, separate from the main phase plan. Reworks panto's Lua +embedding into something that fits Lua's actual concurrency model and +opens the door to a luarocks-based extension ecosystem. + +Phase 3 as shipped is fine and stays. This document describes what +replaces it. + +## Why + +The current Lua embedding has three problems, in increasing order of +how badly they constrain us: + +1. **One `lua_State` per tool call.** Every `LuaTool.invoke` builds and + tears down a fresh interpreter. Module-global state is impossible. + Top-level extension code runs over and over. The API only makes + sense for stateless single-shot handlers, which is not how Lua + wants to be used. +2. **The `Tool` contract is "thread-safe."** Right for native + extensions; wrong for Lua. Lua's concurrency primitive is the + coroutine, not the OS thread. A single `lua_State` is not safe for + concurrent host entry, so the only way to honor "thread-safe" with + Lua is one state per call — which we have, badly. +3. **No path to a real Lua ecosystem.** The current setup discovers + `.lua` files on disk and that's it. There's no answer to "how do + extension authors depend on lua-cjson" or "how does an HTTP-using + tool work." The eventual answer to both is luarocks, and the + current architecture has no place for it. + +## Shape of the fix + +Three independent pieces that compose: + +1. **`ToolSource` in libpanto.** A new kind of registration alongside + `Tool`. A source owns one or more tools and receives all calls + targeting them, as a batch, on one thread per turn. Different + sources still run in parallel. Lua becomes one source. +2. **Long-lived `lua_State` with cooperative scheduling.** The panto + CLI maintains exactly one `lua_State` for its entire lifetime. + Extension top-level code runs once at startup. Each tool call is a + coroutine. A libuv event loop drives suspended coroutines. +3. **luarocks as the package manager.** Both panto's own runtime + battery (luv — see note below) and user-installed extensions + come through luarocks, installed into a tree under + `$XDG_DATA_HOME/panto/`. luarocks itself is embedded in the panto + binary as Lua source. + + Originally planned to ship `coro-*` (coro-fs, coro-http, coro-net, + coro-channel, coro-spawn) as additional batteries; that plan was + dropped during implementation. luv alone provides the libuv + surface (callback-shaped, but coroutine-friendly with a `co = + coroutine.running(); ...; coroutine.yield()` pattern); shipping + the smallest possible set keeps the makeover focused and lets us + write panto's own `std` tools directly against luv. Users who + want coro-* helpers can install them via `panto lua -e ...` + against the embedded luarocks. + +## libpanto: `ToolSource` + +Native tools keep the existing `Tool` API unchanged. Adapters that +back multiple tools through a shared runtime use the new `ToolSource`: + +```zig +pub const ToolSource = struct { + name: []const u8, // diagnostic only ("panto-lua") + tools: []const Tool.Decl, // metadata only; no per-tool vtable + ctx: *anyopaque, + vtable: *const VTable, + + pub const VTable = struct { + /// libpanto guarantees: for a given turn, all ToolUse calls + /// whose tool name belongs to this source are delivered in + /// one invoke_batch call, on one thread. Different sources + /// still execute in parallel. + /// + /// The source decides internal scheduling (coroutines, + /// sequential, internal worker pool). + invoke_batch: *const fn ( + ctx: *anyopaque, + calls: []const Call, + results: []CallResult, // parallel array, pre-allocated + allocator: Allocator, + ) anyerror!void, + + deinit: *const fn (ctx: *anyopaque, allocator: Allocator) void, + }; + + pub const Call = struct { + tool_name: []const u8, + input: []const u8, + }; + + pub const CallResult = union(enum) { + ok: []u8, // owned by allocator + err: anyerror, + }; +}; + +pub const Tool.Decl = struct { + name: []const u8, + description: []const u8, + schema_json: []const u8, +}; +``` + +`ToolRegistry` indexes by tool name with a tagged value: +`{ .single: *Tool }` or `{ .source: *ToolSource, .tool_index: usize }`. + +`Agent.registerToolSource(src: ToolSource) !void` is the new entry +point. + +### Agent loop change + +In `runStep`, after collecting ToolUse blocks for a turn: + +1. Group them by owning source. Single-`Tool` entries form + single-entry groups. +2. Spawn one OS thread per group. +3. Each thread calls either `tool.vtable.invoke` (single) or + `source.vtable.invoke_batch` (batched). +4. Join. Assemble ToolResult blocks in the original order. + +**Concurrency contract becomes:** different groups run in parallel; a +single group is the source's problem. The "thread-safe" promise still +holds for native `Tool`s. For Lua, it relaxes to "coroutine-safe +within the panto-lua runtime." + +## Lua runtime + +### One `lua_State`, many coroutines, one event loop + +The panto CLI creates a single `lua_State` at startup. Every Lua +extension is loaded into it. All top-level extension code runs once. +Module-global state is real and persistent across calls. + +When `invoke_batch` fires for the panto-lua source: + +``` +1. for each call: coroutine.create(handler), coroutine.resume(co, args) +2. uv.run() until all coroutines have completed (or errored) +3. collect results into the CallResult array +``` + +That's the entire scheduler. The work happens inside libuv: when a +coroutine calls a yield-aware libuv operation (HTTP, fs, subprocess, +sleep) it suspends; libuv resumes it when the event fires. + +A wrapper layer translates libuv's callback shape into coroutine +yields. The Luvit project's `coro-*` modules (coro-fs, coro-net, +coro-http, coro-channel, coro-spawn) do this upstream for the common +operations. Where they don't cover something, we write small wrappers +in panto's own Lua code. The pattern is ~10 lines: + +```lua +local function fs_open(path, flags, mode) + local co = coroutine.running() + uv.fs_open(path, flags, mode, function(err, fd) + coroutine.resume(co, err, fd) + end) + return coroutine.yield() +end +``` + +### What this gets us + +- True cooperative parallel I/O within a batch. Three concurrent + `web_fetch` calls go through three concurrent sockets; total + latency is `max(req1, req2, req3)`, not sum. +- First-class async subprocess. A `bash`-style tool that runs three + commands at once does it without blocking the runtime. +- Module-global state for extensions that want it (rate limiters, + caches, lazy connection pools, etc.). +- Extension top-level code runs once. Initialization is real. + +### The honest caveat + +Cooperative scheduling only helps when handlers yield. A handler that +calls a non-yielding C function — raw `os.execute`, `io.read` against +a slow file, an FFI call — blocks its siblings until it returns. +Document loudly. The escape hatch is "use native extensions for +work that can't yield." + +This is the same trade Python's asyncio makes with `requests` vs +`aiohttp`. Panto's recommended posture: handlers should use libuv via +the coro-* wrappers (or other luv-aware libraries) for I/O. Pure +compute is fine. Calling `socket.http.request` or `os.execute` will +work but blocks the batch. + +### Why libuv (not cqueues) + +Considered cqueues + lua-http. Better HTTP story (HTTP/2), +coroutine-native API. Lost on **subprocess**, which has no +maintained cqueues binding and which matters a lot for a coding +agent. Also lost on familiarity — luv is what every Neovim user has +seen. + +Trade accepted: HTTP only via luv's lower-level TCP + manual +framing (or a user-installed rock), and a callback-shaped async +surface rather than a coroutine-native one. Coroutine ergonomics +are an extension-author concern, not a runtime concern. + +## luarocks as the package manager + +### Distribution model + +luarocks is embedded in the panto binary as `@embedFile`'d Lua +source. At startup the runtime: + +1. Computes `$PANTO_HOME = $XDG_DATA_HOME/panto` (default + `~/.local/share/panto`). +2. Configures the embedded `lua_State`'s `package.path` and + `package.cpath` to look under `$PANTO_HOME/share/lua/5.4/` and + `$PANTO_HOME/lib/lua/5.4/`. +3. Bootstraps the embedded luarocks against + `--tree=$PANTO_HOME`. +4. Reconciles a "runtime batteries" manifest (currently just + luv) — installs missing rocks, no-ops if present. +5. Iterates user extensions from config. For `luarocks:foo`-style + references, ensures `foo` is installed. +6. Hands control to the agent loop. + +luarocks 3.12+ no-ops on already-installed rocks and caches the +upstream manifest, so steps 4–5 are cheap on every run after the +first. + +Network failures on later runs are swallowed: the rocks are already +there. First-run-with-no-network degrades to "no Lua extensions +work" — native panto features and the agent loop are unaffected. + +### Why fully luarocks-based + +- One mechanism for everything Lua-distribution. Runtime batteries + and user extensions install the same way. +- Single small binary. No vendored libuv, no vendored luv, no + `@embedFile`'d coro-* sources. luarocks itself is ~1MB of Lua + source that compresses well. +- Version flexibility for batteries without re-shipping panto. +- Matches Neovim's rocks.nvim direction — the most relevant + ecosystem signal for "Lua as a serious distribution target." +- User extension story is genuinely the same as the batteries + story. No special cases. + +### Distributable artifact + +Single `panto` binary contains: + +- Zig CLI + libpanto +- Lua 5.4 (already vendored) +- luarocks Lua source, embedded via `@embedFile` +- A small Zig-side bootstrap that configures `package.path` for the + embedded luarocks code + +Everything else lives under `$PANTO_HOME` and is installed on first +run. + +## Migration shape + +Independent chunks of work, roughly in order: + +1. **`ToolSource` in libpanto.** ✅ **Done.** Type lives in + `libpanto/src/tool_source.zig`; registry tagging in + `tool_registry.zig`; per-group `std.Io.Group` fan-out in + `agent.zig`. Native `Tool` API unchanged. +2. **Long-lived `lua_State` runtime in the CLI.** ✅ **Done.** + `src/lua_runtime.zig` owns one `lua_State` for the process + lifetime, loads all discovered extensions once, stores handlers + as `luaL_ref` slots, and runs each call as a coroutine. + Registers itself with libpanto as the `panto-lua` `ToolSource`. + Step 5 added the libuv-driven scheduler that makes yields + productive. +3. **Embed luarocks.** ✅ **Done.** `build.zig.zon` fetches + luarocks 3.13.0 source. `build/gen_luarocks_embed.zig` enumerates + the Lua sources at build time and produces a Zig module that + `@embedFile`s each entry. `src/luarocks_runtime.zig` installs an + embedded-source `package.searcher` so `require("luarocks.*")` + resolves without disk I/O, injects `luarocks.core.hardcoded`, + configures `package.path`/`cpath`, stages Lua headers under + `/include/`, writes `/etc/luarocks/config-5.4.lua`, + and materializes a `/bin/lua` shell wrapper that `exec`s + `panto lua` (the interpreter luarocks shells out to for rockspec + build scripts). +4. **Install luv as the first battery.** ✅ **Done.** Manifest + pins `luv 1.52.1-0` (`src/manifest.zig`). Bootstrap reconciles + in-process by `load`ing the embedded `src/bin/luarocks` driver + and calling it with `install ` args. luv's + rockspec bundles libuv as a git submodule and builds it via + CMake, so first-run toolchain requirement is `cc`, `make`, and + `cmake`. Subsequent runs detect the version-stamped install + metadata under `/lib/luarocks/rocks-5.4///` + and no-op fast. +5. **Cooperative scheduler around `uv.run()`.** ✅ **Done.** + `LuaRuntime.installScheduler` registers `panto._record_result` + (a C function with the runtime pointer as upvalue), installs a + wrapper closure that runs handlers under `pcall` and reports + results, and caches `require("luv").run` in the registry. The + new `runBatch` path creates one coroutine per call, resumes + each once, then ticks `uv.run("once")` between resumes until + every coroutine has terminated. Coroutines that yield without + any pending libuv handle are surfaced as a deadlock error. + + We deliberately ship no `coro-*` helpers — luv is the entire + async surface. Extension authors `require("luv")` and use its + native callback-shaped APIs; `panto lua -e ...` (plus the + embedded luarocks) is the escape hatch for anything else. +6. ~~User extension config syntax.~~ **Out of scope for this + makeover.** Deferred; see Q7 "Future work". +7. **Delete `LuaTool` and per-call `lua_State` machinery.** ✅ + **Done** as part of step 2 — `LuaTool` and `LuaStatePool` are + gone from the tree. + +Additional work surfaced by Q1–Q7 that lands alongside steps 3–5: + +- **`panto lua` subcommand.** ✅ **Done.** `build/panto_lua_repl.c` + includes upstream `lua.c` with `#define static` stripped so we + can call `pmain` directly. `src/subcommand.zig` opens a fresh + `lua_State`, runs the same bootstrap pipeline as `panto run`, + then hands the state to `panto_lua_pmain(L, argc, argv)` so + upstream argv handling (`-i`, `-l`, `-e`, `script.lua args...`) + works verbatim. (Q4) +- **`panto bootstrap` subcommand.** ✅ **Done.** Same bootstrap path + as the agent and `panto lua`; just exits cleanly afterwards + instead of entering an interactive surface. Idempotent: a clean + tree on subsequent runs no-ops in milliseconds. (Q2) + + Supports `--force`: wipes the per-Lua-version tree + (`/rocks/lua-X.Y.Z/`) before running the regular bootstrap. + Useful for recovering from a corrupted installation or testing + the cold-start path. Sibling trees from other Lua versions are + not touched. +- **Batteries manifest + reconcile.** ✅ **Done.** `src/manifest.zig` + ships pinned versions; `reconcileBatteries` runs in-process by + `load`ing the embedded luarocks driver, calling its `cmd.run_command` + with `install ` for any rock that isn't already + on disk. (Q5) +- **Lua headers staged at `$PANTO_HOME/rocks/lua-X.Y.Z/include/`.** + ✅ **Done.** `build/gen_lua_headers_embed.zig` embeds the headers + from the Lua source tarball at build time; `stageLuaHeaders` in + `luarocks_runtime.zig` writes them to disk on first run and + when the contents differ (e.g. a panto upgrade that bumped + the bundled Lua version). (Q3) + +Documentation updates: phase-3.md gains a "superseded by +LUA_MAKEOVER.md for the Lua runtime; native extension contract +unchanged" note. The contract for native extensions ("thread-safe") +stays as-is. + +## Open questions + +These came up during design and need resolution before +implementation. We'll edit answers in here as decisions land. + +### Q1: luarocks's own C dependencies + +luarocks attempts to `require` several optional Lua modules via +`pcall` and falls back to shelling out or to its own pure-Lua +implementations when they're missing. The optional set: + +- **LuaSocket** (`socket.http`, `socket.ftp`) — for HTTP/FTP + downloads. Without it: shell out to a configured downloader + (`curl` or `wget`). +- **LuaSec** (`ssl.https`) — for HTTPS. Without it *and* without + the LuaSocket+luarocks-internal HTTPS path: must use `curl` or + `wget` for HTTPS. **luarocks.org is HTTPS-only**, so this is + effectively mandatory in some form. +- **LuaFileSystem** (`lfs`) — for directory operations, + `chdir`, file attributes. Without it: degraded fallbacks using + only `io.*` and `os.*`. Some operations become impossible. +- **lua-bz2** (`bz2`) — for `.bz2` archives. Almost never + encountered; rocks ship as `.tar.gz` or `.zip`. +- **LuaPosix** (`posix`) — for chmod and other POSIX ops. + Without it: fall back to shelling out to `chmod` etc. +- **md5** — for checksums. luarocks has a pure-Lua fallback. +- **`luarocks.tools.zip`** (bundled with luarocks itself) — pure + Lua zip/gzip. No external dependency. +- **`luarocks.tools.tar`** (bundled) — pure Lua tar. + +**Correction:** there is no `--with-lua=embedded` flag — that was a +hallucination from earlier in the design conversation. luarocks's +`./configure` accepts `--with-lua=DIR`, `--with-lua-include=DIR`, +`--with-lua-lib=DIR`, `--with-lua-interpreter=NAME`, +`--lua-version=VERSION`. These point luarocks at *a* Lua install +(which can be ours under `$PANTO_HOME`); they don't enable a +separate "embedded" mode. + +**Decision:** minimize luarocks's optional Lua deps. Bootstrap runs +luarocks in its degraded-but-functional mode using its bundled +pure-Lua `tools.zip` and `tools.tar`, plus shell-out to `curl` (or +`wget`) for HTTPS downloads. If any of the optional deps turn out +to be effectively mandatory in practice, we statically link the +native C library and embed the Lua wrapper as `@embedFile` in +`panto` (sibling to Lua itself in `build.zig`). Likely candidates: +LuaFileSystem (small, pure C wrapper around POSIX, very widely +used), and possibly LuaSocket+LuaSec if shelling out to `curl` +proves too clunky. + +We also depend on `curl` (or `wget`) being on PATH for downloads. +This is universal on Unix dev machines and we accept the +dependency. If it's missing, bootstrap surfaces a clear error. + +### Q2: C toolchain on first run + +luv has a C component. Building it requires `cc`, `make`, and Lua +headers. On dev machines (panto's target audience) these are +universal. On bare end-user machines they aren't. + +**Decision:** add a `panto bootstrap` subcommand. It's effectively +a no-op `panto` invocation that exercises the same +fetch-and-install path that every normal startup runs, just +without entering the agent loop afterwards. On a clean machine +it's where the slow first-run download-and-compile happens with +visible output; on subsequent runs it's a fast no-op equivalent +to what every `panto` startup already does. + +Every normal `panto` startup runs the same sync logic. The +`bootstrap` subcommand isn't a *separate* mechanism — it's a way +to run the sync explicitly without the agent loop, for users who +want to do setup ahead of time, or for CI/scripted installs. + +When the toolchain is missing, surface a friendly error from +the bootstrap code path before luarocks itself barfs. "You need +a C compiler and make installed to compile Lua extensions" or +similar. Native panto features keep working regardless — only +the Lua tool runtime is gated on successful bootstrap. + +### Q3: Lua headers for C rocks + +Building C rocks against panto's embedded Lua requires the Lua +headers to be findable on disk. + +**Decision:** drop the headers into `$PANTO_HOME/rocks/lua-X.Y.Z/include/` +at bootstrap time, where `X.Y.Z` is the Lua version panto is +built against. Embed the header sources via `@embedFile` (they're +already available via the `lua_src` build dep). Bootstrap writes +them out on first run and on any panto upgrade that changes the +Lua version. + +**Why a versioned subdirectory:** rocks compiled against Lua +5.4.7's headers are not safe to load into a Lua 5.5 interpreter +(ABI changes happen across minor versions). The whole rock tree +lives under `$PANTO_HOME/rocks/lua-X.Y.Z/` and each Lua version +gets its own. A panto upgrade that bumps Lua creates a new tree +and reinstalls everything against it. The old tree is left in +place for rollback; users (or a future `panto gc` command) can +delete stale ones. + +Directory layout: + +``` +$PANTO_HOME/ + rocks/ + lua-5.4.7/ + include/ ← Lua headers + share/lua/5.4/ ← installed pure-Lua rocks + lib/lua/5.4/ ← installed C rocks + ...luarocks metadata... + lua-5.5.0/ ← after a future upgrade + ... +``` + +luarocks is invoked with `--tree=$PANTO_HOME/rocks/lua-5.4.7` and +configured (via its config file or CLI flags) to know that the +Lua headers live at `$PANTO_HOME/rocks/lua-5.4.7/include/`. The +tree contains everything needed for that Lua version including +the headers, which keeps rebuilds reproducible and rollback +clean. + +### Q4: The `lua` interpreter that luarocks expects on PATH + +luarocks uses an external `lua` binary for some operations (running +rockspec build scripts, primarily). It needs to be on PATH and +needs to be the same version as the embedded interpreter. + +**Decision:** `panto lua` is a first-class user-visible subcommand +that wraps the **upstream `lua.c` standalone interpreter** with +panto's environment pre-configured. + +Mechanically: + +- Compile `lua.c` (the upstream standalone interpreter, ~600 lines) + into the panto binary as a subcommand entry point. Currently + excluded from `lua_files` in `build.zig`; include it for the + `lua` subcommand path. +- `panto lua` arguments pass through to `lua.c`'s normal + command-line handling (`-i`, `-l`, `-e`, `script.lua args...`, + etc.). Full standalone-interpreter behavior, not a luarocks-only + subset. +- Before handing control to `lua.c`'s main, panto's subcommand + setup runs the same bootstrap as `panto run` (verify batteries + installed, install missing rocks, configure `package.path` / + `package.cpath` to find `$PANTO_HOME/rocks/lua-X.Y.Z/...`). +- Configure luarocks (via its config file written to + `$PANTO_HOME/rocks/lua-X.Y.Z/config.lua`) to use + ` lua` as its Lua interpreter. luarocks's + `--with-lua-interpreter=...` flag accepts an executable name; + we either symlink or use the full argv mechanism. + +This gives users a real `lua` they can use to test their +extensions in panto's environment — `require "luv"` and +`require "coro-http"` work, plus anything else they've installed +via `panto lua -e 'require("luarocks.cmd").run(...)'` or similar. + +**Side benefit (Q7-related):** until we have a proper user-facing +`panto rocks install foo` command, `panto lua` is also the user's +escape hatch for installing extra rocks into `$PANTO_HOME`. We +can invoke luarocks itself through it. + +### Q5: Reproducibility / lockfile + +luarocks installs latest-matching by default. For a CLI tool we +want reproducible: the same panto version installs the same +battery versions on every fresh machine. + +**Decision:** pin exact versions in a panto-internal manifest +shipped with the binary. No user-facing `panto lock` or +`panto sync` commands — sync is what every startup (and +`panto bootstrap`) does automatically. + +A manifest file (likely `runtime-batteries.zon` or similar in the +panto source tree) lists exact versions: + +```zig +.{ + .lua_version = "5.4.7", + .luarocks_version = "3.12.2", + .batteries = .{ + .luv = "1.51.0-1", + .{ .@"coro-fs" = "3.0.4-1" }, + .{ .@"coro-http" = "3.2.1-1" }, + .{ .@"coro-net" = "3.2.1-1" }, + .{ .@"coro-channel" = "3.0.4-1" }, + .{ .@"coro-spawn" = "3.2.1-1" }, + }, +} +``` + +Bumping any of these is a deliberate edit + commit + version bump +of panto itself. Each panto release pins one consistent set. + +Bootstrap reads the embedded manifest and ensures the tree matches: +any rock not present at the pinned version gets installed; any +stale versions get removed. A panto upgrade that bumps Lua +creates an entirely new tree (per Q3) and installs everything +fresh against it. + +### Q6: Where in the agent loop does the runtime live + +**Decision:** CLI-side. libpanto continues to be native-only and +Lua-unaware. + +The long-lived `lua_State` is constructed in the panto CLI's +`main` (or a module it calls) before the `Agent` is built. +Bootstrap runs first, then the runtime loads all discovered +Lua extensions into the state, then the runtime registers itself +with the `Agent` as a single `ToolSource` named `panto-lua`. +The source's `ctx` holds the runtime; the `lua_State` lives +inside it. + +libpanto's only concept is `Tool` and `ToolSource`. It has no +idea that one of its sources happens to be Lua-backed. + +### Q7: What "user extension" actually means in the new world + +**Decision (for now):** keep the phase-3 directory-based discovery +as the only user extension mechanism. Local `.lua` files in +`~/.config/panto/extensions/` and `./.panto/extensions/`. No +config file, no `luarocks:foo` references yet. + +Directory-discovered extensions get access to whatever's in +`$PANTO_HOME/rocks/lua-X.Y.Z/` — the runtime batteries (luv, +coro-*, future additions) and nothing else by default. They can +`require` any of those modules and pure-Lua code they write +themselves; that's the supported surface. + +**Escape hatch for extra rocks:** users who need a third-party +Lua library (lua-cjson, lpeg, etc.) for their local extension +can install it manually via `panto lua` + the embedded luarocks: + +``` +panto lua -e 'require("luarocks.cmd").run("install", "lua-cjson")' +``` + +or more sugared if we feel like making that look better. The +rock lands in `$PANTO_HOME/rocks/lua-X.Y.Z/` and survives across +panto runs. + +**Future work (not part of this makeover):** a proper config +file with `luarocks:panto-subagents`-style references, where +panto-published extensions can declare their own Lua library +dependencies in their rockspec and bootstrap installs the +whole graph automatically. The infrastructure built here +(luarocks-as-runtime, `$PANTO_HOME` tree, version pinning) +directly enables this — it's just an orthogonal piece of +user-facing surface that hasn't been designed yet. diff --git a/docs/archive/ideas.md b/docs/archive/ideas.md new file mode 100644 index 0000000..e83d543 --- /dev/null +++ b/docs/archive/ideas.md @@ -0,0 +1,38 @@ +# pantograph Ideas + +`pantograph` is a minimal coding agent inspired by `pi`, with a focus on performance, efficiency, correctness, and a small core that can be extended deliberately. + +## Core architecture + +- The core agent loop will be written in Zig for performance, efficiency, and a small runtime footprint. +- In addition to the CLI/application, the project should expose a `libpanto` library through a C ABI so other programs can embed or build on the agent functionality. + +## Extension system + +- Extensions will initially be written in Lua, chosen for speed, simplicity, and low overhead. +- Because poorly written extensions can easily destabilize the host, `pantograph` should explore ways to improve extension correctness and crash protection. +- Future extension support should include loading shared object libraries through a C ABI, allowing extensions to be written in Zig, Rust, C, C++, or other native languages. + +## Minimal built-in feature set + +- `pantograph` should follow `pi`'s philosophy of avoiding unnecessary built-ins such as native subagents, MCP support, and permission systems. +- `pantograph` will go further by not building in AGENTS.md automation, skills, or customizable `/prompts`. +- Those features should be possible to implement as extensions rather than being part of the core runtime. + +## Provider API support + +- Provider support should be careful and conservative. +- The core should support Anthropic-shaped and OpenAI-shaped APIs with arbitrary base URLs. +- The goal is to avoid the common failure mode where provider integrations exist but only partially work, break important agent features, or crash the process. + +## Server/proxy mode + +- `pantograph` should support running as a server that exposes OpenAI-compatible and Anthropic-compatible APIs. +- In this mode, `pantograph` can act as a lightweight provider router/proxy to its configured backends. +- This is intended to provide the useful parts of tools like `omniroute` while avoiding excessive memory usage, fragile integrations, and runtime instability. + +## Core tools as extensions + +- Basic tools such as `read`, `write`, `edit`, and `bash` should be supplied by extensions rather than hardcoded into the core. +- These tools can be included in the standard distribution but should be individually disableable. +- Once shared object extension support exists, the standard core tools should be ported to Zig/native extensions. diff --git a/docs/archive/phase-1.md b/docs/archive/phase-1.md new file mode 100644 index 0000000..6b149aa --- /dev/null +++ b/docs/archive/phase-1.md @@ -0,0 +1,449 @@ +# Phase 1: libpanto — Minimal Chat Library + +**Status: complete.** Streaming chat works end-to-end against OpenAI-compatible APIs; conversation history persists across turns; thinking blocks (`reasoning_content` / `reasoning`) are streamed; `reasoning_effort` is configurable. Open questions resolved: thinking support implemented; mid-stream errors propagate via Zig errors; connections are one-per-turn intentionally (uniform reentry into each turn); long-conversation memory deferred to a later phase. Session persistence is phase 4. + +## Goal + +A Zig library that can hold a streaming conversation with an LLM via an OpenAI-compatible API. No tools, no extensions — just chat. Includes a minimal CLI for live testing. + +## Deliverable + +A `libpanto` Zig module importable by other Zig code, plus a `panto` binary that wires it into a basic read/print loop. At the end of this phase, you can: + +- Start `panto`, type a message, and receive a streamed response from an OpenAI-compatible LLM. +- Send follow-up messages that include full conversation history. +- See thinking tokens and text tokens stream in as they arrive. +- Have the complete conversation available in memory for the duration of the session. + +## What is usable at the end + +| Capability | How to exercise it | +|---|---| +| Open a conversation | `libpanto.conversation.Conversation.init(allocator)` | +| Add a user message | `conversation.addUserMessage("hello")` | +| Run an agent step (streaming) | `agent.runStep(conversation, &receiver)` | +| See streamed output | CLI prints thinking/text chunks as they arrive | +| Conversation persists across turns | Follow-up messages include prior history | + +## What is explicitly out of scope + +- Tools and tool-use (phase 3+) +- Extensions and extension API (phase 3+) +- C ABI (phase 3+, when needed for Lua extensions) +- Anthropic provider (phase 2) +- Disk persistence / session save (later phase) +- Server/proxy mode (future, undefined phase) +- System prompt construction framework (later phase — the model supports system messages, but no opinionated assembly system yet) + +--- + +## Data Model + +### TextualBlock (shared streaming buffer) + +``` +TextualBlock = struct { + buf: std.ArrayList(u8), + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) TextualBlock + pub fn content(self: *const TextualBlock) []const u8 // self.buf.items + pub fn append(self: *TextualBlock, delta: []const u8) !void + pub fn deinit(self: *TextualBlock) void // self.buf.deinit() +} +``` + +`Text` and `Thinking` blocks both use `TextualBlock` as their payload. They share the same append-based streaming behavior — deltas arrive incrementally and are appended via an internal `ArrayList(u8)`, giving amortized O(1) appends and avoiding the O(n²) re-copying that would result from storing `[]const u8` slices. + +The `TextualBlock` stores its own allocator reference so that `deinit()` needs no external context. Each `TextualBlock` has an `init()`/`deinit()` pair. This also means a `ContentBlock` can clean itself up without the caller providing an allocator. + +### ContentBlock (tagged union) + +``` +ContentBlock = union(enum) { + Text: TextualBlock, + Thinking: TextualBlock, + ToolUse: ToolUseBlock, // phase 3+ + ToolResult: ToolResultBlock, // phase 3+ + + pub fn deinit(self: *ContentBlock) void { + switch (self.*) { + .Text, .Thinking => |*b| b.deinit(), + .ToolUse => |b| { /* free id, name, input */ }, + .ToolResult => |b| { /* free tool_use_id, content */ }, + } + } +} + +ToolUseBlock = struct { + id: []const u8, // owned copy + name: []const u8, // owned copy + input: []const u8, // raw JSON bytes, owned copy +} + +ToolResultBlock = struct { + tool_use_id: []const u8, // owned copy + content: []const u8, // owned copy +} +``` + +`ToolUse` and `ToolResult` are defined in the model now but not populated or processed until the extensions phase. This avoids a model refactor later — the types exist, we just never encounter them in phase 1. + +The `input` field of `ToolUse` is stored as raw JSON bytes (`[]const u8`) rather than a parsed structure. We are not in the business of understanding tool input schemas; we pass them through. + +`ToolUse` blocks also stream incrementally — both providers send tool input as JSON fragments across multiple deltas. Therefore `ToolUse.input` also uses `TextualBlock` for assembly. `id` and `name` arrive at block-start time and are stored as owned `[]const u8` copies. + +`ToolResult` blocks are constructed by `pantograph` itself (not streamed from a provider), so `content` could be a simple `[]const u8`. However, for consistency and to allow progressive construction of results, it also uses `TextualBlock`. + +Updated types: +``` +ToolUseBlock = struct { + id: []const u8, // owned copy, from onBlockStart metadata + name: []const u8, // owned copy, from onBlockStart metadata + input: TextualBlock, // accumulated from onContentDelta +} + +ToolResultBlock = struct { + tool_use_id: []const u8, // owned copy + content: TextualBlock, // accumulated content +} +``` + +**Memory discipline**: When a `ContentBlock` is moved into a `Message`'s content list (stored in `std.ArrayList(ContentBlock)`), the TextualBlock's internal ArrayList buffer pointer remains valid — it points to the same heap allocation. The caller must ensure each block's `deinit()` is called exactly once, and must not copy a ContentBlock without clearing the source (standard Zig move semantics). + +### Message + +``` +Message = { + role: enum { system, user, assistant }, + content: []ContentBlock, +} +``` + +A system message may contain multiple `Text` blocks. When serializing to Anthropic's API (phase 2), these are concatenated into the single system prompt string. + +An assistant message is assembled incrementally during streaming. A user message containing tool results (phase 3+) naturally groups multiple `ToolResult` blocks. + +### Conversation + +``` +Conversation = { + messages: std.ArrayList(Message), + allocator: std.mem.Allocator, +} +``` + +Ordered list of messages. Methods: + +- `init(allocator)` → Conversation +- `addSystemMessage(text)` → appends `Message{ .system, [TextBlock(text)] }` +- `addUserMessage(text)` → appends `Message{ .user, [TextBlock(text)] }` +- `addAssistantMessage(blocks)` → appends `Message{ .assistant, blocks }` (called by agent loop after streaming completes) +- `deinit()` → frees all owned memory + +All `[]const u8` fields in ContentBlocks and Messages are owned by the Conversation and freed on `deinit()`. Content is stored as copies, not slices into external buffers. `TextualBlock` fields back their content with a heap-allocated `ArrayList(u8)` that grows incrementally during streaming and is freed on `deinit()`. + +--- + +## Module Structure + +``` +src/ + root.zig // public API re-exports + conversation.zig // Message, ContentBlock, Conversation + provider.zig // Provider interface, StreamEvent, StreamResult + provider_openai.zig // OpenAI-compatible implementation + sse.zig // SSE line parser + agent.zig // Agent loop: runStep, Receiver interface + config.zig // Config struct (api_key, base_url, model) + json.zig // Serialization helpers (model → wire JSON, deltas → ContentBlocks) +``` + +### `conversation.zig` + +Defines `Message`, `ContentBlock`, `Conversation`. All serialization to/from provider wire formats lives in `json.zig` — conversation.zig is pure data structure. + +Tests: create conversations, add messages, verify content, free without leaks. + +### `provider.zig` + +Defines the `Provider` interface: + +``` +Provider = struct { + ptr: *anyopaque, + vtable: *const VTable, + + VTable = struct { + streamStep: *const fn(*anyopaque, conversation: *Conversation, receiver: *Receiver) anyerror!void, + deinit: *const fn(*anyopaque) void, + }; + + pub fn streamStep(self, conversation, receiver) !void + pub fn deinit(self) void +}; +``` + +And the `Receiver` interface for streaming callbacks: + +``` +Receiver = struct { + ptr: *anyopaque, + vtable: *const ReceiverVTable, + + ReceiverVTable = struct { + onMessageStart: *const fn(*anyopaque, role: MessageRole) void, + onBlockStart: *const fn(*anyopaque, block_type: ContentBlockType, index: usize, meta: ?BlockMeta) void, + onContentDelta: *const fn(*anyopaque, block_index: usize, delta: []const u8) void, + onBlockComplete: *const fn(*anyopaque, block_index: usize, block: ContentBlock) void, + onMessageComplete:*const fn(*anyopaque, message: Message) void, + }; + + pub fn onMessageStart(self, role) void + pub fn onBlockStart(self, block_type, index, meta) void + pub fn onContentDelta(self, block_index, delta) void + pub fn onBlockComplete(self, block_index, block) void + pub fn onMessageComplete(self, message) void +}; + +BlockMeta = struct { + // Only populated for ToolUse blocks. Null for Text/Thinking. + tool_id: ?[]const u8, + tool_name: ?[]const u8, +}; +``` + +**Callback contract:** + +- Callbacks are always invoked in this order for every block, regardless of which provider is active. +- `onMessageStart` is called when the stream begins delivering a new message. +- `onBlockStart` is called when a new content block begins. `meta` carries block-type-specific metadata (tool id/name for ToolUse, null for Text/Thinking). +- `onContentDelta` is called zero or more times per block with raw byte fragments. For Text/Thinking these are word fragments; for ToolUse these are JSON fragments. The receiver does not need to interpret them. `delta` is a `[]const u8` — libpanto does not parse tool input content, it passes bytes through. +- `onBlockComplete` is called when a block is finished. The `block` parameter contains the fully assembled ContentBlock. The receiver that only needs complete content can ignore deltas and use this. +- `onMessageComplete` is called when the stream ends. The `message` parameter contains the fully assembled Message with all blocks. +- Providers guarantee that `onBlockComplete`'s `block` and `onMessageComplete`'s `message` are always fully assembled and valid. + +This uniform callback sequence means the TUI and agent loop don't need to know which provider is active. Anthropic (phase 2) maps its structured events directly to these callbacks; OpenAI synthesizes block boundaries from delta field transitions (see below). + +### `provider_openai.zig` + +Implements `Provider` for OpenAI-compatible APIs. + +- Converts `Conversation` → OpenAI wire JSON (see [OpenAI serialization](#openai-serialization) below) +- Makes HTTP POST to `{base_url}/chat/completions` with `stream: true` +- Reads SSE events, parses each `data: {...}` line as complete JSON +- Synthesizes block boundaries from delta field transitions (OpenAI has no explicit block boundary events) +- Calls the full Receiver callback sequence (onMessageStart → onBlockStart → onContentDelta → onBlockComplete → onMessageComplete) +- Accumulates deltas into ContentBlocks via TextualBlocks + +Construction: + +``` +OpenAIProvider.init(allocator, config) !OpenAIProvider +``` + +### OpenAI block boundary synthesis + +OpenAI's streaming deltas have no explicit block boundaries. The provider tracks a state machine to infer when blocks start and end: + +``` +StreamingState = struct { + active_block_type: enum { none, thinking, text, tool_use }, + active_block_index: usize, + // assembly buffers per block +} + +On each SSE event: + 1. If delta.role == "assistant" → emit onMessageStart(.assistant) + 2. If delta.reasoning_content present: + - If active_block_type != .thinking: + - If active_block_type != .none → emit onBlockComplete for prior block + - Emit onBlockStart(.Thinking, index, null) + - active_block_type = .thinking + - Emit onContentDelta(index, delta.reasoning_content) + 3. If delta.content present: + - If active_block_type != .text: + - If active_block_type != .none → emit onBlockComplete for prior block + - Emit onBlockStart(.Text, index, null) + - active_block_type = .text + - Emit onContentDelta(index, delta.content) + 4. If delta.tool_calls present: + - If active_block_type != .tool_use: + - If active_block_type != .none → emit onBlockComplete for prior block + - Emit onBlockStart(.ToolUse, index, .{ .tool_id = ..., .tool_name = ... }) + - active_block_type = .tool_use + - Emit onContentDelta(index, delta.tool_calls[].function.arguments) + 5. If finish_reason != null: + - If active_block_type != .none → emit onBlockComplete for current block + - Emit onMessageComplete(assembled_message) +``` + +A transition in `active_block_type` means the previous block is done and a new one has started. The state machine also handles the case where the same block type appears again after an intervening type (e.g., thinking → text → thinking), which would open a new Thinking block at a new index. + +### `sse.zig` + +Incremental SSE line parser. The HTTP client delivers arbitrary-sized read buffers; this module reassembles them into complete `data: ...\n\n` events. + +``` +SSEParser = struct { + buf: std.ArrayList(u8), + + pub fn init(allocator) SSEParser + pub fn feed(self, chunk: []const u8) ![]const []const u8 // returns slice of complete event strings + pub fn deinit(self) void +}; +``` + +`feed()` may return zero events (partial line buffered) or multiple events (chunk contained several). The caller does not need to worry about line boundaries. + +Tests: feed partial chunks, verify events emitted at correct boundaries; multi-event in single chunk; empty lines; `data: [DONE]`. + +### `json.zig` + +Two responsibilities: + +1. **Serialize Conversation → OpenAI request body** — Convert our `Message`/`ContentBlock` model into the JSON shape OpenAI expects. See below. +2. **Parse SSE chunk deltas → ContentBlock updates** — Each SSE event's JSON contains a `choices[0].delta` object. Extract text/thinking content from it. + +### `agent.zig` + +The agent loop. In phase 1, it's simple: + +``` +Agent = struct { + provider: Provider, + allocator: std.mem.Allocator, + + pub fn init(allocator, provider) Agent + pub fn runStep(self, conversation: *Conversation, receiver: *Receiver) !void + pub fn deinit(self) void +}; +``` + +`runStep` does: +1. Call `provider.streamStep(conversation, receiver)` — this streams the response and calls the full Receiver callback sequence on the receiver +2. The `onMessageComplete` callback appends the finished Message to the conversation (the agent itself can wire this, or the caller handles it — TBD during implementation) + +In later phases, `runStep` gains the tool-call loop: check for ToolUse blocks, execute them, feed results back, call provider again. But the shape stays the same — one `runStep` invocation carries the conversation through a full agent turn. + +### `config.zig` + +``` +Config = struct { + api_key: []const u8, + base_url: []const u8, // e.g. "https://api.openai.com/v1" + model: []const u8, // e.g. "gpt-4o" +}; +``` + +Populated from environment variables (`PANTO_API_KEY`, `PANTO_BASE_URL`, `PANTO_MODEL`) with defaults for base_url and model. + +### `root.zig` + +Public API. Re-exports the types and functions that external Zig code needs: + +``` +pub const conversation = @import("conversation.zig"); +pub const provider = @import("provider.zig"); +pub const agent = @import("agent.zig"); +pub const config = @import("config.zig"); +``` + +Does not re-export provider_openai, sse, or json — those are internal. + +--- + +## OpenAI Serialization + +### Request + +Our `Conversation` → OpenAI `chat/completions` request body: + +``` +{ + "model": config.model, + "stream": true, + "messages": [ + // For each Message in conversation: + // + // role=.system → { "role": "system", "content": "" } + // role=.user → { "role": "user", "content": "" } + // (ToolResult blocks pulled out into separate role:tool messages in phase 3+) + // role=.assistant → { "role": "assistant", "content": [ + // ...text blocks as { "type": "text", "text": "..." }, + // ...thinking blocks as { "type": "thinking", "thinking": "..." }, + // ...tool_use blocks become function_call/function entries in phase 3+ + // ] } + ] +} +``` + +For phase 1, all content blocks we encounter are `Text` or `Thinking`, so serialization is straightforward. + +### Response (streaming) + +Each SSE event is a complete JSON object: + +``` +data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]} +data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]} +data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{"content":" world"},"finish_reason":null}]} +data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} +data: [DONE] +``` + +We parse each event's `choices[0].delta` and drive the block boundary state machine: +- `delta.role == "assistant"` → emit onMessageStart, marks the start of a new assistant message +- `delta.reasoning_content` → transition to Thinking block if needed, append via onContentDelta +- `delta.content` → transition to Text block if needed, append via onContentDelta +- `delta.tool_calls` → transition to ToolUse block if needed, append arguments via onContentDelta (phase 3+) + +The `finish_reason: "stop"` signals stream end → emit onBlockComplete for any active block, then onMessageComplete. + +--- + +## Minimal CLI + +``` +panto/ + src/ + main.zig // CLI entry point +``` + +Behavior: +1. Read `PANTO_API_KEY`, `PANTO_BASE_URL`, `PANTO_MODEL` from environment +2. Create a Conversation, add a system message (default: "You are a helpful assistant.") +3. Print a prompt (`> `), read a line from stdin +4. Add user message, call `agent.runStep()`, print streamed deltas to stdout +5. Repeat step 3 until EOF (Ctrl+D) + +There is no line editing, no scrolling, no syntax highlighting. Just `readline` → `print`. The sole purpose is exercising libpanto against a real API. + +--- + +## Testing Strategy + +### Unit tests (automated, per module) + +| Module | What to test | +|---|---| +| `conversation.zig` | Create conversation, add messages of each role, verify content block storage, free without leaks | +| `sse.zig` | Feed partial chunks, verify event boundaries; multi-event chunks; `data: [DONE]`; empty lines between events | +| `json.zig` | Serialize conversation → OpenAI JSON; parse delta JSON objects → content updates | +| `config.zig` | Parse from env vars; defaults for missing optional fields | + +### Integration test (manual) + +- Run `panto` binary with a real API key +- Hold a multi-turn conversation +- Verify responses stream to stdout +- Verify follow-up messages include prior context (ask the model "what did I just say?") + +--- + +## Open Questions (to resolve during implementation) + +1. **Thinking token support in OpenAI API**: OpenAI's `reasoning_content` field in streaming deltas is not universally present across models/endpoints. We need to handle its absence gracefully (just skip it, don't crash). +2. **Error handling in streams**: Mid-stream HTTP errors, rate limiting, truncated responses. How do we represent these to the caller? An `onError` callback on the Receiver seems likely. +3. **HTTP connection lifecycle**: Does `std.http.Client` support long-lived streaming connections cleanly? We may need to manage connection pooling or timeouts. +4. **Memory strategy for long conversations**: We're storing full message content in memory. For phase 1 this is fine, but we should define the interface so a later phase can introduce message summarization or offloading without changing the agent loop. diff --git a/docs/archive/phase-2.md b/docs/archive/phase-2.md new file mode 100644 index 0000000..4cb7ed9 --- /dev/null +++ b/docs/archive/phase-2.md @@ -0,0 +1,157 @@ +# Phase 2: Anthropic Provider + +**Status: complete.** Anthropic Messages API streams end-to-end alongside the phase-1 OpenAI provider; the Receiver callback sequence is identical across both; system prompts extract to the top-level field; thinking blocks round-trip via captured `signature` deltas. Refinements that diverged from the original plan: file/type names use the explicit dialect (`provider_anthropic_messages`, `AnthropicMessagesConfig`) to leave room for `openai_responses` and future Anthropic shapes; `Config` is a tagged union keyed by `APIStyle` rather than separate types at call sites; concrete provider types are internal — callers construct via `provider.Provider.init(allocator, io, config)`. Wire-level gotchas worth remembering: both providers send `accept-encoding: identity` because gzip buffers small SSE frames and defeats streaming; the read loop uses `readVec` rather than `readSliceShort` because the latter fills its buffer before returning, which also defeats streaming. Tool use / tool result blocks remain stubbed for phase 3+. + +## Goal + +Add Anthropic as a second provider, validating that the Provider abstraction and internal conversation model are genuinely provider-agnostic — not just OpenAI in disguise. + +## Deliverable + +A working `provider_anthropic.zig` that can hold a streaming conversation via Anthropic's API. At the end of this phase, you can: + +- Switch between OpenAI and Anthropic providers with no changes to the agent loop or conversation model. +- Stream responses from either provider and see the same callback sequence (onMessageStart, onBlockStart, onContentDelta, onBlockComplete, onMessageComplete). +- Observe thinking and text content streaming correctly from Anthropic models. + +## What is usable at the end + +| Capability | How to exercise it | +|---|---| +| Create an Anthropic provider | `AnthropicProvider.init(allocator, config)` | +| Run a chat via Anthropic | Same `agent.runStep()` call, different provider | +| Stream Anthropic responses | Same Receiver interface, same callback sequence | +| System prompts with Anthropic | System messages extracted and sent as top-level system field | + +## What is explicitly out of scope + +- Tools and tool-use (phase 3+) +- Extensions (phase 3+) +- Conversation serialization / disk persistence (phase 4) +- Server/proxy mode (future) +- Google API provider (future) + +--- + +## Receiver Interface + +The Receiver interface with the full 5-callback lifecycle is defined in phase 1. Both providers must produce the same callback sequence: + +- onMessageStart → onBlockStart → onContentDelta(s) → onBlockComplete → ... → onMessageComplete + +See `phase-1.md` for the full definition and contract. + +### How OpenAI synthesizes the callbacks + +Defined in phase 1. OpenAI has no explicit block boundaries; the provider infers them via a state machine that tracks the active block type and emits start/complete callbacks on transitions. + +### How Anthropic maps to the callbacks + +Anthropic's structured events map directly: + +| Anthropic event | Callback | +|---|---| +| `message_start` | onMessageStart(.assistant) | +| `content_block_start` | onBlockStart(type, index, meta if ToolUse) | +| `content_block_delta` | onContentDelta(index, delta bytes) | +| `content_block_stop` | onBlockComplete(index, assembled block) | +| `message_delta` + `message_stop` | onMessageComplete(assembled message) | + +No inference needed — Anthropic gives us explicit boundaries. + +--- + +## Anthropic Request Serialization + +### Wire format differences from OpenAI + +| Aspect | OpenAI | Anthropic | +|---|---|---| +| System prompt | Messages with `role: "system"` | Top-level `system` field (string) | +| Content shape | String or array of parts | Always array of content blocks | +| Tool results | Separate `role: "tool"` messages | Content blocks on `role: "user"` messages | +| Auth | `Authorization: Bearer ` | `x-api-key: ` + `anthropic-version` header | +| Streaming | `stream: true` in request body | `stream: true` in request body | + +### Serialization rules + +**System messages**: Extract all `role=.system` messages from the conversation. Concatenate their Text block contents into a single string. Set as the top-level `system` field. Do not include them in the messages array. + +**User messages**: Emit as `role: "user"`. Content blocks become Anthropic content block format: +- Text → `{ "type": "text", "text": "..." }` +- ToolResult → `{ "type": "tool_result", "tool_use_id": "...", "content": "..." }` (phase 3+) + +**Assistant messages**: Emit as `role: "assistant"`. Content blocks: +- Text → `{ "type": "text", "text": "..." }` +- Thinking → `{ "type": "thinking", "thinking": "..." }` +- ToolUse → `{ "type": "tool_use", "id": "...", "name": "...", "input": {...} }` (phase 3+) + +Note: Anthropic expects ToolUse's `input` as a parsed JSON object, not a string. Since we store `input` as raw bytes in a TextualBlock, we will need to parse it into a `std.json.Value` during Anthropic serialization. This is the one place where libpanto does parse tool input JSON — it's a serialization requirement, not an interpretation of the tool schema. The round-trip guarantee is: the bytes we stored serialize back to equivalent JSON when sent to Anthropic. + +--- + +## Anthropic Streaming Event Parser + +Each SSE event is a complete JSON object. The event type is in a top-level `type` field. + +### Event types and handling + +| Event type | What we extract | Action | +|---|---|---| +| `message_start` | `message.role`, `message.id`, `message.model` | Emit onMessageStart; begin assembling Message | +| `content_block_start` | `content_block.type`, `content_block.index`, `content_block.id`, `content_block.name` | Emit onBlockStart; create new TextualBlock or ToolUseBlock | +| `content_block_delta` | `delta.type` (text_delta, thinking_delta, input_json_delta), `delta.text` or `delta.thinking` or `delta.partial_json` | Append to current block's buffer; emit onContentDelta | +| `content_block_stop` | `index` | Emit onBlockComplete with assembled block | +| `message_delta` | `delta.stop_reason`, `usage` | Track stop reason for onMessageComplete | +| `message_stop` | (none) | Emit onMessageComplete with fully assembled Message | + +The parser is a separate concern from the SSE line parser (`sse.zig`). The SSE parser reassembles byte chunks into complete `data: {...}` events. The Anthropic event parser interprets the JSON of each event. They compose: `HTTP read → SSE parser → event strings → Anthropic event parser → callbacks`. + +--- + +## Module Changes + +### New files + +``` +src/provider_anthropic.zig // Anthropic provider implementation +``` + +### Modified files + +- `provider.zig` — ReceiverVTable expanded to the 5-callback lifecycle +- `provider_openai.zig` — No changes needed (block synthesis already built in phase 1) +- `agent.zig` — Any changes needed for expanded Receiver (should be minimal since Receiver is an interface) + +### Config + +``` +AnthropicConfig = struct { + api_key: []const u8, + base_url: []const u8, // e.g. "https://api.anthropic.com" + model: []const u8, // e.g. "claude-sonnet-4-20250514" + api_version: []const u8, // e.g. "2023-06-01" +}; +``` + +Separate from `OpenAIConfig`. The CLI and any higher-level config can unify them; libpanto treats them as distinct. + +--- + +## Testing Strategy + +### Unit tests + +| What | How | +|---|---| +| Anthropic serialization | Create conversations with system/user/assistant messages, serialize to Anthropic JSON, verify structure and system prompt extraction | +| Streaming event parser | Feed canned Anthropic SSE events (message_start, content_block_start, etc.) and verify correct callback sequence and assembled output | +| Block boundary synthesis | Feed OpenAI-style deltas (reasoning → content transitions) and verify onBlockStart/onBlockComplete emitted correctly | +| Receiver contract | Verify that both providers produce the same callback sequence for equivalent conversations | + +### Integration test (manual) + +- Run `panto` binary against Anthropic API with a real API key +- Hold a multi-turn conversation with thinking model +- Verify thinking and text stream correctly +- Switch to OpenAI provider, verify same experience diff --git a/docs/archive/phase-3.md b/docs/archive/phase-3.md new file mode 100644 index 0000000..eb52ee9 --- /dev/null +++ b/docs/archive/phase-3.md @@ -0,0 +1,391 @@ +# Phase 3: Extension System + +> **Status:** Complete. The Lua runtime described in this doc has been **superseded by [`LUA_MAKEOVER.md`](../LUA_MAKEOVER.md)**. The current `panto` CLI uses a long-lived `lua_State` exposed to libpanto as a single `ToolSource`; the per-call `LuaStatePool` / `LuaTool` design described below no longer exists. The **native extension contract (`Tool`) is unchanged** — the rest of this document remains accurate for that path. + +## Goal + +Introduce a native-code tool extension API on `libpanto` and a Lua extension runtime in the `panto` CLI. Together these transform `pantograph` from a chat client into an agent that can act on the world. The extension system is the primary mechanism for adding capability — tools are extensions, not built-ins. + +This phase also marks the point where the `Agent` abstraction starts earning its keep. Until now, `agent.zig` has been a thin pass-through to the provider. In phase 3 the agent grows into the thing that owns the tool registry and drives the tool-call loop. Providers stay dumb (stream blocks); the agent is what makes a conversation iterative rather than one-shot. + +## Deliverable + +Two layered deliverables: + +1. **libpanto** — a Zig-native tool extension API. `Agent` exposes `registerTool(Tool)` and runs a tool-call loop in `runStep`. The agent loop detects ToolUse blocks in LLM responses, dispatches to registered tools, feeds ToolResult blocks back, and repeats until the model stops calling tools. libpanto has no awareness of Lua, Python, or any specific extension language. + +2. **panto CLI** — a Lua extension runtime that discovers Lua scripts on disk, loads them, and registers each declared tool with `libpanto` via a `LuaTool` adapter that satisfies libpanto's `Tool` interface. + +At the end of this phase, you can: + +- Embed `libpanto` in a Zig program, implement a `Tool` struct natively, and register it with the agent — no Lua involved. +- Write a Lua extension that registers a tool, place it in `.agent/extensions/` (or `~/.config/panto/extensions/`), and have the `panto` CLI discover and load it. +- Have a conversation where the LLM calls your tool and receives the result. +- See tool calls execute in parallel when the LLM returns multiple ToolUse blocks. +- See a meaningful error message when a Lua extension crashes, instead of a process abort. + +## What is usable at the end + +| Capability | How to exercise it | +|---|---| +| Native tool registration | In a Zig embedder, construct a `Tool` and call `agent.registerTool(tool)` | +| Write a Lua tool extension | Create `.agent/extensions/mytool.lua` calling `panto.register_tool(...)` | +| Discover Lua extensions | Place `.lua` files or directories in extension directories; `panto` CLI loads them on startup | +| LLM calls a tool | Ask the LLM to use a registered tool; it emits a ToolUse block; the agent dispatches to the registered handler | +| Tool result fed back | The tool's return bytes become a ToolResult block sent back to the LLM | +| Parallel tool calls | LLM returns multiple ToolUse blocks; they execute concurrently | +| Lua crash handling | A crashing Lua handler prints `the "mytool" extension crashed: ` and aborts the turn | + +## What is explicitly out of scope + +- Core tools as extensions (phase 5 — `std.read`, `std.write`, etc.) +- Conversation serialization / disk persistence (phase 4) +- C ABI distribution of libpanto for external consumers (future) +- GitHub or luarocks extension loaders (future — local filesystem only in phase 3) +- Shared-object extensions (future) +- Extension sandboxing beyond `xpcall` crash protection in the Lua adapter (future) +- Config file for specifying which extensions to load (phase 6 — phase 3 loads everything it discovers) +- Non-Lua extension runtimes (future — the libpanto API is general, but only the Lua adapter ships in phase 3) + +--- + +## libpanto: Native Tool API + +### The `Tool` interface + +`libpanto` defines a `Tool` as an opaque-context value with a vtable. This is the boundary between the agent loop and any extension runtime. + +```zig +pub const Tool = struct { + name: []const u8, // borrowed; lifetime owned by the registrar + schema_json: []const u8, // borrowed JSON Schema bytes; lifetime owned by the registrar + ctx: *anyopaque, + vtable: *const VTable, + + pub const VTable = struct { + /// Invoke the tool. MUST be thread-safe — the agent may call `invoke` + /// concurrently from multiple threads when the LLM returns multiple + /// ToolUse blocks in a single response. + /// + /// `input` is the raw JSON bytes the provider sent. The tool is + /// responsible for parsing them if it cares about their structure. + /// + /// Returns owned bytes allocated with `allocator`. These bytes become + /// the `content` of the ToolResult block. + /// + /// Returning an error aborts the current turn. The agent will surface + /// the error to the user. Native tool implementations are responsible + /// for catching their own panics — a panic in `invoke` will crash the + /// process. Adapters that bridge to safer languages (Lua, Python, Go) + /// should convert panics/exceptions into errors here. + invoke: *const fn (ctx: *anyopaque, input: []const u8, allocator: std.mem.Allocator) anyerror![]u8, + + /// Called when the tool is removed from the registry or the agent is + /// torn down. Frees any resources owned by `ctx`. + deinit: *const fn (ctx: *anyopaque, allocator: std.mem.Allocator) void, + }; +}; +``` + +The contract: + +- **Thread-safety is mandatory.** Tool implementations promise that concurrent `invoke` calls are safe. This pushes the burden onto adapters (where it belongs — the Lua adapter solves it with a `lua_State` pool; a native Zig tool may just be re-entrant) and out of the agent loop. +- **Input is opaque bytes.** libpanto never parses tool input. The tool decides whether to parse JSON, treat it as a string, or ignore it. +- **Output is opaque bytes.** Same principle in reverse. +- **Errors abort the turn.** No partial recovery in phase 3. +- **Panics crash the process.** Native code that wants safety implements it itself. This is trivially obvious from the API surface: it's a function pointer with `anyerror!` return — there's no magic. Adapters for safer languages can wrap panics/exceptions and convert them to errors. + +### `Agent` changes + +```zig +pub const Agent = struct { + provider: provider.Provider, + allocator: std.mem.Allocator, + registry: ToolRegistry, + + pub fn init(allocator: Allocator, prov: provider.Provider) Agent { ... } + pub fn deinit(self: *Agent) void { ... } + + pub fn registerTool(self: *Agent, tool: Tool) !void { ... } + pub fn unregisterTool(self: *Agent, name: []const u8) void { ... } + + pub fn runStep(self: *Agent, conv: *Conversation, receiver: *Receiver) !void { ... } +}; +``` + +`registerTool` takes ownership of the `Tool` value (the agent calls `tool.vtable.deinit` on teardown or unregister). Tool name uniqueness is enforced — registering a name that already exists is an error. + +### `ToolRegistry` + +A small internal module: + +``` +ToolRegistry = struct { + tools: std.StringHashMap(Tool), + allocator: Allocator, + + fn register(name, tool) !void + fn unregister(name) void + fn lookup(name) ?*const Tool + fn iterator() Iterator // for serializing into provider requests + fn deinit() void // invokes each tool's vtable.deinit +}; +``` + +### Agent loop with tools + +`runStep` gains the tool-call loop: + +``` +runStep(conv, receiver): + loop: + provider.streamStep(conv, receiver) + inspect the assistant message just appended to conv for ToolUse blocks + if none: return — turn complete + for each ToolUse block, in parallel: + tool = registry.lookup(block.name) or error + result_bytes = tool.vtable.invoke(tool.ctx, block.input, allocator) + build a ToolResult block { tool_use_id = block.id, content = result_bytes } + append a user Message containing all ToolResult blocks to conv + continue loop +``` + +A single `runStep` may invoke the provider multiple times if the LLM chains tool calls. + +### Parallel execution + +When a response contains multiple ToolUse blocks, the agent dispatches them concurrently. Implementation likely uses a small thread pool or `std.Thread.spawn` per call (TBD during implementation). Because `Tool.invoke` is declared thread-safe, the agent loop has no further constraints — it just fans out and joins. + +### Tool serialization in provider requests + +When the agent calls the provider, the provider receives the registry (or an iterator over it) and emits the tools array. This means provider request serialization gains a tools-emitting branch when the registry is non-empty. + +**OpenAI**: +```json +{ + "tools": [ + { "type": "function", + "function": { "name": "echo", "parameters": } } + ] +} +``` + +**Anthropic**: +```json +{ + "tools": [ + { "name": "echo", "input_schema": } + ] +} +``` + +The `schema_json` bytes are emitted verbatim. Description fields are deferred — `Tool` does not yet carry a description (see open questions). + +### ToolUse decoding in responses + +Already handled by the existing Receiver callback sequence from phase 1/2: + +- OpenAI: `delta.tool_calls` → `onBlockStart(.ToolUse, ...)` with `meta.tool_id` and `meta.tool_name`, then `onContentDelta` with JSON argument fragments. +- Anthropic: `content_block_start` with `type: "tool_use"` → `onBlockStart(.ToolUse, ...)`, then `content_block_delta` with `input_json_delta` fragments. + +The assembled ToolUseBlock contains `id`, `name`, and `input` (TextualBlock with the full JSON string). + +### ToolResult encoding in requests + +**OpenAI** — each ToolResult block becomes a separate top-level `tool` message: +```json +{ "role": "tool", "tool_call_id": "", "content": "" } +``` + +**Anthropic** — ToolResult blocks are content blocks on a user message: +```json +{ "role": "user", "content": [ + { "type": "tool_result", "tool_use_id": "", "content": "" } +] } +``` + +--- + +## panto CLI: Lua Extension Runtime + +The Lua runtime lives entirely in the CLI. libpanto has no Lua dependency. + +### Extension discovery + +The CLI scans two directories in order: + +1. `.panto/extensions/` — project-local, relative to current working directory +2. `~/.config/panto/extensions/` — user-level + +`.panto/` follows the established per-agent pattern (`.claude/`, `.opencode/`, `.pi/`). + +### Naming and structure + +- **Single-file**: `.lua` → extension name is `` +- **Directory**: `/init.lua` → extension name is `` + +Hierarchical names via dot/directory mapping, mirroring `require("a.b.c")`: + +- `utils/json.lua` → `utils.json` +- `coding/edit/init.lua` → `coding.edit` + +Sub-modules under a directory extension (e.g., `coding/edit/helpers.lua`) are the extension's internal business — the CLI only loads the top-level entry point. + +### Loading behavior + +- Recursively scan both directories. +- Construct extension names from relative paths. +- For each unique extension name, load its entry point into the `lua_State` pool (see below). +- Project-local entries shadow user-level entries of the same name. +- Extension top-level code runs at load time; it is expected to call `panto.register_tool(...)`. + +### The Lua bridge + +A small Zig module (`lua_bridge.zig`) registers Zig functions into a `lua_State`. The bridge exposes: + +#### `panto.register_tool(name, schema, handler)` + +- `name` (string) — tool name, e.g. `"bash"` +- `schema` (table) — JSON Schema as a Lua table. The bridge serializes this to JSON bytes at registration time. +- `handler` (function) — invoked when the LLM calls this tool. Receives a single argument: a Lua table parsed from the input JSON. Must return a string. + +```lua +panto.register_tool("echo", { + type = "object", + properties = { + message = { type = "string", description = "The message to echo back" } + }, + required = { "message" } +}, function(input) + return input.message +end) +``` + +The bridge stores the handler in the Lua registry (`luaL_ref`) and constructs a `LuaTool` (see below) that it then registers with the agent. + +### `LuaTool` — the adapter to libpanto + +```zig +const LuaTool = struct { + pool: *LuaStatePool, + handler_ref: i32, // registry ref to the Lua function + name_owned: []u8, + schema_owned: []u8, + + // Implements Tool.VTable.invoke: + // 1. pool.acquire() → lua_State + // 2. lua_rawgeti(L, LUA_REGISTRYINDEX, handler_ref) — push handler + // 3. parse input bytes into a Lua table via std.json + // 4. xpcall the handler with the table + // 5. read returned string (or capture trace on error) + // 6. pool.release(L) + // 7. return result bytes or error +}; +``` + +The adapter is what makes the CLI side responsible for Lua-specific concerns: + +- **JSON → Lua table conversion**: happens here, not in libpanto. +- **`xpcall` crash protection**: wraps the handler call, captures `debug.traceback`, returns an error so libpanto aborts the turn cleanly with `the "" extension crashed: `. +- **Thread-safety**: provided by the `lua_State` pool. Each concurrent `invoke` checks out a separate state. libpanto's contract is satisfied. + +### `LuaStatePool` + +On-demand pool of `lua_State` instances: + +``` +LuaStatePool = struct { + states: std.ArrayList(*lua_State), + available: std.BitSet, + allocator: Allocator, + extension_paths: []const []const u8, // all discovered entry points + + fn acquire() *lua_State // returns a free state, or creates a new one + fn release(L: *lua_State) void // returns L to the pool + fn deinit() void // closes all states +}; +``` + +- States are created lazily — no pre-allocation. +- Each new state loads all discovered extensions identically, so any state can handle any tool. +- A state is checked out for the duration of a single `invoke` call. + +There's a coupling here: the handler `luaL_ref` is per-`lua_State`. The pool must guarantee that the same registry slot in every state points at the same handler. The cleanest way is: when an extension is first loaded into the prototype state, the bridge records the registry index. When a new state is constructed, the bridge re-loads all extensions in the same order, producing the same ref indices. This needs care during implementation but is mechanically straightforward. + +--- + +## Module Changes + +### libpanto + +New: +- `libpanto/src/tool.zig` — `Tool` and `VTable` definitions +- `libpanto/src/tool_registry.zig` — `ToolRegistry` + +Modified: +- `libpanto/src/agent.zig` — owns registry, gains `registerTool`, drives tool-call loop in `runStep` +- `libpanto/src/provider.zig` — provider receives a way to see registered tools (likely a `*const ToolRegistry` on the agent passed via `streamStep` context, or a callback) +- `libpanto/src/provider_openai_chat.zig` — request serialization emits `tools` array +- `libpanto/src/provider_anthropic_messages.zig` — request serialization emits `tools` array +- `libpanto/src/openai_chat_json.zig` — ToolResult message encoding +- `libpanto/src/anthropic_messages_json.zig` — ToolResult content block encoding +- `libpanto/src/root.zig` — re-exports `Tool`, `ToolRegistry` + +### panto CLI + +New: +- `src/lua_bridge.zig` — Zig functions registered into Lua states; `panto.register_tool` implementation +- `src/lua_tool.zig` — `LuaTool` adapter implementing libpanto's `Tool` interface +- `src/lua_state_pool.zig` — on-demand `lua_State` pool +- `src/extension_loader.zig` — directory scanning, name construction, loading + +Modified: +- `src/main.zig` — wires extension discovery + loading into agent startup + +### External dependency + +Lua 5.4 linked into the `panto` binary. Zig's build system can fetch and compile Lua from source (~30KLOC C). No system dependency required. libpanto does not link Lua. + +--- + +## Testing Strategy + +### Unit tests (libpanto) + +| What | How | +|---|---| +| Tool registration | Construct a native `Tool`, register with `Agent`, verify lookup | +| Duplicate registration | Verify registering the same name twice returns an error | +| Tool dispatch | Hand-craft a conversation with a ToolUse block, run `runStep` against a stub provider, verify the registered tool's `invoke` is called with the right bytes | +| ToolResult round-trip | Verify the ToolResult message appended to the conversation has the right `tool_use_id` and `content` | +| Parallel dispatch | Register two tools with sleep+timestamps, verify they ran concurrently (elapsed wall time < sum of individual times) | +| Provider request serialization | Snapshot tests on OpenAI and Anthropic request bodies with tools registered | + +### Unit tests (panto CLI) + +| What | How | +|---|---| +| Extension discovery | Create temp directory with single-file and directory extensions, verify names constructed correctly | +| Project shadows user | Same extension name in both directories — project wins | +| Lua bridge input | Feed JSON bytes through `LuaTool.invoke`, verify the Lua handler sees the expected table | +| Lua bridge output | Handler returns a string, verify it round-trips to bytes | +| `xpcall` crash protection | Extension whose handler throws — verify `invoke` returns an error and includes the trace | +| Concurrent `invoke` | Two threads call `invoke` on the same `LuaTool`, verify they don't share a `lua_State` | + +### Integration tests (manual) + +- `echo.lua` extension: ask the LLM to use it, verify result is fed back, LLM continues. +- `crash.lua` extension: verify the crash is caught, printed, turn aborts gracefully. +- Multi-tool response: ask the LLM to call two tools in one message, verify both execute concurrently. + +--- + +## Resolved Decisions + +- **Project-local extension directory**: `.panto/extensions/`, matching `.claude/`, `.opencode/`, `.pi/`. +- **Lua version**: Lua 5.4. LuaJIT is stuck at 5.1 semantics and the JIT speedup is largely irrelevant for tool handlers (which spend most of their time in Zig-side I/O). 5.4 keeps us aligned with current luarocks and Lua documentation. +- **Tool description**: a top-level `description: []const u8` field on `Tool`. The Lua bridge gains it as a parameter: `panto.register_tool(name, description, schema, handler)`. Schema describes input only; description is the human-readable purpose. +- **Provider sees the registry**: `provider.streamStep` gains a `*const ToolRegistry` parameter. Embedders normally pass the same registry on every call. +- **Parallel dispatch**: `std.Thread.spawn` per ToolUse block. Thread creation is sub-millisecond on Linux/macOS; tool calls are orders of magnitude longer. Recycling buys nothing meaningful. +- **Streaming tool results**: deferred, possibly indefinitely. Provider APIs only accept a final result payload, so streaming would be display-only — not worth the vtable complication in phase 3. +- **Handler timeout / cancellation**: deferred. POSIX has no clean way to cancel a thread (`pthread_kill` with SIGKILL terminates the whole process; `pthread_cancel` is unusable in practice; signal-based interruption is not async-signal-safe for arbitrary code). Lua's `lua_sethook` only fires between VM steps, so it can't interrupt a handler blocked in a C call. A real cancellation primitive requires process isolation, which is a phase of its own — see the Punted list in [overview.md](overview.md). Phase 3 ships with the honest contract: **tool handlers run to completion. If a tool hangs, the user kills the `panto` process.** diff --git a/docs/phase-1.md b/docs/phase-1.md deleted file mode 100644 index 6b149aa..0000000 --- a/docs/phase-1.md +++ /dev/null @@ -1,449 +0,0 @@ -# Phase 1: libpanto — Minimal Chat Library - -**Status: complete.** Streaming chat works end-to-end against OpenAI-compatible APIs; conversation history persists across turns; thinking blocks (`reasoning_content` / `reasoning`) are streamed; `reasoning_effort` is configurable. Open questions resolved: thinking support implemented; mid-stream errors propagate via Zig errors; connections are one-per-turn intentionally (uniform reentry into each turn); long-conversation memory deferred to a later phase. Session persistence is phase 4. - -## Goal - -A Zig library that can hold a streaming conversation with an LLM via an OpenAI-compatible API. No tools, no extensions — just chat. Includes a minimal CLI for live testing. - -## Deliverable - -A `libpanto` Zig module importable by other Zig code, plus a `panto` binary that wires it into a basic read/print loop. At the end of this phase, you can: - -- Start `panto`, type a message, and receive a streamed response from an OpenAI-compatible LLM. -- Send follow-up messages that include full conversation history. -- See thinking tokens and text tokens stream in as they arrive. -- Have the complete conversation available in memory for the duration of the session. - -## What is usable at the end - -| Capability | How to exercise it | -|---|---| -| Open a conversation | `libpanto.conversation.Conversation.init(allocator)` | -| Add a user message | `conversation.addUserMessage("hello")` | -| Run an agent step (streaming) | `agent.runStep(conversation, &receiver)` | -| See streamed output | CLI prints thinking/text chunks as they arrive | -| Conversation persists across turns | Follow-up messages include prior history | - -## What is explicitly out of scope - -- Tools and tool-use (phase 3+) -- Extensions and extension API (phase 3+) -- C ABI (phase 3+, when needed for Lua extensions) -- Anthropic provider (phase 2) -- Disk persistence / session save (later phase) -- Server/proxy mode (future, undefined phase) -- System prompt construction framework (later phase — the model supports system messages, but no opinionated assembly system yet) - ---- - -## Data Model - -### TextualBlock (shared streaming buffer) - -``` -TextualBlock = struct { - buf: std.ArrayList(u8), - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) TextualBlock - pub fn content(self: *const TextualBlock) []const u8 // self.buf.items - pub fn append(self: *TextualBlock, delta: []const u8) !void - pub fn deinit(self: *TextualBlock) void // self.buf.deinit() -} -``` - -`Text` and `Thinking` blocks both use `TextualBlock` as their payload. They share the same append-based streaming behavior — deltas arrive incrementally and are appended via an internal `ArrayList(u8)`, giving amortized O(1) appends and avoiding the O(n²) re-copying that would result from storing `[]const u8` slices. - -The `TextualBlock` stores its own allocator reference so that `deinit()` needs no external context. Each `TextualBlock` has an `init()`/`deinit()` pair. This also means a `ContentBlock` can clean itself up without the caller providing an allocator. - -### ContentBlock (tagged union) - -``` -ContentBlock = union(enum) { - Text: TextualBlock, - Thinking: TextualBlock, - ToolUse: ToolUseBlock, // phase 3+ - ToolResult: ToolResultBlock, // phase 3+ - - pub fn deinit(self: *ContentBlock) void { - switch (self.*) { - .Text, .Thinking => |*b| b.deinit(), - .ToolUse => |b| { /* free id, name, input */ }, - .ToolResult => |b| { /* free tool_use_id, content */ }, - } - } -} - -ToolUseBlock = struct { - id: []const u8, // owned copy - name: []const u8, // owned copy - input: []const u8, // raw JSON bytes, owned copy -} - -ToolResultBlock = struct { - tool_use_id: []const u8, // owned copy - content: []const u8, // owned copy -} -``` - -`ToolUse` and `ToolResult` are defined in the model now but not populated or processed until the extensions phase. This avoids a model refactor later — the types exist, we just never encounter them in phase 1. - -The `input` field of `ToolUse` is stored as raw JSON bytes (`[]const u8`) rather than a parsed structure. We are not in the business of understanding tool input schemas; we pass them through. - -`ToolUse` blocks also stream incrementally — both providers send tool input as JSON fragments across multiple deltas. Therefore `ToolUse.input` also uses `TextualBlock` for assembly. `id` and `name` arrive at block-start time and are stored as owned `[]const u8` copies. - -`ToolResult` blocks are constructed by `pantograph` itself (not streamed from a provider), so `content` could be a simple `[]const u8`. However, for consistency and to allow progressive construction of results, it also uses `TextualBlock`. - -Updated types: -``` -ToolUseBlock = struct { - id: []const u8, // owned copy, from onBlockStart metadata - name: []const u8, // owned copy, from onBlockStart metadata - input: TextualBlock, // accumulated from onContentDelta -} - -ToolResultBlock = struct { - tool_use_id: []const u8, // owned copy - content: TextualBlock, // accumulated content -} -``` - -**Memory discipline**: When a `ContentBlock` is moved into a `Message`'s content list (stored in `std.ArrayList(ContentBlock)`), the TextualBlock's internal ArrayList buffer pointer remains valid — it points to the same heap allocation. The caller must ensure each block's `deinit()` is called exactly once, and must not copy a ContentBlock without clearing the source (standard Zig move semantics). - -### Message - -``` -Message = { - role: enum { system, user, assistant }, - content: []ContentBlock, -} -``` - -A system message may contain multiple `Text` blocks. When serializing to Anthropic's API (phase 2), these are concatenated into the single system prompt string. - -An assistant message is assembled incrementally during streaming. A user message containing tool results (phase 3+) naturally groups multiple `ToolResult` blocks. - -### Conversation - -``` -Conversation = { - messages: std.ArrayList(Message), - allocator: std.mem.Allocator, -} -``` - -Ordered list of messages. Methods: - -- `init(allocator)` → Conversation -- `addSystemMessage(text)` → appends `Message{ .system, [TextBlock(text)] }` -- `addUserMessage(text)` → appends `Message{ .user, [TextBlock(text)] }` -- `addAssistantMessage(blocks)` → appends `Message{ .assistant, blocks }` (called by agent loop after streaming completes) -- `deinit()` → frees all owned memory - -All `[]const u8` fields in ContentBlocks and Messages are owned by the Conversation and freed on `deinit()`. Content is stored as copies, not slices into external buffers. `TextualBlock` fields back their content with a heap-allocated `ArrayList(u8)` that grows incrementally during streaming and is freed on `deinit()`. - ---- - -## Module Structure - -``` -src/ - root.zig // public API re-exports - conversation.zig // Message, ContentBlock, Conversation - provider.zig // Provider interface, StreamEvent, StreamResult - provider_openai.zig // OpenAI-compatible implementation - sse.zig // SSE line parser - agent.zig // Agent loop: runStep, Receiver interface - config.zig // Config struct (api_key, base_url, model) - json.zig // Serialization helpers (model → wire JSON, deltas → ContentBlocks) -``` - -### `conversation.zig` - -Defines `Message`, `ContentBlock`, `Conversation`. All serialization to/from provider wire formats lives in `json.zig` — conversation.zig is pure data structure. - -Tests: create conversations, add messages, verify content, free without leaks. - -### `provider.zig` - -Defines the `Provider` interface: - -``` -Provider = struct { - ptr: *anyopaque, - vtable: *const VTable, - - VTable = struct { - streamStep: *const fn(*anyopaque, conversation: *Conversation, receiver: *Receiver) anyerror!void, - deinit: *const fn(*anyopaque) void, - }; - - pub fn streamStep(self, conversation, receiver) !void - pub fn deinit(self) void -}; -``` - -And the `Receiver` interface for streaming callbacks: - -``` -Receiver = struct { - ptr: *anyopaque, - vtable: *const ReceiverVTable, - - ReceiverVTable = struct { - onMessageStart: *const fn(*anyopaque, role: MessageRole) void, - onBlockStart: *const fn(*anyopaque, block_type: ContentBlockType, index: usize, meta: ?BlockMeta) void, - onContentDelta: *const fn(*anyopaque, block_index: usize, delta: []const u8) void, - onBlockComplete: *const fn(*anyopaque, block_index: usize, block: ContentBlock) void, - onMessageComplete:*const fn(*anyopaque, message: Message) void, - }; - - pub fn onMessageStart(self, role) void - pub fn onBlockStart(self, block_type, index, meta) void - pub fn onContentDelta(self, block_index, delta) void - pub fn onBlockComplete(self, block_index, block) void - pub fn onMessageComplete(self, message) void -}; - -BlockMeta = struct { - // Only populated for ToolUse blocks. Null for Text/Thinking. - tool_id: ?[]const u8, - tool_name: ?[]const u8, -}; -``` - -**Callback contract:** - -- Callbacks are always invoked in this order for every block, regardless of which provider is active. -- `onMessageStart` is called when the stream begins delivering a new message. -- `onBlockStart` is called when a new content block begins. `meta` carries block-type-specific metadata (tool id/name for ToolUse, null for Text/Thinking). -- `onContentDelta` is called zero or more times per block with raw byte fragments. For Text/Thinking these are word fragments; for ToolUse these are JSON fragments. The receiver does not need to interpret them. `delta` is a `[]const u8` — libpanto does not parse tool input content, it passes bytes through. -- `onBlockComplete` is called when a block is finished. The `block` parameter contains the fully assembled ContentBlock. The receiver that only needs complete content can ignore deltas and use this. -- `onMessageComplete` is called when the stream ends. The `message` parameter contains the fully assembled Message with all blocks. -- Providers guarantee that `onBlockComplete`'s `block` and `onMessageComplete`'s `message` are always fully assembled and valid. - -This uniform callback sequence means the TUI and agent loop don't need to know which provider is active. Anthropic (phase 2) maps its structured events directly to these callbacks; OpenAI synthesizes block boundaries from delta field transitions (see below). - -### `provider_openai.zig` - -Implements `Provider` for OpenAI-compatible APIs. - -- Converts `Conversation` → OpenAI wire JSON (see [OpenAI serialization](#openai-serialization) below) -- Makes HTTP POST to `{base_url}/chat/completions` with `stream: true` -- Reads SSE events, parses each `data: {...}` line as complete JSON -- Synthesizes block boundaries from delta field transitions (OpenAI has no explicit block boundary events) -- Calls the full Receiver callback sequence (onMessageStart → onBlockStart → onContentDelta → onBlockComplete → onMessageComplete) -- Accumulates deltas into ContentBlocks via TextualBlocks - -Construction: - -``` -OpenAIProvider.init(allocator, config) !OpenAIProvider -``` - -### OpenAI block boundary synthesis - -OpenAI's streaming deltas have no explicit block boundaries. The provider tracks a state machine to infer when blocks start and end: - -``` -StreamingState = struct { - active_block_type: enum { none, thinking, text, tool_use }, - active_block_index: usize, - // assembly buffers per block -} - -On each SSE event: - 1. If delta.role == "assistant" → emit onMessageStart(.assistant) - 2. If delta.reasoning_content present: - - If active_block_type != .thinking: - - If active_block_type != .none → emit onBlockComplete for prior block - - Emit onBlockStart(.Thinking, index, null) - - active_block_type = .thinking - - Emit onContentDelta(index, delta.reasoning_content) - 3. If delta.content present: - - If active_block_type != .text: - - If active_block_type != .none → emit onBlockComplete for prior block - - Emit onBlockStart(.Text, index, null) - - active_block_type = .text - - Emit onContentDelta(index, delta.content) - 4. If delta.tool_calls present: - - If active_block_type != .tool_use: - - If active_block_type != .none → emit onBlockComplete for prior block - - Emit onBlockStart(.ToolUse, index, .{ .tool_id = ..., .tool_name = ... }) - - active_block_type = .tool_use - - Emit onContentDelta(index, delta.tool_calls[].function.arguments) - 5. If finish_reason != null: - - If active_block_type != .none → emit onBlockComplete for current block - - Emit onMessageComplete(assembled_message) -``` - -A transition in `active_block_type` means the previous block is done and a new one has started. The state machine also handles the case where the same block type appears again after an intervening type (e.g., thinking → text → thinking), which would open a new Thinking block at a new index. - -### `sse.zig` - -Incremental SSE line parser. The HTTP client delivers arbitrary-sized read buffers; this module reassembles them into complete `data: ...\n\n` events. - -``` -SSEParser = struct { - buf: std.ArrayList(u8), - - pub fn init(allocator) SSEParser - pub fn feed(self, chunk: []const u8) ![]const []const u8 // returns slice of complete event strings - pub fn deinit(self) void -}; -``` - -`feed()` may return zero events (partial line buffered) or multiple events (chunk contained several). The caller does not need to worry about line boundaries. - -Tests: feed partial chunks, verify events emitted at correct boundaries; multi-event in single chunk; empty lines; `data: [DONE]`. - -### `json.zig` - -Two responsibilities: - -1. **Serialize Conversation → OpenAI request body** — Convert our `Message`/`ContentBlock` model into the JSON shape OpenAI expects. See below. -2. **Parse SSE chunk deltas → ContentBlock updates** — Each SSE event's JSON contains a `choices[0].delta` object. Extract text/thinking content from it. - -### `agent.zig` - -The agent loop. In phase 1, it's simple: - -``` -Agent = struct { - provider: Provider, - allocator: std.mem.Allocator, - - pub fn init(allocator, provider) Agent - pub fn runStep(self, conversation: *Conversation, receiver: *Receiver) !void - pub fn deinit(self) void -}; -``` - -`runStep` does: -1. Call `provider.streamStep(conversation, receiver)` — this streams the response and calls the full Receiver callback sequence on the receiver -2. The `onMessageComplete` callback appends the finished Message to the conversation (the agent itself can wire this, or the caller handles it — TBD during implementation) - -In later phases, `runStep` gains the tool-call loop: check for ToolUse blocks, execute them, feed results back, call provider again. But the shape stays the same — one `runStep` invocation carries the conversation through a full agent turn. - -### `config.zig` - -``` -Config = struct { - api_key: []const u8, - base_url: []const u8, // e.g. "https://api.openai.com/v1" - model: []const u8, // e.g. "gpt-4o" -}; -``` - -Populated from environment variables (`PANTO_API_KEY`, `PANTO_BASE_URL`, `PANTO_MODEL`) with defaults for base_url and model. - -### `root.zig` - -Public API. Re-exports the types and functions that external Zig code needs: - -``` -pub const conversation = @import("conversation.zig"); -pub const provider = @import("provider.zig"); -pub const agent = @import("agent.zig"); -pub const config = @import("config.zig"); -``` - -Does not re-export provider_openai, sse, or json — those are internal. - ---- - -## OpenAI Serialization - -### Request - -Our `Conversation` → OpenAI `chat/completions` request body: - -``` -{ - "model": config.model, - "stream": true, - "messages": [ - // For each Message in conversation: - // - // role=.system → { "role": "system", "content": "" } - // role=.user → { "role": "user", "content": "" } - // (ToolResult blocks pulled out into separate role:tool messages in phase 3+) - // role=.assistant → { "role": "assistant", "content": [ - // ...text blocks as { "type": "text", "text": "..." }, - // ...thinking blocks as { "type": "thinking", "thinking": "..." }, - // ...tool_use blocks become function_call/function entries in phase 3+ - // ] } - ] -} -``` - -For phase 1, all content blocks we encounter are `Text` or `Thinking`, so serialization is straightforward. - -### Response (streaming) - -Each SSE event is a complete JSON object: - -``` -data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]} -data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]} -data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{"content":" world"},"finish_reason":null}]} -data: {"id":"chatcmpl-123","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} -data: [DONE] -``` - -We parse each event's `choices[0].delta` and drive the block boundary state machine: -- `delta.role == "assistant"` → emit onMessageStart, marks the start of a new assistant message -- `delta.reasoning_content` → transition to Thinking block if needed, append via onContentDelta -- `delta.content` → transition to Text block if needed, append via onContentDelta -- `delta.tool_calls` → transition to ToolUse block if needed, append arguments via onContentDelta (phase 3+) - -The `finish_reason: "stop"` signals stream end → emit onBlockComplete for any active block, then onMessageComplete. - ---- - -## Minimal CLI - -``` -panto/ - src/ - main.zig // CLI entry point -``` - -Behavior: -1. Read `PANTO_API_KEY`, `PANTO_BASE_URL`, `PANTO_MODEL` from environment -2. Create a Conversation, add a system message (default: "You are a helpful assistant.") -3. Print a prompt (`> `), read a line from stdin -4. Add user message, call `agent.runStep()`, print streamed deltas to stdout -5. Repeat step 3 until EOF (Ctrl+D) - -There is no line editing, no scrolling, no syntax highlighting. Just `readline` → `print`. The sole purpose is exercising libpanto against a real API. - ---- - -## Testing Strategy - -### Unit tests (automated, per module) - -| Module | What to test | -|---|---| -| `conversation.zig` | Create conversation, add messages of each role, verify content block storage, free without leaks | -| `sse.zig` | Feed partial chunks, verify event boundaries; multi-event chunks; `data: [DONE]`; empty lines between events | -| `json.zig` | Serialize conversation → OpenAI JSON; parse delta JSON objects → content updates | -| `config.zig` | Parse from env vars; defaults for missing optional fields | - -### Integration test (manual) - -- Run `panto` binary with a real API key -- Hold a multi-turn conversation -- Verify responses stream to stdout -- Verify follow-up messages include prior context (ask the model "what did I just say?") - ---- - -## Open Questions (to resolve during implementation) - -1. **Thinking token support in OpenAI API**: OpenAI's `reasoning_content` field in streaming deltas is not universally present across models/endpoints. We need to handle its absence gracefully (just skip it, don't crash). -2. **Error handling in streams**: Mid-stream HTTP errors, rate limiting, truncated responses. How do we represent these to the caller? An `onError` callback on the Receiver seems likely. -3. **HTTP connection lifecycle**: Does `std.http.Client` support long-lived streaming connections cleanly? We may need to manage connection pooling or timeouts. -4. **Memory strategy for long conversations**: We're storing full message content in memory. For phase 1 this is fine, but we should define the interface so a later phase can introduce message summarization or offloading without changing the agent loop. diff --git a/docs/phase-2.md b/docs/phase-2.md deleted file mode 100644 index 4cb7ed9..0000000 --- a/docs/phase-2.md +++ /dev/null @@ -1,157 +0,0 @@ -# Phase 2: Anthropic Provider - -**Status: complete.** Anthropic Messages API streams end-to-end alongside the phase-1 OpenAI provider; the Receiver callback sequence is identical across both; system prompts extract to the top-level field; thinking blocks round-trip via captured `signature` deltas. Refinements that diverged from the original plan: file/type names use the explicit dialect (`provider_anthropic_messages`, `AnthropicMessagesConfig`) to leave room for `openai_responses` and future Anthropic shapes; `Config` is a tagged union keyed by `APIStyle` rather than separate types at call sites; concrete provider types are internal — callers construct via `provider.Provider.init(allocator, io, config)`. Wire-level gotchas worth remembering: both providers send `accept-encoding: identity` because gzip buffers small SSE frames and defeats streaming; the read loop uses `readVec` rather than `readSliceShort` because the latter fills its buffer before returning, which also defeats streaming. Tool use / tool result blocks remain stubbed for phase 3+. - -## Goal - -Add Anthropic as a second provider, validating that the Provider abstraction and internal conversation model are genuinely provider-agnostic — not just OpenAI in disguise. - -## Deliverable - -A working `provider_anthropic.zig` that can hold a streaming conversation via Anthropic's API. At the end of this phase, you can: - -- Switch between OpenAI and Anthropic providers with no changes to the agent loop or conversation model. -- Stream responses from either provider and see the same callback sequence (onMessageStart, onBlockStart, onContentDelta, onBlockComplete, onMessageComplete). -- Observe thinking and text content streaming correctly from Anthropic models. - -## What is usable at the end - -| Capability | How to exercise it | -|---|---| -| Create an Anthropic provider | `AnthropicProvider.init(allocator, config)` | -| Run a chat via Anthropic | Same `agent.runStep()` call, different provider | -| Stream Anthropic responses | Same Receiver interface, same callback sequence | -| System prompts with Anthropic | System messages extracted and sent as top-level system field | - -## What is explicitly out of scope - -- Tools and tool-use (phase 3+) -- Extensions (phase 3+) -- Conversation serialization / disk persistence (phase 4) -- Server/proxy mode (future) -- Google API provider (future) - ---- - -## Receiver Interface - -The Receiver interface with the full 5-callback lifecycle is defined in phase 1. Both providers must produce the same callback sequence: - -- onMessageStart → onBlockStart → onContentDelta(s) → onBlockComplete → ... → onMessageComplete - -See `phase-1.md` for the full definition and contract. - -### How OpenAI synthesizes the callbacks - -Defined in phase 1. OpenAI has no explicit block boundaries; the provider infers them via a state machine that tracks the active block type and emits start/complete callbacks on transitions. - -### How Anthropic maps to the callbacks - -Anthropic's structured events map directly: - -| Anthropic event | Callback | -|---|---| -| `message_start` | onMessageStart(.assistant) | -| `content_block_start` | onBlockStart(type, index, meta if ToolUse) | -| `content_block_delta` | onContentDelta(index, delta bytes) | -| `content_block_stop` | onBlockComplete(index, assembled block) | -| `message_delta` + `message_stop` | onMessageComplete(assembled message) | - -No inference needed — Anthropic gives us explicit boundaries. - ---- - -## Anthropic Request Serialization - -### Wire format differences from OpenAI - -| Aspect | OpenAI | Anthropic | -|---|---|---| -| System prompt | Messages with `role: "system"` | Top-level `system` field (string) | -| Content shape | String or array of parts | Always array of content blocks | -| Tool results | Separate `role: "tool"` messages | Content blocks on `role: "user"` messages | -| Auth | `Authorization: Bearer ` | `x-api-key: ` + `anthropic-version` header | -| Streaming | `stream: true` in request body | `stream: true` in request body | - -### Serialization rules - -**System messages**: Extract all `role=.system` messages from the conversation. Concatenate their Text block contents into a single string. Set as the top-level `system` field. Do not include them in the messages array. - -**User messages**: Emit as `role: "user"`. Content blocks become Anthropic content block format: -- Text → `{ "type": "text", "text": "..." }` -- ToolResult → `{ "type": "tool_result", "tool_use_id": "...", "content": "..." }` (phase 3+) - -**Assistant messages**: Emit as `role: "assistant"`. Content blocks: -- Text → `{ "type": "text", "text": "..." }` -- Thinking → `{ "type": "thinking", "thinking": "..." }` -- ToolUse → `{ "type": "tool_use", "id": "...", "name": "...", "input": {...} }` (phase 3+) - -Note: Anthropic expects ToolUse's `input` as a parsed JSON object, not a string. Since we store `input` as raw bytes in a TextualBlock, we will need to parse it into a `std.json.Value` during Anthropic serialization. This is the one place where libpanto does parse tool input JSON — it's a serialization requirement, not an interpretation of the tool schema. The round-trip guarantee is: the bytes we stored serialize back to equivalent JSON when sent to Anthropic. - ---- - -## Anthropic Streaming Event Parser - -Each SSE event is a complete JSON object. The event type is in a top-level `type` field. - -### Event types and handling - -| Event type | What we extract | Action | -|---|---|---| -| `message_start` | `message.role`, `message.id`, `message.model` | Emit onMessageStart; begin assembling Message | -| `content_block_start` | `content_block.type`, `content_block.index`, `content_block.id`, `content_block.name` | Emit onBlockStart; create new TextualBlock or ToolUseBlock | -| `content_block_delta` | `delta.type` (text_delta, thinking_delta, input_json_delta), `delta.text` or `delta.thinking` or `delta.partial_json` | Append to current block's buffer; emit onContentDelta | -| `content_block_stop` | `index` | Emit onBlockComplete with assembled block | -| `message_delta` | `delta.stop_reason`, `usage` | Track stop reason for onMessageComplete | -| `message_stop` | (none) | Emit onMessageComplete with fully assembled Message | - -The parser is a separate concern from the SSE line parser (`sse.zig`). The SSE parser reassembles byte chunks into complete `data: {...}` events. The Anthropic event parser interprets the JSON of each event. They compose: `HTTP read → SSE parser → event strings → Anthropic event parser → callbacks`. - ---- - -## Module Changes - -### New files - -``` -src/provider_anthropic.zig // Anthropic provider implementation -``` - -### Modified files - -- `provider.zig` — ReceiverVTable expanded to the 5-callback lifecycle -- `provider_openai.zig` — No changes needed (block synthesis already built in phase 1) -- `agent.zig` — Any changes needed for expanded Receiver (should be minimal since Receiver is an interface) - -### Config - -``` -AnthropicConfig = struct { - api_key: []const u8, - base_url: []const u8, // e.g. "https://api.anthropic.com" - model: []const u8, // e.g. "claude-sonnet-4-20250514" - api_version: []const u8, // e.g. "2023-06-01" -}; -``` - -Separate from `OpenAIConfig`. The CLI and any higher-level config can unify them; libpanto treats them as distinct. - ---- - -## Testing Strategy - -### Unit tests - -| What | How | -|---|---| -| Anthropic serialization | Create conversations with system/user/assistant messages, serialize to Anthropic JSON, verify structure and system prompt extraction | -| Streaming event parser | Feed canned Anthropic SSE events (message_start, content_block_start, etc.) and verify correct callback sequence and assembled output | -| Block boundary synthesis | Feed OpenAI-style deltas (reasoning → content transitions) and verify onBlockStart/onBlockComplete emitted correctly | -| Receiver contract | Verify that both providers produce the same callback sequence for equivalent conversations | - -### Integration test (manual) - -- Run `panto` binary against Anthropic API with a real API key -- Hold a multi-turn conversation with thinking model -- Verify thinking and text stream correctly -- Switch to OpenAI provider, verify same experience diff --git a/docs/phase-3.md b/docs/phase-3.md deleted file mode 100644 index eb52ee9..0000000 --- a/docs/phase-3.md +++ /dev/null @@ -1,391 +0,0 @@ -# Phase 3: Extension System - -> **Status:** Complete. The Lua runtime described in this doc has been **superseded by [`LUA_MAKEOVER.md`](../LUA_MAKEOVER.md)**. The current `panto` CLI uses a long-lived `lua_State` exposed to libpanto as a single `ToolSource`; the per-call `LuaStatePool` / `LuaTool` design described below no longer exists. The **native extension contract (`Tool`) is unchanged** — the rest of this document remains accurate for that path. - -## Goal - -Introduce a native-code tool extension API on `libpanto` and a Lua extension runtime in the `panto` CLI. Together these transform `pantograph` from a chat client into an agent that can act on the world. The extension system is the primary mechanism for adding capability — tools are extensions, not built-ins. - -This phase also marks the point where the `Agent` abstraction starts earning its keep. Until now, `agent.zig` has been a thin pass-through to the provider. In phase 3 the agent grows into the thing that owns the tool registry and drives the tool-call loop. Providers stay dumb (stream blocks); the agent is what makes a conversation iterative rather than one-shot. - -## Deliverable - -Two layered deliverables: - -1. **libpanto** — a Zig-native tool extension API. `Agent` exposes `registerTool(Tool)` and runs a tool-call loop in `runStep`. The agent loop detects ToolUse blocks in LLM responses, dispatches to registered tools, feeds ToolResult blocks back, and repeats until the model stops calling tools. libpanto has no awareness of Lua, Python, or any specific extension language. - -2. **panto CLI** — a Lua extension runtime that discovers Lua scripts on disk, loads them, and registers each declared tool with `libpanto` via a `LuaTool` adapter that satisfies libpanto's `Tool` interface. - -At the end of this phase, you can: - -- Embed `libpanto` in a Zig program, implement a `Tool` struct natively, and register it with the agent — no Lua involved. -- Write a Lua extension that registers a tool, place it in `.agent/extensions/` (or `~/.config/panto/extensions/`), and have the `panto` CLI discover and load it. -- Have a conversation where the LLM calls your tool and receives the result. -- See tool calls execute in parallel when the LLM returns multiple ToolUse blocks. -- See a meaningful error message when a Lua extension crashes, instead of a process abort. - -## What is usable at the end - -| Capability | How to exercise it | -|---|---| -| Native tool registration | In a Zig embedder, construct a `Tool` and call `agent.registerTool(tool)` | -| Write a Lua tool extension | Create `.agent/extensions/mytool.lua` calling `panto.register_tool(...)` | -| Discover Lua extensions | Place `.lua` files or directories in extension directories; `panto` CLI loads them on startup | -| LLM calls a tool | Ask the LLM to use a registered tool; it emits a ToolUse block; the agent dispatches to the registered handler | -| Tool result fed back | The tool's return bytes become a ToolResult block sent back to the LLM | -| Parallel tool calls | LLM returns multiple ToolUse blocks; they execute concurrently | -| Lua crash handling | A crashing Lua handler prints `the "mytool" extension crashed: ` and aborts the turn | - -## What is explicitly out of scope - -- Core tools as extensions (phase 5 — `std.read`, `std.write`, etc.) -- Conversation serialization / disk persistence (phase 4) -- C ABI distribution of libpanto for external consumers (future) -- GitHub or luarocks extension loaders (future — local filesystem only in phase 3) -- Shared-object extensions (future) -- Extension sandboxing beyond `xpcall` crash protection in the Lua adapter (future) -- Config file for specifying which extensions to load (phase 6 — phase 3 loads everything it discovers) -- Non-Lua extension runtimes (future — the libpanto API is general, but only the Lua adapter ships in phase 3) - ---- - -## libpanto: Native Tool API - -### The `Tool` interface - -`libpanto` defines a `Tool` as an opaque-context value with a vtable. This is the boundary between the agent loop and any extension runtime. - -```zig -pub const Tool = struct { - name: []const u8, // borrowed; lifetime owned by the registrar - schema_json: []const u8, // borrowed JSON Schema bytes; lifetime owned by the registrar - ctx: *anyopaque, - vtable: *const VTable, - - pub const VTable = struct { - /// Invoke the tool. MUST be thread-safe — the agent may call `invoke` - /// concurrently from multiple threads when the LLM returns multiple - /// ToolUse blocks in a single response. - /// - /// `input` is the raw JSON bytes the provider sent. The tool is - /// responsible for parsing them if it cares about their structure. - /// - /// Returns owned bytes allocated with `allocator`. These bytes become - /// the `content` of the ToolResult block. - /// - /// Returning an error aborts the current turn. The agent will surface - /// the error to the user. Native tool implementations are responsible - /// for catching their own panics — a panic in `invoke` will crash the - /// process. Adapters that bridge to safer languages (Lua, Python, Go) - /// should convert panics/exceptions into errors here. - invoke: *const fn (ctx: *anyopaque, input: []const u8, allocator: std.mem.Allocator) anyerror![]u8, - - /// Called when the tool is removed from the registry or the agent is - /// torn down. Frees any resources owned by `ctx`. - deinit: *const fn (ctx: *anyopaque, allocator: std.mem.Allocator) void, - }; -}; -``` - -The contract: - -- **Thread-safety is mandatory.** Tool implementations promise that concurrent `invoke` calls are safe. This pushes the burden onto adapters (where it belongs — the Lua adapter solves it with a `lua_State` pool; a native Zig tool may just be re-entrant) and out of the agent loop. -- **Input is opaque bytes.** libpanto never parses tool input. The tool decides whether to parse JSON, treat it as a string, or ignore it. -- **Output is opaque bytes.** Same principle in reverse. -- **Errors abort the turn.** No partial recovery in phase 3. -- **Panics crash the process.** Native code that wants safety implements it itself. This is trivially obvious from the API surface: it's a function pointer with `anyerror!` return — there's no magic. Adapters for safer languages can wrap panics/exceptions and convert them to errors. - -### `Agent` changes - -```zig -pub const Agent = struct { - provider: provider.Provider, - allocator: std.mem.Allocator, - registry: ToolRegistry, - - pub fn init(allocator: Allocator, prov: provider.Provider) Agent { ... } - pub fn deinit(self: *Agent) void { ... } - - pub fn registerTool(self: *Agent, tool: Tool) !void { ... } - pub fn unregisterTool(self: *Agent, name: []const u8) void { ... } - - pub fn runStep(self: *Agent, conv: *Conversation, receiver: *Receiver) !void { ... } -}; -``` - -`registerTool` takes ownership of the `Tool` value (the agent calls `tool.vtable.deinit` on teardown or unregister). Tool name uniqueness is enforced — registering a name that already exists is an error. - -### `ToolRegistry` - -A small internal module: - -``` -ToolRegistry = struct { - tools: std.StringHashMap(Tool), - allocator: Allocator, - - fn register(name, tool) !void - fn unregister(name) void - fn lookup(name) ?*const Tool - fn iterator() Iterator // for serializing into provider requests - fn deinit() void // invokes each tool's vtable.deinit -}; -``` - -### Agent loop with tools - -`runStep` gains the tool-call loop: - -``` -runStep(conv, receiver): - loop: - provider.streamStep(conv, receiver) - inspect the assistant message just appended to conv for ToolUse blocks - if none: return — turn complete - for each ToolUse block, in parallel: - tool = registry.lookup(block.name) or error - result_bytes = tool.vtable.invoke(tool.ctx, block.input, allocator) - build a ToolResult block { tool_use_id = block.id, content = result_bytes } - append a user Message containing all ToolResult blocks to conv - continue loop -``` - -A single `runStep` may invoke the provider multiple times if the LLM chains tool calls. - -### Parallel execution - -When a response contains multiple ToolUse blocks, the agent dispatches them concurrently. Implementation likely uses a small thread pool or `std.Thread.spawn` per call (TBD during implementation). Because `Tool.invoke` is declared thread-safe, the agent loop has no further constraints — it just fans out and joins. - -### Tool serialization in provider requests - -When the agent calls the provider, the provider receives the registry (or an iterator over it) and emits the tools array. This means provider request serialization gains a tools-emitting branch when the registry is non-empty. - -**OpenAI**: -```json -{ - "tools": [ - { "type": "function", - "function": { "name": "echo", "parameters": } } - ] -} -``` - -**Anthropic**: -```json -{ - "tools": [ - { "name": "echo", "input_schema": } - ] -} -``` - -The `schema_json` bytes are emitted verbatim. Description fields are deferred — `Tool` does not yet carry a description (see open questions). - -### ToolUse decoding in responses - -Already handled by the existing Receiver callback sequence from phase 1/2: - -- OpenAI: `delta.tool_calls` → `onBlockStart(.ToolUse, ...)` with `meta.tool_id` and `meta.tool_name`, then `onContentDelta` with JSON argument fragments. -- Anthropic: `content_block_start` with `type: "tool_use"` → `onBlockStart(.ToolUse, ...)`, then `content_block_delta` with `input_json_delta` fragments. - -The assembled ToolUseBlock contains `id`, `name`, and `input` (TextualBlock with the full JSON string). - -### ToolResult encoding in requests - -**OpenAI** — each ToolResult block becomes a separate top-level `tool` message: -```json -{ "role": "tool", "tool_call_id": "", "content": "" } -``` - -**Anthropic** — ToolResult blocks are content blocks on a user message: -```json -{ "role": "user", "content": [ - { "type": "tool_result", "tool_use_id": "", "content": "" } -] } -``` - ---- - -## panto CLI: Lua Extension Runtime - -The Lua runtime lives entirely in the CLI. libpanto has no Lua dependency. - -### Extension discovery - -The CLI scans two directories in order: - -1. `.panto/extensions/` — project-local, relative to current working directory -2. `~/.config/panto/extensions/` — user-level - -`.panto/` follows the established per-agent pattern (`.claude/`, `.opencode/`, `.pi/`). - -### Naming and structure - -- **Single-file**: `.lua` → extension name is `` -- **Directory**: `/init.lua` → extension name is `` - -Hierarchical names via dot/directory mapping, mirroring `require("a.b.c")`: - -- `utils/json.lua` → `utils.json` -- `coding/edit/init.lua` → `coding.edit` - -Sub-modules under a directory extension (e.g., `coding/edit/helpers.lua`) are the extension's internal business — the CLI only loads the top-level entry point. - -### Loading behavior - -- Recursively scan both directories. -- Construct extension names from relative paths. -- For each unique extension name, load its entry point into the `lua_State` pool (see below). -- Project-local entries shadow user-level entries of the same name. -- Extension top-level code runs at load time; it is expected to call `panto.register_tool(...)`. - -### The Lua bridge - -A small Zig module (`lua_bridge.zig`) registers Zig functions into a `lua_State`. The bridge exposes: - -#### `panto.register_tool(name, schema, handler)` - -- `name` (string) — tool name, e.g. `"bash"` -- `schema` (table) — JSON Schema as a Lua table. The bridge serializes this to JSON bytes at registration time. -- `handler` (function) — invoked when the LLM calls this tool. Receives a single argument: a Lua table parsed from the input JSON. Must return a string. - -```lua -panto.register_tool("echo", { - type = "object", - properties = { - message = { type = "string", description = "The message to echo back" } - }, - required = { "message" } -}, function(input) - return input.message -end) -``` - -The bridge stores the handler in the Lua registry (`luaL_ref`) and constructs a `LuaTool` (see below) that it then registers with the agent. - -### `LuaTool` — the adapter to libpanto - -```zig -const LuaTool = struct { - pool: *LuaStatePool, - handler_ref: i32, // registry ref to the Lua function - name_owned: []u8, - schema_owned: []u8, - - // Implements Tool.VTable.invoke: - // 1. pool.acquire() → lua_State - // 2. lua_rawgeti(L, LUA_REGISTRYINDEX, handler_ref) — push handler - // 3. parse input bytes into a Lua table via std.json - // 4. xpcall the handler with the table - // 5. read returned string (or capture trace on error) - // 6. pool.release(L) - // 7. return result bytes or error -}; -``` - -The adapter is what makes the CLI side responsible for Lua-specific concerns: - -- **JSON → Lua table conversion**: happens here, not in libpanto. -- **`xpcall` crash protection**: wraps the handler call, captures `debug.traceback`, returns an error so libpanto aborts the turn cleanly with `the "" extension crashed: `. -- **Thread-safety**: provided by the `lua_State` pool. Each concurrent `invoke` checks out a separate state. libpanto's contract is satisfied. - -### `LuaStatePool` - -On-demand pool of `lua_State` instances: - -``` -LuaStatePool = struct { - states: std.ArrayList(*lua_State), - available: std.BitSet, - allocator: Allocator, - extension_paths: []const []const u8, // all discovered entry points - - fn acquire() *lua_State // returns a free state, or creates a new one - fn release(L: *lua_State) void // returns L to the pool - fn deinit() void // closes all states -}; -``` - -- States are created lazily — no pre-allocation. -- Each new state loads all discovered extensions identically, so any state can handle any tool. -- A state is checked out for the duration of a single `invoke` call. - -There's a coupling here: the handler `luaL_ref` is per-`lua_State`. The pool must guarantee that the same registry slot in every state points at the same handler. The cleanest way is: when an extension is first loaded into the prototype state, the bridge records the registry index. When a new state is constructed, the bridge re-loads all extensions in the same order, producing the same ref indices. This needs care during implementation but is mechanically straightforward. - ---- - -## Module Changes - -### libpanto - -New: -- `libpanto/src/tool.zig` — `Tool` and `VTable` definitions -- `libpanto/src/tool_registry.zig` — `ToolRegistry` - -Modified: -- `libpanto/src/agent.zig` — owns registry, gains `registerTool`, drives tool-call loop in `runStep` -- `libpanto/src/provider.zig` — provider receives a way to see registered tools (likely a `*const ToolRegistry` on the agent passed via `streamStep` context, or a callback) -- `libpanto/src/provider_openai_chat.zig` — request serialization emits `tools` array -- `libpanto/src/provider_anthropic_messages.zig` — request serialization emits `tools` array -- `libpanto/src/openai_chat_json.zig` — ToolResult message encoding -- `libpanto/src/anthropic_messages_json.zig` — ToolResult content block encoding -- `libpanto/src/root.zig` — re-exports `Tool`, `ToolRegistry` - -### panto CLI - -New: -- `src/lua_bridge.zig` — Zig functions registered into Lua states; `panto.register_tool` implementation -- `src/lua_tool.zig` — `LuaTool` adapter implementing libpanto's `Tool` interface -- `src/lua_state_pool.zig` — on-demand `lua_State` pool -- `src/extension_loader.zig` — directory scanning, name construction, loading - -Modified: -- `src/main.zig` — wires extension discovery + loading into agent startup - -### External dependency - -Lua 5.4 linked into the `panto` binary. Zig's build system can fetch and compile Lua from source (~30KLOC C). No system dependency required. libpanto does not link Lua. - ---- - -## Testing Strategy - -### Unit tests (libpanto) - -| What | How | -|---|---| -| Tool registration | Construct a native `Tool`, register with `Agent`, verify lookup | -| Duplicate registration | Verify registering the same name twice returns an error | -| Tool dispatch | Hand-craft a conversation with a ToolUse block, run `runStep` against a stub provider, verify the registered tool's `invoke` is called with the right bytes | -| ToolResult round-trip | Verify the ToolResult message appended to the conversation has the right `tool_use_id` and `content` | -| Parallel dispatch | Register two tools with sleep+timestamps, verify they ran concurrently (elapsed wall time < sum of individual times) | -| Provider request serialization | Snapshot tests on OpenAI and Anthropic request bodies with tools registered | - -### Unit tests (panto CLI) - -| What | How | -|---|---| -| Extension discovery | Create temp directory with single-file and directory extensions, verify names constructed correctly | -| Project shadows user | Same extension name in both directories — project wins | -| Lua bridge input | Feed JSON bytes through `LuaTool.invoke`, verify the Lua handler sees the expected table | -| Lua bridge output | Handler returns a string, verify it round-trips to bytes | -| `xpcall` crash protection | Extension whose handler throws — verify `invoke` returns an error and includes the trace | -| Concurrent `invoke` | Two threads call `invoke` on the same `LuaTool`, verify they don't share a `lua_State` | - -### Integration tests (manual) - -- `echo.lua` extension: ask the LLM to use it, verify result is fed back, LLM continues. -- `crash.lua` extension: verify the crash is caught, printed, turn aborts gracefully. -- Multi-tool response: ask the LLM to call two tools in one message, verify both execute concurrently. - ---- - -## Resolved Decisions - -- **Project-local extension directory**: `.panto/extensions/`, matching `.claude/`, `.opencode/`, `.pi/`. -- **Lua version**: Lua 5.4. LuaJIT is stuck at 5.1 semantics and the JIT speedup is largely irrelevant for tool handlers (which spend most of their time in Zig-side I/O). 5.4 keeps us aligned with current luarocks and Lua documentation. -- **Tool description**: a top-level `description: []const u8` field on `Tool`. The Lua bridge gains it as a parameter: `panto.register_tool(name, description, schema, handler)`. Schema describes input only; description is the human-readable purpose. -- **Provider sees the registry**: `provider.streamStep` gains a `*const ToolRegistry` parameter. Embedders normally pass the same registry on every call. -- **Parallel dispatch**: `std.Thread.spawn` per ToolUse block. Thread creation is sub-millisecond on Linux/macOS; tool calls are orders of magnitude longer. Recycling buys nothing meaningful. -- **Streaming tool results**: deferred, possibly indefinitely. Provider APIs only accept a final result payload, so streaming would be display-only — not worth the vtable complication in phase 3. -- **Handler timeout / cancellation**: deferred. POSIX has no clean way to cancel a thread (`pthread_kill` with SIGKILL terminates the whole process; `pthread_cancel` is unusable in practice; signal-based interruption is not async-signal-safe for arbitrary code). Lua's `lua_sethook` only fires between VM steps, so it can't interrupt a handler blocked in a C call. A real cancellation primitive requires process isolation, which is a phase of its own — see the Punted list in [overview.md](overview.md). Phase 3 ships with the honest contract: **tool handlers run to completion. If a tool hangs, the user kills the `panto` process.** diff --git a/ideas.md b/ideas.md deleted file mode 100644 index e83d543..0000000 --- a/ideas.md +++ /dev/null @@ -1,38 +0,0 @@ -# pantograph Ideas - -`pantograph` is a minimal coding agent inspired by `pi`, with a focus on performance, efficiency, correctness, and a small core that can be extended deliberately. - -## Core architecture - -- The core agent loop will be written in Zig for performance, efficiency, and a small runtime footprint. -- In addition to the CLI/application, the project should expose a `libpanto` library through a C ABI so other programs can embed or build on the agent functionality. - -## Extension system - -- Extensions will initially be written in Lua, chosen for speed, simplicity, and low overhead. -- Because poorly written extensions can easily destabilize the host, `pantograph` should explore ways to improve extension correctness and crash protection. -- Future extension support should include loading shared object libraries through a C ABI, allowing extensions to be written in Zig, Rust, C, C++, or other native languages. - -## Minimal built-in feature set - -- `pantograph` should follow `pi`'s philosophy of avoiding unnecessary built-ins such as native subagents, MCP support, and permission systems. -- `pantograph` will go further by not building in AGENTS.md automation, skills, or customizable `/prompts`. -- Those features should be possible to implement as extensions rather than being part of the core runtime. - -## Provider API support - -- Provider support should be careful and conservative. -- The core should support Anthropic-shaped and OpenAI-shaped APIs with arbitrary base URLs. -- The goal is to avoid the common failure mode where provider integrations exist but only partially work, break important agent features, or crash the process. - -## Server/proxy mode - -- `pantograph` should support running as a server that exposes OpenAI-compatible and Anthropic-compatible APIs. -- In this mode, `pantograph` can act as a lightweight provider router/proxy to its configured backends. -- This is intended to provide the useful parts of tools like `omniroute` while avoiding excessive memory usage, fragile integrations, and runtime instability. - -## Core tools as extensions - -- Basic tools such as `read`, `write`, `edit`, and `bash` should be supplied by extensions rather than hardcoded into the core. -- These tools can be included in the standard distribution but should be individually disableable. -- Once shared object extension support exists, the standard core tools should be ported to Zig/native extensions. diff --git a/src/lua_runtime.zig b/src/lua_runtime.zig index fb00951..71e4ae7 100644 --- a/src/lua_runtime.zig +++ b/src/lua_runtime.zig @@ -58,6 +58,23 @@ pub const LuaRuntime = struct { /// the Lua registry (`luaL_ref` index). handlers: std.StringHashMap(c_int), + /// Registry ref to the wrapper closure that runs a user handler + /// inside a `pcall` and reports the result back to Zig via + /// `panto._record_result`. Allocated once at `create`; reused for + /// every call. `0` until `installScheduler` runs. + wrapper_ref: c_int = 0, + /// Registry ref to `require("luv").run`, the function we call to + /// tick libuv between coroutine resumes. `0` until + /// `installScheduler` runs. + uv_run_ref: c_int = 0, + + /// Pointer to the in-flight batch, valid only for the duration of + /// one `invoke_batch` call. The `panto._record_result` C function + /// writes through this. `null` between batches; not concurrently + /// accessible (libpanto's source-grouped dispatch guarantees one + /// thread per source per turn). + current_batch: ?*BatchState = null, + /// Create a new runtime. The `lua_State` is opened, standard libs /// loaded, and the `panto.register_tool` bridge installed. pub fn create(allocator: Allocator) !*LuaRuntime { @@ -79,6 +96,22 @@ pub const LuaRuntime = struct { return self; } + /// Install the libuv-driven coroutine scheduler: + /// - Register `panto._record_result` (C function with `self` as + /// light-userdata upvalue) so the wrapper closure can hand + /// results back to Zig. + /// - Create the wrapper closure that runs a user handler in + /// `pcall` and reports the result. + /// - Cache `require("luv").run` for fast per-tick access. + /// + /// Must be called after luarocks bootstrap has installed luv, + /// otherwise the `require("luv")` step will fail. + pub fn installScheduler(self: *LuaRuntime) !void { + try installRecordResult(self); + try installWrapperClosure(self); + try cacheUvRun(self); + } + /// Tear down the runtime: free every owned string, unref every /// handler, close the Lua state. pub fn deinit(self: *LuaRuntime) void { @@ -90,6 +123,12 @@ pub const LuaRuntime = struct { c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, entry.value_ptr.*); } self.handlers.deinit(); + if (self.wrapper_ref != 0) { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, self.wrapper_ref); + } + if (self.uv_run_ref != 0) { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, self.uv_run_ref); + } c.lua_close(self.L); @@ -320,6 +359,52 @@ fn deinitSrc(_: *anyopaque, _: Allocator) void { // libpanto's source.deinit here is a no-op. } +// =========================================================================== +// Scheduler: libuv-driven cooperative coroutine dispatch +// =========================================================================== +// +// libpanto's `invoke_batch` delivers all of a turn's tool-call requests +// at once, on a single thread. We answer the contract by running each +// call as a Lua coroutine inside our long-lived `lua_State`, then +// driving `uv.run("once")` to wake any of those coroutines that are +// blocked on libuv-aware I/O. This is the entire scheduler — luv's +// libuv binding does the actual event-loop work; we just resume +// coroutines and call `run` between resumes. +// +// Capturing return values requires a wrapper. When a coroutine is +// resumed by a luv callback after yielding, the eventual return value +// of the coroutine flows back to *that callback*, not to us. So we +// install a Lua wrapper closure that does +// +// pcall(handler, input) → panto._record_result(idx, ok, val) +// +// before the handler returns. `_record_result` is a C function that +// stores into a per-runtime `BatchState`, accessed via a light-userdata +// upvalue carrying the runtime pointer. + +/// One coroutine's outcome, recorded by `_record_result` and read by +/// `invokeBatch` once the coroutine has terminated. +const Slot = struct { + /// Set true the moment `_record_result` writes a result for this + /// index. Used to detect coroutines that terminated without + /// calling the wrapper (a bug / API misuse). + recorded: bool = false, + /// `true` if the handler returned cleanly, `false` if it raised + /// via the `pcall` wrapping. + ok: bool = false, + /// Result payload as owned bytes. Allocated from `allocator`. + /// Caller frees. + value: ?[]u8 = null, + /// On `ok = false`, an owned copy of the error message. + err_msg: ?[]u8 = null, +}; + +/// State shared between Zig and the in-flight Lua wrapper closure. +const BatchState = struct { + allocator: Allocator, + slots: []Slot, +}; + fn invokeBatch( ctx: *anyopaque, calls: []const panto.ToolCall, @@ -327,21 +412,205 @@ fn invokeBatch( allocator: Allocator, ) anyerror!void { const self: *LuaRuntime = @ptrCast(@alignCast(ctx)); + return runBatch(self, calls, results, allocator); +} - // Step 2 of LUA_MAKEOVER.md: no batteries yet — each call is run - // as a coroutine, but the scheduler doesn't drive an event loop. - // A handler that yields with no batteries available has nothing - // to wake it; we surface that as `LuaHandlerYielded`. - // - // Once `luv` and the `coro-*` wrappers are installed, this loop - // becomes "drive uv.run() until every coroutine is dead/erroring, - // then collect results". +fn runBatch( + self: *LuaRuntime, + calls: []const panto.ToolCall, + results: []panto.ToolCallResult, + allocator: Allocator, +) !void { + if (self.wrapper_ref == 0 or self.uv_run_ref == 0) { + // Scheduler not installed. We can still run synchronous + // handlers — use the legacy path that drives one coroutine at + // a time without an event loop. + for (calls, 0..) |call, i| { + results[i] = runLegacySync(self, call, allocator); + } + return; + } + + var slots = try allocator.alloc(Slot, calls.len); + defer allocator.free(slots); + for (slots) |*s| s.* = .{}; + + var batch_state: BatchState = .{ .allocator = allocator, .slots = slots }; + self.current_batch = &batch_state; + defer self.current_batch = null; + + // Track each call's coroutine reference in the parent stack (we + // hold them in registry refs so they survive across `uv.run` + // ticks). `0` after a coroutine has been reaped. + var thread_refs = try allocator.alloc(c_int, calls.len); + defer allocator.free(thread_refs); + @memset(thread_refs, 0); + defer for (thread_refs) |r| { + if (r != 0) c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, r); + }; + + var pending: usize = 0; for (calls, 0..) |call, i| { - results[i] = runOneCall(self, call, allocator); + const handler_ref = self.handlers.get(call.tool_name) orelse { + // Synthesize a recorded "err" result; don't even bother + // spawning a coroutine. + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe(u8, "panto: unknown tool name"), + }; + continue; + }; + const t = try startCoroutine(self, i, handler_ref, call.input, allocator); + thread_refs[i] = t.thread_ref; + if (t.still_pending) pending += 1; + } + + while (pending > 0) { + const active = try driveUvOnce(self); + // Reap any coroutines that terminated during the tick. + var reaped: usize = 0; + for (thread_refs, 0..) |tref, i| { + if (tref == 0) continue; + // Push the thread, check status, pop. + _ = c.lua_rawgeti(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + const co: *c.lua_State = @ptrCast(c.lua_tothread(self.L, -1).?); + const status = c.lua_status(co); + c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (status != c.LUA_YIELD) { + // Terminated (LUA_OK or error). The wrapper should + // have called `_record_result` already; if not, synthesize. + if (!slots[i].recorded) { + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe( + u8, + "panto: handler terminated without recording a result", + ), + }; + } + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + thread_refs[i] = 0; + reaped += 1; + } + } + if (reaped > 0) { + if (reaped > pending) { + // Defensive: keep the counter sane. + pending = 0; + } else { + pending -= reaped; + } + continue; + } + if (active == 0) { + // No libuv handles are pending, but we still have alive + // coroutines. They yielded without arranging to be woken. + // Mark them as failed and break. + for (thread_refs, 0..) |tref, i| { + if (tref == 0) continue; + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe( + u8, + "panto: handler yielded but no libuv handle is pending; " ++ + "did you forget to await with luv?", + ), + }; + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + thread_refs[i] = 0; + } + break; + } + } + + // Translate slots into the libpanto-shaped results. + for (slots, 0..) |slot, i| { + if (!slot.recorded) { + results[i] = .{ .err = RuntimeError.BadHandlerReturn }; + continue; + } + if (slot.ok) { + results[i] = .{ .ok = slot.value orelse try allocator.dupe(u8, "") }; + // Free the err_msg if both ended up set somehow. + if (slot.err_msg) |m| allocator.free(m); + } else { + if (slot.value) |v| allocator.free(v); + // The error message was logged for the user; we still + // surface a typed error to libpanto so it can route the + // failure to the agent's error path. + std.log.warn( + "panto-lua: tool '{s}' failed: {s}", + .{ + calls[i].tool_name, + slot.err_msg orelse "(no message)", + }, + ); + if (slot.err_msg) |m| allocator.free(m); + results[i] = .{ .err = RuntimeError.LuaHandlerCrashed }; + } } } -fn runOneCall( +/// Start one coroutine: create a thread under the runtime's lua_State, +/// push the wrapper closure + (idx, handler, input), `lua_resume` once. +/// +/// If the coroutine returns immediately (sync handler), the wrapper +/// has already recorded its result via `panto._record_result` — +/// `still_pending` will be `false`. +fn startCoroutine( + self: *LuaRuntime, + idx: usize, + handler_ref: c_int, + input: []const u8, + allocator: Allocator, +) !struct { thread_ref: c_int, still_pending: bool } { + const L = self.L; + + const co = c.lua_newthread(L) orelse return RuntimeError.LuaInitFailed; + // luaL_ref pops the topmost value (the thread) and returns a + // registry ref to it. We keep the ref alive for the lifetime of + // the call so GC doesn't collect the thread mid-yield. + const thread_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); + + // Push the wrapper onto the coroutine's stack. + _ = c.lua_rawgeti(co, lua_bridge.LUA_REGISTRYINDEX, @intCast(self.wrapper_ref)); + // Push (idx, handler, input) as the resume args. + c.lua_pushinteger(co, @intCast(idx)); + _ = c.lua_rawgeti(co, lua_bridge.LUA_REGISTRYINDEX, @intCast(handler_ref)); + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + try lua_bridge.pushJsonAsLua(co, arena_state.allocator(), input); + + var nres: c_int = 0; + const status = c.lua_resume(co, L, 3, &nres); + return .{ + .thread_ref = thread_ref, + .still_pending = status == c.LUA_YIELD, + }; +} + +/// Call `uv.run("once")`. Returns the number of active handles luv +/// reports remain pending (0 means the loop is drained). +fn driveUvOnce(self: *LuaRuntime) !c_int { + const L = self.L; + _ = c.lua_rawgeti(L, lua_bridge.LUA_REGISTRYINDEX, @intCast(self.uv_run_ref)); + _ = c.lua_pushlstring(L, "once", 4); + if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: uv.run failed"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.UvRunFailed; + } + const n = c.lua_tointegerx(L, -1, null); + c.lua_settop(L, c.lua_gettop(L) - 1); + return @intCast(n); +} + +/// Pre-scheduler fallback (used in unit tests and during early +/// startup before `installScheduler` has run). +fn runLegacySync( self: *LuaRuntime, call: panto.ToolCall, allocator: Allocator, @@ -349,62 +618,40 @@ fn runOneCall( const handler_ref = self.handlers.get(call.tool_name) orelse { return .{ .err = RuntimeError.LuaHandlerNotFound }; }; - - const out_bytes = invokeCoroutine(self.L, handler_ref, call.input, allocator) catch |e| { + const out_bytes = invokeCoroutineSync(self.L, handler_ref, call.input, allocator) catch |e| { return .{ .err = e }; }; return .{ .ok = out_bytes }; } -/// Create a fresh coroutine, push the handler + JSON-decoded input as -/// the resume args, then resume. Returns the handler's return value as -/// owned JSON bytes (the slot `ok` in CallResult). -/// -/// Resume outcomes: -/// - LUA_OK: coroutine returned. Read its top value as the result. -/// - LUA_YIELD: coroutine yielded. With no event loop installed, we -/// treat this as an error so the user sees that their handler is -/// trying to do async I/O that isn't yet supported. -/// - other (errors): error message is on the coroutine's stack; -/// copy it to a log line and return LuaHandlerCrashed. -fn invokeCoroutine( +fn invokeCoroutineSync( L: *c.lua_State, handler_ref: c_int, input: []const u8, allocator: Allocator, ) ![]u8 { - // Create the coroutine thread. After this, `co` is the child - // thread; the parent stack also gains a thread value at the top. const co = c.lua_newthread(L) orelse return RuntimeError.LuaInitFailed; - defer c.lua_settop(L, c.lua_gettop(L) - 1); // pop the thread when done + defer c.lua_settop(L, c.lua_gettop(L) - 1); - // Push handler from the registry onto the coroutine's stack. _ = c.lua_rawgeti(co, lua_bridge.LUA_REGISTRYINDEX, @intCast(handler_ref)); if (c.lua_type(co, -1) != lua_bridge.T_FUNCTION) { return RuntimeError.LuaHandlerNotFound; } - // Push the parsed JSON input as the resume arg. var arena_state = std.heap.ArenaAllocator.init(allocator); defer arena_state.deinit(); try lua_bridge.pushJsonAsLua(co, arena_state.allocator(), input); - // Resume with 1 arg. var nresults: c_int = 0; const status = c.lua_resume(co, L, 1, &nresults); switch (status) { c.LUA_OK => { - // Coroutine returned. We expect exactly one string return - // value (the tool result). If there are zero or extra - // values we still try to read top-of-stack. if (nresults < 1) return RuntimeError.BadHandlerReturn; return try lua_bridge.readHandlerResult(co, -1, allocator); }, c.LUA_YIELD => { - // Nothing to wake this coroutine without an event loop. - // Surface the situation so the user knows what's wrong. - const msg = "lua: tool handler yielded with no event loop installed (step 4+ of LUA_MAKEOVER.md not yet implemented)"; + const msg = "lua: tool handler yielded with no event loop installed; call installScheduler() before dispatching"; if (@import("builtin").is_test) { std.log.warn("{s}", .{msg}); } else { @@ -419,6 +666,124 @@ fn invokeCoroutine( } } +// --------------------------------------------------------------------------- +// Scheduler setup (called once at startup, after luarocks bootstrap) +// --------------------------------------------------------------------------- + +/// Register `panto._record_result(idx, ok, value)` on the `panto` +/// global. The C function carries the runtime pointer as an upvalue, +/// reaches the in-flight `BatchState` through `current_batch`, and +/// stores into the matching slot. +fn installRecordResult(self: *LuaRuntime) !void { + const L = self.L; + _ = c.lua_getglobal(L, "panto"); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + c.lua_pushlightuserdata(L, @ptrCast(self)); + c.lua_pushcclosure(L, recordResultC, 1); + c.lua_setfield(L, -2, "_record_result"); + c.lua_settop(L, c.lua_gettop(L) - 1); // pop `panto` +} + +fn recordResultC(L: ?*c.lua_State) callconv(.c) c_int { + const Lst = L.?; + const self_ptr = c.lua_touserdata(Lst, c.lua_upvalueindex(1)); + if (self_ptr == null) return 0; + const self: *LuaRuntime = @ptrCast(@alignCast(self_ptr.?)); + const batch = self.current_batch orelse return 0; + + const idx_i64 = c.lua_tointegerx(Lst, 1, null); + const ok = c.lua_toboolean(Lst, 2) != 0; + const idx: usize = @intCast(idx_i64); + if (idx >= batch.slots.len) return 0; + + if (ok) { + // Result value at index 3. The handler's return type is + // free-form; we serialize via the existing `readHandlerResult` + // helper which already knows how to JSON-encode any Lua value. + const value = lua_bridge.readHandlerResult(Lst, 3, batch.allocator) catch |e| { + // Allocation failure mid-callback is unrecoverable from + // Lua's POV; record a synthetic error and bail. + const msg = std.fmt.allocPrint( + batch.allocator, + "panto: failed to serialize handler result: {s}", + .{@errorName(e)}, + ) catch null; + batch.slots[idx] = .{ .recorded = true, .ok = false, .err_msg = msg }; + return 0; + }; + batch.slots[idx] = .{ .recorded = true, .ok = true, .value = value }; + } else { + // Error message at index 3. May be any Lua value; coerce to + // string via `tostring`-equivalent semantics. + var len: usize = 0; + const ptr = c.luaL_tolstring(Lst, 3, &len); + const owned = if (ptr != null) + batch.allocator.dupe(u8, ptr[0..len]) catch null + else + null; + c.lua_settop(Lst, c.lua_gettop(Lst) - 1); // pop tolstring's pushed string + batch.slots[idx] = .{ .recorded = true, .ok = false, .err_msg = owned }; + } + return 0; +} + +/// Create the per-call wrapper closure: +/// +/// local function wrapper(idx, handler, input) +/// local ok, val = pcall(handler, input) +/// panto._record_result(idx, ok, val) +/// end +/// +/// Stored in the Lua registry under `self.wrapper_ref`. +fn installWrapperClosure(self: *LuaRuntime) !void { + const L = self.L; + const snippet = + \\return function(idx, handler, input) + \\ local ok, val = pcall(handler, input) + \\ panto._record_result(idx, ok, val) + \\end + ; + if (c.luaL_loadstring(L, snippet) != 0) { + logTopAsError(L, "panto-lua: wrapper closure failed to compile"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: wrapper closure failed to evaluate"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + // Top of stack: the wrapper function. luaL_ref pops it. + self.wrapper_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); +} + +/// Cache `require("luv").run` in the registry so the scheduler can +/// invoke it cheaply per tick. +fn cacheUvRun(self: *LuaRuntime) !void { + const L = self.L; + const snippet = + \\return require("luv").run + ; + if (c.luaL_loadstring(L, snippet) != 0) { + logTopAsError(L, "panto-lua: failed to compile luv lookup"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: require('luv') failed (was the bootstrap successful?)"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_type(L, -1) != lua_bridge.T_FUNCTION) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + self.uv_run_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); +} + // --------------------------------------------------------------------------- // Internals // --------------------------------------------------------------------------- @@ -715,6 +1080,91 @@ test "yielding handler with no event loop surfaces LuaHandlerYielded" { try testing.expectEqual(@as(anyerror, RuntimeError.LuaHandlerYielded), results[0].err); } +// Integration test: requires a `$PANTO_HOME` with luv already +// installed. Skipped if luv isn't on disk — unit tests stay offline. +test "scheduler: yielding handler is resumed by libuv" { + const home_z = std.c.getenv("PANTO_HOME") orelse return error.SkipZigTest; + const panto_home_env = std.mem.sliceTo(home_z, 0); + // Check for `/rocks/lua-/lib/lua/5.4/luv.so`. + const manifest = @import("manifest.zig"); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const so_path = try std.fmt.bufPrint( + &path_buf, + "{s}/rocks/lua-{s}/lib/lua/{s}/luv.so", + .{ panto_home_env, manifest.lua_version, manifest.lua_short_version }, + ); + std.Io.Dir.cwd().access(testing.io, so_path, .{}) catch return error.SkipZigTest; + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + + const source = + \\local uv = require("luv") + \\panto.register_tool { + \\ name = "timer_say", description = "sleep then return", + \\ schema = { type = "object" }, + \\ handler = function(input) + \\ local co = coroutine.running() + \\ local timer = uv.new_timer() + \\ uv.timer_start(timer, 5, 0, function() + \\ uv.timer_stop(timer) + \\ uv.close(timer) + \\ coroutine.resume(co) + \\ end) + \\ coroutine.yield() + \\ return "awake" + \\ end, + \\} + ; + const path = try writeTempScript(tmp.dir, "timer.lua", source); + defer testing.allocator.free(path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + // Bootstrap luarocks (so `require("luv")` works), then install + // the scheduler. We use the real environment so the test picks + // up the same PANTO_HOME the developer's machine has. + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("PANTO_HOME", panto_home_env); + + const luarocks_runtime = @import("luarocks_runtime.zig"); + // The bootstrap needs a panto executable path for the wrapper + // script; tests don't actually invoke it, so a placeholder is + // fine (the wrapper is only consulted when luarocks itself + // shells out, which the test never triggers). + const luarocks_rt = try luarocks_runtime.bootstrap( + testing.allocator, + testing.io, + &env, + rt.L, + "/usr/bin/true", + ); + defer luarocks_rt.deinit(); + + try rt.installScheduler(); + try rt.loadExtension(path, null); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{ + .{ .tool_name = "timer_say", .input = "{}" }, + .{ .tool_name = "timer_say", .input = "{}" }, + }; + var results: [2]panto.ToolCallResult = .{ + .{ .err = error.SourceDroppedCall }, + .{ .err = error.SourceDroppedCall }, + }; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer for (results) |r| switch (r) { + .ok => |b| testing.allocator.free(b), + .err => {}, + }; + + try testing.expectEqualStrings("awake", results[0].ok); + try testing.expectEqualStrings("awake", results[1].ok); +} + test "loadExtension: duplicate tool name from a second extension errors" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); diff --git a/src/luarocks_runtime.zig b/src/luarocks_runtime.zig new file mode 100644 index 0000000..b67a8ad --- /dev/null +++ b/src/luarocks_runtime.zig @@ -0,0 +1,763 @@ +//! Embedded-luarocks runtime bootstrap. +//! +//! Responsibilities at startup (per LUA_MAKEOVER.md steps 3-5 and Q1-Q5): +//! +//! 1. Resolve `$PANTO_HOME` and the per-Lua-version rocks tree +//! (`panto_home.zig`). Create the directory layout if missing. +//! 2. Stage Lua headers under `/include/` (from `@embedFile`) +//! so luarocks can compile C rocks against them. Idempotent: a +//! file is only rewritten if its checksum differs. +//! 3. Materialize `/etc/luarocks/config-.lua` with the +//! pinned interpreter, rock trees, and toolchain variables. +//! 4. Install a `package.searcher` that serves `require("luarocks.*")` +//! and `require("compat53.*")` from embedded sources \u2014 the +//! luarocks Lua libraries never touch disk. +//! 5. Inject `luarocks.core.hardcoded` into `package.loaded` with the +//! runtime-resolved `SYSCONFDIR`. Without this, `luarocks.core.cfg` +//! can't find its config file. +//! 6. Configure `package.path` / `package.cpath` so user rocks +//! installed in `/share/lua/` and +//! `/lib/lua/` are visible to `require`. +//! 7. Reconcile the batteries manifest: for each pinned rock, check +//! `/lib/luarocks/rocks-///` and invoke +//! `luarocks install` for anything missing. (Slow path; only the +//! first run after a fresh `$PANTO_HOME` actually downloads.) +//! +//! Step 7 needs a usable `lua` executable on PATH from luarocks's point +//! of view \u2014 it shells out for rockspec build scripts. We satisfy +//! this via the `panto lua` subcommand, addressed by writing +//! ` lua` (with the absolute path of the running `panto` +//! binary) into the luarocks config as `variables.LUA`. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; + +const manifest = @import("manifest.zig"); +const panto_home = @import("panto_home.zig"); +const embedded_luarocks = @import("embedded_luarocks"); +const embedded_lua_headers = @import("embedded_lua_headers"); +const lua_bridge = @import("lua_bridge.zig"); + +const c = lua_bridge.c; + +/// Owned state for the runtime side of luarocks. Holds onto the +/// resolved layout, the `lua_State` we attached to, and a hash map +/// used by the embedded-module searcher. +pub const LuarocksRuntime = struct { + allocator: Allocator, + layout: panto_home.Layout, + L: *c.lua_State, + /// Module-name to source-bytes lookup for the embedded-source + /// `package.searcher` callback. Keys borrow from + /// `embedded_luarocks.files`; values likewise. + modules: std.StringHashMapUnmanaged([]const u8), + + pub fn deinit(self: *LuarocksRuntime) void { + self.modules.deinit(self.allocator); + self.layout.deinit(); + self.allocator.destroy(self); + } +}; + +/// Errors surfaced by the bootstrap pipeline. The `Luarocks*` variants +/// indicate that we were able to invoke luarocks but it exited non-zero +/// (or otherwise failed); the message is in stderr at that point. +pub const BootstrapError = error{ + HeadersMissing, + LuarocksInjectionFailed, + LuarocksInstallFailed, + LuarocksSearcherInstallFailed, + PathConfigFailed, + PantoExecutablePathUnknown, +} || Allocator.Error; + +/// Full startup-time bootstrap. Walks the entire setup pipeline against +/// the given `lua_State`, leaving it ready for callers to `require` +/// luarocks modules and for any user code to find rocks under the +/// configured tree. +/// +/// `panto_executable_path` is the absolute path of the currently +/// running `panto` binary. Used to construct the ` lua` +/// invocation that luarocks will use as its interpreter when running +/// rockspec build scripts. +/// +/// Wipe the current Lua-version's rocks tree before bootstrapping. +/// Used by `panto bootstrap --force` to recover from a corrupted or +/// outdated installation. Only the `/rocks/lua-/` +/// subtree is removed; sibling trees from other Lua versions stay +/// untouched, matching the rationale in Q3 (each Lua version owns +/// its own tree for clean rollback). +pub fn wipeTree( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, +) !void { + var layout = try panto_home.resolve(allocator, environ_map); + defer layout.deinit(); + + // `deleteTree` is the cwd-relative entrypoint; we want absolute. + // Open the parent and delete by basename to avoid path-traversal + // surprises if `layout.tree` ever contains a symlink. + const parent = std.fs.path.dirname(layout.tree) orelse { + std.log.warn("panto bootstrap --force: tree has no parent? '{s}'", .{layout.tree}); + return; + }; + const base = std.fs.path.basename(layout.tree); + + var parent_dir = Io.Dir.cwd().openDir(io, parent, .{}) catch |err| switch (err) { + error.FileNotFound => return, // already gone; nothing to wipe + else => return err, + }; + defer parent_dir.close(io); + + // `deleteTree` does not raise `FileNotFound` — a missing leaf is + // treated as success, matching our "force wipe is idempotent" + // intent here. + try parent_dir.deleteTree(io, base); + + std.log.info("panto bootstrap --force: removed {s}", .{layout.tree}); +} + +/// Every `panto` invocation runs through this one pipeline — `panto +/// lua`, `panto bootstrap`, and the default agent loop are all just +/// surfaces on top of "start the Lua runtime, install missing +/// batteries, then do your thing." +pub fn bootstrap( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + L: *c.lua_State, + panto_executable_path: []const u8, +) !*LuarocksRuntime { + const layout = try panto_home.resolve(allocator, environ_map); + errdefer layout.deinit(); + + try panto_home.ensureDirsExist(layout, io); + try stageLuaHeaders(allocator, io, layout); + const lua_wrapper_path = try writeLuaWrapper(allocator, io, layout, panto_executable_path); + defer allocator.free(lua_wrapper_path); + try writeLuarocksConfig(allocator, io, layout, lua_wrapper_path); + + // The embedded-source `package.searcher` keeps a pointer to the + // module map; we need the map's storage to live at a stable + // address for the process lifetime. Build the runtime first so + // `self.modules` is the address the singleton can capture, then + // install the searcher pointing at `self.modules` specifically. + const self = try allocator.create(LuarocksRuntime); + errdefer allocator.destroy(self); + self.* = .{ + .allocator = allocator, + .layout = layout, + .L = L, + .modules = .empty, + }; + try self.modules.ensureTotalCapacity(allocator, @intCast(embedded_luarocks.files.len + 2)); + for (embedded_luarocks.files) |entry| { + self.modules.putAssumeCapacityNoClobber(entry.name, entry.contents); + } + + try installEmbeddedSearcher(L, &self.modules); + try injectHardcoded(L, layout); + try configurePackagePaths(allocator, L, layout); + + // Reconcile the batteries manifest. This may take a while on a + // fresh install; subsequent runs no-op. Runs in-process — we + // already have luarocks loaded in `L` via the embedded searcher, + // so there's no need (and no reason) to spawn a child Lua. + // + // `PANTO_BOOTSTRAP_NO_RECONCILE` is a re-entry guard. When the + // reconcile loop is already running in an ancestor process (a + // luarocks build step shells out to `/bin/lua`, which is + // our `panto lua` wrapper), we don't want the child to start its + // own reconcile. The variable is set by `reconcileBatteries` + // before any potentially-recursive work and cleared afterward. + if (environ_map.get("PANTO_BOOTSTRAP_NO_RECONCILE") == null) { + try reconcileBatteries(allocator, io, self); + } + + return self; +} + +// --------------------------------------------------------------------------- +// Step 2: stage Lua headers +// --------------------------------------------------------------------------- + +/// Write every embedded Lua header to `/include/`. Skips any file +/// whose existing on-disk contents already match \u2014 keeps file mtimes +/// stable across reruns and lets luarocks's mtime-based caching work. +fn stageLuaHeaders( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, +) !void { + var dir = try Io.Dir.cwd().openDir(io, layout.include_dir, .{}); + defer dir.close(io); + + for (embedded_lua_headers.files) |entry| { + try writeIfDifferent(allocator, io, dir, entry.name, entry.contents); + } +} + +/// Write `contents` into `dir/name` only if the existing file differs. +/// Creates the file if absent. +fn writeIfDifferent( + allocator: Allocator, + io: Io, + dir: Io.Dir, + name: []const u8, + contents: []const u8, +) !void { + if (dir.readFileAlloc(io, name, allocator, .limited(1 << 22))) |existing| { + defer allocator.free(existing); + if (std.mem.eql(u8, existing, contents)) return; + } else |err| switch (err) { + error.FileNotFound => {}, + else => return err, + } + try dir.writeFile(io, .{ .sub_path = name, .data = contents }); +} + +// --------------------------------------------------------------------------- +// Step 3: write luarocks config +// --------------------------------------------------------------------------- + +/// Write a tiny shell wrapper at `/bin/lua` that `exec`s +/// ` lua "$@"`. luarocks invokes its configured `LUA` variable +/// as a real executable when running rockspec build scripts; we can't +/// give it `"panto lua"` directly because it splits argv naively, and +/// we can't give it an absolute path to the panto binary because then +/// it'd run panto's agent loop instead of `lua.c`'s REPL. +/// +/// The wrapper makes panto's lua subcommand visible to luarocks as if +/// it were a standalone `lua` binary. Returns the wrapper path; caller +/// frees. +fn writeLuaWrapper( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, + panto_executable_path: []const u8, +) ![]u8 { + const bin_dir = try std.fs.path.join(allocator, &.{ layout.tree, "bin" }); + defer allocator.free(bin_dir); + Io.Dir.cwd().createDirPath(io, bin_dir) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, + }; + + const wrapper_path = try std.fs.path.join(allocator, &.{ bin_dir, "lua" }); + errdefer allocator.free(wrapper_path); + + var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + const w = &aw.writer; + try w.writeAll("#!/bin/sh\n"); + try w.writeAll("# Auto-generated by panto. Bridges luarocks's external\n"); + try w.writeAll("# `lua` invocations to `panto lua`.\n"); + try w.writeAll("exec "); + try writeShellQuoted(w, panto_executable_path); + try w.writeAll(" lua \"$@\"\n"); + + var bin = try Io.Dir.cwd().openDir(io, bin_dir, .{}); + defer bin.close(io); + try bin.writeFile(io, .{ + .sub_path = "lua", + .data = aw.written(), + .flags = .{ .permissions = .executable_file }, + }); + return wrapper_path; +} + +fn writeShellQuoted(w: anytype, s: []const u8) !void { + try w.writeByte('\''); + for (s) |ch| { + if (ch == '\'') { + try w.writeAll("'\\''"); + } else { + try w.writeByte(ch); + } + } + try w.writeByte('\''); +} + +/// Materialize a luarocks config-.lua under `layout.sysconfdir`. +/// This is the file luarocks reads from `SYSCONFDIR`; we point every +/// path-typed variable at the in-tree directories so installs land in +/// `$PANTO_HOME/rocks/lua-X.Y.Z/`. +/// +/// Format reference: docs/config_file_format.md in the luarocks repo. +fn writeLuarocksConfig( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, + lua_wrapper_path: []const u8, +) !void { + var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + const w = &aw.writer; + + try w.writeAll("-- Auto-generated by panto. Do not edit.\n"); + try w.writeAll("-- This file is rewritten on every panto startup.\n\n"); + + try w.writeAll("rocks_trees = {\n { name = \"user\", root = "); + try writeLuaString(w, layout.tree); + try w.writeAll(" },\n}\n\n"); + + try w.writeAll("lua_interpreter = \"panto\"\n\n"); + + const bin_dir = std.fs.path.dirname(lua_wrapper_path) orelse layout.tree; + try w.writeAll("variables = {\n"); + try writeLuaKV(w, "LUA", lua_wrapper_path); + try writeLuaKV(w, "LUA_BINDIR", bin_dir); + try writeLuaKV(w, "LUA_INCDIR", layout.include_dir); + try writeLuaKV(w, "LUA_LIBDIR", layout.lib_lua_dir); + try writeLuaKV(w, "LUA_DIR", layout.tree); + try writeLuaKV(w, "CURL", "curl"); + try w.writeAll("}\n\n"); + + // We do not run luarocks's external `lua` invocation as a + // separate Lua install; the panto binary itself answers to + // `panto lua` and behaves like upstream lua.c. + try w.writeAll("lua_version = "); + try writeLuaString(w, manifest.lua_short_version); + try w.writeAll("\n\n"); + + // Default deps mode: only consider the panto tree, ignore any + // system-wide luarocks installations. Keeps reproducibility. + try w.writeAll("deps_mode = \"one\"\n"); + + // Write atomically: stage as `.tmp`, rename into place. luarocks + // reads the config eagerly on every invocation; an interrupted + // write would corrupt the next startup. + var sysconf = try Io.Dir.cwd().openDir(io, layout.sysconfdir, .{}); + defer sysconf.close(io); + + const tmp_name = "config-staging.lua"; + try sysconf.writeFile(io, .{ .sub_path = tmp_name, .data = aw.written() }); + try sysconf.rename(tmp_name, sysconf, std.fs.path.basename(layout.config_file), io); +} + +// --------------------------------------------------------------------------- +// Step 4: embedded-source `package.searcher` +// --------------------------------------------------------------------------- + +fn buildEmbeddedModuleMap(allocator: Allocator) !std.StringHashMapUnmanaged([]const u8) { + var modules: std.StringHashMapUnmanaged([]const u8) = .empty; + try modules.ensureTotalCapacity(allocator, @intCast(embedded_luarocks.files.len + 2)); + for (embedded_luarocks.files) |entry| { + modules.putAssumeCapacityNoClobber(entry.name, entry.contents); + } + return modules; +} + +/// Process-singleton handle the C searcher uses to find module sources. +/// Populated at bootstrap, read on every `require`. +var module_map_singleton: ?*const std.StringHashMapUnmanaged([]const u8) = null; + +/// Install our embedded-source searcher as `package.searchers[2]`, +/// after the preload searcher (slot 1) but before path-based searchers. +/// Slot 2 is conventional for embedded code (luarocks's own all_in_one +/// does the same). +fn installEmbeddedSearcher( + L: *c.lua_State, + modules: *std.StringHashMapUnmanaged([]const u8), +) !void { + module_map_singleton = modules; + + // Push package.searchers onto the stack. + _ = c.lua_getglobal(L, "package"); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return BootstrapError.LuarocksSearcherInstallFailed; + } + _ = c.lua_getfield(L, -1, "searchers"); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 2); + return BootstrapError.LuarocksSearcherInstallFailed; + } + + // We want to insert our searcher at slot 2 \u2014 push every existing + // entry from slot 2 onward up by one, then assign our function. + const n = c.lua_rawlen(L, -1); + // Shift slots upward: searchers[i+1] = searchers[i] for i in [n..2]. + var i: c.lua_Integer = @intCast(n); + while (i >= 2) : (i -= 1) { + _ = c.lua_rawgeti(L, -1, i); + c.lua_rawseti(L, -2, i + 1); + } + c.lua_pushcfunction(L, embeddedSearcher); + c.lua_rawseti(L, -2, 2); + + c.lua_settop(L, c.lua_gettop(L) - 2); // pop searchers + package +} + +/// Lua C function: given a module name, look it up in the embedded map +/// and return a loader closure if found; otherwise push an explanatory +/// string and return 1 \u2014 standard `package.searchers` contract. +fn embeddedSearcher(L: ?*c.lua_State) callconv(.c) c_int { + const Lst = L.?; + var name_len: usize = 0; + const name_ptr = c.lua_tolstring(Lst, 1, &name_len); + if (name_ptr == null) { + _ = c.lua_pushlstring(Lst, "\n\t[panto: invalid require argument]", 35); + return 1; + } + const name = name_ptr[0..name_len]; + + const map = module_map_singleton orelse { + _ = c.lua_pushlstring(Lst, "\n\t[panto: embedded modules not initialized]", 43); + return 1; + }; + + // Resolve `name` first, then `name.init` if missing — matches + // stock Lua path-searcher behavior where `require("foo")` checks + // both `foo.lua` and `foo/init.lua`. + var contents_opt = map.get(name); + var resolved_name: []const u8 = name; + var init_buf: [256]u8 = undefined; + if (contents_opt == null) { + const init_name = std.fmt.bufPrint(&init_buf, "{s}.init", .{name}) catch name; + if (map.get(init_name)) |contents2| { + contents_opt = contents2; + resolved_name = init_name; + } + } + if (contents_opt) |contents| { + // Push loader: a Lua function that, when called, executes the + // chunk and returns its result. `luaL_loadbufferx` with mode + // "t" forbids binary chunks so we can't be tricked into loading + // pre-compiled bytecode through this path. + const chunkname_buf = blk: { + // "=panto/" \u2014 the `=` prefix makes Lua print the name + // verbatim in stack traces instead of prefixing with `[string]`. + var sbuf: [256]u8 = undefined; + const s = std.fmt.bufPrintZ(&sbuf, "=panto/{s}", .{resolved_name}) catch "=panto/embedded"; + break :blk s; + }; + const status = c.luaL_loadbufferx( + Lst, + contents.ptr, + contents.len, + chunkname_buf.ptr, + "t", + ); + if (status != c.LUA_OK) { + // Error message left on stack by luaL_loadbufferx \u2014 return + // it as the searcher's diagnostic. + return 1; + } + return 1; + } + + _ = c.lua_pushlstring(Lst, "\n\t[panto: no embedded match]", 27); + return 1; +} + +// --------------------------------------------------------------------------- +// Step 5: inject luarocks.core.hardcoded +// --------------------------------------------------------------------------- + +/// Construct a `luarocks.core.hardcoded` table with the runtime-resolved +/// SYSCONFDIR (and FORCE_CONFIG = true, so luarocks doesn't go hunting +/// for system configs) and stash it in `package.loaded` so that the +/// later `require("luarocks.core.hardcoded")` returns our table instead +/// of going to disk. This is exactly the trick the upstream GNUmakefile +/// uses. +fn injectHardcoded(L: *c.lua_State, layout: panto_home.Layout) !void { + const snippet = + \\local sysconfdir = ... + \\package.loaded["luarocks.core.hardcoded"] = { + \\ SYSCONFDIR = sysconfdir, + \\ FORCE_CONFIG = true, + \\} + ; + if (c.luaL_loadstring(L, snippet) != 0) { + return BootstrapError.LuarocksInjectionFailed; + } + _ = c.lua_pushlstring(L, layout.sysconfdir.ptr, layout.sysconfdir.len); + if (c.lua_pcallk(L, 1, 0, 0, 0, null) != 0) { + return BootstrapError.LuarocksInjectionFailed; + } +} + +// --------------------------------------------------------------------------- +// Step 6: package.path / package.cpath +// --------------------------------------------------------------------------- + +/// Add `/share/lua//?.lua` and `/share/lua//?/init.lua` +/// to `package.path`, and the matching `.so`/`.dylib` patterns to +/// `package.cpath`, so rocks installed under our tree are visible to +/// `require`. The original path entries follow ours \u2014 we shadow the +/// system in case anything matches by accident. +fn configurePackagePaths( + allocator: Allocator, + L: *c.lua_State, + layout: panto_home.Layout, +) !void { + const so_suffix = comptime soSuffix(); + + const path_segment = try std.fmt.allocPrint( + allocator, + "{0s}/?.lua;{0s}/?/init.lua", + .{layout.share_lua_dir}, + ); + defer allocator.free(path_segment); + + const cpath_segment = try std.fmt.allocPrint( + allocator, + "{0s}/?{1s}", + .{ layout.lib_lua_dir, so_suffix }, + ); + defer allocator.free(cpath_segment); + + // Snippet: prepend the segments to package.path/cpath. + const snippet = + \\local path_seg, cpath_seg = ... + \\package.path = path_seg .. ";" .. package.path + \\package.cpath = cpath_seg .. ";" .. package.cpath + ; + if (c.luaL_loadstring(L, snippet) != 0) { + return BootstrapError.PathConfigFailed; + } + _ = c.lua_pushlstring(L, path_segment.ptr, path_segment.len); + _ = c.lua_pushlstring(L, cpath_segment.ptr, cpath_segment.len); + if (c.lua_pcallk(L, 2, 0, 0, 0, null) != 0) { + return BootstrapError.PathConfigFailed; + } +} + +fn soSuffix() []const u8 { + return switch (@import("builtin").os.tag) { + .macos, .ios, .watchos, .tvos => ".so", + .windows => ".dll", + else => ".so", + }; +} + +/// Write `value` as a quoted Lua string. Uses the `"..."` short-string +/// form, escaping backslashes, double-quotes, newlines, and any other +/// control characters via `\NNN` decimal escapes. Suitable for any +/// path on any platform. +fn writeLuaString(w: anytype, value: []const u8) !void { + try w.writeByte('"'); + for (value) |ch| { + switch (ch) { + '\\' => try w.writeAll("\\\\"), + '"' => try w.writeAll("\\\""), + '\n' => try w.writeAll("\\n"), + '\r' => try w.writeAll("\\r"), + '\t' => try w.writeAll("\\t"), + 0...8, 11, 12, 14...31, 127 => try w.print("\\{d}", .{ch}), + else => try w.writeByte(ch), + } + } + try w.writeByte('"'); +} + +fn writeLuaKV(w: anytype, key: []const u8, value: []const u8) !void { + try w.print(" {s} = ", .{key}); + try writeLuaString(w, value); + try w.writeAll(",\n"); +} + +// --------------------------------------------------------------------------- +// Step 7: reconcile manifest +// --------------------------------------------------------------------------- + +/// For each pinned battery, check whether the version-stamped install +/// metadata exists under `/lib/luarocks/rocks-///` +/// (luarocks's standard metadata location). If absent, call +/// `luarocks.cmd.run("install", name, version)` directly against our +/// `lua_State` — no subprocess. +/// +/// Subsequent panto runs hit the fast path: the metadata directory +/// exists, no luarocks invocation happens. +fn reconcileBatteries( + allocator: Allocator, + io: Io, + rt: *LuarocksRuntime, +) !void { + var any_missing = false; + for (manifest.batteries) |battery| { + if (try batteryInstalled(allocator, io, rt.layout, battery)) continue; + any_missing = true; + break; + } + if (!any_missing) return; + + // Tell any child processes that go through panto (luarocks's + // CMake/make subprocesses ultimately invoke `/bin/lua`, + // which is our `panto lua` wrapper) to skip their own reconcile + // step — we're already in the middle of it. + // + // Set process-wide via the C `setenv` so spawn calls inherit it. + // Cleared after the loop so subsequent panto invocations of this + // process see a clean environment. + _ = c_setenv("PANTO_BOOTSTRAP_NO_RECONCILE", "1", 1); + defer _ = c_unsetenv("PANTO_BOOTSTRAP_NO_RECONCILE"); + + for (manifest.batteries) |battery| { + if (try batteryInstalled(allocator, io, rt.layout, battery)) continue; + std.log.info( + "panto: installing battery {s} {s} via luarocks (first run; this may take a minute)", + .{ battery.name, battery.version }, + ); + try installBattery(allocator, rt.L, battery); + } +} + +extern "c" fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; +extern "c" fn unsetenv(name: [*:0]const u8) c_int; +const c_setenv = setenv; +const c_unsetenv = unsetenv; + +/// Check whether `/lib/luarocks/rocks-///` +/// exists. luarocks creates this directory atomically as part of an +/// install, so its presence is a reliable signal that the rock landed. +fn batteryInstalled( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, + battery: manifest.Battery, +) !bool { + const subpath = try std.fs.path.join( + allocator, + &.{ layout.rocks_metadata_dir, battery.name, battery.version }, + ); + defer allocator.free(subpath); + + var dir = Io.Dir.cwd().openDir(io, subpath, .{}) catch |err| switch (err) { + error.FileNotFound, error.NotDir => return false, + else => return err, + }; + dir.close(io); + return true; +} + +/// Spawn ` lua -e 'require("luarocks.cmd").run("install", name, version)'` +/// inheriting stdout/stderr so the user sees compilation output. +/// +/// We set `PANTO_BOOTSTRAP_NO_RECONCILE=1` in the child so that the +/// nested `panto lua` doesn't itself try to reconcile (which would +/// recurse forever — installing luv would re-trigger installing luv). +/// The child still runs the full filesystem-side bootstrap (searcher, +/// hardcoded, package paths) — only the manifest reconcile is skipped. +/// Run luarocks's `install` command directly against our `lua_State`. +/// +/// We mirror what `src/bin/luarocks` does — it's a thin Lua driver +/// that builds a command table and calls `cmd.run_command(...)`. We +/// embed the driver's source (`embedded_luarocks.luarocks_main`) and +/// load it as a chunk, passing `{"install", name, version}` as the +/// vararg the chunk reads via `...`. +/// +/// luarocks's `run_command` prints diagnostics to stdout/stderr as +/// it goes, so the user sees compilation progress in real time. +/// On failure it calls `os.exit(1)`, which our `pcall` wrapper +/// intercepts before it can terminate the process. +fn installBattery( + allocator: Allocator, + L: *c.lua_State, + battery: manifest.Battery, +) !void { + // We wrap the embedded driver in a function that overrides + // `os.exit` for the duration of the call. luarocks's `die()` + // ultimately calls `os.exit(1)`, which would terminate panto + // entirely; we want to catch the failure as a regular error. + const driver = embedded_luarocks.luarocks_main; + + const wrapper = + \\local orig_exit = os.exit + \\local args = { ... } + \\local fail_code = nil + \\os.exit = function(code) fail_code = code or 0; error({ panto_exit = code or 0 }) end + \\arg = arg or {} + \\arg[0] = arg[0] or "luarocks" + \\local driver = args[1] + \\local chunk, err = load(driver, "=panto/luarocks-driver", "t") + \\if not chunk then error("failed to load luarocks driver: " .. tostring(err)) end + \\local ok, err = pcall(chunk, table.unpack(args, 2)) + \\os.exit = orig_exit + \\if not ok then + \\ if type(err) == "table" and err.panto_exit ~= nil then + \\ if err.panto_exit ~= 0 then + \\ error("luarocks exited with code " .. tostring(err.panto_exit)) + \\ end + \\ else + \\ error(err) + \\ end + \\end + ; + + if (c.luaL_loadstring(L, wrapper) != 0) { + return BootstrapError.LuarocksInstallFailed; + } + + // Strip the shebang line if present. `luaL_loadfile` does this + // for files, but we're going through `load(...)` from Lua which + // doesn't — it'll choke on `#!/usr/bin/env lua` as a syntax error. + var driver_slice: []const u8 = driver; + if (driver_slice.len >= 2 and driver_slice[0] == '#' and driver_slice[1] == '!') { + if (std.mem.indexOfScalar(u8, driver_slice, '\n')) |nl| { + driver_slice = driver_slice[nl + 1 ..]; + } + } + + // Pass the driver source as the first arg, then the luarocks + // CLI args. We use `lua_pushlstring` to push everything as + // Lua strings (no NUL terminator needed, since lua_pushlstring + // takes an explicit length). + _ = c.lua_pushlstring(L, driver_slice.ptr, driver_slice.len); + _ = c.lua_pushlstring(L, "install", 7); + _ = c.lua_pushlstring(L, battery.name.ptr, battery.name.len); + _ = c.lua_pushlstring(L, battery.version.ptr, battery.version.len); + + _ = allocator; // currently unused; reserved for future scratch needs + + if (c.lua_pcallk(L, 4, 0, 0, 0, null) != 0) { + var len: usize = 0; + const msg = c.lua_tolstring(L, -1, &len); + if (msg != null) { + std.log.err( + "panto: luarocks install of {s} {s} failed: {s}", + .{ battery.name, battery.version, msg[0..len] }, + ); + } + c.lua_settop(L, c.lua_gettop(L) - 1); + return BootstrapError.LuarocksInstallFailed; + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "embedded module map contains luarocks.core.cfg, luarocks.cmd, compat53.init" { + var modules = try buildEmbeddedModuleMap(testing.allocator); + defer modules.deinit(testing.allocator); + + try testing.expect(modules.get("luarocks.core.cfg") != null); + try testing.expect(modules.get("luarocks.cmd") != null); + // `luarocks.cmd.init` is a real submodule (the `init` subcommand); + // luarocks.cmd is the module itself — distinct entries. + try testing.expect(modules.get("luarocks.cmd.init") != null); + try testing.expect(modules.get("compat53.init") != null); + try testing.expect(modules.get("compat53.module") != null); + // Not there: + try testing.expect(modules.get("luarocks.nonexistent") == null); +} + +test "embedded searcher fallback resolves bare name via name.init" { + // Mirrors stock Lua searcher semantics: require("compat53") should + // succeed even though our map only stores `compat53.init`. We test + // the lookup logic directly here — the C-level installation is + // exercised by integration runs of the panto binary. + var modules = try buildEmbeddedModuleMap(testing.allocator); + defer modules.deinit(testing.allocator); + + const bare = modules.get("compat53"); + try testing.expect(bare == null); + const via_init = modules.get("compat53.init"); + try testing.expect(via_init != null); +} diff --git a/src/main.zig b/src/main.zig index 15b2dd2..6654c95 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4,6 +4,10 @@ const ping_tool = @import("ping_tool.zig"); const lua_bridge = @import("lua_bridge.zig"); const lua_runtime = @import("lua_runtime.zig"); const extension_loader = @import("extension_loader.zig"); +const panto_home = @import("panto_home.zig"); +const luarocks_runtime = @import("luarocks_runtime.zig"); +const self_exe = @import("self_exe.zig"); +const subcommand = @import("subcommand.zig"); // Shorthand alias for the Lua C API. The bridge module owns the actual // `@cImport`; we re-use it here so the smoke check uses identical types. @@ -23,6 +27,10 @@ test { _ = lua_bridge; _ = lua_runtime; _ = extension_loader; + _ = panto_home; + _ = luarocks_runtime; + _ = self_exe; + _ = subcommand; } const Receiver = panto.provider.Receiver; @@ -231,6 +239,26 @@ pub fn main(init: std.process.Init) !void { // wired up yet, this just confirms the static lib comes through. luaSmokeCheck(); + // Resolve the absolute path of the running panto binary. Needed + // both by `panto lua` (we re-exec ourselves through a wrapper + // luarocks invokes) and by the agent's bootstrap. + const panto_path = try self_exe.selfExePathAlloc(alloc); + defer alloc.free(panto_path); + + // Subcommand dispatch: `panto lua` and `panto bootstrap` short + // out of the agent loop, but still run the same luarocks bootstrap + // pipeline so first-run setup happens consistently. + switch (try subcommand.dispatch( + alloc, + io, + init.environ_map, + init.minimal.args, + panto_path, + )) { + .done => return, + .agent => {}, + } + const config = try loadConfig(init.environ_map); var stdout_buffer: [4096]u8 = undefined; @@ -260,6 +288,24 @@ pub fn main(init: std.process.Init) !void { var rt = try lua_runtime.LuaRuntime.create(alloc); defer rt.deinit(); + // Bootstrap luarocks against the Lua runtime's lua_State — same + // pipeline as `panto lua` and `panto bootstrap`. After this, + // `require("luarocks.*")` works and any pinned batteries from the + // manifest are installed under $PANTO_HOME. + const luarocks_rt = try luarocks_runtime.bootstrap( + alloc, + io, + init.environ_map, + rt.L, + panto_path, + ); + defer luarocks_rt.deinit(); + + // luv is installed (or already present) at this point; wire the + // libuv-driven coroutine scheduler before any extensions get a + // chance to register tools that might want to yield. + try rt.installScheduler(); + // Discover Lua extensions from $XDG_CONFIG_HOME/panto/extensions (or // $HOME/.config/panto/extensions) and ./.panto/extensions. Project // entries shadow user entries with the same name; tool-name collisions diff --git a/src/manifest.zig b/src/manifest.zig new file mode 100644 index 0000000..b12b5c4 --- /dev/null +++ b/src/manifest.zig @@ -0,0 +1,43 @@ +//! Pinned versions of every Lua component shipped with panto: the +//! upstream Lua language version, the embedded luarocks version, and +//! the rocks ("batteries") panto requires for its own runtime to work. +//! +//! Bumping any of these is a deliberate edit + commit + version bump of +//! panto itself. Each panto release pins one consistent set; bootstrap +//! reconciles the installed rocks tree against this manifest on every +//! startup, installing missing rocks and removing stale ones (per Q5 of +//! LUA_MAKEOVER.md). +//! +//! The Lua and luarocks versions ride in via the `versions` build option +//! so build.zig stays the single source of truth. The batteries are +//! pinned here directly because they don't otherwise need to be visible +//! to the build. + +const versions = @import("versions"); + +pub const lua_version: []const u8 = versions.lua_version; +pub const lua_short_version: []const u8 = versions.lua_short_version; +pub const luarocks_version: []const u8 = versions.luarocks_version; + +pub const Battery = struct { + /// Rock name as published on luarocks.org. + name: []const u8, + /// Pinned rockspec version, e.g. `"1.52.1-1"`. luarocks uses these + /// MAJOR.MINOR.PATCH-REV strings throughout — we pass them straight + /// through to `luarocks install`. + version: []const u8, +}; + +/// Rocks installed automatically at bootstrap. Compiled into the panto +/// binary; users do not configure this list. +/// +/// We deliberately ship the smallest viable set: luv is the only +/// runtime dependency — it provides libuv's event loop, which panto's +/// coroutine scheduler drives, and gives extension authors a single +/// rich async I/O surface. Anything else (coro-* helpers, lua-cjson, +/// lpeg, etc.) is the user's responsibility, installable via +/// `panto lua -e 'arg[0]="luarocks"; require("luarocks.cmd").run_command(...)'` +/// or, eventually, a higher-level `panto rocks install` subcommand. +pub const batteries: []const Battery = &.{ + .{ .name = "luv", .version = "1.52.1-0" }, +}; diff --git a/src/panto_home.zig b/src/panto_home.zig new file mode 100644 index 0000000..b7c1799 --- /dev/null +++ b/src/panto_home.zig @@ -0,0 +1,206 @@ +//! Filesystem layout resolution for `$PANTO_HOME` and the per-Lua +//! version rocks tree. +//! +//! $PANTO_HOME = $XDG_DATA_HOME/panto +//! (or $HOME/.local/share/panto if XDG_DATA_HOME unset) +//! +//! $PANTO_HOME/ +//! rocks/ +//! lua-5.4.7/ ← current tree +//! include/ ← Lua headers staged by bootstrap +//! share/lua/5.4/ ← installed pure-Lua rocks +//! lib/lua/5.4/ ← installed C rocks (.so/.dylib) +//! lib/luarocks/rocks-5.4/ ← luarocks's own rock metadata +//! etc/luarocks/ ← luarocks config-5.4.lua +//! +//! See Q3 of LUA_MAKEOVER.md for the rationale behind the versioned +//! subdirectory. We never write to the tree of a different Lua version +//! — a panto upgrade that bumps Lua creates a fresh tree alongside the +//! old one, leaving the old one in place for rollback. + +const std = @import("std"); +const manifest = @import("manifest.zig"); + +const Allocator = std.mem.Allocator; +const Io = std.Io; + +/// Resolved set of filesystem paths panto uses for its rocks tree. All +/// fields are owned by the same allocator passed to `resolve`. +pub const Layout = struct { + allocator: Allocator, + /// `$PANTO_HOME` itself. + home: []u8, + /// `$PANTO_HOME/rocks/lua-/` — the versioned tree. + tree: []u8, + /// `/include/` — where Lua headers are staged. + include_dir: []u8, + /// `/share/lua//` — pure-Lua rocks. + share_lua_dir: []u8, + /// `/lib/lua//` — C rocks (.so/.dylib). + lib_lua_dir: []u8, + /// `/lib/luarocks/rocks-/` — luarocks's rock metadata + /// per the standard tree layout. + rocks_metadata_dir: []u8, + /// `/etc/luarocks/` — luarocks's own config dir, where the + /// per-version `config-.lua` lives. We set `SYSCONFDIR` to + /// this when constructing `luarocks.core.hardcoded`. + sysconfdir: []u8, + /// `/etc/luarocks/config-.lua` — the path of the config + /// file we materialize at bootstrap. + config_file: []u8, + + pub fn deinit(self: Layout) void { + const a = self.allocator; + a.free(self.home); + a.free(self.tree); + a.free(self.include_dir); + a.free(self.share_lua_dir); + a.free(self.lib_lua_dir); + a.free(self.rocks_metadata_dir); + a.free(self.sysconfdir); + a.free(self.config_file); + } +}; + +/// Resolve every path the runtime cares about. Environment-driven: +/// - `PANTO_HOME` (explicit override) +/// - `XDG_DATA_HOME` (XDG default) +/// - `HOME` (fallback) +/// +/// Returns `error.NoHomeDirectory` if none of those are available and +/// no `PANTO_HOME` was set. +pub fn resolve( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, +) !Layout { + const home = try resolveHome(allocator, environ_map); + errdefer allocator.free(home); + + // `/rocks/lua-` + const tree_subdir = try std.fmt.allocPrint( + allocator, + "lua-{s}", + .{manifest.lua_version}, + ); + defer allocator.free(tree_subdir); + const tree = try std.fs.path.join(allocator, &.{ home, "rocks", tree_subdir }); + errdefer allocator.free(tree); + + const short = manifest.lua_short_version; + + const include_dir = try std.fs.path.join(allocator, &.{ tree, "include" }); + errdefer allocator.free(include_dir); + const share_lua_dir = try std.fs.path.join(allocator, &.{ tree, "share", "lua", short }); + errdefer allocator.free(share_lua_dir); + const lib_lua_dir = try std.fs.path.join(allocator, &.{ tree, "lib", "lua", short }); + errdefer allocator.free(lib_lua_dir); + const rocks_subdir = try std.fmt.allocPrint(allocator, "rocks-{s}", .{short}); + defer allocator.free(rocks_subdir); + const rocks_metadata_dir = try std.fs.path.join( + allocator, + &.{ tree, "lib", "luarocks", rocks_subdir }, + ); + errdefer allocator.free(rocks_metadata_dir); + const sysconfdir = try std.fs.path.join(allocator, &.{ tree, "etc", "luarocks" }); + errdefer allocator.free(sysconfdir); + const config_basename = try std.fmt.allocPrint(allocator, "config-{s}.lua", .{short}); + defer allocator.free(config_basename); + const config_file = try std.fs.path.join(allocator, &.{ sysconfdir, config_basename }); + errdefer allocator.free(config_file); + + return .{ + .allocator = allocator, + .home = home, + .tree = tree, + .include_dir = include_dir, + .share_lua_dir = share_lua_dir, + .lib_lua_dir = lib_lua_dir, + .rocks_metadata_dir = rocks_metadata_dir, + .sysconfdir = sysconfdir, + .config_file = config_file, + }; +} + +/// Resolve `$PANTO_HOME` honoring overrides in the documented order. +/// Returns owned bytes. +fn resolveHome( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, +) ![]u8 { + if (environ_map.get("PANTO_HOME")) |explicit| { + return allocator.dupe(u8, explicit); + } + if (environ_map.get("XDG_DATA_HOME")) |xdg| { + return std.fs.path.join(allocator, &.{ xdg, "panto" }); + } + if (environ_map.get("HOME")) |hh| { + return std.fs.path.join(allocator, &.{ hh, ".local", "share", "panto" }); + } + return error.NoHomeDirectory; +} + +/// Create every directory referenced by `layout`, recursively. Idempotent +/// — existing directories are left alone. +pub fn ensureDirsExist(layout: Layout, io: Io) !void { + try makePathRecursive(io, layout.home); + try makePathRecursive(io, layout.tree); + try makePathRecursive(io, layout.include_dir); + try makePathRecursive(io, layout.share_lua_dir); + try makePathRecursive(io, layout.lib_lua_dir); + try makePathRecursive(io, layout.rocks_metadata_dir); + try makePathRecursive(io, layout.sysconfdir); +} + +fn makePathRecursive(io: Io, path: []const u8) !void { + Io.Dir.cwd().createDirPath(io, path) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "resolve: PANTO_HOME explicit override wins" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("PANTO_HOME", "/tmp/some/home"); + try env.put("XDG_DATA_HOME", "/should/be/ignored"); + + var layout = try resolve(testing.allocator, &env); + defer layout.deinit(); + + try testing.expectEqualStrings("/tmp/some/home", layout.home); + try testing.expect(std.mem.indexOf(u8, layout.tree, "/tmp/some/home/rocks/lua-") != null); + try testing.expect(std.mem.endsWith(u8, layout.share_lua_dir, "/share/lua/" ++ manifest.lua_short_version)); +} + +test "resolve: XDG_DATA_HOME is honored before HOME" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("XDG_DATA_HOME", "/x/data"); + try env.put("HOME", "/h"); + + var layout = try resolve(testing.allocator, &env); + defer layout.deinit(); + try testing.expectEqualStrings("/x/data/panto", layout.home); +} + +test "resolve: HOME fallback used if neither override is present" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("HOME", "/home/user"); + + var layout = try resolve(testing.allocator, &env); + defer layout.deinit(); + try testing.expectEqualStrings("/home/user/.local/share/panto", layout.home); +} + +test "resolve: error when no environment hint at all" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try testing.expectError(error.NoHomeDirectory, resolve(testing.allocator, &env)); +} diff --git a/src/self_exe.zig b/src/self_exe.zig new file mode 100644 index 0000000..6eda92d --- /dev/null +++ b/src/self_exe.zig @@ -0,0 +1,90 @@ +//! Resolve the absolute path of the currently running executable. +//! +//! Used to find the panto binary so we can launch ourselves as `panto +//! lua` from inside the bootstrap (the embedded luarocks shells out to +//! its configured `LUA` interpreter, which we point at a wrapper script +//! that exec's panto's `lua` subcommand). +//! +//! Zig's standard library doesn't expose a portable `selfExePath` in +//! the current API, so this module wraps the per-OS syscall. +//! +//! Supported: +//! - macOS / iOS / tvOS / watchOS: `_NSGetExecutablePath` + realpath +//! - Linux: `readlink("/proc/self/exe")` +//! - FreeBSD / NetBSD / DragonFly: `readlink("/proc/curproc/file")` +//! +//! Other targets currently return `error.SelfExePathUnavailable`. + +const std = @import("std"); +const builtin = @import("builtin"); + +const Allocator = std.mem.Allocator; + +pub const ResolveError = error{ + SelfExePathUnavailable, + SelfExePathTooLong, + SelfExePathReadFailed, + OutOfMemory, +}; + +/// Return the absolute, symlink-resolved path of the current process's +/// executable. Caller owns the returned slice. +pub fn selfExePathAlloc(allocator: Allocator) ResolveError![]u8 { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try selfExePath(&buf); + return allocator.dupe(u8, path); +} + +/// Fill `buf` with the absolute path. Returns the slice that was used. +pub fn selfExePath(buf: []u8) ResolveError![]u8 { + return switch (builtin.os.tag) { + .macos, .ios, .tvos, .watchos => try macosSelfExePath(buf), + .linux => try readLinkSelfExe(buf, "/proc/self/exe"), + .freebsd, .netbsd, .dragonfly => try readLinkSelfExe(buf, "/proc/curproc/file"), + else => return ResolveError.SelfExePathUnavailable, + }; +} + +fn readLinkSelfExe(buf: []u8, link_path: []const u8) ResolveError![]u8 { + const link_z = std.posix.toPosixPath(link_path) catch return ResolveError.SelfExePathTooLong; + const n = posixReadlink(&link_z, buf) catch return ResolveError.SelfExePathReadFailed; + return buf[0..n]; +} + +/// Thin wrapper around the POSIX `readlink` syscall using libc directly, +/// since std.posix's surface here has been in flux. libc is linked into +/// panto already (Lua needs it), so this stays cheap. +extern "c" fn readlink(path: [*:0]const u8, buf: [*]u8, bufsize: usize) isize; + +fn posixReadlink(path: [*:0]const u8, buf: []u8) !usize { + const n = readlink(path, buf.ptr, buf.len); + if (n < 0) return error.SelfExePathReadFailed; + return @intCast(n); +} + +extern "c" fn _NSGetExecutablePath(buf: [*]u8, bufsize: *u32) c_int; + +fn macosSelfExePath(buf: []u8) ResolveError![]u8 { + var stage: [std.fs.max_path_bytes]u8 = undefined; + var size: u32 = @intCast(stage.len); + if (_NSGetExecutablePath(&stage, &size) != 0) return ResolveError.SelfExePathTooLong; + // _NSGetExecutablePath populates `size` on overflow; on success + // we need to find the NUL ourselves. + const len = std.mem.indexOfScalar(u8, &stage, 0) orelse return ResolveError.SelfExePathTooLong; + + // The returned path may include `..` and symlinks; canonicalize so + // downstream consumers don't have to. realpath(3) on POSIX, both + // libc-resident on macOS and Linux. + const path_z = std.posix.toPosixPath(stage[0..len]) catch return ResolveError.SelfExePathTooLong; + + // libc realpath: NULL second arg => malloc; we want stack buffer. + var real_buf: [std.fs.max_path_bytes]u8 = undefined; + if (libc_realpath(&path_z, &real_buf) == null) return ResolveError.SelfExePathReadFailed; + const real_len = std.mem.indexOfScalar(u8, &real_buf, 0) orelse return ResolveError.SelfExePathTooLong; + if (real_len > buf.len) return ResolveError.SelfExePathTooLong; + @memcpy(buf[0..real_len], real_buf[0..real_len]); + return buf[0..real_len]; +} + +extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*]u8; +const libc_realpath = realpath; diff --git a/src/subcommand.zig b/src/subcommand.zig new file mode 100644 index 0000000..97eb977 --- /dev/null +++ b/src/subcommand.zig @@ -0,0 +1,231 @@ +//! Subcommand dispatch for the `panto` CLI. +//! +//! Routes argv[1] to one of: +//! - `lua` — drop into the embedded standalone Lua interpreter +//! (panto's `lua.c` build), with luarocks's runtime +//! bootstrap completed first so `require("luarocks.*")` +//! and the configured rocks tree work the same as in +//! the agent process. +//! - `bootstrap` — run the luarocks runtime bootstrap pipeline only; +//! exit before entering any agent loop. Lets users +//! do first-run setup on a fresh machine without +//! starting a chat session. +//! - anything else (or absent) — fall through to the agent REPL. +//! +//! Both `lua` and `bootstrap` end up calling `luarocks_runtime.bootstrap` +//! before doing their thing. The agent path does the same; the only +//! difference is whether the agent loop runs afterward. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; + +const lua_bridge = @import("lua_bridge.zig"); +const luarocks_runtime = @import("luarocks_runtime.zig"); +const self_exe = @import("self_exe.zig"); + +const c = lua_bridge.c; + +pub const Action = enum { + /// Continue with the default agent REPL. + agent, + /// Bootstrap is already done; the dispatcher consumed the subcommand. + /// `main` should exit immediately. + done, +}; + +/// Inspect `argv[1]`, run the appropriate subcommand, and return what +/// the caller should do next. On `.agent`, the dispatcher leaves argv +/// untouched and `main` continues as before. On `.done`, the caller +/// must return promptly (the subcommand has already produced output). +/// +/// `panto_executable_path` is the absolute path of the running panto +/// binary, used both to wire up the embedded luarocks `LUA` variable +/// and to `exec` ourselves where needed. +pub fn dispatch( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + args: std.process.Args, + panto_executable_path: []const u8, +) !Action { + var it = args.iterate(); + defer it.deinit(); + _ = it.next(); // argv[0] + + const sub = it.next() orelse return .agent; + + if (std.mem.eql(u8, sub, "lua")) { + try runLuaSubcommand(allocator, io, environ_map, args, panto_executable_path); + return .done; + } + if (std.mem.eql(u8, sub, "bootstrap")) { + var force = false; + while (it.next()) |flag| { + if (std.mem.eql(u8, flag, "--force")) { + force = true; + } else { + std.log.err("panto bootstrap: unknown flag '{s}'", .{flag}); + return error.UnknownFlag; + } + } + try runBootstrapSubcommand(allocator, io, environ_map, panto_executable_path, .{ .force = force }); + return .done; + } + return .agent; +} + +pub const BootstrapOptions = struct { + /// Wipe the per-Lua-version tree before reinstalling everything. + /// Surfaced as `panto bootstrap --force`. Equivalent to deleting + /// `$PANTO_HOME/rocks/lua-X.Y.Z/` by hand and then running + /// `panto bootstrap`. + force: bool = false, +}; + +// --------------------------------------------------------------------------- +// `panto lua` +// --------------------------------------------------------------------------- + +extern "c" fn panto_lua_pmain(L: *c.lua_State, argc: c_int, argv: [*]?[*:0]u8) c_int; + +/// Drop into the embedded Lua standalone interpreter, with the +/// luarocks runtime bootstrap completed so `require("luarocks.*")` +/// and rocks installed under `$PANTO_HOME` are visible. +/// +/// argv is rewritten so the interpreter sees `lua [...args]` rather +/// than `panto lua [...args]` — matching upstream behavior. The first +/// argument visible to `pmain` is the program name; this matters for +/// `arg[0]` and error reporting. +fn runLuaSubcommand( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + args: std.process.Args, + panto_executable_path: []const u8, +) !void { + // Build a fresh lua_State that we own, configure it like luarocks + // expects, then hand it to `pmain`. + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + + // Run bootstrap against this state. This installs the embedded + // searcher, configures package.path/cpath, and stages on-disk + // resources. We deliberately do NOT call `luaL_openlibs` here — + // `pmain` does that itself, and we want exactly the upstream + // ordering for everything that runs inside the REPL. + // + // The searcher install only requires `package.searchers` to be + // present; the stock libs ship it. We open libs once here just + // to satisfy that, then pmain's own `luaL_openlibs` is idempotent. + c.luaL_openlibs(L); + const rt = try luarocks_runtime.bootstrap( + allocator, + io, + environ_map, + L, + panto_executable_path, + ); + defer rt.deinit(); + + // Re-create the argv the standalone interpreter expects. argv[0] + // is the program name; argv[1..] are the user's args. + var raw_args = args.iterate(); + defer raw_args.deinit(); + _ = raw_args.next(); // panto + _ = raw_args.next(); // lua + + var argv_list: std.array_list.Managed([:0]u8) = .init(allocator); + defer { + for (argv_list.items) |s| allocator.free(s); + argv_list.deinit(); + } + + // Program name first. + try argv_list.append(try allocator.dupeZ(u8, "lua")); + while (raw_args.next()) |a| { + try argv_list.append(try allocator.dupeZ(u8, a)); + } + + // Build a `[*]?[*:0]u8` argv pointer array. lua.c expects a + // NULL-terminated array (it uses `argv[i]` indexed access through + // argc; the trailing NULL is conventional for C `main`). + var argv_c: std.array_list.Managed(?[*:0]u8) = .init(allocator); + defer argv_c.deinit(); + for (argv_list.items) |s| { + try argv_c.append(s.ptr); + } + try argv_c.append(null); + + const exit_code = panto_lua_pmain(L, @intCast(argv_list.items.len), argv_c.items.ptr); + if (exit_code != 0) std.process.exit(@intCast(exit_code)); +} + +// --------------------------------------------------------------------------- +// `panto bootstrap` +// --------------------------------------------------------------------------- + +/// Run the luarocks bootstrap and exit. Useful for first-run setup on +/// a clean machine (downloads + compiles batteries, stages headers, +/// materializes config) and for CI/scripted installs. +/// +/// Idempotent: subsequent invocations no-op fast, unless `force` was +/// passed — then the entire per-Lua-version tree is wiped before the +/// regular bootstrap pipeline runs. +fn runBootstrapSubcommand( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + panto_executable_path: []const u8, + opts: BootstrapOptions, +) !void { + if (opts.force) { + try luarocks_runtime.wipeTree(allocator, io, environ_map); + } + + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + c.luaL_openlibs(L); + + const rt = try luarocks_runtime.bootstrap( + allocator, + io, + environ_map, + L, + panto_executable_path, + ); + defer rt.deinit(); + + // Pleasant single-line confirmation. The interesting bits (rock + // installs etc.) print their own progress. + std.log.info( + "panto bootstrap: tree ready at {s}", + .{rt.layout.tree}, + ); +} + +// --------------------------------------------------------------------------- +// `panto lua` argv plumbing — sketched against the older Args API for +// reference (kept here so the design notes survive the implementation). +// --------------------------------------------------------------------------- +// +// Because we own the `lua_State` end-to-end, the subcommand can also +// expose extra panto-specific globals to user code (e.g. surface the +// resolved $PANTO_HOME) without disturbing upstream `lua.c` behavior. +// Step out of scope for the current makeover; add when needed. + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +// Note: `dispatch` reads from the process's real argv, which isn't +// controllable from a unit test. The behavior is exercised by +// integration runs of the panto binary. We test the smaller pieces. +// +// Suppress dead-code warnings for `self_exe` (it's used by main, not +// by tests in this module). +test { + _ = self_exe; +} -- cgit v1.3