diff options
| -rw-r--r-- | build.zig | 251 | ||||
| -rw-r--r-- | build.zig.zon | 4 | ||||
| -rw-r--r-- | build/gen_lua_anchor.zig | 144 | ||||
| -rw-r--r-- | build/gen_lua_headers_embed.zig | 68 | ||||
| -rw-r--r-- | build/gen_luarocks_embed.zig | 157 | ||||
| -rw-r--r-- | build/panto_lua_repl.c | 49 | ||||
| -rw-r--r-- | docs/archive/LUA_MAKEOVER.md | 594 | ||||
| -rw-r--r-- | docs/archive/ideas.md (renamed from ideas.md) | 0 | ||||
| -rw-r--r-- | docs/archive/phase-1.md (renamed from docs/phase-1.md) | 0 | ||||
| -rw-r--r-- | docs/archive/phase-2.md (renamed from docs/phase-2.md) | 0 | ||||
| -rw-r--r-- | docs/archive/phase-3.md (renamed from docs/phase-3.md) | 0 | ||||
| -rw-r--r-- | src/lua_runtime.zig | 522 | ||||
| -rw-r--r-- | src/luarocks_runtime.zig | 763 | ||||
| -rw-r--r-- | src/main.zig | 46 | ||||
| -rw-r--r-- | src/manifest.zig | 43 | ||||
| -rw-r--r-- | src/panto_home.zig | 206 | ||||
| -rw-r--r-- | src/self_exe.zig | 90 | ||||
| -rw-r--r-- | src/subcommand.zig | 231 |
18 files changed, 3111 insertions, 57 deletions
@@ -1,5 +1,9 @@ const std = @import("std"); +const lua_version = "5.4.7"; +const lua_short_version = "5.4"; +const luarocks_version = "3.13.0"; + pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); @@ -9,10 +13,31 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); - // Fetch upstream Lua 5.4.7 (source-only tarball from lua.org). - // Reproducibility comes from the content-addressed hash in build.zig.zon. + // Fetch upstream Lua source (used both for our static library and + // staged at runtime as the `include/` headers under $PANTO_HOME). + // Reproducibility comes from the content-addressed hash in + // build.zig.zon. const lua_src = b.dependency("lua_src", .{}); - const lua = buildLua(b, target, optimize, lua_src); + const lua_anchor_path = generateLuaAnchor(b, lua_src); + + // Fetch luarocks source. We don't compile any of it — it's pure + // Lua. The codegen step below produces a `embedded_luarocks.zig` + // module that `@embedFile`s every script so the runtime can serve + // them as `require("luarocks.*")` results without disk I/O. + const luarocks_src = b.dependency("luarocks_src", .{}); + const luarocks_embed_path = generateLuarocksEmbed(b, luarocks_src); + + // Same trick for the Lua headers: bundle them so first-run + // bootstrap can stage them under $PANTO_HOME/rocks/lua-X.Y.Z/include/ + // for luarocks to find when compiling C rocks. + const lua_headers_embed_path = generateLuaHeadersEmbed(b, lua_src); + + // Constants module: makes Zig know the Lua and luarocks versions + // without duplicating literals everywhere. + const versions_mod = b.addOptions(); + versions_mod.addOption([]const u8, "lua_version", lua_version); + versions_mod.addOption([]const u8, "lua_short_version", lua_short_version); + versions_mod.addOption([]const u8, "luarocks_version", luarocks_version); // CLI executable const exe_mod = b.createModule(.{ @@ -22,13 +47,42 @@ pub fn build(b: *std.Build) void { .link_libc = true, }); exe_mod.addImport("panto", panto_lib_dep.module("panto")); - exe_mod.linkLibrary(lua); + exe_mod.addImport("versions", versions_mod.createModule()); + exe_mod.addImport("embedded_luarocks", b.createModule(.{ + .root_source_file = luarocks_embed_path, + })); + exe_mod.addImport("embedded_lua_headers", b.createModule(.{ + .root_source_file = lua_headers_embed_path, + })); + // Link Lua. We can't use a static library here because the + // standard archive-linking behavior drops object files whose + // symbols nothing in our code references — e.g. `lua_settable`, + // which luv.so needs at runtime via `dlopen` but neither panto + // itself nor the embedded lua.c repl ever call directly. Instead, + // compile the Lua sources directly *into* the executable's own + // module so every object lands in the binary. `rdynamic = true` + // below ensures the symbols are exported into the dynamic symbol + // table for `dlopen`'d rocks to resolve. + addLuaSources(exe_mod, lua_src, lua_anchor_path); exe_mod.addIncludePath(lua_src.path("src")); + // Compile lua.c (the upstream standalone interpreter, ~600 lines) + // as a separate static library so we can route the `panto lua` + // subcommand into its main() without it conflicting with our own + // main(). + const lua_repl = buildLuaRepl(b, target, optimize, lua_src); + exe_mod.linkLibrary(lua_repl); + const exe = b.addExecutable(.{ .name = "panto", .root_module = exe_mod, }); + // C rocks (luv.so and anything else loaded via dlopen) need + // Lua's C API symbols to be findable in the running process's + // symbol table. Without `-rdynamic`, the linker strips them as + // unused-from-outside; `dlopen` then fails at load time with + // "symbol not found in flat namespace '_lua_settable'" etc. + exe.rdynamic = true; b.installArtifact(exe); @@ -41,7 +95,7 @@ pub fn build(b: *std.Build) void { const run_step = b.step("run", "Run the app"); run_step.dependOn(&run_cmd.step); - // CLI unit tests (minimal — most tests live in libpanto) + // CLI unit tests const test_mod = b.createModule(.{ .root_source_file = b.path("src/main.zig"), .target = target, @@ -49,8 +103,16 @@ pub fn build(b: *std.Build) void { .link_libc = true, }); test_mod.addImport("panto", panto_lib_dep.module("panto")); - test_mod.linkLibrary(lua); + test_mod.addImport("versions", versions_mod.createModule()); + test_mod.addImport("embedded_luarocks", b.createModule(.{ + .root_source_file = luarocks_embed_path, + })); + test_mod.addImport("embedded_lua_headers", b.createModule(.{ + .root_source_file = lua_headers_embed_path, + })); + addLuaSources(test_mod, lua_src, lua_anchor_path); test_mod.addIncludePath(lua_src.path("src")); + test_mod.linkLibrary(lua_repl); const unit_tests = b.addTest(.{ .name = "panto-tests", @@ -66,16 +128,20 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&lib_test_step.step); } -/// Compile Lua 5.4.7 as a static library against the upstream source tarball. -/// Excludes `lua.c` (interpreter REPL) and `luac.c` (bytecode compiler) — we -/// only want the embedded library. -fn buildLua( +/// Compile a thin wrapper around the upstream `lua.c` standalone +/// interpreter that exposes `pmain` for panto's `lua` subcommand to +/// invoke against a pre-configured `lua_State`. +/// +/// We rename `main` to `lua_unused_main` so the symbol doesn't collide +/// with panto's `main` (the wrapper never references it; the linker +/// just needs a place for it to land). +fn buildLuaRepl( b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, lua_src: *std.Build.Dependency, ) *std.Build.Step.Compile { - const lua_mod = b.createModule(.{ + const mod = b.createModule(.{ .target = target, .optimize = optimize, .link_libc = true, @@ -84,33 +150,176 @@ fn buildLua( const cflags = [_][]const u8{ "-std=gnu99", "-Wall", - "-Wextra", "-Wno-unused-parameter", + "-Wno-unused-function", + // Rename upstream `main` so it doesn't collide with panto's. + // We never call it; the wrapper invokes `pmain` directly. + "-Dmain=lua_unused_main", }; - lua_mod.addCSourceFiles(.{ + mod.addCSourceFile(.{ + .file = b.path("build/panto_lua_repl.c"), + .flags = &cflags, + }); + // The wrapper `#include`s lua.c by name; we add lua_src/src to + // the search path so `#include "lua.c"` resolves. + mod.addIncludePath(lua_src.path("src")); + addLuaPlatformMacros(mod, target); + + return b.addLibrary(.{ + .name = "lua-repl", + .linkage = .static, + .root_module = mod, + }); +} + +/// Add all the Lua 5.4 core + standard-library C sources directly to +/// `mod`. Compared to building a separate static library and linking, +/// this guarantees the linker doesn't drop "unused" objects — we need +/// every Lua API symbol present in the final binary so C rocks loaded +/// via `dlopen` can resolve them against the host process's symbol +/// table. +fn addLuaSources( + mod: *std.Build.Module, + lua_src: *std.Build.Dependency, + lua_anchor_path: std.Build.LazyPath, +) void { + const cflags = [_][]const u8{ + "-std=gnu99", + "-Wall", + "-Wextra", + "-Wno-unused-parameter", + }; + mod.addCSourceFiles(.{ .root = lua_src.path("src"), .files = &lua_files, .flags = &cflags, }); + // Tiny extra TU (codegen'd at build time): takes the address of + // every Lua API function into a `__attribute__((used))` array. + // This forces the linker to keep every API function in the binary + // even under ReleaseFast's LTO and gc-sections, regardless of + // whether our own code references them. C rocks loaded via + // `dlopen` (luv.so etc.) need every public Lua symbol resolvable + // in the host process's symbol table; combined with + // `rdynamic = true` on the executable, this gets them all into + // the dynamic symbol table. + mod.addCSourceFile(.{ + .file = lua_anchor_path, + .flags = &cflags, + }); + addLuaPlatformMacros(mod, mod.resolved_target.?); +} +fn addLuaPlatformMacros( + mod: *std.Build.Module, + target: std.Build.ResolvedTarget, +) void { // Tell Lua which platform features are available. The macros change // which `dlopen`/readline/etc. paths Lua compiles in. switch (target.result.os.tag) { - .macos => lua_mod.addCMacro("LUA_USE_MACOSX", ""), - .linux => lua_mod.addCMacro("LUA_USE_LINUX", ""), - .freebsd, .netbsd, .openbsd => lua_mod.addCMacro("LUA_USE_POSIX", ""), + .macos => mod.addCMacro("LUA_USE_MACOSX", ""), + .linux => mod.addCMacro("LUA_USE_LINUX", ""), + .freebsd, .netbsd, .openbsd => mod.addCMacro("LUA_USE_POSIX", ""), else => {}, } +} - return b.addLibrary(.{ - .name = "lua", - .linkage = .static, - .root_module = lua_mod, +/// Codegen step: walk the luarocks tarball, emit a Zig module that +/// exposes every Lua source as a `@embedFile`d entry plus a top-level +/// table mapping `require` paths (e.g. `"luarocks.core.cfg"`) to those +/// contents. The runtime installs this table into `package.preload` so +/// `require("luarocks.*")` resolves without touching disk. +/// +/// The generator emits `@embedFile("luarocks/<...>.lua")`-style paths. +/// We then materialize the luarocks source tree alongside the +/// generated Zig file via `addCopyDirectory`, so the embedded paths +/// resolve against the same root the module compiles from. (`@embedFile` +/// only allows reading files inside the module's import-path tree.) +/// +/// Two outputs are present in the resulting module: +/// - `pub const files: []const Entry` — { require_name, contents } +/// - `pub const luarocks_main` / `pub const luarocks_admin_main` +/// for the two `src/bin/` driver scripts. +fn generateLuaAnchor( + b: *std.Build, + lua_src: *std.Build.Dependency, +) std.Build.LazyPath { + const tool = b.addExecutable(.{ + .name = "gen-lua-anchor", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_lua_anchor.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), + }); + + const run = b.addRunArtifact(tool); + run.addDirectoryArg(lua_src.path("src")); + return run.addOutputFileArg("lua_anchor.c"); +} + +fn generateLuarocksEmbed( + b: *std.Build, + luarocks_src: *std.Build.Dependency, +) std.Build.LazyPath { + const tool = b.addExecutable(.{ + .name = "gen-luarocks-embed", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_luarocks_embed.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), }); + + const run = b.addRunArtifact(tool); + // Pass a marker arg the tool will treat as the "embed-relative" + // subpath — every `@embedFile` reference uses this prefix so + // they resolve against the same directory the generated zig file + // lives in (the one we set up next via addWriteFiles). + run.addArg("luarocks_src"); + run.addDirectoryArg(luarocks_src.path("src")); + const gen_file = run.addOutputFileArg("embedded_luarocks.zig"); + + // Stage the generated zig file next to a copy of the luarocks src + // tree, so `@embedFile("luarocks_src/...")` resolves. + const wf = b.addWriteFiles(); + const out_zig = wf.addCopyFile(gen_file, "embedded_luarocks.zig"); + _ = wf.addCopyDirectory(luarocks_src.path("src"), "luarocks_src", .{}); + return out_zig; +} + +/// Codegen step: emit a Zig module that exposes every Lua public header +/// from the Lua source tarball as embedded bytes. The bootstrap stages +/// these under `$PANTO_HOME/rocks/lua-X.Y.Z/include/` so luarocks can +/// compile C rocks against them. +fn generateLuaHeadersEmbed( + b: *std.Build, + lua_src: *std.Build.Dependency, +) std.Build.LazyPath { + const tool = b.addExecutable(.{ + .name = "gen-lua-headers-embed", + .root_module = b.createModule(.{ + .root_source_file = b.path("build/gen_lua_headers_embed.zig"), + .target = b.graph.host, + .optimize = .Debug, + }), + }); + + const run = b.addRunArtifact(tool); + // Marker subpath: each emitted `@embedFile` reference is + // `lua_src/<header>`, and we stage the headers under that name. + run.addArg("lua_src"); + run.addDirectoryArg(lua_src.path("src")); + const gen_file = run.addOutputFileArg("embedded_lua_headers.zig"); + + const wf = b.addWriteFiles(); + const out_zig = wf.addCopyFile(gen_file, "embedded_lua_headers.zig"); + _ = wf.addCopyDirectory(lua_src.path("src"), "lua_src", .{}); + return out_zig; } -// Lua 5.4.7 core VM + standard library. Excludes lua.c (interpreter entry +// Lua 5.4.x core VM + standard library. Excludes lua.c (interpreter entry // point) and luac.c (compiler entry point). const lua_files = [_][]const u8{ // Core VM diff --git a/build.zig.zon b/build.zig.zon index 82794fc..f7eb08b 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -10,6 +10,10 @@ .url = "https://www.lua.org/ftp/lua-5.4.7.tar.gz", .hash = "N-V-__8AAIMvFABt-Qcpk24RD10ldEN743D8Q2e19Er8x3dJ", }, + .luarocks_src = .{ + .url = "https://luarocks.github.io/luarocks/releases/luarocks-3.13.0.tar.gz", + .hash = "N-V-__8AADZhHwD_K21NLLYGySn7TnLhpLyP8ca2JnXSLgbp", + }, }, .paths = .{ "build.zig", diff --git a/build/gen_lua_anchor.zig b/build/gen_lua_anchor.zig new file mode 100644 index 0000000..1143c2c --- /dev/null +++ b/build/gen_lua_anchor.zig @@ -0,0 +1,144 @@ +//! Build-time codegen: scan Lua's `lua.h` and `lauxlib.h` for every +//! `LUA_API` / `LUALIB_API` function declaration, and emit a C source +//! file that takes the address of each into a single `__attribute__ +//! ((used))` array. +//! +//! Result: the linker can't drop those functions even under LTO + +//! gc-sections, because they're reachable from the surviving array. +//! Combined with `rdynamic = true` on the executable, every Lua API +//! function ends up in the dynamic symbol table for C rocks (luv.so +//! etc.) to resolve at `dlopen` time. +//! +//! Invoked from `build.zig`: +//! +//! gen-lua-anchor <lua-src-dir> <out-file> + +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); + const src_dir = args.next() orelse return error.MissingSrcDir; + const out_path = args.next() orelse return error.MissingOutPath; + + var names: std.array_list.Managed([]const u8) = .init(arena); + try collectFrom(arena, io, src_dir, "lua.h", &names); + try collectFrom(arena, io, src_dir, "lauxlib.h", &names); + + // Deduplicate via a string set. + var seen: std.StringHashMapUnmanaged(void) = .empty; + try seen.ensureTotalCapacity(arena, @intCast(names.items.len * 2)); + var unique: std.array_list.Managed([]const u8) = .init(arena); + for (names.items) |n| { + const gop = seen.getOrPutAssumeCapacity(n); + if (!gop.found_existing) try unique.append(n); + } + std.mem.sort([]const u8, unique.items, {}, lessThanString); + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.writeAll( + \\/* Auto-generated by build/gen_lua_anchor.zig. Do not edit. + \\ * + \\ * Holds a reference to every Lua public API function so the + \\ * linker can't drop them under LTO + gc-sections. C rocks + \\ * loaded at runtime (luv.so etc.) resolve these symbols + \\ * against the host process's symbol table; combined with + \\ * rdynamic on the executable, this keeps the whole API + \\ * surface exported. + \\ */ + \\ + \\#include "lua.h" + \\#include "lauxlib.h" + \\ + \\__attribute__((used, visibility("default"))) + \\const void *const _panto_lua_api_anchor[] = { + \\ + ); + for (unique.items) |n| { + try w.print(" (const void *)&{s},\n", .{n}); + } + try w.writeAll("};\n"); + try w.flush(); +} + +fn lessThanString(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); +} + +fn collectFrom( + arena: std.mem.Allocator, + io: std.Io, + src_dir: []const u8, + filename: []const u8, + out: *std.array_list.Managed([]const u8), +) !void { + const path = try std.fs.path.join(arena, &.{ src_dir, filename }); + const contents = try std.Io.Dir.cwd().readFileAlloc(io, path, arena, .unlimited); + + // Walk line by line. When we hit a line starting with `LUA_API`, + // accumulate continuation lines until we see `);`. From the joined + // string, extract the function name \u2014 either parenthesized + // (`(lua_foo)`) or bare (`lua_foo(...)`). + var line_it = std.mem.splitScalar(u8, contents, '\n'); + while (line_it.next()) |first_line| { + const trimmed = std.mem.trimStart(u8, first_line, " \t"); + if (!std.mem.startsWith(u8, trimmed, "LUA_API") and + !std.mem.startsWith(u8, trimmed, "LUALIB_API")) continue; + + var joined: std.array_list.Managed(u8) = .init(arena); + try joined.appendSlice(first_line); + while (std.mem.indexOf(u8, joined.items, ");") == null) { + const next_line = line_it.next() orelse break; + try joined.append(' '); + try joined.appendSlice(next_line); + } + + if (extractName(joined.items)) |name| { + try out.append(try arena.dupe(u8, name)); + } + } +} + +/// Return the function-name portion of a Lua API declaration. Handles +/// both `LUA_API <ret> (lua_foo) (...)` and `LUA_API <ret> lua_foo(...)`. +fn extractName(decl: []const u8) ?[]const u8 { + // Find a `(name)` token first. + var i: usize = 0; + while (i + 1 < decl.len) : (i += 1) { + if (decl[i] != '(') continue; + const end = std.mem.indexOfScalar(u8, decl[i + 1 ..], ')') orelse return null; + const candidate = decl[i + 1 .. i + 1 + end]; + if (isApiName(candidate)) return candidate; + // Skip this candidate; try further on. + } + // Fall back: bare-name pattern (no parens around the name). + var j: usize = 0; + while (j < decl.len) : (j += 1) { + if (j + 4 > decl.len) break; + if (!std.mem.eql(u8, decl[j .. j + 4], "lua_") and + !std.mem.eql(u8, decl[j .. j + 5], "luaL_")) continue; + var end = j; + while (end < decl.len and (std.ascii.isAlphanumeric(decl[end]) or decl[end] == '_')) end += 1; + return decl[j..end]; + } + return null; +} + +fn isApiName(s: []const u8) bool { + if (s.len < 4) return false; + const ok_prefix = std.mem.startsWith(u8, s, "lua_") or std.mem.startsWith(u8, s, "luaL_"); + if (!ok_prefix) return false; + for (s) |ch| { + if (!std.ascii.isAlphanumeric(ch) and ch != '_') return false; + } + return true; +} diff --git a/build/gen_lua_headers_embed.zig b/build/gen_lua_headers_embed.zig new file mode 100644 index 0000000..1ac4004 --- /dev/null +++ b/build/gen_lua_headers_embed.zig @@ -0,0 +1,68 @@ +//! Build-time codegen: emit a Zig module exposing every Lua public +//! header from the Lua source tarball as embedded bytes. Bootstrap +//! writes these to `$PANTO_HOME/rocks/lua-X.Y.Z/include/` on first run +//! so luarocks can compile C rocks against them. +//! +//! Invoked from `build.zig`: +//! +//! gen-lua-headers-embed <lua-src-dir> <out-file> +//! +//! `<lua-src-dir>` is the `src/` directory of the Lua tarball. + +const std = @import("std"); + +const headers = [_][]const u8{ + "lua.h", + "luaconf.h", + "lualib.h", + "lauxlib.h", + // hpp is the C++-friendly include shim shipped upstream; some C + // rocks reference it. + "lua.hpp", +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); + const embed_prefix = args.next() orelse return error.MissingEmbedPrefix; + const src_dir = args.next() orelse return error.MissingSrcDir; + const out_path = args.next() orelse return error.MissingOutPath; + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.writeAll( + \\//! Auto-generated. Do not edit. See build/gen_lua_headers_embed.zig. + \\ + \\pub const Entry = struct { + \\ name: []const u8, + \\ contents: []const u8, + \\}; + \\ + \\pub const files: []const Entry = &.{ + \\ + ); + + for (headers) |name| { + const abs = try std.fs.path.join(arena, &.{ src_dir, name }); + // Verify the file exists; emit nothing for missing files (lua.hpp + // exists in current Lua releases but we don't want to hard-fail + // if the tarball ever drops it). + std.Io.Dir.cwd().access(io, abs, .{}) catch continue; + const embed_path = try std.fs.path.join(arena, &.{ embed_prefix, name }); + try w.print( + " .{{ .name = \"{s}\", .contents = @embedFile(\"{s}\") }},\n", + .{ name, embed_path }, + ); + } + + try w.writeAll("};\n"); + try w.flush(); +} diff --git a/build/gen_luarocks_embed.zig b/build/gen_luarocks_embed.zig new file mode 100644 index 0000000..3f85177 --- /dev/null +++ b/build/gen_luarocks_embed.zig @@ -0,0 +1,157 @@ +//! Build-time codegen: walk the luarocks `src/` tree and emit a Zig +//! module that exposes every Lua file as an `@embedFile` entry, with a +//! comptime table mapping `require`-style module names to contents. +//! +//! Invoked from `build.zig`: +//! +//! gen-luarocks-embed <src-dir> <out-file> +//! +//! `<src-dir>` is `luarocks_src/src` (contains `luarocks/`, `compat53/`, +//! and `bin/`). `<out-file>` is the Zig source to write. +//! +//! The generated file looks like: +//! +//! pub const Entry = struct { name: []const u8, contents: []const u8 }; +//! pub const files: []const Entry = &.{ +//! .{ .name = "luarocks.cmd", .contents = @embedFile("...") }, +//! ... +//! }; +//! pub const luarocks_main: []const u8 = @embedFile("..."); +//! pub const luarocks_admin_main: []const u8 = @embedFile("..."); +//! +//! The `@embedFile` paths are absolute paths into the Zig cache. That's +//! safe because the codegen step depends on the luarocks dependency, so +//! the cache entry exists by the time the generated source is compiled. + +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const arena = init.arena.allocator(); + const io = init.io; + + var args = init.minimal.args.iterate(); + defer args.deinit(); + _ = args.next(); // skip program name + const embed_prefix = args.next() orelse return error.MissingEmbedPrefix; + const src_dir_arg = args.next() orelse return error.MissingSrcDir; + const out_path_arg = args.next() orelse return error.MissingOutPath; + + // Walk src/luarocks and src/compat53 collecting *.lua files; pick + // up src/bin/luarocks and src/bin/luarocks-admin separately. + var entries: std.array_list.Managed(Entry) = .init(arena); + try collect(arena, io, src_dir_arg, embed_prefix, "luarocks", &entries); + try collect(arena, io, src_dir_arg, embed_prefix, "compat53", &entries); + + // Sort for deterministic output. + std.mem.sort(Entry, entries.items, {}, Entry.lessThan); + + const bin_luarocks = try std.fs.path.join(arena, &.{ embed_prefix, "bin/luarocks" }); + const bin_luarocks_admin = try std.fs.path.join(arena, &.{ embed_prefix, "bin/luarocks-admin" }); + + var out_file = try std.Io.Dir.cwd().createFile(io, out_path_arg, .{}); + defer out_file.close(io); + var buf: [4096]u8 = undefined; + var writer = out_file.writer(io, &buf); + const w = &writer.interface; + + try w.writeAll( + \\//! Auto-generated. Do not edit. See build/gen_luarocks_embed.zig. + \\ + \\pub const Entry = struct { + \\ /// Lua `require` path, e.g. "luarocks.core.cfg". + \\ name: []const u8, + \\ /// File contents (Lua source). + \\ contents: []const u8, + \\}; + \\ + \\pub const files: []const Entry = &.{ + \\ + ); + + for (entries.items) |e| { + try w.print( + " .{{ .name = \"{s}\", .contents = @embedFile(\"{s}\") }},\n", + .{ e.require_name, e.embed_path }, + ); + } + + try w.writeAll("};\n\n"); + try w.print( + "pub const luarocks_main: []const u8 = @embedFile(\"{s}\");\n", + .{bin_luarocks}, + ); + try w.print( + "pub const luarocks_admin_main: []const u8 = @embedFile(\"{s}\");\n", + .{bin_luarocks_admin}, + ); + + try w.flush(); +} + +const Entry = struct { + require_name: []const u8, + embed_path: []const u8, + + fn lessThan(_: void, a: Entry, b: Entry) bool { + return std.mem.lessThan(u8, a.require_name, b.require_name); + } +}; + +/// Walk `<root>/<subdir>` recursively, collecting every `.lua` file. The +/// `require` name is computed as the path relative to `<root>` with `/` +/// replaced by `.` and `.lua` stripped, then `/init.lua` collapsed to +/// the parent directory. +fn collect( + arena: std.mem.Allocator, + io: std.Io, + root_abs: []const u8, + embed_prefix: []const u8, + subdir: []const u8, + out: *std.array_list.Managed(Entry), +) !void { + const start_abs = try std.fs.path.join(arena, &.{ root_abs, subdir }); + var dir = std.Io.Dir.cwd().openDir(io, start_abs, .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => return, + else => return err, + }; + defer dir.close(io); + + var walker = try dir.walk(arena); + defer walker.deinit(); + + while (try walker.next(io)) |entry| { + if (entry.kind != .file) continue; + if (!std.mem.endsWith(u8, entry.basename, ".lua")) continue; + + // `embed_path` lives under the staged tree (`<embed_prefix>/...`) + // so `@embedFile` can resolve it relative to the generated zig. + const embed_path = try std.fs.path.join( + arena, + &.{ embed_prefix, subdir, entry.path }, + ); + const rel = try std.mem.concat(arena, u8, &.{ subdir, "/", entry.path }); + + // Compute `require` name: drop ".lua", convert `/` to `.`. + // + // We deliberately do NOT collapse `/init.lua` into the parent + // module name. Lua's stock path searcher checks both + // `foo.lua` and `foo/init.lua` when you `require("foo")`, so + // the on-disk layout would resolve fine without our help — + // and luarocks's own tree contains both `luarocks/cmd.lua` + // (the package) and `luarocks/cmd/init.lua` (the subcommand + // module named `luarocks.cmd.init`). Collapsing would alias + // them onto one name. The embedded searcher serves whichever + // name appears literally in the source tree; the standard + // searcher behavior emerges from cmd.lua being a sibling. + const stem = rel[0 .. rel.len - ".lua".len]; + const require_name = try arena.alloc(u8, stem.len); + for (stem, 0..) |ch, i| { + require_name[i] = if (ch == '/') '.' else ch; + } + + try out.append(.{ + .require_name = require_name, + .embed_path = embed_path, + }); + } +} diff --git a/build/panto_lua_repl.c b/build/panto_lua_repl.c new file mode 100644 index 0000000..ce99cb1 --- /dev/null +++ b/build/panto_lua_repl.c @@ -0,0 +1,49 @@ +/* + * panto's hook into the upstream `lua.c` standalone interpreter. + * + * We need to call into `pmain` (the body of the standalone REPL/driver) + * with a pre-configured `lua_State` — specifically, one where panto's + * embedded-luarocks searcher and configured `package.path` have already + * been installed. Upstream `pmain` is `static`, so we can't link to it + * from Zig directly. + * + * Trick: include `lua.c` verbatim with `static` redefined to empty, so + * every static gets external linkage. We rename `main` to a stub via + * macro so it doesn't conflict with panto's own `main`. Then we expose + * a single function, `panto_lua_pmain`, that runs `pmain` against a + * caller-provided state. + * + * `lua.c` also defines its own signal handlers via `setsignal` / + * `laction`. We leave those alone — they're useful in `panto lua` too, + * giving Ctrl-C the same behavior it has in upstream `lua`. + */ + +/* Remove `static` everywhere lua.c uses it so symbols are externally + * accessible. We undef `static` back to default for any system headers + * that follow, in case they care. */ +#define static +#include "lua.c" +#undef static + +/* Forward declarations of the (now non-static) symbols we use. */ +extern int pmain(lua_State *L); + +/* + * Trampoline: push (argc, argv) onto `L` and run `pmain` in protected + * mode, mirroring `lua.c::main`'s call pattern. Returns the exit code + * appropriate for `lua` (0 on success, 1 on failure). + * + * The caller is responsible for state lifecycle (lua_close, etc.) — + * we don't open or close `L` ourselves. This is what lets panto's + * subcommand install the embedded-luarocks searcher into `L` *before* + * calling us. + */ +int panto_lua_pmain(lua_State *L, int argc, char **argv) { + lua_pushcfunction(L, &pmain); + lua_pushinteger(L, argc); + lua_pushlightuserdata(L, argv); + int status = lua_pcall(L, 2, 1, 0); + int result = lua_toboolean(L, -1); + report(L, status); + return (result && status == LUA_OK) ? 0 : 1; +} diff --git a/docs/archive/LUA_MAKEOVER.md b/docs/archive/LUA_MAKEOVER.md new file mode 100644 index 0000000..f701a29 --- /dev/null +++ b/docs/archive/LUA_MAKEOVER.md @@ -0,0 +1,594 @@ +# Lua Runtime Makeover + +Side project, separate from the main phase plan. Reworks panto's Lua +embedding into something that fits Lua's actual concurrency model and +opens the door to a luarocks-based extension ecosystem. + +Phase 3 as shipped is fine and stays. This document describes what +replaces it. + +## Why + +The current Lua embedding has three problems, in increasing order of +how badly they constrain us: + +1. **One `lua_State` per tool call.** Every `LuaTool.invoke` builds and + tears down a fresh interpreter. Module-global state is impossible. + Top-level extension code runs over and over. The API only makes + sense for stateless single-shot handlers, which is not how Lua + wants to be used. +2. **The `Tool` contract is "thread-safe."** Right for native + extensions; wrong for Lua. Lua's concurrency primitive is the + coroutine, not the OS thread. A single `lua_State` is not safe for + concurrent host entry, so the only way to honor "thread-safe" with + Lua is one state per call — which we have, badly. +3. **No path to a real Lua ecosystem.** The current setup discovers + `.lua` files on disk and that's it. There's no answer to "how do + extension authors depend on lua-cjson" or "how does an HTTP-using + tool work." The eventual answer to both is luarocks, and the + current architecture has no place for it. + +## Shape of the fix + +Three independent pieces that compose: + +1. **`ToolSource` in libpanto.** A new kind of registration alongside + `Tool`. A source owns one or more tools and receives all calls + targeting them, as a batch, on one thread per turn. Different + sources still run in parallel. Lua becomes one source. +2. **Long-lived `lua_State` with cooperative scheduling.** The panto + CLI maintains exactly one `lua_State` for its entire lifetime. + Extension top-level code runs once at startup. Each tool call is a + coroutine. A libuv event loop drives suspended coroutines. +3. **luarocks as the package manager.** Both panto's own runtime + battery (luv — see note below) and user-installed extensions + come through luarocks, installed into a tree under + `$XDG_DATA_HOME/panto/`. luarocks itself is embedded in the panto + binary as Lua source. + + Originally planned to ship `coro-*` (coro-fs, coro-http, coro-net, + coro-channel, coro-spawn) as additional batteries; that plan was + dropped during implementation. luv alone provides the libuv + surface (callback-shaped, but coroutine-friendly with a `co = + coroutine.running(); ...; coroutine.yield()` pattern); shipping + the smallest possible set keeps the makeover focused and lets us + write panto's own `std` tools directly against luv. Users who + want coro-* helpers can install them via `panto lua -e ...` + against the embedded luarocks. + +## libpanto: `ToolSource` + +Native tools keep the existing `Tool` API unchanged. Adapters that +back multiple tools through a shared runtime use the new `ToolSource`: + +```zig +pub const ToolSource = struct { + name: []const u8, // diagnostic only ("panto-lua") + tools: []const Tool.Decl, // metadata only; no per-tool vtable + ctx: *anyopaque, + vtable: *const VTable, + + pub const VTable = struct { + /// libpanto guarantees: for a given turn, all ToolUse calls + /// whose tool name belongs to this source are delivered in + /// one invoke_batch call, on one thread. Different sources + /// still execute in parallel. + /// + /// The source decides internal scheduling (coroutines, + /// sequential, internal worker pool). + invoke_batch: *const fn ( + ctx: *anyopaque, + calls: []const Call, + results: []CallResult, // parallel array, pre-allocated + allocator: Allocator, + ) anyerror!void, + + deinit: *const fn (ctx: *anyopaque, allocator: Allocator) void, + }; + + pub const Call = struct { + tool_name: []const u8, + input: []const u8, + }; + + pub const CallResult = union(enum) { + ok: []u8, // owned by allocator + err: anyerror, + }; +}; + +pub const Tool.Decl = struct { + name: []const u8, + description: []const u8, + schema_json: []const u8, +}; +``` + +`ToolRegistry` indexes by tool name with a tagged value: +`{ .single: *Tool }` or `{ .source: *ToolSource, .tool_index: usize }`. + +`Agent.registerToolSource(src: ToolSource) !void` is the new entry +point. + +### Agent loop change + +In `runStep`, after collecting ToolUse blocks for a turn: + +1. Group them by owning source. Single-`Tool` entries form + single-entry groups. +2. Spawn one OS thread per group. +3. Each thread calls either `tool.vtable.invoke` (single) or + `source.vtable.invoke_batch` (batched). +4. Join. Assemble ToolResult blocks in the original order. + +**Concurrency contract becomes:** different groups run in parallel; a +single group is the source's problem. The "thread-safe" promise still +holds for native `Tool`s. For Lua, it relaxes to "coroutine-safe +within the panto-lua runtime." + +## Lua runtime + +### One `lua_State`, many coroutines, one event loop + +The panto CLI creates a single `lua_State` at startup. Every Lua +extension is loaded into it. All top-level extension code runs once. +Module-global state is real and persistent across calls. + +When `invoke_batch` fires for the panto-lua source: + +``` +1. for each call: coroutine.create(handler), coroutine.resume(co, args) +2. uv.run() until all coroutines have completed (or errored) +3. collect results into the CallResult array +``` + +That's the entire scheduler. The work happens inside libuv: when a +coroutine calls a yield-aware libuv operation (HTTP, fs, subprocess, +sleep) it suspends; libuv resumes it when the event fires. + +A wrapper layer translates libuv's callback shape into coroutine +yields. The Luvit project's `coro-*` modules (coro-fs, coro-net, +coro-http, coro-channel, coro-spawn) do this upstream for the common +operations. Where they don't cover something, we write small wrappers +in panto's own Lua code. The pattern is ~10 lines: + +```lua +local function fs_open(path, flags, mode) + local co = coroutine.running() + uv.fs_open(path, flags, mode, function(err, fd) + coroutine.resume(co, err, fd) + end) + return coroutine.yield() +end +``` + +### What this gets us + +- True cooperative parallel I/O within a batch. Three concurrent + `web_fetch` calls go through three concurrent sockets; total + latency is `max(req1, req2, req3)`, not sum. +- First-class async subprocess. A `bash`-style tool that runs three + commands at once does it without blocking the runtime. +- Module-global state for extensions that want it (rate limiters, + caches, lazy connection pools, etc.). +- Extension top-level code runs once. Initialization is real. + +### The honest caveat + +Cooperative scheduling only helps when handlers yield. A handler that +calls a non-yielding C function — raw `os.execute`, `io.read` against +a slow file, an FFI call — blocks its siblings until it returns. +Document loudly. The escape hatch is "use native extensions for +work that can't yield." + +This is the same trade Python's asyncio makes with `requests` vs +`aiohttp`. Panto's recommended posture: handlers should use libuv via +the coro-* wrappers (or other luv-aware libraries) for I/O. Pure +compute is fine. Calling `socket.http.request` or `os.execute` will +work but blocks the batch. + +### Why libuv (not cqueues) + +Considered cqueues + lua-http. Better HTTP story (HTTP/2), +coroutine-native API. Lost on **subprocess**, which has no +maintained cqueues binding and which matters a lot for a coding +agent. Also lost on familiarity — luv is what every Neovim user has +seen. + +Trade accepted: HTTP only via luv's lower-level TCP + manual +framing (or a user-installed rock), and a callback-shaped async +surface rather than a coroutine-native one. Coroutine ergonomics +are an extension-author concern, not a runtime concern. + +## luarocks as the package manager + +### Distribution model + +luarocks is embedded in the panto binary as `@embedFile`'d Lua +source. At startup the runtime: + +1. Computes `$PANTO_HOME = $XDG_DATA_HOME/panto` (default + `~/.local/share/panto`). +2. Configures the embedded `lua_State`'s `package.path` and + `package.cpath` to look under `$PANTO_HOME/share/lua/5.4/` and + `$PANTO_HOME/lib/lua/5.4/`. +3. Bootstraps the embedded luarocks against + `--tree=$PANTO_HOME`. +4. Reconciles a "runtime batteries" manifest (currently just + luv) — installs missing rocks, no-ops if present. +5. Iterates user extensions from config. For `luarocks:foo`-style + references, ensures `foo` is installed. +6. Hands control to the agent loop. + +luarocks 3.12+ no-ops on already-installed rocks and caches the +upstream manifest, so steps 4–5 are cheap on every run after the +first. + +Network failures on later runs are swallowed: the rocks are already +there. First-run-with-no-network degrades to "no Lua extensions +work" — native panto features and the agent loop are unaffected. + +### Why fully luarocks-based + +- One mechanism for everything Lua-distribution. Runtime batteries + and user extensions install the same way. +- Single small binary. No vendored libuv, no vendored luv, no + `@embedFile`'d coro-* sources. luarocks itself is ~1MB of Lua + source that compresses well. +- Version flexibility for batteries without re-shipping panto. +- Matches Neovim's rocks.nvim direction — the most relevant + ecosystem signal for "Lua as a serious distribution target." +- User extension story is genuinely the same as the batteries + story. No special cases. + +### Distributable artifact + +Single `panto` binary contains: + +- Zig CLI + libpanto +- Lua 5.4 (already vendored) +- luarocks Lua source, embedded via `@embedFile` +- A small Zig-side bootstrap that configures `package.path` for the + embedded luarocks code + +Everything else lives under `$PANTO_HOME` and is installed on first +run. + +## Migration shape + +Independent chunks of work, roughly in order: + +1. **`ToolSource` in libpanto.** ✅ **Done.** Type lives in + `libpanto/src/tool_source.zig`; registry tagging in + `tool_registry.zig`; per-group `std.Io.Group` fan-out in + `agent.zig`. Native `Tool` API unchanged. +2. **Long-lived `lua_State` runtime in the CLI.** ✅ **Done.** + `src/lua_runtime.zig` owns one `lua_State` for the process + lifetime, loads all discovered extensions once, stores handlers + as `luaL_ref` slots, and runs each call as a coroutine. + Registers itself with libpanto as the `panto-lua` `ToolSource`. + Step 5 added the libuv-driven scheduler that makes yields + productive. +3. **Embed luarocks.** ✅ **Done.** `build.zig.zon` fetches + luarocks 3.13.0 source. `build/gen_luarocks_embed.zig` enumerates + the Lua sources at build time and produces a Zig module that + `@embedFile`s each entry. `src/luarocks_runtime.zig` installs an + embedded-source `package.searcher` so `require("luarocks.*")` + resolves without disk I/O, injects `luarocks.core.hardcoded`, + configures `package.path`/`cpath`, stages Lua headers under + `<tree>/include/`, writes `<tree>/etc/luarocks/config-5.4.lua`, + and materializes a `<tree>/bin/lua` shell wrapper that `exec`s + `panto lua` (the interpreter luarocks shells out to for rockspec + build scripts). +4. **Install luv as the first battery.** ✅ **Done.** Manifest + pins `luv 1.52.1-0` (`src/manifest.zig`). Bootstrap reconciles + in-process by `load`ing the embedded `src/bin/luarocks` driver + and calling it with `install <name> <version>` args. luv's + rockspec bundles libuv as a git submodule and builds it via + CMake, so first-run toolchain requirement is `cc`, `make`, and + `cmake`. Subsequent runs detect the version-stamped install + metadata under `<tree>/lib/luarocks/rocks-5.4/<name>/<version>/` + and no-op fast. +5. **Cooperative scheduler around `uv.run()`.** ✅ **Done.** + `LuaRuntime.installScheduler` registers `panto._record_result` + (a C function with the runtime pointer as upvalue), installs a + wrapper closure that runs handlers under `pcall` and reports + results, and caches `require("luv").run` in the registry. The + new `runBatch` path creates one coroutine per call, resumes + each once, then ticks `uv.run("once")` between resumes until + every coroutine has terminated. Coroutines that yield without + any pending libuv handle are surfaced as a deadlock error. + + We deliberately ship no `coro-*` helpers — luv is the entire + async surface. Extension authors `require("luv")` and use its + native callback-shaped APIs; `panto lua -e ...` (plus the + embedded luarocks) is the escape hatch for anything else. +6. ~~User extension config syntax.~~ **Out of scope for this + makeover.** Deferred; see Q7 "Future work". +7. **Delete `LuaTool` and per-call `lua_State` machinery.** ✅ + **Done** as part of step 2 — `LuaTool` and `LuaStatePool` are + gone from the tree. + +Additional work surfaced by Q1–Q7 that lands alongside steps 3–5: + +- **`panto lua` subcommand.** ✅ **Done.** `build/panto_lua_repl.c` + includes upstream `lua.c` with `#define static` stripped so we + can call `pmain` directly. `src/subcommand.zig` opens a fresh + `lua_State`, runs the same bootstrap pipeline as `panto run`, + then hands the state to `panto_lua_pmain(L, argc, argv)` so + upstream argv handling (`-i`, `-l`, `-e`, `script.lua args...`) + works verbatim. (Q4) +- **`panto bootstrap` subcommand.** ✅ **Done.** Same bootstrap path + as the agent and `panto lua`; just exits cleanly afterwards + instead of entering an interactive surface. Idempotent: a clean + tree on subsequent runs no-ops in milliseconds. (Q2) + + Supports `--force`: wipes the per-Lua-version tree + (`<home>/rocks/lua-X.Y.Z/`) before running the regular bootstrap. + Useful for recovering from a corrupted installation or testing + the cold-start path. Sibling trees from other Lua versions are + not touched. +- **Batteries manifest + reconcile.** ✅ **Done.** `src/manifest.zig` + ships pinned versions; `reconcileBatteries` runs in-process by + `load`ing the embedded luarocks driver, calling its `cmd.run_command` + with `install <name> <version>` for any rock that isn't already + on disk. (Q5) +- **Lua headers staged at `$PANTO_HOME/rocks/lua-X.Y.Z/include/`.** + ✅ **Done.** `build/gen_lua_headers_embed.zig` embeds the headers + from the Lua source tarball at build time; `stageLuaHeaders` in + `luarocks_runtime.zig` writes them to disk on first run and + when the contents differ (e.g. a panto upgrade that bumped + the bundled Lua version). (Q3) + +Documentation updates: phase-3.md gains a "superseded by +LUA_MAKEOVER.md for the Lua runtime; native extension contract +unchanged" note. The contract for native extensions ("thread-safe") +stays as-is. + +## Open questions + +These came up during design and need resolution before +implementation. We'll edit answers in here as decisions land. + +### Q1: luarocks's own C dependencies + +luarocks attempts to `require` several optional Lua modules via +`pcall` and falls back to shelling out or to its own pure-Lua +implementations when they're missing. The optional set: + +- **LuaSocket** (`socket.http`, `socket.ftp`) — for HTTP/FTP + downloads. Without it: shell out to a configured downloader + (`curl` or `wget`). +- **LuaSec** (`ssl.https`) — for HTTPS. Without it *and* without + the LuaSocket+luarocks-internal HTTPS path: must use `curl` or + `wget` for HTTPS. **luarocks.org is HTTPS-only**, so this is + effectively mandatory in some form. +- **LuaFileSystem** (`lfs`) — for directory operations, + `chdir`, file attributes. Without it: degraded fallbacks using + only `io.*` and `os.*`. Some operations become impossible. +- **lua-bz2** (`bz2`) — for `.bz2` archives. Almost never + encountered; rocks ship as `.tar.gz` or `.zip`. +- **LuaPosix** (`posix`) — for chmod and other POSIX ops. + Without it: fall back to shelling out to `chmod` etc. +- **md5** — for checksums. luarocks has a pure-Lua fallback. +- **`luarocks.tools.zip`** (bundled with luarocks itself) — pure + Lua zip/gzip. No external dependency. +- **`luarocks.tools.tar`** (bundled) — pure Lua tar. + +**Correction:** there is no `--with-lua=embedded` flag — that was a +hallucination from earlier in the design conversation. luarocks's +`./configure` accepts `--with-lua=DIR`, `--with-lua-include=DIR`, +`--with-lua-lib=DIR`, `--with-lua-interpreter=NAME`, +`--lua-version=VERSION`. These point luarocks at *a* Lua install +(which can be ours under `$PANTO_HOME`); they don't enable a +separate "embedded" mode. + +**Decision:** minimize luarocks's optional Lua deps. Bootstrap runs +luarocks in its degraded-but-functional mode using its bundled +pure-Lua `tools.zip` and `tools.tar`, plus shell-out to `curl` (or +`wget`) for HTTPS downloads. If any of the optional deps turn out +to be effectively mandatory in practice, we statically link the +native C library and embed the Lua wrapper as `@embedFile` in +`panto` (sibling to Lua itself in `build.zig`). Likely candidates: +LuaFileSystem (small, pure C wrapper around POSIX, very widely +used), and possibly LuaSocket+LuaSec if shelling out to `curl` +proves too clunky. + +We also depend on `curl` (or `wget`) being on PATH for downloads. +This is universal on Unix dev machines and we accept the +dependency. If it's missing, bootstrap surfaces a clear error. + +### Q2: C toolchain on first run + +luv has a C component. Building it requires `cc`, `make`, and Lua +headers. On dev machines (panto's target audience) these are +universal. On bare end-user machines they aren't. + +**Decision:** add a `panto bootstrap` subcommand. It's effectively +a no-op `panto` invocation that exercises the same +fetch-and-install path that every normal startup runs, just +without entering the agent loop afterwards. On a clean machine +it's where the slow first-run download-and-compile happens with +visible output; on subsequent runs it's a fast no-op equivalent +to what every `panto` startup already does. + +Every normal `panto` startup runs the same sync logic. The +`bootstrap` subcommand isn't a *separate* mechanism — it's a way +to run the sync explicitly without the agent loop, for users who +want to do setup ahead of time, or for CI/scripted installs. + +When the toolchain is missing, surface a friendly error from +the bootstrap code path before luarocks itself barfs. "You need +a C compiler and make installed to compile Lua extensions" or +similar. Native panto features keep working regardless — only +the Lua tool runtime is gated on successful bootstrap. + +### Q3: Lua headers for C rocks + +Building C rocks against panto's embedded Lua requires the Lua +headers to be findable on disk. + +**Decision:** drop the headers into `$PANTO_HOME/rocks/lua-X.Y.Z/include/` +at bootstrap time, where `X.Y.Z` is the Lua version panto is +built against. Embed the header sources via `@embedFile` (they're +already available via the `lua_src` build dep). Bootstrap writes +them out on first run and on any panto upgrade that changes the +Lua version. + +**Why a versioned subdirectory:** rocks compiled against Lua +5.4.7's headers are not safe to load into a Lua 5.5 interpreter +(ABI changes happen across minor versions). The whole rock tree +lives under `$PANTO_HOME/rocks/lua-X.Y.Z/` and each Lua version +gets its own. A panto upgrade that bumps Lua creates a new tree +and reinstalls everything against it. The old tree is left in +place for rollback; users (or a future `panto gc` command) can +delete stale ones. + +Directory layout: + +``` +$PANTO_HOME/ + rocks/ + lua-5.4.7/ + include/ ← Lua headers + share/lua/5.4/ ← installed pure-Lua rocks + lib/lua/5.4/ ← installed C rocks + ...luarocks metadata... + lua-5.5.0/ ← after a future upgrade + ... +``` + +luarocks is invoked with `--tree=$PANTO_HOME/rocks/lua-5.4.7` and +configured (via its config file or CLI flags) to know that the +Lua headers live at `$PANTO_HOME/rocks/lua-5.4.7/include/`. The +tree contains everything needed for that Lua version including +the headers, which keeps rebuilds reproducible and rollback +clean. + +### Q4: The `lua` interpreter that luarocks expects on PATH + +luarocks uses an external `lua` binary for some operations (running +rockspec build scripts, primarily). It needs to be on PATH and +needs to be the same version as the embedded interpreter. + +**Decision:** `panto lua` is a first-class user-visible subcommand +that wraps the **upstream `lua.c` standalone interpreter** with +panto's environment pre-configured. + +Mechanically: + +- Compile `lua.c` (the upstream standalone interpreter, ~600 lines) + into the panto binary as a subcommand entry point. Currently + excluded from `lua_files` in `build.zig`; include it for the + `lua` subcommand path. +- `panto lua` arguments pass through to `lua.c`'s normal + command-line handling (`-i`, `-l`, `-e`, `script.lua args...`, + etc.). Full standalone-interpreter behavior, not a luarocks-only + subset. +- Before handing control to `lua.c`'s main, panto's subcommand + setup runs the same bootstrap as `panto run` (verify batteries + installed, install missing rocks, configure `package.path` / + `package.cpath` to find `$PANTO_HOME/rocks/lua-X.Y.Z/...`). +- Configure luarocks (via its config file written to + `$PANTO_HOME/rocks/lua-X.Y.Z/config.lua`) to use + `<absolute panto path> lua` as its Lua interpreter. luarocks's + `--with-lua-interpreter=...` flag accepts an executable name; + we either symlink or use the full argv mechanism. + +This gives users a real `lua` they can use to test their +extensions in panto's environment — `require "luv"` and +`require "coro-http"` work, plus anything else they've installed +via `panto lua -e 'require("luarocks.cmd").run(...)'` or similar. + +**Side benefit (Q7-related):** until we have a proper user-facing +`panto rocks install foo` command, `panto lua` is also the user's +escape hatch for installing extra rocks into `$PANTO_HOME`. We +can invoke luarocks itself through it. + +### Q5: Reproducibility / lockfile + +luarocks installs latest-matching by default. For a CLI tool we +want reproducible: the same panto version installs the same +battery versions on every fresh machine. + +**Decision:** pin exact versions in a panto-internal manifest +shipped with the binary. No user-facing `panto lock` or +`panto sync` commands — sync is what every startup (and +`panto bootstrap`) does automatically. + +A manifest file (likely `runtime-batteries.zon` or similar in the +panto source tree) lists exact versions: + +```zig +.{ + .lua_version = "5.4.7", + .luarocks_version = "3.12.2", + .batteries = .{ + .luv = "1.51.0-1", + .{ .@"coro-fs" = "3.0.4-1" }, + .{ .@"coro-http" = "3.2.1-1" }, + .{ .@"coro-net" = "3.2.1-1" }, + .{ .@"coro-channel" = "3.0.4-1" }, + .{ .@"coro-spawn" = "3.2.1-1" }, + }, +} +``` + +Bumping any of these is a deliberate edit + commit + version bump +of panto itself. Each panto release pins one consistent set. + +Bootstrap reads the embedded manifest and ensures the tree matches: +any rock not present at the pinned version gets installed; any +stale versions get removed. A panto upgrade that bumps Lua +creates an entirely new tree (per Q3) and installs everything +fresh against it. + +### Q6: Where in the agent loop does the runtime live + +**Decision:** CLI-side. libpanto continues to be native-only and +Lua-unaware. + +The long-lived `lua_State` is constructed in the panto CLI's +`main` (or a module it calls) before the `Agent` is built. +Bootstrap runs first, then the runtime loads all discovered +Lua extensions into the state, then the runtime registers itself +with the `Agent` as a single `ToolSource` named `panto-lua`. +The source's `ctx` holds the runtime; the `lua_State` lives +inside it. + +libpanto's only concept is `Tool` and `ToolSource`. It has no +idea that one of its sources happens to be Lua-backed. + +### Q7: What "user extension" actually means in the new world + +**Decision (for now):** keep the phase-3 directory-based discovery +as the only user extension mechanism. Local `.lua` files in +`~/.config/panto/extensions/` and `./.panto/extensions/`. No +config file, no `luarocks:foo` references yet. + +Directory-discovered extensions get access to whatever's in +`$PANTO_HOME/rocks/lua-X.Y.Z/` — the runtime batteries (luv, +coro-*, future additions) and nothing else by default. They can +`require` any of those modules and pure-Lua code they write +themselves; that's the supported surface. + +**Escape hatch for extra rocks:** users who need a third-party +Lua library (lua-cjson, lpeg, etc.) for their local extension +can install it manually via `panto lua` + the embedded luarocks: + +``` +panto lua -e 'require("luarocks.cmd").run("install", "lua-cjson")' +``` + +or more sugared if we feel like making that look better. The +rock lands in `$PANTO_HOME/rocks/lua-X.Y.Z/` and survives across +panto runs. + +**Future work (not part of this makeover):** a proper config +file with `luarocks:panto-subagents`-style references, where +panto-published extensions can declare their own Lua library +dependencies in their rockspec and bootstrap installs the +whole graph automatically. The infrastructure built here +(luarocks-as-runtime, `$PANTO_HOME` tree, version pinning) +directly enables this — it's just an orthogonal piece of +user-facing surface that hasn't been designed yet. diff --git a/ideas.md b/docs/archive/ideas.md index e83d543..e83d543 100644 --- a/ideas.md +++ b/docs/archive/ideas.md diff --git a/docs/phase-1.md b/docs/archive/phase-1.md index 6b149aa..6b149aa 100644 --- a/docs/phase-1.md +++ b/docs/archive/phase-1.md diff --git a/docs/phase-2.md b/docs/archive/phase-2.md index 4cb7ed9..4cb7ed9 100644 --- a/docs/phase-2.md +++ b/docs/archive/phase-2.md diff --git a/docs/phase-3.md b/docs/archive/phase-3.md index eb52ee9..eb52ee9 100644 --- a/docs/phase-3.md +++ b/docs/archive/phase-3.md diff --git a/src/lua_runtime.zig b/src/lua_runtime.zig index fb00951..71e4ae7 100644 --- a/src/lua_runtime.zig +++ b/src/lua_runtime.zig @@ -58,6 +58,23 @@ pub const LuaRuntime = struct { /// the Lua registry (`luaL_ref` index). handlers: std.StringHashMap(c_int), + /// Registry ref to the wrapper closure that runs a user handler + /// inside a `pcall` and reports the result back to Zig via + /// `panto._record_result`. Allocated once at `create`; reused for + /// every call. `0` until `installScheduler` runs. + wrapper_ref: c_int = 0, + /// Registry ref to `require("luv").run`, the function we call to + /// tick libuv between coroutine resumes. `0` until + /// `installScheduler` runs. + uv_run_ref: c_int = 0, + + /// Pointer to the in-flight batch, valid only for the duration of + /// one `invoke_batch` call. The `panto._record_result` C function + /// writes through this. `null` between batches; not concurrently + /// accessible (libpanto's source-grouped dispatch guarantees one + /// thread per source per turn). + current_batch: ?*BatchState = null, + /// Create a new runtime. The `lua_State` is opened, standard libs /// loaded, and the `panto.register_tool` bridge installed. pub fn create(allocator: Allocator) !*LuaRuntime { @@ -79,6 +96,22 @@ pub const LuaRuntime = struct { return self; } + /// Install the libuv-driven coroutine scheduler: + /// - Register `panto._record_result` (C function with `self` as + /// light-userdata upvalue) so the wrapper closure can hand + /// results back to Zig. + /// - Create the wrapper closure that runs a user handler in + /// `pcall` and reports the result. + /// - Cache `require("luv").run` for fast per-tick access. + /// + /// Must be called after luarocks bootstrap has installed luv, + /// otherwise the `require("luv")` step will fail. + pub fn installScheduler(self: *LuaRuntime) !void { + try installRecordResult(self); + try installWrapperClosure(self); + try cacheUvRun(self); + } + /// Tear down the runtime: free every owned string, unref every /// handler, close the Lua state. pub fn deinit(self: *LuaRuntime) void { @@ -90,6 +123,12 @@ pub const LuaRuntime = struct { c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, entry.value_ptr.*); } self.handlers.deinit(); + if (self.wrapper_ref != 0) { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, self.wrapper_ref); + } + if (self.uv_run_ref != 0) { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, self.uv_run_ref); + } c.lua_close(self.L); @@ -320,6 +359,52 @@ fn deinitSrc(_: *anyopaque, _: Allocator) void { // libpanto's source.deinit here is a no-op. } +// =========================================================================== +// Scheduler: libuv-driven cooperative coroutine dispatch +// =========================================================================== +// +// libpanto's `invoke_batch` delivers all of a turn's tool-call requests +// at once, on a single thread. We answer the contract by running each +// call as a Lua coroutine inside our long-lived `lua_State`, then +// driving `uv.run("once")` to wake any of those coroutines that are +// blocked on libuv-aware I/O. This is the entire scheduler — luv's +// libuv binding does the actual event-loop work; we just resume +// coroutines and call `run` between resumes. +// +// Capturing return values requires a wrapper. When a coroutine is +// resumed by a luv callback after yielding, the eventual return value +// of the coroutine flows back to *that callback*, not to us. So we +// install a Lua wrapper closure that does +// +// pcall(handler, input) → panto._record_result(idx, ok, val) +// +// before the handler returns. `_record_result` is a C function that +// stores into a per-runtime `BatchState`, accessed via a light-userdata +// upvalue carrying the runtime pointer. + +/// One coroutine's outcome, recorded by `_record_result` and read by +/// `invokeBatch` once the coroutine has terminated. +const Slot = struct { + /// Set true the moment `_record_result` writes a result for this + /// index. Used to detect coroutines that terminated without + /// calling the wrapper (a bug / API misuse). + recorded: bool = false, + /// `true` if the handler returned cleanly, `false` if it raised + /// via the `pcall` wrapping. + ok: bool = false, + /// Result payload as owned bytes. Allocated from `allocator`. + /// Caller frees. + value: ?[]u8 = null, + /// On `ok = false`, an owned copy of the error message. + err_msg: ?[]u8 = null, +}; + +/// State shared between Zig and the in-flight Lua wrapper closure. +const BatchState = struct { + allocator: Allocator, + slots: []Slot, +}; + fn invokeBatch( ctx: *anyopaque, calls: []const panto.ToolCall, @@ -327,21 +412,205 @@ fn invokeBatch( allocator: Allocator, ) anyerror!void { const self: *LuaRuntime = @ptrCast(@alignCast(ctx)); + return runBatch(self, calls, results, allocator); +} - // Step 2 of LUA_MAKEOVER.md: no batteries yet — each call is run - // as a coroutine, but the scheduler doesn't drive an event loop. - // A handler that yields with no batteries available has nothing - // to wake it; we surface that as `LuaHandlerYielded`. - // - // Once `luv` and the `coro-*` wrappers are installed, this loop - // becomes "drive uv.run() until every coroutine is dead/erroring, - // then collect results". +fn runBatch( + self: *LuaRuntime, + calls: []const panto.ToolCall, + results: []panto.ToolCallResult, + allocator: Allocator, +) !void { + if (self.wrapper_ref == 0 or self.uv_run_ref == 0) { + // Scheduler not installed. We can still run synchronous + // handlers — use the legacy path that drives one coroutine at + // a time without an event loop. + for (calls, 0..) |call, i| { + results[i] = runLegacySync(self, call, allocator); + } + return; + } + + var slots = try allocator.alloc(Slot, calls.len); + defer allocator.free(slots); + for (slots) |*s| s.* = .{}; + + var batch_state: BatchState = .{ .allocator = allocator, .slots = slots }; + self.current_batch = &batch_state; + defer self.current_batch = null; + + // Track each call's coroutine reference in the parent stack (we + // hold them in registry refs so they survive across `uv.run` + // ticks). `0` after a coroutine has been reaped. + var thread_refs = try allocator.alloc(c_int, calls.len); + defer allocator.free(thread_refs); + @memset(thread_refs, 0); + defer for (thread_refs) |r| { + if (r != 0) c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, r); + }; + + var pending: usize = 0; for (calls, 0..) |call, i| { - results[i] = runOneCall(self, call, allocator); + const handler_ref = self.handlers.get(call.tool_name) orelse { + // Synthesize a recorded "err" result; don't even bother + // spawning a coroutine. + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe(u8, "panto: unknown tool name"), + }; + continue; + }; + const t = try startCoroutine(self, i, handler_ref, call.input, allocator); + thread_refs[i] = t.thread_ref; + if (t.still_pending) pending += 1; + } + + while (pending > 0) { + const active = try driveUvOnce(self); + // Reap any coroutines that terminated during the tick. + var reaped: usize = 0; + for (thread_refs, 0..) |tref, i| { + if (tref == 0) continue; + // Push the thread, check status, pop. + _ = c.lua_rawgeti(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + const co: *c.lua_State = @ptrCast(c.lua_tothread(self.L, -1).?); + const status = c.lua_status(co); + c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (status != c.LUA_YIELD) { + // Terminated (LUA_OK or error). The wrapper should + // have called `_record_result` already; if not, synthesize. + if (!slots[i].recorded) { + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe( + u8, + "panto: handler terminated without recording a result", + ), + }; + } + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + thread_refs[i] = 0; + reaped += 1; + } + } + if (reaped > 0) { + if (reaped > pending) { + // Defensive: keep the counter sane. + pending = 0; + } else { + pending -= reaped; + } + continue; + } + if (active == 0) { + // No libuv handles are pending, but we still have alive + // coroutines. They yielded without arranging to be woken. + // Mark them as failed and break. + for (thread_refs, 0..) |tref, i| { + if (tref == 0) continue; + slots[i] = .{ + .recorded = true, + .ok = false, + .err_msg = try allocator.dupe( + u8, + "panto: handler yielded but no libuv handle is pending; " ++ + "did you forget to await with luv?", + ), + }; + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, tref); + thread_refs[i] = 0; + } + break; + } + } + + // Translate slots into the libpanto-shaped results. + for (slots, 0..) |slot, i| { + if (!slot.recorded) { + results[i] = .{ .err = RuntimeError.BadHandlerReturn }; + continue; + } + if (slot.ok) { + results[i] = .{ .ok = slot.value orelse try allocator.dupe(u8, "") }; + // Free the err_msg if both ended up set somehow. + if (slot.err_msg) |m| allocator.free(m); + } else { + if (slot.value) |v| allocator.free(v); + // The error message was logged for the user; we still + // surface a typed error to libpanto so it can route the + // failure to the agent's error path. + std.log.warn( + "panto-lua: tool '{s}' failed: {s}", + .{ + calls[i].tool_name, + slot.err_msg orelse "(no message)", + }, + ); + if (slot.err_msg) |m| allocator.free(m); + results[i] = .{ .err = RuntimeError.LuaHandlerCrashed }; + } } } -fn runOneCall( +/// Start one coroutine: create a thread under the runtime's lua_State, +/// push the wrapper closure + (idx, handler, input), `lua_resume` once. +/// +/// If the coroutine returns immediately (sync handler), the wrapper +/// has already recorded its result via `panto._record_result` — +/// `still_pending` will be `false`. +fn startCoroutine( + self: *LuaRuntime, + idx: usize, + handler_ref: c_int, + input: []const u8, + allocator: Allocator, +) !struct { thread_ref: c_int, still_pending: bool } { + const L = self.L; + + const co = c.lua_newthread(L) orelse return RuntimeError.LuaInitFailed; + // luaL_ref pops the topmost value (the thread) and returns a + // registry ref to it. We keep the ref alive for the lifetime of + // the call so GC doesn't collect the thread mid-yield. + const thread_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); + + // Push the wrapper onto the coroutine's stack. + _ = c.lua_rawgeti(co, lua_bridge.LUA_REGISTRYINDEX, @intCast(self.wrapper_ref)); + // Push (idx, handler, input) as the resume args. + c.lua_pushinteger(co, @intCast(idx)); + _ = c.lua_rawgeti(co, lua_bridge.LUA_REGISTRYINDEX, @intCast(handler_ref)); + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + try lua_bridge.pushJsonAsLua(co, arena_state.allocator(), input); + + var nres: c_int = 0; + const status = c.lua_resume(co, L, 3, &nres); + return .{ + .thread_ref = thread_ref, + .still_pending = status == c.LUA_YIELD, + }; +} + +/// Call `uv.run("once")`. Returns the number of active handles luv +/// reports remain pending (0 means the loop is drained). +fn driveUvOnce(self: *LuaRuntime) !c_int { + const L = self.L; + _ = c.lua_rawgeti(L, lua_bridge.LUA_REGISTRYINDEX, @intCast(self.uv_run_ref)); + _ = c.lua_pushlstring(L, "once", 4); + if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: uv.run failed"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.UvRunFailed; + } + const n = c.lua_tointegerx(L, -1, null); + c.lua_settop(L, c.lua_gettop(L) - 1); + return @intCast(n); +} + +/// Pre-scheduler fallback (used in unit tests and during early +/// startup before `installScheduler` has run). +fn runLegacySync( self: *LuaRuntime, call: panto.ToolCall, allocator: Allocator, @@ -349,62 +618,40 @@ fn runOneCall( const handler_ref = self.handlers.get(call.tool_name) orelse { return .{ .err = RuntimeError.LuaHandlerNotFound }; }; - - const out_bytes = invokeCoroutine(self.L, handler_ref, call.input, allocator) catch |e| { + const out_bytes = invokeCoroutineSync(self.L, handler_ref, call.input, allocator) catch |e| { return .{ .err = e }; }; return .{ .ok = out_bytes }; } -/// Create a fresh coroutine, push the handler + JSON-decoded input as -/// the resume args, then resume. Returns the handler's return value as -/// owned JSON bytes (the slot `ok` in CallResult). -/// -/// Resume outcomes: -/// - LUA_OK: coroutine returned. Read its top value as the result. -/// - LUA_YIELD: coroutine yielded. With no event loop installed, we -/// treat this as an error so the user sees that their handler is -/// trying to do async I/O that isn't yet supported. -/// - other (errors): error message is on the coroutine's stack; -/// copy it to a log line and return LuaHandlerCrashed. -fn invokeCoroutine( +fn invokeCoroutineSync( L: *c.lua_State, handler_ref: c_int, input: []const u8, allocator: Allocator, ) ![]u8 { - // Create the coroutine thread. After this, `co` is the child - // thread; the parent stack also gains a thread value at the top. const co = c.lua_newthread(L) orelse return RuntimeError.LuaInitFailed; - defer c.lua_settop(L, c.lua_gettop(L) - 1); // pop the thread when done + defer c.lua_settop(L, c.lua_gettop(L) - 1); - // Push handler from the registry onto the coroutine's stack. _ = c.lua_rawgeti(co, lua_bridge.LUA_REGISTRYINDEX, @intCast(handler_ref)); if (c.lua_type(co, -1) != lua_bridge.T_FUNCTION) { return RuntimeError.LuaHandlerNotFound; } - // Push the parsed JSON input as the resume arg. var arena_state = std.heap.ArenaAllocator.init(allocator); defer arena_state.deinit(); try lua_bridge.pushJsonAsLua(co, arena_state.allocator(), input); - // Resume with 1 arg. var nresults: c_int = 0; const status = c.lua_resume(co, L, 1, &nresults); switch (status) { c.LUA_OK => { - // Coroutine returned. We expect exactly one string return - // value (the tool result). If there are zero or extra - // values we still try to read top-of-stack. if (nresults < 1) return RuntimeError.BadHandlerReturn; return try lua_bridge.readHandlerResult(co, -1, allocator); }, c.LUA_YIELD => { - // Nothing to wake this coroutine without an event loop. - // Surface the situation so the user knows what's wrong. - const msg = "lua: tool handler yielded with no event loop installed (step 4+ of LUA_MAKEOVER.md not yet implemented)"; + const msg = "lua: tool handler yielded with no event loop installed; call installScheduler() before dispatching"; if (@import("builtin").is_test) { std.log.warn("{s}", .{msg}); } else { @@ -420,6 +667,124 @@ fn invokeCoroutine( } // --------------------------------------------------------------------------- +// Scheduler setup (called once at startup, after luarocks bootstrap) +// --------------------------------------------------------------------------- + +/// Register `panto._record_result(idx, ok, value)` on the `panto` +/// global. The C function carries the runtime pointer as an upvalue, +/// reaches the in-flight `BatchState` through `current_batch`, and +/// stores into the matching slot. +fn installRecordResult(self: *LuaRuntime) !void { + const L = self.L; + _ = c.lua_getglobal(L, "panto"); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + c.lua_pushlightuserdata(L, @ptrCast(self)); + c.lua_pushcclosure(L, recordResultC, 1); + c.lua_setfield(L, -2, "_record_result"); + c.lua_settop(L, c.lua_gettop(L) - 1); // pop `panto` +} + +fn recordResultC(L: ?*c.lua_State) callconv(.c) c_int { + const Lst = L.?; + const self_ptr = c.lua_touserdata(Lst, c.lua_upvalueindex(1)); + if (self_ptr == null) return 0; + const self: *LuaRuntime = @ptrCast(@alignCast(self_ptr.?)); + const batch = self.current_batch orelse return 0; + + const idx_i64 = c.lua_tointegerx(Lst, 1, null); + const ok = c.lua_toboolean(Lst, 2) != 0; + const idx: usize = @intCast(idx_i64); + if (idx >= batch.slots.len) return 0; + + if (ok) { + // Result value at index 3. The handler's return type is + // free-form; we serialize via the existing `readHandlerResult` + // helper which already knows how to JSON-encode any Lua value. + const value = lua_bridge.readHandlerResult(Lst, 3, batch.allocator) catch |e| { + // Allocation failure mid-callback is unrecoverable from + // Lua's POV; record a synthetic error and bail. + const msg = std.fmt.allocPrint( + batch.allocator, + "panto: failed to serialize handler result: {s}", + .{@errorName(e)}, + ) catch null; + batch.slots[idx] = .{ .recorded = true, .ok = false, .err_msg = msg }; + return 0; + }; + batch.slots[idx] = .{ .recorded = true, .ok = true, .value = value }; + } else { + // Error message at index 3. May be any Lua value; coerce to + // string via `tostring`-equivalent semantics. + var len: usize = 0; + const ptr = c.luaL_tolstring(Lst, 3, &len); + const owned = if (ptr != null) + batch.allocator.dupe(u8, ptr[0..len]) catch null + else + null; + c.lua_settop(Lst, c.lua_gettop(Lst) - 1); // pop tolstring's pushed string + batch.slots[idx] = .{ .recorded = true, .ok = false, .err_msg = owned }; + } + return 0; +} + +/// Create the per-call wrapper closure: +/// +/// local function wrapper(idx, handler, input) +/// local ok, val = pcall(handler, input) +/// panto._record_result(idx, ok, val) +/// end +/// +/// Stored in the Lua registry under `self.wrapper_ref`. +fn installWrapperClosure(self: *LuaRuntime) !void { + const L = self.L; + const snippet = + \\return function(idx, handler, input) + \\ local ok, val = pcall(handler, input) + \\ panto._record_result(idx, ok, val) + \\end + ; + if (c.luaL_loadstring(L, snippet) != 0) { + logTopAsError(L, "panto-lua: wrapper closure failed to compile"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: wrapper closure failed to evaluate"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + // Top of stack: the wrapper function. luaL_ref pops it. + self.wrapper_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); +} + +/// Cache `require("luv").run` in the registry so the scheduler can +/// invoke it cheaply per tick. +fn cacheUvRun(self: *LuaRuntime) !void { + const L = self.L; + const snippet = + \\return require("luv").run + ; + if (c.luaL_loadstring(L, snippet) != 0) { + logTopAsError(L, "panto-lua: failed to compile luv lookup"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { + logTopAsError(L, "panto-lua: require('luv') failed (was the bootstrap successful?)"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + if (c.lua_type(L, -1) != lua_bridge.T_FUNCTION) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return RuntimeError.LuaInitFailed; + } + self.uv_run_ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); +} + +// --------------------------------------------------------------------------- // Internals // --------------------------------------------------------------------------- @@ -715,6 +1080,91 @@ test "yielding handler with no event loop surfaces LuaHandlerYielded" { try testing.expectEqual(@as(anyerror, RuntimeError.LuaHandlerYielded), results[0].err); } +// Integration test: requires a `$PANTO_HOME` with luv already +// installed. Skipped if luv isn't on disk — unit tests stay offline. +test "scheduler: yielding handler is resumed by libuv" { + const home_z = std.c.getenv("PANTO_HOME") orelse return error.SkipZigTest; + const panto_home_env = std.mem.sliceTo(home_z, 0); + // Check for `<home>/rocks/lua-<version>/lib/lua/5.4/luv.so`. + const manifest = @import("manifest.zig"); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const so_path = try std.fmt.bufPrint( + &path_buf, + "{s}/rocks/lua-{s}/lib/lua/{s}/luv.so", + .{ panto_home_env, manifest.lua_version, manifest.lua_short_version }, + ); + std.Io.Dir.cwd().access(testing.io, so_path, .{}) catch return error.SkipZigTest; + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + + const source = + \\local uv = require("luv") + \\panto.register_tool { + \\ name = "timer_say", description = "sleep then return", + \\ schema = { type = "object" }, + \\ handler = function(input) + \\ local co = coroutine.running() + \\ local timer = uv.new_timer() + \\ uv.timer_start(timer, 5, 0, function() + \\ uv.timer_stop(timer) + \\ uv.close(timer) + \\ coroutine.resume(co) + \\ end) + \\ coroutine.yield() + \\ return "awake" + \\ end, + \\} + ; + const path = try writeTempScript(tmp.dir, "timer.lua", source); + defer testing.allocator.free(path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + // Bootstrap luarocks (so `require("luv")` works), then install + // the scheduler. We use the real environment so the test picks + // up the same PANTO_HOME the developer's machine has. + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("PANTO_HOME", panto_home_env); + + const luarocks_runtime = @import("luarocks_runtime.zig"); + // The bootstrap needs a panto executable path for the wrapper + // script; tests don't actually invoke it, so a placeholder is + // fine (the wrapper is only consulted when luarocks itself + // shells out, which the test never triggers). + const luarocks_rt = try luarocks_runtime.bootstrap( + testing.allocator, + testing.io, + &env, + rt.L, + "/usr/bin/true", + ); + defer luarocks_rt.deinit(); + + try rt.installScheduler(); + try rt.loadExtension(path, null); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{ + .{ .tool_name = "timer_say", .input = "{}" }, + .{ .tool_name = "timer_say", .input = "{}" }, + }; + var results: [2]panto.ToolCallResult = .{ + .{ .err = error.SourceDroppedCall }, + .{ .err = error.SourceDroppedCall }, + }; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer for (results) |r| switch (r) { + .ok => |b| testing.allocator.free(b), + .err => {}, + }; + + try testing.expectEqualStrings("awake", results[0].ok); + try testing.expectEqualStrings("awake", results[1].ok); +} + test "loadExtension: duplicate tool name from a second extension errors" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); diff --git a/src/luarocks_runtime.zig b/src/luarocks_runtime.zig new file mode 100644 index 0000000..b67a8ad --- /dev/null +++ b/src/luarocks_runtime.zig @@ -0,0 +1,763 @@ +//! Embedded-luarocks runtime bootstrap. +//! +//! Responsibilities at startup (per LUA_MAKEOVER.md steps 3-5 and Q1-Q5): +//! +//! 1. Resolve `$PANTO_HOME` and the per-Lua-version rocks tree +//! (`panto_home.zig`). Create the directory layout if missing. +//! 2. Stage Lua headers under `<tree>/include/` (from `@embedFile`) +//! so luarocks can compile C rocks against them. Idempotent: a +//! file is only rewritten if its checksum differs. +//! 3. Materialize `<tree>/etc/luarocks/config-<short>.lua` with the +//! pinned interpreter, rock trees, and toolchain variables. +//! 4. Install a `package.searcher` that serves `require("luarocks.*")` +//! and `require("compat53.*")` from embedded sources \u2014 the +//! luarocks Lua libraries never touch disk. +//! 5. Inject `luarocks.core.hardcoded` into `package.loaded` with the +//! runtime-resolved `SYSCONFDIR`. Without this, `luarocks.core.cfg` +//! can't find its config file. +//! 6. Configure `package.path` / `package.cpath` so user rocks +//! installed in `<tree>/share/lua/<short>` and +//! `<tree>/lib/lua/<short>` are visible to `require`. +//! 7. Reconcile the batteries manifest: for each pinned rock, check +//! `<tree>/lib/luarocks/rocks-<short>/<name>/<version>/` and invoke +//! `luarocks install` for anything missing. (Slow path; only the +//! first run after a fresh `$PANTO_HOME` actually downloads.) +//! +//! Step 7 needs a usable `lua` executable on PATH from luarocks's point +//! of view \u2014 it shells out for rockspec build scripts. We satisfy +//! this via the `panto lua` subcommand, addressed by writing +//! `<panto-binary> lua` (with the absolute path of the running `panto` +//! binary) into the luarocks config as `variables.LUA`. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; + +const manifest = @import("manifest.zig"); +const panto_home = @import("panto_home.zig"); +const embedded_luarocks = @import("embedded_luarocks"); +const embedded_lua_headers = @import("embedded_lua_headers"); +const lua_bridge = @import("lua_bridge.zig"); + +const c = lua_bridge.c; + +/// Owned state for the runtime side of luarocks. Holds onto the +/// resolved layout, the `lua_State` we attached to, and a hash map +/// used by the embedded-module searcher. +pub const LuarocksRuntime = struct { + allocator: Allocator, + layout: panto_home.Layout, + L: *c.lua_State, + /// Module-name to source-bytes lookup for the embedded-source + /// `package.searcher` callback. Keys borrow from + /// `embedded_luarocks.files`; values likewise. + modules: std.StringHashMapUnmanaged([]const u8), + + pub fn deinit(self: *LuarocksRuntime) void { + self.modules.deinit(self.allocator); + self.layout.deinit(); + self.allocator.destroy(self); + } +}; + +/// Errors surfaced by the bootstrap pipeline. The `Luarocks*` variants +/// indicate that we were able to invoke luarocks but it exited non-zero +/// (or otherwise failed); the message is in stderr at that point. +pub const BootstrapError = error{ + HeadersMissing, + LuarocksInjectionFailed, + LuarocksInstallFailed, + LuarocksSearcherInstallFailed, + PathConfigFailed, + PantoExecutablePathUnknown, +} || Allocator.Error; + +/// Full startup-time bootstrap. Walks the entire setup pipeline against +/// the given `lua_State`, leaving it ready for callers to `require` +/// luarocks modules and for any user code to find rocks under the +/// configured tree. +/// +/// `panto_executable_path` is the absolute path of the currently +/// running `panto` binary. Used to construct the `<panto> lua` +/// invocation that luarocks will use as its interpreter when running +/// rockspec build scripts. +/// +/// Wipe the current Lua-version's rocks tree before bootstrapping. +/// Used by `panto bootstrap --force` to recover from a corrupted or +/// outdated installation. Only the `<home>/rocks/lua-<version>/` +/// subtree is removed; sibling trees from other Lua versions stay +/// untouched, matching the rationale in Q3 (each Lua version owns +/// its own tree for clean rollback). +pub fn wipeTree( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, +) !void { + var layout = try panto_home.resolve(allocator, environ_map); + defer layout.deinit(); + + // `deleteTree` is the cwd-relative entrypoint; we want absolute. + // Open the parent and delete by basename to avoid path-traversal + // surprises if `layout.tree` ever contains a symlink. + const parent = std.fs.path.dirname(layout.tree) orelse { + std.log.warn("panto bootstrap --force: tree has no parent? '{s}'", .{layout.tree}); + return; + }; + const base = std.fs.path.basename(layout.tree); + + var parent_dir = Io.Dir.cwd().openDir(io, parent, .{}) catch |err| switch (err) { + error.FileNotFound => return, // already gone; nothing to wipe + else => return err, + }; + defer parent_dir.close(io); + + // `deleteTree` does not raise `FileNotFound` — a missing leaf is + // treated as success, matching our "force wipe is idempotent" + // intent here. + try parent_dir.deleteTree(io, base); + + std.log.info("panto bootstrap --force: removed {s}", .{layout.tree}); +} + +/// Every `panto` invocation runs through this one pipeline — `panto +/// lua`, `panto bootstrap`, and the default agent loop are all just +/// surfaces on top of "start the Lua runtime, install missing +/// batteries, then do your thing." +pub fn bootstrap( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + L: *c.lua_State, + panto_executable_path: []const u8, +) !*LuarocksRuntime { + const layout = try panto_home.resolve(allocator, environ_map); + errdefer layout.deinit(); + + try panto_home.ensureDirsExist(layout, io); + try stageLuaHeaders(allocator, io, layout); + const lua_wrapper_path = try writeLuaWrapper(allocator, io, layout, panto_executable_path); + defer allocator.free(lua_wrapper_path); + try writeLuarocksConfig(allocator, io, layout, lua_wrapper_path); + + // The embedded-source `package.searcher` keeps a pointer to the + // module map; we need the map's storage to live at a stable + // address for the process lifetime. Build the runtime first so + // `self.modules` is the address the singleton can capture, then + // install the searcher pointing at `self.modules` specifically. + const self = try allocator.create(LuarocksRuntime); + errdefer allocator.destroy(self); + self.* = .{ + .allocator = allocator, + .layout = layout, + .L = L, + .modules = .empty, + }; + try self.modules.ensureTotalCapacity(allocator, @intCast(embedded_luarocks.files.len + 2)); + for (embedded_luarocks.files) |entry| { + self.modules.putAssumeCapacityNoClobber(entry.name, entry.contents); + } + + try installEmbeddedSearcher(L, &self.modules); + try injectHardcoded(L, layout); + try configurePackagePaths(allocator, L, layout); + + // Reconcile the batteries manifest. This may take a while on a + // fresh install; subsequent runs no-op. Runs in-process — we + // already have luarocks loaded in `L` via the embedded searcher, + // so there's no need (and no reason) to spawn a child Lua. + // + // `PANTO_BOOTSTRAP_NO_RECONCILE` is a re-entry guard. When the + // reconcile loop is already running in an ancestor process (a + // luarocks build step shells out to `<tree>/bin/lua`, which is + // our `panto lua` wrapper), we don't want the child to start its + // own reconcile. The variable is set by `reconcileBatteries` + // before any potentially-recursive work and cleared afterward. + if (environ_map.get("PANTO_BOOTSTRAP_NO_RECONCILE") == null) { + try reconcileBatteries(allocator, io, self); + } + + return self; +} + +// --------------------------------------------------------------------------- +// Step 2: stage Lua headers +// --------------------------------------------------------------------------- + +/// Write every embedded Lua header to `<tree>/include/`. Skips any file +/// whose existing on-disk contents already match \u2014 keeps file mtimes +/// stable across reruns and lets luarocks's mtime-based caching work. +fn stageLuaHeaders( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, +) !void { + var dir = try Io.Dir.cwd().openDir(io, layout.include_dir, .{}); + defer dir.close(io); + + for (embedded_lua_headers.files) |entry| { + try writeIfDifferent(allocator, io, dir, entry.name, entry.contents); + } +} + +/// Write `contents` into `dir/name` only if the existing file differs. +/// Creates the file if absent. +fn writeIfDifferent( + allocator: Allocator, + io: Io, + dir: Io.Dir, + name: []const u8, + contents: []const u8, +) !void { + if (dir.readFileAlloc(io, name, allocator, .limited(1 << 22))) |existing| { + defer allocator.free(existing); + if (std.mem.eql(u8, existing, contents)) return; + } else |err| switch (err) { + error.FileNotFound => {}, + else => return err, + } + try dir.writeFile(io, .{ .sub_path = name, .data = contents }); +} + +// --------------------------------------------------------------------------- +// Step 3: write luarocks config +// --------------------------------------------------------------------------- + +/// Write a tiny shell wrapper at `<tree>/bin/lua` that `exec`s +/// `<panto> lua "$@"`. luarocks invokes its configured `LUA` variable +/// as a real executable when running rockspec build scripts; we can't +/// give it `"panto lua"` directly because it splits argv naively, and +/// we can't give it an absolute path to the panto binary because then +/// it'd run panto's agent loop instead of `lua.c`'s REPL. +/// +/// The wrapper makes panto's lua subcommand visible to luarocks as if +/// it were a standalone `lua` binary. Returns the wrapper path; caller +/// frees. +fn writeLuaWrapper( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, + panto_executable_path: []const u8, +) ![]u8 { + const bin_dir = try std.fs.path.join(allocator, &.{ layout.tree, "bin" }); + defer allocator.free(bin_dir); + Io.Dir.cwd().createDirPath(io, bin_dir) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, + }; + + const wrapper_path = try std.fs.path.join(allocator, &.{ bin_dir, "lua" }); + errdefer allocator.free(wrapper_path); + + var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + const w = &aw.writer; + try w.writeAll("#!/bin/sh\n"); + try w.writeAll("# Auto-generated by panto. Bridges luarocks's external\n"); + try w.writeAll("# `lua` invocations to `panto lua`.\n"); + try w.writeAll("exec "); + try writeShellQuoted(w, panto_executable_path); + try w.writeAll(" lua \"$@\"\n"); + + var bin = try Io.Dir.cwd().openDir(io, bin_dir, .{}); + defer bin.close(io); + try bin.writeFile(io, .{ + .sub_path = "lua", + .data = aw.written(), + .flags = .{ .permissions = .executable_file }, + }); + return wrapper_path; +} + +fn writeShellQuoted(w: anytype, s: []const u8) !void { + try w.writeByte('\''); + for (s) |ch| { + if (ch == '\'') { + try w.writeAll("'\\''"); + } else { + try w.writeByte(ch); + } + } + try w.writeByte('\''); +} + +/// Materialize a luarocks config-<short>.lua under `layout.sysconfdir`. +/// This is the file luarocks reads from `SYSCONFDIR`; we point every +/// path-typed variable at the in-tree directories so installs land in +/// `$PANTO_HOME/rocks/lua-X.Y.Z/`. +/// +/// Format reference: docs/config_file_format.md in the luarocks repo. +fn writeLuarocksConfig( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, + lua_wrapper_path: []const u8, +) !void { + var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + const w = &aw.writer; + + try w.writeAll("-- Auto-generated by panto. Do not edit.\n"); + try w.writeAll("-- This file is rewritten on every panto startup.\n\n"); + + try w.writeAll("rocks_trees = {\n { name = \"user\", root = "); + try writeLuaString(w, layout.tree); + try w.writeAll(" },\n}\n\n"); + + try w.writeAll("lua_interpreter = \"panto\"\n\n"); + + const bin_dir = std.fs.path.dirname(lua_wrapper_path) orelse layout.tree; + try w.writeAll("variables = {\n"); + try writeLuaKV(w, "LUA", lua_wrapper_path); + try writeLuaKV(w, "LUA_BINDIR", bin_dir); + try writeLuaKV(w, "LUA_INCDIR", layout.include_dir); + try writeLuaKV(w, "LUA_LIBDIR", layout.lib_lua_dir); + try writeLuaKV(w, "LUA_DIR", layout.tree); + try writeLuaKV(w, "CURL", "curl"); + try w.writeAll("}\n\n"); + + // We do not run luarocks's external `lua` invocation as a + // separate Lua install; the panto binary itself answers to + // `panto lua` and behaves like upstream lua.c. + try w.writeAll("lua_version = "); + try writeLuaString(w, manifest.lua_short_version); + try w.writeAll("\n\n"); + + // Default deps mode: only consider the panto tree, ignore any + // system-wide luarocks installations. Keeps reproducibility. + try w.writeAll("deps_mode = \"one\"\n"); + + // Write atomically: stage as `.tmp`, rename into place. luarocks + // reads the config eagerly on every invocation; an interrupted + // write would corrupt the next startup. + var sysconf = try Io.Dir.cwd().openDir(io, layout.sysconfdir, .{}); + defer sysconf.close(io); + + const tmp_name = "config-staging.lua"; + try sysconf.writeFile(io, .{ .sub_path = tmp_name, .data = aw.written() }); + try sysconf.rename(tmp_name, sysconf, std.fs.path.basename(layout.config_file), io); +} + +// --------------------------------------------------------------------------- +// Step 4: embedded-source `package.searcher` +// --------------------------------------------------------------------------- + +fn buildEmbeddedModuleMap(allocator: Allocator) !std.StringHashMapUnmanaged([]const u8) { + var modules: std.StringHashMapUnmanaged([]const u8) = .empty; + try modules.ensureTotalCapacity(allocator, @intCast(embedded_luarocks.files.len + 2)); + for (embedded_luarocks.files) |entry| { + modules.putAssumeCapacityNoClobber(entry.name, entry.contents); + } + return modules; +} + +/// Process-singleton handle the C searcher uses to find module sources. +/// Populated at bootstrap, read on every `require`. +var module_map_singleton: ?*const std.StringHashMapUnmanaged([]const u8) = null; + +/// Install our embedded-source searcher as `package.searchers[2]`, +/// after the preload searcher (slot 1) but before path-based searchers. +/// Slot 2 is conventional for embedded code (luarocks's own all_in_one +/// does the same). +fn installEmbeddedSearcher( + L: *c.lua_State, + modules: *std.StringHashMapUnmanaged([]const u8), +) !void { + module_map_singleton = modules; + + // Push package.searchers onto the stack. + _ = c.lua_getglobal(L, "package"); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + return BootstrapError.LuarocksSearcherInstallFailed; + } + _ = c.lua_getfield(L, -1, "searchers"); + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 2); + return BootstrapError.LuarocksSearcherInstallFailed; + } + + // We want to insert our searcher at slot 2 \u2014 push every existing + // entry from slot 2 onward up by one, then assign our function. + const n = c.lua_rawlen(L, -1); + // Shift slots upward: searchers[i+1] = searchers[i] for i in [n..2]. + var i: c.lua_Integer = @intCast(n); + while (i >= 2) : (i -= 1) { + _ = c.lua_rawgeti(L, -1, i); + c.lua_rawseti(L, -2, i + 1); + } + c.lua_pushcfunction(L, embeddedSearcher); + c.lua_rawseti(L, -2, 2); + + c.lua_settop(L, c.lua_gettop(L) - 2); // pop searchers + package +} + +/// Lua C function: given a module name, look it up in the embedded map +/// and return a loader closure if found; otherwise push an explanatory +/// string and return 1 \u2014 standard `package.searchers` contract. +fn embeddedSearcher(L: ?*c.lua_State) callconv(.c) c_int { + const Lst = L.?; + var name_len: usize = 0; + const name_ptr = c.lua_tolstring(Lst, 1, &name_len); + if (name_ptr == null) { + _ = c.lua_pushlstring(Lst, "\n\t[panto: invalid require argument]", 35); + return 1; + } + const name = name_ptr[0..name_len]; + + const map = module_map_singleton orelse { + _ = c.lua_pushlstring(Lst, "\n\t[panto: embedded modules not initialized]", 43); + return 1; + }; + + // Resolve `name` first, then `name.init` if missing — matches + // stock Lua path-searcher behavior where `require("foo")` checks + // both `foo.lua` and `foo/init.lua`. + var contents_opt = map.get(name); + var resolved_name: []const u8 = name; + var init_buf: [256]u8 = undefined; + if (contents_opt == null) { + const init_name = std.fmt.bufPrint(&init_buf, "{s}.init", .{name}) catch name; + if (map.get(init_name)) |contents2| { + contents_opt = contents2; + resolved_name = init_name; + } + } + if (contents_opt) |contents| { + // Push loader: a Lua function that, when called, executes the + // chunk and returns its result. `luaL_loadbufferx` with mode + // "t" forbids binary chunks so we can't be tricked into loading + // pre-compiled bytecode through this path. + const chunkname_buf = blk: { + // "=panto/<name>" \u2014 the `=` prefix makes Lua print the name + // verbatim in stack traces instead of prefixing with `[string]`. + var sbuf: [256]u8 = undefined; + const s = std.fmt.bufPrintZ(&sbuf, "=panto/{s}", .{resolved_name}) catch "=panto/embedded"; + break :blk s; + }; + const status = c.luaL_loadbufferx( + Lst, + contents.ptr, + contents.len, + chunkname_buf.ptr, + "t", + ); + if (status != c.LUA_OK) { + // Error message left on stack by luaL_loadbufferx \u2014 return + // it as the searcher's diagnostic. + return 1; + } + return 1; + } + + _ = c.lua_pushlstring(Lst, "\n\t[panto: no embedded match]", 27); + return 1; +} + +// --------------------------------------------------------------------------- +// Step 5: inject luarocks.core.hardcoded +// --------------------------------------------------------------------------- + +/// Construct a `luarocks.core.hardcoded` table with the runtime-resolved +/// SYSCONFDIR (and FORCE_CONFIG = true, so luarocks doesn't go hunting +/// for system configs) and stash it in `package.loaded` so that the +/// later `require("luarocks.core.hardcoded")` returns our table instead +/// of going to disk. This is exactly the trick the upstream GNUmakefile +/// uses. +fn injectHardcoded(L: *c.lua_State, layout: panto_home.Layout) !void { + const snippet = + \\local sysconfdir = ... + \\package.loaded["luarocks.core.hardcoded"] = { + \\ SYSCONFDIR = sysconfdir, + \\ FORCE_CONFIG = true, + \\} + ; + if (c.luaL_loadstring(L, snippet) != 0) { + return BootstrapError.LuarocksInjectionFailed; + } + _ = c.lua_pushlstring(L, layout.sysconfdir.ptr, layout.sysconfdir.len); + if (c.lua_pcallk(L, 1, 0, 0, 0, null) != 0) { + return BootstrapError.LuarocksInjectionFailed; + } +} + +// --------------------------------------------------------------------------- +// Step 6: package.path / package.cpath +// --------------------------------------------------------------------------- + +/// Add `<tree>/share/lua/<short>/?.lua` and `<tree>/share/lua/<short>/?/init.lua` +/// to `package.path`, and the matching `.so`/`.dylib` patterns to +/// `package.cpath`, so rocks installed under our tree are visible to +/// `require`. The original path entries follow ours \u2014 we shadow the +/// system in case anything matches by accident. +fn configurePackagePaths( + allocator: Allocator, + L: *c.lua_State, + layout: panto_home.Layout, +) !void { + const so_suffix = comptime soSuffix(); + + const path_segment = try std.fmt.allocPrint( + allocator, + "{0s}/?.lua;{0s}/?/init.lua", + .{layout.share_lua_dir}, + ); + defer allocator.free(path_segment); + + const cpath_segment = try std.fmt.allocPrint( + allocator, + "{0s}/?{1s}", + .{ layout.lib_lua_dir, so_suffix }, + ); + defer allocator.free(cpath_segment); + + // Snippet: prepend the segments to package.path/cpath. + const snippet = + \\local path_seg, cpath_seg = ... + \\package.path = path_seg .. ";" .. package.path + \\package.cpath = cpath_seg .. ";" .. package.cpath + ; + if (c.luaL_loadstring(L, snippet) != 0) { + return BootstrapError.PathConfigFailed; + } + _ = c.lua_pushlstring(L, path_segment.ptr, path_segment.len); + _ = c.lua_pushlstring(L, cpath_segment.ptr, cpath_segment.len); + if (c.lua_pcallk(L, 2, 0, 0, 0, null) != 0) { + return BootstrapError.PathConfigFailed; + } +} + +fn soSuffix() []const u8 { + return switch (@import("builtin").os.tag) { + .macos, .ios, .watchos, .tvos => ".so", + .windows => ".dll", + else => ".so", + }; +} + +/// Write `value` as a quoted Lua string. Uses the `"..."` short-string +/// form, escaping backslashes, double-quotes, newlines, and any other +/// control characters via `\NNN` decimal escapes. Suitable for any +/// path on any platform. +fn writeLuaString(w: anytype, value: []const u8) !void { + try w.writeByte('"'); + for (value) |ch| { + switch (ch) { + '\\' => try w.writeAll("\\\\"), + '"' => try w.writeAll("\\\""), + '\n' => try w.writeAll("\\n"), + '\r' => try w.writeAll("\\r"), + '\t' => try w.writeAll("\\t"), + 0...8, 11, 12, 14...31, 127 => try w.print("\\{d}", .{ch}), + else => try w.writeByte(ch), + } + } + try w.writeByte('"'); +} + +fn writeLuaKV(w: anytype, key: []const u8, value: []const u8) !void { + try w.print(" {s} = ", .{key}); + try writeLuaString(w, value); + try w.writeAll(",\n"); +} + +// --------------------------------------------------------------------------- +// Step 7: reconcile manifest +// --------------------------------------------------------------------------- + +/// For each pinned battery, check whether the version-stamped install +/// metadata exists under `<tree>/lib/luarocks/rocks-<short>/<name>/<version>/` +/// (luarocks's standard metadata location). If absent, call +/// `luarocks.cmd.run("install", name, version)` directly against our +/// `lua_State` — no subprocess. +/// +/// Subsequent panto runs hit the fast path: the metadata directory +/// exists, no luarocks invocation happens. +fn reconcileBatteries( + allocator: Allocator, + io: Io, + rt: *LuarocksRuntime, +) !void { + var any_missing = false; + for (manifest.batteries) |battery| { + if (try batteryInstalled(allocator, io, rt.layout, battery)) continue; + any_missing = true; + break; + } + if (!any_missing) return; + + // Tell any child processes that go through panto (luarocks's + // CMake/make subprocesses ultimately invoke `<tree>/bin/lua`, + // which is our `panto lua` wrapper) to skip their own reconcile + // step — we're already in the middle of it. + // + // Set process-wide via the C `setenv` so spawn calls inherit it. + // Cleared after the loop so subsequent panto invocations of this + // process see a clean environment. + _ = c_setenv("PANTO_BOOTSTRAP_NO_RECONCILE", "1", 1); + defer _ = c_unsetenv("PANTO_BOOTSTRAP_NO_RECONCILE"); + + for (manifest.batteries) |battery| { + if (try batteryInstalled(allocator, io, rt.layout, battery)) continue; + std.log.info( + "panto: installing battery {s} {s} via luarocks (first run; this may take a minute)", + .{ battery.name, battery.version }, + ); + try installBattery(allocator, rt.L, battery); + } +} + +extern "c" fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; +extern "c" fn unsetenv(name: [*:0]const u8) c_int; +const c_setenv = setenv; +const c_unsetenv = unsetenv; + +/// Check whether `<tree>/lib/luarocks/rocks-<short>/<name>/<version>/` +/// exists. luarocks creates this directory atomically as part of an +/// install, so its presence is a reliable signal that the rock landed. +fn batteryInstalled( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, + battery: manifest.Battery, +) !bool { + const subpath = try std.fs.path.join( + allocator, + &.{ layout.rocks_metadata_dir, battery.name, battery.version }, + ); + defer allocator.free(subpath); + + var dir = Io.Dir.cwd().openDir(io, subpath, .{}) catch |err| switch (err) { + error.FileNotFound, error.NotDir => return false, + else => return err, + }; + dir.close(io); + return true; +} + +/// Spawn `<panto> lua -e 'require("luarocks.cmd").run("install", name, version)'` +/// inheriting stdout/stderr so the user sees compilation output. +/// +/// We set `PANTO_BOOTSTRAP_NO_RECONCILE=1` in the child so that the +/// nested `panto lua` doesn't itself try to reconcile (which would +/// recurse forever — installing luv would re-trigger installing luv). +/// The child still runs the full filesystem-side bootstrap (searcher, +/// hardcoded, package paths) — only the manifest reconcile is skipped. +/// Run luarocks's `install` command directly against our `lua_State`. +/// +/// We mirror what `src/bin/luarocks` does — it's a thin Lua driver +/// that builds a command table and calls `cmd.run_command(...)`. We +/// embed the driver's source (`embedded_luarocks.luarocks_main`) and +/// load it as a chunk, passing `{"install", name, version}` as the +/// vararg the chunk reads via `...`. +/// +/// luarocks's `run_command` prints diagnostics to stdout/stderr as +/// it goes, so the user sees compilation progress in real time. +/// On failure it calls `os.exit(1)`, which our `pcall` wrapper +/// intercepts before it can terminate the process. +fn installBattery( + allocator: Allocator, + L: *c.lua_State, + battery: manifest.Battery, +) !void { + // We wrap the embedded driver in a function that overrides + // `os.exit` for the duration of the call. luarocks's `die()` + // ultimately calls `os.exit(1)`, which would terminate panto + // entirely; we want to catch the failure as a regular error. + const driver = embedded_luarocks.luarocks_main; + + const wrapper = + \\local orig_exit = os.exit + \\local args = { ... } + \\local fail_code = nil + \\os.exit = function(code) fail_code = code or 0; error({ panto_exit = code or 0 }) end + \\arg = arg or {} + \\arg[0] = arg[0] or "luarocks" + \\local driver = args[1] + \\local chunk, err = load(driver, "=panto/luarocks-driver", "t") + \\if not chunk then error("failed to load luarocks driver: " .. tostring(err)) end + \\local ok, err = pcall(chunk, table.unpack(args, 2)) + \\os.exit = orig_exit + \\if not ok then + \\ if type(err) == "table" and err.panto_exit ~= nil then + \\ if err.panto_exit ~= 0 then + \\ error("luarocks exited with code " .. tostring(err.panto_exit)) + \\ end + \\ else + \\ error(err) + \\ end + \\end + ; + + if (c.luaL_loadstring(L, wrapper) != 0) { + return BootstrapError.LuarocksInstallFailed; + } + + // Strip the shebang line if present. `luaL_loadfile` does this + // for files, but we're going through `load(...)` from Lua which + // doesn't — it'll choke on `#!/usr/bin/env lua` as a syntax error. + var driver_slice: []const u8 = driver; + if (driver_slice.len >= 2 and driver_slice[0] == '#' and driver_slice[1] == '!') { + if (std.mem.indexOfScalar(u8, driver_slice, '\n')) |nl| { + driver_slice = driver_slice[nl + 1 ..]; + } + } + + // Pass the driver source as the first arg, then the luarocks + // CLI args. We use `lua_pushlstring` to push everything as + // Lua strings (no NUL terminator needed, since lua_pushlstring + // takes an explicit length). + _ = c.lua_pushlstring(L, driver_slice.ptr, driver_slice.len); + _ = c.lua_pushlstring(L, "install", 7); + _ = c.lua_pushlstring(L, battery.name.ptr, battery.name.len); + _ = c.lua_pushlstring(L, battery.version.ptr, battery.version.len); + + _ = allocator; // currently unused; reserved for future scratch needs + + if (c.lua_pcallk(L, 4, 0, 0, 0, null) != 0) { + var len: usize = 0; + const msg = c.lua_tolstring(L, -1, &len); + if (msg != null) { + std.log.err( + "panto: luarocks install of {s} {s} failed: {s}", + .{ battery.name, battery.version, msg[0..len] }, + ); + } + c.lua_settop(L, c.lua_gettop(L) - 1); + return BootstrapError.LuarocksInstallFailed; + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "embedded module map contains luarocks.core.cfg, luarocks.cmd, compat53.init" { + var modules = try buildEmbeddedModuleMap(testing.allocator); + defer modules.deinit(testing.allocator); + + try testing.expect(modules.get("luarocks.core.cfg") != null); + try testing.expect(modules.get("luarocks.cmd") != null); + // `luarocks.cmd.init` is a real submodule (the `init` subcommand); + // luarocks.cmd is the module itself — distinct entries. + try testing.expect(modules.get("luarocks.cmd.init") != null); + try testing.expect(modules.get("compat53.init") != null); + try testing.expect(modules.get("compat53.module") != null); + // Not there: + try testing.expect(modules.get("luarocks.nonexistent") == null); +} + +test "embedded searcher fallback resolves bare name via name.init" { + // Mirrors stock Lua searcher semantics: require("compat53") should + // succeed even though our map only stores `compat53.init`. We test + // the lookup logic directly here — the C-level installation is + // exercised by integration runs of the panto binary. + var modules = try buildEmbeddedModuleMap(testing.allocator); + defer modules.deinit(testing.allocator); + + const bare = modules.get("compat53"); + try testing.expect(bare == null); + const via_init = modules.get("compat53.init"); + try testing.expect(via_init != null); +} diff --git a/src/main.zig b/src/main.zig index 15b2dd2..6654c95 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4,6 +4,10 @@ const ping_tool = @import("ping_tool.zig"); const lua_bridge = @import("lua_bridge.zig"); const lua_runtime = @import("lua_runtime.zig"); const extension_loader = @import("extension_loader.zig"); +const panto_home = @import("panto_home.zig"); +const luarocks_runtime = @import("luarocks_runtime.zig"); +const self_exe = @import("self_exe.zig"); +const subcommand = @import("subcommand.zig"); // Shorthand alias for the Lua C API. The bridge module owns the actual // `@cImport`; we re-use it here so the smoke check uses identical types. @@ -23,6 +27,10 @@ test { _ = lua_bridge; _ = lua_runtime; _ = extension_loader; + _ = panto_home; + _ = luarocks_runtime; + _ = self_exe; + _ = subcommand; } const Receiver = panto.provider.Receiver; @@ -231,6 +239,26 @@ pub fn main(init: std.process.Init) !void { // wired up yet, this just confirms the static lib comes through. luaSmokeCheck(); + // Resolve the absolute path of the running panto binary. Needed + // both by `panto lua` (we re-exec ourselves through a wrapper + // luarocks invokes) and by the agent's bootstrap. + const panto_path = try self_exe.selfExePathAlloc(alloc); + defer alloc.free(panto_path); + + // Subcommand dispatch: `panto lua` and `panto bootstrap` short + // out of the agent loop, but still run the same luarocks bootstrap + // pipeline so first-run setup happens consistently. + switch (try subcommand.dispatch( + alloc, + io, + init.environ_map, + init.minimal.args, + panto_path, + )) { + .done => return, + .agent => {}, + } + const config = try loadConfig(init.environ_map); var stdout_buffer: [4096]u8 = undefined; @@ -260,6 +288,24 @@ pub fn main(init: std.process.Init) !void { var rt = try lua_runtime.LuaRuntime.create(alloc); defer rt.deinit(); + // Bootstrap luarocks against the Lua runtime's lua_State — same + // pipeline as `panto lua` and `panto bootstrap`. After this, + // `require("luarocks.*")` works and any pinned batteries from the + // manifest are installed under $PANTO_HOME. + const luarocks_rt = try luarocks_runtime.bootstrap( + alloc, + io, + init.environ_map, + rt.L, + panto_path, + ); + defer luarocks_rt.deinit(); + + // luv is installed (or already present) at this point; wire the + // libuv-driven coroutine scheduler before any extensions get a + // chance to register tools that might want to yield. + try rt.installScheduler(); + // Discover Lua extensions from $XDG_CONFIG_HOME/panto/extensions (or // $HOME/.config/panto/extensions) and ./.panto/extensions. Project // entries shadow user entries with the same name; tool-name collisions diff --git a/src/manifest.zig b/src/manifest.zig new file mode 100644 index 0000000..b12b5c4 --- /dev/null +++ b/src/manifest.zig @@ -0,0 +1,43 @@ +//! Pinned versions of every Lua component shipped with panto: the +//! upstream Lua language version, the embedded luarocks version, and +//! the rocks ("batteries") panto requires for its own runtime to work. +//! +//! Bumping any of these is a deliberate edit + commit + version bump of +//! panto itself. Each panto release pins one consistent set; bootstrap +//! reconciles the installed rocks tree against this manifest on every +//! startup, installing missing rocks and removing stale ones (per Q5 of +//! LUA_MAKEOVER.md). +//! +//! The Lua and luarocks versions ride in via the `versions` build option +//! so build.zig stays the single source of truth. The batteries are +//! pinned here directly because they don't otherwise need to be visible +//! to the build. + +const versions = @import("versions"); + +pub const lua_version: []const u8 = versions.lua_version; +pub const lua_short_version: []const u8 = versions.lua_short_version; +pub const luarocks_version: []const u8 = versions.luarocks_version; + +pub const Battery = struct { + /// Rock name as published on luarocks.org. + name: []const u8, + /// Pinned rockspec version, e.g. `"1.52.1-1"`. luarocks uses these + /// MAJOR.MINOR.PATCH-REV strings throughout — we pass them straight + /// through to `luarocks install`. + version: []const u8, +}; + +/// Rocks installed automatically at bootstrap. Compiled into the panto +/// binary; users do not configure this list. +/// +/// We deliberately ship the smallest viable set: luv is the only +/// runtime dependency — it provides libuv's event loop, which panto's +/// coroutine scheduler drives, and gives extension authors a single +/// rich async I/O surface. Anything else (coro-* helpers, lua-cjson, +/// lpeg, etc.) is the user's responsibility, installable via +/// `panto lua -e 'arg[0]="luarocks"; require("luarocks.cmd").run_command(...)'` +/// or, eventually, a higher-level `panto rocks install` subcommand. +pub const batteries: []const Battery = &.{ + .{ .name = "luv", .version = "1.52.1-0" }, +}; diff --git a/src/panto_home.zig b/src/panto_home.zig new file mode 100644 index 0000000..b7c1799 --- /dev/null +++ b/src/panto_home.zig @@ -0,0 +1,206 @@ +//! Filesystem layout resolution for `$PANTO_HOME` and the per-Lua +//! version rocks tree. +//! +//! $PANTO_HOME = $XDG_DATA_HOME/panto +//! (or $HOME/.local/share/panto if XDG_DATA_HOME unset) +//! +//! $PANTO_HOME/ +//! rocks/ +//! lua-5.4.7/ ← current tree +//! include/ ← Lua headers staged by bootstrap +//! share/lua/5.4/ ← installed pure-Lua rocks +//! lib/lua/5.4/ ← installed C rocks (.so/.dylib) +//! lib/luarocks/rocks-5.4/ ← luarocks's own rock metadata +//! etc/luarocks/ ← luarocks config-5.4.lua +//! +//! See Q3 of LUA_MAKEOVER.md for the rationale behind the versioned +//! subdirectory. We never write to the tree of a different Lua version +//! — a panto upgrade that bumps Lua creates a fresh tree alongside the +//! old one, leaving the old one in place for rollback. + +const std = @import("std"); +const manifest = @import("manifest.zig"); + +const Allocator = std.mem.Allocator; +const Io = std.Io; + +/// Resolved set of filesystem paths panto uses for its rocks tree. All +/// fields are owned by the same allocator passed to `resolve`. +pub const Layout = struct { + allocator: Allocator, + /// `$PANTO_HOME` itself. + home: []u8, + /// `$PANTO_HOME/rocks/lua-<lua_version>/` — the versioned tree. + tree: []u8, + /// `<tree>/include/` — where Lua headers are staged. + include_dir: []u8, + /// `<tree>/share/lua/<short>/` — pure-Lua rocks. + share_lua_dir: []u8, + /// `<tree>/lib/lua/<short>/` — C rocks (.so/.dylib). + lib_lua_dir: []u8, + /// `<tree>/lib/luarocks/rocks-<short>/` — luarocks's rock metadata + /// per the standard tree layout. + rocks_metadata_dir: []u8, + /// `<tree>/etc/luarocks/` — luarocks's own config dir, where the + /// per-version `config-<short>.lua` lives. We set `SYSCONFDIR` to + /// this when constructing `luarocks.core.hardcoded`. + sysconfdir: []u8, + /// `<tree>/etc/luarocks/config-<short>.lua` — the path of the config + /// file we materialize at bootstrap. + config_file: []u8, + + pub fn deinit(self: Layout) void { + const a = self.allocator; + a.free(self.home); + a.free(self.tree); + a.free(self.include_dir); + a.free(self.share_lua_dir); + a.free(self.lib_lua_dir); + a.free(self.rocks_metadata_dir); + a.free(self.sysconfdir); + a.free(self.config_file); + } +}; + +/// Resolve every path the runtime cares about. Environment-driven: +/// - `PANTO_HOME` (explicit override) +/// - `XDG_DATA_HOME` (XDG default) +/// - `HOME` (fallback) +/// +/// Returns `error.NoHomeDirectory` if none of those are available and +/// no `PANTO_HOME` was set. +pub fn resolve( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, +) !Layout { + const home = try resolveHome(allocator, environ_map); + errdefer allocator.free(home); + + // `<home>/rocks/lua-<lua_version>` + const tree_subdir = try std.fmt.allocPrint( + allocator, + "lua-{s}", + .{manifest.lua_version}, + ); + defer allocator.free(tree_subdir); + const tree = try std.fs.path.join(allocator, &.{ home, "rocks", tree_subdir }); + errdefer allocator.free(tree); + + const short = manifest.lua_short_version; + + const include_dir = try std.fs.path.join(allocator, &.{ tree, "include" }); + errdefer allocator.free(include_dir); + const share_lua_dir = try std.fs.path.join(allocator, &.{ tree, "share", "lua", short }); + errdefer allocator.free(share_lua_dir); + const lib_lua_dir = try std.fs.path.join(allocator, &.{ tree, "lib", "lua", short }); + errdefer allocator.free(lib_lua_dir); + const rocks_subdir = try std.fmt.allocPrint(allocator, "rocks-{s}", .{short}); + defer allocator.free(rocks_subdir); + const rocks_metadata_dir = try std.fs.path.join( + allocator, + &.{ tree, "lib", "luarocks", rocks_subdir }, + ); + errdefer allocator.free(rocks_metadata_dir); + const sysconfdir = try std.fs.path.join(allocator, &.{ tree, "etc", "luarocks" }); + errdefer allocator.free(sysconfdir); + const config_basename = try std.fmt.allocPrint(allocator, "config-{s}.lua", .{short}); + defer allocator.free(config_basename); + const config_file = try std.fs.path.join(allocator, &.{ sysconfdir, config_basename }); + errdefer allocator.free(config_file); + + return .{ + .allocator = allocator, + .home = home, + .tree = tree, + .include_dir = include_dir, + .share_lua_dir = share_lua_dir, + .lib_lua_dir = lib_lua_dir, + .rocks_metadata_dir = rocks_metadata_dir, + .sysconfdir = sysconfdir, + .config_file = config_file, + }; +} + +/// Resolve `$PANTO_HOME` honoring overrides in the documented order. +/// Returns owned bytes. +fn resolveHome( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, +) ![]u8 { + if (environ_map.get("PANTO_HOME")) |explicit| { + return allocator.dupe(u8, explicit); + } + if (environ_map.get("XDG_DATA_HOME")) |xdg| { + return std.fs.path.join(allocator, &.{ xdg, "panto" }); + } + if (environ_map.get("HOME")) |hh| { + return std.fs.path.join(allocator, &.{ hh, ".local", "share", "panto" }); + } + return error.NoHomeDirectory; +} + +/// Create every directory referenced by `layout`, recursively. Idempotent +/// — existing directories are left alone. +pub fn ensureDirsExist(layout: Layout, io: Io) !void { + try makePathRecursive(io, layout.home); + try makePathRecursive(io, layout.tree); + try makePathRecursive(io, layout.include_dir); + try makePathRecursive(io, layout.share_lua_dir); + try makePathRecursive(io, layout.lib_lua_dir); + try makePathRecursive(io, layout.rocks_metadata_dir); + try makePathRecursive(io, layout.sysconfdir); +} + +fn makePathRecursive(io: Io, path: []const u8) !void { + Io.Dir.cwd().createDirPath(io, path) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "resolve: PANTO_HOME explicit override wins" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("PANTO_HOME", "/tmp/some/home"); + try env.put("XDG_DATA_HOME", "/should/be/ignored"); + + var layout = try resolve(testing.allocator, &env); + defer layout.deinit(); + + try testing.expectEqualStrings("/tmp/some/home", layout.home); + try testing.expect(std.mem.indexOf(u8, layout.tree, "/tmp/some/home/rocks/lua-") != null); + try testing.expect(std.mem.endsWith(u8, layout.share_lua_dir, "/share/lua/" ++ manifest.lua_short_version)); +} + +test "resolve: XDG_DATA_HOME is honored before HOME" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("XDG_DATA_HOME", "/x/data"); + try env.put("HOME", "/h"); + + var layout = try resolve(testing.allocator, &env); + defer layout.deinit(); + try testing.expectEqualStrings("/x/data/panto", layout.home); +} + +test "resolve: HOME fallback used if neither override is present" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try env.put("HOME", "/home/user"); + + var layout = try resolve(testing.allocator, &env); + defer layout.deinit(); + try testing.expectEqualStrings("/home/user/.local/share/panto", layout.home); +} + +test "resolve: error when no environment hint at all" { + var env: std.process.Environ.Map = .init(testing.allocator); + defer env.deinit(); + try testing.expectError(error.NoHomeDirectory, resolve(testing.allocator, &env)); +} diff --git a/src/self_exe.zig b/src/self_exe.zig new file mode 100644 index 0000000..6eda92d --- /dev/null +++ b/src/self_exe.zig @@ -0,0 +1,90 @@ +//! Resolve the absolute path of the currently running executable. +//! +//! Used to find the panto binary so we can launch ourselves as `panto +//! lua` from inside the bootstrap (the embedded luarocks shells out to +//! its configured `LUA` interpreter, which we point at a wrapper script +//! that exec's panto's `lua` subcommand). +//! +//! Zig's standard library doesn't expose a portable `selfExePath` in +//! the current API, so this module wraps the per-OS syscall. +//! +//! Supported: +//! - macOS / iOS / tvOS / watchOS: `_NSGetExecutablePath` + realpath +//! - Linux: `readlink("/proc/self/exe")` +//! - FreeBSD / NetBSD / DragonFly: `readlink("/proc/curproc/file")` +//! +//! Other targets currently return `error.SelfExePathUnavailable`. + +const std = @import("std"); +const builtin = @import("builtin"); + +const Allocator = std.mem.Allocator; + +pub const ResolveError = error{ + SelfExePathUnavailable, + SelfExePathTooLong, + SelfExePathReadFailed, + OutOfMemory, +}; + +/// Return the absolute, symlink-resolved path of the current process's +/// executable. Caller owns the returned slice. +pub fn selfExePathAlloc(allocator: Allocator) ResolveError![]u8 { + var buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try selfExePath(&buf); + return allocator.dupe(u8, path); +} + +/// Fill `buf` with the absolute path. Returns the slice that was used. +pub fn selfExePath(buf: []u8) ResolveError![]u8 { + return switch (builtin.os.tag) { + .macos, .ios, .tvos, .watchos => try macosSelfExePath(buf), + .linux => try readLinkSelfExe(buf, "/proc/self/exe"), + .freebsd, .netbsd, .dragonfly => try readLinkSelfExe(buf, "/proc/curproc/file"), + else => return ResolveError.SelfExePathUnavailable, + }; +} + +fn readLinkSelfExe(buf: []u8, link_path: []const u8) ResolveError![]u8 { + const link_z = std.posix.toPosixPath(link_path) catch return ResolveError.SelfExePathTooLong; + const n = posixReadlink(&link_z, buf) catch return ResolveError.SelfExePathReadFailed; + return buf[0..n]; +} + +/// Thin wrapper around the POSIX `readlink` syscall using libc directly, +/// since std.posix's surface here has been in flux. libc is linked into +/// panto already (Lua needs it), so this stays cheap. +extern "c" fn readlink(path: [*:0]const u8, buf: [*]u8, bufsize: usize) isize; + +fn posixReadlink(path: [*:0]const u8, buf: []u8) !usize { + const n = readlink(path, buf.ptr, buf.len); + if (n < 0) return error.SelfExePathReadFailed; + return @intCast(n); +} + +extern "c" fn _NSGetExecutablePath(buf: [*]u8, bufsize: *u32) c_int; + +fn macosSelfExePath(buf: []u8) ResolveError![]u8 { + var stage: [std.fs.max_path_bytes]u8 = undefined; + var size: u32 = @intCast(stage.len); + if (_NSGetExecutablePath(&stage, &size) != 0) return ResolveError.SelfExePathTooLong; + // _NSGetExecutablePath populates `size` on overflow; on success + // we need to find the NUL ourselves. + const len = std.mem.indexOfScalar(u8, &stage, 0) orelse return ResolveError.SelfExePathTooLong; + + // The returned path may include `..` and symlinks; canonicalize so + // downstream consumers don't have to. realpath(3) on POSIX, both + // libc-resident on macOS and Linux. + const path_z = std.posix.toPosixPath(stage[0..len]) catch return ResolveError.SelfExePathTooLong; + + // libc realpath: NULL second arg => malloc; we want stack buffer. + var real_buf: [std.fs.max_path_bytes]u8 = undefined; + if (libc_realpath(&path_z, &real_buf) == null) return ResolveError.SelfExePathReadFailed; + const real_len = std.mem.indexOfScalar(u8, &real_buf, 0) orelse return ResolveError.SelfExePathTooLong; + if (real_len > buf.len) return ResolveError.SelfExePathTooLong; + @memcpy(buf[0..real_len], real_buf[0..real_len]); + return buf[0..real_len]; +} + +extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*]u8; +const libc_realpath = realpath; diff --git a/src/subcommand.zig b/src/subcommand.zig new file mode 100644 index 0000000..97eb977 --- /dev/null +++ b/src/subcommand.zig @@ -0,0 +1,231 @@ +//! Subcommand dispatch for the `panto` CLI. +//! +//! Routes argv[1] to one of: +//! - `lua` — drop into the embedded standalone Lua interpreter +//! (panto's `lua.c` build), with luarocks's runtime +//! bootstrap completed first so `require("luarocks.*")` +//! and the configured rocks tree work the same as in +//! the agent process. +//! - `bootstrap` — run the luarocks runtime bootstrap pipeline only; +//! exit before entering any agent loop. Lets users +//! do first-run setup on a fresh machine without +//! starting a chat session. +//! - anything else (or absent) — fall through to the agent REPL. +//! +//! Both `lua` and `bootstrap` end up calling `luarocks_runtime.bootstrap` +//! before doing their thing. The agent path does the same; the only +//! difference is whether the agent loop runs afterward. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; + +const lua_bridge = @import("lua_bridge.zig"); +const luarocks_runtime = @import("luarocks_runtime.zig"); +const self_exe = @import("self_exe.zig"); + +const c = lua_bridge.c; + +pub const Action = enum { + /// Continue with the default agent REPL. + agent, + /// Bootstrap is already done; the dispatcher consumed the subcommand. + /// `main` should exit immediately. + done, +}; + +/// Inspect `argv[1]`, run the appropriate subcommand, and return what +/// the caller should do next. On `.agent`, the dispatcher leaves argv +/// untouched and `main` continues as before. On `.done`, the caller +/// must return promptly (the subcommand has already produced output). +/// +/// `panto_executable_path` is the absolute path of the running panto +/// binary, used both to wire up the embedded luarocks `LUA` variable +/// and to `exec` ourselves where needed. +pub fn dispatch( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + args: std.process.Args, + panto_executable_path: []const u8, +) !Action { + var it = args.iterate(); + defer it.deinit(); + _ = it.next(); // argv[0] + + const sub = it.next() orelse return .agent; + + if (std.mem.eql(u8, sub, "lua")) { + try runLuaSubcommand(allocator, io, environ_map, args, panto_executable_path); + return .done; + } + if (std.mem.eql(u8, sub, "bootstrap")) { + var force = false; + while (it.next()) |flag| { + if (std.mem.eql(u8, flag, "--force")) { + force = true; + } else { + std.log.err("panto bootstrap: unknown flag '{s}'", .{flag}); + return error.UnknownFlag; + } + } + try runBootstrapSubcommand(allocator, io, environ_map, panto_executable_path, .{ .force = force }); + return .done; + } + return .agent; +} + +pub const BootstrapOptions = struct { + /// Wipe the per-Lua-version tree before reinstalling everything. + /// Surfaced as `panto bootstrap --force`. Equivalent to deleting + /// `$PANTO_HOME/rocks/lua-X.Y.Z/` by hand and then running + /// `panto bootstrap`. + force: bool = false, +}; + +// --------------------------------------------------------------------------- +// `panto lua` +// --------------------------------------------------------------------------- + +extern "c" fn panto_lua_pmain(L: *c.lua_State, argc: c_int, argv: [*]?[*:0]u8) c_int; + +/// Drop into the embedded Lua standalone interpreter, with the +/// luarocks runtime bootstrap completed so `require("luarocks.*")` +/// and rocks installed under `$PANTO_HOME` are visible. +/// +/// argv is rewritten so the interpreter sees `lua [...args]` rather +/// than `panto lua [...args]` — matching upstream behavior. The first +/// argument visible to `pmain` is the program name; this matters for +/// `arg[0]` and error reporting. +fn runLuaSubcommand( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + args: std.process.Args, + panto_executable_path: []const u8, +) !void { + // Build a fresh lua_State that we own, configure it like luarocks + // expects, then hand it to `pmain`. + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + + // Run bootstrap against this state. This installs the embedded + // searcher, configures package.path/cpath, and stages on-disk + // resources. We deliberately do NOT call `luaL_openlibs` here — + // `pmain` does that itself, and we want exactly the upstream + // ordering for everything that runs inside the REPL. + // + // The searcher install only requires `package.searchers` to be + // present; the stock libs ship it. We open libs once here just + // to satisfy that, then pmain's own `luaL_openlibs` is idempotent. + c.luaL_openlibs(L); + const rt = try luarocks_runtime.bootstrap( + allocator, + io, + environ_map, + L, + panto_executable_path, + ); + defer rt.deinit(); + + // Re-create the argv the standalone interpreter expects. argv[0] + // is the program name; argv[1..] are the user's args. + var raw_args = args.iterate(); + defer raw_args.deinit(); + _ = raw_args.next(); // panto + _ = raw_args.next(); // lua + + var argv_list: std.array_list.Managed([:0]u8) = .init(allocator); + defer { + for (argv_list.items) |s| allocator.free(s); + argv_list.deinit(); + } + + // Program name first. + try argv_list.append(try allocator.dupeZ(u8, "lua")); + while (raw_args.next()) |a| { + try argv_list.append(try allocator.dupeZ(u8, a)); + } + + // Build a `[*]?[*:0]u8` argv pointer array. lua.c expects a + // NULL-terminated array (it uses `argv[i]` indexed access through + // argc; the trailing NULL is conventional for C `main`). + var argv_c: std.array_list.Managed(?[*:0]u8) = .init(allocator); + defer argv_c.deinit(); + for (argv_list.items) |s| { + try argv_c.append(s.ptr); + } + try argv_c.append(null); + + const exit_code = panto_lua_pmain(L, @intCast(argv_list.items.len), argv_c.items.ptr); + if (exit_code != 0) std.process.exit(@intCast(exit_code)); +} + +// --------------------------------------------------------------------------- +// `panto bootstrap` +// --------------------------------------------------------------------------- + +/// Run the luarocks bootstrap and exit. Useful for first-run setup on +/// a clean machine (downloads + compiles batteries, stages headers, +/// materializes config) and for CI/scripted installs. +/// +/// Idempotent: subsequent invocations no-op fast, unless `force` was +/// passed — then the entire per-Lua-version tree is wiped before the +/// regular bootstrap pipeline runs. +fn runBootstrapSubcommand( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + panto_executable_path: []const u8, + opts: BootstrapOptions, +) !void { + if (opts.force) { + try luarocks_runtime.wipeTree(allocator, io, environ_map); + } + + const L = c.luaL_newstate() orelse return error.LuaInitFailed; + defer c.lua_close(L); + c.luaL_openlibs(L); + + const rt = try luarocks_runtime.bootstrap( + allocator, + io, + environ_map, + L, + panto_executable_path, + ); + defer rt.deinit(); + + // Pleasant single-line confirmation. The interesting bits (rock + // installs etc.) print their own progress. + std.log.info( + "panto bootstrap: tree ready at {s}", + .{rt.layout.tree}, + ); +} + +// --------------------------------------------------------------------------- +// `panto lua` argv plumbing — sketched against the older Args API for +// reference (kept here so the design notes survive the implementation). +// --------------------------------------------------------------------------- +// +// Because we own the `lua_State` end-to-end, the subcommand can also +// expose extra panto-specific globals to user code (e.g. surface the +// resolved $PANTO_HOME) without disturbing upstream `lua.c` behavior. +// Step out of scope for the current makeover; add when needed. + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +// Note: `dispatch` reads from the process's real argv, which isn't +// controllable from a unit test. The behavior is exercised by +// integration runs of the panto binary. We test the smaller pieces. +// +// Suppress dead-code warnings for `self_exe` (it's used by main, not +// by tests in this module). +test { + _ = self_exe; +} |