From 800b2c4b6115845f6bf15d90484b3e80e81a606f Mon Sep 17 00:00:00 2001 From: t Date: Wed, 1 Jul 2026 15:36:20 -0600 Subject: Load extensions via unified policy, entry/activate, rocks and paths Replace the split [tools]/[extensions] config sections and the two-stage load gate with a single model: - [extensions] is now one policy resolved across all layers. Rules from every layer are kept (not clobbered) and resolved by last-match-wins after sorting by (layer, glob specificity, deny-last), so a base layer can carve one name out of an otherwise-denied group and a higher layer's broad rule still wins. Default is allow; whitelist via deny=["**"]. - Registration is deferred: a source returns an entry {name, activate} (or a list, or the sugar tool table), is always eval'd side-effect-free, and only permitted names get activate()d. Identity is the declared name, not the filename. Collapses the old pre/post-load two-stage gate. - The loader runs two passes (eval -> shadow -> filter -> activate) with precedence project>user>base and, within a layer, rocks, activate = }`, + /// - the sugar tool form `{ name, description, schema, handler }` + /// (no `activate`; activation registers it as a tool), or + /// - a list `{ , , ... }` of the above. /// - /// All `panto.ext.register_tool` calls in the script run during this - /// call. The runtime then harvests the registrations table, - /// transfers handler functions into the Lua registry (one `luaL_ref` - /// per tool), and records each tool's metadata in `self.decls`. - pub fn loadExtension( + /// `package_root`, if provided, is prepended to `package.path`. + pub fn evalEntries( self: *LuaRuntime, script_path: []const u8, package_root: ?[]const u8, + out: *std.array_list.Managed(Entry), ) !void { + const L = self.L; const path_z = try self.allocator.dupeZ(u8, script_path); defer self.allocator.free(path_z); - // Reset the registrations table to empty so we only harvest the - // calls made by *this* script (not accumulated from prior ones). - // The bridge re-installs the registrations table when called; - // we want to call only that subset. Instead of re-installing - // everything (which would also reset the panto global, fine), - // create a fresh registrations table directly via the bridge. - lua_bridge.resetRegistrations(self.L); - if (package_root) |root| { const root_z = try self.allocator.dupeZ(u8, root); defer self.allocator.free(root_z); - try prependPackagePath(self.L, root_z); + try prependPackagePath(L, root_z); } - lua_bridge.loadFile(self.L, path_z) catch |err| { - logTopAsError(self.L, "lua: failed to load extension"); - return err; - }; + if (c.luaL_loadfilex(L, path_z.ptr, null) != 0) { + logTopAsError(L, "lua: failed to load extension"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuaLoadFailed; + } + if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { + logTopAsError(L, "lua: failed to evaluate extension"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuaRunFailed; + } + // Stack: ..., returned_value + defer c.lua_settop(L, c.lua_gettop(L) - 1); // pop returned value + try self.collectEntriesFromTop(script_path, out); + } - // Harvest the registrations table into our state. - try self.harvestAndStoreHandlers(); + /// Evaluate an installed rock (or any `require`-able module) and append + /// the entries it declares to `out`. Same contract as `evalEntries`: the + /// module's chunk must be side-effect-free and return an entry, sugar + /// tool, or list. Used to load `extensions.rocks`. + pub fn evalEntriesFromModule( + self: *LuaRuntime, + module_name: []const u8, + out: *std.array_list.Managed(Entry), + ) !void { + const L = self.L; + _ = c.lua_getglobal(L, "require"); + _ = c.lua_pushlstring(L, module_name.ptr, module_name.len); + if (c.lua_pcallk(L, 1, 1, 0, 0, null) != 0) { + logTopAsError(L, "lua: failed to require rock"); + c.lua_settop(L, c.lua_gettop(L) - 1); + return error.LuaRunFailed; + } + // Stack: ..., returned_value + defer c.lua_settop(L, c.lua_gettop(L) - 1); + try self.collectEntriesFromTop(module_name, out); } - /// Load a single-tool Lua script and register the table it returns - /// as if `panto.ext.register_tool` had been called on that table. - /// - /// The script's top-level chunk must return a table with the same - /// shape that `panto.ext.register_tool` accepts: - /// `{ name, description, schema, handler }`. This is the ergonomic - /// form supported under a `tools/` directory. - pub fn loadTool( + /// Classify the value at the top of the stack (does not pop it) as an + /// entry, a sugar tool, or a list of those, appending each to `out`. + fn collectEntriesFromTop( self: *LuaRuntime, - script_path: []const u8, - package_root: ?[]const u8, + what: []const u8, + out: *std.array_list.Managed(Entry), ) !void { - const path_z = try self.allocator.dupeZ(u8, script_path); - defer self.allocator.free(path_z); + const L = self.L; + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + std.log.err("lua: extension '{s}' must return an entry table or list", .{what}); + return error.BadRegistration; + } - lua_bridge.resetRegistrations(self.L); + // A single entry/tool has a string `name`; otherwise it's a list. + _ = c.lua_getfield(L, -1, "name"); + const has_name = c.lua_type(L, -1) == lua_bridge.T_STRING; + c.lua_settop(L, c.lua_gettop(L) - 1); - if (package_root) |root| { - const root_z = try self.allocator.dupeZ(u8, root); - defer self.allocator.free(root_z); - try prependPackagePath(self.L, root_z); + if (has_name) { + try self.readOneEntry(what, out); + return; } - // Run the file expecting exactly one returned value (the tool - // table). Use luaL_loadfilex + lua_pcallk directly so we can - // ask for a return value (the bridge's loadFile discards them). - // - // Push `panto.ext.register_tool` *first*, then load+run the chunk - // so its return value naturally lands above it; calling pcall then - // consumes both in the right order. We reach `register_tool` - // through the `ext` table directly (it exists from `install`, - // independent of whether the native module table has been wired). - const L = self.L; - lua_bridge.pushExtTable(L); - _ = c.lua_getfield(L, -1, "register_tool"); - // Stack: ..., ext, register_tool. Collapse to just register_tool. - c.lua_copy(L, -1, -2); // overwrite `ext` with `register_tool` - c.lua_settop(L, c.lua_gettop(L) - 1); // pop duplicate - // Stack: ..., register_tool - - if (c.luaL_loadfilex(L, path_z.ptr, null) != 0) { - logTopAsError(L, "lua: failed to load tool"); - c.lua_settop(L, c.lua_gettop(L) - 2); // pop err + register_tool - return error.LuaLoadFailed; + const n: usize = @intCast(c.lua_rawlen(L, -1)); + if (n == 0) { + std.log.err("lua: extension '{s}' returned a table with no `name` and no entries", .{what}); + return error.BadRegistration; } - if (c.lua_pcallk(L, 0, 1, 0, 0, null) != 0) { - logTopAsError(L, "lua: failed to run tool"); - c.lua_settop(L, c.lua_gettop(L) - 2); // pop err + register_tool - return error.LuaRunFailed; + var i: usize = 1; + while (i <= n) : (i += 1) { + _ = c.lua_rawgeti(L, -1, @intCast(i)); // push element + if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { + c.lua_settop(L, c.lua_gettop(L) - 1); + std.log.err("lua: extension '{s}' entry {d} is not a table", .{ what, i }); + return error.BadRegistration; + } + try self.readOneEntry(what, out); + c.lua_settop(L, c.lua_gettop(L) - 1); // pop element } - // Stack: ..., register_tool, returned_value - if (c.lua_type(L, -1) != lua_bridge.T_TABLE) { - c.lua_settop(L, c.lua_gettop(L) - 2); // pop both - std.log.err( - "lua: tool script '{s}' must return a table", - .{script_path}, - ); + } + + /// Read one entry table sitting at the top of the stack (does not pop + /// it), ref its activator, and append an `Entry` to `out`. + fn readOneEntry( + self: *LuaRuntime, + script_path: []const u8, + out: *std.array_list.Managed(Entry), + ) !void { + const L = self.L; + const t = c.lua_gettop(L); // absolute index of the entry table + + _ = c.lua_getfield(L, t, "name"); + if (c.lua_type(L, -1) != lua_bridge.T_STRING) { + c.lua_settop(L, c.lua_gettop(L) - 1); + std.log.err("lua: extension '{s}' entry has no string `name`", .{script_path}); return error.BadRegistration; } + var nlen: usize = 0; + const nptr = c.lua_tolstring(L, -1, &nlen); + const name = try self.allocator.dupe(u8, nptr[0..nlen]); + errdefer self.allocator.free(name); + c.lua_settop(L, c.lua_gettop(L) - 1); // pop name + + var ref: c_int = undefined; + var is_tool_table = false; + + _ = c.lua_getfield(L, t, "activate"); + if (c.lua_type(L, -1) == lua_bridge.T_FUNCTION) { + ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); // refs + pops the fn + } else { + c.lua_settop(L, c.lua_gettop(L) - 1); // pop non-function activate + _ = c.lua_getfield(L, t, "handler"); + const is_handler = c.lua_type(L, -1) == lua_bridge.T_FUNCTION; + c.lua_settop(L, c.lua_gettop(L) - 1); // pop handler probe + if (!is_handler) { + std.log.err("lua: extension '{s}' entry '{s}' has neither `activate` nor `handler`", .{ script_path, name }); + return error.BadRegistration; + } + // Sugar tool form: ref the table itself; activation registers it. + c.lua_pushvalue(L, t); // copy of the tool table + ref = c.luaL_ref(L, lua_bridge.LUA_REGISTRYINDEX); // refs + pops copy + is_tool_table = true; + } - // Invoke register_tool(returned_table). Same validation, schema - // serialization, and registrations-table append logic as an - // extension's `panto.ext.register_tool` call. - if (c.lua_pcallk(L, 1, 0, 0, 0, null) != 0) { - logTopAsError(L, "lua: register_tool failed for tool script"); - return error.LuaRunFailed; + try out.append(.{ .name = name, .ref = ref, .is_tool_table = is_tool_table }); + } + + /// Activate an entry: run its `activate` function (or, for the sugar + /// form, `register_tool()`), then harvest the tools, + /// commands, and event handlers it registered. Consumes the entry + /// (unrefs its ref, frees its name). + pub fn activateEntry(self: *LuaRuntime, entry: Entry) !void { + const L = self.L; + // Isolate this entry's registrations so harvest picks up only its own. + lua_bridge.resetRegistrations(L); + + if (entry.is_tool_table) { + lua_bridge.pushExtTable(L); + _ = c.lua_getfield(L, -1, "register_tool"); + c.lua_copy(L, -1, -2); // overwrite `ext` with `register_tool` + c.lua_settop(L, c.lua_gettop(L) - 1); // Stack: register_tool + _ = c.lua_rawgeti(L, lua_bridge.LUA_REGISTRYINDEX, entry.ref); // push tool table + if (c.lua_pcallk(L, 1, 0, 0, 0, null) != 0) { + logTopAsError(L, "lua: register_tool failed for tool"); + c.lua_settop(L, c.lua_gettop(L) - 1); + self.finishEntry(entry); + return error.LuaRunFailed; + } + } else { + _ = c.lua_rawgeti(L, lua_bridge.LUA_REGISTRYINDEX, entry.ref); // push activate fn + if (c.lua_pcallk(L, 0, 0, 0, 0, null) != 0) { + logTopAsError(L, "lua: extension activate() failed"); + c.lua_settop(L, c.lua_gettop(L) - 1); + self.finishEntry(entry); + return error.LuaRunFailed; + } } + self.finishEntry(entry); try self.harvestAndStoreHandlers(); } + /// Discard an entry without activating it (shadowed or denied). + pub fn dropEntry(self: *LuaRuntime, entry: Entry) void { + self.finishEntry(entry); + } + + fn finishEntry(self: *LuaRuntime, entry: Entry) void { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, entry.ref); + self.allocator.free(entry.name); + } + /// Walk the registrations table that the script just populated. /// For each entry: /// - Copy `name`, `description`, `schema_json` into owned bytes. @@ -1285,17 +1382,68 @@ fn freeResults(results: []panto.ToolCallResult) void { fn writeTempScript(dir: Io.Dir, name: []const u8, source: []const u8) ![]const u8 { // `panto` is not a global; extensions reach it via `require('panto')`. - // Prepend the require so these tests exercise the real preload path - // without each script repeating the boilerplate. + // The register-based `source` body is wrapped in an entry whose + // `activate()` runs it, matching the new deferred-registration model. var src_buf: [16384]u8 = undefined; - const data = try std.fmt.bufPrint(&src_buf, "local panto = require(\"panto\")\n{s}", .{source}); + const data = try std.fmt.bufPrint( + &src_buf, + "local panto = require(\"panto\")\nreturn {{ name = \"_t\", activate = function()\n{s}\nend }}\n", + .{source}, + ); try dir.writeFile(testing.io, .{ .sub_path = name, .data = data }); var buf: [std.fs.max_path_bytes]u8 = undefined; const n = try dir.realPathFile(testing.io, name, &buf); return testing.allocator.dupe(u8, buf[0..n]); } -test "loadExtension records tool decls" { +/// Test helper: eval one script and activate every entry it declares +/// (no policy, no shadowing) — the single-file analogue of the loader. +fn loadScript(rt: *LuaRuntime, path: []const u8, package_root: ?[]const u8) !void { + var entries: std.array_list.Managed(LuaRuntime.Entry) = .init(rt.allocator); + defer entries.deinit(); + rt.evalEntries(path, package_root, &entries) catch |e| { + for (entries.items) |it| rt.dropEntry(it); + return e; + }; + for (entries.items, 0..) |it, idx| { + rt.activateEntry(it) catch |e| { + var j = idx + 1; + while (j < entries.items.len) : (j += 1) rt.dropEntry(entries.items[j]); + return e; + }; + } +} + +test "evalEntriesFromModule loads a rock's entries via require" { + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + // Stand in for an installed rock: preload a module returning entries. + const preload = + \\package.preload["fakerock"] = function() + \\ local panto = require("panto") + \\ return { + \\ { name = "rock.a", activate = function() + \\ panto.ext.register_tool { name = "rock.a", description = "a", + \\ schema = { type = "object" }, handler = function() return "A" end } + \\ end }, + \\ { name = "rock.b", description = "b", schema = { type = "object" }, + \\ handler = function() return "B" end }, + \\ } + \\end + ; + if (c.luaL_loadstring(rt.L, preload) != 0) return error.TestSetupFailed; + if (c.lua_pcallk(rt.L, 0, 0, 0, 0, null) != 0) return error.TestSetupFailed; + + var entries: std.array_list.Managed(LuaRuntime.Entry) = .init(testing.allocator); + defer entries.deinit(); + try rt.evalEntriesFromModule("fakerock", &entries); + try testing.expectEqual(@as(usize, 2), entries.items.len); + for (entries.items) |e| try rt.activateEntry(e); + try testing.expectEqual(@as(usize, 2), rt.toolCount()); +} + +test "loadScript records tool decls" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -1312,12 +1460,12 @@ test "loadExtension records tool decls" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); try testing.expectEqual(@as(usize, 1), rt.toolCount()); try testing.expectEqualStrings("greet", rt.decls.items[0].name); } -test "loadExtension records command decls" { +test "loadScript records command decls" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -1334,7 +1482,7 @@ test "loadExtension records command decls" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); const cmds = rt.commandList(); try testing.expectEqual(@as(usize, 1), cmds.len); try testing.expectEqualStrings("shout", cmds[0].name); @@ -1369,7 +1517,7 @@ test "runCommand surfaces a Lua error" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); const cmds = rt.commandList(); var err_buf: [4096]u8 = undefined; @@ -1394,7 +1542,7 @@ test "duplicate command name within runtime is rejected" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try testing.expectError(error.DuplicateCommand, rt.loadExtension(path, null)); + try testing.expectError(error.DuplicateCommand, loadScript(rt, path, null)); } test "invokeBatch runs each call through a coroutine and returns the result" { @@ -1418,7 +1566,7 @@ test "invokeBatch runs each call through a coroutine and returns the result" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); var src = rt.toolSource(); @@ -1462,7 +1610,7 @@ test "module-global state survives across calls in the same runtime" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); var src = rt.toolSource(); const calls = [_]panto.ToolCall{ @@ -1516,7 +1664,7 @@ test "handler crash: per-call error surfaces, sibling calls succeed" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); var src = rt.toolSource(); const calls = [_]panto.ToolCall{ @@ -1566,7 +1714,7 @@ test "directory-style extension can require sibling modules" { .data = \\local panto = require("panto") \\local util = require("util") - \\panto.ext.register_tool { + \\return { \\ name = "shout", description = "uppercase + bang", \\ schema = { type = "object", properties = { text = { type = "string" } } }, \\ handler = function(input) return util.shout(input.text) end, @@ -1584,7 +1732,7 @@ test "directory-style extension can require sibling modules" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(init_path, ext_dir); + try loadScript(rt, init_path, ext_dir); var src = rt.toolSource(); @@ -1611,7 +1759,7 @@ test "yielding handler with no event loop surfaces LuaHandlerYielded" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); var src = rt.toolSource(); const calls = [_]panto.ToolCall{.{ .tool_name = "sleeper", .input = "{}" }}; @@ -1696,7 +1844,7 @@ test "scheduler: yielding handler is resumed by libuv" { defer luarocks_rt.deinit(); try rt.installScheduler(); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); var src = rt.toolSource(); const calls = [_]panto.ToolCall{ @@ -1742,7 +1890,7 @@ test "scheduler: handler that yields without arming libuv work is surfaced as an ; const path = try writeTempScript(tmp.dir, "abandon.lua", source); defer testing.allocator.free(path); - try rt.loadExtension(path, null); + try loadScript(rt, path, null); var src = rt.toolSource(); const calls = [_]panto.ToolCall{ @@ -1837,7 +1985,7 @@ test "scheduler: two real std.shell calls in one batch do not deadlock" { defer testing.allocator.free(tools_dir); const shell_path = try std.fs.path.join(testing.allocator, &.{ tools_dir, "shell.lua" }); defer testing.allocator.free(shell_path); - try rt.loadTool(shell_path, tools_dir); + try loadScript(rt, shell_path, tools_dir); var src = rt.toolSource(); const calls = [_]panto.ToolCall{ @@ -1871,7 +2019,7 @@ test "scheduler: shell timeout is measured from dispatch, not a stale loop clock defer testing.allocator.free(tools_dir); const shell_path = try std.fs.path.join(testing.allocator, &.{ tools_dir, "shell.lua" }); defer testing.allocator.free(shell_path); - try rt.loadTool(shell_path, tools_dir); + try loadScript(rt, shell_path, tools_dir); var src = rt.toolSource(); @@ -1912,7 +2060,7 @@ test "scheduler: three real std.shell calls with explicit timeout all succeed" { defer testing.allocator.free(tools_dir); const shell_path = try std.fs.path.join(testing.allocator, &.{ tools_dir, "shell.lua" }); defer testing.allocator.free(shell_path); - try rt.loadTool(shell_path, tools_dir); + try loadScript(rt, shell_path, tools_dir); var src = rt.toolSource(); const calls = [_]panto.ToolCall{ @@ -1950,7 +2098,7 @@ test "scheduler: two real std.read calls in one batch do not deadlock" { defer testing.allocator.free(tools_dir); const read_path = try std.fs.path.join(testing.allocator, &.{ tools_dir, "read.lua" }); defer testing.allocator.free(read_path); - try rt.loadTool(read_path, tools_dir); + try loadScript(rt, read_path, tools_dir); const in0 = try std.fmt.allocPrint(testing.allocator, "{{\"path\":\"{s}/shell.lua\"}}", .{tools_dir}); defer testing.allocator.free(in0); @@ -1974,7 +2122,7 @@ test "scheduler: two real std.read calls in one batch do not deadlock" { try testing.expect(okText(results[1]).len > 0); } -test "loadExtension: duplicate tool name from a second extension errors" { +test "loadScript: duplicate tool name from a second extension errors" { var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -1999,7 +2147,7 @@ test "loadExtension: duplicate tool name from a second extension errors" { var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); - try rt.loadExtension(pa, null); + try loadScript(rt, pa, null); - try testing.expectError(error.DuplicateTool, rt.loadExtension(pb, null)); + try testing.expectError(error.DuplicateTool, loadScript(rt, pb, null)); } -- cgit v1.3