From 1beefefc69beee214430eb5bd2528a4f5692d2a8 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 1 Jun 2026 08:21:00 -0600 Subject: Rename system extension layer to base for clarity Replace all references to the "system" layer with "base" to better reflect its role as the foundational extension/tool layer in panto's hierarchy. The layer hierarchy now consistently uses: project > user > base. Includes: - Update agent README and build.zig documentation - Refactor tool registry and config module to support layered lookups - Add TestHarness abstraction for cleaner test setup - Improve JSON serialization with wire-encoded tool names - Add glob pattern matching for tool/extension discovery --- src/extension_loader.zig | 223 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 179 insertions(+), 44 deletions(-) (limited to 'src/extension_loader.zig') diff --git a/src/extension_loader.zig b/src/extension_loader.zig index 2fd86ae..b84eba5 100644 --- a/src/extension_loader.zig +++ b/src/extension_loader.zig @@ -1,22 +1,22 @@ //! Extension and tool discovery: walk well-known directories, locate Lua //! files, and load each into a long-lived `LuaRuntime`. //! -//! Two parallel namespaces are scanned, each at three scopes — system, -//! user, and project. Project shadows user shadows system. +//! Two parallel namespaces are scanned, each at three scopes — base, +//! user, and project. Project shadows user shadows base. //! //! Extensions (full-featured; call `panto.register_tool` from a script //! that may register many tools): -//! 1. `$PANTO_HOME/agent/extensions/` ("system") +//! 1. `$PANTO_HOME/agent/extensions/` ("base") //! 2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/` ("user") //! 3. `./.panto/extensions/` ("project") //! //! Tools (ergonomic single-tool form; the script returns one table //! shaped like the argument to `panto.register_tool`): -//! 1. `$PANTO_HOME/agent/tools/` ("system") +//! 1. `$PANTO_HOME/agent/tools/` ("base") //! 2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/tools/` ("user") //! 3. `./.panto/tools/` ("project") //! -//! The `system` layer is populated at bootstrap from files embedded +//! The `base` layer is populated at bootstrap from files embedded //! into the panto binary (see `build/gen_agent_embed.zig` and //! `luarocks_runtime.stageAgentTree`). It is panto's "batteries" tier: //! ships in the binary, but every entry is individually shadowable @@ -33,7 +33,7 @@ //! Conflict rules: //! - Within one directory, two entries with the same logical name //! are an error. -//! - Project shadows user shadows system *within the same kind* +//! - Project shadows user shadows base *within the same kind* //! (extension or tool). Extensions and tools live in distinct //! namespaces for shadowing; the registered *tool names* still //! share one global namespace across both, and collisions there @@ -44,11 +44,32 @@ const std = @import("std"); const panto = @import("panto"); const lua_runtime = @import("lua_runtime.zig"); +const config_file = @import("config_file.zig"); const Allocator = std.mem.Allocator; const Io = std.Io; const LuaRuntime = lua_runtime.LuaRuntime; +/// Availability policies for the two namespaces. Both are optional; a +/// null policy permits everything. `extensions` gates extension/tool +/// *entries* by their logical (file/dir) name before loading; `tools` +/// gates the *registered tool names* (e.g. `std.read`) after loading. +pub const Policies = struct { + extensions: ?*const config_file.Policy = null, + tools: ?*const config_file.Policy = null, +}; + +fn policyPermits(p: ?*const config_file.Policy, name: []const u8) bool { + const pol = p orelse return true; + return pol.permits(name); +} + +/// Free-function form of `Policy.permits` taking the policy by pointer, +/// matching the `fn(ctx, name) bool` shape `LuaRuntime.filterTools` wants. +fn permitsByPtr(pol: *const config_file.Policy, name: []const u8) bool { + return pol.permits(name); +} + /// A discovered extension or tool before loading. Owns its strings. const Found = struct { /// Logical name (basename without `.lua`, or directory name). @@ -72,13 +93,13 @@ const Found = struct { pub const Source = enum { /// Staged at bootstrap from files embedded in the panto binary. /// Lowest priority — shadowed by user and project. - system, + base, user, project, pub fn label(self: Source) []const u8 { return switch (self) { - .system => "system", + .base => "base", .user => "user", .project => "project", }; @@ -101,9 +122,9 @@ pub const Kind = enum { /// paths into `runtime`. Returns the number of registered tools added /// to the runtime by this call. /// -/// `system_agent_dir`, when non-null, is the path under which embedded -/// system tools/extensions have been staged — typically -/// `$PANTO_HOME/agent/`. Pass `null` to skip the system layer entirely +/// `base_agent_dir`, when non-null, is the path under which embedded +/// base tools/extensions have been staged — typically +/// `$PANTO_HOME/agent/`. Pass `null` to skip the base layer entirely /// (mostly useful for tests). /// /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The @@ -112,15 +133,16 @@ pub fn discoverAndLoad( allocator: Allocator, io: Io, environ_map: *const std.process.Environ.Map, - system_agent_dir: ?[]const u8, + base_agent_dir: ?[]const u8, runtime: *LuaRuntime, + policies: Policies, ) !usize { - const sys_ext = if (system_agent_dir) |d| + const sys_ext = if (base_agent_dir) |d| try std.fs.path.join(allocator, &.{ d, kindSubdir(.extension) }) else null; defer if (sys_ext) |d| allocator.free(d); - const sys_tool = if (system_agent_dir) |d| + const sys_tool = if (base_agent_dir) |d| try std.fs.path.join(allocator, &.{ d, kindSubdir(.tool) }) else null; @@ -141,27 +163,28 @@ pub fn discoverAndLoad( io, runtime, .{ - .system_extensions = sys_ext, + .base_extensions = sys_ext, .user_extensions = user_ext, .project_extensions = project_ext, - .system_tools = sys_tool, + .base_tools = sys_tool, .user_tools = user_tool, .project_tools = project_tool, }, + policies, ); } /// Set of search paths consumed by `loadFromDirs`. Any field may be /// null; missing directories on disk are silently skipped. /// -/// Scan order is system → user → project. `applyShadowing` keeps the +/// Scan order is base → user → project. `applyShadowing` keeps the /// *last* occurrence of each (kind, name), so project entries win, -/// then user, then system. +/// then user, then base. pub const DirSet = struct { - system_extensions: ?[]const u8 = null, + base_extensions: ?[]const u8 = null, user_extensions: ?[]const u8 = null, project_extensions: ?[]const u8 = null, - system_tools: ?[]const u8 = null, + base_tools: ?[]const u8 = null, user_tools: ?[]const u8 = null, project_tools: ?[]const u8 = null, }; @@ -173,6 +196,7 @@ pub fn loadFromDirs( io: Io, runtime: *LuaRuntime, dirs: DirSet, + policies: Policies, ) !usize { var found: std.array_list.Managed(Found) = .init(allocator); defer { @@ -180,15 +204,38 @@ pub fn loadFromDirs( found.deinit(); } - if (dirs.system_extensions) |d| try scanDir(allocator, io, d, .system, .extension, &found); + if (dirs.base_extensions) |d| try scanDir(allocator, io, d, .base, .extension, &found); if (dirs.user_extensions) |d| try scanDir(allocator, io, d, .user, .extension, &found); if (dirs.project_extensions) |d| try scanDir(allocator, io, d, .project, .extension, &found); - if (dirs.system_tools) |d| try scanDir(allocator, io, d, .system, .tool, &found); + if (dirs.base_tools) |d| try scanDir(allocator, io, d, .base, .tool, &found); if (dirs.user_tools) |d| try scanDir(allocator, io, d, .user, .tool, &found); if (dirs.project_tools) |d| try scanDir(allocator, io, d, .project, .tool, &found); try applyShadowing(allocator, &found); + // Gate *entries* by the extensions policy (matched on logical name). + // This drops whole scripts before they run — a denied extension + // never executes, never registers tools. The tools policy is + // applied post-load against registered tool names. + if (policies.extensions) |_| { + var keep: std.array_list.Managed(Found) = .init(allocator); + errdefer { + for (keep.items) |*f| f.deinit(allocator); + keep.deinit(); + } + for (found.items) |*f| { + if (policyPermits(policies.extensions, f.name)) { + try keep.append(f.*); + } else { + std.log.debug("{s}: '{s}' denied by extensions policy", .{ f.kind.label(), f.name }); + f.deinit(allocator); + } + } + found.clearRetainingCapacity(); + try found.appendSlice(keep.items); + keep.deinit(); + } + const before = runtime.toolCount(); for (found.items) |f| { const load_result = switch (f.kind) { @@ -214,6 +261,13 @@ pub fn loadFromDirs( .{ f.kind.label(), f.name, f.source.label() }, ); } + + // Apply the tools policy to *registered tool names* (e.g. `std.read`). + if (policies.tools) |pol| { + const dropped = runtime.filterTools(pol, permitsByPtr); + if (dropped > 0) std.log.debug("tools: {d} tool(s) removed by tools policy", .{dropped}); + } + return runtime.toolCount() - before; } @@ -581,7 +635,7 @@ test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = user_path, .project_extensions = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); // Invoke the tool through the source and verify the project handler ran. @@ -623,7 +677,7 @@ test "loadFromDirs: tool-name collision between extensions errors" { const result = loadFromDirs(testing.allocator, testing.io, rt, .{ .project_extensions = ext_path, - }); + }, .{}); try testing.expectError(error.DuplicateTool, result); } @@ -650,7 +704,7 @@ test "loadFromDirs: tools/ directory — single-file tool form" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -690,7 +744,7 @@ test "loadFromDirs: tools/ directory — directory-style tool with sibling requi const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .project_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -738,7 +792,7 @@ test "loadFromDirs: project tool shadows user tool of the same name" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = user_path, .project_tools = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -749,19 +803,19 @@ test "loadFromDirs: project tool shadows user tool of the same name" { try testing.expectEqualStrings("PROJECT", results[0].ok); } -test "loadFromDirs: project tool shadows user shadows system" { +test "loadFromDirs: project tool shadows user shadows base" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); - try makeDir(tmp.dir, "system_tools"); + try makeDir(tmp.dir, "base_tools"); try makeDir(tmp.dir, "user_tools"); try makeDir(tmp.dir, "project_tools"); - try writeFile(tmp.dir, "system_tools/greet.lua", + try writeFile(tmp.dir, "base_tools/greet.lua", \\return { - \\ name = "greet", description = "system version", + \\ name = "greet", description = "base version", \\ schema = { type = "object" }, - \\ handler = function(input) return "SYSTEM" end, + \\ handler = function(input) return "BASE" end, \\} ); try writeFile(tmp.dir, "user_tools/greet.lua", @@ -780,7 +834,7 @@ test "loadFromDirs: project tool shadows user shadows system" { ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf); + const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf); const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]); defer testing.allocator.free(sys_path); const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); @@ -794,10 +848,10 @@ test "loadFromDirs: project tool shadows user shadows system" { defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ - .system_tools = sys_path, + .base_tools = sys_path, .user_tools = user_path, .project_tools = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -808,18 +862,18 @@ test "loadFromDirs: project tool shadows user shadows system" { try testing.expectEqualStrings("PROJECT", results[0].ok); } -test "loadFromDirs: user tool shadows system tool when no project entry" { +test "loadFromDirs: user tool shadows base tool when no project entry" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); - try makeDir(tmp.dir, "system_tools"); + try makeDir(tmp.dir, "base_tools"); try makeDir(tmp.dir, "user_tools"); - try writeFile(tmp.dir, "system_tools/greet.lua", + try writeFile(tmp.dir, "base_tools/greet.lua", \\return { - \\ name = "greet", description = "system", + \\ name = "greet", description = "base", \\ schema = { type = "object" }, - \\ handler = function(input) return "SYSTEM" end, + \\ handler = function(input) return "BASE" end, \\} ); try writeFile(tmp.dir, "user_tools/greet.lua", @@ -831,7 +885,7 @@ test "loadFromDirs: user tool shadows system tool when no project entry" { ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf); + const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf); const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]); defer testing.allocator.free(sys_path); const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); @@ -842,9 +896,9 @@ test "loadFromDirs: user tool shadows system tool when no project entry" { defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ - .system_tools = sys_path, + .base_tools = sys_path, .user_tools = user_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -895,6 +949,87 @@ test "loadFromDirs: extension and tool share a *file* name independently" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = ext_path, .user_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 2), n_tools); } + +test "loadFromDirs: tools policy removes a denied registered tool name" { + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + try makeDir(tmp.dir, "tools"); + try writeFile(tmp.dir, "tools/reader.lua", + \\return { + \\ name = "std.read", description = "r", + \\ schema = { type = "object" }, + \\ handler = function(input) return "READ" end, + \\} + ); + try writeFile(tmp.dir, "tools/sheller.lua", + \\return { + \\ name = "std.shell", description = "s", + \\ schema = { type = "object" }, + \\ handler = function(input) return "SHELL" end, + \\} + ); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf); + const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); + defer testing.allocator.free(tools_path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + var deny = try testing.allocator.alloc([]const u8, 1); + deny[0] = try testing.allocator.dupe(u8, "std.shell"); + const tools_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny }; + defer tools_policy.deinit(testing.allocator); + + // Both tools register, but std.shell is filtered out post-load. + const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ + .user_tools = tools_path, + }, .{ .tools = &tools_policy }); + try testing.expectEqual(@as(usize, 1), n_tools); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{.{ .tool_name = "std.read", .input = "{}" }}; + var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer testing.allocator.free(results[0].ok); + try testing.expectEqualStrings("READ", results[0].ok); +} + +test "loadFromDirs: extensions policy denies a whole entry before it loads" { + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + try makeDir(tmp.dir, "tools"); + // This tool would crash if its script ran (calls a nil global). The + // extensions policy must stop it from loading at all. + try writeFile(tmp.dir, "danger.lua", + \\error("this script must never run") + ); + try makeDir(tmp.dir, "td"); + try writeFile(tmp.dir, "td/danger.lua", + \\error("this script must never run") + ); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPathFile(testing.io, "td", &path_buf); + const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); + defer testing.allocator.free(tools_path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + var deny = try testing.allocator.alloc([]const u8, 1); + deny[0] = try testing.allocator.dupe(u8, "danger"); + const ext_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny }; + defer ext_policy.deinit(testing.allocator); + + const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ + .user_tools = tools_path, + }, .{ .extensions = &ext_policy }); + try testing.expectEqual(@as(usize, 0), n_tools); +} -- cgit v1.3