Rename system extension layer to base for clarity

Replace all references to the "system" layer with "base" to better reflect
its role as the foundational extension/tool layer in panto's hierarchy.
The layer hierarchy now consistently uses: project > user > base.

Includes:
- Update agent README and build.zig documentation
- Refactor tool registry and config module to support layered lookups
- Add TestHarness abstraction for cleaner test setup
- Improve JSON serialization with wire-encoded tool names
- Add glob pattern matching for tool/extension discovery

1 files changed, 179 insertions, 44 deletions

diff --git a/src/extension_loader.zig b/src/extension_loader.zig
index 2fd86ae..b84eba5 100644
--- a/src/extension_loader.zig
+++ b/src/extension_loader.zig
@@ -1,22 +1,22 @@
 //! Extension and tool discovery: walk well-known directories, locate Lua
 //! files, and load each into a long-lived `LuaRuntime`.
 //!
-//! Two parallel namespaces are scanned, each at three scopes — system,
-//! user, and project. Project shadows user shadows system.
+//! Two parallel namespaces are scanned, each at three scopes — base,
+//! user, and project. Project shadows user shadows base.
 //!
 //!   Extensions (full-featured; call `panto.register_tool` from a script
 //!   that may register many tools):
-//!     1. `$PANTO_HOME/agent/extensions/`                          ("system")
+//!     1. `$PANTO_HOME/agent/extensions/`                          ("base")
 //!     2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/`    ("user")
 //!     3. `./.panto/extensions/`                                   ("project")
 //!
 //!   Tools (ergonomic single-tool form; the script returns one table
 //!   shaped like the argument to `panto.register_tool`):
-//!     1. `$PANTO_HOME/agent/tools/`                               ("system")
+//!     1. `$PANTO_HOME/agent/tools/`                               ("base")
 //!     2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/tools/`         ("user")
 //!     3. `./.panto/tools/`                                        ("project")
 //!
-//! The `system` layer is populated at bootstrap from files embedded
+//! The `base` layer is populated at bootstrap from files embedded
 //! into the panto binary (see `build/gen_agent_embed.zig` and
 //! `luarocks_runtime.stageAgentTree`). It is panto's "batteries" tier:
 //! ships in the binary, but every entry is individually shadowable
@@ -33,7 +33,7 @@
 //! Conflict rules:
 //!   - Within one directory, two entries with the same logical name
 //!     are an error.
-//!   - Project shadows user shadows system *within the same kind*
+//!   - Project shadows user shadows base *within the same kind*
 //!     (extension or tool). Extensions and tools live in distinct
 //!     namespaces for shadowing; the registered *tool names* still
 //!     share one global namespace across both, and collisions there
@@ -44,11 +44,32 @@
 const std = @import("std");
 const panto = @import("panto");
 const lua_runtime = @import("lua_runtime.zig");
+const config_file = @import("config_file.zig");
 
 const Allocator = std.mem.Allocator;
 const Io = std.Io;
 const LuaRuntime = lua_runtime.LuaRuntime;
 
+/// Availability policies for the two namespaces. Both are optional; a
+/// null policy permits everything. `extensions` gates extension/tool
+/// *entries* by their logical (file/dir) name before loading; `tools`
+/// gates the *registered tool names* (e.g. `std.read`) after loading.
+pub const Policies = struct {
+    extensions: ?*const config_file.Policy = null,
+    tools: ?*const config_file.Policy = null,
+};
+
+fn policyPermits(p: ?*const config_file.Policy, name: []const u8) bool {
+    const pol = p orelse return true;
+    return pol.permits(name);
+}
+
+/// Free-function form of `Policy.permits` taking the policy by pointer,
+/// matching the `fn(ctx, name) bool` shape `LuaRuntime.filterTools` wants.
+fn permitsByPtr(pol: *const config_file.Policy, name: []const u8) bool {
+    return pol.permits(name);
+}
+
 /// A discovered extension or tool before loading. Owns its strings.
 const Found = struct {
     /// Logical name (basename without `.lua`, or directory name).
@@ -72,13 +93,13 @@ const Found = struct {
 pub const Source = enum {
     /// Staged at bootstrap from files embedded in the panto binary.
     /// Lowest priority — shadowed by user and project.
-    system,
+    base,
     user,
     project,
 
     pub fn label(self: Source) []const u8 {
         return switch (self) {
-            .system => "system",
+            .base => "base",
             .user => "user",
             .project => "project",
         };
@@ -101,9 +122,9 @@ pub const Kind = enum {
 /// paths into `runtime`. Returns the number of registered tools added
 /// to the runtime by this call.
 ///
-/// `system_agent_dir`, when non-null, is the path under which embedded
-/// system tools/extensions have been staged — typically
-/// `$PANTO_HOME/agent/`. Pass `null` to skip the system layer entirely
+/// `base_agent_dir`, when non-null, is the path under which embedded
+/// base tools/extensions have been staged — typically
+/// `$PANTO_HOME/agent/`. Pass `null` to skip the base layer entirely
 /// (mostly useful for tests).
 ///
 /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The
@@ -112,15 +133,16 @@ pub fn discoverAndLoad(
     allocator: Allocator,
     io: Io,
     environ_map: *const std.process.Environ.Map,
-    system_agent_dir: ?[]const u8,
+    base_agent_dir: ?[]const u8,
     runtime: *LuaRuntime,
+    policies: Policies,
 ) !usize {
-    const sys_ext = if (system_agent_dir) |d|
+    const sys_ext = if (base_agent_dir) |d|
         try std.fs.path.join(allocator, &.{ d, kindSubdir(.extension) })
     else
         null;
     defer if (sys_ext) |d| allocator.free(d);
-    const sys_tool = if (system_agent_dir) |d|
+    const sys_tool = if (base_agent_dir) |d|
         try std.fs.path.join(allocator, &.{ d, kindSubdir(.tool) })
     else
         null;
@@ -141,27 +163,28 @@ pub fn discoverAndLoad(
         io,
         runtime,
         .{
-            .system_extensions = sys_ext,
+            .base_extensions = sys_ext,
             .user_extensions = user_ext,
             .project_extensions = project_ext,
-            .system_tools = sys_tool,
+            .base_tools = sys_tool,
             .user_tools = user_tool,
             .project_tools = project_tool,
         },
+        policies,
     );
 }
 
 /// Set of search paths consumed by `loadFromDirs`. Any field may be
 /// null; missing directories on disk are silently skipped.
 ///
-/// Scan order is system → user → project. `applyShadowing` keeps the
+/// Scan order is base → user → project. `applyShadowing` keeps the
 /// *last* occurrence of each (kind, name), so project entries win,
-/// then user, then system.
+/// then user, then base.
 pub const DirSet = struct {
-    system_extensions: ?[]const u8 = null,
+    base_extensions: ?[]const u8 = null,
     user_extensions: ?[]const u8 = null,
     project_extensions: ?[]const u8 = null,
-    system_tools: ?[]const u8 = null,
+    base_tools: ?[]const u8 = null,
     user_tools: ?[]const u8 = null,
     project_tools: ?[]const u8 = null,
 };
@@ -173,6 +196,7 @@ pub fn loadFromDirs(
     io: Io,
     runtime: *LuaRuntime,
     dirs: DirSet,
+    policies: Policies,
 ) !usize {
     var found: std.array_list.Managed(Found) = .init(allocator);
     defer {
@@ -180,15 +204,38 @@ pub fn loadFromDirs(
         found.deinit();
     }
 
-    if (dirs.system_extensions) |d| try scanDir(allocator, io, d, .system, .extension, &found);
+    if (dirs.base_extensions) |d| try scanDir(allocator, io, d, .base, .extension, &found);
     if (dirs.user_extensions) |d| try scanDir(allocator, io, d, .user, .extension, &found);
     if (dirs.project_extensions) |d| try scanDir(allocator, io, d, .project, .extension, &found);
-    if (dirs.system_tools) |d| try scanDir(allocator, io, d, .system, .tool, &found);
+    if (dirs.base_tools) |d| try scanDir(allocator, io, d, .base, .tool, &found);
     if (dirs.user_tools) |d| try scanDir(allocator, io, d, .user, .tool, &found);
     if (dirs.project_tools) |d| try scanDir(allocator, io, d, .project, .tool, &found);
 
     try applyShadowing(allocator, &found);
 
+    // Gate *entries* by the extensions policy (matched on logical name).
+    // This drops whole scripts before they run — a denied extension
+    // never executes, never registers tools. The tools policy is
+    // applied post-load against registered tool names.
+    if (policies.extensions) |_| {
+        var keep: std.array_list.Managed(Found) = .init(allocator);
+        errdefer {
+            for (keep.items) |*f| f.deinit(allocator);
+            keep.deinit();
+        }
+        for (found.items) |*f| {
+            if (policyPermits(policies.extensions, f.name)) {
+                try keep.append(f.*);
+            } else {
+                std.log.debug("{s}: '{s}' denied by extensions policy", .{ f.kind.label(), f.name });
+                f.deinit(allocator);
+            }
+        }
+        found.clearRetainingCapacity();
+        try found.appendSlice(keep.items);
+        keep.deinit();
+    }
+
     const before = runtime.toolCount();
     for (found.items) |f| {
         const load_result = switch (f.kind) {
@@ -214,6 +261,13 @@ pub fn loadFromDirs(
             .{ f.kind.label(), f.name, f.source.label() },
         );
     }
+
+    // Apply the tools policy to *registered tool names* (e.g. `std.read`).
+    if (policies.tools) |pol| {
+        const dropped = runtime.filterTools(pol, permitsByPtr);
+        if (dropped > 0) std.log.debug("tools: {d} tool(s) removed by tools policy", .{dropped});
+    }
+
     return runtime.toolCount() - before;
 }
 
@@ -581,7 +635,7 @@ test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" {
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
         .user_extensions = user_path,
         .project_extensions = proj_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     // Invoke the tool through the source and verify the project handler ran.
@@ -623,7 +677,7 @@ test "loadFromDirs: tool-name collision between extensions errors" {
 
     const result = loadFromDirs(testing.allocator, testing.io, rt, .{
         .project_extensions = ext_path,
-    });
+    }, .{});
     try testing.expectError(error.DuplicateTool, result);
 }
 
@@ -650,7 +704,7 @@ test "loadFromDirs: tools/ directory — single-file tool form" {
 
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
         .user_tools = tools_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     var src = rt.toolSource();
@@ -690,7 +744,7 @@ test "loadFromDirs: tools/ directory — directory-style tool with sibling requi
 
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
         .project_tools = tools_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     var src = rt.toolSource();
@@ -738,7 +792,7 @@ test "loadFromDirs: project tool shadows user tool of the same name" {
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
         .user_tools = user_path,
         .project_tools = proj_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     var src = rt.toolSource();
@@ -749,19 +803,19 @@ test "loadFromDirs: project tool shadows user tool of the same name" {
     try testing.expectEqualStrings("PROJECT", results[0].ok);
 }
 
-test "loadFromDirs: project tool shadows user shadows system" {
+test "loadFromDirs: project tool shadows user shadows base" {
     var tmp = testing.tmpDir(.{ .iterate = true });
     defer tmp.cleanup();
 
-    try makeDir(tmp.dir, "system_tools");
+    try makeDir(tmp.dir, "base_tools");
     try makeDir(tmp.dir, "user_tools");
     try makeDir(tmp.dir, "project_tools");
 
-    try writeFile(tmp.dir, "system_tools/greet.lua",
+    try writeFile(tmp.dir, "base_tools/greet.lua",
         \\return {
-        \\  name = "greet", description = "system version",
+        \\  name = "greet", description = "base version",
         \\  schema = { type = "object" },
-        \\  handler = function(input) return "SYSTEM" end,
+        \\  handler = function(input) return "BASE" end,
         \\}
     );
     try writeFile(tmp.dir, "user_tools/greet.lua",
@@ -780,7 +834,7 @@ test "loadFromDirs: project tool shadows user shadows system" {
     );
 
     var path_buf: [std.fs.max_path_bytes]u8 = undefined;
-    const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf);
+    const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf);
     const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]);
     defer testing.allocator.free(sys_path);
     const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf);
@@ -794,10 +848,10 @@ test "loadFromDirs: project tool shadows user shadows system" {
     defer rt.deinit();
 
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
-        .system_tools = sys_path,
+        .base_tools = sys_path,
         .user_tools = user_path,
         .project_tools = proj_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     var src = rt.toolSource();
@@ -808,18 +862,18 @@ test "loadFromDirs: project tool shadows user shadows system" {
     try testing.expectEqualStrings("PROJECT", results[0].ok);
 }
 
-test "loadFromDirs: user tool shadows system tool when no project entry" {
+test "loadFromDirs: user tool shadows base tool when no project entry" {
     var tmp = testing.tmpDir(.{ .iterate = true });
     defer tmp.cleanup();
 
-    try makeDir(tmp.dir, "system_tools");
+    try makeDir(tmp.dir, "base_tools");
     try makeDir(tmp.dir, "user_tools");
 
-    try writeFile(tmp.dir, "system_tools/greet.lua",
+    try writeFile(tmp.dir, "base_tools/greet.lua",
         \\return {
-        \\  name = "greet", description = "system",
+        \\  name = "greet", description = "base",
         \\  schema = { type = "object" },
-        \\  handler = function(input) return "SYSTEM" end,
+        \\  handler = function(input) return "BASE" end,
         \\}
     );
     try writeFile(tmp.dir, "user_tools/greet.lua",
@@ -831,7 +885,7 @@ test "loadFromDirs: user tool shadows system tool when no project entry" {
     );
 
     var path_buf: [std.fs.max_path_bytes]u8 = undefined;
-    const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf);
+    const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf);
     const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]);
     defer testing.allocator.free(sys_path);
     const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf);
@@ -842,9 +896,9 @@ test "loadFromDirs: user tool shadows system tool when no project entry" {
     defer rt.deinit();
 
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
-        .system_tools = sys_path,
+        .base_tools = sys_path,
         .user_tools = user_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     var src = rt.toolSource();
@@ -895,6 +949,87 @@ test "loadFromDirs: extension and tool share a *file* name independently" {
     const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
         .user_extensions = ext_path,
         .user_tools = tools_path,
-    });
+    }, .{});
     try testing.expectEqual(@as(usize, 2), n_tools);
 }
+
+test "loadFromDirs: tools policy removes a denied registered tool name" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    try makeDir(tmp.dir, "tools");
+    try writeFile(tmp.dir, "tools/reader.lua",
+        \\return {
+        \\  name = "std.read", description = "r",
+        \\  schema = { type = "object" },
+        \\  handler = function(input) return "READ" end,
+        \\}
+    );
+    try writeFile(tmp.dir, "tools/sheller.lua",
+        \\return {
+        \\  name = "std.shell", description = "s",
+        \\  schema = { type = "object" },
+        \\  handler = function(input) return "SHELL" end,
+        \\}
+    );
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf);
+    const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]);
+    defer testing.allocator.free(tools_path);
+
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
+
+    var deny = try testing.allocator.alloc([]const u8, 1);
+    deny[0] = try testing.allocator.dupe(u8, "std.shell");
+    const tools_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny };
+    defer tools_policy.deinit(testing.allocator);
+
+    // Both tools register, but std.shell is filtered out post-load.
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .user_tools = tools_path,
+    }, .{ .tools = &tools_policy });
+    try testing.expectEqual(@as(usize, 1), n_tools);
+
+    var src = rt.toolSource();
+    const calls = [_]panto.ToolCall{.{ .tool_name = "std.read", .input = "{}" }};
+    var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }};
+    try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator);
+    defer testing.allocator.free(results[0].ok);
+    try testing.expectEqualStrings("READ", results[0].ok);
+}
+
+test "loadFromDirs: extensions policy denies a whole entry before it loads" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    try makeDir(tmp.dir, "tools");
+    // This tool would crash if its script ran (calls a nil global). The
+    // extensions policy must stop it from loading at all.
+    try writeFile(tmp.dir, "danger.lua",
+        \\error("this script must never run")
+    );
+    try makeDir(tmp.dir, "td");
+    try writeFile(tmp.dir, "td/danger.lua",
+        \\error("this script must never run")
+    );
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const n = try tmp.dir.realPathFile(testing.io, "td", &path_buf);
+    const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]);
+    defer testing.allocator.free(tools_path);
+
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
+
+    var deny = try testing.allocator.alloc([]const u8, 1);
+    deny[0] = try testing.allocator.dupe(u8, "danger");
+    const ext_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny };
+    defer ext_policy.deinit(testing.allocator);
+
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .user_tools = tools_path,
+    }, .{ .extensions = &ext_policy });
+    try testing.expectEqual(@as(usize, 0), n_tools);
+}