.panto/tools/ resolution

1 files changed, 314 insertions, 56 deletions

diff --git a/src/extension_loader.zig b/src/extension_loader.zig
index 2a35d15..8a871c9 100644
--- a/src/extension_loader.zig
+++ b/src/extension_loader.zig
@@ -1,24 +1,33 @@
-//! Extension discovery: walk well-known directories, locate Lua extensions,
-//! and load each one into a long-lived `LuaRuntime`.
+//! Extension and tool discovery: walk well-known directories, locate Lua
+//! files, and load each into a long-lived `LuaRuntime`.
 //!
-//! Search order (later entries shadow earlier ones by extension *name*):
-//!   1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/`  ("user")
-//!   2. `./.panto/extensions/`                                  ("project")
+//! Two parallel namespaces are scanned, each at user and project scope:
 //!
-//! Layout per directory:
-//!   - `<name>.lua`        -- single-file extension; the extension name is
-//!                            the basename without the `.lua` suffix.
-//!   - `<name>/init.lua`   -- directory extension; the extension name is
-//!                            the directory name. The directory is added
-//!                            to the extension's `package.path` so it can
+//!   Extensions (full-featured; call `panto.register_tool` from a script
+//!   that may register many tools):
+//!     1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/`  ("user")
+//!     2. `./.panto/extensions/`                                  ("project")
+//!
+//!   Tools (ergonomic single-tool form; the script returns one table
+//!   shaped like the argument to `panto.register_tool`):
+//!     1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/tools/`        ("user")
+//!     2. `./.panto/tools/`                                       ("project")
+//!
+//! Layout per directory (identical for extensions and tools):
+//!   - `<name>.lua`        -- single-file entry; the logical name is the
+//!                            basename without the `.lua` suffix.
+//!   - `<name>/init.lua`   -- directory entry; the logical name is the
+//!                            directory name. The directory is added to
+//!                            the script's `package.path` so it can
 //!                            `require` sibling Lua files.
 //!
 //! Conflict rules:
-//!   - Within one directory, two entries with the same extension name
+//!   - Within one directory, two entries with the same logical name
 //!     are an error.
-//!   - Project shadows user by extension name (debug-logged, not an error).
-//!   - Tool-name collisions *between* loaded extensions are an error: a
-//!     tool name is a contract the LLM relies on.
+//!   - Project shadows user *within the same kind* (extension or tool).
+//!     Extensions and tools live in distinct namespaces for shadowing;
+//!     the registered *tool names* still share one global namespace
+//!     across both, and collisions there are still an error.
 //!
 //! Symlinks: followed normally. Dotfiles: skipped.
 
@@ -30,16 +39,18 @@ const Allocator = std.mem.Allocator;
 const Io = std.Io;
 const LuaRuntime = lua_runtime.LuaRuntime;
 
-/// A discovered extension before loading. Owns its strings.
+/// A discovered extension or tool before loading. Owns its strings.
 const Found = struct {
     /// Logical name (basename without `.lua`, or directory name).
     name: []u8,
     /// Absolute path to the Lua script to execute.
     script_path: []u8,
-    /// For directory-style extensions, the directory containing the script.
+    /// For directory-style entries, the directory containing the script.
     package_root: ?[]u8,
     /// Which search-path source this came from.
     source: Source,
+    /// Whether this is a full extension or a single-tool script.
+    kind: Kind,
 
     pub fn deinit(self: *Found, allocator: Allocator) void {
         allocator.free(self.name);
@@ -60,34 +71,69 @@ pub const Source = enum {
     }
 };
 
-/// Discover and load every extension found in the standard paths into
-/// `runtime`. Returns the number of *tools* (not extensions) declared.
+pub const Kind = enum {
+    extension,
+    tool,
+
+    pub fn label(self: Kind) []const u8 {
+        return switch (self) {
+            .extension => "extension",
+            .tool => "tool",
+        };
+    }
+};
+
+/// Discover and load every extension and tool found in the standard
+/// paths into `runtime`. Returns the number of registered tools added
+/// to the runtime by this call.
 ///
 /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The
-/// project directory is always `cwd()/.panto/extensions`.
+/// project directories are always `cwd()/.panto/{extensions,tools}`.
 pub fn discoverAndLoad(
     allocator: Allocator,
     io: Io,
     environ_map: *const std.process.Environ.Map,
     runtime: *LuaRuntime,
 ) !usize {
-    const user_dir = try userExtensionsDir(allocator, environ_map);
-    defer if (user_dir) |d| allocator.free(d);
+    const user_ext = try userKindDir(allocator, environ_map, .extension);
+    defer if (user_ext) |d| allocator.free(d);
+    const user_tool = try userKindDir(allocator, environ_map, .tool);
+    defer if (user_tool) |d| allocator.free(d);
 
-    const project_dir = try projectExtensionsDir(allocator, io);
-    defer allocator.free(project_dir);
+    const project_ext = try projectKindDir(allocator, io, .extension);
+    defer allocator.free(project_ext);
+    const project_tool = try projectKindDir(allocator, io, .tool);
+    defer allocator.free(project_tool);
 
-    return loadFromDirs(allocator, io, runtime, user_dir, project_dir);
+    return loadFromDirs(
+        allocator,
+        io,
+        runtime,
+        .{
+            .user_extensions = user_ext,
+            .project_extensions = project_ext,
+            .user_tools = user_tool,
+            .project_tools = project_tool,
+        },
+    );
 }
 
+/// Set of search paths consumed by `loadFromDirs`. Any field may be
+/// null; missing directories on disk are silently skipped.
+pub const DirSet = struct {
+    user_extensions: ?[]const u8 = null,
+    project_extensions: ?[]const u8 = null,
+    user_tools: ?[]const u8 = null,
+    project_tools: ?[]const u8 = null,
+};
+
 /// Lower-level entry point: load from explicit user/project paths.
 /// Either path may be null; missing directories are silently skipped.
 pub fn loadFromDirs(
     allocator: Allocator,
     io: Io,
     runtime: *LuaRuntime,
-    user_dir: ?[]const u8,
-    project_dir: ?[]const u8,
+    dirs: DirSet,
 ) !usize {
     var found: std.array_list.Managed(Found) = .init(allocator);
     defer {
@@ -95,30 +141,36 @@ pub fn loadFromDirs(
         found.deinit();
     }
 
-    if (user_dir) |d| try scanDir(allocator, io, d, .user, &found);
-    if (project_dir) |d| try scanDir(allocator, io, d, .project, &found);
+    if (dirs.user_extensions) |d| try scanDir(allocator, io, d, .user, .extension, &found);
+    if (dirs.project_extensions) |d| try scanDir(allocator, io, d, .project, .extension, &found);
+    if (dirs.user_tools) |d| try scanDir(allocator, io, d, .user, .tool, &found);
+    if (dirs.project_tools) |d| try scanDir(allocator, io, d, .project, .tool, &found);
 
     try applyShadowing(allocator, &found);
 
     const before = runtime.toolCount();
     for (found.items) |f| {
-        runtime.loadExtension(f.script_path, f.package_root) catch |err| {
+        const load_result = switch (f.kind) {
+            .extension => runtime.loadExtension(f.script_path, f.package_root),
+            .tool => runtime.loadTool(f.script_path, f.package_root),
+        };
+        load_result catch |err| {
             if (@import("builtin").is_test) {
                 std.log.warn(
-                    "extension '{s}' ({s}: {s}) failed to load: {t}",
-                    .{ f.name, f.source.label(), f.script_path, err },
+                    "{s} '{s}' ({s}: {s}) failed to load: {t}",
+                    .{ f.kind.label(), f.name, f.source.label(), f.script_path, err },
                 );
             } else {
                 std.log.err(
-                    "extension '{s}' ({s}: {s}) failed to load: {t}",
-                    .{ f.name, f.source.label(), f.script_path, err },
+                    "{s} '{s}' ({s}: {s}) failed to load: {t}",
+                    .{ f.kind.label(), f.name, f.source.label(), f.script_path, err },
                 );
             }
             return err;
         };
         std.log.debug(
-            "extension: loaded '{s}' ({s})",
-            .{ f.name, f.source.label() },
+            "{s}: loaded '{s}' ({s})",
+            .{ f.kind.label(), f.name, f.source.label() },
         );
     }
     return runtime.toolCount() - before;
@@ -128,23 +180,32 @@ pub fn loadFromDirs(
 // Path resolution
 // ---------------------------------------------------------------------------
 
-fn userExtensionsDir(
+fn kindSubdir(kind: Kind) []const u8 {
+    return switch (kind) {
+        .extension => "extensions",
+        .tool => "tools",
+    };
+}
+
+fn userKindDir(
     allocator: Allocator,
     environ_map: *const std.process.Environ.Map,
+    kind: Kind,
 ) !?[]u8 {
+    const sub = kindSubdir(kind);
     if (environ_map.get("XDG_CONFIG_HOME")) |xdg| {
-        return try std.fs.path.join(allocator, &.{ xdg, "panto", "extensions" });
+        return try std.fs.path.join(allocator, &.{ xdg, "panto", sub });
     }
     if (environ_map.get("HOME")) |home| {
-        return try std.fs.path.join(allocator, &.{ home, ".config", "panto", "extensions" });
+        return try std.fs.path.join(allocator, &.{ home, ".config", "panto", sub });
     }
     return null;
 }
 
-fn projectExtensionsDir(allocator: Allocator, io: Io) ![]u8 {
+fn projectKindDir(allocator: Allocator, io: Io, kind: Kind) ![]u8 {
     const cwd = try std.process.currentPathAlloc(io, allocator);
     defer allocator.free(cwd);
-    return try std.fs.path.join(allocator, &.{ cwd, ".panto", "extensions" });
+    return try std.fs.path.join(allocator, &.{ cwd, ".panto", kindSubdir(kind) });
 }
 
 // ---------------------------------------------------------------------------
@@ -156,6 +217,7 @@ fn scanDir(
     io: Io,
     dir_path: []const u8,
     source: Source,
+    kind: Kind,
     out: *std.array_list.Managed(Found),
 ) !void {
     var dir = Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch |err| switch (err) {
@@ -176,8 +238,8 @@ fn scanDir(
         if (entry.name.len == 0 or entry.name[0] == '.') continue;
 
         const maybe_found: ?Found = switch (entry.kind) {
-            .file, .sym_link => try classifyFile(allocator, dir_path, entry.name, source),
-            .directory => try classifyDirectory(allocator, io, dir, dir_path, entry.name, source),
+            .file, .sym_link => try classifyFile(allocator, dir_path, entry.name, source, kind),
+            .directory => try classifyDirectory(allocator, io, dir, dir_path, entry.name, source, kind),
             else => null,
         };
         const f = maybe_found orelse continue;
@@ -186,13 +248,13 @@ fn scanDir(
         if (gop.found_existing) {
             if (@import("builtin").is_test) {
                 std.log.warn(
-                    "extension name '{s}' is provided by multiple entries in {s}",
-                    .{ f.name, dir_path },
+                    "{s} name '{s}' is provided by multiple entries in {s}",
+                    .{ kind.label(), f.name, dir_path },
                 );
             } else {
                 std.log.err(
-                    "extension name '{s}' is provided by multiple entries in {s}",
-                    .{ f.name, dir_path },
+                    "{s} name '{s}' is provided by multiple entries in {s}",
+                    .{ kind.label(), f.name, dir_path },
                 );
             }
             var dup = f;
@@ -210,6 +272,7 @@ fn classifyFile(
     dir_path: []const u8,
     entry_name: []const u8,
     source: Source,
+    kind: Kind,
 ) !?Found {
     if (!std.mem.endsWith(u8, entry_name, ".lua")) return null;
     const base = entry_name[0 .. entry_name.len - ".lua".len];
@@ -225,6 +288,7 @@ fn classifyFile(
         .script_path = script_path,
         .package_root = null,
         .source = source,
+        .kind = kind,
     };
 }
 
@@ -235,6 +299,7 @@ fn classifyDirectory(
     dir_path: []const u8,
     entry_name: []const u8,
     source: Source,
+    kind: Kind,
 ) !?Found {
     var sub = parent.openDir(io, entry_name, .{}) catch return null;
     defer sub.close(io);
@@ -256,6 +321,7 @@ fn classifyDirectory(
         .script_path = script_path,
         .package_root = package_root,
         .source = source,
+        .kind = kind,
     };
 }
 
@@ -263,11 +329,31 @@ fn classifyDirectory(
 // Shadowing
 // ---------------------------------------------------------------------------
 
+/// Shadowing key combines (kind, name) so a tool named `foo` does not
+/// shadow an extension named `foo` (or vice versa). Tool-name collisions
+/// across these are still caught later by the runtime/registry.
+const ShadowKey = struct {
+    kind: Kind,
+    name: []const u8,
+};
+
+const ShadowKeyCtx = struct {
+    pub fn hash(_: ShadowKeyCtx, k: ShadowKey) u64 {
+        var hasher = std.hash.Wyhash.init(0);
+        hasher.update(&[_]u8{@intFromEnum(k.kind)});
+        hasher.update(k.name);
+        return hasher.final();
+    }
+    pub fn eql(_: ShadowKeyCtx, a: ShadowKey, b: ShadowKey) bool {
+        return a.kind == b.kind and std.mem.eql(u8, a.name, b.name);
+    }
+};
+
 fn applyShadowing(allocator: Allocator, list: *std.array_list.Managed(Found)) !void {
-    var latest: std.StringHashMap(usize) = .init(allocator);
+    var latest: std.HashMap(ShadowKey, usize, ShadowKeyCtx, std.hash_map.default_max_load_percentage) = .init(allocator);
 
     for (list.items, 0..) |f, i| {
-        try latest.put(f.name, i);
+        try latest.put(.{ .kind = f.kind, .name = f.name }, i);
     }
 
     var keep: std.array_list.Managed(Found) = .init(allocator);
@@ -283,13 +369,13 @@ fn applyShadowing(allocator: Allocator, list: *std.array_list.Managed(Found)) !v
     try drop.ensureTotalCapacity(list.items.len);
 
     for (list.items, 0..) |f, i| {
-        const winner = latest.get(f.name).?;
+        const winner = latest.get(.{ .kind = f.kind, .name = f.name }).?;
         if (winner == i) {
             keep.appendAssumeCapacity(f);
         } else {
             std.log.debug(
-                "extension: '{s}' from {s} shadowed by {s}",
-                .{ f.name, f.source.label(), list.items[winner].source.label() },
+                "{s}: '{s}' from {s} shadowed by {s}",
+                .{ f.kind.label(), f.name, f.source.label(), list.items[winner].source.label() },
             );
             drop.appendAssumeCapacity(f);
         }
@@ -339,7 +425,7 @@ test "scanDir picks up single-file and directory-style extensions" {
         for (list.items) |*f| f.deinit(testing.allocator);
         list.deinit();
     }
-    try scanDir(testing.allocator, testing.io, ext_root, .user, &list);
+    try scanDir(testing.allocator, testing.io, ext_root, .user, .extension, &list);
 
     try testing.expectEqual(@as(usize, 2), list.items.len);
 
@@ -376,7 +462,7 @@ test "duplicate name in same directory is an error" {
         list.deinit();
     }
 
-    const result = scanDir(testing.allocator, testing.io, ext_root, .project, &list);
+    const result = scanDir(testing.allocator, testing.io, ext_root, .project, .extension, &list);
     try testing.expectError(error.DuplicateExtensionInDirectory, result);
 }
 
@@ -398,6 +484,7 @@ test "applyShadowing keeps the latest occurrence" {
             .script_path = try testing.allocator.dupe(u8, row[1]),
             .package_root = null,
             .source = row[2],
+            .kind = .extension,
         });
     }
 
@@ -450,7 +537,10 @@ test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" {
     var rt = try LuaRuntime.create(testing.allocator);
     defer rt.deinit();
 
-    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, user_path, proj_path);
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .user_extensions = user_path,
+        .project_extensions = proj_path,
+    });
     try testing.expectEqual(@as(usize, 1), n_tools);
 
     // Invoke the tool through the source and verify the project handler ran.
@@ -490,6 +580,174 @@ test "loadFromDirs: tool-name collision between extensions errors" {
     var rt = try LuaRuntime.create(testing.allocator);
     defer rt.deinit();
 
-    const result = loadFromDirs(testing.allocator, testing.io, rt, null, ext_path);
+    const result = loadFromDirs(testing.allocator, testing.io, rt, .{
+        .project_extensions = ext_path,
+    });
     try testing.expectError(error.DuplicateTool, result);
 }
+
+test "loadFromDirs: tools/ directory — single-file tool form" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    try makeDir(tmp.dir, "tools");
+    try writeFile(tmp.dir, "tools/echo.lua",
+        \\return {
+        \\  name = "echo", description = "Echo back input.",
+        \\  schema = { type = "object", properties = { msg = { type = "string" } } },
+        \\  handler = function(input) return "echo: " .. input.msg end,
+        \\}
+    );
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf);
+    const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]);
+    defer testing.allocator.free(tools_path);
+
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
+
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .user_tools = tools_path,
+    });
+    try testing.expectEqual(@as(usize, 1), n_tools);
+
+    var src = rt.toolSource();
+    const calls = [_]panto.ToolCall{.{ .tool_name = "echo", .input = "{\"msg\":\"hi\"}" }};
+    var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }};
+    try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator);
+    defer testing.allocator.free(results[0].ok);
+    try testing.expectEqualStrings("echo: hi", results[0].ok);
+}
+
+test "loadFromDirs: tools/ directory — directory-style tool with sibling require" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    try makeDir(tmp.dir, "tools/shout");
+    try writeFile(tmp.dir, "tools/shout/util.lua",
+        \\local M = {}
+        \\function M.shout(s) return s:upper() .. "!" end
+        \\return M
+    );
+    try writeFile(tmp.dir, "tools/shout/init.lua",
+        \\local util = require("util")
+        \\return {
+        \\  name = "shout", description = "uppercase + bang",
+        \\  schema = { type = "object", properties = { text = { type = "string" } } },
+        \\  handler = function(input) return util.shout(input.text) end,
+        \\}
+    );
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf);
+    const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]);
+    defer testing.allocator.free(tools_path);
+
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
+
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .project_tools = tools_path,
+    });
+    try testing.expectEqual(@as(usize, 1), n_tools);
+
+    var src = rt.toolSource();
+    const calls = [_]panto.ToolCall{.{ .tool_name = "shout", .input = "{\"text\":\"hi\"}" }};
+    var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }};
+    try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator);
+    defer testing.allocator.free(results[0].ok);
+    try testing.expectEqualStrings("HI!", results[0].ok);
+}
+
+test "loadFromDirs: project tool shadows user tool of the same name" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    try makeDir(tmp.dir, "user_tools");
+    try makeDir(tmp.dir, "project_tools");
+
+    try writeFile(tmp.dir, "user_tools/greet.lua",
+        \\return {
+        \\  name = "greet", description = "user version",
+        \\  schema = { type = "object" },
+        \\  handler = function(input) return "USER" end,
+        \\}
+    );
+    try writeFile(tmp.dir, "project_tools/greet.lua",
+        \\return {
+        \\  name = "greet", description = "project version",
+        \\  schema = { type = "object" },
+        \\  handler = function(input) return "PROJECT" end,
+        \\}
+    );
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf);
+    const user_path = try testing.allocator.dupe(u8, path_buf[0..user_len]);
+    defer testing.allocator.free(user_path);
+
+    const proj_len = try tmp.dir.realPathFile(testing.io, "project_tools", &path_buf);
+    const proj_path = try testing.allocator.dupe(u8, path_buf[0..proj_len]);
+    defer testing.allocator.free(proj_path);
+
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
+
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .user_tools = user_path,
+        .project_tools = proj_path,
+    });
+    try testing.expectEqual(@as(usize, 1), n_tools);
+
+    var src = rt.toolSource();
+    const calls = [_]panto.ToolCall{.{ .tool_name = "greet", .input = "{}" }};
+    var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }};
+    try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator);
+    defer testing.allocator.free(results[0].ok);
+    try testing.expectEqualStrings("PROJECT", results[0].ok);
+}
+
+test "loadFromDirs: extension and tool share a *file* name independently" {
+    // The shadow-key is (kind, name) so an extension named `foo` and a
+    // tool named `foo` coexist — as long as their *registered* tool names
+    // don't collide.
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    try makeDir(tmp.dir, "ext");
+    try makeDir(tmp.dir, "tools");
+
+    try writeFile(tmp.dir, "ext/foo.lua",
+        \\panto.register_tool {
+        \\  name = "ext_foo", description = "e",
+        \\  schema = { type = "object" },
+        \\  handler = function(input) return "ext" end,
+        \\}
+    );
+    try writeFile(tmp.dir, "tools/foo.lua",
+        \\return {
+        \\  name = "tool_foo", description = "t",
+        \\  schema = { type = "object" },
+        \\  handler = function(input) return "tool" end,
+        \\}
+    );
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const e_len = try tmp.dir.realPathFile(testing.io, "ext", &path_buf);
+    const ext_path = try testing.allocator.dupe(u8, path_buf[0..e_len]);
+    defer testing.allocator.free(ext_path);
+
+    const t_len = try tmp.dir.realPathFile(testing.io, "tools", &path_buf);
+    const tools_path = try testing.allocator.dupe(u8, path_buf[0..t_len]);
+    defer testing.allocator.free(tools_path);
+
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
+
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{
+        .user_extensions = ext_path,
+        .user_tools = tools_path,
+    });
+    try testing.expectEqual(@as(usize, 2), n_tools);
+}