//! Extension and tool discovery: walk well-known directories, locate Lua //! files, and load each into a long-lived `LuaRuntime`. //! //! Two parallel namespaces are scanned, each at user and project scope: //! //! Extensions (full-featured; call `panto.register_tool` from a script //! that may register many tools): //! 1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/` ("user") //! 2. `./.panto/extensions/` ("project") //! //! Tools (ergonomic single-tool form; the script returns one table //! shaped like the argument to `panto.register_tool`): //! 1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/tools/` ("user") //! 2. `./.panto/tools/` ("project") //! //! Layout per directory (identical for extensions and tools): //! - `.lua` -- single-file entry; the logical name is the //! basename without the `.lua` suffix. //! - `/init.lua` -- directory entry; the logical name is the //! directory name. The directory is added to //! the script's `package.path` so it can //! `require` sibling Lua files. //! //! Conflict rules: //! - Within one directory, two entries with the same logical name //! are an error. //! - Project shadows user *within the same kind* (extension or tool). //! Extensions and tools live in distinct namespaces for shadowing; //! the registered *tool names* still share one global namespace //! across both, and collisions there are still an error. //! //! Symlinks: followed normally. Dotfiles: skipped. const std = @import("std"); const panto = @import("panto"); const lua_runtime = @import("lua_runtime.zig"); const Allocator = std.mem.Allocator; const Io = std.Io; const LuaRuntime = lua_runtime.LuaRuntime; /// A discovered extension or tool before loading. Owns its strings. const Found = struct { /// Logical name (basename without `.lua`, or directory name). name: []u8, /// Absolute path to the Lua script to execute. script_path: []u8, /// For directory-style entries, the directory containing the script. package_root: ?[]u8, /// Which search-path source this came from. source: Source, /// Whether this is a full extension or a single-tool script. kind: Kind, pub fn deinit(self: *Found, allocator: Allocator) void { allocator.free(self.name); allocator.free(self.script_path); if (self.package_root) |p| allocator.free(p); } }; pub const Source = enum { user, project, pub fn label(self: Source) []const u8 { return switch (self) { .user => "user", .project => "project", }; } }; pub const Kind = enum { extension, tool, pub fn label(self: Kind) []const u8 { return switch (self) { .extension => "extension", .tool => "tool", }; } }; /// Discover and load every extension and tool found in the standard /// paths into `runtime`. Returns the number of registered tools added /// to the runtime by this call. /// /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The /// project directories are always `cwd()/.panto/{extensions,tools}`. pub fn discoverAndLoad( allocator: Allocator, io: Io, environ_map: *const std.process.Environ.Map, runtime: *LuaRuntime, ) !usize { const user_ext = try userKindDir(allocator, environ_map, .extension); defer if (user_ext) |d| allocator.free(d); const user_tool = try userKindDir(allocator, environ_map, .tool); defer if (user_tool) |d| allocator.free(d); const project_ext = try projectKindDir(allocator, io, .extension); defer allocator.free(project_ext); const project_tool = try projectKindDir(allocator, io, .tool); defer allocator.free(project_tool); return loadFromDirs( allocator, io, runtime, .{ .user_extensions = user_ext, .project_extensions = project_ext, .user_tools = user_tool, .project_tools = project_tool, }, ); } /// Set of search paths consumed by `loadFromDirs`. Any field may be /// null; missing directories on disk are silently skipped. pub const DirSet = struct { user_extensions: ?[]const u8 = null, project_extensions: ?[]const u8 = null, user_tools: ?[]const u8 = null, project_tools: ?[]const u8 = null, }; /// Lower-level entry point: load from explicit user/project paths. /// Either path may be null; missing directories are silently skipped. pub fn loadFromDirs( allocator: Allocator, io: Io, runtime: *LuaRuntime, dirs: DirSet, ) !usize { var found: std.array_list.Managed(Found) = .init(allocator); defer { for (found.items) |*f| f.deinit(allocator); found.deinit(); } if (dirs.user_extensions) |d| try scanDir(allocator, io, d, .user, .extension, &found); if (dirs.project_extensions) |d| try scanDir(allocator, io, d, .project, .extension, &found); if (dirs.user_tools) |d| try scanDir(allocator, io, d, .user, .tool, &found); if (dirs.project_tools) |d| try scanDir(allocator, io, d, .project, .tool, &found); try applyShadowing(allocator, &found); const before = runtime.toolCount(); for (found.items) |f| { const load_result = switch (f.kind) { .extension => runtime.loadExtension(f.script_path, f.package_root), .tool => runtime.loadTool(f.script_path, f.package_root), }; load_result catch |err| { if (@import("builtin").is_test) { std.log.warn( "{s} '{s}' ({s}: {s}) failed to load: {t}", .{ f.kind.label(), f.name, f.source.label(), f.script_path, err }, ); } else { std.log.err( "{s} '{s}' ({s}: {s}) failed to load: {t}", .{ f.kind.label(), f.name, f.source.label(), f.script_path, err }, ); } return err; }; std.log.debug( "{s}: loaded '{s}' ({s})", .{ f.kind.label(), f.name, f.source.label() }, ); } return runtime.toolCount() - before; } // --------------------------------------------------------------------------- // Path resolution // --------------------------------------------------------------------------- fn kindSubdir(kind: Kind) []const u8 { return switch (kind) { .extension => "extensions", .tool => "tools", }; } fn userKindDir( allocator: Allocator, environ_map: *const std.process.Environ.Map, kind: Kind, ) !?[]u8 { const sub = kindSubdir(kind); if (environ_map.get("XDG_CONFIG_HOME")) |xdg| { return try std.fs.path.join(allocator, &.{ xdg, "panto", sub }); } if (environ_map.get("HOME")) |home| { return try std.fs.path.join(allocator, &.{ home, ".config", "panto", sub }); } return null; } fn projectKindDir(allocator: Allocator, io: Io, kind: Kind) ![]u8 { const cwd = try std.process.currentPathAlloc(io, allocator); defer allocator.free(cwd); return try std.fs.path.join(allocator, &.{ cwd, ".panto", kindSubdir(kind) }); } // --------------------------------------------------------------------------- // Directory scanning // --------------------------------------------------------------------------- fn scanDir( allocator: Allocator, io: Io, dir_path: []const u8, source: Source, kind: Kind, out: *std.array_list.Managed(Found), ) !void { var dir = Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch |err| switch (err) { error.FileNotFound, error.NotDir => return, else => |e| return e, }; defer dir.close(io); var local_names: std.StringHashMap(void) = .init(allocator); defer { var it = local_names.keyIterator(); while (it.next()) |k| allocator.free(k.*); local_names.deinit(); } var iter = dir.iterate(); while (try iter.next(io)) |entry| { if (entry.name.len == 0 or entry.name[0] == '.') continue; const maybe_found: ?Found = switch (entry.kind) { .file, .sym_link => try classifyFile(allocator, dir_path, entry.name, source, kind), .directory => try classifyDirectory(allocator, io, dir, dir_path, entry.name, source, kind), else => null, }; const f = maybe_found orelse continue; const gop = try local_names.getOrPut(f.name); if (gop.found_existing) { if (@import("builtin").is_test) { std.log.warn( "{s} name '{s}' is provided by multiple entries in {s}", .{ kind.label(), f.name, dir_path }, ); } else { std.log.err( "{s} name '{s}' is provided by multiple entries in {s}", .{ kind.label(), f.name, dir_path }, ); } var dup = f; dup.deinit(allocator); return error.DuplicateExtensionInDirectory; } gop.key_ptr.* = try allocator.dupe(u8, f.name); try out.append(f); } } fn classifyFile( allocator: Allocator, dir_path: []const u8, entry_name: []const u8, source: Source, kind: Kind, ) !?Found { if (!std.mem.endsWith(u8, entry_name, ".lua")) return null; const base = entry_name[0 .. entry_name.len - ".lua".len]; if (base.len == 0) return null; const script_path = try std.fs.path.join(allocator, &.{ dir_path, entry_name }); errdefer allocator.free(script_path); const name = try allocator.dupe(u8, base); errdefer allocator.free(name); return Found{ .name = name, .script_path = script_path, .package_root = null, .source = source, .kind = kind, }; } fn classifyDirectory( allocator: Allocator, io: Io, parent: Io.Dir, dir_path: []const u8, entry_name: []const u8, source: Source, kind: Kind, ) !?Found { var sub = parent.openDir(io, entry_name, .{}) catch return null; defer sub.close(io); sub.access(io, "init.lua", .{}) catch |err| switch (err) { error.FileNotFound => return null, else => return null, }; const package_root = try std.fs.path.join(allocator, &.{ dir_path, entry_name }); errdefer allocator.free(package_root); const script_path = try std.fs.path.join(allocator, &.{ package_root, "init.lua" }); errdefer allocator.free(script_path); const name = try allocator.dupe(u8, entry_name); errdefer allocator.free(name); return Found{ .name = name, .script_path = script_path, .package_root = package_root, .source = source, .kind = kind, }; } // --------------------------------------------------------------------------- // Shadowing // --------------------------------------------------------------------------- /// Shadowing key combines (kind, name) so a tool named `foo` does not /// shadow an extension named `foo` (or vice versa). Tool-name collisions /// across these are still caught later by the runtime/registry. const ShadowKey = struct { kind: Kind, name: []const u8, }; const ShadowKeyCtx = struct { pub fn hash(_: ShadowKeyCtx, k: ShadowKey) u64 { var hasher = std.hash.Wyhash.init(0); hasher.update(&[_]u8{@intFromEnum(k.kind)}); hasher.update(k.name); return hasher.final(); } pub fn eql(_: ShadowKeyCtx, a: ShadowKey, b: ShadowKey) bool { return a.kind == b.kind and std.mem.eql(u8, a.name, b.name); } }; fn applyShadowing(allocator: Allocator, list: *std.array_list.Managed(Found)) !void { var latest: std.HashMap(ShadowKey, usize, ShadowKeyCtx, std.hash_map.default_max_load_percentage) = .init(allocator); for (list.items, 0..) |f, i| { try latest.put(.{ .kind = f.kind, .name = f.name }, i); } var keep: std.array_list.Managed(Found) = .init(allocator); var drop: std.array_list.Managed(Found) = .init(allocator); errdefer { latest.deinit(); for (keep.items) |*f| f.deinit(allocator); keep.deinit(); for (drop.items) |*f| f.deinit(allocator); drop.deinit(); } try keep.ensureTotalCapacity(list.items.len); try drop.ensureTotalCapacity(list.items.len); for (list.items, 0..) |f, i| { const winner = latest.get(.{ .kind = f.kind, .name = f.name }).?; if (winner == i) { keep.appendAssumeCapacity(f); } else { std.log.debug( "{s}: '{s}' from {s} shadowed by {s}", .{ f.kind.label(), f.name, f.source.label(), list.items[winner].source.label() }, ); drop.appendAssumeCapacity(f); } } latest.deinit(); for (drop.items) |*f| f.deinit(allocator); drop.deinit(); list.clearRetainingCapacity(); list.appendSlice(keep.items) catch unreachable; keep.deinit(); } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- const testing = std.testing; fn writeFile(dir: Io.Dir, sub_path: []const u8, content: []const u8) !void { try dir.writeFile(testing.io, .{ .sub_path = sub_path, .data = content }); } fn makeDir(dir: Io.Dir, sub_path: []const u8) !void { try dir.createDirPath(testing.io, sub_path); } test "scanDir picks up single-file and directory-style extensions" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "ext_root"); try makeDir(tmp.dir, "ext_root/beta"); try writeFile(tmp.dir, "ext_root/alpha.lua", "-- alpha\n"); try writeFile(tmp.dir, "ext_root/beta/init.lua", "-- beta init\n"); try writeFile(tmp.dir, "ext_root/beta/helper.lua", "-- helper\n"); try writeFile(tmp.dir, "ext_root/.ignored.lua", "-- hidden\n"); try writeFile(tmp.dir, "ext_root/readme.txt", "noise\n"); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const ext_root_len = try tmp.dir.realPathFile(testing.io, "ext_root", &path_buf); const ext_root = path_buf[0..ext_root_len]; var list: std.array_list.Managed(Found) = .init(testing.allocator); defer { for (list.items) |*f| f.deinit(testing.allocator); list.deinit(); } try scanDir(testing.allocator, testing.io, ext_root, .user, .extension, &list); try testing.expectEqual(@as(usize, 2), list.items.len); std.mem.sort(Found, list.items, {}, struct { fn lt(_: void, a: Found, b: Found) bool { return std.mem.lessThan(u8, a.name, b.name); } }.lt); try testing.expectEqualStrings("alpha", list.items[0].name); try testing.expect(list.items[0].package_root == null); try testing.expect(std.mem.endsWith(u8, list.items[0].script_path, "alpha.lua")); try testing.expectEqualStrings("beta", list.items[1].name); try testing.expect(list.items[1].package_root != null); try testing.expect(std.mem.endsWith(u8, list.items[1].script_path, "init.lua")); } test "duplicate name in same directory is an error" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "ext_root/foo"); try writeFile(tmp.dir, "ext_root/foo.lua", "-- single\n"); try writeFile(tmp.dir, "ext_root/foo/init.lua", "-- dir\n"); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const n = try tmp.dir.realPathFile(testing.io, "ext_root", &path_buf); const ext_root = path_buf[0..n]; var list: std.array_list.Managed(Found) = .init(testing.allocator); defer { for (list.items) |*f| f.deinit(testing.allocator); list.deinit(); } const result = scanDir(testing.allocator, testing.io, ext_root, .project, .extension, &list); try testing.expectError(error.DuplicateExtensionInDirectory, result); } test "applyShadowing keeps the latest occurrence" { var list: std.array_list.Managed(Found) = .init(testing.allocator); defer { for (list.items) |*f| f.deinit(testing.allocator); list.deinit(); } inline for (.{ .{ "shared", "/u/shared.lua", Source.user }, .{ "only_user", "/u/only_user.lua", Source.user }, .{ "shared", "/p/shared.lua", Source.project }, .{ "only_project", "/p/only_project.lua", Source.project }, }) |row| { try list.append(.{ .name = try testing.allocator.dupe(u8, row[0]), .script_path = try testing.allocator.dupe(u8, row[1]), .package_root = null, .source = row[2], .kind = .extension, }); } try applyShadowing(testing.allocator, &list); try testing.expectEqual(@as(usize, 3), list.items.len); var shared_count: usize = 0; var shared_source: ?Source = null; for (list.items) |f| { if (std.mem.eql(u8, f.name, "shared")) { shared_count += 1; shared_source = f.source; } } try testing.expectEqual(@as(usize, 1), shared_count); try testing.expectEqual(@as(?Source, .project), shared_source); } test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "user_ext"); try makeDir(tmp.dir, "project_ext"); try writeFile(tmp.dir, "user_ext/greet.lua", \\panto.register_tool { \\ name = "greet", description = "user version", \\ schema = { type = "object" }, \\ handler = function(input) return "USER" end, \\} ); try writeFile(tmp.dir, "project_ext/greet.lua", \\panto.register_tool { \\ name = "greet", description = "project version", \\ schema = { type = "object" }, \\ handler = function(input) return "PROJECT" end, \\} ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const user_len = try tmp.dir.realPathFile(testing.io, "user_ext", &path_buf); const user_path = try testing.allocator.dupe(u8, path_buf[0..user_len]); defer testing.allocator.free(user_path); const proj_len = try tmp.dir.realPathFile(testing.io, "project_ext", &path_buf); const proj_path = try testing.allocator.dupe(u8, path_buf[0..proj_len]); defer testing.allocator.free(proj_path); var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = user_path, .project_extensions = proj_path, }); try testing.expectEqual(@as(usize, 1), n_tools); // Invoke the tool through the source and verify the project handler ran. var src = rt.toolSource(); const calls = [_]panto.ToolCall{.{ .tool_name = "greet", .input = "{}" }}; var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); defer testing.allocator.free(results[0].ok); try testing.expectEqualStrings("PROJECT", results[0].ok); } test "loadFromDirs: tool-name collision between extensions errors" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "ext"); try writeFile(tmp.dir, "ext/alpha.lua", \\panto.register_tool { \\ name = "clash", description = "a", \\ schema = { type = "object" }, \\ handler = function(input) return "a" end, \\} ); try writeFile(tmp.dir, "ext/beta.lua", \\panto.register_tool { \\ name = "clash", description = "b", \\ schema = { type = "object" }, \\ handler = function(input) return "b" end, \\} ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const n = try tmp.dir.realPathFile(testing.io, "ext", &path_buf); const ext_path = try testing.allocator.dupe(u8, path_buf[0..n]); defer testing.allocator.free(ext_path); var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); const result = loadFromDirs(testing.allocator, testing.io, rt, .{ .project_extensions = ext_path, }); try testing.expectError(error.DuplicateTool, result); } test "loadFromDirs: tools/ directory — single-file tool form" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "tools"); try writeFile(tmp.dir, "tools/echo.lua", \\return { \\ name = "echo", description = "Echo back input.", \\ schema = { type = "object", properties = { msg = { type = "string" } } }, \\ handler = function(input) return "echo: " .. input.msg end, \\} ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf); const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); defer testing.allocator.free(tools_path); var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = tools_path, }); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); const calls = [_]panto.ToolCall{.{ .tool_name = "echo", .input = "{\"msg\":\"hi\"}" }}; var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); defer testing.allocator.free(results[0].ok); try testing.expectEqualStrings("echo: hi", results[0].ok); } test "loadFromDirs: tools/ directory — directory-style tool with sibling require" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "tools/shout"); try writeFile(tmp.dir, "tools/shout/util.lua", \\local M = {} \\function M.shout(s) return s:upper() .. "!" end \\return M ); try writeFile(tmp.dir, "tools/shout/init.lua", \\local util = require("util") \\return { \\ name = "shout", description = "uppercase + bang", \\ schema = { type = "object", properties = { text = { type = "string" } } }, \\ handler = function(input) return util.shout(input.text) end, \\} ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf); const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); defer testing.allocator.free(tools_path); var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .project_tools = tools_path, }); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); const calls = [_]panto.ToolCall{.{ .tool_name = "shout", .input = "{\"text\":\"hi\"}" }}; var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); defer testing.allocator.free(results[0].ok); try testing.expectEqualStrings("HI!", results[0].ok); } test "loadFromDirs: project tool shadows user tool of the same name" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "user_tools"); try makeDir(tmp.dir, "project_tools"); try writeFile(tmp.dir, "user_tools/greet.lua", \\return { \\ name = "greet", description = "user version", \\ schema = { type = "object" }, \\ handler = function(input) return "USER" end, \\} ); try writeFile(tmp.dir, "project_tools/greet.lua", \\return { \\ name = "greet", description = "project version", \\ schema = { type = "object" }, \\ handler = function(input) return "PROJECT" end, \\} ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); const user_path = try testing.allocator.dupe(u8, path_buf[0..user_len]); defer testing.allocator.free(user_path); const proj_len = try tmp.dir.realPathFile(testing.io, "project_tools", &path_buf); const proj_path = try testing.allocator.dupe(u8, path_buf[0..proj_len]); defer testing.allocator.free(proj_path); var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = user_path, .project_tools = proj_path, }); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); const calls = [_]panto.ToolCall{.{ .tool_name = "greet", .input = "{}" }}; var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); defer testing.allocator.free(results[0].ok); try testing.expectEqualStrings("PROJECT", results[0].ok); } test "loadFromDirs: extension and tool share a *file* name independently" { // The shadow-key is (kind, name) so an extension named `foo` and a // tool named `foo` coexist — as long as their *registered* tool names // don't collide. var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); try makeDir(tmp.dir, "ext"); try makeDir(tmp.dir, "tools"); try writeFile(tmp.dir, "ext/foo.lua", \\panto.register_tool { \\ name = "ext_foo", description = "e", \\ schema = { type = "object" }, \\ handler = function(input) return "ext" end, \\} ); try writeFile(tmp.dir, "tools/foo.lua", \\return { \\ name = "tool_foo", description = "t", \\ schema = { type = "object" }, \\ handler = function(input) return "tool" end, \\} ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; const e_len = try tmp.dir.realPathFile(testing.io, "ext", &path_buf); const ext_path = try testing.allocator.dupe(u8, path_buf[0..e_len]); defer testing.allocator.free(ext_path); const t_len = try tmp.dir.realPathFile(testing.io, "tools", &path_buf); const tools_path = try testing.allocator.dupe(u8, path_buf[0..t_len]); defer testing.allocator.free(tools_path); var rt = try LuaRuntime.create(testing.allocator); defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = ext_path, .user_tools = tools_path, }); try testing.expectEqual(@as(usize, 2), n_tools); }