finish lua runtime makeover

- new multi-tool registration via ToolSource
- thread per source-or-standalone-tool
- switched to zig 0.16 Io threading interface
- cli: include `luv` package and run concurrent lua tools via libuv
- one single long-lived lua_State for the whole cli program

1 files changed, 42 insertions, 174 deletions

diff --git a/src/extension_loader.zig b/src/extension_loader.zig
index 158e6b6..2a35d15 100644
--- a/src/extension_loader.zig
+++ b/src/extension_loader.zig
@@ -1,38 +1,34 @@
 //! Extension discovery: walk well-known directories, locate Lua extensions,
-//! and register their tools with a `ToolRegistry`.
+//! and load each one into a long-lived `LuaRuntime`.
 //!
 //! Search order (later entries shadow earlier ones by extension *name*):
 //!   1. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/`  ("user")
 //!   2. `./.panto/extensions/`                                  ("project")
 //!
-//! Layout per directory: each entry is either
+//! Layout per directory:
 //!   - `<name>.lua`        -- single-file extension; the extension name is
 //!                            the basename without the `.lua` suffix.
-//!   - `<name>/init.lua`   -- directory extension; the extension name is the
-//!                            directory name. The directory is added to the
-//!                            extension's `package.path` so it can `require`
-//!                            sibling Lua files.
+//!   - `<name>/init.lua`   -- directory extension; the extension name is
+//!                            the directory name. The directory is added
+//!                            to the extension's `package.path` so it can
+//!                            `require` sibling Lua files.
 //!
 //! Conflict rules:
-//!   - Within a single directory, two entries with the same extension name
+//!   - Within one directory, two entries with the same extension name
 //!     are an error.
 //!   - Project shadows user by extension name (debug-logged, not an error).
-//!   - Tool-name collisions *between* loaded extensions are an error: a tool
-//!     name is a contract the LLM relies on, and surprising overrides at
-//!     load time are worse than failing fast.
+//!   - Tool-name collisions *between* loaded extensions are an error: a
+//!     tool name is a contract the LLM relies on.
 //!
-//! Symlinks: followed normally (no special handling).
-//! Hidden files: dotfiles (`.foo.lua`) are skipped to leave room for editor
-//! swap files and the like.
+//! Symlinks: followed normally. Dotfiles: skipped.
 
 const std = @import("std");
 const panto = @import("panto");
-const lua_tool = @import("lua_tool.zig");
-const lua_bridge = @import("lua_bridge.zig");
+const lua_runtime = @import("lua_runtime.zig");
 
-const c = lua_bridge.c;
 const Allocator = std.mem.Allocator;
 const Io = std.Io;
+const LuaRuntime = lua_runtime.LuaRuntime;
 
 /// A discovered extension before loading. Owns its strings.
 const Found = struct {
@@ -40,11 +36,9 @@ const Found = struct {
     name: []u8,
     /// Absolute path to the Lua script to execute.
     script_path: []u8,
-    /// For directory-style extensions, the directory containing the script
-    /// (used to extend `package.path`). Null for single-file extensions.
+    /// For directory-style extensions, the directory containing the script.
     package_root: ?[]u8,
-    /// Which search-path source this came from. Informational, used for
-    /// shadowing log messages and tool-conflict error context.
+    /// Which search-path source this came from.
     source: Source,
 
     pub fn deinit(self: *Found, allocator: Allocator) void {
@@ -66,17 +60,16 @@ pub const Source = enum {
     }
 };
 
-/// Discover and register every extension found in the standard paths
-/// derived from the environment + current working directory. Returns the
-/// number of *tools* (not extensions) registered.
+/// Discover and load every extension found in the standard paths into
+/// `runtime`. Returns the number of *tools* (not extensions) declared.
 ///
 /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The
-/// project directory is always taken as `cwd()/.panto/extensions`.
+/// project directory is always `cwd()/.panto/extensions`.
 pub fn discoverAndLoad(
     allocator: Allocator,
     io: Io,
     environ_map: *const std.process.Environ.Map,
-    registry: *panto.ToolRegistry,
+    runtime: *LuaRuntime,
 ) !usize {
     const user_dir = try userExtensionsDir(allocator, environ_map);
     defer if (user_dir) |d| allocator.free(d);
@@ -84,21 +77,18 @@ pub fn discoverAndLoad(
     const project_dir = try projectExtensionsDir(allocator, io);
     defer allocator.free(project_dir);
 
-    return loadFromDirs(allocator, io, registry, user_dir, project_dir);
+    return loadFromDirs(allocator, io, runtime, user_dir, project_dir);
 }
 
-/// Lower-level entry point: load from explicit user/project paths. Useful
-/// for tests that want deterministic behavior without touching
-/// process-global state (HOME, cwd). Either path may be null; missing
-/// directories on disk are silently skipped.
+/// Lower-level entry point: load from explicit user/project paths.
+/// Either path may be null; missing directories are silently skipped.
 pub fn loadFromDirs(
     allocator: Allocator,
     io: Io,
-    registry: *panto.ToolRegistry,
+    runtime: *LuaRuntime,
     user_dir: ?[]const u8,
     project_dir: ?[]const u8,
 ) !usize {
-    // 1. Collect candidates from both sources, project last so it wins.
     var found: std.array_list.Managed(Found) = .init(allocator);
     defer {
         for (found.items) |*f| f.deinit(allocator);
@@ -108,16 +98,11 @@ pub fn loadFromDirs(
     if (user_dir) |d| try scanDir(allocator, io, d, .user, &found);
     if (project_dir) |d| try scanDir(allocator, io, d, .project, &found);
 
-    // 2. Apply project-shadows-user by name. The latest occurrence wins.
     try applyShadowing(allocator, &found);
 
-    // 3. Load each surviving extension. Tool-name conflicts surface as
-    //    `ToolRegistry.register` errors and abort startup.
-    var total_tools: usize = 0;
+    const before = runtime.toolCount();
     for (found.items) |f| {
-        const n = loadOne(allocator, registry, f) catch |err| {
-            // In test builds, log at warn level so a deliberate failure
-            // test doesn't trip the test runner's err-count check.
+        runtime.loadExtension(f.script_path, f.package_root) catch |err| {
             if (@import("builtin").is_test) {
                 std.log.warn(
                     "extension '{s}' ({s}: {s}) failed to load: {t}",
@@ -132,21 +117,17 @@ pub fn loadFromDirs(
             return err;
         };
         std.log.debug(
-            "extension: loaded {d} tool(s) from '{s}' ({s})",
-            .{ n, f.name, f.source.label() },
+            "extension: loaded '{s}' ({s})",
+            .{ f.name, f.source.label() },
         );
-        total_tools += n;
     }
-    return total_tools;
+    return runtime.toolCount() - before;
 }
 
 // ---------------------------------------------------------------------------
 // Path resolution
 // ---------------------------------------------------------------------------
 
-/// Returns the absolute path of the user extensions directory, or null if
-/// `HOME` is unset and `XDG_CONFIG_HOME` is not provided either. Caller
-/// owns the returned slice.
 fn userExtensionsDir(
     allocator: Allocator,
     environ_map: *const std.process.Environ.Map,
@@ -160,8 +141,6 @@ fn userExtensionsDir(
     return null;
 }
 
-/// Returns the absolute path of the project extensions directory.
-/// Caller owns the returned slice.
 fn projectExtensionsDir(allocator: Allocator, io: Io) ![]u8 {
     const cwd = try std.process.currentPathAlloc(io, allocator);
     defer allocator.free(cwd);
@@ -172,8 +151,6 @@ fn projectExtensionsDir(allocator: Allocator, io: Io) ![]u8 {
 // Directory scanning
 // ---------------------------------------------------------------------------
 
-/// Scan `dir_path` and append any candidate extensions to `out`. Missing
-/// directories are not an error: extensions are an optional feature.
 fn scanDir(
     allocator: Allocator,
     io: Io,
@@ -196,7 +173,6 @@ fn scanDir(
 
     var iter = dir.iterate();
     while (try iter.next(io)) |entry| {
-        // Skip dotfiles (editor swap files, .DS_Store, hidden dirs, ...).
         if (entry.name.len == 0 or entry.name[0] == '.') continue;
 
         const maybe_found: ?Found = switch (entry.kind) {
@@ -206,8 +182,6 @@ fn scanDir(
         };
         const f = maybe_found orelse continue;
 
-        // Within one directory, duplicate names are an error. (Can happen
-        // if both `foo.lua` and `foo/init.lua` exist.)
         const gop = try local_names.getOrPut(f.name);
         if (gop.found_existing) {
             if (@import("builtin").is_test) {
@@ -221,13 +195,10 @@ fn scanDir(
                     .{ f.name, dir_path },
                 );
             }
-            // Free the duplicate's resources before bailing.
             var dup = f;
             dup.deinit(allocator);
             return error.DuplicateExtensionInDirectory;
         }
-        // Hash map key is borrowed from f.name; we need an independent copy
-        // since `out` owns f and we'd otherwise double-free.
         gop.key_ptr.* = try allocator.dupe(u8, f.name);
 
         try out.append(f);
@@ -257,8 +228,6 @@ fn classifyFile(
     };
 }
 
-/// Treat `entry_name/init.lua` as a directory-style extension if present.
-/// Returns null if no `init.lua` exists inside.
 fn classifyDirectory(
     allocator: Allocator,
     io: Io,
@@ -267,13 +236,12 @@ fn classifyDirectory(
     entry_name: []const u8,
     source: Source,
 ) !?Found {
-    // Probe for init.lua before doing any allocation.
     var sub = parent.openDir(io, entry_name, .{}) catch return null;
     defer sub.close(io);
 
     sub.access(io, "init.lua", .{}) catch |err| switch (err) {
         error.FileNotFound => return null,
-        else => return null, // permission, etc. — quietly skip
+        else => return null,
     };
 
     const package_root = try std.fs.path.join(allocator, &.{ dir_path, entry_name });
@@ -295,14 +263,6 @@ fn classifyDirectory(
 // Shadowing
 // ---------------------------------------------------------------------------
 
-/// In-place: for every name that appears more than once, keep only the
-/// *last* occurrence and drop earlier ones with a debug log.
-///
-/// We do this in two passes. The first pass populates a `name → winning
-/// index` map (last-write-wins by construction). The second pass walks
-/// the list once, partitioning into kept and dropped sets *without*
-/// freeing strings yet — the map still references them via its keys, so
-/// freeing mid-walk would create dangling keys in the hash table.
 fn applyShadowing(allocator: Allocator, list: *std.array_list.Managed(Found)) !void {
     var latest: std.StringHashMap(usize) = .init(allocator);
 
@@ -335,52 +295,25 @@ fn applyShadowing(allocator: Allocator, list: *std.array_list.Managed(Found)) !v
         }
     }
 
-    // The map's keys still alias `drop`'s entries; deinit it first so we
-    // can then free those entries' strings without leaving dangling keys.
     latest.deinit();
     for (drop.items) |*f| f.deinit(allocator);
     drop.deinit();
 
-    // Replace list contents without re-freeing the entries we kept.
     list.clearRetainingCapacity();
     list.appendSlice(keep.items) catch unreachable;
     keep.deinit();
 }
 
 // ---------------------------------------------------------------------------
-// Loading
-// ---------------------------------------------------------------------------
-
-/// Load one discovered extension and register its tools. Returns the
-/// number of tools registered.
-fn loadOne(
-    allocator: Allocator,
-    registry: *panto.ToolRegistry,
-    found: Found,
-) !usize {
-    if (found.package_root) |root| {
-        return lua_tool.loadExtensionWithPackageRoot(
-            allocator,
-            registry,
-            found.script_path,
-            root,
-        );
-    }
-    return lua_tool.loadExtension(allocator, registry, found.script_path);
-}
-
-// ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
 
 const testing = std.testing;
 
-/// Helper: write a single file inside `dir` at `sub_path`.
 fn writeFile(dir: Io.Dir, sub_path: []const u8, content: []const u8) !void {
     try dir.writeFile(testing.io, .{ .sub_path = sub_path, .data = content });
 }
 
-/// Helper: create a directory (path may contain separators).
 fn makeDir(dir: Io.Dir, sub_path: []const u8) !void {
     try dir.createDirPath(testing.io, sub_path);
 }
@@ -389,14 +322,6 @@ test "scanDir picks up single-file and directory-style extensions" {
     var tmp = testing.tmpDir(.{ .iterate = true });
     defer tmp.cleanup();
 
-    // Layout:
-    //   ext_root/
-    //     alpha.lua
-    //     beta/
-    //       init.lua
-    //       helper.lua
-    //     .ignored.lua        (dotfile -> skipped)
-    //     readme.txt          (non-.lua -> skipped)
     try makeDir(tmp.dir, "ext_root");
     try makeDir(tmp.dir, "ext_root/beta");
     try writeFile(tmp.dir, "ext_root/alpha.lua", "-- alpha\n");
@@ -405,7 +330,6 @@ test "scanDir picks up single-file and directory-style extensions" {
     try writeFile(tmp.dir, "ext_root/.ignored.lua", "-- hidden\n");
     try writeFile(tmp.dir, "ext_root/readme.txt", "noise\n");
 
-    // Absolute path to ext_root.
     var path_buf: [std.fs.max_path_bytes]u8 = undefined;
     const ext_root_len = try tmp.dir.realPathFile(testing.io, "ext_root", &path_buf);
     const ext_root = path_buf[0..ext_root_len];
@@ -419,7 +343,6 @@ test "scanDir picks up single-file and directory-style extensions" {
 
     try testing.expectEqual(@as(usize, 2), list.items.len);
 
-    // Order is filesystem-dependent; sort by name for stable assertions.
     std.mem.sort(Found, list.items, {}, struct {
         fn lt(_: void, a: Found, b: Found) bool {
             return std.mem.lessThan(u8, a.name, b.name);
@@ -439,7 +362,6 @@ test "duplicate name in same directory is an error" {
     var tmp = testing.tmpDir(.{ .iterate = true });
     defer tmp.cleanup();
 
-    // Both `foo.lua` and `foo/init.lua` exist.
     try makeDir(tmp.dir, "ext_root/foo");
     try writeFile(tmp.dir, "ext_root/foo.lua", "-- single\n");
     try writeFile(tmp.dir, "ext_root/foo/init.lua", "-- dir\n");
@@ -482,7 +404,6 @@ test "applyShadowing keeps the latest occurrence" {
     try applyShadowing(testing.allocator, &list);
     try testing.expectEqual(@as(usize, 3), list.items.len);
 
-    // "shared" survives once, from the project source.
     var shared_count: usize = 0;
     var shared_source: ?Source = null;
     for (list.items) |f| {
@@ -495,7 +416,7 @@ test "applyShadowing keeps the latest occurrence" {
     try testing.expectEqual(@as(?Source, .project), shared_source);
 }
 
-test "loadFromDirs: project shadows user end-to-end" {
+test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" {
     var tmp = testing.tmpDir(.{ .iterate = true });
     defer tmp.cleanup();
 
@@ -526,66 +447,19 @@ test "loadFromDirs: project shadows user end-to-end" {
     const proj_path = try testing.allocator.dupe(u8, path_buf[0..proj_len]);
     defer testing.allocator.free(proj_path);
 
-    var registry = panto.ToolRegistry.init(testing.allocator);
-    defer registry.deinit();
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
 
-    const n_tools = try loadFromDirs(
-        testing.allocator,
-        testing.io,
-        &registry,
-        user_path,
-        proj_path,
-    );
+    const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, user_path, proj_path);
     try testing.expectEqual(@as(usize, 1), n_tools);
 
-    const tool = registry.lookup("greet") orelse return error.NotRegistered;
-    try testing.expectEqualStrings("project version", tool.description);
-
-    const out = try tool.vtable.invoke(tool.ctx, "{}", testing.allocator);
-    defer testing.allocator.free(out);
-    try testing.expectEqualStrings("PROJECT", out);
-}
-
-test "loadFromDirs: directory-style extension can require siblings" {
-    var tmp = testing.tmpDir(.{ .iterate = true });
-    defer tmp.cleanup();
-
-    try makeDir(tmp.dir, "ext/composer");
-    try writeFile(tmp.dir, "ext/composer/util.lua",
-        \\local M = {}
-        \\function M.shout(s) return s:upper() .. "!" end
-        \\return M
-    );
-    try writeFile(tmp.dir, "ext/composer/init.lua",
-        \\local util = require("util")
-        \\panto.register_tool {
-        \\  name = "shout", description = "uppercase + bang",
-        \\  schema = { type = "object", properties = { text = { type = "string" } } },
-        \\  handler = function(input) return util.shout(input.text) end,
-        \\}
-    );
-
-    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
-    const ext_len = try tmp.dir.realPathFile(testing.io, "ext", &path_buf);
-    const ext_path = try testing.allocator.dupe(u8, path_buf[0..ext_len]);
-    defer testing.allocator.free(ext_path);
-
-    var registry = panto.ToolRegistry.init(testing.allocator);
-    defer registry.deinit();
-
-    const n = try loadFromDirs(
-        testing.allocator,
-        testing.io,
-        &registry,
-        null,
-        ext_path,
-    );
-    try testing.expectEqual(@as(usize, 1), n);
-
-    const tool = registry.lookup("shout") orelse return error.NotRegistered;
-    const out = try tool.vtable.invoke(tool.ctx, "{\"text\":\"hi\"}", testing.allocator);
-    defer testing.allocator.free(out);
-    try testing.expectEqualStrings("HI!", out);
+    // Invoke the tool through the source and verify the project handler ran.
+    var src = rt.toolSource();
+    const calls = [_]panto.ToolCall{.{ .tool_name = "greet", .input = "{}" }};
+    var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }};
+    try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator);
+    defer testing.allocator.free(results[0].ok);
+    try testing.expectEqualStrings("PROJECT", results[0].ok);
 }
 
 test "loadFromDirs: tool-name collision between extensions errors" {
@@ -613,15 +487,9 @@ test "loadFromDirs: tool-name collision between extensions errors" {
     const ext_path = try testing.allocator.dupe(u8, path_buf[0..n]);
     defer testing.allocator.free(ext_path);
 
-    var registry = panto.ToolRegistry.init(testing.allocator);
-    defer registry.deinit();
+    var rt = try LuaRuntime.create(testing.allocator);
+    defer rt.deinit();
 
-    const result = loadFromDirs(
-        testing.allocator,
-        testing.io,
-        &registry,
-        null,
-        ext_path,
-    );
+    const result = loadFromDirs(testing.allocator, testing.io, rt, null, ext_path);
     try testing.expectError(error.DuplicateTool, result);
 }