diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | agent/README.md | 10 | ||||
| -rw-r--r-- | build.zig | 4 | ||||
| -rw-r--r-- | libpanto/src/agent.zig | 381 | ||||
| -rw-r--r-- | libpanto/src/anthropic_messages_json.zig | 47 | ||||
| -rw-r--r-- | libpanto/src/config.zig | 118 | ||||
| -rw-r--r-- | libpanto/src/openai_chat_json.zig | 36 | ||||
| -rw-r--r-- | libpanto/src/provider.zig | 107 | ||||
| -rw-r--r-- | libpanto/src/provider_anthropic_messages.zig | 113 | ||||
| -rw-r--r-- | libpanto/src/provider_openai_chat.zig | 114 | ||||
| -rw-r--r-- | libpanto/src/root.zig | 12 | ||||
| -rw-r--r-- | libpanto/src/tool_registry.zig | 169 | ||||
| -rw-r--r-- | src/config_file.zig | 977 | ||||
| -rw-r--r-- | src/extension_loader.zig | 223 | ||||
| -rw-r--r-- | src/glob.zig | 121 | ||||
| -rw-r--r-- | src/lua_runtime.zig | 32 | ||||
| -rw-r--r-- | src/luarocks_runtime.zig | 66 | ||||
| -rw-r--r-- | src/main.zig | 196 | ||||
| -rw-r--r-- | src/models_toml.zig | 375 | ||||
| -rw-r--r-- | src/panto_home.zig | 4 | ||||
| -rw-r--r-- | src/subcommand.zig | 14 |
21 files changed, 2536 insertions, 584 deletions
@@ -1,3 +1,4 @@ .zig-cache/ zig-out/ zig-pkg/ +mise.local.toml diff --git a/agent/README.md b/agent/README.md index 00e6e60..44d31fe 100644 --- a/agent/README.md +++ b/agent/README.md @@ -1,14 +1,14 @@ -# panto system agent tree +# panto base agent tree Contents of this directory are embedded into the `panto` binary at build time and staged onto disk at bootstrap into `$PANTO_HOME/agent/` (i.e. `$XDG_DATA_HOME/panto/agent/`, or `~/.local/share/panto/agent/` if `XDG_DATA_HOME` is unset). -The staged tree is the "system" layer of panto's extension/tool +The staged tree is the "base" layer of panto's extension/tool search path — sitting underneath the user layer (`$XDG_CONFIG_HOME/panto/`) and the project layer -(`./.panto/`). Project shadows user shadows system. +(`./.panto/`). Project shadows user shadows base. Files are written with `writeIfDifferent` semantics: on each bootstrap, embedded contents are compared against what's on disk and @@ -16,10 +16,10 @@ only rewritten when they differ. This keeps mtimes stable across reruns. The staged tree is **panto's territory** — local edits will be -overwritten on the next bootstrap. To customize a system tool, +overwritten on the next bootstrap. To customize a base tool, override it at the user or project layer (a file with the same basename under `$XDG_CONFIG_HOME/panto/tools/` or `./.panto/tools/` -shadows the system copy). +shadows the base copy). ## Layout @@ -43,7 +43,7 @@ pub fn build(b: *std.Build) void { // And the in-repo `agent/` tree: bundled into the binary and // staged at $PANTO_HOME/agent/ on first run. This is panto's - // "system" extension/tool layer (read/write/edit/bash etc.). + // "base" extension/tool layer (read/write/edit/bash etc.). const agent_embed_path = generateAgentEmbed(b); // Constants module: makes Zig know the Lua and luarocks versions @@ -314,7 +314,7 @@ fn generateLuarocksEmbed( /// Codegen step: walk the in-repo `agent/` tree and emit a Zig module /// embedding every file. Bootstrap stages the result under /// `$PANTO_HOME/agent/` on first run, where the runtime's extension -/// loader finds it as the "system" layer (below user and project). +/// loader finds it as the "base" layer (below user and project). /// /// Unlike `generateLuarocksEmbed`, the agent tree lives in-repo and /// its contents change as we add/edit tools. `addDirectoryArg` on a diff --git a/libpanto/src/agent.zig b/libpanto/src/agent.zig index 92c5a3a..08cc91c 100644 --- a/libpanto/src/agent.zig +++ b/libpanto/src/agent.zig @@ -29,6 +29,7 @@ const Allocator = std.mem.Allocator; const Io = std.Io; const provider_mod = @import("provider.zig"); +const config_mod = @import("config.zig"); const conversation = @import("conversation.zig"); const tool_mod = @import("tool.zig"); const tool_source_mod = @import("tool_source.zig"); @@ -40,40 +41,46 @@ pub const ToolRegistry = tool_registry_mod.ToolRegistry; const Entry = tool_registry_mod.Entry; +pub const Config = config_mod.Config; + pub const Agent = struct { - provider: provider_mod.Provider, allocator: Allocator, io: Io, - registry: ToolRegistry, + /// The active configuration snapshot, consulted fresh at the top of + /// every turn. Immutable while a turn is in flight; swap this pointer + /// (`setConfig`) between turns to change provider/model/base_url and/or + /// the visible tool set atomically. The pointee and its registry are + /// owned by the embedder, not the agent. + config: *const Config, + /// Injectable streaming seam. Defaults to the real provider dispatch + /// (`provider_mod.streamStep`); tests override it with a stub. + stream_fn: provider_mod.StreamFn = provider_mod.streamStep, - pub fn init(allocator: Allocator, io: Io, prov: provider_mod.Provider) Agent { + pub fn init(allocator: Allocator, io: Io, config: *const Config) Agent { return .{ - .provider = prov, .allocator = allocator, .io = io, - .registry = ToolRegistry.init(allocator), + .config = config, }; } pub fn deinit(self: *Agent) void { - self.registry.deinit(); - self.provider.deinit(); - } - - /// Register a single tool. The agent's registry takes ownership. - pub fn registerTool(self: *Agent, tool: Tool) !void { - try self.registry.register(tool); + // The agent owns neither the config snapshot nor the registry it + // borrows; the embedder tears those down. + _ = self; } - /// Register a tool source. The agent's registry takes ownership. - pub fn registerToolSource(self: *Agent, src: ToolSource) !void { - try self.registry.registerSource(src); + /// Swap the active configuration snapshot. Takes effect at the start of + /// the next turn. Safe to call between `runStep` invocations or from a + /// tool handler that runs between provider steps; never mutates a + /// snapshot a turn is currently reading. + pub fn setConfig(self: *Agent, config: *const Config) void { + self.config = config; } - /// Remove a tool by name. No-op if not registered or if the name - /// belongs to a source. - pub fn unregisterTool(self: *Agent, name: []const u8) void { - self.registry.unregister(name); + /// The registry exposed by the active snapshot. + pub fn registry(self: *const Agent) *const ToolRegistry { + return self.config.registry; } /// Drive the conversation forward until the model stops calling tools. @@ -83,7 +90,10 @@ pub const Agent = struct { receiver: *provider_mod.Receiver, ) !void { while (true) { - try self.provider.streamStep(conv, &self.registry, receiver); + // Re-read the config snapshot at the top of each turn so a + // mid-conversation swap takes effect here, never mid-stream. + const cfg = self.config; + try self.stream_fn(self.allocator, self.io, cfg, conv, receiver); const last = conv.messages.items[conv.messages.items.len - 1]; std.debug.assert(last.role == .assistant); @@ -122,7 +132,7 @@ pub const Agent = struct { for (assistant_msg.content.items) |block| { if (block != .ToolUse) continue; const tu = block.ToolUse; - const entry = self.registry.lookup(tu.name) orelse { + const entry = self.config.registry.lookup(tu.name) orelse { // Unknown tool: abort the turn with a clear error. return error.UnknownTool; }; @@ -379,6 +389,15 @@ fn runSourceGroup(agent: *Agent, sg: Group.SourceGroup, calls: []FlatCall) void const testing = std.testing; +/// Test harness for the injectable `stream_fn` seam. +/// +/// `provider_mod.StreamFn` carries no user context (it mirrors the real +/// free function exactly), so the stub parks its state in a module-level +/// pointer that `stubStreamStep` reads. The Zig test runner executes tests +/// serially in one process, so a single global slot is safe; each test +/// sets it via `install` before driving the agent. +var stub_active: ?*StubProvider = null; + const StubProvider = struct { allocator: Allocator, scripted: []const ScriptedTurn, @@ -397,60 +416,85 @@ const StubProvider = struct { }, }; - fn provider(self: *StubProvider) provider_mod.Provider { - return .{ .ptr = self, .vtable = &vt }; + /// Point the global seam at this stub and return the function to assign + /// to `agent.stream_fn`. Call once per test, after constructing the + /// stub on the stack. + fn install(self: *StubProvider) provider_mod.StreamFn { + stub_active = self; + return stubStreamStep; } +}; - const vt: provider_mod.ProviderVTable = .{ - .streamStep = vtStreamStep, - .deinit = vtDeinit, - }; - - fn vtStreamStep( - ptr: *anyopaque, - conv: *conversation.Conversation, - _: *const ToolRegistry, - _: *provider_mod.Receiver, - ) anyerror!void { - const self: *StubProvider = @ptrCast(@alignCast(ptr)); - if (self.next >= self.scripted.len) return error.NoMoreScriptedTurns; - const turn = self.scripted[self.next]; - self.next += 1; +fn stubStreamStep( + allocator: Allocator, + _: Io, + _: *const config_mod.Config, + conv: *conversation.Conversation, + _: *provider_mod.Receiver, +) anyerror!void { + const self = stub_active orelse return error.NoStubInstalled; + _ = allocator; + if (self.next >= self.scripted.len) return error.NoMoreScriptedTurns; + const turn = self.scripted[self.next]; + self.next += 1; - var blocks: std.ArrayList(conversation.ContentBlock) = .empty; - errdefer { - for (blocks.items) |*b| b.deinit(self.allocator); - blocks.deinit(self.allocator); - } - for (turn.blocks) |tb| { - switch (tb) { - .Text => |s| { - try blocks.append(self.allocator, .{ - .Text = try conversation.textualBlockFromSlice(self.allocator, s), - }); - }, - .ToolUse => |tu| { - const id = try self.allocator.dupe(u8, tu.id); - errdefer self.allocator.free(id); - const name = try self.allocator.dupe(u8, tu.name); - errdefer self.allocator.free(name); - var input_buf: conversation.TextualBlock = .empty; - errdefer input_buf.deinit(self.allocator); - try input_buf.appendSlice(self.allocator, tu.input); - try blocks.append(self.allocator, .{ .ToolUse = .{ - .id = id, - .name = name, - .input = input_buf, - } }); - }, - } + var blocks: std.ArrayList(conversation.ContentBlock) = .empty; + errdefer { + for (blocks.items) |*b| b.deinit(self.allocator); + blocks.deinit(self.allocator); + } + for (turn.blocks) |tb| { + switch (tb) { + .Text => |s| { + try blocks.append(self.allocator, .{ + .Text = try conversation.textualBlockFromSlice(self.allocator, s), + }); + }, + .ToolUse => |tu| { + const id = try self.allocator.dupe(u8, tu.id); + errdefer self.allocator.free(id); + const name = try self.allocator.dupe(u8, tu.name); + errdefer self.allocator.free(name); + var input_buf: conversation.TextualBlock = .empty; + errdefer input_buf.deinit(self.allocator); + try input_buf.appendSlice(self.allocator, tu.input); + try blocks.append(self.allocator, .{ .ToolUse = .{ + .id = id, + .name = name, + .input = input_buf, + } }); + }, } - const moved = try blocks.toOwnedSlice(self.allocator); - defer self.allocator.free(moved); - try conv.addAssistantMessage(moved); } + const moved = try blocks.toOwnedSlice(self.allocator); + defer self.allocator.free(moved); + try conv.addAssistantMessage(moved); +} - fn vtDeinit(_: *anyopaque) void {} +/// Build a stack registry + active `Config` snapshot wired together, for +/// tests that drive the agent. The caller owns both and must keep them +/// alive for the agent's lifetime. +const TestHarness = struct { + registry: ToolRegistry, + config: config_mod.Config, + + fn init(allocator: Allocator) TestHarness { + return .{ .registry = ToolRegistry.init(allocator), .config = undefined }; + } + + /// Finalize the config snapshot to point at this harness's registry. + /// Must be called after `init` and before constructing the agent, once + /// the harness has a stable address. + fn activate(self: *TestHarness) void { + self.config = .{ + .provider = .{ .openai_chat = .{ .api_key = "k", .base_url = "u", .model = "m" } }, + .registry = &self.registry, + }; + } + + fn deinit(self: *TestHarness) void { + self.registry.deinit(); + } }; const EchoTool = struct { @@ -574,7 +618,7 @@ const FailingTool = struct { const NoopReceiver = struct { fn make() provider_mod.Receiver { - return .{ .ptr = @constCast(@ptrCast(&dummy)), .vtable = &vt }; + return .{ .ptr = @ptrCast(@constCast(&dummy)), .vtable = &vt }; } var dummy: u8 = 0; const vt: provider_mod.ReceiverVTable = .{ @@ -772,31 +816,21 @@ const FailingSource = struct { } }; -test "registerTool and lookup via registry" { - var stub = StubProvider{ .allocator = testing.allocator, .scripted = &.{} }; - var threaded: std.Io.Threaded = .init(testing.allocator, .{}); - defer threaded.deinit(); - const io = threaded.io(); - var agent = Agent.init(testing.allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerTool(try EchoTool.create(testing.allocator, "echo", "ECHO:")); - try testing.expectEqual(@as(usize, 1), agent.registry.count()); - try testing.expect(agent.registry.lookup("echo") != null); +test "registry register and lookup" { + var h = TestHarness.init(testing.allocator); + defer h.deinit(); + try h.registry.register(try EchoTool.create(testing.allocator, "echo", "ECHO:")); + try testing.expectEqual(@as(usize, 1), h.registry.count()); + try testing.expect(h.registry.lookup("echo") != null); } -test "duplicate registerTool returns error" { - var stub = StubProvider{ .allocator = testing.allocator, .scripted = &.{} }; - var threaded: std.Io.Threaded = .init(testing.allocator, .{}); - defer threaded.deinit(); - const io = threaded.io(); - var agent = Agent.init(testing.allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerTool(try EchoTool.create(testing.allocator, "echo", "A:")); +test "duplicate register returns error" { + var h = TestHarness.init(testing.allocator); + defer h.deinit(); + try h.registry.register(try EchoTool.create(testing.allocator, "echo", "A:")); var dup = try EchoTool.create(testing.allocator, "echo", "B:"); - try testing.expectError(error.DuplicateTool, agent.registerTool(dup)); + try testing.expectError(error.DuplicateTool, h.registry.register(dup)); dup.vtable.deinit(dup.ctx, testing.allocator); } @@ -815,10 +849,12 @@ test "runStep dispatches a tool call and loops to a final text turn" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerTool(try EchoTool.create(allocator, "echo", "ECHO:")); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.register(try EchoTool.create(allocator, "echo", "ECHO:")); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -862,12 +898,14 @@ test "runStep dispatches multiple tool calls in parallel" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerTool(try BarrierTool.create(allocator, "barrierA", &barrier)); - try agent.registerTool(try BarrierTool.create(allocator, "barrierB", &barrier)); - try agent.registerTool(try BarrierTool.create(allocator, "barrierC", &barrier)); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.register(try BarrierTool.create(allocator, "barrierA", &barrier)); + try h.registry.register(try BarrierTool.create(allocator, "barrierB", &barrier)); + try h.registry.register(try BarrierTool.create(allocator, "barrierC", &barrier)); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -902,10 +940,12 @@ test "runStep propagates tool errors and aborts the turn" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerTool(try FailingTool.create(allocator, "boom")); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.register(try FailingTool.create(allocator, "boom")); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -929,8 +969,11 @@ test "runStep errors UnknownTool when the model calls something unregistered" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); + var h = TestHarness.init(allocator); + defer h.deinit(); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -950,8 +993,11 @@ test "runStep with no tool calls returns after one provider step" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); + var h = TestHarness.init(allocator); + defer h.deinit(); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -974,8 +1020,11 @@ test "runStep surfaces EmptyAssistantResponse when provider commits an empty mes var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); + var h = TestHarness.init(allocator); + defer h.deinit(); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -1002,10 +1051,12 @@ test "runStep delivers all source-backed calls in one batch on one thread" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerToolSource(try TestSource.create(allocator, "panto-lua", &.{ "lua_x", "lua_y" })); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.registerSource(try TestSource.create(allocator, "panto-lua", &.{ "lua_x", "lua_y" })); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -1015,7 +1066,7 @@ test "runStep delivers all source-backed calls in one batch on one thread" { try agent.runStep(&conv, &recv); // Locate the source and inspect its observed batches. - const view = agent.registry.lookup("lua_x") orelse return error.NotFound; + const view = h.registry.lookup("lua_x") orelse return error.NotFound; const src_ptr = view.entry.source.source; const test_src: *TestSource = @ptrCast(@alignCast(src_ptr.ctx)); @@ -1051,11 +1102,13 @@ test "runStep: distinct sources run on distinct threads in parallel" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerToolSource(try TestSource.create(allocator, "src_a", &.{"src_a_t"})); - try agent.registerToolSource(try TestSource.create(allocator, "src_b", &.{"src_b_t"})); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.registerSource(try TestSource.create(allocator, "src_a", &.{"src_a_t"})); + try h.registry.registerSource(try TestSource.create(allocator, "src_b", &.{"src_b_t"})); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -1064,8 +1117,8 @@ test "runStep: distinct sources run on distinct threads in parallel" { var recv = NoopReceiver.make(); try agent.runStep(&conv, &recv); - const view_a = agent.registry.lookup("src_a_t") orelse return error.NotFound; - const view_b = agent.registry.lookup("src_b_t") orelse return error.NotFound; + const view_a = h.registry.lookup("src_a_t") orelse return error.NotFound; + const view_b = h.registry.lookup("src_b_t") orelse return error.NotFound; const sa: *TestSource = @ptrCast(@alignCast(view_a.entry.source.source.ctx)); const sb: *TestSource = @ptrCast(@alignCast(view_b.entry.source.source.ctx)); @@ -1089,10 +1142,12 @@ test "runStep: source whole-batch error aborts the turn" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerToolSource(try FailingSource.create(allocator, "fs", &.{ "fa", "fb" })); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.registerSource(try FailingSource.create(allocator, "fs", &.{ "fa", "fb" })); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -1120,11 +1175,13 @@ test "runStep: mixed single Tools and source-backed tools coexist in one turn" { var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - var agent = Agent.init(allocator, io, stub.provider()); - defer agent.deinit(); - - try agent.registerTool(try EchoTool.create(allocator, "single", "S:")); - try agent.registerToolSource(try TestSource.create(allocator, "src", &.{ "src_t1", "src_t2" })); + var h = TestHarness.init(allocator); + defer h.deinit(); + try h.registry.register(try EchoTool.create(allocator, "single", "S:")); + try h.registry.registerSource(try TestSource.create(allocator, "src", &.{ "src_t1", "src_t2" })); + h.activate(); + var agent = Agent.init(allocator, io, &h.config); + agent.stream_fn = stub.install(); var conv = conversation.Conversation.init(allocator); defer conv.deinit(); @@ -1139,3 +1196,63 @@ test "runStep: mixed single Tools and source-backed tools coexist in one turn" { try testing.expectEqualStrings("src_t1->Y", tr_msg.content.items[1].ToolResult.content.items); try testing.expectEqualStrings("src_t2->Z", tr_msg.content.items[2].ToolResult.content.items); } + +test "setConfig swaps the visible tool set between turns" { + // The core RCU promise: the agent reads `*const Config` fresh each + // turn, so swapping the pointer mid-conversation changes the tool set + // the next turn sees. Config A exposes only `echo`; config B only + // `late`. After `setConfig(&cfg_b)`, a turn that calls `late` resolves + // — proving both the swap and per-turn re-consultation. + const allocator = testing.allocator; + + const scripted = [_]StubProvider.ScriptedTurn{ + .{ .blocks = &.{.{ .ToolUse = .{ .id = "2", .name = "late", .input = "B" } }} }, + .{ .blocks = &.{.{ .Text = "done" }} }, + }; + var stub = StubProvider{ .allocator = allocator, .scripted = &scripted }; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + // Config A: only `echo`. + var reg_a = ToolRegistry.init(allocator); + defer reg_a.deinit(); + try reg_a.register(try EchoTool.create(allocator, "echo", "A:")); + const cfg_a: config_mod.Config = .{ + .provider = .{ .openai_chat = .{ .api_key = "k", .base_url = "u", .model = "m" } }, + .registry = ®_a, + }; + + // Config B: only `late`. + var reg_b = ToolRegistry.init(allocator); + defer reg_b.deinit(); + try reg_b.register(try EchoTool.create(allocator, "late", "B:")); + const cfg_b: config_mod.Config = .{ + .provider = .{ .openai_chat = .{ .api_key = "k", .base_url = "u", .model = "m" } }, + .registry = ®_b, + }; + + var agent = Agent.init(allocator, io, &cfg_a); + agent.stream_fn = stub.install(); + + // Under A: `echo` visible, `late` not. + try testing.expect(agent.config.registry.lookup("echo") != null); + try testing.expect(agent.config.registry.lookup("late") == null); + + // Swap. Under B: the visibility inverts. + agent.setConfig(&cfg_b); + try testing.expect(agent.config.registry.lookup("echo") == null); + try testing.expect(agent.config.registry.lookup("late") != null); + + // A real turn under B resolves `late` (which would have been + // UnknownTool under A), then loops to the final text turn. + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try conv.addUserMessage("go"); + var recv = NoopReceiver.make(); + try agent.runStep(&conv, &recv); + + const tr = conv.messages.items[2].content.items[0].ToolResult; + try testing.expectEqualStrings("2", tr.tool_use_id); + try testing.expectEqualStrings("B:B", tr.content.items); +} diff --git a/libpanto/src/anthropic_messages_json.zig b/libpanto/src/anthropic_messages_json.zig index 46ec17c..5b65c2b 100644 --- a/libpanto/src/anthropic_messages_json.zig +++ b/libpanto/src/anthropic_messages_json.zig @@ -64,10 +64,11 @@ pub fn serializeRequest( if (tools.count() > 0) { try s.objectField("tools"); try s.beginArray(); - var it = tools.iterator(); + var it = tools.toolsForLLM(); while (it.next()) |t| { try s.beginObject(); try s.objectField("name"); + // `t.decl.name` is already wire-encoded by `toolsForLLM`. try s.write(t.decl.name); try s.objectField("description"); try s.write(t.decl.description); @@ -180,7 +181,9 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void { try s.objectField("id"); try s.write(tu.id); try s.objectField("name"); - try s.write(tu.name); + // Replayed assistant tool_use: encode the stored dotted name. + var name_buf: [tool_registry_mod.max_wire_name_len]u8 = undefined; + try s.write(tool_registry_mod.encodeName(&name_buf, tu.name)); try s.objectField("input"); // Anthropic expects `input` as a nested JSON object, not a // string. The block's input bytes were assembled from @@ -801,9 +804,7 @@ test "serializeRequest - emits tools array when registry non-empty" { var tools = tool_registry_mod.ToolRegistry.init(allocator); defer tools.deinit(); - try tools.register(makeStaticTool( - "echo", - "Echo a message back.", + try tools.register(makeStaticTool("echo", "Echo a message back.", \\{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]} )); @@ -824,6 +825,42 @@ test "serializeRequest - emits tools array when registry non-empty" { try testing.expect(schema.get("properties").? == .object); } +test "serializeRequest - dotted names are wire-encoded in tools and history" { + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + // A prior assistant tool_use replayed from history (dotted internally). + const id = try allocator.dupe(u8, "tu_1"); + const name = try allocator.dupe(u8, "std.write"); + var input: conversation.TextualBlock = .empty; + try input.appendSlice(allocator, "{}"); + try conv.addAssistantMessage(&.{ + .{ .ToolUse = .{ .id = id, .name = name, .input = input } }, + }); + + var tools = tool_registry_mod.ToolRegistry.init(allocator); + defer tools.deinit(); + try tools.register(makeStaticTool("std.read", "Read.", "{\"type\":\"object\"}")); + + const cfg = testConfig("claude-x"); + const body = try serializeRequest(allocator, &cfg, &conv, &tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + + // Tool list: `std.read` -> `std__read`. + const tool_name = parsed.value.object.get("tools").?.array.items[0] + .object.get("name").?.string; + try testing.expectEqualStrings("std__read", tool_name); + + // Replayed history tool_use: `std.write` -> `std__write`. + const hist_name = parsed.value.object.get("messages").?.array.items[0] + .object.get("content").?.array.items[0].object.get("name").?.string; + try testing.expectEqualStrings("std__write", hist_name); +} + test "serializeRequest - assistant ToolUse becomes tool_use content block" { const allocator = testing.allocator; diff --git a/libpanto/src/config.zig b/libpanto/src/config.zig index e395230..6744b56 100644 --- a/libpanto/src/config.zig +++ b/libpanto/src/config.zig @@ -1,9 +1,24 @@ -//! Per-provider configuration. +//! Active configuration the agent consults on every turn. //! -//! `Config` is a tagged union keyed by `APIStyle`. Each variant carries the -//! settings specific to one wire dialect. New providers add a new tag and a -//! new payload struct here; nothing else in libpanto needs a central enum -//! refresh. +//! `ProviderConfig` is a tagged union keyed by `APIStyle`; each variant +//! carries the settings specific to one wire dialect. New providers add a +//! new tag and a new payload struct here; nothing else in libpanto needs a +//! central enum refresh. +//! +//! `Config` bundles the active `ProviderConfig` together with the +//! `ToolRegistry` the agent should expose this turn. It is an **immutable +//! snapshot**: the agent holds a `*const Config` and re-reads it at the top +//! of every turn, so swapping that pointer (e.g. from a `panto.configure` +//! hook) changes provider, model, base_url, and/or the visible tool set +//! atomically at the next turn boundary. Because the snapshot is read-only +//! while a turn is in flight, concurrent tool workers reading the old +//! snapshot stay consistent. + +const std = @import("std"); +const Io = std.Io; +const tool_registry_mod = @import("tool_registry.zig"); + +pub const ToolRegistry = tool_registry_mod.ToolRegistry; /// The wire dialect a provider speaks. pub const APIStyle = enum { @@ -43,19 +58,74 @@ pub const AnthropicMessagesConfig = struct { max_tokens: u32 = 4096, }; -pub const Config = union(APIStyle) { +/// Per-provider transport/auth/model configuration. Tagged by `APIStyle`. +pub const ProviderConfig = union(APIStyle) { openai_chat: OpenAIChatConfig, anthropic_messages: AnthropicMessagesConfig, - pub fn style(self: Config) APIStyle { + pub fn style(self: ProviderConfig) APIStyle { return @as(APIStyle, self); } }; -const t = @import("std").testing; +/// An immutable snapshot of everything the agent consults per turn: which +/// provider/model to talk to, and which tools to expose. The agent holds a +/// `*const Config` and re-reads it each turn; replacing the pointer swaps +/// the active configuration wholesale at the next turn boundary. +/// +/// `registry` is borrowed, not owned — its lifetime is managed by whoever +/// built the snapshot (typically the embedder). A `Config` may be copied +/// freely; copies share the same borrowed registry. +pub const Config = struct { + provider: ProviderConfig, + registry: *const ToolRegistry, + + pub fn style(self: Config) APIStyle { + return self.provider.style(); + } +}; + +// =========================================================================== +// Process-global HTTP client +// =========================================================================== +// +// `std.http.Client`'s connection pool is mutex-guarded and keyed by host, +// so a single client safely multiplexes every provider/base_url the agent +// ever switches to, across concurrent turns. We keep exactly one for the +// whole process: switching `base_url` simply leaves the old host's idle +// connections to time out (and reuses them if the user switches back). +// +// Embedders must call `initHttp` once before any turn and `deinitHttp` +// once at shutdown. + +var global_http: ?std.http.Client = null; + +/// Initialize the process-global HTTP client. Call once from the embedder's +/// `main()` before driving any agent turns. Idempotent: a second call with +/// an already-initialized client is a no-op. +pub fn initHttp(allocator: std.mem.Allocator, io: Io) void { + if (global_http != null) return; + global_http = .{ .allocator = allocator, .io = io }; +} + +/// Tear down the process-global HTTP client. Call once at shutdown, after +/// all turns have completed. +pub fn deinitHttp() void { + if (global_http) |*c| { + c.deinit(); + global_http = null; + } +} + +/// Borrow the process-global HTTP client. Asserts `initHttp` has run. +pub fn httpClient() *std.http.Client { + return &(global_http orelse @panic("libpanto: httpClient() called before initHttp()")); +} -test "Config - openai_chat variant" { - const cfg: Config = .{ .openai_chat = .{ +const t = std.testing; + +test "ProviderConfig - openai_chat variant" { + const cfg: ProviderConfig = .{ .openai_chat = .{ .api_key = "sk-test", .base_url = "https://api.openai.com/v1", .model = "gpt-4o", @@ -66,8 +136,8 @@ test "Config - openai_chat variant" { try t.expectEqual(ReasoningEffort.high, cfg.openai_chat.reasoning); } -test "Config - anthropic_messages variant" { - const cfg: Config = .{ .anthropic_messages = .{ +test "ProviderConfig - anthropic_messages variant" { + const cfg: ProviderConfig = .{ .anthropic_messages = .{ .api_key = "sk-ant-test", .base_url = "https://api.anthropic.com", .model = "claude-sonnet-4-20250514", @@ -77,11 +147,31 @@ test "Config - anthropic_messages variant" { try t.expectEqual(@as(u32, 4096), cfg.anthropic_messages.max_tokens); } -test "Config - openai_chat reasoning defaults to .default" { - const cfg: Config = .{ .openai_chat = .{ +test "ProviderConfig - openai_chat reasoning defaults to .default" { + const cfg: ProviderConfig = .{ .openai_chat = .{ .api_key = "k", .base_url = "u", .model = "m", } }; try t.expectEqual(ReasoningEffort.default, cfg.openai_chat.reasoning); } + +test "Config bundles provider + registry and forwards style" { + var reg = ToolRegistry.init(t.allocator); + defer reg.deinit(); + const cfg: Config = .{ + .provider = .{ .openai_chat = .{ .api_key = "k", .base_url = "u", .model = "m" } }, + .registry = ®, + }; + try t.expectEqual(APIStyle.openai_chat, cfg.style()); +} + +test "global http client: init/borrow/deinit" { + var threaded: std.Io.Threaded = .init(t.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + initHttp(t.allocator, io); + defer deinitHttp(); + initHttp(t.allocator, io); // idempotent + _ = httpClient(); +} diff --git a/libpanto/src/openai_chat_json.zig b/libpanto/src/openai_chat_json.zig index 37d347b..6991559 100644 --- a/libpanto/src/openai_chat_json.zig +++ b/libpanto/src/openai_chat_json.zig @@ -117,7 +117,7 @@ pub fn serializeRequest( if (tools.count() > 0) { try s.objectField("tools"); try s.beginArray(); - var it = tools.iterator(); + var it = tools.toolsForLLM(); while (it.next()) |t| { try s.beginObject(); try s.objectField("type"); @@ -125,6 +125,7 @@ pub fn serializeRequest( try s.objectField("function"); try s.beginObject(); try s.objectField("name"); + // `t.decl.name` is already wire-encoded by `toolsForLLM`. try s.write(t.decl.name); try s.objectField("description"); try s.write(t.decl.description); @@ -261,7 +262,11 @@ fn writeMessage(s: *std.json.Stringify, msg: conversation.Message, allocator: Al try s.objectField("function"); try s.beginObject(); try s.objectField("name"); - try s.write(tu.name); + // Replayed assistant tool_use. The conversation stores the + // internal (dotted) name; encode `.` -> `__` so it matches + // what we advertise in `tools`. + var name_buf: [tool_registry_mod.max_wire_name_len]u8 = undefined; + try s.write(tool_registry_mod.encodeName(&name_buf, tu.name)); try s.objectField("arguments"); // `arguments` is a string carrying JSON, per the OpenAI // wire format — not a nested object. @@ -664,9 +669,7 @@ test "serializeRequest - emits tools array when registry non-empty" { var tools = emptyTools(); defer tools.deinit(); - try tools.register(makeStaticTool( - "echo", - "Echo a message back.", + try tools.register(makeStaticTool("echo", "Echo a message back.", \\{"type":"object","properties":{"message":{"type":"string"}},"required":["message"]} )); @@ -691,6 +694,29 @@ test "serializeRequest - emits tools array when registry non-empty" { try testing.expect(params.get("required").? == .array); } +test "serializeRequest - dotted tool name is wire-encoded with __" { + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try conv.addUserMessage("go"); + + var tools = emptyTools(); + defer tools.deinit(); + try tools.register(makeStaticTool("std.read", "Read a file.", "{\"type\":\"object\"}")); + + const cfg = testConfig("gpt-4o"); + const body = try serializeRequest(allocator, &cfg, &conv, &tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + + const f = parsed.value.object.get("tools").?.array.items[0].object.get("function").?.object; + // Internal `std.read` crosses the wire as `std__read` (OpenAI forbids dots). + try testing.expectEqualStrings("std__read", f.get("name").?.string); +} + test "serializeRequest - assistant ToolUse becomes tool_calls" { const allocator = testing.allocator; diff --git a/libpanto/src/provider.zig b/libpanto/src/provider.zig index ab60678..1a8ee4e 100644 --- a/libpanto/src/provider.zig +++ b/libpanto/src/provider.zig @@ -88,59 +88,58 @@ pub const Receiver = struct { } }; -pub const ProviderVTable = struct { - streamStep: *const fn ( - *anyopaque, - *conversation.Conversation, - *const ToolRegistry, - *Receiver, - ) anyerror!void, - deinit: *const fn (*anyopaque) void, -}; - -pub const Provider = struct { - ptr: *anyopaque, - vtable: *const ProviderVTable, - - /// Construct the concrete provider implementation that matches `cfg`'s - /// API style, heap-allocate it, and return a `Provider` interface bound - /// to it. `deinit` tears down the impl and frees the allocation. - /// - /// The concrete provider types (`OpenAIChatProvider`, etc.) are - /// implementation details; callers interact only with this interface. - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - cfg: config_mod.Config, - ) !Provider { - // Imported lazily to break the circular module graph: - // provider.zig <- provider_openai_chat.zig <- provider.zig. - const provider_openai_chat = @import("provider_openai_chat.zig"); - const provider_anthropic_messages = @import("provider_anthropic_messages.zig"); - switch (cfg) { - .openai_chat => |c| { - const impl = try allocator.create(provider_openai_chat.OpenAIChatProvider); - impl.* = provider_openai_chat.OpenAIChatProvider.init(allocator, io, c); - return impl.provider(); - }, - .anthropic_messages => |c| { - const impl = try allocator.create(provider_anthropic_messages.AnthropicMessagesProvider); - impl.* = provider_anthropic_messages.AnthropicMessagesProvider.init(allocator, io, c); - return impl.provider(); - }, - } - } - - pub fn streamStep( - self: Provider, - conv: *conversation.Conversation, - tools: *const ToolRegistry, - receiver: *Receiver, - ) anyerror!void { - return self.vtable.streamStep(self.ptr, conv, tools, receiver); +/// Drive one streaming provider turn against the active config snapshot. +/// +/// This is the single dispatch point: it switches on `cfg.provider`'s +/// `APIStyle` tag, builds a transient per-request object bound to the +/// process-global HTTP client, and runs it. There is no persistent +/// provider object — every turn re-reads `cfg`, so swapping the agent's +/// `*const Config` between turns changes provider, model, base_url, and +/// the visible tool set with no transport teardown. +/// +/// The tool registry is taken from `cfg.registry`; the serializers receive +/// it directly. +pub fn streamStep( + allocator: std.mem.Allocator, + io: std.Io, + cfg: *const config_mod.Config, + conv: *conversation.Conversation, + receiver: *Receiver, +) anyerror!void { + // Imported lazily to break the circular module graph: + // provider.zig <- provider_openai_chat.zig <- provider.zig. + const provider_openai_chat = @import("provider_openai_chat.zig"); + const provider_anthropic_messages = @import("provider_anthropic_messages.zig"); + const client = config_mod.httpClient(); + switch (cfg.provider) { + .openai_chat => |*c| { + var req: provider_openai_chat.OpenAIChatRequest = .{ + .allocator = allocator, + .io = io, + .config = c, + .http_client = client, + }; + return req.streamStep(conv, cfg.registry, receiver); + }, + .anthropic_messages => |*c| { + var req: provider_anthropic_messages.AnthropicMessagesRequest = .{ + .allocator = allocator, + .io = io, + .config = c, + .http_client = client, + }; + return req.streamStep(conv, cfg.registry, receiver); + }, } +} - pub fn deinit(self: Provider) void { - self.vtable.deinit(self.ptr); - } -}; +/// The shape of `streamStep`, exposed as a function-pointer type so the +/// agent can carry an injectable seam (real dispatch in production, a stub +/// in tests) without resurrecting a per-provider vtable. +pub const StreamFn = *const fn ( + allocator: std.mem.Allocator, + io: std.Io, + cfg: *const config_mod.Config, + conv: *conversation.Conversation, + receiver: *Receiver, +) anyerror!void; diff --git a/libpanto/src/provider_anthropic_messages.zig b/libpanto/src/provider_anthropic_messages.zig index 5d5520a..02277be 100644 --- a/libpanto/src/provider_anthropic_messages.zig +++ b/libpanto/src/provider_anthropic_messages.zig @@ -23,62 +23,19 @@ const provider_mod = @import("provider.zig"); const sse_mod = @import("sse.zig"); const json_mod = @import("anthropic_messages_json.zig"); const config_mod = @import("config.zig"); +const tool_registry_mod = @import("tool_registry.zig"); -pub const AnthropicMessagesProvider = struct { +/// A single Anthropic Messages streaming request. Transient: constructed +/// per `streamStep`, holds only borrowed state (allocator, io, the global +/// HTTP client, and the active config). Carries nothing across requests. +pub const AnthropicMessagesRequest = struct { allocator: Allocator, io: Io, - config: config_mod.AnthropicMessagesConfig, - http_client: http.Client, - - pub fn init( - allocator: Allocator, - io: Io, - cfg: config_mod.AnthropicMessagesConfig, - ) AnthropicMessagesProvider { - return .{ - .allocator = allocator, - .io = io, - .config = cfg, - .http_client = .{ .allocator = allocator, .io = io }, - }; - } - - pub fn deinit(self: *AnthropicMessagesProvider) void { - self.http_client.deinit(); - } - - pub fn provider(self: *AnthropicMessagesProvider) provider_mod.Provider { - return .{ .ptr = self, .vtable = &vtable }; - } - - const vtable: provider_mod.ProviderVTable = .{ - .streamStep = vtableStreamStep, - .deinit = vtableDeinit, - }; - - fn vtableStreamStep( - ptr: *anyopaque, - conv: *conversation.Conversation, - tools: *const provider_mod.ToolRegistry, - receiver: *provider_mod.Receiver, - ) anyerror!void { - const self: *AnthropicMessagesProvider = @ptrCast(@alignCast(ptr)); - return self.streamStep(conv, tools, receiver); - } - - /// Called via the `Provider` interface. Tears down the impl AND frees - /// its heap allocation, since `Provider.init` is the one that allocated - /// it. Direct stack-allocated users (tests, embedders) call `deinit` - /// themselves and never hit this path. - fn vtableDeinit(ptr: *anyopaque) void { - const self: *AnthropicMessagesProvider = @ptrCast(@alignCast(ptr)); - const allocator = self.allocator; - self.deinit(); - allocator.destroy(self); - } + config: *const config_mod.AnthropicMessagesConfig, + http_client: *http.Client, pub fn streamStep( - self: *AnthropicMessagesProvider, + self: *AnthropicMessagesRequest, conv: *conversation.Conversation, tools: *const provider_mod.ToolRegistry, receiver: *provider_mod.Receiver, @@ -92,7 +49,7 @@ pub const AnthropicMessagesProvider = struct { } fn streamStepInner( - self: *AnthropicMessagesProvider, + self: *AnthropicMessagesRequest, conv: *conversation.Conversation, tools: *const provider_mod.ToolRegistry, receiver: *provider_mod.Receiver, @@ -106,7 +63,7 @@ pub const AnthropicMessagesProvider = struct { const uri = try Uri.parse(url); - const body = try json_mod.serializeRequest(self.allocator, &self.config, conv, tools); + const body = try json_mod.serializeRequest(self.allocator, self.config, conv, tools); defer self.allocator.free(body); const extra_headers = [_]http.Header{ @@ -294,10 +251,25 @@ const StreamState = struct { .kind = kind, }; // For tool_use blocks, capture the identity fields. Anthropic - // delivers both whole on content_block_start. + // delivers both whole on content_block_start. The wire name is + // encoded (`__` for `.`); decode it here so everything downstream + // — onToolDetails, the stored ContentBlock, session logs, and + // dispatch — sees the internal (dotted) name. The decoded form is + // never longer than the wire form. if (kind == .tool_use) { if (tool_id) |id| ab.tool_id = try self.allocator.dupe(u8, id); - if (tool_name) |n| ab.tool_name = try self.allocator.dupe(u8, n); + if (tool_name) |n| { + // Decode `__` -> `.` into an exact-size owned buffer so the + // stored slice is freeable as a whole allocation. + const owned = try self.allocator.alloc(u8, n.len); + errdefer self.allocator.free(owned); + const decoded = tool_registry_mod.decodeName(owned, n); + if (decoded.len == n.len) { + ab.tool_name = owned; + } else { + ab.tool_name = try self.allocator.realloc(owned, decoded.len); + } + } } self.active = ab; const block_type: ?provider_mod.ContentBlockType = switch (kind) { @@ -933,6 +905,37 @@ test "tool_use blocks are captured with id, name, and assembled input" { try testing.expectEqualStrings("done", asst.content.items[1].Text.items); } +test "inbound wire tool name is decoded to dotted form" { + // Anthropic delivers the (wire-encoded) name whole at + // content_block_start; it is decoded to the internal dotted form for + // the conversation, session logs, and dispatch. + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try conv.addUserMessage("use a tool"); + + var rec = RecordingReceiver.init(allocator); + defer rec.deinit(); + var recv = rec.receiver(); + + const events = [_][]const u8{ + \\{"type":"message_start","message":{"role":"assistant"}} + , + \\{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"t1","name":"calc__sum","input":{}}} + , + \\{"type":"content_block_stop","index":0} + , + \\{"type":"message_stop"} + , + }; + + try runStreamedTurn(allocator, &conv, &recv, &events); + + const tu = conv.messages.items[1].content.items[0].ToolUse; + try testing.expectEqualStrings("calc.sum", tu.name); +} + test "error event propagates as Zig error" { const allocator = testing.allocator; diff --git a/libpanto/src/provider_openai_chat.zig b/libpanto/src/provider_openai_chat.zig index fcbc0bb..d70eb30 100644 --- a/libpanto/src/provider_openai_chat.zig +++ b/libpanto/src/provider_openai_chat.zig @@ -21,63 +21,34 @@ const provider_mod = @import("provider.zig"); const sse_mod = @import("sse.zig"); const json_mod = @import("openai_chat_json.zig"); const config_mod = @import("config.zig"); +const tool_registry_mod = @import("tool_registry.zig"); + +/// Decode a wire tool name (`__` -> `.`) in place within an assembled +/// name buffer. Decoding only ever shrinks the buffer (reads stay ahead +/// of writes), so aliasing src/dst is safe; we then truncate to the +/// decoded length. Unambiguous because internal names never contain a +/// literal `__` (enforced at registration). +fn decodeNameInPlace(name_buf: *conversation.TextualBlock) void { + const decoded = tool_registry_mod.decodeName(name_buf.items, name_buf.items); + name_buf.items.len = decoded.len; +} /// Active streaming block type tracked by the state machine. Mirrors the /// `ContentBlock` union variants but adds `.none` for "no block open yet". const ActiveBlock = enum { none, text, thinking, tool_use }; -pub const OpenAIChatProvider = struct { +/// A single OpenAI Chat streaming request. Transient: constructed per +/// `streamStep`, holds only borrowed state (allocator, io, the global HTTP +/// client, and the active config). Carries nothing across requests, so it +/// is created inline by the free `streamStep` entry point below. +pub const OpenAIChatRequest = struct { allocator: Allocator, io: Io, - config: config_mod.OpenAIChatConfig, - http_client: http.Client, - - pub fn init(allocator: Allocator, io: Io, cfg: config_mod.OpenAIChatConfig) OpenAIChatProvider { - return .{ - .allocator = allocator, - .io = io, - .config = cfg, - .http_client = .{ .allocator = allocator, .io = io }, - }; - } - - pub fn deinit(self: *OpenAIChatProvider) void { - self.http_client.deinit(); - } - - /// Return a `Provider` interface bound to this concrete provider. - pub fn provider(self: *OpenAIChatProvider) provider_mod.Provider { - return .{ .ptr = self, .vtable = &vtable }; - } - - const vtable: provider_mod.ProviderVTable = .{ - .streamStep = vtableStreamStep, - .deinit = vtableDeinit, - }; - - fn vtableStreamStep( - ptr: *anyopaque, - conv: *conversation.Conversation, - tools: *const provider_mod.ToolRegistry, - receiver: *provider_mod.Receiver, - ) anyerror!void { - const self: *OpenAIChatProvider = @ptrCast(@alignCast(ptr)); - return self.streamStep(conv, tools, receiver); - } - - /// Called via the `Provider` interface. Tears down the impl AND frees - /// its heap allocation, since `Provider.init` is the one that allocated - /// it. Direct stack-allocated users (tests, embedders) call `deinit` - /// themselves and never hit this path. - fn vtableDeinit(ptr: *anyopaque) void { - const self: *OpenAIChatProvider = @ptrCast(@alignCast(ptr)); - const allocator = self.allocator; - self.deinit(); - allocator.destroy(self); - } + config: *const config_mod.OpenAIChatConfig, + http_client: *http.Client, pub fn streamStep( - self: *OpenAIChatProvider, + self: *OpenAIChatRequest, conv: *conversation.Conversation, tools: *const provider_mod.ToolRegistry, receiver: *provider_mod.Receiver, @@ -92,7 +63,7 @@ pub const OpenAIChatProvider = struct { } fn streamStepInner( - self: *OpenAIChatProvider, + self: *OpenAIChatRequest, conv: *conversation.Conversation, tools: *const provider_mod.ToolRegistry, receiver: *provider_mod.Receiver, @@ -108,7 +79,7 @@ pub const OpenAIChatProvider = struct { const uri = try Uri.parse(url); // Build the request body. - const body = try json_mod.serializeRequest(self.allocator, &self.config, conv, tools); + const body = try json_mod.serializeRequest(self.allocator, self.config, conv, tools); defer self.allocator.free(body); // Auth header @@ -490,6 +461,12 @@ const StreamState = struct { return; } + // The model echoes the wire-encoded tool name (`__` for `.`). + // Decode in place now that the full name is assembled, so the + // conversation, receiver callbacks, and dispatch all see the + // internal (dotted) name. Decoding never grows the buffer. + decodeNameInPlace(&tu.name_buf); + // If no arguments ever arrived, we haven't emitted onBlockStart // yet — do it now so the receiver sees a balanced start/complete. try self.emitStartIfNeeded(receiver, &tu); @@ -640,7 +617,7 @@ const testing = std.testing; /// about post-stream conversation state rather than callback observability. const NoopReceiver = struct { fn make() provider_mod.Receiver { - return .{ .ptr = @constCast(@ptrCast(&dummy)), .vtable = &vt }; + return .{ .ptr = @ptrCast(@constCast(&dummy)), .vtable = &vt }; } var dummy: u8 = 0; const vt: provider_mod.ReceiverVTable = .{ @@ -676,7 +653,7 @@ fn runStreamedTurn( if (std.mem.eql(u8, payload, "[DONE]")) break; // Process every chunk through to [DONE], including the // post-finish_reason usage chunk. Mirrors the production loop - // in OpenAIChatProvider.streamStep. + // in OpenAIChatRequest.streamStep. try handleEvent(allocator, payload, &state, receiver); } try state.finalize(receiver, conv); @@ -847,6 +824,39 @@ test "fragmented tool_call id and name are reassembled" { try testing.expectEqualStrings("{\"host\":\"a.com\"}", tu.input.items); } +test "inbound wire tool name is decoded to dotted form (even split across __)" { + // The model echoes the wire name it was given (`std__read`). It is + // decoded to the internal `std.read` for the conversation/session/ + // dispatch. The decode happens after full assembly, so a `__` split + // across two deltas (`std_` + `_read`) decodes correctly. + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try conv.addUserMessage("read a file"); + + var recv = NoopReceiver.make(); + + const events = [_][]const u8{ + \\{"choices":[{"delta":{"role":"assistant"}}]} + , + \\{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","type":"function","function":{"name":"std_"}}]}}]} + , + \\{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"name":"_read"}}]}}]} + , + \\{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{}"}}]}}]} + , + \\{"choices":[{"delta":{},"finish_reason":"tool_calls"}]} + , + "[DONE]", + }; + + try runStreamedTurn(allocator, &conv, &recv, &events); + + const tu = conv.messages.items[1].content.items[0].ToolUse; + try testing.expectEqualStrings("std.read", tu.name); +} + /// A Receiver that records the sequence of callback events as compact /// strings. Useful for asserting per-block start/complete ordering. const RecordingReceiver = struct { diff --git a/libpanto/src/root.zig b/libpanto/src/root.zig index de62cfa..68f8e7d 100644 --- a/libpanto/src/root.zig +++ b/libpanto/src/root.zig @@ -19,11 +19,15 @@ pub const ToolDecl = tool_source.ToolDecl; pub const ToolCall = tool_source.Call; pub const ToolCallResult = tool_source.CallResult; pub const ToolRegistry = tool_registry.ToolRegistry; +pub const Config = config.Config; +pub const ProviderConfig = config.ProviderConfig; -// Internal modules. Not part of the public API — callers construct providers -// via `provider.Provider.init(allocator, io, config)`, which picks the right -// concrete implementation based on the config's API style. These are exposed -// here only so `refAllDecls` picks up their tests. +// Internal modules. Not part of the public API — callers drive turns via +// `provider.streamStep(allocator, io, &config, conv, receiver)` (or via the +// `Agent`, which holds a swappable `*const Config`). The process-global HTTP +// client is initialized with `config.initHttp` / torn down with +// `config.deinitHttp`. These impls are exposed here only so `refAllDecls` +// picks up their tests. const openai_chat_json = @import("openai_chat_json.zig"); const provider_openai_chat = @import("provider_openai_chat.zig"); const anthropic_messages_json = @import("anthropic_messages_json.zig"); diff --git a/libpanto/src/tool_registry.zig b/libpanto/src/tool_registry.zig index e8cb4d1..35aff02 100644 --- a/libpanto/src/tool_registry.zig +++ b/libpanto/src/tool_registry.zig @@ -26,6 +26,89 @@ const Tool = tool_mod.Tool; const ToolSource = tool_source_mod.ToolSource; const ToolDecl = tool_source_mod.ToolDecl; +// =========================================================================== +// Wire-name encoding +// =========================================================================== +// +// Internally, tool names use dots for namespacing (`std.read`), which our +// glob-based allow/deny policies rely on. But the OpenAI and Anthropic +// tool-name grammars forbid dots: both require `^[a-zA-Z0-9_-]{1,128}$`. +// +// So names are translated at the wire boundary only: `.` <-> `__`. The +// mapping is a clean bijection because a literal `__` is forbidden in an +// internal name (enforced by `validateName` at registration). Everything +// inside libpanto keeps speaking dots; only the serializers (via +// `toolsForLLM`) and inbound dispatch (via `lookupLLM`) cross the boundary. + +/// The largest a wire (LLM-facing) tool name may be, per the provider +/// grammars. We validate the *encoded* length against this so an encoded +/// name is always acceptable to both providers. +pub const max_wire_name_len = 128; + +pub const NameError = error{ + /// Name is empty or its encoded form exceeds `max_wire_name_len`. + NameTooLong, + /// Name contains a literal `__` (reserved as the encoded form of `.`) + /// or a character outside `[a-zA-Z0-9_.-]`. + InvalidNameChar, +}; + +/// Validate an internal tool name. Permits `[a-zA-Z0-9_.-]` but forbids a +/// literal `__` (which would collide with an encoded `.`), and requires +/// the encoded form to be 1..=`max_wire_name_len` bytes. Each `.` expands +/// to two bytes when encoded, so the cap is checked against that. +pub fn validateName(name: []const u8) NameError!void { + if (name.len == 0) return error.NameTooLong; + var encoded_len: usize = 0; + for (name, 0..) |ch, i| { + const ok = (ch >= 'a' and ch <= 'z') or (ch >= 'A' and ch <= 'Z') or + (ch >= '0' and ch <= '9') or ch == '_' or ch == '-' or ch == '.'; + if (!ok) return error.InvalidNameChar; + // Reject a literal double underscore: it is reserved for `.`. + if (ch == '_' and i + 1 < name.len and name[i + 1] == '_') return error.InvalidNameChar; + encoded_len += if (ch == '.') 2 else 1; + } + if (encoded_len > max_wire_name_len) return error.NameTooLong; +} + +/// Encode an internal name for the wire: `.` -> `__`. Writes into `buf` +/// (which must be at least `max_wire_name_len` bytes) and returns the +/// written slice. Names that passed `validateName` always fit. +pub fn encodeName(buf: []u8, name: []const u8) []const u8 { + var w: usize = 0; + for (name) |ch| { + if (ch == '.') { + buf[w] = '_'; + buf[w + 1] = '_'; + w += 2; + } else { + buf[w] = ch; + w += 1; + } + } + return buf[0..w]; +} + +/// Decode a wire name back to internal form: `__` -> `.`. Writes into +/// `buf` (at least `wire.len` bytes) and returns the written slice. The +/// decode is unambiguous because internal names never contain `__`. +pub fn decodeName(buf: []u8, wire: []const u8) []u8 { + var r: usize = 0; + var w: usize = 0; + while (r < wire.len) { + if (wire[r] == '_' and r + 1 < wire.len and wire[r + 1] == '_') { + buf[w] = '.'; + w += 1; + r += 2; + } else { + buf[w] = wire[r]; + w += 1; + r += 1; + } + } + return buf[0..w]; +} + /// Tagged registry value. The registry stores one of these per *tool /// name*. ToolSources expand to one entry per declared tool, each with a /// distinct `tool_index`. @@ -98,6 +181,7 @@ pub const ToolRegistry = struct { /// In the duplicate case the caller's tool is NOT taken over; the /// caller is responsible for tearing it down. pub fn register(self: *ToolRegistry, tool: Tool) !void { + try validateName(tool.decl.name); const gop = try self.entries.getOrPut(tool.decl.name); if (gop.found_existing) return error.DuplicateTool; gop.value_ptr.* = .{ .single = tool }; @@ -113,8 +197,10 @@ pub const ToolRegistry = struct { /// down) and any tools that *had* been inserted before the collision /// are rolled back. pub fn registerSource(self: *ToolRegistry, src: ToolSource) !void { - // First pass: check for any collision before committing. + // First pass: validate names and check for any collision before + // committing anything. for (src.tools) |decl| { + try validateName(decl.name); if (self.entries.contains(decl.name)) return error.DuplicateTool; } @@ -183,6 +269,27 @@ pub const ToolRegistry = struct { } }; + /// Iterate tools with their names **wire-encoded** (`.` -> `__`) for + /// the LLM. The yielded `ToolView.decl.name` borrows the iterator's + /// internal buffer and is only valid until the next `next()` call; + /// serializers consume it immediately, so this is safe. Description + /// and schema are unchanged. + pub fn toolsForLLM(self: *const ToolRegistry) LLMIterator { + return .{ .inner = self.entries.iterator() }; + } + + pub const LLMIterator = struct { + inner: std.StringHashMap(Entry).Iterator, + name_buf: [max_wire_name_len]u8 = undefined, + + pub fn next(self: *LLMIterator) ?ToolView { + const entry = self.inner.next() orelse return null; + var view = makeView(entry.value_ptr.*); + view.decl.name = encodeName(&self.name_buf, view.decl.name); + return view; + } + }; + fn makeView(entry: Entry) ToolView { return switch (entry) { .single => |t| .{ .decl = t.decl, .entry = entry }, @@ -489,3 +596,63 @@ test "source view exposes per-tool metadata uniformly" { } try testing.expectEqual(@as(usize, 3), n); } + +// --- wire-name encoding --- + +test "validateName: accepts dotted names, rejects literal __ and bad chars" { + try validateName("std.read"); + try validateName("pkg.read_file"); + try validateName("a-b_c.d"); + try testing.expectError(error.InvalidNameChar, validateName("std__read")); + try testing.expectError(error.InvalidNameChar, validateName("has space")); + try testing.expectError(error.InvalidNameChar, validateName("slash/name")); + try testing.expectError(error.NameTooLong, validateName("")); + // 64 dots -> 128 encoded bytes: OK; 65 -> 130: too long. + try validateName("." ** 64); + try testing.expectError(error.NameTooLong, validateName("." ** 65)); +} + +test "encode/decode: dots <-> double underscores, bijective" { + var buf: [max_wire_name_len]u8 = undefined; + var buf2: [max_wire_name_len]u8 = undefined; + + const cases = [_][]const u8{ "std.read", "pkg.read_file", "a.b.c", "plain", "a-b" }; + inline for (cases) |internal| { + const wire = encodeName(&buf, internal); + try testing.expect(std.mem.indexOf(u8, wire, ".") == null); + const back = decodeName(&buf2, wire); + try testing.expectEqualStrings(internal, back); + } + + // Spot-check the exact wire form and the read_file distinction. + try testing.expectEqualStrings("std__read", encodeName(&buf, "std.read")); + try testing.expectEqualStrings("pkg__read_file", encodeName(&buf, "pkg.read_file")); + try testing.expectEqualStrings("pkg__read__file", encodeName(&buf, "pkg.read.file")); +} + +test "register rejects names with literal double underscore" { + const allocator = testing.allocator; + var reg = ToolRegistry.init(allocator); + defer reg.deinit(); + + var bad = try TestTool.create(allocator, "std__read"); + try testing.expectError(error.InvalidNameChar, reg.register(bad)); + // Registration refused ownership; tear the tool down ourselves. + bad.vtable.deinit(bad.ctx, allocator); +} + +test "toolsForLLM yields wire-encoded names; iterator keeps dotted names" { + const allocator = testing.allocator; + var reg = ToolRegistry.init(allocator); + defer reg.deinit(); + try reg.register(try TestTool.create(allocator, "std.read")); + + var llm = reg.toolsForLLM(); + const v = llm.next().?; + try testing.expectEqualStrings("std__read", v.decl.name); + try testing.expect(llm.next() == null); + + // The internal iterator is unchanged. + var it = reg.iterator(); + try testing.expectEqualStrings("std.read", it.next().?.decl.name); +} diff --git a/src/config_file.zig b/src/config_file.zig new file mode 100644 index 0000000..533eb48 --- /dev/null +++ b/src/config_file.zig @@ -0,0 +1,977 @@ +//! Layered `config.toml` loader for the panto CLI. +//! +//! Four files are read and merged, lowest precedence first: +//! +//! 1. base — `${XDG_DATA_HOME:-$HOME/.local/share}/panto/config.toml` +//! 2. user — `${XDG_CONFIG_HOME:-$HOME/.config}/panto/config.toml` +//! 3. project — `./.panto/config.toml` +//! 4. local — `./.panto/config.local.toml` +//! +//! The `local` layer is intended to be git-ignored: it lets an individual +//! developer apply more-opinionated overrides on top of the team-shared +//! `project` layer without committing them. +//! +//! Merge semantics: **tables merge recursively; scalars and arrays from a +//! higher-precedence layer overwrite wholesale.** A project file that sets +//! `tools.deny = [...]` replaces the array entirely — it does not append to +//! a user-level `tools.deny`. Tables (notably `providers`) accumulate: a +//! provider defined only at the base layer survives even when the project +//! layer adds a different provider. +//! +//! After merge, the document is resolved into a `Config`: +//! +//! - Every `providers.<name>` becomes a `Provider`. Its API key is taken +//! from `api_key` if present, else from the environment variable named +//! by `api_key_env_var`. A provider whose only key source is an absent +//! env var is **silently dropped** — this is what lets the base layer +//! ship default OpenAI/Anthropic providers that simply don't appear when +//! the user hasn't exported the corresponding key. +//! - `defaults.model` is parsed as `<provider_name>:<model_alias>`. +//! - `tools` / `extensions` allow- and deny-lists are captured verbatim +//! (as glob pattern lists). A pattern appearing in both allow and deny +//! for the same kind is a hard error. +//! +//! Model *aliases* (the `<model_alias>` half of a model reference) are +//! resolved separately against `models.toml` — this module only validates +//! that the reference names a known provider. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; + +const toml = @import("toml"); +const panto = @import("panto"); + +const glob = @import("glob.zig"); +const models_toml = @import("models_toml.zig"); + +// Document-building helpers live in the toml library's `value` module and +// are not re-exported at its root. `tableIterator` *is* re-exported, so we +// use `toml.tableIterator` directly but reach through `value_mod` for the +// constructors/mutators. +const tvalue = toml.value_mod; + +pub const APIStyle = panto.config.APIStyle; + +// =========================================================================== +// Resolved config model +// =========================================================================== + +/// One fully-resolved provider. The API key has already been read (from the +/// inline value or the named env var); a provider that never resolved a key +/// is absent from `Config.providers` entirely. +pub const Provider = struct { + /// The user-chosen provider name (the `providers.<name>` key). This is + /// the name used on the left of a `<provider>:<model>` reference. + name: []const u8, + style: APIStyle, + base_url: []const u8, + api_key: []const u8, + + pub fn deinit(self: Provider, alloc: Allocator) void { + alloc.free(self.name); + alloc.free(self.base_url); + alloc.free(self.api_key); + } +}; + +/// A parsed `<provider>:<model>` reference. Both halves borrow from the +/// owning `Config` (the `default_model` storage); do not free separately. +pub const ModelRef = struct { + provider: []const u8, + model: []const u8, +}; + +/// Tool/extension availability policy. Empty `allow` means "allow all" +/// (subject to `deny`). A name is permitted when it matches at least one +/// allow pattern (or allow is empty) AND matches no deny pattern. +pub const Policy = struct { + allow: [][]const u8, + deny: [][]const u8, + + pub fn deinit(self: Policy, alloc: Allocator) void { + for (self.allow) |p| alloc.free(p); + alloc.free(self.allow); + for (self.deny) |p| alloc.free(p); + alloc.free(self.deny); + } + + /// True if `name` is permitted under this policy. + pub fn permits(self: Policy, name: []const u8) bool { + for (self.deny) |pat| { + if (glob.match(pat, name)) return false; + } + if (self.allow.len == 0) return true; + for (self.allow) |pat| { + if (glob.match(pat, name)) return true; + } + return false; + } +}; + +/// Build a `panto.config.ProviderConfig` for a chosen `<provider>:<alias>` model +/// reference, combining the resolved provider (transport/auth) with the +/// model definition from `models.toml` (wire name + knobs). +/// +/// `ref` selects the provider and alias. `defs` supplies per-model knobs; +/// a missing alias falls back to using the alias verbatim as the wire +/// model name with default knobs. Returns the assembled `Config` plus the +/// wire model id (borrowed from `defs`/`ref` — valid as long as both +/// outlive the returned config's use). The `panto.config.ProviderConfig` itself +/// borrows the provider/model strings; the caller must keep `cfg` (the +/// `Config`) and `defs` alive for its lifetime. +pub fn buildProviderConfig( + cfg: *const Config, + defs: *const models_toml.ModelRegistry, + ref: ModelRef, +) ResolveError!panto.config.ProviderConfig { + const prov = cfg.provider(ref.provider) orelse return error.UnknownProvider; + + const def_opt = defs.get(ref.provider, ref.model); + const wire_model: []const u8 = if (def_opt) |d| d.model else ref.model; + const reasoning: panto.config.ReasoningEffort = if (def_opt) |d| d.reasoning else .default; + + switch (prov.style) { + .openai_chat => return .{ .openai_chat = .{ + .api_key = prov.api_key, + .base_url = prov.base_url, + .model = wire_model, + .reasoning = reasoning, + } }, + .anthropic_messages => return .{ .anthropic_messages = .{ + .api_key = prov.api_key, + .base_url = prov.base_url, + .model = wire_model, + .api_version = if (def_opt) |d| (d.api_version orelse "2023-06-01") else "2023-06-01", + .max_tokens = if (def_opt) |d| (d.max_tokens orelse 4096) else 4096, + } }, + } +} + +pub const Config = struct { + allocator: Allocator, + providers: []Provider, + /// `defaults.model` storage, owned. `default_model_ref` slices into it. + default_model: ?[]const u8, + default_model_ref: ?ModelRef, + tools: Policy, + extensions: Policy, + + pub fn deinit(self: *Config) void { + for (self.providers) |p| p.deinit(self.allocator); + self.allocator.free(self.providers); + if (self.default_model) |m| self.allocator.free(m); + self.tools.deinit(self.allocator); + self.extensions.deinit(self.allocator); + } + + /// Look up a resolved provider by name. Returns null if absent (e.g. + /// dropped because its env var wasn't set). + pub fn provider(self: *const Config, name: []const u8) ?*const Provider { + for (self.providers) |*p| { + if (std.mem.eql(u8, p.name, name)) return p; + } + return null; + } + + /// Choose the model reference to start with. Precedence: + /// 1. an explicit `model_override` (e.g. a future `--model` flag), + /// 2. `defaults.model` from config, + /// 3. if exactly one provider resolved AND it has exactly one model + /// alias in `defs`, use that, + /// 4. otherwise error (`NoModelSelected`). + /// + /// The returned ref borrows from `self`/`defs`/`model_override`. + pub fn selectModel( + self: *const Config, + defs: *const models_toml.ModelRegistry, + model_override: ?[]const u8, + ) ResolveError!ModelRef { + if (model_override) |m| { + return parseModelRef(m) catch return error.NoModelSelected; + } + if (self.default_model_ref) |ref| return ref; + + // Single-provider, single-alias convenience. + if (self.providers.len == 1) { + const only = self.providers[0].name; + var found: ?ModelRef = null; + for (defs.entries.items) |d| { + if (std.mem.eql(u8, d.provider, only)) { + if (found != null) { + found = null; // more than one alias; ambiguous. + break; + } + found = .{ .provider = d.provider, .model = d.alias }; + } + } + if (found) |ref| return ref; + } + return error.NoModelSelected; + } +}; + +pub const Error = error{ + NoHomeDirectory, + InvalidConfigToml, + InvalidProvider, + InvalidModelRef, + UnknownDefaultProvider, + PolicyConflict, +} || Allocator.Error || FileError; + +pub const ResolveError = error{ + NoModelSelected, + UnknownProvider, +}; + +/// The filesystem errors that can surface while reading a config layer. +/// `FileNotFound` is handled by the caller (skip the layer); any other +/// I/O failure is collapsed to `error.ConfigReadFailed` so this module +/// keeps a small, stable error set. +pub const FileError = Io.Cancelable || error{ FileNotFound, ConfigReadFailed }; + +// =========================================================================== +// Public entry points +// =========================================================================== + +/// Resolve and merge the four config layers from their standard paths, +/// then build a `Config`. Missing files are skipped silently. `cwd` is the +/// project root (used for the `./.panto/config.toml` and +/// `./.panto/config.local.toml` layers). +pub fn load( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, +) Error!Config { + const paths = try layerPaths(allocator, environ_map, cwd); + defer paths.deinit(allocator); + + return loadFromPaths(allocator, io, environ_map, &.{ + paths.base, + paths.user, + paths.project, + paths.local, + }); +} + +const LayerPaths = struct { + base: []u8, + user: []u8, + project: []u8, + local: []u8, + + fn deinit(self: LayerPaths, alloc: Allocator) void { + alloc.free(self.base); + alloc.free(self.user); + alloc.free(self.project); + alloc.free(self.local); + } +}; + +fn layerPaths( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, +) Error!LayerPaths { + const base = try baseConfigPath(allocator, environ_map); + errdefer allocator.free(base); + const user = try userConfigPath(allocator, environ_map); + errdefer allocator.free(user); + const project = try std.fs.path.join(allocator, &.{ cwd, ".panto", "config.toml" }); + errdefer allocator.free(project); + const local = try std.fs.path.join(allocator, &.{ cwd, ".panto", "config.local.toml" }); + return .{ .base = base, .user = user, .project = project, .local = local }; +} + +/// `$PANTO_HOME/config.toml`, falling back to +/// `${XDG_DATA_HOME:-$HOME/.local/share}/panto/config.toml`. +/// +/// `PANTO_HOME` is checked first so this always agrees with where the +/// bootstrap stages the default base config (see `panto_home.resolveHome`). +pub fn baseConfigPath(allocator: Allocator, environ_map: *const std.process.Environ.Map) Error![]u8 { + if (environ_map.get("PANTO_HOME")) |home| { + return try std.fs.path.join(allocator, &.{ home, "config.toml" }); + } + if (environ_map.get("XDG_DATA_HOME")) |xdg| { + return try std.fs.path.join(allocator, &.{ xdg, "panto", "config.toml" }); + } + if (environ_map.get("HOME")) |home| { + return try std.fs.path.join(allocator, &.{ home, ".local", "share", "panto", "config.toml" }); + } + return error.NoHomeDirectory; +} + +/// `${XDG_CONFIG_HOME:-$HOME/.config}/panto/config.toml`. +pub fn userConfigPath(allocator: Allocator, environ_map: *const std.process.Environ.Map) Error![]u8 { + if (environ_map.get("XDG_CONFIG_HOME")) |xdg| { + return try std.fs.path.join(allocator, &.{ xdg, "panto", "config.toml" }); + } + if (environ_map.get("HOME")) |home| { + return try std.fs.path.join(allocator, &.{ home, ".config", "panto", "config.toml" }); + } + return error.NoHomeDirectory; +} + +/// Load from explicit layer paths, lowest precedence first. Useful for +/// tests. Missing files are skipped. +pub fn loadFromPaths( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + paths: []const []const u8, +) Error!Config { + // The merged document is built into its own arena-backed Document so we + // never have to reason about which source layer a given Value came from. + const merged = try tvalue.createDocument(allocator); + defer merged.deinit(); + + for (paths) |path| { + const bytes = readFileAlloc(allocator, io, path) catch |err| switch (err) { + error.FileNotFound => continue, + else => return err, + }; + defer allocator.free(bytes); + + const doc = parseDoc(allocator, bytes) catch return error.InvalidConfigToml; + defer doc.deinit(); + try mergeTable(merged.allocator(), merged.root, doc.root); + } + + return resolve(allocator, environ_map, merged.root); +} + +// =========================================================================== +// Merge +// =========================================================================== + +/// Deep-merge `src` table into `dst` table (both must be `.table`). +/// Tables recurse; everything else (scalars, arrays) overwrites. Values are +/// deep-copied into `alloc` (the merged document's arena) so they outlive +/// the source document. +fn mergeTable(alloc: Allocator, dst: *toml.Value, src: *const toml.Value) Allocator.Error!void { + std.debug.assert(dst.* == .table); + if (src.* != .table) return; + + var it = toml.tableIterator(src); + while (it.next()) |entry| { + const existing = dst.get(entry.key); + if (existing != null and existing.?.* == .table and entry.value.* == .table) { + // Both sides are tables — recurse to accumulate keys. + try mergeTable(alloc, @constCast(existing.?), entry.value); + } else { + // Overwrite (or insert) with a deep copy of the source value. + const copy = try cloneValue(alloc, entry.value); + const key_copy = try alloc.dupe(u8, entry.key); + try tvalue.tableSet(alloc, dst, key_copy, copy); + } + } +} + +fn cloneValue(alloc: Allocator, src: *const toml.Value) Allocator.Error!*toml.Value { + const out = try alloc.create(toml.Value); + switch (src.*) { + .table => { + out.* = .{ .table = .{} }; + var it = toml.tableIterator(src); + while (it.next()) |entry| { + const child = try cloneValue(alloc, entry.value); + const key_copy = try alloc.dupe(u8, entry.key); + try tvalue.tableSet(alloc, out, key_copy, child); + } + }, + .array => |*a| { + out.* = .{ .array = .{} }; + for (a.items.items) |*item| { + const child = try cloneValue(alloc, item); + try tvalue.arrayAppend(alloc, out, child.*); + } + }, + .string => |s| out.* = .{ .string = try alloc.dupe(u8, s) }, + else => out.* = src.*, + } + return out; +} + +// =========================================================================== +// Resolve +// =========================================================================== + +fn resolve( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, + root: *const toml.Value, +) Error!Config { + var providers: std.ArrayList(Provider) = .empty; + errdefer { + for (providers.items) |p| p.deinit(allocator); + providers.deinit(allocator); + } + + if (root.get("providers")) |providers_tbl| { + if (providers_tbl.* == .table) { + var it = toml.tableIterator(providers_tbl); + while (it.next()) |entry| { + const maybe = try resolveProvider(allocator, environ_map, entry.key, entry.value); + if (maybe) |p| try providers.append(allocator, p); + } + } + } + + // Tool / extension policies. + var tools = try resolvePolicy(allocator, root, "tools"); + errdefer tools.deinit(allocator); + var extensions = try resolvePolicy(allocator, root, "extensions"); + errdefer extensions.deinit(allocator); + + // Default model. + var default_model: ?[]const u8 = null; + errdefer if (default_model) |m| allocator.free(m); + var default_model_ref: ?ModelRef = null; + if (root.get("defaults")) |defaults_tbl| { + if (defaults_tbl.get("model")) |model_v| { + if (model_v.* == .string) { + default_model = try allocator.dupe(u8, model_v.string); + default_model_ref = try parseModelRef(default_model.?); + } + } + } + + const providers_slice = try providers.toOwnedSlice(allocator); + errdefer { + for (providers_slice) |p| p.deinit(allocator); + allocator.free(providers_slice); + } + + // Validate the default model names a provider that actually resolved. + if (default_model_ref) |ref| { + var found = false; + for (providers_slice) |p| { + if (std.mem.eql(u8, p.name, ref.provider)) { + found = true; + break; + } + } + if (!found) return error.UnknownDefaultProvider; + } + + return .{ + .allocator = allocator, + .providers = providers_slice, + .default_model = default_model, + .default_model_ref = default_model_ref, + .tools = tools, + .extensions = extensions, + }; +} + +/// Resolve one `providers.<name>` entry. Returns null (provider dropped) if +/// the only key source is an env var that isn't set. +fn resolveProvider( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, + name: []const u8, + val: *const toml.Value, +) Error!?Provider { + if (val.* != .table) return error.InvalidProvider; + + const style_str = (val.get("style") orelse return error.InvalidProvider).asString() orelse + return error.InvalidProvider; + const style = std.meta.stringToEnum(APIStyle, style_str) orelse return error.InvalidProvider; + + const base_url = (val.get("base_url") orelse return error.InvalidProvider).asString() orelse + return error.InvalidProvider; + + // api_key wins over api_key_env_var. + const api_key: []const u8 = blk: { + if (val.get("api_key")) |k| { + if (k.asString()) |s| break :blk s; + } + if (val.get("api_key_env_var")) |ev| { + if (ev.asString()) |env_name| { + if (environ_map.get(env_name)) |actual| break :blk actual; + } + } + // No usable key — silently omit this provider. + return null; + }; + + return .{ + .name = try allocator.dupe(u8, name), + .style = style, + .base_url = try allocator.dupe(u8, base_url), + .api_key = try allocator.dupe(u8, api_key), + }; +} + +fn resolvePolicy(allocator: Allocator, root: *const toml.Value, key: []const u8) Error!Policy { + var allow: [][]const u8 = &.{}; + var deny: [][]const u8 = &.{}; + errdefer { + for (allow) |p| allocator.free(p); + allocator.free(allow); + for (deny) |p| allocator.free(p); + allocator.free(deny); + } + + if (root.get(key)) |tbl| { + if (tbl.* == .table) { + allow = try stringArray(allocator, tbl, "allow"); + deny = try stringArray(allocator, tbl, "deny"); + } + } + + // A pattern present in both allow and deny is contradictory. + for (allow) |a| { + for (deny) |d| { + if (std.mem.eql(u8, a, d)) return error.PolicyConflict; + } + } + + return .{ .allow = allow, .deny = deny }; +} + +fn stringArray(allocator: Allocator, tbl: *const toml.Value, key: []const u8) Error![][]const u8 { + const arr_v = tbl.get(key) orelse return &.{}; + if (arr_v.* != .array) return &.{}; + + var list: std.ArrayList([]const u8) = .empty; + errdefer { + for (list.items) |s| allocator.free(s); + list.deinit(allocator); + } + for (arr_v.array.items.items) |*item| { + if (item.* == .string) { + try list.append(allocator, try allocator.dupe(u8, item.string)); + } + } + return try list.toOwnedSlice(allocator); +} + +/// Parse `<provider>:<model>`. Both halves must be non-empty and there must +/// be exactly one separating colon. +pub fn parseModelRef(ref: []const u8) Error!ModelRef { + const colon = std.mem.indexOfScalar(u8, ref, ':') orelse return error.InvalidModelRef; + const provider = ref[0..colon]; + const model = ref[colon + 1 ..]; + if (provider.len == 0 or model.len == 0) return error.InvalidModelRef; + if (std.mem.indexOfScalar(u8, model, ':') != null) return error.InvalidModelRef; + return .{ .provider = provider, .model = model }; +} + +// =========================================================================== +// File / parse helpers +// =========================================================================== + +fn readFileAlloc(allocator: Allocator, io: Io, path: []const u8) (Allocator.Error || FileError)![]u8 { + const file = Io.Dir.cwd().openFile(io, path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => return error.FileNotFound, + error.Canceled => return error.Canceled, + else => return error.ConfigReadFailed, + }; + defer file.close(io); + const len = file.length(io) catch return error.ConfigReadFailed; + const bytes = try allocator.alloc(u8, @intCast(len)); + errdefer allocator.free(bytes); + _ = file.readPositionalAll(io, bytes, 0) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return error.ConfigReadFailed, + }; + return bytes; +} + +fn parseDoc(allocator: Allocator, source: []const u8) !*toml.Document { + const result = toml.parseWithError(allocator, source, .{}); + switch (result) { + .err => |e| { + if (!@import("builtin").is_test) { + std.log.err( + "config.toml: parse error at line {d}, column {d}: {s}", + .{ e.line, e.column, e.message }, + ); + } + return error.InvalidConfigToml; + }, + .ok => |doc| return doc, + } +} + +// =========================================================================== +// Tests +// =========================================================================== + +const testing = std.testing; + +fn emptyEnv(a: Allocator) std.process.Environ.Map { + return std.process.Environ.Map.init(a); +} + +test "parseModelRef: splits provider and model" { + const ref = try parseModelRef("anthropic:claude-sonnet-4-6"); + try testing.expectEqualStrings("anthropic", ref.provider); + try testing.expectEqualStrings("claude-sonnet-4-6", ref.model); +} + +test "parseModelRef: rejects malformed refs" { + try testing.expectError(error.InvalidModelRef, parseModelRef("no-colon")); + try testing.expectError(error.InvalidModelRef, parseModelRef(":model")); + try testing.expectError(error.InvalidModelRef, parseModelRef("provider:")); + try testing.expectError(error.InvalidModelRef, parseModelRef("a:b:c")); +} + +test "Policy.permits: empty allow means allow-all minus deny" { + const a = testing.allocator; + var deny = try a.alloc([]const u8, 1); + deny[0] = try a.dupe(u8, "std.shell"); + const policy: Policy = .{ .allow = &.{}, .deny = deny }; + defer policy.deinit(a); + + try testing.expect(policy.permits("std.read")); + try testing.expect(!policy.permits("std.shell")); +} + +test "Policy.permits: allow gates everything not matched" { + const a = testing.allocator; + var allow = try a.alloc([]const u8, 1); + allow[0] = try a.dupe(u8, "std.*"); + const policy: Policy = .{ .allow = allow, .deny = &.{} }; + defer policy.deinit(a); + + try testing.expect(policy.permits("std.read")); + try testing.expect(!policy.permits("other.read")); +} + +test "resolve: env-backed provider is dropped when env var is absent" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + try testing.expectEqual(@as(usize, 0), cfg.providers.len); +} + +test "resolve: env-backed provider resolves when env var is set" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + try env.put("ANTHROPIC_API_KEY", "sk-ant-xyz"); + + const src = + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + try testing.expectEqual(@as(usize, 1), cfg.providers.len); + const p = cfg.provider("anthropic").?; + try testing.expectEqual(APIStyle.anthropic_messages, p.style); + try testing.expectEqualStrings("sk-ant-xyz", p.api_key); +} + +test "resolve: inline api_key wins over env var" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + try env.put("KEY_FROM_ENV", "env-value"); + + const src = + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "inline-value" + \\api_key_env_var = "KEY_FROM_ENV" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + const p = cfg.provider("openai").?; + try testing.expectEqualStrings("inline-value", p.api_key); +} + +test "resolve: unknown default-model provider is an error" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[defaults] + \\model = "ghost:some-model" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + try testing.expectError(error.UnknownDefaultProvider, resolve(a, &env, doc.root)); +} + +test "resolvePolicy: pattern in both allow and deny is a conflict" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[tools] + \\allow = ["std.*"] + \\deny = ["std.*"] + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + try testing.expectError(error.PolicyConflict, resolve(a, &env, doc.root)); +} + +test "loadFromPaths: layers merge with tables accumulating and scalars overriding" { + const a = testing.allocator; + const io = testing.io; + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_n = try tmp.dir.realPath(io, &path_buf); + const tmp_path = path_buf[0..tmp_n]; + + // Base layer: defines anthropic provider + a default model. + const sys_path = try std.fs.path.join(a, &.{ tmp_path, "base.toml" }); + defer a.free(sys_path); + try tmp.dir.writeFile(io, .{ .sub_path = "base.toml", .data = + \\[defaults] + \\model = "anthropic:sonnet" + \\ + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key = "sys-key" + \\ + \\[tools] + \\allow = ["std.*"] + }); + + // Project layer: adds an openai provider (table accumulates), and + // overrides the default model (scalar overwrites). + const proj_path = try std.fs.path.join(a, &.{ tmp_path, "project.toml" }); + defer a.free(proj_path); + try tmp.dir.writeFile(io, .{ .sub_path = "project.toml", .data = + \\[defaults] + \\model = "openai:gpt" + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "proj-key" + }); + + var env = emptyEnv(a); + defer env.deinit(); + + var cfg = try loadFromPaths(a, io, &env, &.{ sys_path, proj_path }); + defer cfg.deinit(); + + // Both providers survive (table accumulation). + try testing.expectEqual(@as(usize, 2), cfg.providers.len); + try testing.expect(cfg.provider("anthropic") != null); + try testing.expect(cfg.provider("openai") != null); + + // Default model overridden by the project layer. + try testing.expectEqualStrings("openai:gpt", cfg.default_model.?); + try testing.expectEqualStrings("openai", cfg.default_model_ref.?.provider); + try testing.expectEqualStrings("gpt", cfg.default_model_ref.?.model); + + // tools.allow from the base layer is preserved (no project override). + try testing.expect(cfg.tools.permits("std.read")); +} + +test "loadFromPaths: local layer overrides project layer" { + const a = testing.allocator; + const io = testing.io; + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_n = try tmp.dir.realPath(io, &path_buf); + const tmp_path = path_buf[0..tmp_n]; + + // Project layer: shared default model + provider. + const proj_path = try std.fs.path.join(a, &.{ tmp_path, "project.toml" }); + defer a.free(proj_path); + try tmp.dir.writeFile(io, .{ .sub_path = "project.toml", .data = + \\[defaults] + \\model = "openai:gpt" + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "proj-key" + }); + + // Local layer (git-ignored): a developer's more-opinionated override of + // the default model. Scalar overwrites; the provider table accumulates. + const local_path = try std.fs.path.join(a, &.{ tmp_path, "local.toml" }); + defer a.free(local_path); + try tmp.dir.writeFile(io, .{ .sub_path = "local.toml", .data = + \\[defaults] + \\model = "openai:gpt-local" + }); + + var env = emptyEnv(a); + defer env.deinit(); + + var cfg = try loadFromPaths(a, io, &env, &.{ proj_path, local_path }); + defer cfg.deinit(); + + // Local layer wins for the default model. + try testing.expectEqualStrings("openai:gpt-local", cfg.default_model.?); + // Provider from the project layer survives (table accumulation). + try testing.expect(cfg.provider("openai") != null); +} + +test "buildProviderConfig: anthropic ref pulls knobs from model defs" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + try env.put("ANTHROPIC_API_KEY", "sk-ant"); + + const src = + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + try models.entries.append(a, .{ + .provider = try a.dupe(u8, "anthropic"), + .alias = try a.dupe(u8, "sonnet"), + .model = try a.dupe(u8, "claude-sonnet-4-20250514"), + .reasoning = .high, + .max_tokens = 8192, + .api_version = try a.dupe(u8, "2023-06-01"), + }); + + const ref = try parseModelRef("anthropic:sonnet"); + const pc = try buildProviderConfig(&cfg, &models, ref); + try testing.expectEqual(APIStyle.anthropic_messages, pc.style()); + try testing.expectEqualStrings("claude-sonnet-4-20250514", pc.anthropic_messages.model); + try testing.expectEqual(@as(u32, 8192), pc.anthropic_messages.max_tokens); + try testing.expectEqualStrings("sk-ant", pc.anthropic_messages.api_key); +} + +test "buildProviderConfig: missing alias uses the alias verbatim as wire model" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "sk-test" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + + const ref = try parseModelRef("openai:gpt-4o"); + const pc = try buildProviderConfig(&cfg, &models, ref); + try testing.expectEqual(APIStyle.openai_chat, pc.style()); + try testing.expectEqualStrings("gpt-4o", pc.openai_chat.model); + try testing.expectEqual(panto.config.ReasoningEffort.default, pc.openai_chat.reasoning); +} + +test "buildProviderConfig: unknown provider errors" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + const empty = try parseDoc(a, ""); + defer empty.deinit(); + var cfg = try resolve(a, &env, empty.root); + defer cfg.deinit(); + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + try testing.expectError(error.UnknownProvider, buildProviderConfig(&cfg, &models, .{ .provider = "ghost", .model = "x" })); +} + +test "selectModel: default_model wins; single-provider fallback otherwise" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + // Config with one provider and a default model. + const src = + \\[defaults] + \\model = "openai:gpt" + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "sk" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + + const ref = try cfg.selectModel(&models, null); + try testing.expectEqualStrings("openai", ref.provider); + try testing.expectEqualStrings("gpt", ref.model); + + // Override beats default. + const ref2 = try cfg.selectModel(&models, "openai:other"); + try testing.expectEqualStrings("other", ref2.model); +} + +test "selectModel: no default and no providers errors" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + const empty = try parseDoc(a, ""); + defer empty.deinit(); + var cfg = try resolve(a, &env, empty.root); + defer cfg.deinit(); + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + try testing.expectError(error.NoModelSelected, cfg.selectModel(&models, null)); +} + +test "loadFromPaths: missing files are skipped" { + const a = testing.allocator; + const io = testing.io; + var env = emptyEnv(a); + defer env.deinit(); + + var cfg = try loadFromPaths(a, io, &env, &.{ + "/nonexistent/base.toml", + "/nonexistent/project.toml", + }); + defer cfg.deinit(); + try testing.expectEqual(@as(usize, 0), cfg.providers.len); + try testing.expect(cfg.default_model == null); +} diff --git a/src/extension_loader.zig b/src/extension_loader.zig index 2fd86ae..b84eba5 100644 --- a/src/extension_loader.zig +++ b/src/extension_loader.zig @@ -1,22 +1,22 @@ //! Extension and tool discovery: walk well-known directories, locate Lua //! files, and load each into a long-lived `LuaRuntime`. //! -//! Two parallel namespaces are scanned, each at three scopes — system, -//! user, and project. Project shadows user shadows system. +//! Two parallel namespaces are scanned, each at three scopes — base, +//! user, and project. Project shadows user shadows base. //! //! Extensions (full-featured; call `panto.register_tool` from a script //! that may register many tools): -//! 1. `$PANTO_HOME/agent/extensions/` ("system") +//! 1. `$PANTO_HOME/agent/extensions/` ("base") //! 2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/` ("user") //! 3. `./.panto/extensions/` ("project") //! //! Tools (ergonomic single-tool form; the script returns one table //! shaped like the argument to `panto.register_tool`): -//! 1. `$PANTO_HOME/agent/tools/` ("system") +//! 1. `$PANTO_HOME/agent/tools/` ("base") //! 2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/tools/` ("user") //! 3. `./.panto/tools/` ("project") //! -//! The `system` layer is populated at bootstrap from files embedded +//! The `base` layer is populated at bootstrap from files embedded //! into the panto binary (see `build/gen_agent_embed.zig` and //! `luarocks_runtime.stageAgentTree`). It is panto's "batteries" tier: //! ships in the binary, but every entry is individually shadowable @@ -33,7 +33,7 @@ //! Conflict rules: //! - Within one directory, two entries with the same logical name //! are an error. -//! - Project shadows user shadows system *within the same kind* +//! - Project shadows user shadows base *within the same kind* //! (extension or tool). Extensions and tools live in distinct //! namespaces for shadowing; the registered *tool names* still //! share one global namespace across both, and collisions there @@ -44,11 +44,32 @@ const std = @import("std"); const panto = @import("panto"); const lua_runtime = @import("lua_runtime.zig"); +const config_file = @import("config_file.zig"); const Allocator = std.mem.Allocator; const Io = std.Io; const LuaRuntime = lua_runtime.LuaRuntime; +/// Availability policies for the two namespaces. Both are optional; a +/// null policy permits everything. `extensions` gates extension/tool +/// *entries* by their logical (file/dir) name before loading; `tools` +/// gates the *registered tool names* (e.g. `std.read`) after loading. +pub const Policies = struct { + extensions: ?*const config_file.Policy = null, + tools: ?*const config_file.Policy = null, +}; + +fn policyPermits(p: ?*const config_file.Policy, name: []const u8) bool { + const pol = p orelse return true; + return pol.permits(name); +} + +/// Free-function form of `Policy.permits` taking the policy by pointer, +/// matching the `fn(ctx, name) bool` shape `LuaRuntime.filterTools` wants. +fn permitsByPtr(pol: *const config_file.Policy, name: []const u8) bool { + return pol.permits(name); +} + /// A discovered extension or tool before loading. Owns its strings. const Found = struct { /// Logical name (basename without `.lua`, or directory name). @@ -72,13 +93,13 @@ const Found = struct { pub const Source = enum { /// Staged at bootstrap from files embedded in the panto binary. /// Lowest priority — shadowed by user and project. - system, + base, user, project, pub fn label(self: Source) []const u8 { return switch (self) { - .system => "system", + .base => "base", .user => "user", .project => "project", }; @@ -101,9 +122,9 @@ pub const Kind = enum { /// paths into `runtime`. Returns the number of registered tools added /// to the runtime by this call. /// -/// `system_agent_dir`, when non-null, is the path under which embedded -/// system tools/extensions have been staged — typically -/// `$PANTO_HOME/agent/`. Pass `null` to skip the system layer entirely +/// `base_agent_dir`, when non-null, is the path under which embedded +/// base tools/extensions have been staged — typically +/// `$PANTO_HOME/agent/`. Pass `null` to skip the base layer entirely /// (mostly useful for tests). /// /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The @@ -112,15 +133,16 @@ pub fn discoverAndLoad( allocator: Allocator, io: Io, environ_map: *const std.process.Environ.Map, - system_agent_dir: ?[]const u8, + base_agent_dir: ?[]const u8, runtime: *LuaRuntime, + policies: Policies, ) !usize { - const sys_ext = if (system_agent_dir) |d| + const sys_ext = if (base_agent_dir) |d| try std.fs.path.join(allocator, &.{ d, kindSubdir(.extension) }) else null; defer if (sys_ext) |d| allocator.free(d); - const sys_tool = if (system_agent_dir) |d| + const sys_tool = if (base_agent_dir) |d| try std.fs.path.join(allocator, &.{ d, kindSubdir(.tool) }) else null; @@ -141,27 +163,28 @@ pub fn discoverAndLoad( io, runtime, .{ - .system_extensions = sys_ext, + .base_extensions = sys_ext, .user_extensions = user_ext, .project_extensions = project_ext, - .system_tools = sys_tool, + .base_tools = sys_tool, .user_tools = user_tool, .project_tools = project_tool, }, + policies, ); } /// Set of search paths consumed by `loadFromDirs`. Any field may be /// null; missing directories on disk are silently skipped. /// -/// Scan order is system → user → project. `applyShadowing` keeps the +/// Scan order is base → user → project. `applyShadowing` keeps the /// *last* occurrence of each (kind, name), so project entries win, -/// then user, then system. +/// then user, then base. pub const DirSet = struct { - system_extensions: ?[]const u8 = null, + base_extensions: ?[]const u8 = null, user_extensions: ?[]const u8 = null, project_extensions: ?[]const u8 = null, - system_tools: ?[]const u8 = null, + base_tools: ?[]const u8 = null, user_tools: ?[]const u8 = null, project_tools: ?[]const u8 = null, }; @@ -173,6 +196,7 @@ pub fn loadFromDirs( io: Io, runtime: *LuaRuntime, dirs: DirSet, + policies: Policies, ) !usize { var found: std.array_list.Managed(Found) = .init(allocator); defer { @@ -180,15 +204,38 @@ pub fn loadFromDirs( found.deinit(); } - if (dirs.system_extensions) |d| try scanDir(allocator, io, d, .system, .extension, &found); + if (dirs.base_extensions) |d| try scanDir(allocator, io, d, .base, .extension, &found); if (dirs.user_extensions) |d| try scanDir(allocator, io, d, .user, .extension, &found); if (dirs.project_extensions) |d| try scanDir(allocator, io, d, .project, .extension, &found); - if (dirs.system_tools) |d| try scanDir(allocator, io, d, .system, .tool, &found); + if (dirs.base_tools) |d| try scanDir(allocator, io, d, .base, .tool, &found); if (dirs.user_tools) |d| try scanDir(allocator, io, d, .user, .tool, &found); if (dirs.project_tools) |d| try scanDir(allocator, io, d, .project, .tool, &found); try applyShadowing(allocator, &found); + // Gate *entries* by the extensions policy (matched on logical name). + // This drops whole scripts before they run — a denied extension + // never executes, never registers tools. The tools policy is + // applied post-load against registered tool names. + if (policies.extensions) |_| { + var keep: std.array_list.Managed(Found) = .init(allocator); + errdefer { + for (keep.items) |*f| f.deinit(allocator); + keep.deinit(); + } + for (found.items) |*f| { + if (policyPermits(policies.extensions, f.name)) { + try keep.append(f.*); + } else { + std.log.debug("{s}: '{s}' denied by extensions policy", .{ f.kind.label(), f.name }); + f.deinit(allocator); + } + } + found.clearRetainingCapacity(); + try found.appendSlice(keep.items); + keep.deinit(); + } + const before = runtime.toolCount(); for (found.items) |f| { const load_result = switch (f.kind) { @@ -214,6 +261,13 @@ pub fn loadFromDirs( .{ f.kind.label(), f.name, f.source.label() }, ); } + + // Apply the tools policy to *registered tool names* (e.g. `std.read`). + if (policies.tools) |pol| { + const dropped = runtime.filterTools(pol, permitsByPtr); + if (dropped > 0) std.log.debug("tools: {d} tool(s) removed by tools policy", .{dropped}); + } + return runtime.toolCount() - before; } @@ -581,7 +635,7 @@ test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = user_path, .project_extensions = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); // Invoke the tool through the source and verify the project handler ran. @@ -623,7 +677,7 @@ test "loadFromDirs: tool-name collision between extensions errors" { const result = loadFromDirs(testing.allocator, testing.io, rt, .{ .project_extensions = ext_path, - }); + }, .{}); try testing.expectError(error.DuplicateTool, result); } @@ -650,7 +704,7 @@ test "loadFromDirs: tools/ directory — single-file tool form" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -690,7 +744,7 @@ test "loadFromDirs: tools/ directory — directory-style tool with sibling requi const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .project_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -738,7 +792,7 @@ test "loadFromDirs: project tool shadows user tool of the same name" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = user_path, .project_tools = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -749,19 +803,19 @@ test "loadFromDirs: project tool shadows user tool of the same name" { try testing.expectEqualStrings("PROJECT", results[0].ok); } -test "loadFromDirs: project tool shadows user shadows system" { +test "loadFromDirs: project tool shadows user shadows base" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); - try makeDir(tmp.dir, "system_tools"); + try makeDir(tmp.dir, "base_tools"); try makeDir(tmp.dir, "user_tools"); try makeDir(tmp.dir, "project_tools"); - try writeFile(tmp.dir, "system_tools/greet.lua", + try writeFile(tmp.dir, "base_tools/greet.lua", \\return { - \\ name = "greet", description = "system version", + \\ name = "greet", description = "base version", \\ schema = { type = "object" }, - \\ handler = function(input) return "SYSTEM" end, + \\ handler = function(input) return "BASE" end, \\} ); try writeFile(tmp.dir, "user_tools/greet.lua", @@ -780,7 +834,7 @@ test "loadFromDirs: project tool shadows user shadows system" { ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf); + const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf); const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]); defer testing.allocator.free(sys_path); const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); @@ -794,10 +848,10 @@ test "loadFromDirs: project tool shadows user shadows system" { defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ - .system_tools = sys_path, + .base_tools = sys_path, .user_tools = user_path, .project_tools = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -808,18 +862,18 @@ test "loadFromDirs: project tool shadows user shadows system" { try testing.expectEqualStrings("PROJECT", results[0].ok); } -test "loadFromDirs: user tool shadows system tool when no project entry" { +test "loadFromDirs: user tool shadows base tool when no project entry" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); - try makeDir(tmp.dir, "system_tools"); + try makeDir(tmp.dir, "base_tools"); try makeDir(tmp.dir, "user_tools"); - try writeFile(tmp.dir, "system_tools/greet.lua", + try writeFile(tmp.dir, "base_tools/greet.lua", \\return { - \\ name = "greet", description = "system", + \\ name = "greet", description = "base", \\ schema = { type = "object" }, - \\ handler = function(input) return "SYSTEM" end, + \\ handler = function(input) return "BASE" end, \\} ); try writeFile(tmp.dir, "user_tools/greet.lua", @@ -831,7 +885,7 @@ test "loadFromDirs: user tool shadows system tool when no project entry" { ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf); + const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf); const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]); defer testing.allocator.free(sys_path); const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); @@ -842,9 +896,9 @@ test "loadFromDirs: user tool shadows system tool when no project entry" { defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ - .system_tools = sys_path, + .base_tools = sys_path, .user_tools = user_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -895,6 +949,87 @@ test "loadFromDirs: extension and tool share a *file* name independently" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = ext_path, .user_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 2), n_tools); } + +test "loadFromDirs: tools policy removes a denied registered tool name" { + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + try makeDir(tmp.dir, "tools"); + try writeFile(tmp.dir, "tools/reader.lua", + \\return { + \\ name = "std.read", description = "r", + \\ schema = { type = "object" }, + \\ handler = function(input) return "READ" end, + \\} + ); + try writeFile(tmp.dir, "tools/sheller.lua", + \\return { + \\ name = "std.shell", description = "s", + \\ schema = { type = "object" }, + \\ handler = function(input) return "SHELL" end, + \\} + ); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf); + const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); + defer testing.allocator.free(tools_path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + var deny = try testing.allocator.alloc([]const u8, 1); + deny[0] = try testing.allocator.dupe(u8, "std.shell"); + const tools_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny }; + defer tools_policy.deinit(testing.allocator); + + // Both tools register, but std.shell is filtered out post-load. + const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ + .user_tools = tools_path, + }, .{ .tools = &tools_policy }); + try testing.expectEqual(@as(usize, 1), n_tools); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{.{ .tool_name = "std.read", .input = "{}" }}; + var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer testing.allocator.free(results[0].ok); + try testing.expectEqualStrings("READ", results[0].ok); +} + +test "loadFromDirs: extensions policy denies a whole entry before it loads" { + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + try makeDir(tmp.dir, "tools"); + // This tool would crash if its script ran (calls a nil global). The + // extensions policy must stop it from loading at all. + try writeFile(tmp.dir, "danger.lua", + \\error("this script must never run") + ); + try makeDir(tmp.dir, "td"); + try writeFile(tmp.dir, "td/danger.lua", + \\error("this script must never run") + ); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPathFile(testing.io, "td", &path_buf); + const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); + defer testing.allocator.free(tools_path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + var deny = try testing.allocator.alloc([]const u8, 1); + deny[0] = try testing.allocator.dupe(u8, "danger"); + const ext_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny }; + defer ext_policy.deinit(testing.allocator); + + const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ + .user_tools = tools_path, + }, .{ .extensions = &ext_policy }); + try testing.expectEqual(@as(usize, 0), n_tools); +} diff --git a/src/glob.zig b/src/glob.zig new file mode 100644 index 0000000..83d9fcb --- /dev/null +++ b/src/glob.zig @@ -0,0 +1,121 @@ +//! Minimal glob matcher for tool/extension allow- and deny-lists. +//! +//! Patterns and names are dot-delimited segments (e.g. `std.git.status`). +//! Matching operates on whole segments, never on individual characters. +//! +//! Supported segment wildcards: +//! - `*` matches exactly one segment. `std.*` matches `std.read` but +//! not `std.git.status` (greedy within a segment, but not +//! dot-agnostic). +//! - `**` matches one or more segments. `std.**` matches both +//! `std.read` and `std.git.status`. +//! +//! A wildcard is only recognized as a whole segment: `std.*` and `std.**` +//! are wildcards, but `std.rea*` is a literal segment (the `*` has no +//! special meaning mid-segment). Everything else is matched literally. + +const std = @import("std"); + +/// Returns true if `name` matches `pattern` under the segment rules above. +pub fn match(pattern: []const u8, name: []const u8) bool { + var pat_it = std.mem.splitScalar(u8, pattern, '.'); + var name_it = std.mem.splitScalar(u8, name, '.'); + return matchSegments(&pat_it, &name_it); +} + +const SegmentIter = std.mem.SplitIterator(u8, .scalar); + +fn matchSegments(pat_it: *SegmentIter, name_it: *SegmentIter) bool { + while (pat_it.next()) |seg| { + if (std.mem.eql(u8, seg, "**")) { + // `**` matches one or more remaining segments. Try the + // shortest match first (one segment), then extend. + const rest_pat = pat_it.*; // pattern after `**` + while (name_it.next()) |_| { + var pat_copy = rest_pat; + var name_copy = name_it.*; + if (matchSegments(&pat_copy, &name_copy)) return true; + } + // `**` requires at least one segment, and the trailing + // pattern must have matched the remainder above. + return false; + } + + const name_seg = name_it.next() orelse return false; + if (std.mem.eql(u8, seg, "*")) { + // Matches exactly one (any) segment. + continue; + } + if (!std.mem.eql(u8, seg, name_seg)) return false; + } + + // Pattern exhausted: match iff name is also exhausted. + return name_it.next() == null; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "literal match" { + try testing.expect(match("std.read", "std.read")); + try testing.expect(!match("std.read", "std.write")); + try testing.expect(!match("std.read", "std.read.more")); + try testing.expect(!match("std.read.more", "std.read")); +} + +test "single star matches exactly one segment" { + try testing.expect(match("std.*", "std.read")); + try testing.expect(match("std.*", "std.write")); + try testing.expect(!match("std.*", "std.git.status")); + try testing.expect(!match("std.*", "std")); + try testing.expect(!match("std.*", "other.read")); +} + +test "double star is dot-agnostic, one or more segments" { + try testing.expect(match("std.**", "std.read")); + try testing.expect(match("std.**", "std.git.status")); + try testing.expect(!match("std.**", "std")); + try testing.expect(!match("std.**", "other.read")); +} + +test "bare single star matches one segment" { + try testing.expect(match("*", "std")); + try testing.expect(!match("*", "std.read")); + // An empty name is a single empty segment, which `*` matches. + try testing.expect(match("*", "")); +} + +test "bare double star matches everything non-empty" { + try testing.expect(match("**", "std")); + try testing.expect(match("**", "std.read")); + try testing.expect(match("**", "std.git.status")); +} + +test "star in middle" { + try testing.expect(match("*.read", "std.read")); + try testing.expect(match("*.*", "std.read")); + try testing.expect(!match("*.*", "std.git.status")); + try testing.expect(match("std.*.status", "std.git.status")); + try testing.expect(!match("std.*.status", "std.status")); +} + +test "double star in middle" { + try testing.expect(match("std.**.status", "std.git.status")); + try testing.expect(match("std.**.status", "std.a.b.status")); + try testing.expect(!match("std.**.status", "std.status")); + try testing.expect(match("**.status", "std.git.status")); +} + +test "wildcard only recognized as whole segment" { + // A `*` embedded in a segment is a literal, not a wildcard. + try testing.expect(match("std.rea*", "std.rea*")); + try testing.expect(!match("std.rea*", "std.read")); +} + +test "empty pattern only matches empty name" { + try testing.expect(match("", "")); + try testing.expect(!match("", "x")); +} diff --git a/src/lua_runtime.zig b/src/lua_runtime.zig index 5efadaa..fed56a8 100644 --- a/src/lua_runtime.zig +++ b/src/lua_runtime.zig @@ -354,6 +354,38 @@ pub const LuaRuntime = struct { pub fn toolCount(self: *const LuaRuntime) usize { return self.decls.items.len; } + + /// Drop every declared tool whose registered name is rejected by + /// `permits`. The handler ref is unref'd and the decl removed. + /// Returns the number of tools dropped. + /// + /// String storage for a dropped tool's name/description/schema is + /// left in `self.strings` (freed at `deinit`); only the decl entry + /// and the Lua handler ref are reclaimed eagerly. This keeps the + /// filter simple — we never need to find-and-free individual + /// strings out of the shared pool. + pub fn filterTools( + self: *LuaRuntime, + ctx: anytype, + comptime permits: fn (@TypeOf(ctx), []const u8) bool, + ) usize { + var dropped: usize = 0; + var i: usize = 0; + while (i < self.decls.items.len) { + const name = self.decls.items[i].name; + if (permits(ctx, name)) { + i += 1; + continue; + } + // Unref the handler, if present. + if (self.handlers.fetchRemove(name)) |kv| { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, kv.value); + } + _ = self.decls.orderedRemove(i); + dropped += 1; + } + return dropped; + } }; const source_vtable: panto.ToolSource.VTable = .{ diff --git a/src/luarocks_runtime.zig b/src/luarocks_runtime.zig index 0114b64..e652096 100644 --- a/src/luarocks_runtime.zig +++ b/src/luarocks_runtime.zig @@ -137,6 +137,7 @@ pub fn bootstrap( try panto_home.ensureDirsExist(layout, io); try stageLuaHeaders(allocator, io, layout); try stageAgentTree(allocator, io, layout); + try stageDefaultConfig(allocator, io, layout); const lua_wrapper_path = try writeLuaWrapper(allocator, io, layout, panto_executable_path); defer allocator.free(lua_wrapper_path); try writeLuarocksConfig(allocator, io, layout, lua_wrapper_path); @@ -236,6 +237,71 @@ fn stageAgentTree( } } +/// The default base `config.toml`, materialized at `$PANTO_HOME/config.toml` +/// on first run if absent. It declares OpenAI and Anthropic providers whose +/// API keys come from the conventional env vars — so a provider simply +/// doesn't appear unless its key is exported. Edit or override at the user +/// (`~/.config/panto/config.toml`) or project (`./.panto/config.toml`) layer. +const default_base_config = + \\# panto base config (auto-generated on first run). + \\# + \\# This is the lowest-precedence layer. Override anything here from + \\# ~/.config/panto/config.toml (user) + \\# ./.panto/config.toml (project) + \\# Tables merge across layers; scalars and arrays from a higher layer + \\# replace the lower one wholesale. + \\# + \\# A provider whose API key (or its api_key_env_var) is missing at + \\# runtime is silently dropped — so these defaults disappear unless + \\# you've exported the corresponding key. + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key_env_var = "OPENAI_API_KEY" + \\ + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + \\ + \\# Pick the default model with `<provider>:<alias>`. The alias is + \\# resolved against models.toml; an unknown alias is used verbatim as + \\# the wire model name. + \\# + \\# [defaults] + \\# model = "anthropic:claude-sonnet-4-20250514" + \\ + \\# Tool/extension availability (glob patterns). Empty allow = allow all. + \\# [tools] + \\# allow = ["std.*"] + \\# deny = [] + \\ +; + +/// Materialize the default base config at `$PANTO_HOME/config.toml`, +/// but only if no file exists there yet. We never overwrite — the base +/// config is user-editable, and a stale-but-edited file must win over the +/// shipped default. +fn stageDefaultConfig( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, +) !void { + _ = allocator; + var home = try Io.Dir.cwd().openDir(io, layout.home, .{}); + defer home.close(io); + + home.access(io, "config.toml", .{}) catch |err| switch (err) { + error.FileNotFound => { + try home.writeFile(io, .{ .sub_path = "config.toml", .data = default_base_config }); + return; + }, + else => return err, + }; + // Exists already — leave it untouched. +} + /// Write `contents` into `dir/name` only if the existing file differs. /// Creates the file if absent. fn writeIfDifferent( diff --git a/src/main.zig b/src/main.zig index bdb5390..aaf86b5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,6 +9,8 @@ const self_exe = @import("self_exe.zig"); const subcommand = @import("subcommand.zig"); const session_paths = @import("session_paths.zig"); const models_toml = @import("models_toml.zig"); +const config_file = @import("config_file.zig"); +const glob = @import("glob.zig"); // Shorthand alias for the Lua C API. The bridge module owns the actual // `@cImport`; we re-use it here so the smoke check uses identical types. @@ -33,6 +35,8 @@ test { _ = self_exe; _ = subcommand; _ = models_toml; + _ = config_file; + _ = glob; } const Receiver = panto.provider.Receiver; @@ -195,74 +199,58 @@ fn luaSmokeCheck() void { lua.lua_settop(L, 0); } -fn loadConfig(environ_map: *const std.process.Environ.Map) !panto.config.Config { - const style_str = environ_map.get("PANTO_API_STYLE") orelse "openai_chat"; - const style = std.meta.stringToEnum(panto.config.APIStyle, style_str) orelse { - std.debug.print( - "error: PANTO_API_STYLE must be one of: openai_chat, anthropic_messages (got: {s})\n", - .{style_str}, - ); - return error.InvalidApiStyle; - }; +/// Print a friendly message for a config.toml load failure before the +/// process exits non-zero. +fn reportConfigError(err: anyerror) void { + switch (err) { + error.InvalidConfigToml => std.debug.print( + "error: a config.toml failed to parse (see log above).\n", + .{}, + ), + error.InvalidProvider => std.debug.print( + "error: a [providers.*] entry is malformed (need style + base_url).\n", + .{}, + ), + error.InvalidModelRef => std.debug.print( + "error: defaults.model must look like \"provider:model\".\n", + .{}, + ), + error.UnknownDefaultProvider => std.debug.print( + "error: defaults.model names a provider that isn't configured (or whose API key env var isn't set).\n", + .{}, + ), + error.PolicyConflict => std.debug.print( + "error: a tool/extension pattern appears in both allow and deny.\n", + .{}, + ), + error.NoHomeDirectory => std.debug.print( + "error: cannot resolve config paths — set HOME or XDG_CONFIG_HOME/XDG_DATA_HOME.\n", + .{}, + ), + else => std.debug.print("error: failed to load config: {s}\n", .{@errorName(err)}), + } +} - switch (style) { - .openai_chat => { - const api_key = environ_map.get("OPENAI_API_KEY") orelse { - std.debug.print("error: OPENAI_API_KEY is required\n", .{}); - return error.MissingApiKey; - }; - const base_url = environ_map.get("OPENAI_BASE_URL") orelse "https://api.openai.com/v1"; - const model = environ_map.get("OPENAI_MODEL") orelse { - std.debug.print("error: OPENAI_MODEL is required\n", .{}); - return error.MissingModel; - }; - const reasoning: panto.config.ReasoningEffort = - if (environ_map.get("OPENAI_REASONING")) |val| - std.meta.stringToEnum(panto.config.ReasoningEffort, val) orelse { - std.debug.print( - "error: OPENAI_REASONING must be one of: default, off, minimal, low, medium, high (got: {s})\n", - .{val}, - ); - return error.InvalidReasoning; - } - else - .default; - return .{ .openai_chat = .{ - .api_key = api_key, - .base_url = base_url, - .model = model, - .reasoning = reasoning, - } }; - }, - .anthropic_messages => { - const api_key = environ_map.get("ANTHROPIC_API_KEY") orelse { - std.debug.print("error: ANTHROPIC_API_KEY is required\n", .{}); - return error.MissingApiKey; - }; - const base_url = environ_map.get("ANTHROPIC_BASE_URL") orelse "https://api.anthropic.com"; - const model = environ_map.get("ANTHROPIC_MODEL") orelse { - std.debug.print("error: ANTHROPIC_MODEL is required\n", .{}); - return error.MissingModel; - }; - const api_version = environ_map.get("ANTHROPIC_API_VERSION") orelse "2023-06-01"; - const max_tokens: u32 = if (environ_map.get("ANTHROPIC_MAX_TOKENS")) |val| - std.fmt.parseInt(u32, val, 10) catch { - std.debug.print( - "error: ANTHROPIC_MAX_TOKENS must be a positive integer (got: {s})\n", - .{val}, - ); - return error.InvalidMaxTokens; - } - else - 4096; - return .{ .anthropic_messages = .{ - .api_key = api_key, - .base_url = base_url, - .model = model, - .api_version = api_version, - .max_tokens = max_tokens, - } }; +/// Print a friendly message for a model-selection failure. +fn reportModelError(err: anyerror, cfg: *const config_file.Config) void { + switch (err) { + error.NoModelSelected => { + std.debug.print( + "error: no model selected. Set `defaults.model = \"provider:alias\"` in config.toml.\n", + .{}, + ); + if (cfg.providers.len == 0) { + std.debug.print( + " (no providers resolved — check that an API key or its env var is present.)\n", + .{}, + ); + } }, + error.UnknownProvider => std.debug.print( + "error: the selected model names an unconfigured provider.\n", + .{}, + ), + else => std.debug.print("error: model selection failed: {s}\n", .{@errorName(err)}), } } @@ -294,8 +282,6 @@ pub fn main(init: std.process.Init) !void { .agent => {}, } - const config = try loadConfig(init.environ_map); - // Parse the agent-mode flags. Currently only `--resume [<id>]`. const cli_flags = try parseAgentFlags(alloc, init.minimal.args); defer cli_flags.deinit(alloc); @@ -315,19 +301,47 @@ pub fn main(init: std.process.Init) !void { const session_dir = try session_paths.sessionDirForCwd(alloc, init.environ_map, cwd); defer alloc.free(session_dir); - // Load the user's models.toml — a missing file is fine (empty - // registry). Cost lookups against an empty registry return null, - // and the display layer will format that as "unknown." + // Load the merged config.toml (base → user → project). Providers, + // default model, and tool/extension policy all come from here. + var app_config = config_file.load(alloc, io, init.environ_map, cwd) catch |err| { + reportConfigError(err); + std.process.exit(1); + }; + defer app_config.deinit(); + + // Load models.toml — model aliases (wire name + knobs) and pricing. + // A missing file is fine (empty registries); cost lookups return + // null, formatted as "unknown" by the display layer. const models_toml_path = try models_toml.configPath(alloc, init.environ_map); defer alloc.free(models_toml_path); - var pricing_registry = try models_toml.loadFromPath(alloc, io, models_toml_path); - defer pricing_registry.deinit(); - std.log.debug("models.toml: {d} entries from {s}", .{ pricing_registry.count(), models_toml_path }); + var models = try models_toml.loadFromPath(alloc, io, models_toml_path); + defer models.deinit(); + std.log.debug( + "models.toml: {d} model def(s), {d} price entr(ies) from {s}", + .{ models.defs.count(), models.pricing.count(), models_toml_path }, + ); + // `models.pricing` is parsed and ready; the print-based CLI doesn't + // surface per-turn cost yet (that lands with the TUI frontend). - const banner_model_initial: []const u8 = switch (config) { + // Choose the active model and assemble the provider config. + const model_ref = app_config.selectModel(&models.defs, null) catch |err| { + reportModelError(err, &app_config); + std.process.exit(1); + }; + const provider_config = config_file.buildProviderConfig(&app_config, &models.defs, model_ref) catch |err| { + reportModelError(err, &app_config); + std.process.exit(1); + }; + + // Process-global HTTP client: one connection pool for every provider / + // base_url the agent may switch to. Torn down at shutdown. + panto.config.initHttp(alloc, io); + defer panto.config.deinitHttp(); + + const banner_model_initial: []const u8 = switch (provider_config) { inline else => |c| c.model, }; - const banner_provider_initial: []const u8 = @tagName(config); + const banner_provider_initial: []const u8 = model_ref.provider; // Create or resume the session. Resume failures (missing/ambiguous id) // are user errors — print a tidy message and exit 1 rather than @@ -365,9 +379,11 @@ pub fn main(init: std.process.Init) !void { try appendSystemToSession(alloc, &session_mgr, system_text); } - const prov = try panto.provider.Provider.init(alloc, io, config); - var agent = panto.agent.Agent.init(alloc, io, prov); - defer agent.deinit(); + // The tool registry is owned here and referenced by the active config + // snapshot. Extensions register their tool source into it *before* the + // snapshot is built, so the agent's first turn sees them. + var registry = panto.ToolRegistry.init(alloc); + defer registry.deinit(); // Spin up the long-lived Lua runtime. All Lua extensions load into // one `lua_State`; module-global state survives across calls. The @@ -394,10 +410,10 @@ pub fn main(init: std.process.Init) !void { // chance to register tools that might want to yield. try rt.installScheduler(); - // Discover Lua extensions across three layers — system + // Discover Lua extensions across three layers — base // ($PANTO_HOME/agent), user ($XDG_CONFIG_HOME/panto or // $HOME/.config/panto), and project (./.panto). Project shadows - // user shadows system; tool-name collisions across surviving + // user shadows base; tool-name collisions across surviving // entries abort startup. const n_ext_tools = extension_loader.discoverAndLoad( alloc, @@ -405,6 +421,10 @@ pub fn main(init: std.process.Init) !void { init.environ_map, luarocks_rt.layout.agent_dir, rt, + .{ + .extensions = &app_config.extensions, + .tools = &app_config.tools, + }, ) catch |err| { std.log.err("extension discovery failed: {t}", .{err}); return err; @@ -412,10 +432,20 @@ pub fn main(init: std.process.Init) !void { std.log.debug("extensions: {d} tool(s) registered", .{n_ext_tools}); if (n_ext_tools > 0) { - try agent.registerToolSource(rt.toolSource()); + try registry.registerSource(rt.toolSource()); } - const banner_base: []const u8 = switch (config) { + // Assemble the active configuration snapshot (provider + tool registry) + // and create the agent holding a pointer to it. Swapping this pointer + // later (model/provider/tool changes) takes effect at the next turn. + const active_config: panto.config.Config = .{ + .provider = provider_config, + .registry = ®istry, + }; + var agent = panto.agent.Agent.init(alloc, io, &active_config); + defer agent.deinit(); + + const banner_base: []const u8 = switch (provider_config) { inline else => |c| c.base_url, }; try stdout.print( diff --git a/src/models_toml.zig b/src/models_toml.zig index 5ccb302..36d8c65 100644 --- a/src/models_toml.zig +++ b/src/models_toml.zig @@ -1,46 +1,46 @@ -//! Loader for `~/.config/panto/models.toml`. +//! Loader for `models.toml` — model aliases and per-model pricing. //! -//! Schema: +//! A model entry is keyed by `<provider_name>.<alias>`, where +//! `<provider_name>` matches a provider defined in `config.toml`'s +//! `[providers.<name>]` table, and `<alias>` is the short name the user +//! references via `<provider_name>:<alias>` (e.g. `anthropic:sonnet`). //! -//! [<provider>.<model>] -//! input = <float> # USD per million tokens (fresh input) -//! output = <float> # USD per million tokens -//! cache_read = <float> # USD per million tokens (optional; default "unknown") -//! cache_write = <float> # USD per million tokens (optional; default "unknown") +//! Schema: //! -//! All four price fields are optional at the parse layer. Any field -//! omitted from the TOML comes through as `null` in the in-memory -//! `Pricing`, which means "unknown price for this token category" — -//! NOT "zero." If the model later reports usage in an unknown -//! category (e.g. gpt-4o reads from prompt cache but the TOML omitted -//! `cache_read`), session cost degenerates to "unknown" rather than -//! silently treating that usage as free. To declare a category as a -//! known zero (e.g. OpenAI doesn't bill a cache-write rate), write -//! `cache_write = 0` explicitly. +//! [<provider>.<alias>] +//! model = <string> # wire model id; defaults to <alias> if omitted +//! reasoning = <string> # default | off | minimal | low | medium | high +//! max_tokens = <int> # anthropic_messages only +//! api_version = <string> # anthropic_messages only +//! # pricing (all optional; USD per million tokens): +//! input = <float> +//! output = <float> +//! cache_read = <float> +//! cache_write = <float> //! -//! `<provider>` is `openai` or `anthropic` (matching pantograph's API -//! styles). `<model>` is the model id pantograph sends to the API. Both -//! are TOML "dotted-key" path segments; quote them if they contain -//! characters TOML doesn't allow bare (`-`, `.`, `:` etc. — `-` is OK -//! bare; `.` requires quoting since dots separate path segments). +//! Pricing semantics are unchanged from the original models.toml: any +//! omitted price field is "unknown" (null), not zero. Write `= 0` +//! explicitly to declare a known-zero rate. //! //! Example: //! -//! [anthropic."claude-sonnet-4-20250514"] +//! [anthropic.sonnet] +//! model = "claude-sonnet-4-20250514" +//! reasoning = "high" +//! max_tokens = 8192 //! input = 3.0 //! output = 15.0 //! cache_read = 0.3 //! cache_write = 3.75 //! -//! [openai.gpt-4o] +//! [openai.gpt] +//! model = "gpt-4o" //! input = 2.5 //! output = 10.0 //! -//! The model id in the section header may also be unquoted when it -//! contains no `.` or other reserved chars (e.g. `gpt-4o`); pantograph -//! reads either form. We do not currently bake in default pricing — -//! a missing entry (or a present entry with missing fields) surfaces -//! as "unknown cost" and the display layer formats accordingly. +//! A referenced alias with no entry here is not an error: the alias is +//! used verbatim as the wire model name (zero-config convenience), with +//! default knobs and unknown pricing. const std = @import("std"); const Allocator = std.mem.Allocator; @@ -51,6 +51,71 @@ const panto = @import("panto"); pub const Pricing = panto.pricing.Pricing; pub const Registry = panto.pricing.Registry; +pub const ReasoningEffort = panto.config.ReasoningEffort; + +/// A resolved model definition: the wire model id plus per-model knobs. +/// Strings are owned by the `ModelRegistry`. +pub const ModelDef = struct { + /// `<provider_name>` — matches a `[providers.<name>]` in config.toml. + provider: []const u8, + /// `<alias>` — the short reference name. + alias: []const u8, + /// Wire model id sent to the provider API. Equals `alias` when the + /// TOML omitted `model`. + model: []const u8, + reasoning: ReasoningEffort, + /// anthropic_messages only; null = use the provider/library default. + max_tokens: ?u32, + /// anthropic_messages only; null = use the library default. + api_version: ?[]const u8, +}; + +/// In-memory map of `(provider, alias) -> ModelDef`. Owns all strings. +pub const ModelRegistry = struct { + allocator: Allocator, + entries: std.ArrayList(ModelDef), + + pub fn init(allocator: Allocator) ModelRegistry { + return .{ .allocator = allocator, .entries = .empty }; + } + + pub fn deinit(self: *ModelRegistry) void { + for (self.entries.items) |e| { + self.allocator.free(e.provider); + self.allocator.free(e.alias); + self.allocator.free(e.model); + if (e.api_version) |v| self.allocator.free(v); + } + self.entries.deinit(self.allocator); + } + + pub fn count(self: *const ModelRegistry) usize { + return self.entries.items.len; + } + + /// Look up a model definition by provider name + alias. + pub fn get(self: *const ModelRegistry, provider: []const u8, alias: []const u8) ?ModelDef { + for (self.entries.items) |e| { + if (std.mem.eql(u8, e.provider, provider) and std.mem.eql(u8, e.alias, alias)) { + return e; + } + } + return null; + } +}; + +/// Everything `models.toml` yields: model definitions and pricing. The +/// two registries are independent (pricing keys on the *wire* model id; +/// model defs key on the alias) but parsed in one pass. +pub const Models = struct { + defs: ModelRegistry, + pricing: Registry, + + pub fn deinit(self: *Models) void { + self.defs.deinit(); + self.pricing.deinit(); + } +}; /// Resolve the absolute path to `models.toml`. Honors `XDG_CONFIG_HOME`, /// falling back to `$HOME/.config`. Caller owns the returned slice. @@ -64,21 +129,17 @@ pub fn configPath(allocator: Allocator, environ_map: *const std.process.Environ. return error.NoHomeDirectory; } -/// Load `models.toml` into a fresh `Registry`. If the file does not -/// exist, returns an empty registry (no error — missing config is fine). -/// -/// On parse errors, returns `error.InvalidModelsToml` after logging the -/// line/column of the first error. -pub fn loadFromPath( - allocator: Allocator, - io: Io, - path: []const u8, -) !Registry { - var reg = Registry.init(allocator); - errdefer reg.deinit(); +/// Load `models.toml` from `path`. A missing file yields empty registries +/// (no error). Parse errors return `error.InvalidModelsToml`. +pub fn loadFromPath(allocator: Allocator, io: Io, path: []const u8) !Models { + var models: Models = .{ + .defs = ModelRegistry.init(allocator), + .pricing = Registry.init(allocator), + }; + errdefer models.deinit(); const file = Io.Dir.cwd().openFile(io, path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => return reg, // empty registry; perfectly fine. + error.FileNotFound => return models, // empty; perfectly fine. else => return err, }; defer file.close(io); @@ -88,18 +149,16 @@ pub fn loadFromPath( defer allocator.free(bytes); _ = try file.readPositionalAll(io, bytes, 0); - try parseInto(®, bytes); - return reg; + try parseInto(&models, bytes); + return models; } -/// Parse a TOML string into the given registry. Useful for tests. -pub fn parseInto(reg: *Registry, source: []const u8) !void { - const alloc = reg.allocator; +/// Parse a TOML string into the given registries. Useful for tests. +pub fn parseInto(models: *Models, source: []const u8) !void { + const alloc = models.defs.allocator; const result = toml.parseWithError(alloc, source, .{}); switch (result) { .err => |e| { - // Silenced in tests so the test runner doesn't flag the - // expected-failure case as a real error. if (!@import("builtin").is_test) { std.log.err( "models.toml: parse error at line {d}, column {d}: {s}", @@ -113,31 +172,98 @@ pub fn parseInto(reg: *Registry, source: []const u8) !void { var d = doc; d.deinit(); } - try ingestDocument(reg, doc); + try ingestDocument(models, doc); }, } } -fn ingestDocument(reg: *Registry, doc: *toml.Document) !void { - // Root is a table of provider -> table of model -> { input, output, ... }. +fn ingestDocument(models: *Models, doc: *toml.Document) !void { const root_val: *const toml.Value = doc.root; if (root_val.* != .table) return; + var provider_it = toml.tableIterator(root_val); while (provider_it.next()) |provider_entry| { const provider = provider_entry.key; const provider_val: *const toml.Value = provider_entry.value; if (provider_val.* != .table) continue; - var model_it = toml.tableIterator(provider_val); - while (model_it.next()) |model_entry| { - const model = model_entry.key; - const v: *const toml.Value = model_entry.value; + + var alias_it = toml.tableIterator(provider_val); + while (alias_it.next()) |alias_entry| { + const alias = alias_entry.key; + const v: *const toml.Value = alias_entry.value; if (v.* != .table) continue; - const pricing = pricingFromValue(v); - try reg.set(provider, model, pricing); + + try ingestModel(models, provider, alias, v); } } } +fn ingestModel( + models: *Models, + provider: []const u8, + alias: []const u8, + v: *const toml.Value, +) !void { + const alloc = models.defs.allocator; + + // Wire model id: explicit `model`, else the alias itself. + const wire_model: []const u8 = blk: { + if (v.get("model")) |m| { + if (m.asString()) |s| break :blk s; + } + break :blk alias; + }; + + const reasoning: ReasoningEffort = blk: { + if (v.get("reasoning")) |r| { + if (r.asString()) |s| { + break :blk std.meta.stringToEnum(ReasoningEffort, s) orelse .default; + } + } + break :blk .default; + }; + + const max_tokens: ?u32 = blk: { + if (v.get("max_tokens")) |mt| { + if (mt.asI64()) |i| { + if (i > 0) break :blk @intCast(i); + } + } + break :blk null; + }; + + const api_version_src: ?[]const u8 = blk: { + if (v.get("api_version")) |av| { + if (av.asString()) |s| break :blk s; + } + break :blk null; + }; + + // Own all the strings. + const provider_copy = try alloc.dupe(u8, provider); + errdefer alloc.free(provider_copy); + const alias_copy = try alloc.dupe(u8, alias); + errdefer alloc.free(alias_copy); + const model_copy = try alloc.dupe(u8, wire_model); + errdefer alloc.free(model_copy); + const api_version_copy: ?[]const u8 = if (api_version_src) |s| try alloc.dupe(u8, s) else null; + errdefer if (api_version_copy) |s| alloc.free(s); + + try models.defs.entries.append(alloc, .{ + .provider = provider_copy, + .alias = alias_copy, + .model = model_copy, + .reasoning = reasoning, + .max_tokens = max_tokens, + .api_version = api_version_copy, + }); + + // Pricing keys on the *wire* model id so usage records (which carry + // the wire model) resolve directly. + const pricing = pricingFromValue(v); + try models.pricing.set(provider, wire_model, pricing); +} + fn pricingFromValue(v: *const toml.Value) Pricing { return .{ .input = readPriceField(v, "input"), @@ -148,12 +274,9 @@ fn pricingFromValue(v: *const toml.Value) Pricing { } /// Returns `null` if the field is absent or has a type we can't -/// interpret as a price. An explicit `0` (or `0.0`) comes through as a -/// known zero, not `null` — callers rely on that distinction. +/// interpret as a price. An explicit `0` comes through as a known zero. fn readPriceField(table: *const toml.Value, name: []const u8) ?u64 { const field = table.get(name) orelse return null; - // Accept either floats (the natural form) or integers (the user - // wrote `3` instead of `3.0`). if (field.asF64()) |f| return Pricing.fromDollarsPerMtok(f); if (field.asI64()) |i| return Pricing.fromDollarsPerMtok(@floatFromInt(i)); return null; @@ -165,108 +288,116 @@ fn readPriceField(table: *const toml.Value, name: []const u8) ?u64 { const testing = std.testing; -test "parseInto: two providers, multiple models" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +fn emptyModels(a: Allocator) Models { + return .{ .defs = ModelRegistry.init(a), .pricing = Registry.init(a) }; +} + +test "parseInto: model defs carry wire name and knobs, keyed by provider.alias" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = - \\[anthropic."claude-sonnet-4-20250514"] + \\[anthropic.sonnet] + \\model = "claude-sonnet-4-20250514" + \\reasoning = "high" + \\max_tokens = 8192 \\input = 3.0 \\output = 15.0 - \\cache_read = 0.3 - \\cache_write = 3.75 \\ - \\[openai.gpt-4o] + \\[openai.gpt] + \\model = "gpt-4o" \\input = 2.5 \\output = 10.0 - \\cache_read = 1.25 - \\ - \\[openai.gpt-4o-mini] - \\input = 0.15 - \\output = 0.6 ; - try parseInto(®, src); + try parseInto(&models, src); - const anth = reg.get("anthropic", "claude-sonnet-4-20250514").?; - try testing.expectEqual(@as(?u64, 300), anth.input); - try testing.expectEqual(@as(?u64, 1500), anth.output); - try testing.expectEqual(@as(?u64, 30), anth.cache_read); - try testing.expectEqual(@as(?u64, 375), anth.cache_write); + const sonnet = models.defs.get("anthropic", "sonnet").?; + try testing.expectEqualStrings("claude-sonnet-4-20250514", sonnet.model); + try testing.expectEqual(ReasoningEffort.high, sonnet.reasoning); + try testing.expectEqual(@as(?u32, 8192), sonnet.max_tokens); - const oa = reg.get("openai", "gpt-4o").?; - try testing.expectEqual(@as(?u64, 250), oa.input); - try testing.expectEqual(@as(?u64, 1000), oa.output); - try testing.expectEqual(@as(?u64, 125), oa.cache_read); - // cache_write absent in source — stays unknown, NOT silently 0. - try testing.expectEqual(@as(?u64, null), oa.cache_write); + const gpt = models.defs.get("openai", "gpt").?; + try testing.expectEqualStrings("gpt-4o", gpt.model); + try testing.expectEqual(ReasoningEffort.default, gpt.reasoning); + try testing.expectEqual(@as(?u32, null), gpt.max_tokens); - const mini = reg.get("openai", "gpt-4o-mini").?; - try testing.expectEqual(@as(?u64, 15), mini.input); - try testing.expectEqual(@as(?u64, 60), mini.output); - try testing.expectEqual(@as(?u64, null), mini.cache_read); - try testing.expectEqual(@as(?u64, null), mini.cache_write); + // Pricing keyed on the wire model id. + const sonnet_price = models.pricing.get("anthropic", "claude-sonnet-4-20250514").?; + try testing.expectEqual(@as(?u64, 300), sonnet_price.input); + try testing.expectEqual(@as(?u64, 1500), sonnet_price.output); } -test "parseInto: integer values are accepted (user writes `3` not `3.0`)" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +test "parseInto: omitted model defaults the wire name to the alias" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = \\[openai.gpt-4o] - \\input = 3 - \\output = 15 + \\input = 2.5 + \\output = 10.0 ; - try parseInto(®, src); - const p = reg.get("openai", "gpt-4o").?; - try testing.expectEqual(@as(?u64, 300), p.input); - try testing.expectEqual(@as(?u64, 1500), p.output); + try parseInto(&models, src); + + const def = models.defs.get("openai", "gpt-4o").?; + try testing.expectEqualStrings("gpt-4o", def.model); } -test "parseInto: missing optional fields stay null (unknown, not zero)" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +test "parseInto: pricing omissions stay null (unknown, not zero)" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = - \\[openai.gpt-4o] + \\[openai.gpt] + \\model = "gpt-4o" \\input = 2.5 \\output = 10.0 ; - try parseInto(®, src); - const p = reg.get("openai", "gpt-4o").?; + try parseInto(&models, src); + const p = models.pricing.get("openai", "gpt-4o").?; try testing.expectEqual(@as(?u64, null), p.cache_read); try testing.expectEqual(@as(?u64, null), p.cache_write); } -test "parseInto: explicit 0 is a known zero, distinct from omission" { - // OpenAI doesn't charge for cache writes. Writing `cache_write = 0` - // in the TOML must produce a known 0 — not null — so cost stays - // computable on turns that report cache_write usage. - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +test "parseInto: explicit cache_write = 0 is a known zero" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = - \\[openai.gpt-4o] + \\[openai.gpt] + \\model = "gpt-4o" \\input = 2.5 \\output = 10.0 - \\cache_read = 1.25 \\cache_write = 0 ; - try parseInto(®, src); - const p = reg.get("openai", "gpt-4o").?; + try parseInto(&models, src); + const p = models.pricing.get("openai", "gpt-4o").?; try testing.expectEqual(@as(?u64, 0), p.cache_write); } test "parseInto: malformed TOML returns InvalidModelsToml" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); - try testing.expectError(error.InvalidModelsToml, parseInto(®, "this is not valid toml = =")); + var models = emptyModels(testing.allocator); + defer models.deinit(); + try testing.expectError(error.InvalidModelsToml, parseInto(&models, "not valid = =")); } -test "loadFromPath: missing file returns empty registry, no error" { +test "loadFromPath: missing file returns empty registries, no error" { const io = testing.io; - var reg = try loadFromPath(testing.allocator, io, "/nonexistent/path/models.toml"); - defer reg.deinit(); - try testing.expectEqual(@as(usize, 0), reg.count()); + var models = try loadFromPath(testing.allocator, io, "/nonexistent/models.toml"); + defer models.deinit(); + try testing.expectEqual(@as(usize, 0), models.defs.count()); + try testing.expectEqual(@as(usize, 0), models.pricing.count()); +} + +test "ModelRegistry.get: returns null for unknown alias" { + var models = emptyModels(testing.allocator); + defer models.deinit(); + const src = + \\[anthropic.sonnet] + \\model = "claude-sonnet-4-20250514" + ; + try parseInto(&models, src); + try testing.expect(models.defs.get("anthropic", "opus") == null); + try testing.expect(models.defs.get("openai", "sonnet") == null); } test "configPath: XDG_CONFIG_HOME wins" { diff --git a/src/panto_home.zig b/src/panto_home.zig index 353e54b..e75d14b 100644 --- a/src/panto_home.zig +++ b/src/panto_home.zig @@ -30,10 +30,10 @@ pub const Layout = struct { allocator: Allocator, /// `$PANTO_HOME` itself. home: []u8, - /// `$PANTO_HOME/agent/` — the "system" extension/tool tree, + /// `$PANTO_HOME/agent/` — the "base" extension/tool tree, /// populated at bootstrap from files embedded into the panto /// binary. Searched after user/project layers for tools and - /// extensions; project shadows user shadows system. + /// extensions; project shadows user shadows base. agent_dir: []u8, /// `$PANTO_HOME/rocks/lua-<lua_version>/` — the versioned tree. tree: []u8, diff --git a/src/subcommand.zig b/src/subcommand.zig index f6c3058..c50429e 100644 --- a/src/subcommand.zig +++ b/src/subcommand.zig @@ -102,11 +102,17 @@ fn printHelp(io: Io) !void { \\ panto lua [args...] Drop into the embedded Lua interpreter. \\ panto help Show this message. \\ + \\Configuration (TOML, merged base → user → project): + \\ $XDG_DATA_HOME/panto/config.toml (base; auto-generated) + \\ $XDG_CONFIG_HOME/panto/config.toml (user) + \\ ./.panto/config.toml (project) + \\ Define providers under [providers.<name>], pick a default with + \\ [defaults] model = "<provider>:<alias>", and gate tools/extensions + \\ with [tools]/[extensions] allow/deny globs. Model aliases (wire name, + \\ reasoning, max_tokens, pricing) live in models.toml. + \\ \\Environment: - \\ PANTO_API_STYLE "openai_chat" (default) or "anthropic_messages". - \\ OPENAI_API_KEY, OPENAI_MODEL, OPENAI_BASE_URL, OPENAI_REASONING - \\ ANTHROPIC_API_KEY, ANTHROPIC_MODEL, ANTHROPIC_BASE_URL, - \\ ANTHROPIC_API_VERSION, ANTHROPIC_MAX_TOKENS + \\ OPENAI_API_KEY, ANTHROPIC_API_KEY Consumed by the default providers. \\ PANTO_SESSION_DIR Override the base sessions directory. Defaults to \\ $XDG_DATA_HOME/panto/sessions or ~/.local/share/panto/sessions. \\ PANTO_HOME Override the runtime/rocks tree location. |