diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/config_file.zig | 977 | ||||
| -rw-r--r-- | src/extension_loader.zig | 223 | ||||
| -rw-r--r-- | src/glob.zig | 121 | ||||
| -rw-r--r-- | src/lua_runtime.zig | 32 | ||||
| -rw-r--r-- | src/luarocks_runtime.zig | 66 | ||||
| -rw-r--r-- | src/main.zig | 196 | ||||
| -rw-r--r-- | src/models_toml.zig | 375 | ||||
| -rw-r--r-- | src/panto_home.zig | 4 | ||||
| -rw-r--r-- | src/subcommand.zig | 14 |
9 files changed, 1753 insertions, 255 deletions
diff --git a/src/config_file.zig b/src/config_file.zig new file mode 100644 index 0000000..533eb48 --- /dev/null +++ b/src/config_file.zig @@ -0,0 +1,977 @@ +//! Layered `config.toml` loader for the panto CLI. +//! +//! Four files are read and merged, lowest precedence first: +//! +//! 1. base — `${XDG_DATA_HOME:-$HOME/.local/share}/panto/config.toml` +//! 2. user — `${XDG_CONFIG_HOME:-$HOME/.config}/panto/config.toml` +//! 3. project — `./.panto/config.toml` +//! 4. local — `./.panto/config.local.toml` +//! +//! The `local` layer is intended to be git-ignored: it lets an individual +//! developer apply more-opinionated overrides on top of the team-shared +//! `project` layer without committing them. +//! +//! Merge semantics: **tables merge recursively; scalars and arrays from a +//! higher-precedence layer overwrite wholesale.** A project file that sets +//! `tools.deny = [...]` replaces the array entirely — it does not append to +//! a user-level `tools.deny`. Tables (notably `providers`) accumulate: a +//! provider defined only at the base layer survives even when the project +//! layer adds a different provider. +//! +//! After merge, the document is resolved into a `Config`: +//! +//! - Every `providers.<name>` becomes a `Provider`. Its API key is taken +//! from `api_key` if present, else from the environment variable named +//! by `api_key_env_var`. A provider whose only key source is an absent +//! env var is **silently dropped** — this is what lets the base layer +//! ship default OpenAI/Anthropic providers that simply don't appear when +//! the user hasn't exported the corresponding key. +//! - `defaults.model` is parsed as `<provider_name>:<model_alias>`. +//! - `tools` / `extensions` allow- and deny-lists are captured verbatim +//! (as glob pattern lists). A pattern appearing in both allow and deny +//! for the same kind is a hard error. +//! +//! Model *aliases* (the `<model_alias>` half of a model reference) are +//! resolved separately against `models.toml` — this module only validates +//! that the reference names a known provider. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; + +const toml = @import("toml"); +const panto = @import("panto"); + +const glob = @import("glob.zig"); +const models_toml = @import("models_toml.zig"); + +// Document-building helpers live in the toml library's `value` module and +// are not re-exported at its root. `tableIterator` *is* re-exported, so we +// use `toml.tableIterator` directly but reach through `value_mod` for the +// constructors/mutators. +const tvalue = toml.value_mod; + +pub const APIStyle = panto.config.APIStyle; + +// =========================================================================== +// Resolved config model +// =========================================================================== + +/// One fully-resolved provider. The API key has already been read (from the +/// inline value or the named env var); a provider that never resolved a key +/// is absent from `Config.providers` entirely. +pub const Provider = struct { + /// The user-chosen provider name (the `providers.<name>` key). This is + /// the name used on the left of a `<provider>:<model>` reference. + name: []const u8, + style: APIStyle, + base_url: []const u8, + api_key: []const u8, + + pub fn deinit(self: Provider, alloc: Allocator) void { + alloc.free(self.name); + alloc.free(self.base_url); + alloc.free(self.api_key); + } +}; + +/// A parsed `<provider>:<model>` reference. Both halves borrow from the +/// owning `Config` (the `default_model` storage); do not free separately. +pub const ModelRef = struct { + provider: []const u8, + model: []const u8, +}; + +/// Tool/extension availability policy. Empty `allow` means "allow all" +/// (subject to `deny`). A name is permitted when it matches at least one +/// allow pattern (or allow is empty) AND matches no deny pattern. +pub const Policy = struct { + allow: [][]const u8, + deny: [][]const u8, + + pub fn deinit(self: Policy, alloc: Allocator) void { + for (self.allow) |p| alloc.free(p); + alloc.free(self.allow); + for (self.deny) |p| alloc.free(p); + alloc.free(self.deny); + } + + /// True if `name` is permitted under this policy. + pub fn permits(self: Policy, name: []const u8) bool { + for (self.deny) |pat| { + if (glob.match(pat, name)) return false; + } + if (self.allow.len == 0) return true; + for (self.allow) |pat| { + if (glob.match(pat, name)) return true; + } + return false; + } +}; + +/// Build a `panto.config.ProviderConfig` for a chosen `<provider>:<alias>` model +/// reference, combining the resolved provider (transport/auth) with the +/// model definition from `models.toml` (wire name + knobs). +/// +/// `ref` selects the provider and alias. `defs` supplies per-model knobs; +/// a missing alias falls back to using the alias verbatim as the wire +/// model name with default knobs. Returns the assembled `Config` plus the +/// wire model id (borrowed from `defs`/`ref` — valid as long as both +/// outlive the returned config's use). The `panto.config.ProviderConfig` itself +/// borrows the provider/model strings; the caller must keep `cfg` (the +/// `Config`) and `defs` alive for its lifetime. +pub fn buildProviderConfig( + cfg: *const Config, + defs: *const models_toml.ModelRegistry, + ref: ModelRef, +) ResolveError!panto.config.ProviderConfig { + const prov = cfg.provider(ref.provider) orelse return error.UnknownProvider; + + const def_opt = defs.get(ref.provider, ref.model); + const wire_model: []const u8 = if (def_opt) |d| d.model else ref.model; + const reasoning: panto.config.ReasoningEffort = if (def_opt) |d| d.reasoning else .default; + + switch (prov.style) { + .openai_chat => return .{ .openai_chat = .{ + .api_key = prov.api_key, + .base_url = prov.base_url, + .model = wire_model, + .reasoning = reasoning, + } }, + .anthropic_messages => return .{ .anthropic_messages = .{ + .api_key = prov.api_key, + .base_url = prov.base_url, + .model = wire_model, + .api_version = if (def_opt) |d| (d.api_version orelse "2023-06-01") else "2023-06-01", + .max_tokens = if (def_opt) |d| (d.max_tokens orelse 4096) else 4096, + } }, + } +} + +pub const Config = struct { + allocator: Allocator, + providers: []Provider, + /// `defaults.model` storage, owned. `default_model_ref` slices into it. + default_model: ?[]const u8, + default_model_ref: ?ModelRef, + tools: Policy, + extensions: Policy, + + pub fn deinit(self: *Config) void { + for (self.providers) |p| p.deinit(self.allocator); + self.allocator.free(self.providers); + if (self.default_model) |m| self.allocator.free(m); + self.tools.deinit(self.allocator); + self.extensions.deinit(self.allocator); + } + + /// Look up a resolved provider by name. Returns null if absent (e.g. + /// dropped because its env var wasn't set). + pub fn provider(self: *const Config, name: []const u8) ?*const Provider { + for (self.providers) |*p| { + if (std.mem.eql(u8, p.name, name)) return p; + } + return null; + } + + /// Choose the model reference to start with. Precedence: + /// 1. an explicit `model_override` (e.g. a future `--model` flag), + /// 2. `defaults.model` from config, + /// 3. if exactly one provider resolved AND it has exactly one model + /// alias in `defs`, use that, + /// 4. otherwise error (`NoModelSelected`). + /// + /// The returned ref borrows from `self`/`defs`/`model_override`. + pub fn selectModel( + self: *const Config, + defs: *const models_toml.ModelRegistry, + model_override: ?[]const u8, + ) ResolveError!ModelRef { + if (model_override) |m| { + return parseModelRef(m) catch return error.NoModelSelected; + } + if (self.default_model_ref) |ref| return ref; + + // Single-provider, single-alias convenience. + if (self.providers.len == 1) { + const only = self.providers[0].name; + var found: ?ModelRef = null; + for (defs.entries.items) |d| { + if (std.mem.eql(u8, d.provider, only)) { + if (found != null) { + found = null; // more than one alias; ambiguous. + break; + } + found = .{ .provider = d.provider, .model = d.alias }; + } + } + if (found) |ref| return ref; + } + return error.NoModelSelected; + } +}; + +pub const Error = error{ + NoHomeDirectory, + InvalidConfigToml, + InvalidProvider, + InvalidModelRef, + UnknownDefaultProvider, + PolicyConflict, +} || Allocator.Error || FileError; + +pub const ResolveError = error{ + NoModelSelected, + UnknownProvider, +}; + +/// The filesystem errors that can surface while reading a config layer. +/// `FileNotFound` is handled by the caller (skip the layer); any other +/// I/O failure is collapsed to `error.ConfigReadFailed` so this module +/// keeps a small, stable error set. +pub const FileError = Io.Cancelable || error{ FileNotFound, ConfigReadFailed }; + +// =========================================================================== +// Public entry points +// =========================================================================== + +/// Resolve and merge the four config layers from their standard paths, +/// then build a `Config`. Missing files are skipped silently. `cwd` is the +/// project root (used for the `./.panto/config.toml` and +/// `./.panto/config.local.toml` layers). +pub fn load( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, +) Error!Config { + const paths = try layerPaths(allocator, environ_map, cwd); + defer paths.deinit(allocator); + + return loadFromPaths(allocator, io, environ_map, &.{ + paths.base, + paths.user, + paths.project, + paths.local, + }); +} + +const LayerPaths = struct { + base: []u8, + user: []u8, + project: []u8, + local: []u8, + + fn deinit(self: LayerPaths, alloc: Allocator) void { + alloc.free(self.base); + alloc.free(self.user); + alloc.free(self.project); + alloc.free(self.local); + } +}; + +fn layerPaths( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, +) Error!LayerPaths { + const base = try baseConfigPath(allocator, environ_map); + errdefer allocator.free(base); + const user = try userConfigPath(allocator, environ_map); + errdefer allocator.free(user); + const project = try std.fs.path.join(allocator, &.{ cwd, ".panto", "config.toml" }); + errdefer allocator.free(project); + const local = try std.fs.path.join(allocator, &.{ cwd, ".panto", "config.local.toml" }); + return .{ .base = base, .user = user, .project = project, .local = local }; +} + +/// `$PANTO_HOME/config.toml`, falling back to +/// `${XDG_DATA_HOME:-$HOME/.local/share}/panto/config.toml`. +/// +/// `PANTO_HOME` is checked first so this always agrees with where the +/// bootstrap stages the default base config (see `panto_home.resolveHome`). +pub fn baseConfigPath(allocator: Allocator, environ_map: *const std.process.Environ.Map) Error![]u8 { + if (environ_map.get("PANTO_HOME")) |home| { + return try std.fs.path.join(allocator, &.{ home, "config.toml" }); + } + if (environ_map.get("XDG_DATA_HOME")) |xdg| { + return try std.fs.path.join(allocator, &.{ xdg, "panto", "config.toml" }); + } + if (environ_map.get("HOME")) |home| { + return try std.fs.path.join(allocator, &.{ home, ".local", "share", "panto", "config.toml" }); + } + return error.NoHomeDirectory; +} + +/// `${XDG_CONFIG_HOME:-$HOME/.config}/panto/config.toml`. +pub fn userConfigPath(allocator: Allocator, environ_map: *const std.process.Environ.Map) Error![]u8 { + if (environ_map.get("XDG_CONFIG_HOME")) |xdg| { + return try std.fs.path.join(allocator, &.{ xdg, "panto", "config.toml" }); + } + if (environ_map.get("HOME")) |home| { + return try std.fs.path.join(allocator, &.{ home, ".config", "panto", "config.toml" }); + } + return error.NoHomeDirectory; +} + +/// Load from explicit layer paths, lowest precedence first. Useful for +/// tests. Missing files are skipped. +pub fn loadFromPaths( + allocator: Allocator, + io: Io, + environ_map: *const std.process.Environ.Map, + paths: []const []const u8, +) Error!Config { + // The merged document is built into its own arena-backed Document so we + // never have to reason about which source layer a given Value came from. + const merged = try tvalue.createDocument(allocator); + defer merged.deinit(); + + for (paths) |path| { + const bytes = readFileAlloc(allocator, io, path) catch |err| switch (err) { + error.FileNotFound => continue, + else => return err, + }; + defer allocator.free(bytes); + + const doc = parseDoc(allocator, bytes) catch return error.InvalidConfigToml; + defer doc.deinit(); + try mergeTable(merged.allocator(), merged.root, doc.root); + } + + return resolve(allocator, environ_map, merged.root); +} + +// =========================================================================== +// Merge +// =========================================================================== + +/// Deep-merge `src` table into `dst` table (both must be `.table`). +/// Tables recurse; everything else (scalars, arrays) overwrites. Values are +/// deep-copied into `alloc` (the merged document's arena) so they outlive +/// the source document. +fn mergeTable(alloc: Allocator, dst: *toml.Value, src: *const toml.Value) Allocator.Error!void { + std.debug.assert(dst.* == .table); + if (src.* != .table) return; + + var it = toml.tableIterator(src); + while (it.next()) |entry| { + const existing = dst.get(entry.key); + if (existing != null and existing.?.* == .table and entry.value.* == .table) { + // Both sides are tables — recurse to accumulate keys. + try mergeTable(alloc, @constCast(existing.?), entry.value); + } else { + // Overwrite (or insert) with a deep copy of the source value. + const copy = try cloneValue(alloc, entry.value); + const key_copy = try alloc.dupe(u8, entry.key); + try tvalue.tableSet(alloc, dst, key_copy, copy); + } + } +} + +fn cloneValue(alloc: Allocator, src: *const toml.Value) Allocator.Error!*toml.Value { + const out = try alloc.create(toml.Value); + switch (src.*) { + .table => { + out.* = .{ .table = .{} }; + var it = toml.tableIterator(src); + while (it.next()) |entry| { + const child = try cloneValue(alloc, entry.value); + const key_copy = try alloc.dupe(u8, entry.key); + try tvalue.tableSet(alloc, out, key_copy, child); + } + }, + .array => |*a| { + out.* = .{ .array = .{} }; + for (a.items.items) |*item| { + const child = try cloneValue(alloc, item); + try tvalue.arrayAppend(alloc, out, child.*); + } + }, + .string => |s| out.* = .{ .string = try alloc.dupe(u8, s) }, + else => out.* = src.*, + } + return out; +} + +// =========================================================================== +// Resolve +// =========================================================================== + +fn resolve( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, + root: *const toml.Value, +) Error!Config { + var providers: std.ArrayList(Provider) = .empty; + errdefer { + for (providers.items) |p| p.deinit(allocator); + providers.deinit(allocator); + } + + if (root.get("providers")) |providers_tbl| { + if (providers_tbl.* == .table) { + var it = toml.tableIterator(providers_tbl); + while (it.next()) |entry| { + const maybe = try resolveProvider(allocator, environ_map, entry.key, entry.value); + if (maybe) |p| try providers.append(allocator, p); + } + } + } + + // Tool / extension policies. + var tools = try resolvePolicy(allocator, root, "tools"); + errdefer tools.deinit(allocator); + var extensions = try resolvePolicy(allocator, root, "extensions"); + errdefer extensions.deinit(allocator); + + // Default model. + var default_model: ?[]const u8 = null; + errdefer if (default_model) |m| allocator.free(m); + var default_model_ref: ?ModelRef = null; + if (root.get("defaults")) |defaults_tbl| { + if (defaults_tbl.get("model")) |model_v| { + if (model_v.* == .string) { + default_model = try allocator.dupe(u8, model_v.string); + default_model_ref = try parseModelRef(default_model.?); + } + } + } + + const providers_slice = try providers.toOwnedSlice(allocator); + errdefer { + for (providers_slice) |p| p.deinit(allocator); + allocator.free(providers_slice); + } + + // Validate the default model names a provider that actually resolved. + if (default_model_ref) |ref| { + var found = false; + for (providers_slice) |p| { + if (std.mem.eql(u8, p.name, ref.provider)) { + found = true; + break; + } + } + if (!found) return error.UnknownDefaultProvider; + } + + return .{ + .allocator = allocator, + .providers = providers_slice, + .default_model = default_model, + .default_model_ref = default_model_ref, + .tools = tools, + .extensions = extensions, + }; +} + +/// Resolve one `providers.<name>` entry. Returns null (provider dropped) if +/// the only key source is an env var that isn't set. +fn resolveProvider( + allocator: Allocator, + environ_map: *const std.process.Environ.Map, + name: []const u8, + val: *const toml.Value, +) Error!?Provider { + if (val.* != .table) return error.InvalidProvider; + + const style_str = (val.get("style") orelse return error.InvalidProvider).asString() orelse + return error.InvalidProvider; + const style = std.meta.stringToEnum(APIStyle, style_str) orelse return error.InvalidProvider; + + const base_url = (val.get("base_url") orelse return error.InvalidProvider).asString() orelse + return error.InvalidProvider; + + // api_key wins over api_key_env_var. + const api_key: []const u8 = blk: { + if (val.get("api_key")) |k| { + if (k.asString()) |s| break :blk s; + } + if (val.get("api_key_env_var")) |ev| { + if (ev.asString()) |env_name| { + if (environ_map.get(env_name)) |actual| break :blk actual; + } + } + // No usable key — silently omit this provider. + return null; + }; + + return .{ + .name = try allocator.dupe(u8, name), + .style = style, + .base_url = try allocator.dupe(u8, base_url), + .api_key = try allocator.dupe(u8, api_key), + }; +} + +fn resolvePolicy(allocator: Allocator, root: *const toml.Value, key: []const u8) Error!Policy { + var allow: [][]const u8 = &.{}; + var deny: [][]const u8 = &.{}; + errdefer { + for (allow) |p| allocator.free(p); + allocator.free(allow); + for (deny) |p| allocator.free(p); + allocator.free(deny); + } + + if (root.get(key)) |tbl| { + if (tbl.* == .table) { + allow = try stringArray(allocator, tbl, "allow"); + deny = try stringArray(allocator, tbl, "deny"); + } + } + + // A pattern present in both allow and deny is contradictory. + for (allow) |a| { + for (deny) |d| { + if (std.mem.eql(u8, a, d)) return error.PolicyConflict; + } + } + + return .{ .allow = allow, .deny = deny }; +} + +fn stringArray(allocator: Allocator, tbl: *const toml.Value, key: []const u8) Error![][]const u8 { + const arr_v = tbl.get(key) orelse return &.{}; + if (arr_v.* != .array) return &.{}; + + var list: std.ArrayList([]const u8) = .empty; + errdefer { + for (list.items) |s| allocator.free(s); + list.deinit(allocator); + } + for (arr_v.array.items.items) |*item| { + if (item.* == .string) { + try list.append(allocator, try allocator.dupe(u8, item.string)); + } + } + return try list.toOwnedSlice(allocator); +} + +/// Parse `<provider>:<model>`. Both halves must be non-empty and there must +/// be exactly one separating colon. +pub fn parseModelRef(ref: []const u8) Error!ModelRef { + const colon = std.mem.indexOfScalar(u8, ref, ':') orelse return error.InvalidModelRef; + const provider = ref[0..colon]; + const model = ref[colon + 1 ..]; + if (provider.len == 0 or model.len == 0) return error.InvalidModelRef; + if (std.mem.indexOfScalar(u8, model, ':') != null) return error.InvalidModelRef; + return .{ .provider = provider, .model = model }; +} + +// =========================================================================== +// File / parse helpers +// =========================================================================== + +fn readFileAlloc(allocator: Allocator, io: Io, path: []const u8) (Allocator.Error || FileError)![]u8 { + const file = Io.Dir.cwd().openFile(io, path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => return error.FileNotFound, + error.Canceled => return error.Canceled, + else => return error.ConfigReadFailed, + }; + defer file.close(io); + const len = file.length(io) catch return error.ConfigReadFailed; + const bytes = try allocator.alloc(u8, @intCast(len)); + errdefer allocator.free(bytes); + _ = file.readPositionalAll(io, bytes, 0) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => return error.ConfigReadFailed, + }; + return bytes; +} + +fn parseDoc(allocator: Allocator, source: []const u8) !*toml.Document { + const result = toml.parseWithError(allocator, source, .{}); + switch (result) { + .err => |e| { + if (!@import("builtin").is_test) { + std.log.err( + "config.toml: parse error at line {d}, column {d}: {s}", + .{ e.line, e.column, e.message }, + ); + } + return error.InvalidConfigToml; + }, + .ok => |doc| return doc, + } +} + +// =========================================================================== +// Tests +// =========================================================================== + +const testing = std.testing; + +fn emptyEnv(a: Allocator) std.process.Environ.Map { + return std.process.Environ.Map.init(a); +} + +test "parseModelRef: splits provider and model" { + const ref = try parseModelRef("anthropic:claude-sonnet-4-6"); + try testing.expectEqualStrings("anthropic", ref.provider); + try testing.expectEqualStrings("claude-sonnet-4-6", ref.model); +} + +test "parseModelRef: rejects malformed refs" { + try testing.expectError(error.InvalidModelRef, parseModelRef("no-colon")); + try testing.expectError(error.InvalidModelRef, parseModelRef(":model")); + try testing.expectError(error.InvalidModelRef, parseModelRef("provider:")); + try testing.expectError(error.InvalidModelRef, parseModelRef("a:b:c")); +} + +test "Policy.permits: empty allow means allow-all minus deny" { + const a = testing.allocator; + var deny = try a.alloc([]const u8, 1); + deny[0] = try a.dupe(u8, "std.shell"); + const policy: Policy = .{ .allow = &.{}, .deny = deny }; + defer policy.deinit(a); + + try testing.expect(policy.permits("std.read")); + try testing.expect(!policy.permits("std.shell")); +} + +test "Policy.permits: allow gates everything not matched" { + const a = testing.allocator; + var allow = try a.alloc([]const u8, 1); + allow[0] = try a.dupe(u8, "std.*"); + const policy: Policy = .{ .allow = allow, .deny = &.{} }; + defer policy.deinit(a); + + try testing.expect(policy.permits("std.read")); + try testing.expect(!policy.permits("other.read")); +} + +test "resolve: env-backed provider is dropped when env var is absent" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + try testing.expectEqual(@as(usize, 0), cfg.providers.len); +} + +test "resolve: env-backed provider resolves when env var is set" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + try env.put("ANTHROPIC_API_KEY", "sk-ant-xyz"); + + const src = + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + try testing.expectEqual(@as(usize, 1), cfg.providers.len); + const p = cfg.provider("anthropic").?; + try testing.expectEqual(APIStyle.anthropic_messages, p.style); + try testing.expectEqualStrings("sk-ant-xyz", p.api_key); +} + +test "resolve: inline api_key wins over env var" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + try env.put("KEY_FROM_ENV", "env-value"); + + const src = + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "inline-value" + \\api_key_env_var = "KEY_FROM_ENV" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + const p = cfg.provider("openai").?; + try testing.expectEqualStrings("inline-value", p.api_key); +} + +test "resolve: unknown default-model provider is an error" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[defaults] + \\model = "ghost:some-model" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + try testing.expectError(error.UnknownDefaultProvider, resolve(a, &env, doc.root)); +} + +test "resolvePolicy: pattern in both allow and deny is a conflict" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[tools] + \\allow = ["std.*"] + \\deny = ["std.*"] + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + + try testing.expectError(error.PolicyConflict, resolve(a, &env, doc.root)); +} + +test "loadFromPaths: layers merge with tables accumulating and scalars overriding" { + const a = testing.allocator; + const io = testing.io; + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_n = try tmp.dir.realPath(io, &path_buf); + const tmp_path = path_buf[0..tmp_n]; + + // Base layer: defines anthropic provider + a default model. + const sys_path = try std.fs.path.join(a, &.{ tmp_path, "base.toml" }); + defer a.free(sys_path); + try tmp.dir.writeFile(io, .{ .sub_path = "base.toml", .data = + \\[defaults] + \\model = "anthropic:sonnet" + \\ + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key = "sys-key" + \\ + \\[tools] + \\allow = ["std.*"] + }); + + // Project layer: adds an openai provider (table accumulates), and + // overrides the default model (scalar overwrites). + const proj_path = try std.fs.path.join(a, &.{ tmp_path, "project.toml" }); + defer a.free(proj_path); + try tmp.dir.writeFile(io, .{ .sub_path = "project.toml", .data = + \\[defaults] + \\model = "openai:gpt" + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "proj-key" + }); + + var env = emptyEnv(a); + defer env.deinit(); + + var cfg = try loadFromPaths(a, io, &env, &.{ sys_path, proj_path }); + defer cfg.deinit(); + + // Both providers survive (table accumulation). + try testing.expectEqual(@as(usize, 2), cfg.providers.len); + try testing.expect(cfg.provider("anthropic") != null); + try testing.expect(cfg.provider("openai") != null); + + // Default model overridden by the project layer. + try testing.expectEqualStrings("openai:gpt", cfg.default_model.?); + try testing.expectEqualStrings("openai", cfg.default_model_ref.?.provider); + try testing.expectEqualStrings("gpt", cfg.default_model_ref.?.model); + + // tools.allow from the base layer is preserved (no project override). + try testing.expect(cfg.tools.permits("std.read")); +} + +test "loadFromPaths: local layer overrides project layer" { + const a = testing.allocator; + const io = testing.io; + + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const tmp_n = try tmp.dir.realPath(io, &path_buf); + const tmp_path = path_buf[0..tmp_n]; + + // Project layer: shared default model + provider. + const proj_path = try std.fs.path.join(a, &.{ tmp_path, "project.toml" }); + defer a.free(proj_path); + try tmp.dir.writeFile(io, .{ .sub_path = "project.toml", .data = + \\[defaults] + \\model = "openai:gpt" + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "proj-key" + }); + + // Local layer (git-ignored): a developer's more-opinionated override of + // the default model. Scalar overwrites; the provider table accumulates. + const local_path = try std.fs.path.join(a, &.{ tmp_path, "local.toml" }); + defer a.free(local_path); + try tmp.dir.writeFile(io, .{ .sub_path = "local.toml", .data = + \\[defaults] + \\model = "openai:gpt-local" + }); + + var env = emptyEnv(a); + defer env.deinit(); + + var cfg = try loadFromPaths(a, io, &env, &.{ proj_path, local_path }); + defer cfg.deinit(); + + // Local layer wins for the default model. + try testing.expectEqualStrings("openai:gpt-local", cfg.default_model.?); + // Provider from the project layer survives (table accumulation). + try testing.expect(cfg.provider("openai") != null); +} + +test "buildProviderConfig: anthropic ref pulls knobs from model defs" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + try env.put("ANTHROPIC_API_KEY", "sk-ant"); + + const src = + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + try models.entries.append(a, .{ + .provider = try a.dupe(u8, "anthropic"), + .alias = try a.dupe(u8, "sonnet"), + .model = try a.dupe(u8, "claude-sonnet-4-20250514"), + .reasoning = .high, + .max_tokens = 8192, + .api_version = try a.dupe(u8, "2023-06-01"), + }); + + const ref = try parseModelRef("anthropic:sonnet"); + const pc = try buildProviderConfig(&cfg, &models, ref); + try testing.expectEqual(APIStyle.anthropic_messages, pc.style()); + try testing.expectEqualStrings("claude-sonnet-4-20250514", pc.anthropic_messages.model); + try testing.expectEqual(@as(u32, 8192), pc.anthropic_messages.max_tokens); + try testing.expectEqualStrings("sk-ant", pc.anthropic_messages.api_key); +} + +test "buildProviderConfig: missing alias uses the alias verbatim as wire model" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + const src = + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "sk-test" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + + const ref = try parseModelRef("openai:gpt-4o"); + const pc = try buildProviderConfig(&cfg, &models, ref); + try testing.expectEqual(APIStyle.openai_chat, pc.style()); + try testing.expectEqualStrings("gpt-4o", pc.openai_chat.model); + try testing.expectEqual(panto.config.ReasoningEffort.default, pc.openai_chat.reasoning); +} + +test "buildProviderConfig: unknown provider errors" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + const empty = try parseDoc(a, ""); + defer empty.deinit(); + var cfg = try resolve(a, &env, empty.root); + defer cfg.deinit(); + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + try testing.expectError(error.UnknownProvider, buildProviderConfig(&cfg, &models, .{ .provider = "ghost", .model = "x" })); +} + +test "selectModel: default_model wins; single-provider fallback otherwise" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + + // Config with one provider and a default model. + const src = + \\[defaults] + \\model = "openai:gpt" + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key = "sk" + ; + const doc = try parseDoc(a, src); + defer doc.deinit(); + var cfg = try resolve(a, &env, doc.root); + defer cfg.deinit(); + + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + + const ref = try cfg.selectModel(&models, null); + try testing.expectEqualStrings("openai", ref.provider); + try testing.expectEqualStrings("gpt", ref.model); + + // Override beats default. + const ref2 = try cfg.selectModel(&models, "openai:other"); + try testing.expectEqualStrings("other", ref2.model); +} + +test "selectModel: no default and no providers errors" { + const a = testing.allocator; + var env = emptyEnv(a); + defer env.deinit(); + const empty = try parseDoc(a, ""); + defer empty.deinit(); + var cfg = try resolve(a, &env, empty.root); + defer cfg.deinit(); + var models = models_toml.ModelRegistry.init(a); + defer models.deinit(); + try testing.expectError(error.NoModelSelected, cfg.selectModel(&models, null)); +} + +test "loadFromPaths: missing files are skipped" { + const a = testing.allocator; + const io = testing.io; + var env = emptyEnv(a); + defer env.deinit(); + + var cfg = try loadFromPaths(a, io, &env, &.{ + "/nonexistent/base.toml", + "/nonexistent/project.toml", + }); + defer cfg.deinit(); + try testing.expectEqual(@as(usize, 0), cfg.providers.len); + try testing.expect(cfg.default_model == null); +} diff --git a/src/extension_loader.zig b/src/extension_loader.zig index 2fd86ae..b84eba5 100644 --- a/src/extension_loader.zig +++ b/src/extension_loader.zig @@ -1,22 +1,22 @@ //! Extension and tool discovery: walk well-known directories, locate Lua //! files, and load each into a long-lived `LuaRuntime`. //! -//! Two parallel namespaces are scanned, each at three scopes — system, -//! user, and project. Project shadows user shadows system. +//! Two parallel namespaces are scanned, each at three scopes — base, +//! user, and project. Project shadows user shadows base. //! //! Extensions (full-featured; call `panto.register_tool` from a script //! that may register many tools): -//! 1. `$PANTO_HOME/agent/extensions/` ("system") +//! 1. `$PANTO_HOME/agent/extensions/` ("base") //! 2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/extensions/` ("user") //! 3. `./.panto/extensions/` ("project") //! //! Tools (ergonomic single-tool form; the script returns one table //! shaped like the argument to `panto.register_tool`): -//! 1. `$PANTO_HOME/agent/tools/` ("system") +//! 1. `$PANTO_HOME/agent/tools/` ("base") //! 2. `${XDG_CONFIG_HOME:-$HOME/.config}/panto/tools/` ("user") //! 3. `./.panto/tools/` ("project") //! -//! The `system` layer is populated at bootstrap from files embedded +//! The `base` layer is populated at bootstrap from files embedded //! into the panto binary (see `build/gen_agent_embed.zig` and //! `luarocks_runtime.stageAgentTree`). It is panto's "batteries" tier: //! ships in the binary, but every entry is individually shadowable @@ -33,7 +33,7 @@ //! Conflict rules: //! - Within one directory, two entries with the same logical name //! are an error. -//! - Project shadows user shadows system *within the same kind* +//! - Project shadows user shadows base *within the same kind* //! (extension or tool). Extensions and tools live in distinct //! namespaces for shadowing; the registered *tool names* still //! share one global namespace across both, and collisions there @@ -44,11 +44,32 @@ const std = @import("std"); const panto = @import("panto"); const lua_runtime = @import("lua_runtime.zig"); +const config_file = @import("config_file.zig"); const Allocator = std.mem.Allocator; const Io = std.Io; const LuaRuntime = lua_runtime.LuaRuntime; +/// Availability policies for the two namespaces. Both are optional; a +/// null policy permits everything. `extensions` gates extension/tool +/// *entries* by their logical (file/dir) name before loading; `tools` +/// gates the *registered tool names* (e.g. `std.read`) after loading. +pub const Policies = struct { + extensions: ?*const config_file.Policy = null, + tools: ?*const config_file.Policy = null, +}; + +fn policyPermits(p: ?*const config_file.Policy, name: []const u8) bool { + const pol = p orelse return true; + return pol.permits(name); +} + +/// Free-function form of `Policy.permits` taking the policy by pointer, +/// matching the `fn(ctx, name) bool` shape `LuaRuntime.filterTools` wants. +fn permitsByPtr(pol: *const config_file.Policy, name: []const u8) bool { + return pol.permits(name); +} + /// A discovered extension or tool before loading. Owns its strings. const Found = struct { /// Logical name (basename without `.lua`, or directory name). @@ -72,13 +93,13 @@ const Found = struct { pub const Source = enum { /// Staged at bootstrap from files embedded in the panto binary. /// Lowest priority — shadowed by user and project. - system, + base, user, project, pub fn label(self: Source) []const u8 { return switch (self) { - .system => "system", + .base => "base", .user => "user", .project => "project", }; @@ -101,9 +122,9 @@ pub const Kind = enum { /// paths into `runtime`. Returns the number of registered tools added /// to the runtime by this call. /// -/// `system_agent_dir`, when non-null, is the path under which embedded -/// system tools/extensions have been staged — typically -/// `$PANTO_HOME/agent/`. Pass `null` to skip the system layer entirely +/// `base_agent_dir`, when non-null, is the path under which embedded +/// base tools/extensions have been staged — typically +/// `$PANTO_HOME/agent/`. Pass `null` to skip the base layer entirely /// (mostly useful for tests). /// /// `environ_map` is consulted for `HOME` and `XDG_CONFIG_HOME`. The @@ -112,15 +133,16 @@ pub fn discoverAndLoad( allocator: Allocator, io: Io, environ_map: *const std.process.Environ.Map, - system_agent_dir: ?[]const u8, + base_agent_dir: ?[]const u8, runtime: *LuaRuntime, + policies: Policies, ) !usize { - const sys_ext = if (system_agent_dir) |d| + const sys_ext = if (base_agent_dir) |d| try std.fs.path.join(allocator, &.{ d, kindSubdir(.extension) }) else null; defer if (sys_ext) |d| allocator.free(d); - const sys_tool = if (system_agent_dir) |d| + const sys_tool = if (base_agent_dir) |d| try std.fs.path.join(allocator, &.{ d, kindSubdir(.tool) }) else null; @@ -141,27 +163,28 @@ pub fn discoverAndLoad( io, runtime, .{ - .system_extensions = sys_ext, + .base_extensions = sys_ext, .user_extensions = user_ext, .project_extensions = project_ext, - .system_tools = sys_tool, + .base_tools = sys_tool, .user_tools = user_tool, .project_tools = project_tool, }, + policies, ); } /// Set of search paths consumed by `loadFromDirs`. Any field may be /// null; missing directories on disk are silently skipped. /// -/// Scan order is system → user → project. `applyShadowing` keeps the +/// Scan order is base → user → project. `applyShadowing` keeps the /// *last* occurrence of each (kind, name), so project entries win, -/// then user, then system. +/// then user, then base. pub const DirSet = struct { - system_extensions: ?[]const u8 = null, + base_extensions: ?[]const u8 = null, user_extensions: ?[]const u8 = null, project_extensions: ?[]const u8 = null, - system_tools: ?[]const u8 = null, + base_tools: ?[]const u8 = null, user_tools: ?[]const u8 = null, project_tools: ?[]const u8 = null, }; @@ -173,6 +196,7 @@ pub fn loadFromDirs( io: Io, runtime: *LuaRuntime, dirs: DirSet, + policies: Policies, ) !usize { var found: std.array_list.Managed(Found) = .init(allocator); defer { @@ -180,15 +204,38 @@ pub fn loadFromDirs( found.deinit(); } - if (dirs.system_extensions) |d| try scanDir(allocator, io, d, .system, .extension, &found); + if (dirs.base_extensions) |d| try scanDir(allocator, io, d, .base, .extension, &found); if (dirs.user_extensions) |d| try scanDir(allocator, io, d, .user, .extension, &found); if (dirs.project_extensions) |d| try scanDir(allocator, io, d, .project, .extension, &found); - if (dirs.system_tools) |d| try scanDir(allocator, io, d, .system, .tool, &found); + if (dirs.base_tools) |d| try scanDir(allocator, io, d, .base, .tool, &found); if (dirs.user_tools) |d| try scanDir(allocator, io, d, .user, .tool, &found); if (dirs.project_tools) |d| try scanDir(allocator, io, d, .project, .tool, &found); try applyShadowing(allocator, &found); + // Gate *entries* by the extensions policy (matched on logical name). + // This drops whole scripts before they run — a denied extension + // never executes, never registers tools. The tools policy is + // applied post-load against registered tool names. + if (policies.extensions) |_| { + var keep: std.array_list.Managed(Found) = .init(allocator); + errdefer { + for (keep.items) |*f| f.deinit(allocator); + keep.deinit(); + } + for (found.items) |*f| { + if (policyPermits(policies.extensions, f.name)) { + try keep.append(f.*); + } else { + std.log.debug("{s}: '{s}' denied by extensions policy", .{ f.kind.label(), f.name }); + f.deinit(allocator); + } + } + found.clearRetainingCapacity(); + try found.appendSlice(keep.items); + keep.deinit(); + } + const before = runtime.toolCount(); for (found.items) |f| { const load_result = switch (f.kind) { @@ -214,6 +261,13 @@ pub fn loadFromDirs( .{ f.kind.label(), f.name, f.source.label() }, ); } + + // Apply the tools policy to *registered tool names* (e.g. `std.read`). + if (policies.tools) |pol| { + const dropped = runtime.filterTools(pol, permitsByPtr); + if (dropped > 0) std.log.debug("tools: {d} tool(s) removed by tools policy", .{dropped}); + } + return runtime.toolCount() - before; } @@ -581,7 +635,7 @@ test "loadFromDirs: project shadows user end-to-end (via long-lived runtime)" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = user_path, .project_extensions = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); // Invoke the tool through the source and verify the project handler ran. @@ -623,7 +677,7 @@ test "loadFromDirs: tool-name collision between extensions errors" { const result = loadFromDirs(testing.allocator, testing.io, rt, .{ .project_extensions = ext_path, - }); + }, .{}); try testing.expectError(error.DuplicateTool, result); } @@ -650,7 +704,7 @@ test "loadFromDirs: tools/ directory — single-file tool form" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -690,7 +744,7 @@ test "loadFromDirs: tools/ directory — directory-style tool with sibling requi const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .project_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -738,7 +792,7 @@ test "loadFromDirs: project tool shadows user tool of the same name" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_tools = user_path, .project_tools = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -749,19 +803,19 @@ test "loadFromDirs: project tool shadows user tool of the same name" { try testing.expectEqualStrings("PROJECT", results[0].ok); } -test "loadFromDirs: project tool shadows user shadows system" { +test "loadFromDirs: project tool shadows user shadows base" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); - try makeDir(tmp.dir, "system_tools"); + try makeDir(tmp.dir, "base_tools"); try makeDir(tmp.dir, "user_tools"); try makeDir(tmp.dir, "project_tools"); - try writeFile(tmp.dir, "system_tools/greet.lua", + try writeFile(tmp.dir, "base_tools/greet.lua", \\return { - \\ name = "greet", description = "system version", + \\ name = "greet", description = "base version", \\ schema = { type = "object" }, - \\ handler = function(input) return "SYSTEM" end, + \\ handler = function(input) return "BASE" end, \\} ); try writeFile(tmp.dir, "user_tools/greet.lua", @@ -780,7 +834,7 @@ test "loadFromDirs: project tool shadows user shadows system" { ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf); + const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf); const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]); defer testing.allocator.free(sys_path); const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); @@ -794,10 +848,10 @@ test "loadFromDirs: project tool shadows user shadows system" { defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ - .system_tools = sys_path, + .base_tools = sys_path, .user_tools = user_path, .project_tools = proj_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -808,18 +862,18 @@ test "loadFromDirs: project tool shadows user shadows system" { try testing.expectEqualStrings("PROJECT", results[0].ok); } -test "loadFromDirs: user tool shadows system tool when no project entry" { +test "loadFromDirs: user tool shadows base tool when no project entry" { var tmp = testing.tmpDir(.{ .iterate = true }); defer tmp.cleanup(); - try makeDir(tmp.dir, "system_tools"); + try makeDir(tmp.dir, "base_tools"); try makeDir(tmp.dir, "user_tools"); - try writeFile(tmp.dir, "system_tools/greet.lua", + try writeFile(tmp.dir, "base_tools/greet.lua", \\return { - \\ name = "greet", description = "system", + \\ name = "greet", description = "base", \\ schema = { type = "object" }, - \\ handler = function(input) return "SYSTEM" end, + \\ handler = function(input) return "BASE" end, \\} ); try writeFile(tmp.dir, "user_tools/greet.lua", @@ -831,7 +885,7 @@ test "loadFromDirs: user tool shadows system tool when no project entry" { ); var path_buf: [std.fs.max_path_bytes]u8 = undefined; - const sys_len = try tmp.dir.realPathFile(testing.io, "system_tools", &path_buf); + const sys_len = try tmp.dir.realPathFile(testing.io, "base_tools", &path_buf); const sys_path = try testing.allocator.dupe(u8, path_buf[0..sys_len]); defer testing.allocator.free(sys_path); const user_len = try tmp.dir.realPathFile(testing.io, "user_tools", &path_buf); @@ -842,9 +896,9 @@ test "loadFromDirs: user tool shadows system tool when no project entry" { defer rt.deinit(); const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ - .system_tools = sys_path, + .base_tools = sys_path, .user_tools = user_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 1), n_tools); var src = rt.toolSource(); @@ -895,6 +949,87 @@ test "loadFromDirs: extension and tool share a *file* name independently" { const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ .user_extensions = ext_path, .user_tools = tools_path, - }); + }, .{}); try testing.expectEqual(@as(usize, 2), n_tools); } + +test "loadFromDirs: tools policy removes a denied registered tool name" { + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + try makeDir(tmp.dir, "tools"); + try writeFile(tmp.dir, "tools/reader.lua", + \\return { + \\ name = "std.read", description = "r", + \\ schema = { type = "object" }, + \\ handler = function(input) return "READ" end, + \\} + ); + try writeFile(tmp.dir, "tools/sheller.lua", + \\return { + \\ name = "std.shell", description = "s", + \\ schema = { type = "object" }, + \\ handler = function(input) return "SHELL" end, + \\} + ); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPathFile(testing.io, "tools", &path_buf); + const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); + defer testing.allocator.free(tools_path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + var deny = try testing.allocator.alloc([]const u8, 1); + deny[0] = try testing.allocator.dupe(u8, "std.shell"); + const tools_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny }; + defer tools_policy.deinit(testing.allocator); + + // Both tools register, but std.shell is filtered out post-load. + const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ + .user_tools = tools_path, + }, .{ .tools = &tools_policy }); + try testing.expectEqual(@as(usize, 1), n_tools); + + var src = rt.toolSource(); + const calls = [_]panto.ToolCall{.{ .tool_name = "std.read", .input = "{}" }}; + var results: [1]panto.ToolCallResult = .{.{ .err = error.SourceDroppedCall }}; + try src.vtable.invoke_batch(src.ctx, &calls, &results, testing.allocator); + defer testing.allocator.free(results[0].ok); + try testing.expectEqualStrings("READ", results[0].ok); +} + +test "loadFromDirs: extensions policy denies a whole entry before it loads" { + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + try makeDir(tmp.dir, "tools"); + // This tool would crash if its script ran (calls a nil global). The + // extensions policy must stop it from loading at all. + try writeFile(tmp.dir, "danger.lua", + \\error("this script must never run") + ); + try makeDir(tmp.dir, "td"); + try writeFile(tmp.dir, "td/danger.lua", + \\error("this script must never run") + ); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPathFile(testing.io, "td", &path_buf); + const tools_path = try testing.allocator.dupe(u8, path_buf[0..n]); + defer testing.allocator.free(tools_path); + + var rt = try LuaRuntime.create(testing.allocator); + defer rt.deinit(); + + var deny = try testing.allocator.alloc([]const u8, 1); + deny[0] = try testing.allocator.dupe(u8, "danger"); + const ext_policy: config_file.Policy = .{ .allow = &.{}, .deny = deny }; + defer ext_policy.deinit(testing.allocator); + + const n_tools = try loadFromDirs(testing.allocator, testing.io, rt, .{ + .user_tools = tools_path, + }, .{ .extensions = &ext_policy }); + try testing.expectEqual(@as(usize, 0), n_tools); +} diff --git a/src/glob.zig b/src/glob.zig new file mode 100644 index 0000000..83d9fcb --- /dev/null +++ b/src/glob.zig @@ -0,0 +1,121 @@ +//! Minimal glob matcher for tool/extension allow- and deny-lists. +//! +//! Patterns and names are dot-delimited segments (e.g. `std.git.status`). +//! Matching operates on whole segments, never on individual characters. +//! +//! Supported segment wildcards: +//! - `*` matches exactly one segment. `std.*` matches `std.read` but +//! not `std.git.status` (greedy within a segment, but not +//! dot-agnostic). +//! - `**` matches one or more segments. `std.**` matches both +//! `std.read` and `std.git.status`. +//! +//! A wildcard is only recognized as a whole segment: `std.*` and `std.**` +//! are wildcards, but `std.rea*` is a literal segment (the `*` has no +//! special meaning mid-segment). Everything else is matched literally. + +const std = @import("std"); + +/// Returns true if `name` matches `pattern` under the segment rules above. +pub fn match(pattern: []const u8, name: []const u8) bool { + var pat_it = std.mem.splitScalar(u8, pattern, '.'); + var name_it = std.mem.splitScalar(u8, name, '.'); + return matchSegments(&pat_it, &name_it); +} + +const SegmentIter = std.mem.SplitIterator(u8, .scalar); + +fn matchSegments(pat_it: *SegmentIter, name_it: *SegmentIter) bool { + while (pat_it.next()) |seg| { + if (std.mem.eql(u8, seg, "**")) { + // `**` matches one or more remaining segments. Try the + // shortest match first (one segment), then extend. + const rest_pat = pat_it.*; // pattern after `**` + while (name_it.next()) |_| { + var pat_copy = rest_pat; + var name_copy = name_it.*; + if (matchSegments(&pat_copy, &name_copy)) return true; + } + // `**` requires at least one segment, and the trailing + // pattern must have matched the remainder above. + return false; + } + + const name_seg = name_it.next() orelse return false; + if (std.mem.eql(u8, seg, "*")) { + // Matches exactly one (any) segment. + continue; + } + if (!std.mem.eql(u8, seg, name_seg)) return false; + } + + // Pattern exhausted: match iff name is also exhausted. + return name_it.next() == null; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "literal match" { + try testing.expect(match("std.read", "std.read")); + try testing.expect(!match("std.read", "std.write")); + try testing.expect(!match("std.read", "std.read.more")); + try testing.expect(!match("std.read.more", "std.read")); +} + +test "single star matches exactly one segment" { + try testing.expect(match("std.*", "std.read")); + try testing.expect(match("std.*", "std.write")); + try testing.expect(!match("std.*", "std.git.status")); + try testing.expect(!match("std.*", "std")); + try testing.expect(!match("std.*", "other.read")); +} + +test "double star is dot-agnostic, one or more segments" { + try testing.expect(match("std.**", "std.read")); + try testing.expect(match("std.**", "std.git.status")); + try testing.expect(!match("std.**", "std")); + try testing.expect(!match("std.**", "other.read")); +} + +test "bare single star matches one segment" { + try testing.expect(match("*", "std")); + try testing.expect(!match("*", "std.read")); + // An empty name is a single empty segment, which `*` matches. + try testing.expect(match("*", "")); +} + +test "bare double star matches everything non-empty" { + try testing.expect(match("**", "std")); + try testing.expect(match("**", "std.read")); + try testing.expect(match("**", "std.git.status")); +} + +test "star in middle" { + try testing.expect(match("*.read", "std.read")); + try testing.expect(match("*.*", "std.read")); + try testing.expect(!match("*.*", "std.git.status")); + try testing.expect(match("std.*.status", "std.git.status")); + try testing.expect(!match("std.*.status", "std.status")); +} + +test "double star in middle" { + try testing.expect(match("std.**.status", "std.git.status")); + try testing.expect(match("std.**.status", "std.a.b.status")); + try testing.expect(!match("std.**.status", "std.status")); + try testing.expect(match("**.status", "std.git.status")); +} + +test "wildcard only recognized as whole segment" { + // A `*` embedded in a segment is a literal, not a wildcard. + try testing.expect(match("std.rea*", "std.rea*")); + try testing.expect(!match("std.rea*", "std.read")); +} + +test "empty pattern only matches empty name" { + try testing.expect(match("", "")); + try testing.expect(!match("", "x")); +} diff --git a/src/lua_runtime.zig b/src/lua_runtime.zig index 5efadaa..fed56a8 100644 --- a/src/lua_runtime.zig +++ b/src/lua_runtime.zig @@ -354,6 +354,38 @@ pub const LuaRuntime = struct { pub fn toolCount(self: *const LuaRuntime) usize { return self.decls.items.len; } + + /// Drop every declared tool whose registered name is rejected by + /// `permits`. The handler ref is unref'd and the decl removed. + /// Returns the number of tools dropped. + /// + /// String storage for a dropped tool's name/description/schema is + /// left in `self.strings` (freed at `deinit`); only the decl entry + /// and the Lua handler ref are reclaimed eagerly. This keeps the + /// filter simple — we never need to find-and-free individual + /// strings out of the shared pool. + pub fn filterTools( + self: *LuaRuntime, + ctx: anytype, + comptime permits: fn (@TypeOf(ctx), []const u8) bool, + ) usize { + var dropped: usize = 0; + var i: usize = 0; + while (i < self.decls.items.len) { + const name = self.decls.items[i].name; + if (permits(ctx, name)) { + i += 1; + continue; + } + // Unref the handler, if present. + if (self.handlers.fetchRemove(name)) |kv| { + c.luaL_unref(self.L, lua_bridge.LUA_REGISTRYINDEX, kv.value); + } + _ = self.decls.orderedRemove(i); + dropped += 1; + } + return dropped; + } }; const source_vtable: panto.ToolSource.VTable = .{ diff --git a/src/luarocks_runtime.zig b/src/luarocks_runtime.zig index 0114b64..e652096 100644 --- a/src/luarocks_runtime.zig +++ b/src/luarocks_runtime.zig @@ -137,6 +137,7 @@ pub fn bootstrap( try panto_home.ensureDirsExist(layout, io); try stageLuaHeaders(allocator, io, layout); try stageAgentTree(allocator, io, layout); + try stageDefaultConfig(allocator, io, layout); const lua_wrapper_path = try writeLuaWrapper(allocator, io, layout, panto_executable_path); defer allocator.free(lua_wrapper_path); try writeLuarocksConfig(allocator, io, layout, lua_wrapper_path); @@ -236,6 +237,71 @@ fn stageAgentTree( } } +/// The default base `config.toml`, materialized at `$PANTO_HOME/config.toml` +/// on first run if absent. It declares OpenAI and Anthropic providers whose +/// API keys come from the conventional env vars — so a provider simply +/// doesn't appear unless its key is exported. Edit or override at the user +/// (`~/.config/panto/config.toml`) or project (`./.panto/config.toml`) layer. +const default_base_config = + \\# panto base config (auto-generated on first run). + \\# + \\# This is the lowest-precedence layer. Override anything here from + \\# ~/.config/panto/config.toml (user) + \\# ./.panto/config.toml (project) + \\# Tables merge across layers; scalars and arrays from a higher layer + \\# replace the lower one wholesale. + \\# + \\# A provider whose API key (or its api_key_env_var) is missing at + \\# runtime is silently dropped — so these defaults disappear unless + \\# you've exported the corresponding key. + \\ + \\[providers.openai] + \\style = "openai_chat" + \\base_url = "https://api.openai.com/v1" + \\api_key_env_var = "OPENAI_API_KEY" + \\ + \\[providers.anthropic] + \\style = "anthropic_messages" + \\base_url = "https://api.anthropic.com" + \\api_key_env_var = "ANTHROPIC_API_KEY" + \\ + \\# Pick the default model with `<provider>:<alias>`. The alias is + \\# resolved against models.toml; an unknown alias is used verbatim as + \\# the wire model name. + \\# + \\# [defaults] + \\# model = "anthropic:claude-sonnet-4-20250514" + \\ + \\# Tool/extension availability (glob patterns). Empty allow = allow all. + \\# [tools] + \\# allow = ["std.*"] + \\# deny = [] + \\ +; + +/// Materialize the default base config at `$PANTO_HOME/config.toml`, +/// but only if no file exists there yet. We never overwrite — the base +/// config is user-editable, and a stale-but-edited file must win over the +/// shipped default. +fn stageDefaultConfig( + allocator: Allocator, + io: Io, + layout: panto_home.Layout, +) !void { + _ = allocator; + var home = try Io.Dir.cwd().openDir(io, layout.home, .{}); + defer home.close(io); + + home.access(io, "config.toml", .{}) catch |err| switch (err) { + error.FileNotFound => { + try home.writeFile(io, .{ .sub_path = "config.toml", .data = default_base_config }); + return; + }, + else => return err, + }; + // Exists already — leave it untouched. +} + /// Write `contents` into `dir/name` only if the existing file differs. /// Creates the file if absent. fn writeIfDifferent( diff --git a/src/main.zig b/src/main.zig index bdb5390..aaf86b5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,6 +9,8 @@ const self_exe = @import("self_exe.zig"); const subcommand = @import("subcommand.zig"); const session_paths = @import("session_paths.zig"); const models_toml = @import("models_toml.zig"); +const config_file = @import("config_file.zig"); +const glob = @import("glob.zig"); // Shorthand alias for the Lua C API. The bridge module owns the actual // `@cImport`; we re-use it here so the smoke check uses identical types. @@ -33,6 +35,8 @@ test { _ = self_exe; _ = subcommand; _ = models_toml; + _ = config_file; + _ = glob; } const Receiver = panto.provider.Receiver; @@ -195,74 +199,58 @@ fn luaSmokeCheck() void { lua.lua_settop(L, 0); } -fn loadConfig(environ_map: *const std.process.Environ.Map) !panto.config.Config { - const style_str = environ_map.get("PANTO_API_STYLE") orelse "openai_chat"; - const style = std.meta.stringToEnum(panto.config.APIStyle, style_str) orelse { - std.debug.print( - "error: PANTO_API_STYLE must be one of: openai_chat, anthropic_messages (got: {s})\n", - .{style_str}, - ); - return error.InvalidApiStyle; - }; +/// Print a friendly message for a config.toml load failure before the +/// process exits non-zero. +fn reportConfigError(err: anyerror) void { + switch (err) { + error.InvalidConfigToml => std.debug.print( + "error: a config.toml failed to parse (see log above).\n", + .{}, + ), + error.InvalidProvider => std.debug.print( + "error: a [providers.*] entry is malformed (need style + base_url).\n", + .{}, + ), + error.InvalidModelRef => std.debug.print( + "error: defaults.model must look like \"provider:model\".\n", + .{}, + ), + error.UnknownDefaultProvider => std.debug.print( + "error: defaults.model names a provider that isn't configured (or whose API key env var isn't set).\n", + .{}, + ), + error.PolicyConflict => std.debug.print( + "error: a tool/extension pattern appears in both allow and deny.\n", + .{}, + ), + error.NoHomeDirectory => std.debug.print( + "error: cannot resolve config paths — set HOME or XDG_CONFIG_HOME/XDG_DATA_HOME.\n", + .{}, + ), + else => std.debug.print("error: failed to load config: {s}\n", .{@errorName(err)}), + } +} - switch (style) { - .openai_chat => { - const api_key = environ_map.get("OPENAI_API_KEY") orelse { - std.debug.print("error: OPENAI_API_KEY is required\n", .{}); - return error.MissingApiKey; - }; - const base_url = environ_map.get("OPENAI_BASE_URL") orelse "https://api.openai.com/v1"; - const model = environ_map.get("OPENAI_MODEL") orelse { - std.debug.print("error: OPENAI_MODEL is required\n", .{}); - return error.MissingModel; - }; - const reasoning: panto.config.ReasoningEffort = - if (environ_map.get("OPENAI_REASONING")) |val| - std.meta.stringToEnum(panto.config.ReasoningEffort, val) orelse { - std.debug.print( - "error: OPENAI_REASONING must be one of: default, off, minimal, low, medium, high (got: {s})\n", - .{val}, - ); - return error.InvalidReasoning; - } - else - .default; - return .{ .openai_chat = .{ - .api_key = api_key, - .base_url = base_url, - .model = model, - .reasoning = reasoning, - } }; - }, - .anthropic_messages => { - const api_key = environ_map.get("ANTHROPIC_API_KEY") orelse { - std.debug.print("error: ANTHROPIC_API_KEY is required\n", .{}); - return error.MissingApiKey; - }; - const base_url = environ_map.get("ANTHROPIC_BASE_URL") orelse "https://api.anthropic.com"; - const model = environ_map.get("ANTHROPIC_MODEL") orelse { - std.debug.print("error: ANTHROPIC_MODEL is required\n", .{}); - return error.MissingModel; - }; - const api_version = environ_map.get("ANTHROPIC_API_VERSION") orelse "2023-06-01"; - const max_tokens: u32 = if (environ_map.get("ANTHROPIC_MAX_TOKENS")) |val| - std.fmt.parseInt(u32, val, 10) catch { - std.debug.print( - "error: ANTHROPIC_MAX_TOKENS must be a positive integer (got: {s})\n", - .{val}, - ); - return error.InvalidMaxTokens; - } - else - 4096; - return .{ .anthropic_messages = .{ - .api_key = api_key, - .base_url = base_url, - .model = model, - .api_version = api_version, - .max_tokens = max_tokens, - } }; +/// Print a friendly message for a model-selection failure. +fn reportModelError(err: anyerror, cfg: *const config_file.Config) void { + switch (err) { + error.NoModelSelected => { + std.debug.print( + "error: no model selected. Set `defaults.model = \"provider:alias\"` in config.toml.\n", + .{}, + ); + if (cfg.providers.len == 0) { + std.debug.print( + " (no providers resolved — check that an API key or its env var is present.)\n", + .{}, + ); + } }, + error.UnknownProvider => std.debug.print( + "error: the selected model names an unconfigured provider.\n", + .{}, + ), + else => std.debug.print("error: model selection failed: {s}\n", .{@errorName(err)}), } } @@ -294,8 +282,6 @@ pub fn main(init: std.process.Init) !void { .agent => {}, } - const config = try loadConfig(init.environ_map); - // Parse the agent-mode flags. Currently only `--resume [<id>]`. const cli_flags = try parseAgentFlags(alloc, init.minimal.args); defer cli_flags.deinit(alloc); @@ -315,19 +301,47 @@ pub fn main(init: std.process.Init) !void { const session_dir = try session_paths.sessionDirForCwd(alloc, init.environ_map, cwd); defer alloc.free(session_dir); - // Load the user's models.toml — a missing file is fine (empty - // registry). Cost lookups against an empty registry return null, - // and the display layer will format that as "unknown." + // Load the merged config.toml (base → user → project). Providers, + // default model, and tool/extension policy all come from here. + var app_config = config_file.load(alloc, io, init.environ_map, cwd) catch |err| { + reportConfigError(err); + std.process.exit(1); + }; + defer app_config.deinit(); + + // Load models.toml — model aliases (wire name + knobs) and pricing. + // A missing file is fine (empty registries); cost lookups return + // null, formatted as "unknown" by the display layer. const models_toml_path = try models_toml.configPath(alloc, init.environ_map); defer alloc.free(models_toml_path); - var pricing_registry = try models_toml.loadFromPath(alloc, io, models_toml_path); - defer pricing_registry.deinit(); - std.log.debug("models.toml: {d} entries from {s}", .{ pricing_registry.count(), models_toml_path }); + var models = try models_toml.loadFromPath(alloc, io, models_toml_path); + defer models.deinit(); + std.log.debug( + "models.toml: {d} model def(s), {d} price entr(ies) from {s}", + .{ models.defs.count(), models.pricing.count(), models_toml_path }, + ); + // `models.pricing` is parsed and ready; the print-based CLI doesn't + // surface per-turn cost yet (that lands with the TUI frontend). - const banner_model_initial: []const u8 = switch (config) { + // Choose the active model and assemble the provider config. + const model_ref = app_config.selectModel(&models.defs, null) catch |err| { + reportModelError(err, &app_config); + std.process.exit(1); + }; + const provider_config = config_file.buildProviderConfig(&app_config, &models.defs, model_ref) catch |err| { + reportModelError(err, &app_config); + std.process.exit(1); + }; + + // Process-global HTTP client: one connection pool for every provider / + // base_url the agent may switch to. Torn down at shutdown. + panto.config.initHttp(alloc, io); + defer panto.config.deinitHttp(); + + const banner_model_initial: []const u8 = switch (provider_config) { inline else => |c| c.model, }; - const banner_provider_initial: []const u8 = @tagName(config); + const banner_provider_initial: []const u8 = model_ref.provider; // Create or resume the session. Resume failures (missing/ambiguous id) // are user errors — print a tidy message and exit 1 rather than @@ -365,9 +379,11 @@ pub fn main(init: std.process.Init) !void { try appendSystemToSession(alloc, &session_mgr, system_text); } - const prov = try panto.provider.Provider.init(alloc, io, config); - var agent = panto.agent.Agent.init(alloc, io, prov); - defer agent.deinit(); + // The tool registry is owned here and referenced by the active config + // snapshot. Extensions register their tool source into it *before* the + // snapshot is built, so the agent's first turn sees them. + var registry = panto.ToolRegistry.init(alloc); + defer registry.deinit(); // Spin up the long-lived Lua runtime. All Lua extensions load into // one `lua_State`; module-global state survives across calls. The @@ -394,10 +410,10 @@ pub fn main(init: std.process.Init) !void { // chance to register tools that might want to yield. try rt.installScheduler(); - // Discover Lua extensions across three layers — system + // Discover Lua extensions across three layers — base // ($PANTO_HOME/agent), user ($XDG_CONFIG_HOME/panto or // $HOME/.config/panto), and project (./.panto). Project shadows - // user shadows system; tool-name collisions across surviving + // user shadows base; tool-name collisions across surviving // entries abort startup. const n_ext_tools = extension_loader.discoverAndLoad( alloc, @@ -405,6 +421,10 @@ pub fn main(init: std.process.Init) !void { init.environ_map, luarocks_rt.layout.agent_dir, rt, + .{ + .extensions = &app_config.extensions, + .tools = &app_config.tools, + }, ) catch |err| { std.log.err("extension discovery failed: {t}", .{err}); return err; @@ -412,10 +432,20 @@ pub fn main(init: std.process.Init) !void { std.log.debug("extensions: {d} tool(s) registered", .{n_ext_tools}); if (n_ext_tools > 0) { - try agent.registerToolSource(rt.toolSource()); + try registry.registerSource(rt.toolSource()); } - const banner_base: []const u8 = switch (config) { + // Assemble the active configuration snapshot (provider + tool registry) + // and create the agent holding a pointer to it. Swapping this pointer + // later (model/provider/tool changes) takes effect at the next turn. + const active_config: panto.config.Config = .{ + .provider = provider_config, + .registry = ®istry, + }; + var agent = panto.agent.Agent.init(alloc, io, &active_config); + defer agent.deinit(); + + const banner_base: []const u8 = switch (provider_config) { inline else => |c| c.base_url, }; try stdout.print( diff --git a/src/models_toml.zig b/src/models_toml.zig index 5ccb302..36d8c65 100644 --- a/src/models_toml.zig +++ b/src/models_toml.zig @@ -1,46 +1,46 @@ -//! Loader for `~/.config/panto/models.toml`. +//! Loader for `models.toml` — model aliases and per-model pricing. //! -//! Schema: +//! A model entry is keyed by `<provider_name>.<alias>`, where +//! `<provider_name>` matches a provider defined in `config.toml`'s +//! `[providers.<name>]` table, and `<alias>` is the short name the user +//! references via `<provider_name>:<alias>` (e.g. `anthropic:sonnet`). //! -//! [<provider>.<model>] -//! input = <float> # USD per million tokens (fresh input) -//! output = <float> # USD per million tokens -//! cache_read = <float> # USD per million tokens (optional; default "unknown") -//! cache_write = <float> # USD per million tokens (optional; default "unknown") +//! Schema: //! -//! All four price fields are optional at the parse layer. Any field -//! omitted from the TOML comes through as `null` in the in-memory -//! `Pricing`, which means "unknown price for this token category" — -//! NOT "zero." If the model later reports usage in an unknown -//! category (e.g. gpt-4o reads from prompt cache but the TOML omitted -//! `cache_read`), session cost degenerates to "unknown" rather than -//! silently treating that usage as free. To declare a category as a -//! known zero (e.g. OpenAI doesn't bill a cache-write rate), write -//! `cache_write = 0` explicitly. +//! [<provider>.<alias>] +//! model = <string> # wire model id; defaults to <alias> if omitted +//! reasoning = <string> # default | off | minimal | low | medium | high +//! max_tokens = <int> # anthropic_messages only +//! api_version = <string> # anthropic_messages only +//! # pricing (all optional; USD per million tokens): +//! input = <float> +//! output = <float> +//! cache_read = <float> +//! cache_write = <float> //! -//! `<provider>` is `openai` or `anthropic` (matching pantograph's API -//! styles). `<model>` is the model id pantograph sends to the API. Both -//! are TOML "dotted-key" path segments; quote them if they contain -//! characters TOML doesn't allow bare (`-`, `.`, `:` etc. — `-` is OK -//! bare; `.` requires quoting since dots separate path segments). +//! Pricing semantics are unchanged from the original models.toml: any +//! omitted price field is "unknown" (null), not zero. Write `= 0` +//! explicitly to declare a known-zero rate. //! //! Example: //! -//! [anthropic."claude-sonnet-4-20250514"] +//! [anthropic.sonnet] +//! model = "claude-sonnet-4-20250514" +//! reasoning = "high" +//! max_tokens = 8192 //! input = 3.0 //! output = 15.0 //! cache_read = 0.3 //! cache_write = 3.75 //! -//! [openai.gpt-4o] +//! [openai.gpt] +//! model = "gpt-4o" //! input = 2.5 //! output = 10.0 //! -//! The model id in the section header may also be unquoted when it -//! contains no `.` or other reserved chars (e.g. `gpt-4o`); pantograph -//! reads either form. We do not currently bake in default pricing — -//! a missing entry (or a present entry with missing fields) surfaces -//! as "unknown cost" and the display layer formats accordingly. +//! A referenced alias with no entry here is not an error: the alias is +//! used verbatim as the wire model name (zero-config convenience), with +//! default knobs and unknown pricing. const std = @import("std"); const Allocator = std.mem.Allocator; @@ -51,6 +51,71 @@ const panto = @import("panto"); pub const Pricing = panto.pricing.Pricing; pub const Registry = panto.pricing.Registry; +pub const ReasoningEffort = panto.config.ReasoningEffort; + +/// A resolved model definition: the wire model id plus per-model knobs. +/// Strings are owned by the `ModelRegistry`. +pub const ModelDef = struct { + /// `<provider_name>` — matches a `[providers.<name>]` in config.toml. + provider: []const u8, + /// `<alias>` — the short reference name. + alias: []const u8, + /// Wire model id sent to the provider API. Equals `alias` when the + /// TOML omitted `model`. + model: []const u8, + reasoning: ReasoningEffort, + /// anthropic_messages only; null = use the provider/library default. + max_tokens: ?u32, + /// anthropic_messages only; null = use the library default. + api_version: ?[]const u8, +}; + +/// In-memory map of `(provider, alias) -> ModelDef`. Owns all strings. +pub const ModelRegistry = struct { + allocator: Allocator, + entries: std.ArrayList(ModelDef), + + pub fn init(allocator: Allocator) ModelRegistry { + return .{ .allocator = allocator, .entries = .empty }; + } + + pub fn deinit(self: *ModelRegistry) void { + for (self.entries.items) |e| { + self.allocator.free(e.provider); + self.allocator.free(e.alias); + self.allocator.free(e.model); + if (e.api_version) |v| self.allocator.free(v); + } + self.entries.deinit(self.allocator); + } + + pub fn count(self: *const ModelRegistry) usize { + return self.entries.items.len; + } + + /// Look up a model definition by provider name + alias. + pub fn get(self: *const ModelRegistry, provider: []const u8, alias: []const u8) ?ModelDef { + for (self.entries.items) |e| { + if (std.mem.eql(u8, e.provider, provider) and std.mem.eql(u8, e.alias, alias)) { + return e; + } + } + return null; + } +}; + +/// Everything `models.toml` yields: model definitions and pricing. The +/// two registries are independent (pricing keys on the *wire* model id; +/// model defs key on the alias) but parsed in one pass. +pub const Models = struct { + defs: ModelRegistry, + pricing: Registry, + + pub fn deinit(self: *Models) void { + self.defs.deinit(); + self.pricing.deinit(); + } +}; /// Resolve the absolute path to `models.toml`. Honors `XDG_CONFIG_HOME`, /// falling back to `$HOME/.config`. Caller owns the returned slice. @@ -64,21 +129,17 @@ pub fn configPath(allocator: Allocator, environ_map: *const std.process.Environ. return error.NoHomeDirectory; } -/// Load `models.toml` into a fresh `Registry`. If the file does not -/// exist, returns an empty registry (no error — missing config is fine). -/// -/// On parse errors, returns `error.InvalidModelsToml` after logging the -/// line/column of the first error. -pub fn loadFromPath( - allocator: Allocator, - io: Io, - path: []const u8, -) !Registry { - var reg = Registry.init(allocator); - errdefer reg.deinit(); +/// Load `models.toml` from `path`. A missing file yields empty registries +/// (no error). Parse errors return `error.InvalidModelsToml`. +pub fn loadFromPath(allocator: Allocator, io: Io, path: []const u8) !Models { + var models: Models = .{ + .defs = ModelRegistry.init(allocator), + .pricing = Registry.init(allocator), + }; + errdefer models.deinit(); const file = Io.Dir.cwd().openFile(io, path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => return reg, // empty registry; perfectly fine. + error.FileNotFound => return models, // empty; perfectly fine. else => return err, }; defer file.close(io); @@ -88,18 +149,16 @@ pub fn loadFromPath( defer allocator.free(bytes); _ = try file.readPositionalAll(io, bytes, 0); - try parseInto(®, bytes); - return reg; + try parseInto(&models, bytes); + return models; } -/// Parse a TOML string into the given registry. Useful for tests. -pub fn parseInto(reg: *Registry, source: []const u8) !void { - const alloc = reg.allocator; +/// Parse a TOML string into the given registries. Useful for tests. +pub fn parseInto(models: *Models, source: []const u8) !void { + const alloc = models.defs.allocator; const result = toml.parseWithError(alloc, source, .{}); switch (result) { .err => |e| { - // Silenced in tests so the test runner doesn't flag the - // expected-failure case as a real error. if (!@import("builtin").is_test) { std.log.err( "models.toml: parse error at line {d}, column {d}: {s}", @@ -113,31 +172,98 @@ pub fn parseInto(reg: *Registry, source: []const u8) !void { var d = doc; d.deinit(); } - try ingestDocument(reg, doc); + try ingestDocument(models, doc); }, } } -fn ingestDocument(reg: *Registry, doc: *toml.Document) !void { - // Root is a table of provider -> table of model -> { input, output, ... }. +fn ingestDocument(models: *Models, doc: *toml.Document) !void { const root_val: *const toml.Value = doc.root; if (root_val.* != .table) return; + var provider_it = toml.tableIterator(root_val); while (provider_it.next()) |provider_entry| { const provider = provider_entry.key; const provider_val: *const toml.Value = provider_entry.value; if (provider_val.* != .table) continue; - var model_it = toml.tableIterator(provider_val); - while (model_it.next()) |model_entry| { - const model = model_entry.key; - const v: *const toml.Value = model_entry.value; + + var alias_it = toml.tableIterator(provider_val); + while (alias_it.next()) |alias_entry| { + const alias = alias_entry.key; + const v: *const toml.Value = alias_entry.value; if (v.* != .table) continue; - const pricing = pricingFromValue(v); - try reg.set(provider, model, pricing); + + try ingestModel(models, provider, alias, v); } } } +fn ingestModel( + models: *Models, + provider: []const u8, + alias: []const u8, + v: *const toml.Value, +) !void { + const alloc = models.defs.allocator; + + // Wire model id: explicit `model`, else the alias itself. + const wire_model: []const u8 = blk: { + if (v.get("model")) |m| { + if (m.asString()) |s| break :blk s; + } + break :blk alias; + }; + + const reasoning: ReasoningEffort = blk: { + if (v.get("reasoning")) |r| { + if (r.asString()) |s| { + break :blk std.meta.stringToEnum(ReasoningEffort, s) orelse .default; + } + } + break :blk .default; + }; + + const max_tokens: ?u32 = blk: { + if (v.get("max_tokens")) |mt| { + if (mt.asI64()) |i| { + if (i > 0) break :blk @intCast(i); + } + } + break :blk null; + }; + + const api_version_src: ?[]const u8 = blk: { + if (v.get("api_version")) |av| { + if (av.asString()) |s| break :blk s; + } + break :blk null; + }; + + // Own all the strings. + const provider_copy = try alloc.dupe(u8, provider); + errdefer alloc.free(provider_copy); + const alias_copy = try alloc.dupe(u8, alias); + errdefer alloc.free(alias_copy); + const model_copy = try alloc.dupe(u8, wire_model); + errdefer alloc.free(model_copy); + const api_version_copy: ?[]const u8 = if (api_version_src) |s| try alloc.dupe(u8, s) else null; + errdefer if (api_version_copy) |s| alloc.free(s); + + try models.defs.entries.append(alloc, .{ + .provider = provider_copy, + .alias = alias_copy, + .model = model_copy, + .reasoning = reasoning, + .max_tokens = max_tokens, + .api_version = api_version_copy, + }); + + // Pricing keys on the *wire* model id so usage records (which carry + // the wire model) resolve directly. + const pricing = pricingFromValue(v); + try models.pricing.set(provider, wire_model, pricing); +} + fn pricingFromValue(v: *const toml.Value) Pricing { return .{ .input = readPriceField(v, "input"), @@ -148,12 +274,9 @@ fn pricingFromValue(v: *const toml.Value) Pricing { } /// Returns `null` if the field is absent or has a type we can't -/// interpret as a price. An explicit `0` (or `0.0`) comes through as a -/// known zero, not `null` — callers rely on that distinction. +/// interpret as a price. An explicit `0` comes through as a known zero. fn readPriceField(table: *const toml.Value, name: []const u8) ?u64 { const field = table.get(name) orelse return null; - // Accept either floats (the natural form) or integers (the user - // wrote `3` instead of `3.0`). if (field.asF64()) |f| return Pricing.fromDollarsPerMtok(f); if (field.asI64()) |i| return Pricing.fromDollarsPerMtok(@floatFromInt(i)); return null; @@ -165,108 +288,116 @@ fn readPriceField(table: *const toml.Value, name: []const u8) ?u64 { const testing = std.testing; -test "parseInto: two providers, multiple models" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +fn emptyModels(a: Allocator) Models { + return .{ .defs = ModelRegistry.init(a), .pricing = Registry.init(a) }; +} + +test "parseInto: model defs carry wire name and knobs, keyed by provider.alias" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = - \\[anthropic."claude-sonnet-4-20250514"] + \\[anthropic.sonnet] + \\model = "claude-sonnet-4-20250514" + \\reasoning = "high" + \\max_tokens = 8192 \\input = 3.0 \\output = 15.0 - \\cache_read = 0.3 - \\cache_write = 3.75 \\ - \\[openai.gpt-4o] + \\[openai.gpt] + \\model = "gpt-4o" \\input = 2.5 \\output = 10.0 - \\cache_read = 1.25 - \\ - \\[openai.gpt-4o-mini] - \\input = 0.15 - \\output = 0.6 ; - try parseInto(®, src); + try parseInto(&models, src); - const anth = reg.get("anthropic", "claude-sonnet-4-20250514").?; - try testing.expectEqual(@as(?u64, 300), anth.input); - try testing.expectEqual(@as(?u64, 1500), anth.output); - try testing.expectEqual(@as(?u64, 30), anth.cache_read); - try testing.expectEqual(@as(?u64, 375), anth.cache_write); + const sonnet = models.defs.get("anthropic", "sonnet").?; + try testing.expectEqualStrings("claude-sonnet-4-20250514", sonnet.model); + try testing.expectEqual(ReasoningEffort.high, sonnet.reasoning); + try testing.expectEqual(@as(?u32, 8192), sonnet.max_tokens); - const oa = reg.get("openai", "gpt-4o").?; - try testing.expectEqual(@as(?u64, 250), oa.input); - try testing.expectEqual(@as(?u64, 1000), oa.output); - try testing.expectEqual(@as(?u64, 125), oa.cache_read); - // cache_write absent in source — stays unknown, NOT silently 0. - try testing.expectEqual(@as(?u64, null), oa.cache_write); + const gpt = models.defs.get("openai", "gpt").?; + try testing.expectEqualStrings("gpt-4o", gpt.model); + try testing.expectEqual(ReasoningEffort.default, gpt.reasoning); + try testing.expectEqual(@as(?u32, null), gpt.max_tokens); - const mini = reg.get("openai", "gpt-4o-mini").?; - try testing.expectEqual(@as(?u64, 15), mini.input); - try testing.expectEqual(@as(?u64, 60), mini.output); - try testing.expectEqual(@as(?u64, null), mini.cache_read); - try testing.expectEqual(@as(?u64, null), mini.cache_write); + // Pricing keyed on the wire model id. + const sonnet_price = models.pricing.get("anthropic", "claude-sonnet-4-20250514").?; + try testing.expectEqual(@as(?u64, 300), sonnet_price.input); + try testing.expectEqual(@as(?u64, 1500), sonnet_price.output); } -test "parseInto: integer values are accepted (user writes `3` not `3.0`)" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +test "parseInto: omitted model defaults the wire name to the alias" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = \\[openai.gpt-4o] - \\input = 3 - \\output = 15 + \\input = 2.5 + \\output = 10.0 ; - try parseInto(®, src); - const p = reg.get("openai", "gpt-4o").?; - try testing.expectEqual(@as(?u64, 300), p.input); - try testing.expectEqual(@as(?u64, 1500), p.output); + try parseInto(&models, src); + + const def = models.defs.get("openai", "gpt-4o").?; + try testing.expectEqualStrings("gpt-4o", def.model); } -test "parseInto: missing optional fields stay null (unknown, not zero)" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +test "parseInto: pricing omissions stay null (unknown, not zero)" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = - \\[openai.gpt-4o] + \\[openai.gpt] + \\model = "gpt-4o" \\input = 2.5 \\output = 10.0 ; - try parseInto(®, src); - const p = reg.get("openai", "gpt-4o").?; + try parseInto(&models, src); + const p = models.pricing.get("openai", "gpt-4o").?; try testing.expectEqual(@as(?u64, null), p.cache_read); try testing.expectEqual(@as(?u64, null), p.cache_write); } -test "parseInto: explicit 0 is a known zero, distinct from omission" { - // OpenAI doesn't charge for cache writes. Writing `cache_write = 0` - // in the TOML must produce a known 0 — not null — so cost stays - // computable on turns that report cache_write usage. - var reg = Registry.init(testing.allocator); - defer reg.deinit(); +test "parseInto: explicit cache_write = 0 is a known zero" { + var models = emptyModels(testing.allocator); + defer models.deinit(); const src = - \\[openai.gpt-4o] + \\[openai.gpt] + \\model = "gpt-4o" \\input = 2.5 \\output = 10.0 - \\cache_read = 1.25 \\cache_write = 0 ; - try parseInto(®, src); - const p = reg.get("openai", "gpt-4o").?; + try parseInto(&models, src); + const p = models.pricing.get("openai", "gpt-4o").?; try testing.expectEqual(@as(?u64, 0), p.cache_write); } test "parseInto: malformed TOML returns InvalidModelsToml" { - var reg = Registry.init(testing.allocator); - defer reg.deinit(); - try testing.expectError(error.InvalidModelsToml, parseInto(®, "this is not valid toml = =")); + var models = emptyModels(testing.allocator); + defer models.deinit(); + try testing.expectError(error.InvalidModelsToml, parseInto(&models, "not valid = =")); } -test "loadFromPath: missing file returns empty registry, no error" { +test "loadFromPath: missing file returns empty registries, no error" { const io = testing.io; - var reg = try loadFromPath(testing.allocator, io, "/nonexistent/path/models.toml"); - defer reg.deinit(); - try testing.expectEqual(@as(usize, 0), reg.count()); + var models = try loadFromPath(testing.allocator, io, "/nonexistent/models.toml"); + defer models.deinit(); + try testing.expectEqual(@as(usize, 0), models.defs.count()); + try testing.expectEqual(@as(usize, 0), models.pricing.count()); +} + +test "ModelRegistry.get: returns null for unknown alias" { + var models = emptyModels(testing.allocator); + defer models.deinit(); + const src = + \\[anthropic.sonnet] + \\model = "claude-sonnet-4-20250514" + ; + try parseInto(&models, src); + try testing.expect(models.defs.get("anthropic", "opus") == null); + try testing.expect(models.defs.get("openai", "sonnet") == null); } test "configPath: XDG_CONFIG_HOME wins" { diff --git a/src/panto_home.zig b/src/panto_home.zig index 353e54b..e75d14b 100644 --- a/src/panto_home.zig +++ b/src/panto_home.zig @@ -30,10 +30,10 @@ pub const Layout = struct { allocator: Allocator, /// `$PANTO_HOME` itself. home: []u8, - /// `$PANTO_HOME/agent/` — the "system" extension/tool tree, + /// `$PANTO_HOME/agent/` — the "base" extension/tool tree, /// populated at bootstrap from files embedded into the panto /// binary. Searched after user/project layers for tools and - /// extensions; project shadows user shadows system. + /// extensions; project shadows user shadows base. agent_dir: []u8, /// `$PANTO_HOME/rocks/lua-<lua_version>/` — the versioned tree. tree: []u8, diff --git a/src/subcommand.zig b/src/subcommand.zig index f6c3058..c50429e 100644 --- a/src/subcommand.zig +++ b/src/subcommand.zig @@ -102,11 +102,17 @@ fn printHelp(io: Io) !void { \\ panto lua [args...] Drop into the embedded Lua interpreter. \\ panto help Show this message. \\ + \\Configuration (TOML, merged base → user → project): + \\ $XDG_DATA_HOME/panto/config.toml (base; auto-generated) + \\ $XDG_CONFIG_HOME/panto/config.toml (user) + \\ ./.panto/config.toml (project) + \\ Define providers under [providers.<name>], pick a default with + \\ [defaults] model = "<provider>:<alias>", and gate tools/extensions + \\ with [tools]/[extensions] allow/deny globs. Model aliases (wire name, + \\ reasoning, max_tokens, pricing) live in models.toml. + \\ \\Environment: - \\ PANTO_API_STYLE "openai_chat" (default) or "anthropic_messages". - \\ OPENAI_API_KEY, OPENAI_MODEL, OPENAI_BASE_URL, OPENAI_REASONING - \\ ANTHROPIC_API_KEY, ANTHROPIC_MODEL, ANTHROPIC_BASE_URL, - \\ ANTHROPIC_API_VERSION, ANTHROPIC_MAX_TOKENS + \\ OPENAI_API_KEY, ANTHROPIC_API_KEY Consumed by the default providers. \\ PANTO_SESSION_DIR Override the base sessions directory. Defaults to \\ $XDG_DATA_HOME/panto/sessions or ~/.local/share/panto/sessions. \\ PANTO_HOME Override the runtime/rocks tree location. |