models.dev list validation against actual provider's /models

1 files changed, 440 insertions, 10 deletions

diff --git a/src/subcommand.zig b/src/subcommand.zig
index 5238669..9fdcc4c 100644
--- a/src/subcommand.zig
+++ b/src/subcommand.zig
@@ -25,6 +25,7 @@ const luarocks_runtime = @import("luarocks_runtime.zig");
 const self_exe = @import("self_exe.zig");
 const session_paths = @import("session_paths.zig");
 const config_file = @import("config_file.zig");
+const auth_manager = @import("auth_manager.zig");
 const models_toml = @import("models_toml.zig");
 const panto_home = @import("panto_home.zig");
 const panto = @import("panto");
@@ -378,10 +379,19 @@ fn runModelsSubcommand(
     };
     defer cfg.deinit();
 
-    const catalog_json = try fetchModelsDevCatalog(allocator, io);
+    var layout = try panto_home.resolve(allocator, environ_map);
+    defer layout.deinit();
+
+    panto.init(allocator, io);
+    defer panto.deinit();
+    const client = panto.httpClient();
+    var auth_mgr = auth_manager.AuthManager.init(allocator, io, client, layout.auth_dir, &cfg);
+    defer auth_mgr.deinit();
+
+    const catalog_json = try fetchModelsDevCatalog(allocator, client);
     defer allocator.free(catalog_json);
 
-    const sync = try buildSyncedModelsToml(allocator, &cfg, catalog_json);
+    const sync = try buildSyncedModelsToml(allocator, &cfg, &auth_mgr, catalog_json);
     defer allocator.free(sync.content);
 
     const base_models_path = try models_toml.basePath(allocator, environ_map);
@@ -400,18 +410,14 @@ fn runModelsSubcommand(
     );
     if (sync.providers_skipped > 0) {
         try out.print(
-            "{d} configured provider(s) had no matching catalog entry and were skipped\n",
+            "{d} configured provider(s) had no matching catalog entry or no usable synced models and were skipped\n",
             .{sync.providers_skipped},
         );
     }
     try out_file.flush();
 }
 
-fn fetchModelsDevCatalog(allocator: Allocator, io: Io) ![]u8 {
-    panto.init(allocator, io);
-    defer panto.deinit();
-    const client = panto.httpClient();
-
+fn fetchModelsDevCatalog(allocator: Allocator, client: *std.http.Client) ![]u8 {
     var body: std.Io.Writer.Allocating = .init(allocator);
     defer body.deinit();
 
@@ -428,6 +434,7 @@ fn fetchModelsDevCatalog(allocator: Allocator, io: Io) ![]u8 {
 fn buildSyncedModelsToml(
     allocator: Allocator,
     cfg: *const config_file.Config,
+    auth_mgr: *auth_manager.AuthManager,
     catalog_json: []const u8,
 ) !ModelsSyncResult {
     var parsed = try std.json.parseFromSlice(std.json.Value, allocator, catalog_json, .{});
@@ -449,14 +456,17 @@ fn buildSyncedModelsToml(
             "#   ./.panto/models.local.toml\n" ++
             "#\n" ++
             "# Generated from https://models.dev/api.json for the providers currently\n" ++
-            "# configured in panto. Safe to overwrite by running the sync again.\n\n",
+            "# configured in panto. When a configured provider exposes a JSON `GET\n" ++
+            "# /models` endpoint, the synced list is intersected with that live\n" ++
+            "# provider listing to drop unsupported/outdated entries. Safe to\n" ++
+            "# overwrite by running the sync again.\n\n",
     );
 
     var providers_synced: usize = 0;
     var providers_skipped: usize = 0;
     var models_written: usize = 0;
 
-    for (cfg.providers) |p| {
+    for (cfg.providers) |*p| {
         const catalog_name = p.model_catalog_name orelse p.name;
         const provider_v = root.get(catalog_name) orelse {
             providers_skipped += 1;
@@ -481,6 +491,9 @@ fn buildSyncedModelsToml(
             },
         };
 
+        var provider_models_filter = try fetchProviderModelsFilter(allocator, auth_mgr, cfg, p);
+        defer if (provider_models_filter) |*f| f.deinit();
+
         var wrote_provider = false;
         var it_models = models_obj.iterator();
         while (it_models.next()) |kv| {
@@ -496,6 +509,10 @@ fn buildSyncedModelsToml(
             else
                 kv.key_ptr.*;
 
+            if (provider_models_filter) |*filter| {
+                if (!filter.containsForStyle(model_id, p.style)) continue;
+            }
+
             if (!wrote_provider) {
                 wrote_provider = true;
                 providers_synced += 1;
@@ -546,6 +563,362 @@ fn buildSyncedModelsToml(
     };
 }
 
+const ProviderSupportedEndpoints = struct {
+    chat_completions: bool = false,
+    anthropic_messages: bool = false,
+    responses: bool = false,
+
+    fn supportsStyle(self: ProviderSupportedEndpoints, style: config_file.APIStyle) bool {
+        return switch (style) {
+            .openai_chat => self.chat_completions,
+            .anthropic_messages => self.anthropic_messages,
+            .openai_responses, .openai_codex_responses => self.responses,
+        };
+    }
+
+    fn any(self: ProviderSupportedEndpoints) bool {
+        return self.chat_completions or self.anthropic_messages or self.responses;
+    }
+};
+
+const ProviderModelInfo = struct {
+    supported_endpoints_present: bool = false,
+    supported_endpoints: ProviderSupportedEndpoints = .{},
+};
+
+const ProviderModelsFilter = struct {
+    arena: std.heap.ArenaAllocator,
+    models: std.StringHashMap(ProviderModelInfo),
+
+    fn deinit(self: *ProviderModelsFilter) void {
+        self.models.deinit();
+        self.arena.deinit();
+    }
+
+    fn containsForStyle(self: *const ProviderModelsFilter, model_id: []const u8, style: config_file.APIStyle) bool {
+        const info = self.models.get(model_id) orelse return false;
+        if (!info.supported_endpoints_present) return true;
+        return info.supported_endpoints.supportsStyle(style);
+    }
+};
+
+fn fetchProviderModelsFilter(
+    allocator: Allocator,
+    auth_mgr: *auth_manager.AuthManager,
+    cfg: *const config_file.Config,
+    prov: *const config_file.Provider,
+) !?ProviderModelsFilter {
+    const auth = cfg.auth(prov.auth_name) orelse return null;
+    var live: panto.Config = .{ .provider = liveProviderConfigForSync(prov, auth) };
+
+    auth_mgr.resolveInto(&live, prov.name, false, null) catch |err| switch (err) {
+        error.OutOfMemory => return err,
+        else => {
+            std.log.debug("models sync: skipping live /models filter for {s} ({t})", .{ prov.name, err });
+            return null;
+        },
+    };
+
+    return fetchProviderModelsFilterFromLive(allocator, auth_mgr.client, prov.name, live.provider);
+}
+
+fn liveProviderConfigForSync(
+    prov: *const config_file.Provider,
+    auth: *const config_file.ResolvedAuth,
+) panto.ProviderConfig {
+    const api_key: []const u8 = auth.resolved_api_key orelse "";
+    const anthropic_use_bearer_auth = auth.config == .oauth_device;
+    const dummy_model = "__models_probe__";
+
+    return switch (prov.style) {
+        .openai_chat => .{ .openai_chat = .{
+            .api_key = api_key,
+            .base_url = prov.base_url,
+            .model = dummy_model,
+            .extra_headers = prov.extra_headers,
+        } },
+        .anthropic_messages => .{ .anthropic_messages = .{
+            .api_key = api_key,
+            .base_url = prov.base_url,
+            .model = dummy_model,
+            .use_bearer_auth = anthropic_use_bearer_auth,
+            .extra_headers = prov.extra_headers,
+        } },
+        .openai_responses => .{ .openai_responses = .{
+            .api_key = api_key,
+            .base_url = prov.base_url,
+            .model = dummy_model,
+            .extra_headers = prov.extra_headers,
+        } },
+        .openai_codex_responses => .{ .openai_codex_responses = .{
+            .api_key = api_key,
+            .base_url = prov.base_url,
+            .model = dummy_model,
+            .extra_headers = prov.extra_headers,
+        } },
+    };
+}
+
+fn fetchProviderModelsFilterFromLive(
+    allocator: Allocator,
+    client: *std.http.Client,
+    provider_name: []const u8,
+    prov_cfg: panto.ProviderConfig,
+) !?ProviderModelsFilter {
+    const url = try providerModelsURL(allocator, switch (prov_cfg) {
+        .openai_chat => |pcfg| pcfg.base_url,
+        .anthropic_messages => |pcfg| pcfg.base_url,
+        .openai_responses => |pcfg| pcfg.base_url,
+        .openai_codex_responses => |pcfg| pcfg.base_url,
+    });
+    defer allocator.free(url);
+
+    var headers: std.ArrayList(panto.Header) = .empty;
+    defer headers.deinit(allocator);
+    try headers.append(allocator, .{ .name = "User-Agent", .value = "panto models sync" });
+
+    var auth_value: ?[]u8 = null;
+    defer if (auth_value) |v| allocator.free(v);
+
+    switch (prov_cfg) {
+        .openai_chat => |pcfg| {
+            auth_value = try std.fmt.allocPrint(allocator, "Bearer {s}", .{pcfg.api_key});
+            try headers.append(allocator, .{ .name = "authorization", .value = auth_value.? });
+            try headers.appendSlice(allocator, pcfg.extra_headers);
+        },
+        .anthropic_messages => |pcfg| {
+            if (pcfg.use_bearer_auth) {
+                auth_value = try std.fmt.allocPrint(allocator, "Bearer {s}", .{pcfg.api_key});
+                try headers.append(allocator, .{ .name = "authorization", .value = auth_value.? });
+            } else {
+                try headers.append(allocator, .{ .name = "x-api-key", .value = pcfg.api_key });
+            }
+            try headers.append(allocator, .{ .name = "anthropic-version", .value = pcfg.api_version });
+            try headers.appendSlice(allocator, pcfg.extra_headers);
+        },
+        .openai_responses => |pcfg| {
+            auth_value = try std.fmt.allocPrint(allocator, "Bearer {s}", .{pcfg.api_key});
+            try headers.append(allocator, .{ .name = "authorization", .value = auth_value.? });
+            try headers.appendSlice(allocator, pcfg.extra_headers);
+        },
+        .openai_codex_responses => |pcfg| {
+            auth_value = try std.fmt.allocPrint(allocator, "Bearer {s}", .{pcfg.api_key});
+            try headers.append(allocator, .{ .name = "authorization", .value = auth_value.? });
+            try headers.appendSlice(allocator, pcfg.extra_headers);
+        },
+    }
+
+    const res = panto.http.request(allocator, client, .GET, url, .{ .headers = headers.items }) catch |err| switch (err) {
+        error.OutOfMemory => return err,
+        else => {
+            std.log.debug("models sync: {s} /models probe failed ({t})", .{ provider_name, err });
+            return null;
+        },
+    };
+    defer res.deinit();
+
+    if (!res.ok()) {
+        std.log.debug("models sync: {s} /models probe returned HTTP {d}", .{ provider_name, res.status });
+        return null;
+    }
+
+    return parseProviderModelsFilter(allocator, res.body) catch |err| switch (err) {
+        error.OutOfMemory => return err,
+        else => {
+            std.log.debug("models sync: {s} /models probe was not a usable JSON model listing ({t})", .{ provider_name, err });
+            return null;
+        },
+    };
+}
+
+fn providerModelsURL(allocator: Allocator, base_url: []const u8) ![]u8 {
+    const trimmed = std.mem.trim(u8, base_url, "/");
+    return std.fmt.allocPrint(allocator, "{s}/models", .{trimmed});
+}
+
+fn parseProviderModelsFilter(allocator: Allocator, body: []const u8) !?ProviderModelsFilter {
+    var arena = std.heap.ArenaAllocator.init(allocator);
+    errdefer arena.deinit();
+    const aa = arena.allocator();
+
+    const parsed = std.json.parseFromSlice(std.json.Value, aa, body, .{}) catch |err| switch (err) {
+        error.OutOfMemory => return err,
+        else => return null,
+    };
+
+    var models: std.StringHashMap(ProviderModelInfo) = .init(aa);
+    const found = try collectProviderModels(aa, &models, parsed.value);
+    if (!found or models.count() == 0) {
+        models.deinit();
+        arena.deinit();
+        return null;
+    }
+
+    return .{ .arena = arena, .models = models };
+}
+
+fn collectProviderModels(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    root: std.json.Value,
+) !bool {
+    return switch (root) {
+        .array => |arr| collectModelsFromArray(allocator, models, arr.items),
+        .object => |obj| blk: {
+            if (obj.get("data")) |data_v| {
+                if (try collectModelsFromArrayLike(allocator, models, data_v)) break :blk true;
+            }
+            if (obj.get("models")) |models_v| {
+                if (try collectModelsFromModelsNode(allocator, models, models_v)) break :blk true;
+            }
+            break :blk try collectModelsFromTopLevelObjectMap(allocator, models, obj);
+        },
+        else => false,
+    };
+}
+
+fn collectModelsFromArrayLike(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    v: std.json.Value,
+) !bool {
+    return switch (v) {
+        .array => |arr| collectModelsFromArray(allocator, models, arr.items),
+        .object => |obj| collectModelsFromModelsObject(allocator, models, obj),
+        else => false,
+    };
+}
+
+fn collectModelsFromModelsNode(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    v: std.json.Value,
+) !bool {
+    return switch (v) {
+        .array => |arr| collectModelsFromArray(allocator, models, arr.items),
+        .object => |obj| collectModelsFromModelsObject(allocator, models, obj),
+        else => false,
+    };
+}
+
+fn collectModelsFromArray(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    items: []const std.json.Value,
+) !bool {
+    var found = false;
+    for (items) |item| switch (item) {
+        .string => |s| {
+            try putProviderModel(allocator, models, s, null);
+            found = true;
+        },
+        .object => |obj| {
+            const model_id = switch (obj.get("id") orelse continue) {
+                .string => |s| s,
+                else => continue,
+            };
+            try putProviderModel(allocator, models, model_id, obj.get("supported_endpoints"));
+            found = true;
+        },
+        else => {},
+    };
+    return found;
+}
+
+fn collectModelsFromModelsObject(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    obj: std.json.ObjectMap,
+) !bool {
+    var found = false;
+    var it = obj.iterator();
+    while (it.next()) |kv| {
+        const model_id, const supported_endpoints = switch (kv.value_ptr.*) {
+            .object => |child| blk: {
+                const child_id = if (child.get("id")) |id_v|
+                    switch (id_v) {
+                        .string => |s| s,
+                        else => kv.key_ptr.*,
+                    }
+                else
+                    kv.key_ptr.*;
+                break :blk .{ child_id, child.get("supported_endpoints") };
+            },
+            .string => |s| .{ s, null },
+            else => .{ kv.key_ptr.*, null },
+        };
+        try putProviderModel(allocator, models, model_id, supported_endpoints);
+        found = true;
+    }
+    return found;
+}
+
+fn collectModelsFromTopLevelObjectMap(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    obj: std.json.ObjectMap,
+) !bool {
+    var total: usize = 0;
+    var it = obj.iterator();
+    while (it.next()) |kv| {
+        total += 1;
+        switch (kv.value_ptr.*) {
+            .object, .string => {},
+            else => return false,
+        }
+    }
+    if (total == 0) return false;
+    return collectModelsFromModelsObject(allocator, models, obj);
+}
+
+fn parseSupportedEndpoints(v: ?std.json.Value) ProviderSupportedEndpoints {
+    const raw = v orelse return .{};
+    const arr = switch (raw) {
+        .array => |a| a.items,
+        else => return .{},
+    };
+
+    var out: ProviderSupportedEndpoints = .{};
+    for (arr) |item| {
+        const s = switch (item) {
+            .string => |str| str,
+            else => continue,
+        };
+        if (std.mem.eql(u8, s, "/chat/completions")) {
+            out.chat_completions = true;
+        } else if (std.mem.eql(u8, s, "/v1/messages")) {
+            out.anthropic_messages = true;
+        } else if (std.mem.eql(u8, s, "/responses")) {
+            out.responses = true;
+        }
+    }
+    return out;
+}
+
+fn putProviderModel(
+    allocator: Allocator,
+    models: *std.StringHashMap(ProviderModelInfo),
+    model_id: []const u8,
+    supported_endpoints_value: ?std.json.Value,
+) !void {
+    if (model_id.len == 0) return;
+    const supported_endpoints_present = supported_endpoints_value != null;
+    const supported_endpoints = parseSupportedEndpoints(supported_endpoints_value);
+    const gop = try models.getOrPut(model_id);
+    if (!gop.found_existing) {
+        gop.key_ptr.* = try allocator.dupe(u8, model_id);
+        gop.value_ptr.* = .{
+            .supported_endpoints_present = supported_endpoints_present,
+            .supported_endpoints = supported_endpoints,
+        };
+        return;
+    }
+    if (!gop.value_ptr.supported_endpoints_present and supported_endpoints_present) {
+        gop.value_ptr.supported_endpoints_present = true;
+        gop.value_ptr.supported_endpoints = supported_endpoints;
+    }
+}
+
 fn writeTomlTableHeader(w: *std.Io.Writer, provider: []const u8, alias: []const u8) !void {
     try w.writeByte('[');
     try writeTomlBasicString(w, provider);
@@ -826,6 +1199,63 @@ const testing = std.testing;
 //
 // Suppress dead-code warnings for `self_exe` (it's used by main, not
 // by tests in this module).
+test "providerModelsURL appends /models once" {
+    const url = try providerModelsURL(testing.allocator, "https://api.example.com/v1/");
+    defer testing.allocator.free(url);
+    try testing.expectEqualStrings("https://api.example.com/v1/models", url);
+}
+
+test "parseProviderModelsFilter: openai-style data array" {
+    const body =
+        \\{"object":"list","data":[{"id":"gpt-4o"},{"id":"o4-mini"}]}
+    ;
+    var filter = (try parseProviderModelsFilter(testing.allocator, body)).?;
+    defer filter.deinit();
+    try testing.expect(filter.containsForStyle("gpt-4o", .openai_chat));
+    try testing.expect(filter.containsForStyle("o4-mini", .openai_responses));
+    try testing.expect(!filter.containsForStyle("nope", .openai_chat));
+}
+
+test "parseProviderModelsFilter: supported_endpoints further restrict by style" {
+    const body =
+        \\{"data":[{"id":"claude-sonnet-4.6","supported_endpoints":["/chat/completions","/v1/messages"]},{"id":"gpt-5.5","supported_endpoints":["/responses","ws:/responses"]},{"id":"gemini-3.5-flash","supported_endpoints":["/chat/completions"]},{"id":"gpt-4o"}]}
+    ;
+    var filter = (try parseProviderModelsFilter(testing.allocator, body)).?;
+    defer filter.deinit();
+    try testing.expect(filter.containsForStyle("claude-sonnet-4.6", .anthropic_messages));
+    try testing.expect(filter.containsForStyle("claude-sonnet-4.6", .openai_chat));
+    try testing.expect(!filter.containsForStyle("claude-sonnet-4.6", .openai_responses));
+    try testing.expect(filter.containsForStyle("gpt-5.5", .openai_responses));
+    try testing.expect(!filter.containsForStyle("gpt-5.5", .openai_chat));
+    try testing.expect(filter.containsForStyle("gemini-3.5-flash", .openai_chat));
+    try testing.expect(!filter.containsForStyle("gemini-3.5-flash", .anthropic_messages));
+    try testing.expect(filter.containsForStyle("gpt-4o", .openai_chat));
+    try testing.expect(filter.containsForStyle("gpt-4o", .openai_responses));
+}
+
+test "parseProviderModelsFilter: websocket-only responses endpoint does not imply SSE responses support" {
+    const body =
+        \\{"data":[{"id":"ws-only","supported_endpoints":["ws:/responses"]}]}
+    ;
+    var filter = (try parseProviderModelsFilter(testing.allocator, body)).?;
+    defer filter.deinit();
+    try testing.expect(!filter.containsForStyle("ws-only", .openai_responses));
+}
+
+test "parseProviderModelsFilter: keyed models object" {
+    const body =
+        \\{"models":{"claude-sonnet-4":{},"claude-haiku-4":{"id":"claude-haiku-4"}}}
+    ;
+    var filter = (try parseProviderModelsFilter(testing.allocator, body)).?;
+    defer filter.deinit();
+    try testing.expect(filter.containsForStyle("claude-sonnet-4", .anthropic_messages));
+    try testing.expect(filter.containsForStyle("claude-haiku-4", .openai_chat));
+}
+
+test "parseProviderModelsFilter: unrecognized json returns null" {
+    try testing.expect((try parseProviderModelsFilter(testing.allocator, "{\"ok\":true}")) == null);
+}
+
 test {
     _ = self_exe;
 }