anthropic prompt caching

3 files changed, 266 insertions, 8 deletions

diff --git a/libpanto/src/anthropic_messages_json.zig b/libpanto/src/anthropic_messages_json.zig
index 99d89eb..f7df80b 100644
--- a/libpanto/src/anthropic_messages_json.zig
+++ b/libpanto/src/anthropic_messages_json.zig
@@ -120,11 +120,25 @@ pub fn serializeRequest(
     // Emit messages (everything that isn't .system). If the conversation
     // has been compacted, only the latest compaction summary and the
     // messages after it are active; the superseded prefix is dropped.
+    //
+    // Prompt caching: we replicate Anthropic's "automatic caching" by
+    // placing one `cache_control` breakpoint on the last cacheable block of
+    // the request. Anthropic's native API offers a single top-level
+    // `cache_control` field for this, but Anthropic-style proxies
+    // (Bedrock, Vertex, OpenRouter) reject that field outright (400). A
+    // per-block breakpoint produces the same caching behavior and is the
+    // broadly-supported form, so we mark the block directly. As the
+    // append-only conversation grows, this breakpoint advances each request
+    // and stays well inside the 20-block lookback window, so every turn
+    // reads the prior turn's write and writes only its own new suffix.
+    const window = conversation.activeMessageWindow(conv.messages.items);
+    const cache_at: ?CacheMark = if (cfg.prompt_cache) cacheableTailBlock(window) else null;
     try s.objectField("messages");
     try s.beginArray();
-    for (conversation.activeMessageWindow(conv.messages.items)) |msg| {
+    for (window, 0..) |msg, mi| {
         if (msg.role == .system) continue;
-        try writeMessage(&s, msg, allocator);
+        const mark: ?usize = if (cache_at) |c| (if (c.message == mi) c.block else null) else null;
+        try writeMessage(&s, msg, mark);
     }
     try s.endArray();
 
@@ -194,12 +208,50 @@ fn collectSystemPrompt(
     }
 }
 
+/// Location of the prompt-cache breakpoint: the block to stamp with
+/// `cache_control`. Indices are into the active message window and into
+/// that message's `content` array.
+const CacheMark = struct { message: usize, block: usize };
+
+/// Returns whether a content block can carry a `cache_control` marker and
+/// is actually emitted on the wire. Thinking blocks can't be cached
+/// directly, and unsigned thinking blocks are dropped entirely
+/// (`writeBlock` returns early), so neither is a valid breakpoint target.
+/// System blocks are filtered before emission.
+fn isCacheableBlock(block: conversation.ContentBlock) bool {
+    return switch (block) {
+        .Text, .ToolUse, .ToolResult, .CompactionSummary => true,
+        .Thinking, .System => false,
+    };
+}
+
+/// Find the last cacheable, actually-emitted block in the active window:
+/// the spot to place the single cache breakpoint. Scans messages from the
+/// end, skipping system messages and any trailing non-cacheable blocks
+/// (e.g. a thinking block as the final assistant block). Returns null if
+/// nothing cacheable is present (then no breakpoint is emitted).
+fn cacheableTailBlock(window: []const conversation.Message) ?CacheMark {
+    var mi = window.len;
+    while (mi > 0) {
+        mi -= 1;
+        const msg = window[mi];
+        if (msg.role == .system) continue;
+        var bi = msg.content.items.len;
+        while (bi > 0) {
+            bi -= 1;
+            if (isCacheableBlock(msg.content.items[bi])) {
+                return .{ .message = mi, .block = bi };
+            }
+        }
+    }
+    return null;
+}
+
 fn writeMessage(
     s: *std.json.Stringify,
     msg: conversation.Message,
-    allocator: Allocator,
+    cache_block: ?usize,
 ) !void {
-    _ = allocator;
     try s.beginObject();
 
     try s.objectField("role");
@@ -207,15 +259,26 @@ fn writeMessage(
 
     try s.objectField("content");
     try s.beginArray();
-    for (msg.content.items) |block| {
-        try writeBlock(s, block);
+    for (msg.content.items, 0..) |block, bi| {
+        const mark = if (cache_block) |cb| cb == bi else false;
+        try writeBlock(s, block, mark);
     }
     try s.endArray();
 
     try s.endObject();
 }
 
-fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
+/// Emit a `"cache_control": {"type": "ephemeral"}` field. The caller must
+/// be positioned inside an open block object, after its other fields.
+fn writeCacheControl(s: *std.json.Stringify) !void {
+    try s.objectField("cache_control");
+    try s.beginObject();
+    try s.objectField("type");
+    try s.write("ephemeral");
+    try s.endObject();
+}
+
+fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock, mark_cache: bool) !void {
     switch (block) {
         .Text => |tb| {
             try s.beginObject();
@@ -223,6 +286,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
             try s.write("text");
             try s.objectField("text");
             try s.write(tb.items);
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
         .Thinking => |tb| {
@@ -262,6 +326,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
             } else {
                 try writeToolUseInput(s, input_bytes);
             }
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
         .ToolResult => |tr| {
@@ -310,6 +375,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
                 }
             }
             try s.endArray();
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
         // System blocks never reach here: `serializeRequest` filters out
@@ -325,6 +391,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
             try s.write("text");
             try s.objectField("text");
             try s.write(cs.text.items);
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
     }
@@ -655,6 +722,113 @@ test "serializeRequest - system extracted into top-level field" {
     try testing.expectEqualStrings("Hello!", content[0].object.get("text").?.string);
 }
 
+test "serializeRequest - prompt_cache marks last block with cache_control" {
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+    try addUserText(&conv, "Hi");
+
+    var cfg = testConfig("claude-x");
+    cfg.prompt_cache = true;
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    // No top-level field (would break Bedrock/Vertex/OpenRouter).
+    try testing.expect(parsed.value.object.get("cache_control") == null);
+    // The single user text block carries the breakpoint.
+    const block = parsed.value.object.get("messages").?.array.items[0]
+        .object.get("content").?.array.items[0].object;
+    try testing.expectEqualStrings("ephemeral", block.get("cache_control").?.object.get("type").?.string);
+}
+
+test "serializeRequest - cache breakpoint lands on the final block only" {
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+    try addUserText(&conv, "first");
+    try conv.addAssistantMessage(&.{
+        .{ .Text = try conversation.textualBlockFromSlice(allocator, "reply") },
+    }, null);
+    try addUserText(&conv, "second");
+
+    var cfg = testConfig("claude-x");
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    const msgs = parsed.value.object.get("messages").?.array.items;
+    try testing.expectEqual(@as(usize, 3), msgs.len);
+    // Only the last message's only block is marked.
+    try testing.expect(msgs[0].object.get("content").?.array.items[0].object.get("cache_control") == null);
+    try testing.expect(msgs[1].object.get("content").?.array.items[0].object.get("cache_control") == null);
+    try testing.expect(msgs[2].object.get("content").?.array.items[0].object.get("cache_control") != null);
+}
+
+test "serializeRequest - cache breakpoint skips a trailing thinking block" {
+    // A thinking block can't carry cache_control; the breakpoint must fall
+    // back to the prior cacheable block (here the assistant text).
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+
+    const sig = try allocator.dupe(u8, "EqQBCgIYAhIM1gbcDa9GJwZA");
+    try conv.addAssistantMessage(&.{
+        .{ .Text = try conversation.textualBlockFromSlice(allocator, "answer") },
+        .{ .Thinking = .{
+            .text = try conversation.textualBlockFromSlice(allocator, "trailing thought"),
+            .signature = sig,
+        } },
+    }, null);
+
+    var cfg = testConfig("claude-x");
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    const content = parsed.value.object.get("messages").?.array.items[0]
+        .object.get("content").?.array.items;
+    // text block marked, thinking block not.
+    try testing.expectEqualStrings("text", content[0].object.get("type").?.string);
+    try testing.expect(content[0].object.get("cache_control") != null);
+    try testing.expectEqualStrings("thinking", content[1].object.get("type").?.string);
+    try testing.expect(content[1].object.get("cache_control") == null);
+}
+
+test "serializeRequest - prompt_cache disabled omits all cache_control" {
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+    try addUserText(&conv, "Hi");
+
+    var cfg = testConfig("claude-x");
+    cfg.prompt_cache = false;
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    try testing.expect(parsed.value.object.get("cache_control") == null);
+    const block = parsed.value.object.get("messages").?.array.items[0]
+        .object.get("content").?.array.items[0].object;
+    try testing.expect(block.get("cache_control") == null);
+}
+
 test "serializeRequest - multiple system messages joined with horizontal rule" {
     const allocator = testing.allocator;
 
diff --git a/libpanto/src/config.zig b/libpanto/src/config.zig
index 4aae81f..7e1e8e1 100644
--- a/libpanto/src/config.zig
+++ b/libpanto/src/config.zig
@@ -101,6 +101,18 @@ pub const AnthropicMessagesConfig = struct {
     /// between tool calls. Ignored when `thinking == .adaptive` (interleaving
     /// is automatic there) or `.disabled`.
     thinking_interleaved: bool = false,
+    /// Place one `cache_control` breakpoint on the last cacheable block of
+    /// each request, replicating Anthropic's "automatic caching" (a single
+    /// advancing breakpoint) via the broadly-supported per-block marker
+    /// rather than the top-level field.
+    ///
+    /// Defaults on: for a normal append-only multi-turn session it's a
+    /// near-pure win (reads 0.1x base input vs. a one-time 1.25x write).
+    /// Set false when the workload makes that write premium unrecoverable
+    /// — e.g. one-shot requests, or an embedder that aggressively rewrites
+    /// history so prefixes are never reused. There the 1.25x write is pure
+    /// overhead with no read to amortize it.
+    prompt_cache: bool = true,
 };
 
 /// Per-provider transport/auth/model configuration. Tagged by `APIStyle`.
diff --git a/src/config_file.zig b/src/config_file.zig
index f5ed79c..1858520 100644
--- a/src/config_file.zig
+++ b/src/config_file.zig
@@ -25,7 +25,10 @@
 //!     by `api_key_env_var`. A provider whose only key source is an absent
 //!     env var is **silently dropped** — this is what lets the base layer
 //!     ship default OpenAI/Anthropic providers that simply don't appear when
-//!     the user hasn't exported the corresponding key.
+//!     the user hasn't exported the corresponding key. An `anthropic_messages`
+//!     provider may also set `prompt_cache = <bool>` (default true) to control
+//!     the advancing `cache_control` breakpoint; it is ignored for
+//!     `openai_chat` providers.
 //!   - `defaults.model` is parsed as `<provider_name>:<model_alias>`.
 //!   - `tools` / `extensions` allow- and deny-lists are captured verbatim
 //!     (as glob pattern lists). A pattern appearing in both allow and deny
@@ -67,6 +70,10 @@ pub const Provider = struct {
     style: APIStyle,
     base_url: []const u8,
     api_key: []const u8,
+    /// anthropic_messages only. Place one advancing `cache_control`
+    /// breakpoint on each request. Defaults to the library default (true);
+    /// ignored for `openai_chat` providers.
+    prompt_cache: bool = true,
 
     pub fn deinit(self: Provider, alloc: Allocator) void {
         alloc.free(self.name);
@@ -150,6 +157,7 @@ pub fn buildProviderConfig(
             .effort = if (def_opt) |d| d.effort else .medium,
             .thinking_budget_tokens = if (def_opt) |d| d.thinking_budget_tokens else 32_000,
             .thinking_interleaved = if (def_opt) |d| d.thinking_interleaved else false,
+            .prompt_cache = prov.prompt_cache,
         } },
     }
 }
@@ -545,11 +553,20 @@ fn resolveProvider(
         return null;
     };
 
+    // anthropic_messages only; absent => library default (true).
+    const prompt_cache: bool = blk: {
+        if (val.get("prompt_cache")) |pc| {
+            if (pc.asBool()) |b| break :blk b;
+        }
+        break :blk true;
+    };
+
     return .{
         .name = try allocator.dupe(u8, name),
         .style = style,
         .base_url = try allocator.dupe(u8, base_url),
         .api_key = try allocator.dupe(u8, api_key),
+        .prompt_cache = prompt_cache,
     };
 }
 
@@ -732,6 +749,61 @@ test "resolve: env-backed provider resolves when env var is set" {
     try testing.expectEqualStrings("sk-ant-xyz", p.api_key);
 }
 
+test "resolve: prompt_cache defaults true and is parsed when set" {
+    const a = testing.allocator;
+    var env = emptyEnv(a);
+    defer env.deinit();
+
+    const src =
+        \\[providers.anthropic]
+        \\style = "anthropic_messages"
+        \\base_url = "https://api.anthropic.com"
+        \\api_key = "sk-ant"
+        \\
+        \\[providers.anthropic_nocache]
+        \\style = "anthropic_messages"
+        \\base_url = "https://api.anthropic.com"
+        \\api_key = "sk-ant"
+        \\prompt_cache = false
+    ;
+    const doc = try parseDoc(a, src);
+    defer doc.deinit();
+
+    var cfg = try resolve(a, &env, doc.root);
+    defer cfg.deinit();
+
+    // Absent => library default (true).
+    try testing.expectEqual(true, cfg.provider("anthropic").?.prompt_cache);
+    // Explicit false is honoured.
+    try testing.expectEqual(false, cfg.provider("anthropic_nocache").?.prompt_cache);
+}
+
+test "buildProviderConfig: prompt_cache flows from provider into anthropic config" {
+    const a = testing.allocator;
+    var env = emptyEnv(a);
+    defer env.deinit();
+
+    const src =
+        \\[providers.anthropic]
+        \\style = "anthropic_messages"
+        \\base_url = "https://api.anthropic.com"
+        \\api_key = "sk-ant"
+        \\prompt_cache = false
+    ;
+    const doc = try parseDoc(a, src);
+    defer doc.deinit();
+
+    var cfg = try resolve(a, &env, doc.root);
+    defer cfg.deinit();
+
+    var defs = models_toml.ModelRegistry.init(a);
+    defer defs.deinit();
+
+    const pc = try buildProviderConfig(&cfg, &defs, .{ .provider = "anthropic", .model = "sonnet" });
+    try testing.expectEqual(APIStyle.anthropic_messages, pc.style());
+    try testing.expectEqual(false, pc.anthropic_messages.prompt_cache);
+}
+
 test "resolve: inline api_key wins over env var" {
     const a = testing.allocator;
     var env = emptyEnv(a);