anthropic prompt caching

2 files changed, 193 insertions, 7 deletions

diff --git a/libpanto/src/anthropic_messages_json.zig b/libpanto/src/anthropic_messages_json.zig
index 99d89eb..f7df80b 100644
--- a/libpanto/src/anthropic_messages_json.zig
+++ b/libpanto/src/anthropic_messages_json.zig
@@ -120,11 +120,25 @@ pub fn serializeRequest(
     // Emit messages (everything that isn't .system). If the conversation
     // has been compacted, only the latest compaction summary and the
     // messages after it are active; the superseded prefix is dropped.
+    //
+    // Prompt caching: we replicate Anthropic's "automatic caching" by
+    // placing one `cache_control` breakpoint on the last cacheable block of
+    // the request. Anthropic's native API offers a single top-level
+    // `cache_control` field for this, but Anthropic-style proxies
+    // (Bedrock, Vertex, OpenRouter) reject that field outright (400). A
+    // per-block breakpoint produces the same caching behavior and is the
+    // broadly-supported form, so we mark the block directly. As the
+    // append-only conversation grows, this breakpoint advances each request
+    // and stays well inside the 20-block lookback window, so every turn
+    // reads the prior turn's write and writes only its own new suffix.
+    const window = conversation.activeMessageWindow(conv.messages.items);
+    const cache_at: ?CacheMark = if (cfg.prompt_cache) cacheableTailBlock(window) else null;
     try s.objectField("messages");
     try s.beginArray();
-    for (conversation.activeMessageWindow(conv.messages.items)) |msg| {
+    for (window, 0..) |msg, mi| {
         if (msg.role == .system) continue;
-        try writeMessage(&s, msg, allocator);
+        const mark: ?usize = if (cache_at) |c| (if (c.message == mi) c.block else null) else null;
+        try writeMessage(&s, msg, mark);
     }
     try s.endArray();
 
@@ -194,12 +208,50 @@ fn collectSystemPrompt(
     }
 }
 
+/// Location of the prompt-cache breakpoint: the block to stamp with
+/// `cache_control`. Indices are into the active message window and into
+/// that message's `content` array.
+const CacheMark = struct { message: usize, block: usize };
+
+/// Returns whether a content block can carry a `cache_control` marker and
+/// is actually emitted on the wire. Thinking blocks can't be cached
+/// directly, and unsigned thinking blocks are dropped entirely
+/// (`writeBlock` returns early), so neither is a valid breakpoint target.
+/// System blocks are filtered before emission.
+fn isCacheableBlock(block: conversation.ContentBlock) bool {
+    return switch (block) {
+        .Text, .ToolUse, .ToolResult, .CompactionSummary => true,
+        .Thinking, .System => false,
+    };
+}
+
+/// Find the last cacheable, actually-emitted block in the active window:
+/// the spot to place the single cache breakpoint. Scans messages from the
+/// end, skipping system messages and any trailing non-cacheable blocks
+/// (e.g. a thinking block as the final assistant block). Returns null if
+/// nothing cacheable is present (then no breakpoint is emitted).
+fn cacheableTailBlock(window: []const conversation.Message) ?CacheMark {
+    var mi = window.len;
+    while (mi > 0) {
+        mi -= 1;
+        const msg = window[mi];
+        if (msg.role == .system) continue;
+        var bi = msg.content.items.len;
+        while (bi > 0) {
+            bi -= 1;
+            if (isCacheableBlock(msg.content.items[bi])) {
+                return .{ .message = mi, .block = bi };
+            }
+        }
+    }
+    return null;
+}
+
 fn writeMessage(
     s: *std.json.Stringify,
     msg: conversation.Message,
-    allocator: Allocator,
+    cache_block: ?usize,
 ) !void {
-    _ = allocator;
     try s.beginObject();
 
     try s.objectField("role");
@@ -207,15 +259,26 @@ fn writeMessage(
 
     try s.objectField("content");
     try s.beginArray();
-    for (msg.content.items) |block| {
-        try writeBlock(s, block);
+    for (msg.content.items, 0..) |block, bi| {
+        const mark = if (cache_block) |cb| cb == bi else false;
+        try writeBlock(s, block, mark);
     }
     try s.endArray();
 
     try s.endObject();
 }
 
-fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
+/// Emit a `"cache_control": {"type": "ephemeral"}` field. The caller must
+/// be positioned inside an open block object, after its other fields.
+fn writeCacheControl(s: *std.json.Stringify) !void {
+    try s.objectField("cache_control");
+    try s.beginObject();
+    try s.objectField("type");
+    try s.write("ephemeral");
+    try s.endObject();
+}
+
+fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock, mark_cache: bool) !void {
     switch (block) {
         .Text => |tb| {
             try s.beginObject();
@@ -223,6 +286,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
             try s.write("text");
             try s.objectField("text");
             try s.write(tb.items);
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
         .Thinking => |tb| {
@@ -262,6 +326,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
             } else {
                 try writeToolUseInput(s, input_bytes);
             }
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
         .ToolResult => |tr| {
@@ -310,6 +375,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
                 }
             }
             try s.endArray();
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
         // System blocks never reach here: `serializeRequest` filters out
@@ -325,6 +391,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void {
             try s.write("text");
             try s.objectField("text");
             try s.write(cs.text.items);
+            if (mark_cache) try writeCacheControl(s);
             try s.endObject();
         },
     }
@@ -655,6 +722,113 @@ test "serializeRequest - system extracted into top-level field" {
     try testing.expectEqualStrings("Hello!", content[0].object.get("text").?.string);
 }
 
+test "serializeRequest - prompt_cache marks last block with cache_control" {
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+    try addUserText(&conv, "Hi");
+
+    var cfg = testConfig("claude-x");
+    cfg.prompt_cache = true;
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    // No top-level field (would break Bedrock/Vertex/OpenRouter).
+    try testing.expect(parsed.value.object.get("cache_control") == null);
+    // The single user text block carries the breakpoint.
+    const block = parsed.value.object.get("messages").?.array.items[0]
+        .object.get("content").?.array.items[0].object;
+    try testing.expectEqualStrings("ephemeral", block.get("cache_control").?.object.get("type").?.string);
+}
+
+test "serializeRequest - cache breakpoint lands on the final block only" {
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+    try addUserText(&conv, "first");
+    try conv.addAssistantMessage(&.{
+        .{ .Text = try conversation.textualBlockFromSlice(allocator, "reply") },
+    }, null);
+    try addUserText(&conv, "second");
+
+    var cfg = testConfig("claude-x");
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    const msgs = parsed.value.object.get("messages").?.array.items;
+    try testing.expectEqual(@as(usize, 3), msgs.len);
+    // Only the last message's only block is marked.
+    try testing.expect(msgs[0].object.get("content").?.array.items[0].object.get("cache_control") == null);
+    try testing.expect(msgs[1].object.get("content").?.array.items[0].object.get("cache_control") == null);
+    try testing.expect(msgs[2].object.get("content").?.array.items[0].object.get("cache_control") != null);
+}
+
+test "serializeRequest - cache breakpoint skips a trailing thinking block" {
+    // A thinking block can't carry cache_control; the breakpoint must fall
+    // back to the prior cacheable block (here the assistant text).
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+
+    const sig = try allocator.dupe(u8, "EqQBCgIYAhIM1gbcDa9GJwZA");
+    try conv.addAssistantMessage(&.{
+        .{ .Text = try conversation.textualBlockFromSlice(allocator, "answer") },
+        .{ .Thinking = .{
+            .text = try conversation.textualBlockFromSlice(allocator, "trailing thought"),
+            .signature = sig,
+        } },
+    }, null);
+
+    var cfg = testConfig("claude-x");
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    const content = parsed.value.object.get("messages").?.array.items[0]
+        .object.get("content").?.array.items;
+    // text block marked, thinking block not.
+    try testing.expectEqualStrings("text", content[0].object.get("type").?.string);
+    try testing.expect(content[0].object.get("cache_control") != null);
+    try testing.expectEqualStrings("thinking", content[1].object.get("type").?.string);
+    try testing.expect(content[1].object.get("cache_control") == null);
+}
+
+test "serializeRequest - prompt_cache disabled omits all cache_control" {
+    const allocator = testing.allocator;
+
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+    try addUserText(&conv, "Hi");
+
+    var cfg = testConfig("claude-x");
+    cfg.prompt_cache = false;
+    var empty_tools = tool_registry_mod.ToolRegistry.init(allocator);
+    defer empty_tools.deinit();
+    const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools);
+    defer allocator.free(body);
+
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+    try testing.expect(parsed.value.object.get("cache_control") == null);
+    const block = parsed.value.object.get("messages").?.array.items[0]
+        .object.get("content").?.array.items[0].object;
+    try testing.expect(block.get("cache_control") == null);
+}
+
 test "serializeRequest - multiple system messages joined with horizontal rule" {
     const allocator = testing.allocator;
 
diff --git a/libpanto/src/config.zig b/libpanto/src/config.zig
index 4aae81f..7e1e8e1 100644
--- a/libpanto/src/config.zig
+++ b/libpanto/src/config.zig
@@ -101,6 +101,18 @@ pub const AnthropicMessagesConfig = struct {
     /// between tool calls. Ignored when `thinking == .adaptive` (interleaving
     /// is automatic there) or `.disabled`.
     thinking_interleaved: bool = false,
+    /// Place one `cache_control` breakpoint on the last cacheable block of
+    /// each request, replicating Anthropic's "automatic caching" (a single
+    /// advancing breakpoint) via the broadly-supported per-block marker
+    /// rather than the top-level field.
+    ///
+    /// Defaults on: for a normal append-only multi-turn session it's a
+    /// near-pure win (reads 0.1x base input vs. a one-time 1.25x write).
+    /// Set false when the workload makes that write premium unrecoverable
+    /// — e.g. one-shot requests, or an embedder that aggressively rewrites
+    /// history so prefixes are never reused. There the 1.25x write is pure
+    /// overhead with no read to amortize it.
+    prompt_cache: bool = true,
 };
 
 /// Per-provider transport/auth/model configuration. Tagged by `APIStyle`.