From ecadcfa7a1e549d6ddb8611bdf44cac852064cea Mon Sep 17 00:00:00 2001 From: t Date: Fri, 12 Jun 2026 13:57:26 -0600 Subject: anthropic prompt caching --- libpanto/src/anthropic_messages_json.zig | 188 +++++++++++++++++++++++++++++-- 1 file changed, 181 insertions(+), 7 deletions(-) (limited to 'libpanto/src/anthropic_messages_json.zig') diff --git a/libpanto/src/anthropic_messages_json.zig b/libpanto/src/anthropic_messages_json.zig index 99d89eb..f7df80b 100644 --- a/libpanto/src/anthropic_messages_json.zig +++ b/libpanto/src/anthropic_messages_json.zig @@ -120,11 +120,25 @@ pub fn serializeRequest( // Emit messages (everything that isn't .system). If the conversation // has been compacted, only the latest compaction summary and the // messages after it are active; the superseded prefix is dropped. + // + // Prompt caching: we replicate Anthropic's "automatic caching" by + // placing one `cache_control` breakpoint on the last cacheable block of + // the request. Anthropic's native API offers a single top-level + // `cache_control` field for this, but Anthropic-style proxies + // (Bedrock, Vertex, OpenRouter) reject that field outright (400). A + // per-block breakpoint produces the same caching behavior and is the + // broadly-supported form, so we mark the block directly. As the + // append-only conversation grows, this breakpoint advances each request + // and stays well inside the 20-block lookback window, so every turn + // reads the prior turn's write and writes only its own new suffix. + const window = conversation.activeMessageWindow(conv.messages.items); + const cache_at: ?CacheMark = if (cfg.prompt_cache) cacheableTailBlock(window) else null; try s.objectField("messages"); try s.beginArray(); - for (conversation.activeMessageWindow(conv.messages.items)) |msg| { + for (window, 0..) |msg, mi| { if (msg.role == .system) continue; - try writeMessage(&s, msg, allocator); + const mark: ?usize = if (cache_at) |c| (if (c.message == mi) c.block else null) else null; + try writeMessage(&s, msg, mark); } try s.endArray(); @@ -194,12 +208,50 @@ fn collectSystemPrompt( } } +/// Location of the prompt-cache breakpoint: the block to stamp with +/// `cache_control`. Indices are into the active message window and into +/// that message's `content` array. +const CacheMark = struct { message: usize, block: usize }; + +/// Returns whether a content block can carry a `cache_control` marker and +/// is actually emitted on the wire. Thinking blocks can't be cached +/// directly, and unsigned thinking blocks are dropped entirely +/// (`writeBlock` returns early), so neither is a valid breakpoint target. +/// System blocks are filtered before emission. +fn isCacheableBlock(block: conversation.ContentBlock) bool { + return switch (block) { + .Text, .ToolUse, .ToolResult, .CompactionSummary => true, + .Thinking, .System => false, + }; +} + +/// Find the last cacheable, actually-emitted block in the active window: +/// the spot to place the single cache breakpoint. Scans messages from the +/// end, skipping system messages and any trailing non-cacheable blocks +/// (e.g. a thinking block as the final assistant block). Returns null if +/// nothing cacheable is present (then no breakpoint is emitted). +fn cacheableTailBlock(window: []const conversation.Message) ?CacheMark { + var mi = window.len; + while (mi > 0) { + mi -= 1; + const msg = window[mi]; + if (msg.role == .system) continue; + var bi = msg.content.items.len; + while (bi > 0) { + bi -= 1; + if (isCacheableBlock(msg.content.items[bi])) { + return .{ .message = mi, .block = bi }; + } + } + } + return null; +} + fn writeMessage( s: *std.json.Stringify, msg: conversation.Message, - allocator: Allocator, + cache_block: ?usize, ) !void { - _ = allocator; try s.beginObject(); try s.objectField("role"); @@ -207,15 +259,26 @@ fn writeMessage( try s.objectField("content"); try s.beginArray(); - for (msg.content.items) |block| { - try writeBlock(s, block); + for (msg.content.items, 0..) |block, bi| { + const mark = if (cache_block) |cb| cb == bi else false; + try writeBlock(s, block, mark); } try s.endArray(); try s.endObject(); } -fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void { +/// Emit a `"cache_control": {"type": "ephemeral"}` field. The caller must +/// be positioned inside an open block object, after its other fields. +fn writeCacheControl(s: *std.json.Stringify) !void { + try s.objectField("cache_control"); + try s.beginObject(); + try s.objectField("type"); + try s.write("ephemeral"); + try s.endObject(); +} + +fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock, mark_cache: bool) !void { switch (block) { .Text => |tb| { try s.beginObject(); @@ -223,6 +286,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void { try s.write("text"); try s.objectField("text"); try s.write(tb.items); + if (mark_cache) try writeCacheControl(s); try s.endObject(); }, .Thinking => |tb| { @@ -262,6 +326,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void { } else { try writeToolUseInput(s, input_bytes); } + if (mark_cache) try writeCacheControl(s); try s.endObject(); }, .ToolResult => |tr| { @@ -310,6 +375,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void { } } try s.endArray(); + if (mark_cache) try writeCacheControl(s); try s.endObject(); }, // System blocks never reach here: `serializeRequest` filters out @@ -325,6 +391,7 @@ fn writeBlock(s: *std.json.Stringify, block: conversation.ContentBlock) !void { try s.write("text"); try s.objectField("text"); try s.write(cs.text.items); + if (mark_cache) try writeCacheControl(s); try s.endObject(); }, } @@ -655,6 +722,113 @@ test "serializeRequest - system extracted into top-level field" { try testing.expectEqualStrings("Hello!", content[0].object.get("text").?.string); } +test "serializeRequest - prompt_cache marks last block with cache_control" { + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try addUserText(&conv, "Hi"); + + var cfg = testConfig("claude-x"); + cfg.prompt_cache = true; + var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); + defer empty_tools.deinit(); + const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + // No top-level field (would break Bedrock/Vertex/OpenRouter). + try testing.expect(parsed.value.object.get("cache_control") == null); + // The single user text block carries the breakpoint. + const block = parsed.value.object.get("messages").?.array.items[0] + .object.get("content").?.array.items[0].object; + try testing.expectEqualStrings("ephemeral", block.get("cache_control").?.object.get("type").?.string); +} + +test "serializeRequest - cache breakpoint lands on the final block only" { + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try addUserText(&conv, "first"); + try conv.addAssistantMessage(&.{ + .{ .Text = try conversation.textualBlockFromSlice(allocator, "reply") }, + }, null); + try addUserText(&conv, "second"); + + var cfg = testConfig("claude-x"); + var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); + defer empty_tools.deinit(); + const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + const msgs = parsed.value.object.get("messages").?.array.items; + try testing.expectEqual(@as(usize, 3), msgs.len); + // Only the last message's only block is marked. + try testing.expect(msgs[0].object.get("content").?.array.items[0].object.get("cache_control") == null); + try testing.expect(msgs[1].object.get("content").?.array.items[0].object.get("cache_control") == null); + try testing.expect(msgs[2].object.get("content").?.array.items[0].object.get("cache_control") != null); +} + +test "serializeRequest - cache breakpoint skips a trailing thinking block" { + // A thinking block can't carry cache_control; the breakpoint must fall + // back to the prior cacheable block (here the assistant text). + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + + const sig = try allocator.dupe(u8, "EqQBCgIYAhIM1gbcDa9GJwZA"); + try conv.addAssistantMessage(&.{ + .{ .Text = try conversation.textualBlockFromSlice(allocator, "answer") }, + .{ .Thinking = .{ + .text = try conversation.textualBlockFromSlice(allocator, "trailing thought"), + .signature = sig, + } }, + }, null); + + var cfg = testConfig("claude-x"); + var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); + defer empty_tools.deinit(); + const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + const content = parsed.value.object.get("messages").?.array.items[0] + .object.get("content").?.array.items; + // text block marked, thinking block not. + try testing.expectEqualStrings("text", content[0].object.get("type").?.string); + try testing.expect(content[0].object.get("cache_control") != null); + try testing.expectEqualStrings("thinking", content[1].object.get("type").?.string); + try testing.expect(content[1].object.get("cache_control") == null); +} + +test "serializeRequest - prompt_cache disabled omits all cache_control" { + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + try addUserText(&conv, "Hi"); + + var cfg = testConfig("claude-x"); + cfg.prompt_cache = false; + var empty_tools = tool_registry_mod.ToolRegistry.init(allocator); + defer empty_tools.deinit(); + const body = try serializeRequest(allocator, &cfg, &conv, &empty_tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + try testing.expect(parsed.value.object.get("cache_control") == null); + const block = parsed.value.object.get("messages").?.array.items[0] + .object.get("content").?.array.items[0].object; + try testing.expect(block.get("cache_control") == null); +} + test "serializeRequest - multiple system messages joined with horizontal rule" { const allocator = testing.allocator; -- cgit v1.3