Stage and apply event-field overrides at tool lifecycle boundaries

Stage writable event-field overrides by tool call id so they can be applied at the correct point in the turn lifecycle. Capture input overrides from tool_call_complete, capture output overrides from tool_result, and clear staged overrides defensively at turn start and shutdown.

Also honor user_message text overrides when starting a turn, add the helper that applies staged overrides to the agent at dispatch boundaries, and cover the behavior with tests for staged input/output overrides.

2 files changed, 213 insertions, 0 deletions

diff --git a/libpanto/src/agent.zig b/libpanto/src/agent.zig
index faf0a4a..3616ab6 100644
--- a/libpanto/src/agent.zig
+++ b/libpanto/src/agent.zig
@@ -374,6 +374,68 @@ pub const Agent = struct {
         self.flushPersist();
     }
 
+    /// Replace the text of the ToolResult block matching `tool_use_id` in
+    /// the conversation's LAST message. Media parts are preserved (the new
+    /// text becomes the single leading text part); `is_error` is untouched.
+    /// Returns false when the last message holds no such block.
+    ///
+    /// This is the application half of the embedder's "tool_result output
+    /// override" (a `tool_result` event handler assigning `ev.output`):
+    /// called between tool dispatch and the next provider request, the
+    /// override becomes canonical — wire, persistence, and compaction all
+    /// see the replaced text.
+    pub fn overrideToolResultOutput(self: *Agent, tool_use_id: []const u8, text: []const u8) !bool {
+        const msgs = self.conversation.messages.items;
+        if (msgs.len == 0) return false;
+        const last = &msgs[msgs.len - 1];
+        if (last.role != .user) return false;
+        for (last.content.items) |*block| {
+            if (block.* != .ToolResult) continue;
+            const tr = &block.ToolResult;
+            if (!std.mem.eql(u8, tr.tool_use_id, tool_use_id)) continue;
+
+            var new_parts: std.ArrayList(conversation.ResultPartStored) = .empty;
+            errdefer new_parts.deinit(self._allocator);
+            try new_parts.ensureTotalCapacity(self._allocator, tr.parts.items.len + 1);
+            var new_text: conversation.TextualBlock = .empty;
+            try new_text.appendSlice(self._allocator, text);
+            errdefer new_text.deinit(self._allocator);
+            new_parts.appendAssumeCapacity(.{ .text = new_text });
+            for (tr.parts.items) |*p| switch (p.*) {
+                .text => |*t| t.deinit(self._allocator),
+                .media => |m| new_parts.appendAssumeCapacity(.{ .media = m }),
+            };
+            tr.parts.deinit(self._allocator);
+            tr.parts = new_parts;
+            return true;
+        }
+        return false;
+    }
+
+    /// Replace the input JSON of the ToolUse block matching `tool_use_id`
+    /// in the conversation's LAST message (the committed assistant message,
+    /// between response completion and tool dispatch). Returns false when
+    /// no such block exists there.
+    ///
+    /// Application half of the "tool_call_complete input override": the
+    /// rewritten input is what the tool executes with AND what the model
+    /// sees of its own call in future requests.
+    pub fn overrideToolUseInput(self: *Agent, tool_use_id: []const u8, input_json: []const u8) !bool {
+        const msgs = self.conversation.messages.items;
+        if (msgs.len == 0) return false;
+        const last = &msgs[msgs.len - 1];
+        if (last.role != .assistant) return false;
+        for (last.content.items) |*block| {
+            if (block.* != .ToolUse) continue;
+            const tu = &block.ToolUse;
+            if (!std.mem.eql(u8, tu.id, tool_use_id)) continue;
+            tu.input.clearRetainingCapacity();
+            try tu.input.appendSlice(self._allocator, input_json);
+            return true;
+        }
+        return false;
+    }
+
     /// Submit a user message and begin a turn, returning a resumable pull
     /// `Stream`.
     ///
@@ -2216,6 +2278,60 @@ test "agent runs a turn against NullStore without persisting or erroring" {
     try testing.expectEqual(@as(usize, 2), agent.conversation.messages.items.len);
 }
 
+test "override mutators rewrite the conversation tail's tool blocks" {
+    const allocator = testing.allocator;
+    var threaded: std.Io.Threaded = .init(allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+    var h = TestHarness.init(allocator);
+    defer h.deinit();
+    h.activate();
+    var ns = null_store_mod.NullStore.init(allocator);
+    const agent = try Agent.init(allocator, io, &h.config, ns.store().create(), null);
+    defer agent.deinit();
+
+    const conv = &agent.conversation;
+    // Committed assistant message carrying one ToolUse.
+    var a_content: std.ArrayList(conversation.ContentBlock) = .empty;
+    try a_content.append(allocator, .{ .ToolUse = .{
+        .id = try allocator.dupe(u8, "tc_1"),
+        .name = try allocator.dupe(u8, "read"),
+        .input = try conversation.textualBlockFromSlice(allocator, "{\"path\":\"orig\"}"),
+    } });
+    try conv.messages.append(allocator, .{ .role = .assistant, .content = a_content });
+
+    // Input override rewrites the tail's matching ToolUse; unknown ids miss.
+    try testing.expect(try agent.overrideToolUseInput("tc_1", "{\"path\":\"new\"}"));
+    try testing.expect(!try agent.overrideToolUseInput("tc_missing", "{}"));
+    const tu = conv.messages.items[0].content.items[0].ToolUse;
+    try testing.expectEqualStrings("{\"path\":\"new\"}", tu.input.items);
+
+    // Tool-result user message: a text part followed by a media part.
+    var parts: std.ArrayList(conversation.ResultPartStored) = .empty;
+    try parts.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "original output") });
+    try parts.append(allocator, .{ .media = .{
+        .media_type = try allocator.dupe(u8, "image/png"),
+        .data = try conversation.textualBlockFromSlice(allocator, "aW1n"),
+    } });
+    var u_content: std.ArrayList(conversation.ContentBlock) = .empty;
+    try u_content.append(allocator, .{ .ToolResult = .{
+        .tool_use_id = try allocator.dupe(u8, "tc_1"),
+        .parts = parts,
+    } });
+    try conv.messages.append(allocator, .{ .role = .user, .content = u_content });
+
+    // The ToolUse override only targets the tail — now a user message.
+    try testing.expect(!try agent.overrideToolUseInput("tc_1", "{}"));
+
+    // Output override replaces the text, keeps the media part.
+    try testing.expect(try agent.overrideToolResultOutput("tc_1", "replaced output"));
+    try testing.expect(!try agent.overrideToolResultOutput("tc_other", "x"));
+    const tr = conv.messages.items[1].content.items[0].ToolResult;
+    try testing.expectEqual(@as(usize, 2), tr.parts.items.len);
+    try testing.expectEqualStrings("replaced output", tr.parts.items[0].text.items);
+    try testing.expectEqualStrings("image/png", tr.parts.items[1].media.media_type);
+}
+
 /// A configurable ToolSource for testing the grouped-dispatch path.
 /// Stores every batch it receives so tests can assert "calls X and Y
 /// arrived in the same batch on the same thread".
diff --git a/libpanto/src/openai_responses_json.zig b/libpanto/src/openai_responses_json.zig
index cb58c2c..358fc45 100644
--- a/libpanto/src/openai_responses_json.zig
+++ b/libpanto/src/openai_responses_json.zig
@@ -169,9 +169,14 @@ fn writeInputForMessage(
                 if (b == .ToolResult) has_tool_result = true;
             }
             if (has_tool_result) {
+                // `function_call_output` is text-only. Each result carries
+                // its text (plus a short note when media is present); the
+                // media rides along afterward in a synthetic user message.
+                var any_media = false;
                 for (msg.content.items) |block| {
                     if (block != .ToolResult) continue;
                     const tr = block.ToolResult;
+                    if (tr.hasMedia()) any_media = true;
                     try s.beginObject();
                     try s.objectField("type");
                     try s.write("function_call_output");
@@ -181,9 +186,52 @@ fn writeInputForMessage(
                     var tbuf: std.ArrayList(u8) = .empty;
                     defer tbuf.deinit(allocator);
                     try tr.appendTextInto(allocator, &tbuf);
+                    if (tr.hasMedia()) {
+                        if (tbuf.items.len > 0) try tbuf.append(allocator, '\n');
+                        try tbuf.appendSlice(allocator, "[attachment(s) provided in the following user message]");
+                    }
                     try s.write(tbuf.items);
                     try s.endObject();
                 }
+                // Synthetic user message holding the media: images as
+                // `input_image` data URLs, PDFs as `input_file` (the
+                // Responses API rejects non-image data URLs in
+                // `input_image`).
+                if (any_media) {
+                    try s.beginObject();
+                    try s.objectField("role");
+                    try s.write("user");
+                    try s.objectField("content");
+                    try s.beginArray();
+                    for (msg.content.items) |block| {
+                        if (block != .ToolResult) continue;
+                        for (block.ToolResult.parts.items) |part| {
+                            if (part != .media) continue;
+                            const m = part.media;
+                            var url_buf: std.ArrayList(u8) = .empty;
+                            defer url_buf.deinit(allocator);
+                            try url_buf.appendSlice(allocator, "data:");
+                            try url_buf.appendSlice(allocator, m.media_type);
+                            try url_buf.appendSlice(allocator, ";base64,");
+                            try url_buf.appendSlice(allocator, m.data.items);
+                            const is_pdf = std.mem.eql(u8, m.media_type, "application/pdf");
+                            try s.beginObject();
+                            try s.objectField("type");
+                            try s.write(if (is_pdf) "input_file" else "input_image");
+                            if (is_pdf) {
+                                try s.objectField("filename");
+                                try s.write("attachment.pdf");
+                                try s.objectField("file_data");
+                            } else {
+                                try s.objectField("image_url");
+                            }
+                            try s.write(url_buf.items);
+                            try s.endObject();
+                        }
+                    }
+                    try s.endArray();
+                    try s.endObject();
+                }
             }
             // Plain user text (skip if the message was purely tool results).
             var text_buf: std.ArrayList(u8) = .empty;
@@ -714,6 +762,55 @@ test "responses serializeRequest - assistant tool_use + tool result round-trip"
     try testing.expectEqualStrings("42", input[2].object.get("output").?.string);
 }
 
+test "responses serializeRequest - tool result media splits into output note + synthetic user" {
+    const allocator = testing.allocator;
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+
+    const rid = try allocator.dupe(u8, "call_img");
+    var parts: std.ArrayList(conversation.ResultPartStored) = .empty;
+    try parts.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "the file:") });
+    try parts.append(allocator, .{ .media = .{
+        .media_type = try allocator.dupe(u8, "image/png"),
+        .data = try conversation.textualBlockFromSlice(allocator, "iVBOR=="),
+    } });
+    try parts.append(allocator, .{ .media = .{
+        .media_type = try allocator.dupe(u8, "application/pdf"),
+        .data = try conversation.textualBlockFromSlice(allocator, "JVBERg=="),
+    } });
+    var content: std.ArrayList(conversation.ContentBlock) = .empty;
+    try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = rid, .parts = parts } });
+    try conv.messages.append(allocator, .{ .role = .user, .content = content });
+
+    var tools = emptyTools();
+    defer tools.deinit();
+    const cfg = testConfig("gpt-5.1-codex");
+    const body = try serializeRequest(allocator, &cfg, &conv, &tools, .public);
+    defer allocator.free(body);
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+
+    const input = parsed.value.object.get("input").?.array.items;
+    // function_call_output + synthetic user message = 2.
+    try testing.expectEqual(@as(usize, 2), input.len);
+
+    // The output carries the text plus the attachment note.
+    try testing.expectEqualStrings("function_call_output", input[0].object.get("type").?.string);
+    const out_text = input[0].object.get("output").?.string;
+    try testing.expect(std.mem.indexOf(u8, out_text, "the file:") != null);
+    try testing.expect(std.mem.indexOf(u8, out_text, "[attachment(s) provided in the following user message]") != null);
+
+    // The synthetic user message carries the image and the PDF.
+    try testing.expectEqualStrings("user", input[1].object.get("role").?.string);
+    const uc = input[1].object.get("content").?.array.items;
+    try testing.expectEqual(@as(usize, 2), uc.len);
+    try testing.expectEqualStrings("input_image", uc[0].object.get("type").?.string);
+    try testing.expectEqualStrings("data:image/png;base64,iVBOR==", uc[0].object.get("image_url").?.string);
+    try testing.expectEqualStrings("input_file", uc[1].object.get("type").?.string);
+    try testing.expectEqualStrings("attachment.pdf", uc[1].object.get("filename").?.string);
+    try testing.expectEqualStrings("data:application/pdf;base64,JVBERg==", uc[1].object.get("file_data").?.string);
+}
+
 test "responses parseStreamEvent - output_text delta" {
     const allocator = testing.allocator;
     var ev = try parseStreamEvent(allocator,