Stage and apply event-field overrides at tool lifecycle boundaries

Stage writable event-field overrides by tool call id so they can be applied at the correct point in the turn lifecycle. Capture input overrides from tool_call_complete, capture output overrides from tool_result, and clear staged overrides defensively at turn start and shutdown.

Also honor user_message text overrides when starting a turn, add the helper that applies staged overrides to the agent at dispatch boundaries, and cover the behavior with tests for staged input/output overrides.

1 files changed, 97 insertions, 0 deletions

diff --git a/libpanto/src/openai_responses_json.zig b/libpanto/src/openai_responses_json.zig
index cb58c2c..358fc45 100644
--- a/libpanto/src/openai_responses_json.zig
+++ b/libpanto/src/openai_responses_json.zig
@@ -169,9 +169,14 @@ fn writeInputForMessage(
                 if (b == .ToolResult) has_tool_result = true;
             }
             if (has_tool_result) {
+                // `function_call_output` is text-only. Each result carries
+                // its text (plus a short note when media is present); the
+                // media rides along afterward in a synthetic user message.
+                var any_media = false;
                 for (msg.content.items) |block| {
                     if (block != .ToolResult) continue;
                     const tr = block.ToolResult;
+                    if (tr.hasMedia()) any_media = true;
                     try s.beginObject();
                     try s.objectField("type");
                     try s.write("function_call_output");
@@ -181,9 +186,52 @@ fn writeInputForMessage(
                     var tbuf: std.ArrayList(u8) = .empty;
                     defer tbuf.deinit(allocator);
                     try tr.appendTextInto(allocator, &tbuf);
+                    if (tr.hasMedia()) {
+                        if (tbuf.items.len > 0) try tbuf.append(allocator, '\n');
+                        try tbuf.appendSlice(allocator, "[attachment(s) provided in the following user message]");
+                    }
                     try s.write(tbuf.items);
                     try s.endObject();
                 }
+                // Synthetic user message holding the media: images as
+                // `input_image` data URLs, PDFs as `input_file` (the
+                // Responses API rejects non-image data URLs in
+                // `input_image`).
+                if (any_media) {
+                    try s.beginObject();
+                    try s.objectField("role");
+                    try s.write("user");
+                    try s.objectField("content");
+                    try s.beginArray();
+                    for (msg.content.items) |block| {
+                        if (block != .ToolResult) continue;
+                        for (block.ToolResult.parts.items) |part| {
+                            if (part != .media) continue;
+                            const m = part.media;
+                            var url_buf: std.ArrayList(u8) = .empty;
+                            defer url_buf.deinit(allocator);
+                            try url_buf.appendSlice(allocator, "data:");
+                            try url_buf.appendSlice(allocator, m.media_type);
+                            try url_buf.appendSlice(allocator, ";base64,");
+                            try url_buf.appendSlice(allocator, m.data.items);
+                            const is_pdf = std.mem.eql(u8, m.media_type, "application/pdf");
+                            try s.beginObject();
+                            try s.objectField("type");
+                            try s.write(if (is_pdf) "input_file" else "input_image");
+                            if (is_pdf) {
+                                try s.objectField("filename");
+                                try s.write("attachment.pdf");
+                                try s.objectField("file_data");
+                            } else {
+                                try s.objectField("image_url");
+                            }
+                            try s.write(url_buf.items);
+                            try s.endObject();
+                        }
+                    }
+                    try s.endArray();
+                    try s.endObject();
+                }
             }
             // Plain user text (skip if the message was purely tool results).
             var text_buf: std.ArrayList(u8) = .empty;
@@ -714,6 +762,55 @@ test "responses serializeRequest - assistant tool_use + tool result round-trip"
     try testing.expectEqualStrings("42", input[2].object.get("output").?.string);
 }
 
+test "responses serializeRequest - tool result media splits into output note + synthetic user" {
+    const allocator = testing.allocator;
+    var conv = conversation.Conversation.init(allocator);
+    defer conv.deinit();
+
+    const rid = try allocator.dupe(u8, "call_img");
+    var parts: std.ArrayList(conversation.ResultPartStored) = .empty;
+    try parts.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "the file:") });
+    try parts.append(allocator, .{ .media = .{
+        .media_type = try allocator.dupe(u8, "image/png"),
+        .data = try conversation.textualBlockFromSlice(allocator, "iVBOR=="),
+    } });
+    try parts.append(allocator, .{ .media = .{
+        .media_type = try allocator.dupe(u8, "application/pdf"),
+        .data = try conversation.textualBlockFromSlice(allocator, "JVBERg=="),
+    } });
+    var content: std.ArrayList(conversation.ContentBlock) = .empty;
+    try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = rid, .parts = parts } });
+    try conv.messages.append(allocator, .{ .role = .user, .content = content });
+
+    var tools = emptyTools();
+    defer tools.deinit();
+    const cfg = testConfig("gpt-5.1-codex");
+    const body = try serializeRequest(allocator, &cfg, &conv, &tools, .public);
+    defer allocator.free(body);
+    var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{});
+    defer parsed.deinit();
+
+    const input = parsed.value.object.get("input").?.array.items;
+    // function_call_output + synthetic user message = 2.
+    try testing.expectEqual(@as(usize, 2), input.len);
+
+    // The output carries the text plus the attachment note.
+    try testing.expectEqualStrings("function_call_output", input[0].object.get("type").?.string);
+    const out_text = input[0].object.get("output").?.string;
+    try testing.expect(std.mem.indexOf(u8, out_text, "the file:") != null);
+    try testing.expect(std.mem.indexOf(u8, out_text, "[attachment(s) provided in the following user message]") != null);
+
+    // The synthetic user message carries the image and the PDF.
+    try testing.expectEqualStrings("user", input[1].object.get("role").?.string);
+    const uc = input[1].object.get("content").?.array.items;
+    try testing.expectEqual(@as(usize, 2), uc.len);
+    try testing.expectEqualStrings("input_image", uc[0].object.get("type").?.string);
+    try testing.expectEqualStrings("data:image/png;base64,iVBOR==", uc[0].object.get("image_url").?.string);
+    try testing.expectEqualStrings("input_file", uc[1].object.get("type").?.string);
+    try testing.expectEqualStrings("attachment.pdf", uc[1].object.get("filename").?.string);
+    try testing.expectEqualStrings("data:application/pdf;base64,JVBERg==", uc[1].object.get("file_data").?.string);
+}
+
 test "responses parseStreamEvent - output_text delta" {
     const allocator = testing.allocator;
     var ev = try parseStreamEvent(allocator,