From c4d7f70ff831cfcdad0f2a6224f9a14f86905de6 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 2 Jul 2026 10:29:53 -0600 Subject: Stage and apply event-field overrides at tool lifecycle boundaries Stage writable event-field overrides by tool call id so they can be applied at the correct point in the turn lifecycle. Capture input overrides from tool_call_complete, capture output overrides from tool_result, and clear staged overrides defensively at turn start and shutdown. Also honor user_message text overrides when starting a turn, add the helper that applies staged overrides to the agent at dispatch boundaries, and cover the behavior with tests for staged input/output overrides. --- libpanto/src/openai_responses_json.zig | 97 ++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) (limited to 'libpanto/src/openai_responses_json.zig') diff --git a/libpanto/src/openai_responses_json.zig b/libpanto/src/openai_responses_json.zig index cb58c2c..358fc45 100644 --- a/libpanto/src/openai_responses_json.zig +++ b/libpanto/src/openai_responses_json.zig @@ -169,9 +169,14 @@ fn writeInputForMessage( if (b == .ToolResult) has_tool_result = true; } if (has_tool_result) { + // `function_call_output` is text-only. Each result carries + // its text (plus a short note when media is present); the + // media rides along afterward in a synthetic user message. + var any_media = false; for (msg.content.items) |block| { if (block != .ToolResult) continue; const tr = block.ToolResult; + if (tr.hasMedia()) any_media = true; try s.beginObject(); try s.objectField("type"); try s.write("function_call_output"); @@ -181,9 +186,52 @@ fn writeInputForMessage( var tbuf: std.ArrayList(u8) = .empty; defer tbuf.deinit(allocator); try tr.appendTextInto(allocator, &tbuf); + if (tr.hasMedia()) { + if (tbuf.items.len > 0) try tbuf.append(allocator, '\n'); + try tbuf.appendSlice(allocator, "[attachment(s) provided in the following user message]"); + } try s.write(tbuf.items); try s.endObject(); } + // Synthetic user message holding the media: images as + // `input_image` data URLs, PDFs as `input_file` (the + // Responses API rejects non-image data URLs in + // `input_image`). + if (any_media) { + try s.beginObject(); + try s.objectField("role"); + try s.write("user"); + try s.objectField("content"); + try s.beginArray(); + for (msg.content.items) |block| { + if (block != .ToolResult) continue; + for (block.ToolResult.parts.items) |part| { + if (part != .media) continue; + const m = part.media; + var url_buf: std.ArrayList(u8) = .empty; + defer url_buf.deinit(allocator); + try url_buf.appendSlice(allocator, "data:"); + try url_buf.appendSlice(allocator, m.media_type); + try url_buf.appendSlice(allocator, ";base64,"); + try url_buf.appendSlice(allocator, m.data.items); + const is_pdf = std.mem.eql(u8, m.media_type, "application/pdf"); + try s.beginObject(); + try s.objectField("type"); + try s.write(if (is_pdf) "input_file" else "input_image"); + if (is_pdf) { + try s.objectField("filename"); + try s.write("attachment.pdf"); + try s.objectField("file_data"); + } else { + try s.objectField("image_url"); + } + try s.write(url_buf.items); + try s.endObject(); + } + } + try s.endArray(); + try s.endObject(); + } } // Plain user text (skip if the message was purely tool results). var text_buf: std.ArrayList(u8) = .empty; @@ -714,6 +762,55 @@ test "responses serializeRequest - assistant tool_use + tool result round-trip" try testing.expectEqualStrings("42", input[2].object.get("output").?.string); } +test "responses serializeRequest - tool result media splits into output note + synthetic user" { + const allocator = testing.allocator; + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + + const rid = try allocator.dupe(u8, "call_img"); + var parts: std.ArrayList(conversation.ResultPartStored) = .empty; + try parts.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "the file:") }); + try parts.append(allocator, .{ .media = .{ + .media_type = try allocator.dupe(u8, "image/png"), + .data = try conversation.textualBlockFromSlice(allocator, "iVBOR=="), + } }); + try parts.append(allocator, .{ .media = .{ + .media_type = try allocator.dupe(u8, "application/pdf"), + .data = try conversation.textualBlockFromSlice(allocator, "JVBERg=="), + } }); + var content: std.ArrayList(conversation.ContentBlock) = .empty; + try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = rid, .parts = parts } }); + try conv.messages.append(allocator, .{ .role = .user, .content = content }); + + var tools = emptyTools(); + defer tools.deinit(); + const cfg = testConfig("gpt-5.1-codex"); + const body = try serializeRequest(allocator, &cfg, &conv, &tools, .public); + defer allocator.free(body); + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + + const input = parsed.value.object.get("input").?.array.items; + // function_call_output + synthetic user message = 2. + try testing.expectEqual(@as(usize, 2), input.len); + + // The output carries the text plus the attachment note. + try testing.expectEqualStrings("function_call_output", input[0].object.get("type").?.string); + const out_text = input[0].object.get("output").?.string; + try testing.expect(std.mem.indexOf(u8, out_text, "the file:") != null); + try testing.expect(std.mem.indexOf(u8, out_text, "[attachment(s) provided in the following user message]") != null); + + // The synthetic user message carries the image and the PDF. + try testing.expectEqualStrings("user", input[1].object.get("role").?.string); + const uc = input[1].object.get("content").?.array.items; + try testing.expectEqual(@as(usize, 2), uc.len); + try testing.expectEqualStrings("input_image", uc[0].object.get("type").?.string); + try testing.expectEqualStrings("data:image/png;base64,iVBOR==", uc[0].object.get("image_url").?.string); + try testing.expectEqualStrings("input_file", uc[1].object.get("type").?.string); + try testing.expectEqualStrings("attachment.pdf", uc[1].object.get("filename").?.string); + try testing.expectEqualStrings("data:application/pdf;base64,JVBERg==", uc[1].object.get("file_data").?.string); +} + test "responses parseStreamEvent - output_text delta" { const allocator = testing.allocator; var ev = try parseStreamEvent(allocator, -- cgit v1.3