diff options
Diffstat (limited to 'libpanto/src/openai_chat_json.zig')
| -rw-r--r-- | libpanto/src/openai_chat_json.zig | 105 |
1 files changed, 97 insertions, 8 deletions
diff --git a/libpanto/src/openai_chat_json.zig b/libpanto/src/openai_chat_json.zig index 8035338..f0d12ed 100644 --- a/libpanto/src/openai_chat_json.zig +++ b/libpanto/src/openai_chat_json.zig @@ -217,16 +217,62 @@ fn writeMessage(s: *std.json.Stringify, msg: conversation.Message, allocator: Al } } if (has_tool_result) { + // OpenAI forbids images in `role:"tool"` messages. Each tool + // result emits a `role:"tool"` message carrying only its text + // (plus a short note when media is present), and any media + // rides along afterward in a synthetic `role:"user"` message. + var any_media = false; for (msg.content.items) |block| { if (block != .ToolResult) continue; const tr = block.ToolResult; + if (tr.hasMedia()) any_media = true; try s.beginObject(); try s.objectField("role"); try s.write("tool"); try s.objectField("tool_call_id"); try s.write(tr.tool_use_id); try s.objectField("content"); - try s.write(tr.content.items); + var tbuf: std.ArrayList(u8) = .empty; + defer tbuf.deinit(allocator); + try tr.appendTextInto(allocator, &tbuf); + if (tr.hasMedia()) { + if (tbuf.items.len > 0) try tbuf.append(allocator, '\n'); + try tbuf.appendSlice(allocator, "[attachment(s) provided in the following user message]"); + } + try s.write(tbuf.items); + try s.endObject(); + } + // Synthetic user message holding the media as image_url + // data-URL parts (OpenAI's only image channel). + if (any_media) { + try s.beginObject(); + try s.objectField("role"); + try s.write("user"); + try s.objectField("content"); + try s.beginArray(); + for (msg.content.items) |block| { + if (block != .ToolResult) continue; + for (block.ToolResult.parts.items) |part| { + if (part != .media) continue; + const m = part.media; + try s.beginObject(); + try s.objectField("type"); + try s.write("image_url"); + try s.objectField("image_url"); + try s.beginObject(); + try s.objectField("url"); + var url_buf: std.ArrayList(u8) = .empty; + defer url_buf.deinit(allocator); + try url_buf.appendSlice(allocator, "data:"); + try url_buf.appendSlice(allocator, m.media_type); + try url_buf.appendSlice(allocator, ";base64,"); + try url_buf.appendSlice(allocator, m.data.items); + try s.write(url_buf.items); + try s.endObject(); + try s.endObject(); + } + } + try s.endArray(); try s.endObject(); } // Trailing plain Text blocks (rare in practice) ride along as @@ -720,7 +766,7 @@ const tool_mod = @import("tool.zig"); /// Minimal in-test tool: borrows its name/description/schema slices from /// the test's stack. The vtable's deinit is a no-op since nothing is owned. const StaticToolVT = struct { - fn invoke(_: *anyopaque, _: []const u8, _: Allocator) anyerror![]u8 { + fn invoke(_: *anyopaque, _: []const u8, _: Allocator) anyerror![]tool_mod.ResultPart { return error.NotImplementedInTest; } fn deinit_(_: *anyopaque, _: Allocator) void {} @@ -852,16 +898,16 @@ test "serializeRequest - user ToolResult fans out into tool messages" { defer conv.deinit(); const id1 = try allocator.dupe(u8, "call_a"); - var c1: conversation.TextualBlock = .empty; - try c1.appendSlice(allocator, "42"); + var p1: std.ArrayList(conversation.ResultPartStored) = .empty; + try p1.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "42") }); const id2 = try allocator.dupe(u8, "call_b"); - var c2: conversation.TextualBlock = .empty; - try c2.appendSlice(allocator, "oops"); + var p2: std.ArrayList(conversation.ResultPartStored) = .empty; + try p2.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "oops") }); var content: std.ArrayList(conversation.ContentBlock) = .empty; - try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = id1, .content = c1 } }); - try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = id2, .content = c2 } }); + try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = id1, .parts = p1 } }); + try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = id2, .parts = p2 } }); try conv.messages.append(allocator, .{ .role = .user, .content = content }); var tools = emptyTools(); @@ -885,6 +931,49 @@ test "serializeRequest - user ToolResult fans out into tool messages" { try testing.expectEqualStrings("oops", msgs[1].object.get("content").?.string); } +test "serializeRequest - tool result with image splits into tool + synthetic user" { + const allocator = testing.allocator; + + var conv = conversation.Conversation.init(allocator); + defer conv.deinit(); + + const id = try allocator.dupe(u8, "call_img"); + var parts: std.ArrayList(conversation.ResultPartStored) = .empty; + try parts.append(allocator, .{ .text = try conversation.textualBlockFromSlice(allocator, "the file:") }); + try parts.append(allocator, .{ .media = .{ + .media_type = try allocator.dupe(u8, "image/png"), + .data = try conversation.textualBlockFromSlice(allocator, "iVBOR=="), + } }); + var content: std.ArrayList(conversation.ContentBlock) = .empty; + try content.append(allocator, .{ .ToolResult = .{ .tool_use_id = id, .parts = parts } }); + try conv.messages.append(allocator, .{ .role = .user, .content = content }); + + var tools = emptyTools(); + defer tools.deinit(); + const cfg = testConfig("gpt-4o"); + const body = try serializeRequest(allocator, &cfg, &conv, &tools); + defer allocator.free(body); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, body, .{}); + defer parsed.deinit(); + + const msgs = parsed.value.object.get("messages").?.array.items; + try testing.expectEqual(@as(usize, 2), msgs.len); + + // First: the tool message (text only, no image). + try testing.expectEqualStrings("tool", msgs[0].object.get("role").?.string); + try testing.expectEqualStrings("call_img", msgs[0].object.get("tool_call_id").?.string); + try testing.expect(std.mem.indexOf(u8, msgs[0].object.get("content").?.string, "the file:") != null); + + // Second: synthetic user message carrying the image as a data URL. + try testing.expectEqualStrings("user", msgs[1].object.get("role").?.string); + const uc = msgs[1].object.get("content").?.array.items; + try testing.expectEqual(@as(usize, 1), uc.len); + try testing.expectEqualStrings("image_url", uc[0].object.get("type").?.string); + const url = uc[0].object.get("image_url").?.object.get("url").?.string; + try testing.expectEqualStrings("data:image/png;base64,iVBOR==", url); +} + test "parseStreamEvent - tool_calls delta with id and partial arguments" { const allocator = testing.allocator; const payload = |