diff options
Diffstat (limited to 'libpanto/src/agent.zig')
| -rw-r--r-- | libpanto/src/agent.zig | 143 |
1 files changed, 110 insertions, 33 deletions
diff --git a/libpanto/src/agent.zig b/libpanto/src/agent.zig index 411abc5..c078c24 100644 --- a/libpanto/src/agent.zig +++ b/libpanto/src/agent.zig @@ -33,6 +33,7 @@ const config_mod = @import("config.zig"); const conversation = @import("conversation.zig"); const compaction_mod = @import("compaction.zig"); const tool_mod = @import("tool.zig"); +const image_mod = @import("image.zig"); const tool_source_mod = @import("tool_source.zig"); const tool_registry_mod = @import("tool_registry.zig"); @@ -86,8 +87,27 @@ fn cloneBlock(alloc: Allocator, block: conversation.ContentBlock) !conversation. .ToolResult => |b| blk: { const tuid = try alloc.dupe(u8, b.tool_use_id); errdefer alloc.free(tuid); - const content = try conversation.textualBlockFromSlice(alloc, b.content.items); - break :blk .{ .ToolResult = .{ .tool_use_id = tuid, .content = content } }; + var parts: std.ArrayList(conversation.ResultPartStored) = .empty; + errdefer { + for (parts.items) |*p| p.deinit(alloc); + parts.deinit(alloc); + } + try parts.ensureTotalCapacity(alloc, b.parts.items.len); + for (b.parts.items) |src| { + switch (src) { + .text => |tb| { + const t = try conversation.textualBlockFromSlice(alloc, tb.items); + parts.appendAssumeCapacity(.{ .text = t }); + }, + .media => |m| { + const mt = try alloc.dupe(u8, m.media_type); + errdefer alloc.free(mt); + const data = try conversation.textualBlockFromSlice(alloc, m.data.items); + parts.appendAssumeCapacity(.{ .media = .{ .media_type = mt, .data = data } }); + }, + } + } + break :blk .{ .ToolResult = .{ .tool_use_id = tuid, .parts = parts } }; }, .System => |b| .{ .System = .{ .text = try conversation.textualBlockFromSlice(alloc, b.text.items), @@ -137,12 +157,13 @@ fn isValidToolInput(input: []const u8) bool { return parsed.value == .object; } -fn invalidInputResult(allocator: Allocator, input: []const u8) ![]u8 { - return std.fmt.allocPrint( +fn invalidInputResult(allocator: Allocator, input: []const u8) ![]tool_mod.ResultPart { + const msg = try std.fmt.allocPrint( allocator, "Tool call was not executed: tool input was incomplete or invalid JSON. Partial input: {s}", .{input}, ); + return tool_mod.ownedTextResult(allocator, msg); } @@ -536,7 +557,7 @@ pub const Agent = struct { defer task_group.cancel(self.io); errdefer { for (calls.items) |*c| { - if (c.result) |r| self.allocator.free(r); + if (c.result) |r| tool_mod.freeResultParts(self.allocator, r); } } @@ -563,25 +584,68 @@ pub const Agent = struct { first_err = e; continue; } - const result_bytes = c.result orelse { + const result_parts = c.result orelse { // Internal error: every successful call should have left - // bytes behind. Treat as MissingToolResult. + // parts behind. Treat as MissingToolResult. first_err = error.MissingToolResult; continue; }; c.result = null; // ownership transferred below + defer tool_mod.freeResultParts(self.allocator, result_parts); const id_copy = try self.allocator.dupe(u8, c.tool_use_id); errdefer self.allocator.free(id_copy); - var content_buf: conversation.TextualBlock = .empty; - errdefer content_buf.deinit(self.allocator); - try content_buf.appendSlice(self.allocator, result_bytes); - self.allocator.free(result_bytes); + var stored: std.ArrayList(conversation.ResultPartStored) = .empty; + errdefer { + for (stored.items) |*p| p.deinit(self.allocator); + stored.deinit(self.allocator); + } + try stored.ensureTotalCapacity(self.allocator, result_parts.len); + for (result_parts) |part| { + switch (part) { + .text => |t| { + var buf: conversation.TextualBlock = .empty; + errdefer buf.deinit(self.allocator); + try buf.appendSlice(self.allocator, t); + stored.appendAssumeCapacity(.{ .text = buf }); + }, + .media => |m| { + // libpanto owns the heavy lifting: detect the type + // (when the tool gave no hint), resize large + // rasters, then base64-encode for storage. Tools + // hand over raw bytes only. + const processed = image_mod.process(self.allocator, m.data, m.media_type) catch |e| { + // Unrecognized bytes: keep the turn alive by + // dropping the attachment and noting it as text. + if (e == error.UnknownMediaType) { + var note: conversation.TextualBlock = .empty; + errdefer note.deinit(self.allocator); + try note.appendSlice(self.allocator, "[unrecognized binary attachment dropped]"); + stored.appendAssumeCapacity(.{ .text = note }); + continue; + } + return e; + }; + defer self.allocator.free(processed.data); + + const mt = try self.allocator.dupe(u8, processed.media_type); + errdefer self.allocator.free(mt); + + const enc = std.base64.standard.Encoder; + var buf: conversation.TextualBlock = .empty; + errdefer buf.deinit(self.allocator); + try buf.resize(self.allocator, enc.calcSize(processed.data.len)); + _ = enc.encode(buf.items, processed.data); + + stored.appendAssumeCapacity(.{ .media = .{ .media_type = mt, .data = buf } }); + }, + } + } content.appendAssumeCapacity(.{ .ToolResult = .{ .tool_use_id = id_copy, - .content = content_buf, + .parts = stored, } }); } @@ -603,10 +667,10 @@ const FlatCall = struct { input: []const u8, // borrowed from assistant_msg entry: ?Entry, - /// Owned result bytes from `Tool.invoke` or `ToolSource.invoke_batch`. + /// Owned result parts from `Tool.invoke` or `ToolSource.invoke_batch`. /// Allocated with the agent's allocator. Transferred into a /// ToolResultBlock on success. - result: ?[]u8, + result: ?[]tool_mod.ResultPart, /// If non-null, the call failed and the turn must abort. err: ?anyerror, @@ -733,7 +797,7 @@ fn runSourceGroup(agent: *Agent, sg: Group.SourceGroup, calls: []FlatCall) void // Whole-batch failure: free any partial successes the source // already wrote, then mark every member as failed. for (batch_results) |r| switch (r) { - .ok => |b| agent.allocator.free(b), + .ok => |b| tool_mod.freeResultParts(agent.allocator, b), .err => {}, }; for (sg.member_indices) |i| calls[i].err = e; @@ -755,6 +819,14 @@ fn runSourceGroup(agent: *Agent, sg: Group.SourceGroup, calls: []FlatCall) void const testing = std.testing; +/// Test helper: the items of a ToolResultBlock's first text part. +fn trText(tr: conversation.ToolResultBlock) []const u8 { + for (tr.parts.items) |p| { + if (p == .text) return p.text.items; + } + return ""; +} + /// Test harness for the injectable `stream_fn` seam. /// /// `provider_mod.StreamFn` carries no user context (it mirrors the real @@ -894,9 +966,10 @@ const EchoTool = struct { const vt: Tool.VTable = .{ .invoke = invoke, .deinit = deinit }; - fn invoke(ctx: *anyopaque, input: []const u8, allocator: Allocator) anyerror![]u8 { + fn invoke(ctx: *anyopaque, input: []const u8, allocator: Allocator) anyerror![]tool_mod.ResultPart { const self: *EchoTool = @ptrCast(@alignCast(ctx)); - return try std.fmt.allocPrint(allocator, "{s}{s}", .{ self.prefix_owned, input }); + const msg = try std.fmt.allocPrint(allocator, "{s}{s}", .{ self.prefix_owned, input }); + return tool_mod.ownedTextResult(allocator, msg); } fn deinit(ctx: *anyopaque, allocator: Allocator) void { @@ -937,7 +1010,7 @@ const BarrierTool = struct { const vt: Tool.VTable = .{ .invoke = invoke, .deinit = deinit }; - fn invoke(ctx: *anyopaque, _: []const u8, allocator: Allocator) anyerror![]u8 { + fn invoke(ctx: *anyopaque, _: []const u8, allocator: Allocator) anyerror![]tool_mod.ResultPart { const self: *BarrierTool = @ptrCast(@alignCast(ctx)); const arrived = self.barrier.arrived.fetchAdd(1, .acq_rel); if (arrived < self.barrier.thread_ids.len) { @@ -949,7 +1022,7 @@ const BarrierTool = struct { if (i > 50_000) return error.BarrierTimeout; std.Thread.yield() catch {}; } - return try allocator.dupe(u8, "done"); + return tool_mod.textResult(allocator, "done"); } fn deinit(ctx: *anyopaque, allocator: Allocator) void { @@ -979,7 +1052,7 @@ const FailingTool = struct { const vt: Tool.VTable = .{ .invoke = invoke, .deinit = deinit }; - fn invoke(_: *anyopaque, _: []const u8, _: Allocator) anyerror![]u8 { + fn invoke(_: *anyopaque, _: []const u8, _: Allocator) anyerror![]tool_mod.ResultPart { return error.ToolExploded; } @@ -1100,12 +1173,16 @@ const TestSource = struct { try self.batches.append(batch); for (calls, 0..) |c, i| { + const msg = std.fmt.allocPrint( + allocator, + "{s}->{s}", + .{ c.tool_name, c.input }, + ) catch |e| { + results[i] = .{ .err = e }; + continue; + }; results[i] = .{ - .ok = std.fmt.allocPrint( - allocator, - "{s}->{s}", - .{ c.tool_name, c.input }, - ) catch |e| { + .ok = tool_mod.ownedTextResult(allocator, msg) catch |e| { results[i] = .{ .err = e }; continue; }, @@ -1247,7 +1324,7 @@ test "runStep dispatches a tool call and loops to a final text turn" { try testing.expectEqual(@as(usize, 1), conv.messages.items[2].content.items.len); const tr = conv.messages.items[2].content.items[0].ToolResult; try testing.expectEqualStrings("tc_1", tr.tool_use_id); - try testing.expectEqualStrings("ECHO:hello", tr.content.items); + try testing.expectEqualStrings("ECHO:hello", trText(tr)); try testing.expectEqual(conversation.MessageRole.assistant, conv.messages.items[3].role); try testing.expectEqualStrings("ok", conv.messages.items[3].content.items[0].Text.items); @@ -1455,11 +1532,11 @@ test "runStep delivers all source-backed calls in one batch on one thread" { const tr_msg = conv.messages.items[2]; try testing.expectEqual(@as(usize, 3), tr_msg.content.items.len); try testing.expectEqualStrings("a", tr_msg.content.items[0].ToolResult.tool_use_id); - try testing.expectEqualStrings("lua_x->1", tr_msg.content.items[0].ToolResult.content.items); + try testing.expectEqualStrings("lua_x->1", trText(tr_msg.content.items[0].ToolResult)); try testing.expectEqualStrings("b", tr_msg.content.items[1].ToolResult.tool_use_id); - try testing.expectEqualStrings("lua_y->2", tr_msg.content.items[1].ToolResult.content.items); + try testing.expectEqualStrings("lua_y->2", trText(tr_msg.content.items[1].ToolResult)); try testing.expectEqualStrings("c", tr_msg.content.items[2].ToolResult.tool_use_id); - try testing.expectEqualStrings("lua_x->3", tr_msg.content.items[2].ToolResult.content.items); + try testing.expectEqualStrings("lua_x->3", trText(tr_msg.content.items[2].ToolResult)); } test "runStep: distinct sources run on distinct threads in parallel" { @@ -1566,9 +1643,9 @@ test "runStep: mixed single Tools and source-backed tools coexist in one turn" { const tr_msg = conv.messages.items[2]; try testing.expectEqual(@as(usize, 3), tr_msg.content.items.len); - try testing.expectEqualStrings("S:X", tr_msg.content.items[0].ToolResult.content.items); - try testing.expectEqualStrings("src_t1->Y", tr_msg.content.items[1].ToolResult.content.items); - try testing.expectEqualStrings("src_t2->Z", tr_msg.content.items[2].ToolResult.content.items); + try testing.expectEqualStrings("S:X", trText(tr_msg.content.items[0].ToolResult)); + try testing.expectEqualStrings("src_t1->Y", trText(tr_msg.content.items[1].ToolResult)); + try testing.expectEqualStrings("src_t2->Z", trText(tr_msg.content.items[2].ToolResult)); } test "setConfig swaps the visible tool set between turns" { @@ -1628,7 +1705,7 @@ test "setConfig swaps the visible tool set between turns" { const tr = conv.messages.items[2].content.items[0].ToolResult; try testing.expectEqualStrings("2", tr.tool_use_id); - try testing.expectEqualStrings("B:B", tr.content.items); + try testing.expectEqualStrings("B:B", trText(tr)); } test "compact: summarizes prefix, keeps suffix, system survives" { |