diff options
Diffstat (limited to 'libpanto/src/openai_chat_json.zig')
| -rw-r--r-- | libpanto/src/openai_chat_json.zig | 64 |
1 files changed, 64 insertions, 0 deletions
diff --git a/libpanto/src/openai_chat_json.zig b/libpanto/src/openai_chat_json.zig index 9531131..37d347b 100644 --- a/libpanto/src/openai_chat_json.zig +++ b/libpanto/src/openai_chat_json.zig @@ -31,6 +31,31 @@ pub const StreamDelta = struct { /// treat the turn as failed. error_message: ?[]const u8 = null, error_type: ?[]const u8 = null, + /// Token usage from the final chunk's top-level `usage` block. + /// Only present on the final chunk when the request was sent with + /// `stream_options.include_usage: true`. Earlier chunks have null. + usage: ?StreamUsage = null, +}; + +/// Token usage payload from OpenAI's terminating SSE chunk. Field +/// semantics: +/// +/// - `prompt_tokens`: total input tokens, **including** cached tokens. +/// - `completion_tokens`: output tokens (including reasoning tokens). +/// - `cached_prompt_tokens`: subset of `prompt_tokens` served from +/// the prompt cache (billed at a discount). +/// - `reasoning_tokens`: subset of `completion_tokens` spent on +/// internal reasoning (o-series models). +/// +/// To map to `Usage`: `input = prompt_tokens - cached_prompt_tokens`, +/// `cache_read = cached_prompt_tokens`, `output = completion_tokens`, +/// `reasoning = reasoning_tokens`, `cache_write = 0` (OpenAI doesn't +/// bill a cache-write premium). +pub const StreamUsage = struct { + prompt_tokens: ?u64 = null, + completion_tokens: ?u64 = null, + cached_prompt_tokens: ?u64 = null, + reasoning_tokens: ?u64 = null, }; /// A single entry from a streaming `tool_calls` array. Multiple parallel @@ -66,6 +91,17 @@ pub fn serializeRequest( try s.objectField("stream"); try s.write(true); + // Ask for the final-chunk usage block. Without this the server + // sends `usage: null` and we can't stamp token counts on the + // session log. Most OpenAI-compatible proxies accept this; ones + // that don't will either ignore it or 400 — in the latter case + // the user has bigger problems than missing token counts. + try s.objectField("stream_options"); + try s.beginObject(); + try s.objectField("include_usage"); + try s.write(true); + try s.endObject(); + switch (cfg.reasoning) { .default => {}, .off => { @@ -285,6 +321,27 @@ pub fn parseStreamEvent(allocator: Allocator, payload: []const u8) !ParsedDelta // Top-level `error` field. Some providers (and OpenAI itself on rare // mid-stream failures) emit `data: {"error":{"message":...,"type":...}}` // with HTTP 200, so we look for this BEFORE the choices array. + // Top-level `usage` field on the terminating chunk. Independent of + // the (often empty) choices array. + if (root.object.get("usage")) |u| { + if (u == .object) { + var su: StreamUsage = .{}; + su.prompt_tokens = readOptU64(u.object, "prompt_tokens"); + su.completion_tokens = readOptU64(u.object, "completion_tokens"); + if (u.object.get("prompt_tokens_details")) |ptd| { + if (ptd == .object) { + su.cached_prompt_tokens = readOptU64(ptd.object, "cached_tokens"); + } + } + if (u.object.get("completion_tokens_details")) |ctd| { + if (ctd == .object) { + su.reasoning_tokens = readOptU64(ctd.object, "reasoning_tokens"); + } + } + delta.usage = su; + } + } + if (root.object.get("error")) |e| { switch (e) { .object => |obj| { @@ -375,6 +432,13 @@ pub fn parseStreamEvent(allocator: Allocator, payload: []const u8) !ParsedDelta }; } +fn readOptU64(obj: std.json.ObjectMap, name: []const u8) ?u64 { + const v = obj.get(name) orelse return null; + if (v != .integer) return null; + if (v.integer < 0) return null; + return @intCast(v.integer); +} + // ----------------------------------------------------------------------------- // Tests // ----------------------------------------------------------------------------- |