diff options
| -rw-r--r-- | libpanto-c/include/panto.h | 16 | ||||
| -rw-r--r-- | libpanto-c/src/lib.zig | 238 | ||||
| -rw-r--r-- | libpanto-lua/src/module.zig | 368 | ||||
| -rw-r--r-- | libpanto/src/file_system_jsonl_store.zig | 66 | ||||
| -rw-r--r-- | libpanto/src/session.zig | 35 | ||||
| -rw-r--r-- | src/main.zig | 4 | ||||
| -rw-r--r-- | src/subcommand.zig | 2 | ||||
| -rw-r--r-- | src/system_prompt.zig | 2 |
8 files changed, 645 insertions, 86 deletions
diff --git a/libpanto-c/include/panto.h b/libpanto-c/include/panto.h index 4430810..4f69465 100644 --- a/libpanto-c/include/panto.h +++ b/libpanto-c/include/panto.h @@ -39,10 +39,10 @@ typedef struct { PantoAPIStyle api_style; PantoSlice base_url; PantoSlice model; typedef struct { PantoSlice id, created, modified, last_user_message, base_url, model; size_t message_count; PantoAPIStyle api_style; PantoReasoningEffort reasoning; } PantoSessionInfo; typedef struct { PantoSessionInfo *ptr; size_t len; } PantoSessionInfoList; typedef struct { bool found; PantoSession *session; } PantoSessionResult; -typedef struct { PantoSlice api_key, base_url, model; PantoReasoningEffort reasoning; uint32_t max_tokens; } PantoOpenAIChatConfig; -typedef struct { PantoSlice api_key, base_url, model, api_version; uint32_t max_tokens; PantoThinking thinking; PantoEffort effort; bool has_thinking_budget_tokens; uint32_t thinking_budget_tokens; bool thinking_interleaved; } PantoAnthropicMessagesConfig; +typedef struct { const char *api_key, *base_url, *model; PantoReasoningEffort reasoning; uint32_t max_tokens; } PantoOpenAIChatConfig; +typedef struct { const char *api_key, *base_url, *model, *api_version; uint32_t max_tokens; PantoThinking thinking; PantoEffort effort; bool has_thinking_budget_tokens; uint32_t thinking_budget_tokens; bool thinking_interleaved; } PantoAnthropicMessagesConfig; typedef struct { PantoAPIStyle tag; union { PantoOpenAIChatConfig openai_chat; PantoAnthropicMessagesConfig anthropic_messages; } data; } PantoProviderConfig; -typedef struct { uint32_t keep_verbatim; bool has_model; PantoProviderConfig model; bool has_compaction_prompt; PantoSlice compaction_prompt; } PantoCompactionConfig; +typedef struct { uint32_t keep_verbatim; bool has_model; PantoProviderConfig model; bool has_compaction_prompt; const char *compaction_prompt; } PantoCompactionConfig; typedef struct { size_t max_attempts; uint64_t initial_delay_ms, max_delay_ms; double multiplier; bool jitter; } PantoRetryConfig; typedef struct { PantoProviderConfig provider; PantoCompactionConfig compaction; PantoRetryConfig retry; } PantoConfig; @@ -81,19 +81,21 @@ PANTO_EXPORT PantoStatus panto_session_create(PantoSessionStore *store, PantoSes PANTO_EXPORT void panto_session_destroy(PantoSession *session); PANTO_EXPORT PantoStatus panto_session_store_create(void *ctx, const PantoSessionStoreVTable *vtable, PantoSessionStore **out); PANTO_EXPORT void panto_session_store_destroy(PantoSessionStore *store); +PANTO_EXPORT PantoStatus panto_null_store_create(PantoSessionStore **out); +PANTO_EXPORT PantoStatus panto_file_system_jsonl_store_create(const char *dir, const char *metadata, PantoSessionStore **out); PANTO_EXPORT PantoStatus panto_session_store_create_session(PantoSessionStore *store, PantoSession **out); PANTO_EXPORT PantoStatus panto_session_store_list(PantoSessionStore *store, PantoSessionInfoList *out); PANTO_EXPORT void panto_session_info_list_destroy(PantoSessionInfoList infos); -PANTO_EXPORT PantoStatus panto_session_store_resolve(PantoSessionStore *store, PantoSlice id, PantoSessionResult *out); +PANTO_EXPORT PantoStatus panto_session_store_resolve(PantoSessionStore *store, const char *id, PantoSessionResult *out); PANTO_EXPORT PantoStatus panto_session_store_latest(PantoSessionStore *store, PantoSessionResult *out); PANTO_EXPORT PantoStatus panto_session_load(PantoSession *session, PantoConversation **out); PANTO_EXPORT PantoStatus panto_agent_create(const PantoConfig *config, PantoSession *session, PantoConversation *conversation, PantoAgent **out); PANTO_EXPORT void panto_agent_destroy(PantoAgent *agent); PANTO_EXPORT PantoSlice panto_agent_session_id(PantoAgent *agent); PANTO_EXPORT PantoStatus panto_agent_set_config(PantoAgent *agent, const PantoConfig *config); -PANTO_EXPORT PantoStatus panto_agent_add_system_message(PantoAgent *agent, PantoSlice text); -PANTO_EXPORT PantoStatus panto_agent_set_system_prompt(PantoAgent *agent, PantoSlice text); -PANTO_EXPORT PantoStatus panto_agent_run(PantoAgent *agent, PantoSlice user_text, PantoStream **out); +PANTO_EXPORT PantoStatus panto_agent_add_system_message(PantoAgent *agent, const char *text); +PANTO_EXPORT PantoStatus panto_agent_set_system_prompt(PantoAgent *agent, const char *text); +PANTO_EXPORT PantoStatus panto_agent_run(PantoAgent *agent, const char *user_text, PantoStream **out); PANTO_EXPORT void panto_stream_destroy(PantoStream *stream); PANTO_EXPORT PantoNextStatus panto_stream_next(PantoStream *stream, PantoEvent *out); PANTO_EXPORT void panto_event_free(PantoEvent *event); diff --git a/libpanto-c/src/lib.zig b/libpanto-c/src/lib.zig index dd01026..046236a 100644 --- a/libpanto-c/src/lib.zig +++ b/libpanto-c/src/lib.zig @@ -16,6 +16,8 @@ const AgentHandle = struct { agent: *panto.Agent, config: *panto.Config }; const ConversationHandle = struct { conversation: panto.Conversation }; const SessionHandle = struct { session: panto.Session, owns_info: bool = true }; const StoreHandle = struct { ctx: *anyopaque, vtable: *const PantoSessionStoreVTable }; +const NullStoreHandle = struct { store: panto.NullStore }; +const FileSystemJSONLStoreHandle = struct { store: panto.FileSystemJSONLStore }; pub const PantoSlice = extern struct { ptr: ?[*]const u8, len: usize }; pub const PantoStatus = enum(c_int) { ok = 0, err = 1 }; @@ -33,10 +35,10 @@ pub const PantoUsage = extern struct { input: u64, output: u64, cache_read: u64, pub const PantoSessionInfo = extern struct { id: PantoSlice, created: PantoSlice, modified: PantoSlice, last_user_message: PantoSlice, base_url: PantoSlice, model: PantoSlice, message_count: usize, api_style: PantoAPIStyle, reasoning: PantoReasoningEffort }; pub const PantoSessionInfoList = extern struct { ptr: ?[*]PantoSessionInfo, len: usize }; pub const PantoSessionResult = extern struct { found: bool, session: ?*PantoSession }; -pub const PantoOpenAIChatConfig = extern struct { api_key: PantoSlice, base_url: PantoSlice, model: PantoSlice, reasoning: PantoReasoningEffort, max_tokens: u32 }; -pub const PantoAnthropicMessagesConfig = extern struct { api_key: PantoSlice, base_url: PantoSlice, model: PantoSlice, api_version: PantoSlice, max_tokens: u32, thinking: PantoThinking, effort: PantoEffort, has_thinking_budget_tokens: bool, thinking_budget_tokens: u32, thinking_interleaved: bool }; +pub const PantoOpenAIChatConfig = extern struct { api_key: ?[*:0]const u8, base_url: ?[*:0]const u8, model: ?[*:0]const u8, reasoning: PantoReasoningEffort, max_tokens: u32 }; +pub const PantoAnthropicMessagesConfig = extern struct { api_key: ?[*:0]const u8, base_url: ?[*:0]const u8, model: ?[*:0]const u8, api_version: ?[*:0]const u8, max_tokens: u32, thinking: PantoThinking, effort: PantoEffort, has_thinking_budget_tokens: bool, thinking_budget_tokens: u32, thinking_interleaved: bool }; pub const PantoProviderConfig = extern struct { tag: PantoAPIStyle, data: extern union { openai_chat: PantoOpenAIChatConfig, anthropic_messages: PantoAnthropicMessagesConfig } }; -pub const PantoCompactionConfig = extern struct { keep_verbatim: u32, has_model: bool, model: PantoProviderConfig, has_compaction_prompt: bool, compaction_prompt: PantoSlice }; +pub const PantoCompactionConfig = extern struct { keep_verbatim: u32, has_model: bool, model: PantoProviderConfig, has_compaction_prompt: bool, compaction_prompt: ?[*:0]const u8 }; pub const PantoRetryConfig = extern struct { max_attempts: usize, initial_delay_ms: u64, max_delay_ms: u64, multiplier: f64, jitter: bool }; pub const PantoConfig = extern struct { provider: PantoProviderConfig, compaction: PantoCompactionConfig, retry: PantoRetryConfig }; @@ -82,6 +84,12 @@ fn outSlice(s: []const u8) PantoSlice { fn dupSlice(s: PantoSlice) ![]u8 { return allocator.dupe(u8, slice(s)); } +fn cstr(s: ?[*:0]const u8) []const u8 { + return if (s) |p| std.mem.span(p) else &.{}; +} +fn dupCStr(s: ?[*:0]const u8) ![]u8 { + return allocator.dupe(u8, cstr(s)); +} fn dupOut(s: []const u8) !PantoSlice { return outSlice(try allocator.dupe(u8, s)); } @@ -91,9 +99,22 @@ fn cloneSessionInfo(info: PantoSessionInfo) !panto.SessionInfo { fn marshalSessionInfo(info: panto.SessionInfo) !PantoSessionInfo { return .{ .id = try dupOut(info.id), .created = try dupOut(info.created), .modified = try dupOut(info.modified), .last_user_message = try dupOut(info.last_user_message), .base_url = try dupOut(info.base_url), .model = try dupOut(info.model), .message_count = info.message_count, .api_style = @enumFromInt(@intFromEnum(info.api_style)), .reasoning = @enumFromInt(@intFromEnum(info.reasoning)) }; } -fn freePantoSessionInfo(info: PantoSessionInfo) void { freePS(info.id); freePS(info.created); freePS(info.modified); freePS(info.last_user_message); freePS(info.base_url); freePS(info.model); } -fn wrapSession(session: panto.Session, owns_info: bool) !*PantoSession { const h = try allocator.create(SessionHandle); h.* = .{ .session = session, .owns_info = owns_info }; return @ptrCast(h); } -fn unwrapStore(store: *PantoSessionStore) panto.SessionStore { return .{ .ptr = store, .vtable = &c_store_vtable }; } +fn freePantoSessionInfo(info: PantoSessionInfo) void { + freePS(info.id); + freePS(info.created); + freePS(info.modified); + freePS(info.last_user_message); + freePS(info.base_url); + freePS(info.model); +} +fn wrapSession(session: panto.Session, owns_info: bool) !*PantoSession { + const h = try allocator.create(SessionHandle); + h.* = .{ .session = session, .owns_info = owns_info }; + return @ptrCast(h); +} +fn unwrapStore(store: *PantoSessionStore) panto.SessionStore { + return .{ .ptr = store, .vtable = &c_store_vtable }; +} export fn panto_init() void { if (threaded == null) { @@ -152,13 +173,56 @@ export fn panto_session_store_create(ctx: *anyopaque, vt: *const PantoSessionSto return .ok; } export fn panto_session_store_destroy(store: ?*PantoSessionStore) void { - if (store) |ptr| { const h: *StoreHandle = @ptrCast(@alignCast(ptr)); if (h.vtable.destroy) |d| d(h.ctx); allocator.destroy(h); } + if (store) |ptr| { + const h: *StoreHandle = @ptrCast(@alignCast(ptr)); + if (h.vtable.destroy) |d| d(h.ctx); + allocator.destroy(h); + } +} +export fn panto_null_store_create(out: **PantoSessionStore) PantoStatus { + const impl = allocator.create(NullStoreHandle) catch return setErr("OutOfMemory", .{}); + impl.* = .{ .store = panto.NullStore.init(allocator) }; + const h = allocator.create(StoreHandle) catch { + allocator.destroy(impl); + return setErr("OutOfMemory", .{}); + }; + h.* = .{ .ctx = &impl.store, .vtable = &null_store_vtable }; + out.* = @ptrCast(h); + return .ok; +} +export fn panto_file_system_jsonl_store_create(dir: ?[*:0]const u8, metadata: ?[*:0]const u8, out: **PantoSessionStore) PantoStatus { + const impl = allocator.create(FileSystemJSONLStoreHandle) catch return setErr("OutOfMemory", .{}); + impl.* = .{ .store = panto.FileSystemJSONLStore.initWithMetadata(allocator, threaded.?.io(), cstr(dir), if (metadata == null or cstr(metadata).len == 0) null else cstr(metadata)) catch |e| { + allocator.destroy(impl); + return setErr("{t}", .{e}); + } }; + const h = allocator.create(StoreHandle) catch { + impl.store.deinit(); + allocator.destroy(impl); + return setErr("OutOfMemory", .{}); + }; + h.* = .{ .ctx = &impl.store, .vtable = &fs_jsonl_store_vtable }; + out.* = @ptrCast(h); + return .ok; +} +export fn panto_session_store_create_session(store: *PantoSessionStore, out: **PantoSession) PantoStatus { + return cStoreCreate(store, out); +} +export fn panto_session_store_list(store: *PantoSessionStore, out: *PantoSessionInfoList) PantoStatus { + return cStoreList(store, out); +} +export fn panto_session_info_list_destroy(infos: PantoSessionInfoList) void { + if (infos.ptr) |p| { + for (p[0..infos.len]) |info| freePantoSessionInfo(info); + allocator.free(p[0..infos.len]); + } +} +export fn panto_session_store_resolve(store: *PantoSessionStore, id: ?[*:0]const u8, out: *PantoSessionResult) PantoStatus { + return cStoreResolve(store, outSlice(cstr(id)), out); +} +export fn panto_session_store_latest(store: *PantoSessionStore, out: *PantoSessionResult) PantoStatus { + return cStoreLatest(store, out); } -export fn panto_session_store_create_session(store: *PantoSessionStore, out: **PantoSession) PantoStatus { return cStoreCreate(store, out); } -export fn panto_session_store_list(store: *PantoSessionStore, out: *PantoSessionInfoList) PantoStatus { return cStoreList(store, out); } -export fn panto_session_info_list_destroy(infos: PantoSessionInfoList) void { if (infos.ptr) |p| { for (p[0..infos.len]) |info| freePantoSessionInfo(info); allocator.free(p[0..infos.len]); } } -export fn panto_session_store_resolve(store: *PantoSessionStore, id: PantoSlice, out: *PantoSessionResult) PantoStatus { return cStoreResolve(store, id, out); } -export fn panto_session_store_latest(store: *PantoSessionStore, out: *PantoSessionResult) PantoStatus { return cStoreLatest(store, out); } export fn panto_session_load(session: *PantoSession, out: **PantoConversation) PantoStatus { const h: *SessionHandle = @ptrCast(@alignCast(session)); const conv = h.session.load() catch |e| return setErr("{t}", .{e}); @@ -171,13 +235,13 @@ export fn panto_session_load(session: *PantoSession, out: **PantoConversation) P fn convertConfig(c: *const PantoConfig) !*panto.Config { const out = try allocator.create(panto.Config); errdefer allocator.destroy(out); - out.* = .{ .provider = try convertProvider(c.provider), .compaction = .{ .keep_verbatim = c.compaction.keep_verbatim, .model = if (c.compaction.has_model) try convertProvider(c.compaction.model) else null, .compaction_prompt = if (c.compaction.has_compaction_prompt) try dupSlice(c.compaction.compaction_prompt) else null }, .retry = .{ .max_attempts = c.retry.max_attempts, .initial_delay_ms = c.retry.initial_delay_ms, .max_delay_ms = c.retry.max_delay_ms, .multiplier = c.retry.multiplier, .jitter = c.retry.jitter } }; + out.* = .{ .provider = try convertProvider(c.provider), .compaction = .{ .keep_verbatim = c.compaction.keep_verbatim, .model = if (c.compaction.has_model) try convertProvider(c.compaction.model) else null, .compaction_prompt = if (c.compaction.has_compaction_prompt) try dupCStr(c.compaction.compaction_prompt) else null }, .retry = .{ .max_attempts = c.retry.max_attempts, .initial_delay_ms = c.retry.initial_delay_ms, .max_delay_ms = c.retry.max_delay_ms, .multiplier = c.retry.multiplier, .jitter = c.retry.jitter } }; return out; } fn convertProvider(p: PantoProviderConfig) !panto.ProviderConfig { return switch (p.tag) { - .openai_chat => .{ .openai_chat = .{ .api_key = try dupSlice(p.data.openai_chat.api_key), .base_url = try dupSlice(p.data.openai_chat.base_url), .model = try dupSlice(p.data.openai_chat.model), .reasoning = @enumFromInt(@intFromEnum(p.data.openai_chat.reasoning)), .max_tokens = p.data.openai_chat.max_tokens } }, - .anthropic_messages => .{ .anthropic_messages = .{ .api_key = try dupSlice(p.data.anthropic_messages.api_key), .base_url = try dupSlice(p.data.anthropic_messages.base_url), .model = try dupSlice(p.data.anthropic_messages.model), .api_version = if (p.data.anthropic_messages.api_version.len == 0) "2023-06-01" else try dupSlice(p.data.anthropic_messages.api_version), .max_tokens = p.data.anthropic_messages.max_tokens, .thinking = @enumFromInt(@intFromEnum(p.data.anthropic_messages.thinking)), .effort = @enumFromInt(@intFromEnum(p.data.anthropic_messages.effort)), .thinking_budget_tokens = if (p.data.anthropic_messages.has_thinking_budget_tokens) p.data.anthropic_messages.thinking_budget_tokens else null, .thinking_interleaved = p.data.anthropic_messages.thinking_interleaved } }, + .openai_chat => .{ .openai_chat = .{ .api_key = try dupCStr(p.data.openai_chat.api_key), .base_url = try dupCStr(p.data.openai_chat.base_url), .model = try dupCStr(p.data.openai_chat.model), .reasoning = @enumFromInt(@intFromEnum(p.data.openai_chat.reasoning)), .max_tokens = p.data.openai_chat.max_tokens } }, + .anthropic_messages => .{ .anthropic_messages = .{ .api_key = try dupCStr(p.data.anthropic_messages.api_key), .base_url = try dupCStr(p.data.anthropic_messages.base_url), .model = try dupCStr(p.data.anthropic_messages.model), .api_version = if (cstr(p.data.anthropic_messages.api_version).len == 0) "2023-06-01" else try dupCStr(p.data.anthropic_messages.api_version), .max_tokens = p.data.anthropic_messages.max_tokens, .thinking = @enumFromInt(@intFromEnum(p.data.anthropic_messages.thinking)), .effort = @enumFromInt(@intFromEnum(p.data.anthropic_messages.effort)), .thinking_budget_tokens = if (p.data.anthropic_messages.has_thinking_budget_tokens) p.data.anthropic_messages.thinking_budget_tokens else null, .thinking_interleaved = p.data.anthropic_messages.thinking_interleaved } }, }; } fn freeConfig(c: *panto.Config) void { @@ -202,12 +266,39 @@ fn freeProvider(pv: *panto.ProviderConfig) void { } } -fn cStoreCreate(ctx: *anyopaque, out: **PantoSession) PantoStatus { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); return h.vtable.create(h.ctx, out); } -fn cStoreList(ctx: *anyopaque, out: *PantoSessionInfoList) PantoStatus { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); return h.vtable.list(h.ctx, out); } -fn cStoreResolve(ctx: *anyopaque, id: PantoSlice, out: *PantoSessionResult) PantoStatus { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); return h.vtable.resolve(h.ctx, id, out); } -fn cStoreLatest(ctx: *anyopaque, out: *PantoSessionResult) PantoStatus { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); return h.vtable.latest(h.ctx, out); } -fn cStoreLoad(ctx: *anyopaque, id: []const u8) anyerror!?panto.Conversation { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); var out: *PantoConversation = undefined; if (h.vtable.load(h.ctx, outSlice(id), &out) != .ok) return error.CStoreLoadFailed; const ch: *ConversationHandle = @ptrCast(@alignCast(out)); const conv = ch.conversation; ch.conversation = panto.Conversation.init(allocator); panto_conversation_destroy(out); return conv; } -fn cStoreAppend(ctx: *anyopaque, session_id: []const u8, messages: []panto.PersistentMessage) anyerror!void { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); const cmsgs = try allocator.alloc(PantoPersistentMessage, messages.len); defer allocator.free(cmsgs); for (messages, 0..) |m, i| cmsgs[i] = .{ .role = @enumFromInt(@intFromEnum(m.message.role)), .has_usage = m.usage != null, .usage = if (m.usage) |u| usage(u) else .{ .input = 0, .output = 0, .cache_read = 0, .cache_write = 0, .reasoning = 0 }, .has_metadata = m.message.metadata != null, .metadata = outSlice(m.message.metadata orelse "") }; if (h.vtable.append_messages(h.ctx, outSlice(session_id), cmsgs.ptr, cmsgs.len) != .ok) return error.CStoreAppendFailed; } +fn cStoreCreate(ctx: *anyopaque, out: **PantoSession) PantoStatus { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + return h.vtable.create(h.ctx, out); +} +fn cStoreList(ctx: *anyopaque, out: *PantoSessionInfoList) PantoStatus { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + return h.vtable.list(h.ctx, out); +} +fn cStoreResolve(ctx: *anyopaque, id: PantoSlice, out: *PantoSessionResult) PantoStatus { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + return h.vtable.resolve(h.ctx, id, out); +} +fn cStoreLatest(ctx: *anyopaque, out: *PantoSessionResult) PantoStatus { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + return h.vtable.latest(h.ctx, out); +} +fn cStoreLoad(ctx: *anyopaque, id: []const u8) anyerror!?panto.Conversation { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + var out: *PantoConversation = undefined; + if (h.vtable.load(h.ctx, outSlice(id), &out) != .ok) return error.CStoreLoadFailed; + const ch: *ConversationHandle = @ptrCast(@alignCast(out)); + const conv = ch.conversation; + ch.conversation = panto.Conversation.init(allocator); + panto_conversation_destroy(out); + return conv; +} +fn cStoreAppend(ctx: *anyopaque, session_id: []const u8, messages: []panto.PersistentMessage) anyerror!void { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + const cmsgs = try allocator.alloc(PantoPersistentMessage, messages.len); + defer allocator.free(cmsgs); + for (messages, 0..) |m, i| cmsgs[i] = .{ .role = @enumFromInt(@intFromEnum(m.message.role)), .has_usage = m.usage != null, .usage = if (m.usage) |u| usage(u) else .{ .input = 0, .output = 0, .cache_read = 0, .cache_write = 0, .reasoning = 0 }, .has_metadata = m.message.metadata != null, .metadata = outSlice(m.message.metadata orelse "") }; + if (h.vtable.append_messages(h.ctx, outSlice(session_id), cmsgs.ptr, cmsgs.len) != .ok) return error.CStoreAppendFailed; +} fn cStoreListZ(ctx: *anyopaque) anyerror![]panto.SessionInfo { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); var list: PantoSessionInfoList = undefined; @@ -220,9 +311,26 @@ fn cStoreListZ(ctx: *anyopaque) anyerror![]panto.SessionInfo { } return out; } -fn cStoreFreeInfos(_: *anyopaque, infos: []panto.SessionInfo) void { for (infos) |info| info.deinit(allocator); allocator.free(infos); } -fn cStoreResolveZ(ctx: *anyopaque, id: []const u8) anyerror!?panto.Session { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); var res: PantoSessionResult = undefined; if (h.vtable.resolve(h.ctx, outSlice(id), &res) != .ok) return error.CStoreResolveFailed; if (!res.found) return null; const sh: *SessionHandle = @ptrCast(@alignCast(res.session.?)); return sh.session; } -fn cStoreLatestZ(ctx: *anyopaque) anyerror!?panto.Session { const h: *StoreHandle = @ptrCast(@alignCast(ctx)); var res: PantoSessionResult = undefined; if (h.vtable.latest(h.ctx, &res) != .ok) return error.CStoreLatestFailed; if (!res.found) return null; const sh: *SessionHandle = @ptrCast(@alignCast(res.session.?)); return sh.session; } +fn cStoreFreeInfos(_: *anyopaque, infos: []panto.SessionInfo) void { + for (infos) |info| info.deinit(allocator); + allocator.free(infos); +} +fn cStoreResolveZ(ctx: *anyopaque, id: []const u8) anyerror!?panto.Session { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + var res: PantoSessionResult = undefined; + if (h.vtable.resolve(h.ctx, outSlice(id), &res) != .ok) return error.CStoreResolveFailed; + if (!res.found) return null; + const sh: *SessionHandle = @ptrCast(@alignCast(res.session.?)); + return sh.session; +} +fn cStoreLatestZ(ctx: *anyopaque) anyerror!?panto.Session { + const h: *StoreHandle = @ptrCast(@alignCast(ctx)); + var res: PantoSessionResult = undefined; + if (h.vtable.latest(h.ctx, &res) != .ok) return error.CStoreLatestFailed; + if (!res.found) return null; + const sh: *SessionHandle = @ptrCast(@alignCast(res.session.?)); + return sh.session; +} fn cStoreCreateZ(ctx: *anyopaque) panto.Session { var out: *PantoSession = undefined; if (cStoreCreate(ctx, &out) != .ok) { @@ -234,6 +342,76 @@ fn cStoreCreateZ(ctx: *anyopaque) panto.Session { } const c_store_vtable: panto.SessionStore.VTable = .{ .create = cStoreCreateZ, .list = cStoreListZ, .freeSessionInfos = cStoreFreeInfos, .resolve = cStoreResolveZ, .latest = cStoreLatestZ, .load = cStoreLoad, .appendMessages = cStoreAppend }; +fn builtinStoreCreate(ctx: *anyopaque, out: **PantoSession) callconv(.c) PantoStatus { + return wrapBuiltinSession(@as(*panto.SessionStore, @ptrCast(@alignCast(ctx))).create(), out); +} +fn builtinStoreList(ctx: *anyopaque, out: *PantoSessionInfoList) callconv(.c) PantoStatus { + const s: *panto.SessionStore = @ptrCast(@alignCast(ctx)); + const infos = s.list() catch |e| return setErr("{t}", .{e}); + defer s.freeSessionInfos(infos); + const c_infos = allocator.alloc(PantoSessionInfo, infos.len) catch return setErr("OutOfMemory", .{}); + for (infos, 0..) |info, i| c_infos[i] = marshalSessionInfo(info) catch return setErr("OutOfMemory", .{}); + out.* = .{ .ptr = c_infos.ptr, .len = c_infos.len }; + return .ok; +} +fn builtinStoreFreeInfos(_: *anyopaque, infos: PantoSessionInfoList) callconv(.c) void { + panto_session_info_list_destroy(infos); +} +fn builtinStoreResolve(ctx: *anyopaque, id: PantoSlice, out: *PantoSessionResult) callconv(.c) PantoStatus { + const s: *panto.SessionStore = @ptrCast(@alignCast(ctx)); + if (s.resolve(slice(id)) catch |e| return setErr("{t}", .{e})) |sess| return wrapBuiltinResult(sess, out); + out.* = .{ .found = false, .session = null }; + return .ok; +} +fn builtinStoreLatest(ctx: *anyopaque, out: *PantoSessionResult) callconv(.c) PantoStatus { + const s: *panto.SessionStore = @ptrCast(@alignCast(ctx)); + if (s.latest() catch |e| return setErr("{t}", .{e})) |sess| return wrapBuiltinResult(sess, out); + out.* = .{ .found = false, .session = null }; + return .ok; +} +fn builtinStoreLoad(ctx: *anyopaque, id: PantoSlice, out: **PantoConversation) callconv(.c) PantoStatus { + const s: *panto.SessionStore = @ptrCast(@alignCast(ctx)); + const conv = s.load(slice(id)) catch |e| return setErr("{t}", .{e}); + if (conv == null) return setErr("NotFound", .{}); + const ch = allocator.create(ConversationHandle) catch return setErr("OutOfMemory", .{}); + ch.* = .{ .conversation = conv.? }; + out.* = @ptrCast(ch); + return .ok; +} +fn builtinStoreAppend(ctx: *anyopaque, session_id: PantoSlice, messages: [*]const PantoPersistentMessage, len: usize) callconv(.c) PantoStatus { + const s: *panto.SessionStore = @ptrCast(@alignCast(ctx)); + const pmsgs = allocator.alloc(panto.PersistentMessage, len) catch return setErr("OutOfMemory", .{}); + defer allocator.free(pmsgs); + for (messages[0..len], 0..) |m, i| pmsgs[i] = .{ .message = .{ .role = @enumFromInt(@intFromEnum(m.role)), .content = &.{}, .metadata = if (m.has_metadata) slice(m.metadata) else null }, .usage = if (m.has_usage) m.usage else null }; + s.appendMessages(slice(session_id), pmsgs) catch |e| return setErr("{t}", .{e}); + return .ok; +} +fn wrapBuiltinSession(sess: panto.Session, out: **PantoSession) PantoStatus { + out.* = wrapSession(sess, true) catch |e| return setErr("{t}", .{e}); + return .ok; +} +fn wrapBuiltinResult(sess: panto.Session, out: *PantoSessionResult) PantoStatus { + var ps: *PantoSession = undefined; + const st = wrapBuiltinSession(sess, &ps); + if (st != .ok) return st; + out.* = .{ .found = true, .session = ps }; + return .ok; +} +fn nullStoreDestroy(ctx: *anyopaque) callconv(.c) void { + const s: *panto.NullStore = @ptrCast(@alignCast(ctx)); + const h: *NullStoreHandle = @fieldParentPtr("store", s); + allocator.destroy(h); +} +fn fsJSONLStoreDestroy(ctx: *anyopaque) callconv(.c) void { + const s: *panto.FileSystemJSONLStore = @ptrCast(@alignCast(ctx)); + const h: *FileSystemJSONLStoreHandle = @fieldParentPtr("store", s); + h.store.deinit(); + allocator.destroy(h); +} + +const null_store_vtable: PantoSessionStoreVTable = .{ .create = builtinStoreCreate, .list = builtinStoreList, .free_session_infos = builtinStoreFreeInfos, .resolve = builtinStoreResolve, .latest = builtinStoreLatest, .load = builtinStoreLoad, .append_messages = builtinStoreAppend, .destroy = nullStoreDestroy }; +const fs_jsonl_store_vtable: PantoSessionStoreVTable = .{ .create = builtinStoreCreate, .list = builtinStoreList, .free_session_infos = builtinStoreFreeInfos, .resolve = builtinStoreResolve, .latest = builtinStoreLatest, .load = builtinStoreLoad, .append_messages = builtinStoreAppend, .destroy = fsJSONLStoreDestroy }; + export fn panto_agent_create(c: *const PantoConfig, s: ?*PantoSession, conv: ?*PantoConversation, out: **PantoAgent) PantoStatus { const cfg = convertConfig(c) catch |e| return setErr("{t}", .{e}); const sess = if (s) |sp| blk: { @@ -280,19 +458,19 @@ export fn panto_agent_set_config(a: *PantoAgent, c: *const PantoConfig) PantoSta freeConfig(old); return .ok; } -export fn panto_agent_add_system_message(a: *PantoAgent, text: PantoSlice) PantoStatus { +export fn panto_agent_add_system_message(a: *PantoAgent, text: ?[*:0]const u8) PantoStatus { const h: *AgentHandle = @ptrCast(@alignCast(a)); - h.agent.addSystemMessage(slice(text)) catch |e| return setErr("{t}", .{e}); + h.agent.addSystemMessage(cstr(text)) catch |e| return setErr("{t}", .{e}); return .ok; } -export fn panto_agent_set_system_prompt(a: *PantoAgent, text: PantoSlice) PantoStatus { +export fn panto_agent_set_system_prompt(a: *PantoAgent, text: ?[*:0]const u8) PantoStatus { const h: *AgentHandle = @ptrCast(@alignCast(a)); - h.agent.setSystemPrompt(slice(text)) catch |e| return setErr("{t}", .{e}); + h.agent.setSystemPrompt(cstr(text)) catch |e| return setErr("{t}", .{e}); return .ok; } -export fn panto_agent_run(a: *PantoAgent, text: PantoSlice, out: **PantoStream) PantoStatus { +export fn panto_agent_run(a: *PantoAgent, text: ?[*:0]const u8, out: **PantoStream) PantoStatus { const h: *AgentHandle = @ptrCast(@alignCast(a)); - const st = h.agent.run(.{ .text = slice(text) }) catch |e| return setErr("{t}", .{e}); + const st = h.agent.run(.{ .text = cstr(text) }) catch |e| return setErr("{t}", .{e}); out.* = @ptrCast(st); return .ok; } diff --git a/libpanto-lua/src/module.zig b/libpanto-lua/src/module.zig index 7f7f838..f905765 100644 --- a/libpanto-lua/src/module.zig +++ b/libpanto-lua/src/module.zig @@ -146,6 +146,7 @@ fn releaseIo() void { const AGENT_MT = "panto.Agent"; const STREAM_MT = "panto.Stream"; const CONV_MT = "panto.Conversation"; +const STORE_MT = "panto.SessionStore"; /// Boxed `*Agent` userdata. /// @@ -171,7 +172,7 @@ const AgentBox = struct { config: *panto.Config, /// Superseded configs, freed at teardown (see the struct doc). old_configs: std.ArrayList(*panto.Config), - store_impl: panto.NullStore, + store_ref: c_int = c.LUA_NOREF, /// Arena owning every config string (api_key/base_url/model/prompt…) /// across the live config and all superseded ones. Freed at teardown. string_arena: std.heap.ArenaAllocator, @@ -191,6 +192,10 @@ const AgentBox = struct { for (self.old_configs.items) |cfg| c_allocator.destroy(cfg); self.old_configs.deinit(c_allocator); self.string_arena.deinit(); + if (self.store_ref != c.LUA_NOREF) { + c.luaL_unref(L, LUA_REGISTRYINDEX, self.store_ref); + self.store_ref = c.LUA_NOREF; + } releaseIo(); } }; @@ -229,8 +234,56 @@ const StreamBox = struct { /// /// `view` is the pointer methods operate through: for `.owned` it is /// `&self.inner`; for `.borrowed` it is `&agent.conversation`. +const StoreBox = struct { + impl: union(enum) { + null_store: panto.NullStore, + fs_jsonl: panto.FileSystemJSONLStore, + lua: LuaSessionStore, + }, + closed: bool = false, + + fn store(self: *StoreBox) panto.SessionStore { + return switch (self.impl) { + .null_store => |*s| s.store(), + .fs_jsonl => |*s| s.store(), + .lua => |*s| s.store(), + }; + } + + fn deinit(self: *StoreBox, L: *c.lua_State) void { + if (self.closed) return; + self.closed = true; + switch (self.impl) { + .null_store => {}, + .fs_jsonl => |*s| s.deinit(), + .lua => |*s| s.deinit(L), + } + } +}; + +const LuaSessionStore = struct { + L: *c.lua_State, + table_ref: c_int, + + fn init(L: *c.lua_State, idx: c_int) LuaSessionStore { + c.lua_pushvalue(L, idx); + return .{ .L = L, .table_ref = c.luaL_ref(L, LUA_REGISTRYINDEX) }; + } + + fn deinit(self: *LuaSessionStore, L: *c.lua_State) void { + if (self.table_ref != c.LUA_NOREF) { + c.luaL_unref(L, LUA_REGISTRYINDEX, self.table_ref); + self.table_ref = c.LUA_NOREF; + } + } + + fn store(self: *LuaSessionStore) panto.SessionStore { + return .{ .ptr = self, .vtable = &lua_store_vtable }; + } +}; + const ConvBox = struct { - state: enum { owned, adopted, borrowed }, + state: enum { owned, adopted, borrowed }, /// Storage for an owned conversation. Unused (undefined) when borrowed. inner: panto.Conversation, /// The conversation methods act on. Always valid while usable. @@ -268,11 +321,15 @@ pub export fn luaopen_panto(L_opt: ?*c.lua_State) callconv(.c) c_int { registerMetatables(L); - c.lua_createtable(L, 0, 3); + c.lua_createtable(L, 0, 5); c.lua_pushcclosure(L, agentNew, 0); c.lua_setfield(L, -2, "agent"); c.lua_pushcclosure(L, conversationNew, 0); c.lua_setfield(L, -2, "conversation"); + c.lua_pushcclosure(L, nullStoreNew, 0); + c.lua_setfield(L, -2, "null_store"); + c.lua_pushcclosure(L, fsJSONLStoreNew, 0); + c.lua_setfield(L, -2, "file_system_jsonl_store"); _ = c.lua_pushstring(L, "libpanto-lua 0.0.0 (Lua 5.4)"); c.lua_setfield(L, -2, "_VERSION"); return 1; @@ -307,6 +364,17 @@ fn registerMetatables(L: *c.lua_State) void { } c.lua_settop(L, c.lua_gettop(L) - 1); + if (c.luaL_newmetatable(L, STORE_MT) != 0) { + c.lua_createtable(L, 0, 5); // methods + setMethod(L, "list", storeList); + setMethod(L, "resolve", storeResolve); + setMethod(L, "latest", storeLatest); + setMethod(L, "load", storeLoad); + c.lua_setfield(L, -2, "__index"); + setMethod(L, "__gc", storeGc); + } + c.lua_settop(L, c.lua_gettop(L) - 1); + if (c.luaL_newmetatable(L, CONV_MT) != 0) { c.lua_createtable(L, 0, 9); // methods setMethod(L, "add_system_message", convAddSystemMessage); @@ -344,7 +412,7 @@ fn agentNew(L_opt: ?*c.lua_State) callconv(.c) c_int { .agent = undefined, .config = undefined, .old_configs = .empty, - .store_impl = undefined, + .store_ref = c.LUA_NOREF, .string_arena = std.heap.ArenaAllocator.init(c_allocator), .tools = null, .closed = false, @@ -386,16 +454,22 @@ fn buildAgent(L: *c.lua_State, box: *AgentBox) ConfigError!void { retainIo(); errdefer releaseIo(); + const store_box = try parseStore(L, 1); + const session = store_box.store().create(); + box.config = cfg; - box.store_impl = panto.NullStore.init(c_allocator); box.agent = panto.Agent.init( c_allocator, io_ctx.io, box.config, - box.store_impl.store().create(), + session, if (adopt_box) |cb| cb.inner else null, ) catch return error.AgentInit; + // Pin the store after Agent.init succeeds; the agent/session borrow it. + c.lua_pushvalue(L, -1); + box.store_ref = c.luaL_ref(L, LUA_REGISTRYINDEX); + // Ownership of the inner conversation moved into the agent; neuter the // source box so its `__gc` won't double-free. if (adopt_box) |cb| { @@ -574,6 +648,135 @@ fn conversationNew(L_opt: ?*c.lua_State) callconv(.c) c_int { return 1; } +// =========================================================================== +// panto.null_store() / panto.file_system_jsonl_store{} / custom Lua stores +// =========================================================================== + +fn nullStoreNew(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const ud = c.lua_newuserdatauv(L, @sizeOf(StoreBox), 0) orelse + return luaErr(L, "panto.null_store: out of memory"); + const sb: *StoreBox = @ptrCast(@alignCast(ud)); + sb.* = .{ .impl = .{ .null_store = panto.NullStore.init(c_allocator) } }; + _ = c.luaL_setmetatable(L, STORE_MT); + return 1; +} + +fn fsJSONLStoreNew(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + c.luaL_checktype(L, 1, T_TABLE); + const dir = requiredString(L, 1, "dir") catch return luaErr(L, "panto.file_system_jsonl_store: missing dir"); + const metadata = optString(L, 1, "metadata") catch return luaErr(L, "panto.file_system_jsonl_store: bad metadata"); + const store = panto.FileSystemJSONLStore.initWithMetadata(c_allocator, io_ctx.io, dir, metadata) catch + return luaErr(L, "panto.file_system_jsonl_store: failed to initialize store"); + + const ud = c.lua_newuserdatauv(L, @sizeOf(StoreBox), 0) orelse { + var s = store; + s.deinit(); + return luaErr(L, "panto.file_system_jsonl_store: out of memory"); + }; + const sb: *StoreBox = @ptrCast(@alignCast(ud)); + sb.* = .{ .impl = .{ .fs_jsonl = store } }; + _ = c.luaL_setmetatable(L, STORE_MT); + return 1; +} + +fn parseStore(L: *c.lua_State, agent_tbl: c_int) ConfigError!*StoreBox { + const t = c.lua_getfield(L, agent_tbl, "store"); + if (t == T_NIL) { + c.lua_settop(L, c.lua_gettop(L) - 1); + if (nullStoreNew(L) != 1) return error.OutOfMemory; + return checkStore(L, -1); + } + if (c.luaL_testudata(L, -1, STORE_MT)) |ud| return @ptrCast(@alignCast(ud)); + if (t != T_TABLE) return error.BadField; + + const ud = c.lua_newuserdatauv(L, @sizeOf(StoreBox), 0) orelse return error.OutOfMemory; + const sb: *StoreBox = @ptrCast(@alignCast(ud)); + sb.* = .{ .impl = .{ .lua = LuaSessionStore.init(L, -2) } }; + _ = c.luaL_setmetatable(L, STORE_MT); + return sb; +} + +fn checkStore(L: *c.lua_State, idx: c_int) *StoreBox { + return @ptrCast(@alignCast(c.luaL_checkudata(L, idx, STORE_MT))); +} + +fn storeGc(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + checkStore(L, 1).deinit(L); + return 0; +} + +fn storeList(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sb = checkStore(L, 1); + const infos = sb.store().list() catch return luaErr(L, "panto: store list failed"); + defer sb.store().freeSessionInfos(infos); + pushSessionInfoList(L, infos); + return 1; +} + +fn storeResolve(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sb = checkStore(L, 1); + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + const s = sb.store().resolve(ptr[0..len]) catch return luaErr(L, "panto: store resolve failed"); + if (s) |sess| pushSessionInfo(L, sess.info) else c.lua_pushnil(L); + return 1; +} + +fn storeLatest(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sb = checkStore(L, 1); + const s = sb.store().latest() catch return luaErr(L, "panto: store latest failed"); + if (s) |sess| pushSessionInfo(L, sess.info) else c.lua_pushnil(L); + return 1; +} + +fn storeLoad(L_opt: ?*c.lua_State) callconv(.c) c_int { + const L = L_opt.?; + const sb = checkStore(L, 1); + var len: usize = 0; + const ptr = c.luaL_checklstring(L, 2, &len); + const conv = (sb.store().load(ptr[0..len]) catch return luaErr(L, "panto: store load failed")) orelse { + c.lua_pushnil(L); + return 1; + }; + pushConversationOwned(L, conv); + return 1; +} + +fn pushConversationOwned(L: *c.lua_State, conv: panto.Conversation) void { + const ud = c.lua_newuserdatauv(L, @sizeOf(ConvBox), 0).?; + const cb: *ConvBox = @ptrCast(@alignCast(ud)); + cb.* = .{ .state = .owned, .inner = conv, .view = undefined, .agent_ref = c.LUA_NOREF }; + cb.view = &cb.inner; + _ = c.luaL_setmetatable(L, CONV_MT); +} + +fn pushSessionInfoList(L: *c.lua_State, infos: []panto.SessionInfo) void { + c.lua_createtable(L, @intCast(infos.len), 0); + for (infos, 0..) |info, i| { + pushSessionInfo(L, info); + c.lua_rawseti(L, -2, @intCast(i + 1)); + } +} + +fn pushSessionInfo(L: *c.lua_State, info: panto.SessionInfo) void { + c.lua_createtable(L, 0, 9); + setStringField(L, "id", info.id); + setStringField(L, "created", info.created); + setStringField(L, "modified", info.modified); + setIntField(L, "message_count", @intCast(info.message_count)); + setStringField(L, "last_user_message", info.last_user_message); + setStringField(L, "api_style", @tagName(info.api_style)); + setStringField(L, "base_url", info.base_url); + setStringField(L, "model", info.model); + setStringField(L, "reasoning", @tagName(info.reasoning)); +} + fn checkConv(L: *c.lua_State, idx: c_int) *ConvBox { return @ptrCast(@alignCast(c.luaL_checkudata(L, idx, CONV_MT))); } @@ -1673,6 +1876,159 @@ fn contentBlockTypeName(block: panto.ContentBlock) []const u8 { // package has no cross-package dependency on the CLI) // =========================================================================== +const lua_store_vtable: panto.SessionStore.VTable = .{ + .create = luaStoreCreate, + .list = luaStoreList, + .freeSessionInfos = luaStoreFreeInfos, + .resolve = luaStoreResolve, + .latest = luaStoreLatest, + .load = luaStoreLoad, + .appendMessages = luaStoreAppend, +}; + +fn luaStoreCreate(ctx: *anyopaque) panto.Session { + const self: *LuaSessionStore = @ptrCast(@alignCast(ctx)); + callStoreMethod(self, "create", 0) catch return fallbackNullSession(); + defer c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + return sessionFromLuaInfo(self, -1) catch fallbackNullSession(); +} + +fn luaStoreList(ctx: *anyopaque) anyerror![]panto.SessionInfo { + const self: *LuaSessionStore = @ptrCast(@alignCast(ctx)); + try callStoreMethod(self, "list", 0); + defer c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (c.lua_type(self.L, -1) != T_TABLE) return error.BadLuaStore; + const n: usize = @intCast(c.lua_rawlen(self.L, -1)); + const out = try c_allocator.alloc(panto.SessionInfo, n); + var built: usize = 0; + errdefer { for (out[0..built]) |i| i.deinit(c_allocator); c_allocator.free(out); } + var i: usize = 1; + while (i <= n) : (i += 1) { + _ = c.lua_rawgeti(self.L, -1, @intCast(i)); + defer c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + out[built] = try parseSessionInfo(self.L, -1); + built += 1; + } + return out; +} + +fn luaStoreFreeInfos(_: *anyopaque, infos: []panto.SessionInfo) void { + for (infos) |i| i.deinit(c_allocator); + c_allocator.free(infos); +} + +fn luaStoreResolve(ctx: *anyopaque, id: []const u8) anyerror!?panto.Session { + const self: *LuaSessionStore = @ptrCast(@alignCast(ctx)); + _ = c.lua_pushlstring(self.L, id.ptr, id.len); + try callStoreMethod(self, "resolve", 1); + defer c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (c.lua_type(self.L, -1) == T_NIL) return null; + return try sessionFromLuaInfo(self, -1); +} + +fn luaStoreLatest(ctx: *anyopaque) anyerror!?panto.Session { + const self: *LuaSessionStore = @ptrCast(@alignCast(ctx)); + try callStoreMethod(self, "latest", 0); + defer c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (c.lua_type(self.L, -1) == T_NIL) return null; + return try sessionFromLuaInfo(self, -1); +} + +fn luaStoreLoad(ctx: *anyopaque, id: []const u8) anyerror!?panto.Conversation { + const self: *LuaSessionStore = @ptrCast(@alignCast(ctx)); + _ = c.lua_pushlstring(self.L, id.ptr, id.len); + try callStoreMethod(self, "load", 1); + defer c.lua_settop(self.L, c.lua_gettop(self.L) - 1); + if (c.lua_type(self.L, -1) == T_NIL) return null; + const cb = @as(?*ConvBox, if (c.luaL_testudata(self.L, -1, CONV_MT)) |ud| @ptrCast(@alignCast(ud)) else null) orelse return error.BadLuaStore; + if (cb.state != .owned) return error.BadLuaStore; + const conv = cb.inner; + cb.inner = panto.Conversation.init(c_allocator); + return conv; +} + +fn luaStoreAppend(ctx: *anyopaque, session_id: []const u8, messages: []panto.PersistentMessage) anyerror!void { + const self: *LuaSessionStore = @ptrCast(@alignCast(ctx)); + _ = c.lua_pushlstring(self.L, session_id.ptr, session_id.len); + c.lua_createtable(self.L, @intCast(messages.len), 0); + for (messages, 0..) |m, i| { + pushPersistentMessage(self.L, m); + c.lua_rawseti(self.L, -2, @intCast(i + 1)); + } + try callStoreMethod(self, "append_messages", 2); + c.lua_settop(self.L, c.lua_gettop(self.L) - 1); +} + +fn callStoreMethod(self: *LuaSessionStore, name: [:0]const u8, nargs: c_int) !void { + const L = self.L; + _ = c.lua_rawgeti(L, LUA_REGISTRYINDEX, @intCast(self.table_ref)); + const tt = c.lua_getfield(L, -1, name.ptr); + if (tt != T_FUNCTION) return error.BadLuaStore; + c.lua_insert(L, -(nargs + 2)); + c.lua_settop(L, c.lua_gettop(L) - 1); + if (c.lua_pcallk(L, nargs, 1, 0, 0, null) != 0) return error.BadLuaStore; +} + +fn sessionFromLuaInfo(self: *LuaSessionStore, idx: c_int) !panto.Session { + return .{ .info = try parseSessionInfo(self.L, idx), .store = self.store() }; +} + +fn parseSessionInfo(L: *c.lua_State, idx: c_int) !panto.SessionInfo { + if (c.lua_type(L, idx) != T_TABLE) return error.BadLuaStore; + return .{ + .id = try dupLuaField(L, idx, "id"), + .created = try dupLuaField(L, idx, "created"), + .modified = try dupLuaField(L, idx, "modified"), + .message_count = @intCast(intLuaField(L, idx, "message_count")), + .last_user_message = try dupLuaField(L, idx, "last_user_message"), + .api_style = parseEnum(panto.APIStyle, luaFieldOrDefault(L, idx, "api_style", "openai_chat") catch "openai_chat") orelse .openai_chat, + .base_url = try dupLuaField(L, idx, "base_url"), + .model = try dupLuaField(L, idx, "model"), + .reasoning = parseEnum(panto.ReasoningEffort, luaFieldOrDefault(L, idx, "reasoning", "default") catch "default") orelse .default, + }; +} + +fn dupLuaField(L: *c.lua_State, idx: c_int, name: [:0]const u8) ![]u8 { + return c_allocator.dupe(u8, try luaFieldOrDefault(L, idx, name, "")); +} + +fn luaFieldOrDefault(L: *c.lua_State, idx: c_int, name: [:0]const u8, default: []const u8) ![]const u8 { + const abs = c.lua_absindex(L, idx); + const t = c.lua_getfield(L, abs, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t == T_NIL) return default; + if (t != T_STRING) return error.BadLuaStore; + var len: usize = 0; + const ptr = c.lua_tolstring(L, -1, &len) orelse return error.BadLuaStore; + return ptr[0..len]; +} + +fn intLuaField(L: *c.lua_State, idx: c_int, name: [:0]const u8) c.lua_Integer { + const abs = c.lua_absindex(L, idx); + const t = c.lua_getfield(L, abs, name.ptr); + defer c.lua_settop(L, c.lua_gettop(L) - 1); + if (t != T_NUMBER) return 0; + return c.lua_tointegerx(L, -1, null); +} + +fn fallbackNullSession() panto.Session { + var ns = panto.NullStore.init(c_allocator); + return ns.store().create(); +} + +fn pushPersistentMessage(L: *c.lua_State, pm: panto.PersistentMessage) void { + c.lua_createtable(L, 0, 4); + setStringField(L, "role", roleName(pm.message.role)); + if (pm.usage) |u| pushUsage(L, u); + setStringField(L, "metadata", pm.message.metadata orelse ""); + c.lua_createtable(L, @intCast(pm.message.content.items.len), 0); + for (pm.message.content.items, 0..) |block, i| { + pushInspectBlock(L, block); + c.lua_rawseti(L, -2, @intCast(i + 1)); + } + c.lua_setfield(L, -2, "blocks"); +} + const JsonError = error{ OutOfMemory, BadHandlerReturn, InputNotJsonObject }; /// Parse JSON bytes (must be a top-level object — tool input convention) diff --git a/libpanto/src/file_system_jsonl_store.zig b/libpanto/src/file_system_jsonl_store.zig index 979b35e..1d56ae9 100644 --- a/libpanto/src/file_system_jsonl_store.zig +++ b/libpanto/src/file_system_jsonl_store.zig @@ -210,11 +210,11 @@ pub const SessionFile = struct { allocator: Allocator, io: Io, session_dir: []const u8, - cwd: []const u8, + metadata: ?[]const u8, ) !SessionFile { const id = try newUuidV7(allocator, io); defer allocator.free(id); - return initWithId(allocator, io, session_dir, cwd, id); + return initWithId(allocator, io, session_dir, metadata, id); } /// Like `init`, but uses a caller-supplied session id (duped here) @@ -224,7 +224,7 @@ pub const SessionFile = struct { allocator: Allocator, io: Io, session_dir: []const u8, - cwd: []const u8, + metadata: ?[]const u8, session_id: []const u8, ) !SessionFile { const dir = try allocator.dupe(u8, session_dir); @@ -236,8 +236,8 @@ pub const SessionFile = struct { const timestamp = try isoTimestamp(allocator, io); errdefer allocator.free(timestamp); - const cwd_copy = try allocator.dupe(u8, cwd); - errdefer allocator.free(cwd_copy); + const metadata_copy: ?[]const u8 = if (metadata) |m| try allocator.dupe(u8, m) else null; + errdefer if (metadata_copy) |m| allocator.free(m); const filename = try std.fmt.allocPrint(allocator, "{s}.jsonl", .{id}); defer allocator.free(filename); @@ -253,7 +253,7 @@ pub const SessionFile = struct { .version = CURRENT_VERSION, .id = id, .timestamp = timestamp, - .cwd = cwd_copy, + .metadata = metadata_copy, }, .entries = .empty, .by_id = std.StringHashMap(usize).init(allocator), @@ -419,8 +419,8 @@ pub const SessionFile = struct { // ---------- Accessors ---------- - pub fn getCwd(self: *const SessionFile) []const u8 { - return self.header.cwd; + pub fn getMetadata(self: *const SessionFile) ?[]const u8 { + return self.header.metadata; } pub fn getSessionId(self: *const SessionFile) []const u8 { @@ -1157,24 +1157,28 @@ pub fn resolveSessionId( /// separate user-prompt and assistant-turn appends of a single turn. /// /// `dir` is the already-resolved sessions directory (the panto CLI derives -/// the per-cwd grouping; the store itself is cwd-agnostic). `cwd` is stamped -/// into new session headers for display/provenance only. +/// the per-cwd grouping; the store itself is cwd-agnostic). Optional +/// `metadata` is stamped into new session headers for display/provenance only. pub const FileSystemJSONLStore = struct { allocator: Allocator, io: Io, dir: []u8, // owned: the sessions directory - cwd: []u8, // owned: recorded in new session headers + metadata: ?[]u8, // owned: recorded in new session headers open: std.StringHashMap(*SessionFile), - pub fn init(allocator: Allocator, io: Io, dir: []const u8, cwd: []const u8) !FileSystemJSONLStore { + pub fn init(allocator: Allocator, io: Io, dir: []const u8) !FileSystemJSONLStore { + return initWithMetadata(allocator, io, dir, null); + } + + pub fn initWithMetadata(allocator: Allocator, io: Io, dir: []const u8, metadata: ?[]const u8) !FileSystemJSONLStore { const dir_copy = try allocator.dupe(u8, dir); errdefer allocator.free(dir_copy); - const cwd_copy = try allocator.dupe(u8, cwd); + const metadata_copy: ?[]u8 = if (metadata) |m| try allocator.dupe(u8, m) else null; return .{ .allocator = allocator, .io = io, .dir = dir_copy, - .cwd = cwd_copy, + .metadata = metadata_copy, .open = std.StringHashMap(*SessionFile).init(allocator), }; } @@ -1188,7 +1192,7 @@ pub const FileSystemJSONLStore = struct { } self.open.deinit(); self.allocator.free(self.dir); - self.allocator.free(self.cwd); + if (self.metadata) |m| self.allocator.free(m); } /// Borrow (opening if needed) the `SessionFile` for `id`, caching it. @@ -1213,7 +1217,7 @@ pub const FileSystemJSONLStore = struct { sf.* = if (exists) try SessionFile.open(self.allocator, self.io, full) else - try SessionFile.initWithId(self.allocator, self.io, self.dir, self.cwd, id); + try SessionFile.initWithId(self.allocator, self.io, self.dir, self.metadata, id); errdefer sf.deinit(); const key = try self.allocator.dupe(u8, id); @@ -1504,7 +1508,7 @@ test "SessionFile.init: does not create file yet" { testing.allocator, io, sessions, - "/some/cwd", + "{\"cwd\":\"/some/cwd\"}", ); defer mgr.deinit(); @@ -1529,7 +1533,7 @@ test "SessionFile: full flow — buffer, flush on assistant, append, resume" { testing.allocator, io, sessions, - "/proj/foo", + "{\"cwd\":\"/proj/foo\"}", ); defer mgr.deinit(); @@ -1599,7 +1603,7 @@ test "SessionFile: full flow — buffer, flush on assistant, append, resume" { defer resumed.deinit(); try testing.expect(resumed.isFlushed()); try testing.expectEqual(@as(usize, 5), resumed.getEntries().len); - try testing.expectEqualStrings("/proj/foo", resumed.getCwd()); + try testing.expectEqualStrings("{\"cwd\":\"/proj/foo\"}", resumed.getMetadata().?); // Continue the conversation. const u_three = try testing.allocator.alloc(StoredContentBlock, 1); @@ -1616,7 +1620,7 @@ test "SessionFile: assistant message tags the message metadata and the entry lea const sessions = try std.fs.path.join(testing.allocator, &.{ td.abs_path, "sessions" }); defer testing.allocator.free(sessions); - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const u_blocks = try testing.allocator.alloc(StoredContentBlock, 1); @@ -1645,7 +1649,7 @@ test "SessionFile: activeStamp is null before any user message, then tracks the const sessions = try std.fs.path.join(testing.allocator, &.{ td.abs_path, "sessions" }); defer testing.allocator.free(sessions); - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); // No user messages yet — there is no "active" model on disk yet. @@ -1671,7 +1675,7 @@ test "SessionFile: rebuildConversation reconstructs system/user/assistant turn" const sessions = try std.fs.path.join(testing.allocator, &.{ td.abs_path, "sessions" }); defer testing.allocator.free(sessions); - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const sys = try testing.allocator.alloc(StoredContentBlock, 1); @@ -1707,7 +1711,7 @@ test "SessionFile: crash recovery truncates corrupted trailing line" { // Build a valid session first. const session_file: []u8 = blk: { - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const u = try testing.allocator.alloc(StoredContentBlock, 1); u[0] = .{ .text = .{ .text = try testing.allocator.dupe(u8, "ping") } }; @@ -1760,7 +1764,7 @@ test "listSessions: returns most recent first, with counts" { // Create two sessions. for (0..2) |i| { - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const u = try testing.allocator.alloc(StoredContentBlock, 1); u[0] = .{ .text = .{ .text = try testing.allocator.dupe(u8, "u") } }; @@ -1799,7 +1803,7 @@ test "findMostRecentSession: picks lexicographically greatest" { defer if (second_file) |s| testing.allocator.free(s); for (0..2) |i| { - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const u = try testing.allocator.alloc(StoredContentBlock, 1); u[0] = .{ .text = .{ .text = try testing.allocator.dupe(u8, "u") } }; @@ -1824,7 +1828,7 @@ test "SessionFile: tool-use round-trip — assistant w/ ToolUse, user w/ ToolRes defer testing.allocator.free(sessions); const session_file: []u8 = blk: { - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const u = try testing.allocator.alloc(StoredContentBlock, 1); @@ -1895,7 +1899,7 @@ test "SessionFile: linear chain — each entry's parent_id is the previous entry const sessions = try std.fs.path.join(testing.allocator, &.{ td.abs_path, "sessions" }); defer testing.allocator.free(sessions); - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); // Three rounds: sys, user, asst, user, asst. @@ -1936,7 +1940,7 @@ test "resolveSessionId: unique prefix → match, ambiguous → error" { defer testing.allocator.free(sessions); // Create one session. - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); const u = try testing.allocator.alloc(StoredContentBlock, 1); u[0] = .{ .text = .{ .text = try testing.allocator.dupe(u8, "u") } }; @@ -1964,7 +1968,7 @@ test "compaction summary round-trips through persist + resume + rebuild" { defer testing.allocator.free(sessions); const session_file: []u8 = blk: { - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); // System + an old turn that will be superseded. @@ -2024,7 +2028,7 @@ test "loadConversation: trailing user prompt is split out as dangling, excluded const sessions = try std.fs.path.join(testing.allocator, &.{ td.abs_path, "sessions" }); defer testing.allocator.free(sessions); - var mgr = try SessionFile.init(testing.allocator, io, sessions, "/c"); + var mgr = try SessionFile.init(testing.allocator, io, sessions, null); defer mgr.deinit(); // A completed user/assistant round, then a trailing user prompt with no @@ -2056,7 +2060,7 @@ test "FileSystemJSONLStore catalog: create → append → load round-trips" { const sessions = try std.fs.path.join(testing.allocator, &.{ td.abs_path, "sessions" }); defer testing.allocator.free(sessions); - var catalog = try FileSystemJSONLStore.init(testing.allocator, io, sessions, "/c"); + var catalog = try FileSystemJSONLStore.init(testing.allocator, io, sessions); defer catalog.deinit(); const store = catalog.store(); diff --git a/libpanto/src/session.zig b/libpanto/src/session.zig index 492f94b..843989a 100644 --- a/libpanto/src/session.zig +++ b/libpanto/src/session.zig @@ -87,12 +87,14 @@ pub const SessionHeader = struct { version: u32, id: []const u8, // UUIDv7 string, owned timestamp: []const u8, // ISO 8601, owned - cwd: []const u8, // owned + /// Opaque session-wide metadata bag. Round-trips verbatim; `libpanto` + /// never interprets it. The panto CLI records `{ "cwd": ... }` here. + metadata: ?[]const u8 = null, // owned pub fn deinit(self: SessionHeader, alloc: Allocator) void { alloc.free(self.id); alloc.free(self.timestamp); - alloc.free(self.cwd); + if (self.metadata) |m| alloc.free(m); } }; @@ -308,8 +310,12 @@ pub fn serializeHeader(allocator: Allocator, header: SessionHeader) ![]u8 { try s.write(header.id); try s.objectField("timestamp"); try s.write(header.timestamp); - try s.objectField("cwd"); - try s.write(header.cwd); + if (header.metadata) |md| { + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, md, .{}); + defer parsed.deinit(); + try s.objectField("metadata"); + try s.write(parsed.value); + } try s.endObject(); return try aw.toOwnedSlice(); @@ -568,13 +574,24 @@ fn parseHeaderFromObject(allocator: Allocator, obj: std.json.ObjectMap) ParseErr errdefer allocator.free(id); const timestamp = try dupeStringField(allocator, obj, "timestamp"); errdefer allocator.free(timestamp); - const cwd = try dupeStringField(allocator, obj, "cwd"); - errdefer allocator.free(cwd); + const metadata: ?[]const u8 = blk: { + if (obj.get("metadata")) |mv| { + break :blk try std.json.Stringify.valueAlloc(allocator, mv, .{}); + } + if (obj.get("cwd")) |cv| { + if (cv != .string) return error.MissingField; + const cwd_json = try std.json.Stringify.valueAlloc(allocator, cv, .{}); + defer allocator.free(cwd_json); + break :blk try std.fmt.allocPrint(allocator, "{{\"cwd\":{s}}}", .{cwd_json}); + } + break :blk null; + }; + errdefer if (metadata) |m| allocator.free(m); return .{ .version = version, .id = id, .timestamp = timestamp, - .cwd = cwd, + .metadata = metadata, }; } @@ -956,7 +973,7 @@ test "serialize/parse header round-trip" { .version = 1, .id = try dupe(a, "019dc5ba-53f6-71a5-ab8f-b1f8709c2572"), .timestamp = try dupe(a, "2026-04-25T17:40:15.990Z"), - .cwd = try dupe(a, "/Users/travis/Code/pantograph"), + .metadata = try dupe(a, "{\"cwd\":\"/Users/travis/Code/pantograph\"}"), }; defer header.deinit(a); @@ -968,7 +985,7 @@ test "serialize/parse header round-trip" { try testing.expect(fe == .header); try testing.expectEqual(@as(u32, 1), fe.header.version); try testing.expectEqualStrings(header.id, fe.header.id); - try testing.expectEqualStrings(header.cwd, fe.header.cwd); + try testing.expectEqualStrings(header.metadata.?, fe.header.metadata.?); } test "serialize/parse user message entry round-trip (with provider/model stamp)" { diff --git a/src/main.zig b/src/main.zig index 06fd29e..693e88c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -251,7 +251,9 @@ pub fn main(init: std.process.Init) !void { // per-cwd `session_dir` (the CLI owns the cwd→dir grouping; the store // is cwd-agnostic). Must outlive the agent, which appends through a // `Session` handle minted/resolved below. - var session_store_impl = try panto.FileSystemJSONLStore.init(alloc, io, session_dir, cwd); + const session_metadata = try std.fmt.allocPrint(alloc, "{{\"cwd\":{s}}}", .{try std.json.Stringify.valueAlloc(alloc, cwd, .{})}); + defer alloc.free(session_metadata); + var session_store_impl = try panto.FileSystemJSONLStore.initWithMetadata(alloc, io, session_dir, session_metadata); defer session_store_impl.deinit(); const session_store = session_store_impl.store(); diff --git a/src/subcommand.zig b/src/subcommand.zig index f0fdfc9..21d2065 100644 --- a/src/subcommand.zig +++ b/src/subcommand.zig @@ -272,7 +272,7 @@ fn runSessionsSubcommand( const session_dir = try session_paths.sessionDirForCwd(allocator, environ_map, cwd); defer allocator.free(session_dir); - var store_impl = try panto.FileSystemJSONLStore.init(allocator, io, session_dir, cwd); + var store_impl = try panto.FileSystemJSONLStore.init(allocator, io, session_dir); defer store_impl.deinit(); const store = store_impl.store(); diff --git a/src/system_prompt.zig b/src/system_prompt.zig index 9dce3e5..e2b2907 100644 --- a/src/system_prompt.zig +++ b/src/system_prompt.zig @@ -473,7 +473,7 @@ const TmpLayers = struct { fn openTmpStore(arena: Allocator, root: []const u8) !panto.FileSystemJSONLStore { const sessions = try std.fs.path.join(arena, &.{ root, "sessions" }); - return panto.FileSystemJSONLStore.init(testing.allocator, testing.io, sessions, "/cwd"); + return panto.FileSystemJSONLStore.init(testing.allocator, testing.io, sessions); } /// Minimal agent harness for system-prompt tests: a throwaway provider |