system prompt building and logging

1 files changed, 541 insertions, 0 deletions

diff --git a/src/system_prompt.zig b/src/system_prompt.zig
new file mode 100644
index 0000000..b796394
--- /dev/null
+++ b/src/system_prompt.zig
@@ -0,0 +1,541 @@
+//! System-prompt sourcing and resume reconciliation for the `panto` CLI.
+//!
+//! The system prompt is **conversation state**, not config state (see
+//! `docs/system-prompt.md`). It is sourced — by convention, with no TOML
+//! key — from two files discovered across the same three config layers the
+//! CLI already uses, in precedence order base → user → project (project
+//! highest):
+//!
+//!     base    = $PANTO_HOME/agent/            (= $XDG_DATA_HOME/panto/agent/)
+//!     user    = ${XDG_CONFIG_HOME:-$HOME/.config}/panto/
+//!     project = ./.panto/
+//!
+//! In each layer we look for:
+//!   - `SYSTEM.md`        — the base/seed prompt (highest layer present wins).
+//!   - `APPEND_SYSTEM.md` — an additional appended block (every layer kept).
+//!
+//! Resolution:
+//!   1. Seed = the highest-layer `SYSTEM.md`, else a built-in default.
+//!   2. Appends = each layer's `APPEND_SYSTEM.md`, collected base → user →
+//!      project, then emitted reversed: project → user → base. The seed
+//!      leads; the most-specific (project) append follows it.
+
+const std = @import("std");
+const panto = @import("panto");
+
+const Allocator = std.mem.Allocator;
+const Io = std.Io;
+
+/// Last-resort fallback seed when no `SYSTEM.md` exists at any layer.
+///
+/// In normal operation this is never used: the base layer ships a bundled
+/// `agent/SYSTEM.md` (embedded in the binary and staged to
+/// `$PANTO_HOME/agent/SYSTEM.md` at bootstrap), so a seed is always found.
+/// This constant only matters if that staged file is missing or
+/// unreadable.
+pub const default_seed = "You are a helpful assistant.";
+
+const seed_filename = "SYSTEM.md";
+const append_filename = "APPEND_SYSTEM.md";
+
+/// A resolved set of system blocks for a fresh session: the seed followed
+/// by zero or more appends, already in emission order. All slices
+/// are owned by `arena`.
+pub const Resolved = struct {
+    /// Seed text (always present — a built-in default if no file existed).
+    seed: []const u8,
+    /// Append blocks in emission order (project → user → base).
+    appends: []const []const u8,
+
+    /// The full ordered block list: `[seed, appends...]`. Owned by `arena`.
+    pub fn blocks(self: Resolved, arena: Allocator) ![]const []const u8 {
+        var list = try arena.alloc([]const u8, 1 + self.appends.len);
+        list[0] = self.seed;
+        @memcpy(list[1..], self.appends);
+        return list;
+    }
+};
+
+/// Resolve the system-prompt blocks from the three config layers.
+///
+/// `base_dir` is the base layer directory (`$PANTO_HOME/agent`). The user
+/// layer is derived from `XDG_CONFIG_HOME`/`HOME`; the project layer is
+/// `cwd()/.panto`. Allocations are made in `arena` (caller owns it).
+pub fn resolve(
+    arena: Allocator,
+    io: Io,
+    environ_map: *const std.process.Environ.Map,
+    base_dir: []const u8,
+) !Resolved {
+    const user_dir = try userLayerDir(arena, environ_map);
+    const project_dir = try projectLayerDir(arena, io);
+    return resolveLayers(arena, io, base_dir, user_dir, project_dir);
+}
+
+/// Core resolution against explicit layer directories (base → user →
+/// project, project highest). Any layer dir may be null (absent). Exposed
+/// for testing; `resolve` derives the user/project dirs from the
+/// environment and cwd.
+pub fn resolveLayers(
+    arena: Allocator,
+    io: Io,
+    base_dir: ?[]const u8,
+    user_dir: ?[]const u8,
+    project_dir: ?[]const u8,
+) !Resolved {
+    // Seed: highest layer present wins (project > user > base).
+    var seed: []const u8 = default_seed;
+    for ([_]?[]const u8{ base_dir, user_dir, project_dir }) |maybe_dir| {
+        const dir = maybe_dir orelse continue;
+        if (try readLayerFile(arena, io, dir, seed_filename)) |text| {
+            seed = text;
+        }
+    }
+
+    // Appends: collect base → user → project, then reverse for emission.
+    var collected: std.ArrayList([]const u8) = .empty;
+    for ([_]?[]const u8{ base_dir, user_dir, project_dir }) |maybe_dir| {
+        const dir = maybe_dir orelse continue;
+        if (try readLayerFile(arena, io, dir, append_filename)) |text| {
+            try collected.append(arena, text);
+        }
+    }
+    std.mem.reverse([]const u8, collected.items);
+
+    return .{ .seed = seed, .appends = try collected.toOwnedSlice(arena) };
+}
+
+/// Seed a fresh conversation with the resolved blocks and record matching
+/// `append`-mode entries in the session log.
+pub fn seedFresh(
+    arena: Allocator,
+    io: Io,
+    environ_map: *const std.process.Environ.Map,
+    base_dir: []const u8,
+    conv: *panto.conversation.Conversation,
+    mgr: *panto.session_manager.SessionManager,
+) !void {
+    const user_dir = try userLayerDir(arena, environ_map);
+    const project_dir = try projectLayerDir(arena, io);
+    return seedFreshLayers(arena, io, base_dir, user_dir, project_dir, conv, mgr);
+}
+
+/// Layer-explicit variant of `seedFresh` (see `resolveLayers`).
+pub fn seedFreshLayers(
+    arena: Allocator,
+    io: Io,
+    base_dir: ?[]const u8,
+    user_dir: ?[]const u8,
+    project_dir: ?[]const u8,
+    conv: *panto.conversation.Conversation,
+    mgr: *panto.session_manager.SessionManager,
+) !void {
+    const resolved = try resolveLayers(arena, io, base_dir, user_dir, project_dir);
+    const blocks = try resolved.blocks(arena);
+    for (blocks) |text| {
+        try conv.addSystemMessage(text);
+        try appendSystemEntry(mgr.allocator, mgr, text, .append);
+    }
+}
+
+/// Reconcile a resumed conversation against the current on-disk config.
+///
+/// Compares the freshly-resolved config blocks, by position and exact
+/// text, against the conversation's **current effective config window**:
+/// the last `replace`-mode system block plus every system block after it.
+/// If they match, this is a no-op. Otherwise it appends a fresh
+/// `replace + N×append` sequence to both the conversation and the log,
+/// re-seeding the prompt wholesale without rewriting history.
+pub fn reconcileResume(
+    arena: Allocator,
+    io: Io,
+    environ_map: *const std.process.Environ.Map,
+    base_dir: []const u8,
+    conv: *panto.conversation.Conversation,
+    mgr: *panto.session_manager.SessionManager,
+) !void {
+    const user_dir = try userLayerDir(arena, environ_map);
+    const project_dir = try projectLayerDir(arena, io);
+    return reconcileResumeLayers(arena, io, base_dir, user_dir, project_dir, conv, mgr);
+}
+
+/// Layer-explicit variant of `reconcileResume` (see `resolveLayers`).
+pub fn reconcileResumeLayers(
+    arena: Allocator,
+    io: Io,
+    base_dir: ?[]const u8,
+    user_dir: ?[]const u8,
+    project_dir: ?[]const u8,
+    conv: *panto.conversation.Conversation,
+    mgr: *panto.session_manager.SessionManager,
+) !void {
+    const resolved = try resolveLayers(arena, io, base_dir, user_dir, project_dir);
+    const config_blocks = try resolved.blocks(arena);
+
+    const window = try effectiveConfigWindow(arena, conv.messages.items);
+
+    if (blocksEqual(config_blocks, window)) return;
+
+    // Re-seed: one `replace` for the seed, then one `append` per append.
+    try conv.replaceSystemMessage(config_blocks[0]);
+    try appendSystemEntry(mgr.allocator, mgr, config_blocks[0], .replace);
+    for (config_blocks[1..]) |text| {
+        try conv.addSystemMessage(text);
+        try appendSystemEntry(mgr.allocator, mgr, text, .append);
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Internals
+// ---------------------------------------------------------------------------
+
+/// The current effective config window: the text of the last `replace`-mode
+/// system block plus every system block after it, in order. If there is no
+/// `replace` block, the window is every leading system block (the original
+/// seed sequence). Returned slices borrow from `messages`.
+fn effectiveConfigWindow(
+    arena: Allocator,
+    messages: []const panto.conversation.Message,
+) ![]const []const u8 {
+    // Find the index of the last message carrying a `replace`-mode block.
+    var anchor: ?usize = null;
+    for (messages, 0..) |msg, i| {
+        if (msg.role != .system) continue;
+        for (msg.content.items) |block| {
+            switch (block) {
+                .System => |sb| if (sb.mode == .replace) {
+                    anchor = i;
+                },
+                else => {},
+            }
+        }
+    }
+
+    var out: std.ArrayList([]const u8) = .empty;
+    const start = anchor orelse 0;
+    for (messages[start..]) |msg| {
+        if (msg.role != .system) continue;
+        for (msg.content.items) |block| {
+            switch (block) {
+                .System => |sb| try out.append(arena, sb.text.items),
+                .Text => |tb| try out.append(arena, tb.items),
+                else => {},
+            }
+        }
+    }
+    return out.toOwnedSlice(arena);
+}
+
+fn blocksEqual(a: []const []const u8, b: []const []const u8) bool {
+    if (a.len != b.len) return false;
+    for (a, b) |x, y| {
+        if (!std.mem.eql(u8, x, y)) return false;
+    }
+    return true;
+}
+
+/// Append a system-role entry to the session log carrying `mode`.
+fn appendSystemEntry(
+    alloc: Allocator,
+    mgr: *panto.session_manager.SessionManager,
+    text: []const u8,
+    mode: panto.session_manager.DiskSystemMode,
+) !void {
+    const blocks = try alloc.alloc(panto.session_manager.DiskContentBlock, 1);
+    blocks[0] = .{ .text = .{ .text = try alloc.dupe(u8, text) } };
+    _ = try mgr.appendMessage(
+        .{ .role = .system, .mode = mode, .content = blocks },
+        null,
+        null,
+    );
+}
+
+/// Read `dir/name`, returning its contents (owned by `arena`) or null if the
+/// file does not exist. Other I/O errors propagate.
+fn readLayerFile(
+    arena: Allocator,
+    io: Io,
+    dir: []const u8,
+    name: []const u8,
+) !?[]u8 {
+    const path = try std.fs.path.join(arena, &.{ dir, name });
+    const file = Io.Dir.cwd().openFile(io, path, .{ .mode = .read_only }) catch |err| switch (err) {
+        error.FileNotFound => return null,
+        else => return err,
+    };
+    defer file.close(io);
+    const len = file.length(io) catch return error.SystemPromptReadFailed;
+    const bytes = try arena.alloc(u8, @intCast(len));
+    errdefer arena.free(bytes);
+    _ = file.readPositionalAll(io, bytes, 0) catch return error.SystemPromptReadFailed;
+    return bytes;
+}
+
+fn userLayerDir(arena: Allocator, environ_map: *const std.process.Environ.Map) !?[]u8 {
+    if (environ_map.get("XDG_CONFIG_HOME")) |xdg| {
+        return try std.fs.path.join(arena, &.{ xdg, "panto" });
+    }
+    if (environ_map.get("HOME")) |home| {
+        return try std.fs.path.join(arena, &.{ home, ".config", "panto" });
+    }
+    return null;
+}
+
+fn projectLayerDir(arena: Allocator, io: Io) ![]u8 {
+    const cwd = try std.process.currentPathAlloc(io, arena);
+    return try std.fs.path.join(arena, &.{ cwd, ".panto" });
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+const testing = std.testing;
+
+test "blocksEqual - length and content" {
+    try testing.expect(blocksEqual(&.{ "a", "b" }, &.{ "a", "b" }));
+    try testing.expect(!blocksEqual(&.{ "a", "b" }, &.{ "a", "c" }));
+    try testing.expect(!blocksEqual(&.{"a"}, &.{ "a", "b" }));
+    try testing.expect(blocksEqual(&.{}, &.{}));
+}
+
+test "effectiveConfigWindow - no replace anchors to leading system blocks" {
+    const alloc = testing.allocator;
+    var conv = panto.conversation.Conversation.init(alloc);
+    defer conv.deinit();
+
+    try conv.addSystemMessage("seed");
+    try conv.addSystemMessage("append-1");
+    try conv.addUserMessage("hi");
+
+    var arena_state = std.heap.ArenaAllocator.init(alloc);
+    defer arena_state.deinit();
+    const window = try effectiveConfigWindow(arena_state.allocator(), conv.messages.items);
+
+    try testing.expectEqual(@as(usize, 2), window.len);
+    try testing.expectEqualStrings("seed", window[0]);
+    try testing.expectEqualStrings("append-1", window[1]);
+}
+
+test "effectiveConfigWindow - anchors to last replace block" {
+    const alloc = testing.allocator;
+    var conv = panto.conversation.Conversation.init(alloc);
+    defer conv.deinit();
+
+    try conv.addSystemMessage("old seed");
+    try conv.addUserMessage("hi");
+    try conv.replaceSystemMessage("new seed");
+    try conv.addSystemMessage("new append");
+    try conv.addUserMessage("again");
+
+    var arena_state = std.heap.ArenaAllocator.init(alloc);
+    defer arena_state.deinit();
+    const window = try effectiveConfigWindow(arena_state.allocator(), conv.messages.items);
+
+    try testing.expectEqual(@as(usize, 2), window.len);
+    try testing.expectEqualStrings("new seed", window[0]);
+    try testing.expectEqualStrings("new append", window[1]);
+}
+
+fn writeTmpFile(dir: Io.Dir, sub_path: []const u8, content: []const u8) !void {
+    try dir.writeFile(testing.io, .{ .sub_path = sub_path, .data = content });
+}
+
+test "resolveLayers - highest SYSTEM.md wins, appends reversed project-first" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    // base: SYSTEM.md + APPEND_SYSTEM.md; user: APPEND only; project:
+    // SYSTEM.md + APPEND_SYSTEM.md (matches the design's worked example).
+    try tmp.dir.createDirPath(testing.io, "base");
+    try tmp.dir.createDirPath(testing.io, "user");
+    try tmp.dir.createDirPath(testing.io, "project");
+    try writeTmpFile(tmp.dir, "base/SYSTEM.md", "base seed");
+    try writeTmpFile(tmp.dir, "base/APPEND_SYSTEM.md", "base append");
+    try writeTmpFile(tmp.dir, "user/APPEND_SYSTEM.md", "user append");
+    try writeTmpFile(tmp.dir, "project/SYSTEM.md", "project seed");
+    try writeTmpFile(tmp.dir, "project/APPEND_SYSTEM.md", "project append");
+
+    var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+
+    var buf: [std.fs.max_path_bytes]u8 = undefined;
+    const root_len = try tmp.dir.realPathFile(testing.io, ".", &buf);
+    const root = buf[0..root_len];
+    const base_dir = try std.fs.path.join(arena, &.{ root, "base" });
+    const user_dir = try std.fs.path.join(arena, &.{ root, "user" });
+    const project_dir = try std.fs.path.join(arena, &.{ root, "project" });
+
+    const resolved = try resolveLayers(arena, testing.io, base_dir, user_dir, project_dir);
+    try testing.expectEqualStrings("project seed", resolved.seed);
+
+    const blocks = try resolved.blocks(arena);
+    try testing.expectEqual(@as(usize, 4), blocks.len);
+    try testing.expectEqualStrings("project seed", blocks[0]);
+    try testing.expectEqualStrings("project append", blocks[1]);
+    try testing.expectEqualStrings("user append", blocks[2]);
+    try testing.expectEqualStrings("base append", blocks[3]);
+}
+
+test "resolveLayers - no SYSTEM.md anywhere falls back to built-in default" {
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+
+    var buf: [std.fs.max_path_bytes]u8 = undefined;
+    const root_len = try tmp.dir.realPathFile(testing.io, ".", &buf);
+    const root = buf[0..root_len];
+
+    const resolved = try resolveLayers(arena, testing.io, root, null, null);
+    try testing.expectEqualStrings(default_seed, resolved.seed);
+    try testing.expectEqual(@as(usize, 0), resolved.appends.len);
+}
+
+const TmpLayers = struct {
+    tmp: std.testing.TmpDir,
+    root: []u8,
+
+    fn init(arena: Allocator) !TmpLayers {
+        var tmp = std.testing.tmpDir(.{ .iterate = true });
+        errdefer tmp.cleanup();
+        var buf: [std.fs.max_path_bytes]u8 = undefined;
+        const n = try tmp.dir.realPathFile(testing.io, ".", &buf);
+        const root = try arena.dupe(u8, buf[0..n]);
+        try tmp.dir.createDirPath(testing.io, "project");
+        return .{ .tmp = tmp, .root = root };
+    }
+
+    fn projectDir(self: TmpLayers, arena: Allocator) ![]u8 {
+        return std.fs.path.join(arena, &.{ self.root, "project" });
+    }
+
+    fn writeProject(self: *TmpLayers, name: []const u8, content: []const u8) !void {
+        const sub = try std.fmt.allocPrint(testing.allocator, "project/{s}", .{name});
+        defer testing.allocator.free(sub);
+        try self.tmp.dir.writeFile(testing.io, .{ .sub_path = sub, .data = content });
+    }
+
+    fn deinit(self: *TmpLayers) void {
+        self.tmp.cleanup();
+    }
+};
+
+fn openTmpSession(arena: Allocator, root: []const u8) !panto.session_manager.SessionManager {
+    const sessions = try std.fs.path.join(arena, &.{ root, "sessions" });
+    return panto.session_manager.SessionManager.init(testing.allocator, testing.io, sessions, "/cwd");
+}
+
+/// Force the session to flush to disk by appending an assistant entry.
+/// (System/user entries buffer in memory until the first assistant entry;
+/// a realistic resume only sees what a completed turn persisted.)
+fn forceFlush(mgr: *panto.session_manager.SessionManager) !void {
+    const blocks = try mgr.allocator.alloc(panto.session_manager.DiskContentBlock, 1);
+    blocks[0] = .{ .text = .{ .text = try mgr.allocator.dupe(u8, "ok") } };
+    _ = try mgr.appendMessage(
+        .{ .role = .assistant, .content = blocks },
+        "prov",
+        "model",
+    );
+}
+
+test "seedFresh then no-config-change resume is a no-op" {
+    const alloc = testing.allocator;
+    var arena_state = std.heap.ArenaAllocator.init(alloc);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+
+    var layers = try TmpLayers.init(arena);
+    defer layers.deinit();
+    try layers.writeProject("SYSTEM.md", "project seed");
+    try layers.writeProject("APPEND_SYSTEM.md", "project append");
+    const project_dir = try layers.projectDir(arena);
+
+    // Fresh session: seed + persist.
+    var mgr = try openTmpSession(arena, layers.root);
+    {
+        var conv = panto.conversation.Conversation.init(alloc);
+        defer conv.deinit();
+        try seedFreshLayers(arena, testing.io, null, null, project_dir, &conv, &mgr);
+    }
+    try forceFlush(&mgr); // persist seed + append (+ assistant) to disk
+    const entries_on_disk = mgr.getEntries().len;
+    try testing.expectEqual(@as(usize, 3), entries_on_disk); // seed + append + assistant
+    const session_file = try arena.dupe(u8, mgr.getSessionFile());
+    mgr.deinit();
+
+    // Resume: reopen the file, rebuild + reconcile against unchanged config
+    // → no new entries.
+    var mgr2 = try panto.session_manager.SessionManager.open(alloc, testing.io, session_file);
+    defer mgr2.deinit();
+    var conv2 = try mgr2.rebuildConversation();
+    defer conv2.deinit();
+    try reconcileResumeLayers(arena, testing.io, null, null, project_dir, &conv2, &mgr2);
+    try testing.expectEqual(entries_on_disk, mgr2.getEntries().len);
+}
+
+test "resume after config change appends replace + append sequence" {
+    const alloc = testing.allocator;
+    var arena_state = std.heap.ArenaAllocator.init(alloc);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+
+    var layers = try TmpLayers.init(arena);
+    defer layers.deinit();
+    try layers.writeProject("SYSTEM.md", "old seed");
+    const project_dir = try layers.projectDir(arena);
+
+    var mgr = try openTmpSession(arena, layers.root);
+    {
+        var conv = panto.conversation.Conversation.init(alloc);
+        defer conv.deinit();
+        try seedFreshLayers(arena, testing.io, null, null, project_dir, &conv, &mgr);
+    }
+    try forceFlush(&mgr); // persist old seed (+ assistant) to disk
+    try testing.expectEqual(@as(usize, 2), mgr.getEntries().len); // seed + assistant
+    const session_file = try arena.dupe(u8, mgr.getSessionFile());
+    mgr.deinit();
+
+    // Change the config: new seed + a new append.
+    try layers.writeProject("SYSTEM.md", "new seed");
+    try layers.writeProject("APPEND_SYSTEM.md", "new append");
+
+    var mgr2 = try panto.session_manager.SessionManager.open(alloc, testing.io, session_file);
+    defer mgr2.deinit();
+    var conv2 = try mgr2.rebuildConversation();
+    defer conv2.deinit();
+    try reconcileResumeLayers(arena, testing.io, null, null, project_dir, &conv2, &mgr2);
+
+    // old seed + assistant + replace(new seed) + append(new append) = 4.
+    try testing.expectEqual(@as(usize, 4), mgr2.getEntries().len);
+
+    // The effective prompt now reflects only the new config blocks.
+    const eff = try panto.conversation.effectiveSystemBlocks(arena, conv2.messages.items);
+    try testing.expectEqual(@as(usize, 2), eff.items.len);
+    try testing.expectEqualStrings("new seed", eff.items[0]);
+    try testing.expectEqualStrings("new append", eff.items[1]);
+
+    // A second no-op resume must not append again (anchors to the new
+    // `replace` window, not the stale original seed).
+    const after_first = mgr2.getEntries().len;
+    try reconcileResumeLayers(arena, testing.io, null, null, project_dir, &conv2, &mgr2);
+    try testing.expectEqual(after_first, mgr2.getEntries().len);
+}
+
+test "Resolved.blocks - seed leads appends" {
+    const alloc = testing.allocator;
+    var arena_state = std.heap.ArenaAllocator.init(alloc);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+
+    const resolved: Resolved = .{ .seed = "S", .appends = &.{ "P", "U", "B" } };
+    const blocks = try resolved.blocks(arena);
+    try testing.expectEqual(@as(usize, 4), blocks.len);
+    try testing.expectEqualStrings("S", blocks[0]);
+    try testing.expectEqualStrings("P", blocks[1]);
+    try testing.expectEqualStrings("U", blocks[2]);
+    try testing.expectEqualStrings("B", blocks[3]);
+}