proper terminal capability negotiation

determine support for the kitty protocol and then pick output sequences
accordingly.

tested on ghostty (supports kitty) and tmux-in-ghostty (tmux does not
support kitty protocol) with shift+enter as newline.

1 files changed, 340 insertions, 27 deletions

diff --git a/src/tui_input.zig b/src/tui_input.zig
index fdd174c..276b7e1 100644
--- a/src/tui_input.zig
+++ b/src/tui_input.zig
@@ -16,16 +16,18 @@
 //!     nothing here requires a response, and unrecognized sequences are
 //!     consumed and dropped rather than mis-decoded.
 //!
-//! Shift+Enter limitation: without the Kitty protocol most terminals send the
-//! exact same bytes for Enter and Shift+Enter (`\r`), so this decoder cannot
-//! distinguish them and reports both as `.enter` with no shift modifier. When
-//! the Kitty protocol IS active, Shift+Enter arrives as a CSI-u sequence
-//! (`\x1b[13;2u`) which we DO decode, setting `mods.shift`. The `Key`/`Mods`
-//! model can therefore represent the distinction even on terminals that can't
-//! produce it; the input-box component (later sub-phase) keys off
-//! `mods.shift` on an `.enter` press. On terminals that can't express it,
-//! Enter submits and a separate binding (e.g. Alt+Enter / a config key) must
-//! insert a newline — that policy is the component's, not ours.
+//! Shift+Enter: in the bare legacy protocol Enter and Shift+Enter send the same
+//! byte (`\r`), so they're indistinguishable. To recover the distinction we
+//! negotiate at startup (see `negotiate_query`): we push the Kitty keyboard
+//! protocol (flags 1|2|4) and query the terminal; if it confirms Kitty we use
+//! that, otherwise we fall back to xterm modifyOtherKeys mode 2. Either way:
+//!   - Kitty-protocol terminals (ghostty/kitty/foot/…) send `\x1b[13;2u`.
+//!   - xterm and tmux (which does NOT honor the Kitty push but DOES forward
+//!     modifyOtherKeys with `extended-keys on`) send `\x1b[27;2;13~`.
+//! This decoder handles BOTH forms, setting `mods.shift` on an `.enter`. The
+//! input-box component keys off `mods.shift` to insert a newline vs submit. On
+//! terminals that support neither (e.g. macOS Terminal.app), the two remain
+//! indistinguishable and Enter submits; a separate binding must insert newline.
 //!
 //! The splitter (`decodeOne`) takes a byte buffer and returns the next decoded
 //! key plus how many bytes it consumed, so a batched read of several escape
@@ -36,6 +38,29 @@ const std = @import("std");
 const key = @import("tui_key.zig");
 const Key = key.Key;
 const KeyCode = key.KeyCode;
+
+// ---- Global Kitty-protocol state -----------------------------------------
+//
+// Whether the Kitty keyboard protocol is confirmed active for this session.
+// Set by the app after the startup negotiation handshake resolves (see
+// `negotiate_query` / `Decoded.negotiation`). A few decode decisions depend on
+// it — most notably, when Kitty is active some terminals (Ghostty) deliver
+// Shift+Enter as a bare `\n`, whereas in the legacy path `\n` is plain Enter.
+//
+// This mirrors pi's `setKittyProtocolActive`. It is process-global and
+// single-threaded (the TUI runs one terminal); tests set/reset it explicitly.
+var _kitty_active: bool = false;
+
+/// Mark the Kitty keyboard protocol active/inactive. The app calls this once
+/// the negotiation handshake resolves, and again (false) on teardown.
+pub fn setKittyActive(active: bool) void {
+    _kitty_active = active;
+}
+
+/// Whether the Kitty keyboard protocol is currently believed active.
+pub fn kittyActive() bool {
+    return _kitty_active;
+}
 const Mods = key.Mods;
 
 // ---- Terminal control sequences this layer owns --------------------------
@@ -49,23 +74,73 @@ pub const disable_bracketed_paste = "\x1b[?2004l";
 pub const paste_begin = "\x1b[200~";
 pub const paste_end = "\x1b[201~";
 
-/// Kitty keyboard protocol: push a flags entry that asks for disambiguated
-/// escape codes (flag 1). We push (not set) so teardown can pop cleanly. This
-/// is best-effort; terminals that don't support it ignore it.
-pub const enable_kitty_keyboard = "\x1b[>1u";
+/// Kitty keyboard protocol: push a flags entry that asks for:
+///   flag 1 (0b001) disambiguate escape codes,
+///   flag 2 (0b010) report event types (press/repeat/release),
+///   flag 4 (0b100) report alternate keys (shifted/base-layout key).
+/// 1|2|4 = 7. This matches what pi requests. Flag 1 makes the terminal report
+/// modified special keys — including Shift+Enter — as distinct CSI-u sequences
+/// (`\x1b[13;2u`), which is exactly what we need; we deliberately do NOT set
+/// flag 8 (report-all-keys), since that suppresses normal text input and forces
+/// every printable key through the escape-sequence path. We push (not set) so
+/// teardown can pop cleanly. Best-effort; unsupported terminals ignore it and
+/// we fall back to modifyOtherKeys / legacy decoding.
+pub const enable_kitty_keyboard = "\x1b[>7u";
+/// Query the terminal's currently-active Kitty flags: it replies `\x1b[?<n>u`.
+pub const query_kitty_flags = "\x1b[?u";
+/// Primary Device Attributes query. Every terminal answers this (`\x1b[?...c`),
+/// so it acts as a sentinel: if the DA reply arrives WITHOUT a preceding Kitty
+/// flags reply, the terminal does not support the Kitty protocol and we fall
+/// back to modifyOtherKeys — no startup timeout needed.
+pub const query_primary_device_attributes = "\x1b[c";
 /// Pop the Kitty keyboard flags entry we pushed.
 pub const disable_kitty_keyboard = "\x1b[<u";
 
-/// Modest negotiation setup to write at startup. Bracketed paste is the only
-/// thing P1 strictly needs; the Kitty enable is opportunistic. Order: paste
-/// first, then Kitty.
-pub const negotiate_setup = enable_bracketed_paste ++ enable_kitty_keyboard;
+/// xterm `modifyOtherKeys` mode 2. The OLDER, pre-Kitty mechanism that xterm,
+/// mintty, and — critically — tmux understand. tmux does NOT honor the Kitty
+/// `>u` push, but with `extended-keys` on it forwards modifyOtherKeys sequences
+/// (`\x1b[27;2;13~` for Shift+Enter). We enable this ONLY as a fallback when the
+/// startup handshake shows the Kitty protocol is unavailable — enabling both at
+/// once can perturb Kitty's state (Kitty treats `>4;Nm` as an alias for its own
+/// flags) and trigger parse-complaints on some terminals.
+pub const enable_modify_other_keys = "\x1b[>4;2m";
+/// Reset modifyOtherKeys back to the default (mode 0).
+pub const disable_modify_other_keys = "\x1b[>4;0m";
 
-/// Teardown to write on exit; the reverse of `negotiate_setup`.
+/// Startup negotiation to write once raw mode is engaged. We:
+///   1. enable bracketed paste,
+///   2. push the Kitty flags we want (1|2|4),
+///   3. query the now-active Kitty flags, then
+///   4. send a Primary Device Attributes query as a sentinel.
+/// The app then reads the responses (`Decoded.negotiation`): a non-zero Kitty
+/// flags reply confirms the protocol (→ `setKittyActive(true)`); a DA reply with
+/// no preceding Kitty reply means fall back to `enable_modify_other_keys`.
+pub const negotiate_query =
+    enable_bracketed_paste ++
+    enable_kitty_keyboard ++
+    query_kitty_flags ++
+    query_primary_device_attributes;
+
+/// Base teardown common to every exit path: pop the Kitty push and disable
+/// bracketed paste. The app additionally writes `disable_modify_other_keys`
+/// when it enabled the fallback. (Popping the Kitty stack is harmless even if
+/// the push was ignored by a non-Kitty terminal.)
 pub const negotiate_teardown = disable_kitty_keyboard ++ disable_bracketed_paste;
 
 // ---- Decode results -------------------------------------------------------
 
+/// A keyboard-protocol negotiation response from the terminal, surfaced so the
+/// app's startup handshake can resolve which protocol to use. These are NOT
+/// keypresses and must not reach the input box.
+pub const Negotiation = union(enum) {
+    /// Terminal's reply to `\x1b[?u`: its currently-active Kitty flags. A
+    /// non-zero value confirms the Kitty keyboard protocol is active.
+    kitty_flags: u32,
+    /// Terminal's reply to `\x1b[c` (Primary Device Attributes). Used purely as
+    /// the handshake sentinel — its payload is irrelevant to us.
+    device_attributes,
+};
+
 /// What a single decode step produced.
 pub const Decoded = union(enum) {
     /// A decoded key.
@@ -73,6 +148,9 @@ pub const Decoded = union(enum) {
     /// A run of pasted text (between bracketed-paste markers), surfaced
     /// literally. `text` borrows from the input buffer.
     paste: []const u8,
+    /// A keyboard-protocol negotiation response (consumed by the app, never
+    /// routed to components).
+    negotiation: Negotiation,
 };
 
 /// Result of one decode step: what was produced, and how many input bytes it
@@ -107,7 +185,15 @@ pub fn decodeOne(buf: []const u8) ?Step {
 
     // Control bytes (C0).
     switch (b) {
-        '\r', '\n' => return keyStep(.{ .code = .enter }, 1),
+        '\r' => return keyStep(.{ .code = .enter }, 1),
+        // `\n` (0x0a): plain Enter in the legacy path. But when the Kitty
+        // protocol is active, a bare `\n` is some terminals' Shift+Enter
+        // mapping (notably Ghostty's `keybind = shift+enter=text:\n`), since
+        // real Enter then arrives as `\x1b[13u`. Mirror pi's handling.
+        '\n' => return keyStep(.{
+            .code = .enter,
+            .mods = .{ .shift = _kitty_active },
+        }, 1),
         '\t' => return keyStep(.{ .code = .tab }, 1),
         0x7f, 0x08 => return keyStep(.{ .code = .backspace }, 1),
         0x03 => return keyStep(.{ .code = .{ .char = 'c' }, .mods = .{ .ctrl = true } }, 1), // Ctrl+C
@@ -164,8 +250,9 @@ fn decodeEscape(buf: []const u8) ?Step {
                 k.mods.alt = true;
                 return .{ .decoded = .{ .key = k }, .consumed = inner.consumed + 1 };
             },
-            // A paste right after ESC is nonsensical; fall through to Escape.
-            .paste => {},
+            // A paste / negotiation reply right after a lone ESC is
+            // nonsensical; fall through to reporting Escape.
+            .paste, .negotiation => {},
         }
     }
 
@@ -222,6 +309,24 @@ fn decodeCSI(buf: []const u8) ?Step {
     const params = buf[2..i];
     const total = i + 1;
 
+    // Keyboard-protocol negotiation responses (private `?` marker):
+    //   - Kitty active flags reply:  CSI ? <n> u
+    //   - Primary Device Attributes: CSI ? <...> c   (handshake sentinel)
+    // Surface these as `.negotiation` so the app handshake consumes them; they
+    // are never keypresses.
+    if (params.len >= 1 and params[0] == '?') {
+        const body = params[1..];
+        if (final == 'u') {
+            const flags = std.fmt.parseInt(u32, body, 10) catch 0;
+            return .{ .decoded = .{ .negotiation = .{ .kitty_flags = flags } }, .consumed = total };
+        }
+        if (final == 'c') {
+            return .{ .decoded = .{ .negotiation = .device_attributes }, .consumed = total };
+        }
+        // Some other `?`-prefixed report (e.g. mode status): consume + drop.
+        return keyStep(.{ .code = .escape }, total);
+    }
+
     // Bracketed paste begin: CSI 200 ~  -> scan to the end marker.
     if (final == '~' and std.mem.eql(u8, params, "200")) {
         return decodePaste(buf, total);
@@ -230,6 +335,16 @@ fn decodeCSI(buf: []const u8) ?Step {
     // CSI-u (Kitty): final 'u', params = "<codepoint>[;<mods>[:<event>]]".
     if (final == 'u') return decodeKittyU(params, total);
 
+    // xterm modifyOtherKeys (formatOtherKeys=0): CSI 27 ; <mods> ; <cp> ~
+    // This is the LEGACY fallback that tmux + xterm emit (with modifyOtherKeys
+    // mode 2) when the Kitty protocol is not active. Notably tmux sends Shift+
+    // Enter as "\x1b[27;2;13~" — NOT the Kitty "\x1b[13;2u" form — so we must
+    // decode it here or Shift+Enter is lost inside tmux. (Must be checked before
+    // the generic tilde-numeric handling, which assumes a different layout.)
+    if (final == '~') {
+        if (decodeModifyOtherKeys(params, total)) |step| return step;
+    }
+
     // Modifier suffix: many sequences are "1;<mods><final>" or
     // "<num>;<mods>~". Split params on ';'.
     var first: []const u8 = params;
@@ -317,10 +432,23 @@ fn decodeKittyU(params: []const u8, total: usize) ?Step {
 
     const cp = std.fmt.parseInt(u21, cp_field, 10) catch return keyStep(.{ .code = .escape }, total);
 
-    // Modifier + event live in `rest` as "<mods>[:<event>]".
-    var mod_field: ?[]const u8 = rest;
-    var event = key.KeyEvent.press;
+    // After the codepoint, params are "<mods>[:<event>][;<text>]". With flag 16
+    // (report associated text) active, the text section carries the literal
+    // codepoint(s) the key produces, colon-separated decimal. Split off that
+    // text section (after the SECOND ';') before parsing mods/event.
+    var mod_event_field: ?[]const u8 = rest;
+    var text_field: ?[]const u8 = null;
     if (rest) |r| {
+        if (std.mem.indexOfScalar(u8, r, ';')) |semi2| {
+            mod_event_field = r[0..semi2];
+            text_field = r[semi2 + 1 ..];
+        }
+    }
+
+    // Modifier + event live in `mod_event_field` as "<mods>[:<event>]".
+    var mod_field: ?[]const u8 = mod_event_field;
+    var event = key.KeyEvent.press;
+    if (mod_event_field) |r| {
         if (std.mem.indexOfScalar(u8, r, ':')) |colon| {
             mod_field = r[0..colon];
             event = parseKittyEvent(r[colon + 1 ..]);
@@ -328,14 +456,85 @@ fn decodeKittyU(params: []const u8, total: usize) ?Step {
     }
     const mods = parseMods(mod_field);
 
+    // Kitty encodes special keys with either legacy codepoints (13/9/127/27)
+    // or functional-key numbers in the Unicode PUA (57344+). With flag 8 active
+    // we also receive standalone modifier-key events (left_shift, etc.) in that
+    // PUA range — those are not text and must be dropped, not inserted.
+    const code: KeyCode = switch (cp) {
+        13, 57345 => .enter,
+        9, 57346 => .tab,
+        127, 57347 => .backspace,
+        27, 57344 => .escape,
+        57349 => .delete,
+        57350 => .left,
+        57351 => .right,
+        57352 => .up,
+        57353 => .down,
+        57354 => .page_up,
+        57355 => .page_down,
+        57356 => .home,
+        57357 => .end,
+        // Modifier keys and other functional keys we don't model: drop the
+        // event by consuming the bytes and reporting nothing insertable.
+        57358...57363, 57441...57452 => return keyStep(.{ .code = .escape, .event = event }, total),
+        else => .{ .char = cp },
+    };
+
+    // Resolve associated text into the per-decode static buffer so the input
+    // box inserts exactly what the terminal reported (rather than re-encoding
+    // the keycap codepoint, which can differ for shifted/alternate layouts).
+    const text: ?[]const u8 = if (text_field) |tf| decodeKittyText(tf) else null;
+
+    return .{ .decoded = .{ .key = .{ .code = code, .mods = mods, .event = event, .text = text } }, .consumed = total };
+}
+
+/// Decode a Kitty associated-text field (colon-separated decimal Unicode
+/// codepoints) into UTF-8. Returns a slice into a per-call static buffer, valid
+/// until the next `decodeKittyText` call. Single-threaded use only — the caller
+/// consumes `Key.text` before the next decode. Returns null if empty or invalid.
+fn decodeKittyText(field: []const u8) ?[]const u8 {
+    const S = struct {
+        var buf: [64]u8 = undefined;
+    };
+    if (field.len == 0) return null;
+    var len: usize = 0;
+    var it = std.mem.splitScalar(u8, field, ':');
+    while (it.next()) |part| {
+        if (part.len == 0) continue;
+        const cp = std.fmt.parseInt(u21, part, 10) catch return null;
+        var tmp: [4]u8 = undefined;
+        const n = std.unicode.utf8Encode(cp, &tmp) catch return null;
+        if (len + n > S.buf.len) break; // overflow: truncate defensively
+        @memcpy(S.buf[len .. len + n], tmp[0..n]);
+        len += n;
+    }
+    if (len == 0) return null;
+    return S.buf[0..len];
+}
+
+/// Decode an xterm modifyOtherKeys sequence: params are "27;<mods>;<cp>" with a
+/// trailing '~' (already consumed by the caller). The middle field is the CSI
+/// modifier code (1 + bitmask); the last is the key codepoint. Returns null if
+/// the params don't match this exact 3-field shape (so the caller falls through
+/// to other tilde forms).
+fn decodeModifyOtherKeys(params: []const u8, total: usize) ?Step {
+    var it = std.mem.splitScalar(u8, params, ';');
+    const lead = it.next() orelse return null;
+    if (!std.mem.eql(u8, lead, "27")) return null;
+    const mod_str = it.next() orelse return null;
+    const cp_str = it.next() orelse return null;
+    if (it.next() != null) return null; // more than 3 fields: not this form
+
+    const cp = std.fmt.parseInt(u21, cp_str, 10) catch return null;
+    const mods = parseMods(mod_str);
     const code: KeyCode = switch (cp) {
         13 => .enter,
         9 => .tab,
-        127 => .backspace,
+        127, 8 => .backspace,
         27 => .escape,
         else => .{ .char = cp },
     };
-    return .{ .decoded = .{ .key = .{ .code = code, .mods = mods, .event = event } }, .consumed = total };
+    return keyStep(.{ .code = code, .mods = mods }, total);
 }
 
 fn parseKittyEvent(field: []const u8) key.KeyEvent {
@@ -458,6 +657,120 @@ test "kitty release event" {
     try std.testing.expectEqual(key.KeyEvent.release, s.event);
 }
 
+test "negotiation: kitty flags reply" {
+    const s = decodeOne("\x1b[?7u").?;
+    try std.testing.expectEqual(@as(usize, 5), s.consumed);
+    switch (s.decoded) {
+        .negotiation => |n| switch (n) {
+            .kitty_flags => |f| try std.testing.expectEqual(@as(u32, 7), f),
+            else => return error.WrongNegotiation,
+        },
+        else => return error.NotNegotiation,
+    }
+    // Zero flags = Kitty present but no enhancements active.
+    const z = decodeOne("\x1b[?0u").?.decoded;
+    try std.testing.expectEqual(@as(u32, 0), z.negotiation.kitty_flags);
+}
+
+test "negotiation: device attributes sentinel" {
+    // A typical DA reply: CSI ? 62 ; 22 c
+    const s = decodeOne("\x1b[?62;22c").?;
+    switch (s.decoded) {
+        .negotiation => |n| try std.testing.expect(n == .device_attributes),
+        else => return error.NotNegotiation,
+    }
+}
+
+test "negotiation: incomplete reply needs more bytes" {
+    try std.testing.expect(decodeOne("\x1b[?7") == null);
+    try std.testing.expect(decodeOne("\x1b[?") == null);
+}
+
+test "newline is shift+enter only when kitty active" {
+    setKittyActive(false);
+    try std.testing.expect(!decodeOne("\n").?.decoded.key.mods.shift);
+    setKittyActive(true);
+    {
+        const k = decodeOne("\n").?.decoded.key;
+        try std.testing.expectEqual(KeyCode.enter, k.code);
+        try std.testing.expect(k.mods.shift);
+    }
+    // \r is always plain Enter regardless of protocol state.
+    try std.testing.expect(!decodeOne("\r").?.decoded.key.mods.shift);
+    setKittyActive(false); // reset global for other tests
+}
+
+test "modifyOtherKeys tilde form: shift+enter (tmux)" {
+    // tmux with modifyOtherKeys mode 2 sends Shift+Enter as CSI 27;2;13~
+    const s = decodeOne("\x1b[27;2;13~").?.decoded.key;
+    try std.testing.expectEqual(KeyCode.enter, s.code);
+    try std.testing.expect(s.mods.shift);
+    // Plain Enter (no mods) is unaffected.
+    try std.testing.expect(!decodeOne("\r").?.decoded.key.mods.shift);
+}
+
+test "modifyOtherKeys tilde form: ctrl+enter and a printable" {
+    const ce = decodeOne("\x1b[27;5;13~").?.decoded.key;
+    try std.testing.expectEqual(KeyCode.enter, ce.code);
+    try std.testing.expect(ce.mods.ctrl);
+    // 'A' with ctrl: CSI 27;5;97~
+    const ca = decodeOne("\x1b[27;5;97~").?.decoded.key;
+    try std.testing.expectEqual(@as(u21, 97), ca.code.char);
+    try std.testing.expect(ca.mods.ctrl);
+}
+
+test "tilde numeric forms still work alongside modifyOtherKeys" {
+    // Ensure the new 27;... branch didn't shadow normal ~ forms.
+    try std.testing.expectEqual(KeyCode.delete, decodeOne("\x1b[3~").?.decoded.key.code);
+    try std.testing.expectEqual(KeyCode.page_up, decodeOne("\x1b[5~").?.decoded.key.code);
+    try std.testing.expectEqual(KeyCode.f5, decodeOne("\x1b[15~").?.decoded.key.code);
+}
+
+test "modifyOtherKeys ctrl+c via csi-u still recognized as ctrl" {
+    // Under modifyOtherKeys mode 2 (tmux), Ctrl+C arrives as CSI 99 ; 5 u, not
+    // raw 0x03. It must still decode as .char 'c' + ctrl so app quit works.
+    const s = decodeOne("\x1b[99;5u").?.decoded.key;
+    try std.testing.expectEqual(@as(u21, 'c'), s.code.char);
+    try std.testing.expect(s.mods.ctrl);
+    try std.testing.expect(s.isCtrl('c'));
+}
+
+test "kitty pua enter and shift+enter" {
+    // Under flag 8, Enter reports as functional-key 57345 (not legacy 13).
+    const e = decodeOne("\x1b[57345u").?.decoded.key;
+    try std.testing.expectEqual(KeyCode.enter, e.code);
+    try std.testing.expect(!e.mods.shift);
+    // Shift+Enter via PUA codepoint + shift modifier.
+    const se = decodeOne("\x1b[57345;2u").?.decoded.key;
+    try std.testing.expectEqual(KeyCode.enter, se.code);
+    try std.testing.expect(se.mods.shift);
+}
+
+test "kitty associated text populates key.text" {
+    // 'a' pressed with flag 16: CSI 97 ; 1 ; 97 u -> code 'a', text "a".
+    const s = decodeOne("\x1b[97;1;97u").?.decoded.key;
+    try std.testing.expectEqual(@as(u21, 'a'), s.code.char);
+    try std.testing.expect(s.text != null);
+    try std.testing.expectEqualStrings("a", s.text.?);
+}
+
+test "kitty associated text with multibyte codepoint" {
+    // U+00E9 (é) = 233 decimal -> 2-byte UTF-8.
+    const s = decodeOne("\x1b[233;1;233u").?.decoded.key;
+    try std.testing.expectEqualStrings("\xc3\xa9", s.text.?);
+}
+
+test "kitty modifier-only keypress is dropped, not inserted" {
+    // left_shift press (57441) under flag 8 must not produce an insertable char.
+    const s = decodeOne("\x1b[57441u").?.decoded.key;
+    try std.testing.expectEqual(KeyCode.escape, s.code); // sentinel "drop"
+    // The input box ignores .escape, so nothing is inserted.
+    switch (s.code) {
+        .char => return error.ModifierLeakedAsChar,
+        else => {},
+    }
+}
+
 test "bracketed paste surfaces literal text" {
     const input = paste_begin ++ "hello\nworld" ++ paste_end ++ "x";
     const s = decodeOne(input).?;