From 3b0764450af79dbb80ace3e90541886eccfce75a Mon Sep 17 00:00:00 2001 From: Richard Zink Date: Wed, 11 Nov 2026 07:32:18 +0100 Subject: [PATCH] Long chats stay fast; search in a chat; media overview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Performance (measured with 5,900 messages: open 9 s -> 0.7 s, typing 440 ms -> 25 ms) - GET /sessions/{id}?limit=N loads the end of a chat (from the last summary on, at most N); /sessions/{id}/messages?before|after|around loads more (scrolling up stops at a summary unless asked). Without parameters the full history as before (terminal client). - Chat view: memoised message list (typing, streaming and status changes no longer re-render the history), finished turns fold into "N steps · M files" with the answer visible, tool card bodies and summaries are built only when opened, scroll-to-bottom only when content changes. Search: /sessions/{id}/search over the whole chat; a hit loads its part (around) and marks the message, "back to the latest messages" returns. Media: /sessions/{id}/media lists images, videos and audio of a chat and its subagents (generated images, tool output, files written, media in answers, attachments); ⋯ -> Media shows them with filters, a large view and a jump to the message. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 19 ++ server/src/api.rs | 76 ++++- server/src/main.rs | 1 + server/src/media.rs | 200 +++++++++++++ server/src/store.rs | 193 +++++++++++-- web/src/components/ChatView.tsx | 434 +++++++++++++++++++++++----- web/src/components/Icon.tsx | 6 + web/src/components/MediaGallery.tsx | 125 ++++++++ web/src/components/ToolCard.tsx | 11 +- web/src/i18n/de.ts | 27 ++ web/src/i18n/en.ts | 27 ++ web/src/store.ts | 41 ++- web/src/styles.css | 213 ++++++++++++++ web/src/whatsnew.ts | 4 + 14 files changed, 1278 insertions(+), 99 deletions(-) create mode 100644 server/src/media.rs create mode 100644 web/src/components/MediaGallery.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 944e29d..4041a3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ All notable changes to CodeCraft. Versions follow [Semantic Versioning](https://semver.org/); until 1.0, minor versions may contain breaking changes. +## [Unreleased] + +### Added + +- **Search in a chat:** the magnifier in the chat's top bar searches all of it (messages, tool + output) on the server; a hit loads its part of the chat and marks the message. +- **Media of a chat:** *⋯ → Media* shows every image, video and audio file of the chat and its + subagents – generated images, screenshots, files the agent wrote, media in answers, your + attachments – filterable, large view, "show in chat" jumps to the message. + +### Changed + +- **Long chats stay fast:** a chat opens with its messages since the last summary (at most 200); + older ones load when scrolling up, the history before a summary on request. Finished turns + fold into one line ("42 steps · 3 files", the answer stays visible). Typing, streaming and + status changes no longer re-render the whole history, and tool output is only built when a + card is opened. Measured with a chat of 5,900 messages: opening 9 s → 0.7 s (phone-like + CPU: 22 s → 1 s), typing 440 ms → 25 ms per key. + ## [v202611-00018] – 2026-11-10 ### Added diff --git a/server/src/api.rs b/server/src/api.rs index eb0d17b..8b21ea0 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -81,6 +81,9 @@ pub fn router(app: AppState) -> Router { .route("/sessions/{id}/tabs", get(session_tabs).put(put_file_tabs)) .route("/sessions/{id}/export", get(export_session)) .route("/blobs/{hash}", get(get_blob)) + .route("/sessions/{id}/messages", get(session_messages)) + .route("/sessions/{id}/search", get(search_in_session)) + .route("/sessions/{id}/media", get(session_media)) .route("/sessions/{id}", get(get_session).patch(update_session).delete(delete_session)) .route("/sessions/{id}/prompt", post(prompt)) .route("/sessions/{id}/abort", post(abort)) @@ -813,14 +816,81 @@ async fn export_session(Extension(user): Extension, State(app): State, State(app): State, Path(id): Path) -> ApiResult { +#[derive(Deserialize)] +struct WindowQ { + /// Only the end of the chat (from the last summary on, at most this many messages); + /// without it, all messages (terminal client). + limit: Option, + before: Option, + after: Option, + around: Option, + /// Loading older messages stops at a summary (the history before it on request). + #[serde(default)] + stop_at_summary: bool, +} + +impl WindowQ { + fn window(&self) -> Option { + use crate::store::MessageWindow as W; + let limit = self.limit.unwrap_or(200).clamp(1, 1000); + Some(match (&self.before, &self.after, &self.around) { + (Some(id), _, _) => W::Before { id: id.clone(), limit, stop_at_summary: self.stop_at_summary }, + (_, Some(id), _) => W::After { id: id.clone(), limit }, + (_, _, Some(id)) => W::Around { id: id.clone(), limit }, + _ if self.limit.is_some() => W::Tail { limit }, + _ => return None, + }) + } +} + +async fn get_session(Extension(user): Extension, State(app): State, Path(id): Path, Query(q): Query) -> ApiResult { let s = readable(&app, &user, &id)?; - let messages = app.store.messages(&id)?; let busy = app.is_busy(&id); let can_write = app.access(&user, &id) == Access::Full; let queue = app.store.queue_list(&id)?; let step = app.with_runtime(&id, |r| r.step.clone()); - Ok(Json(json!({"session": s, "messages": messages, "busy": busy, "can_write": can_write, "queue": queue, "step": step}))) + let mut out = json!({"session": s, "busy": busy, "can_write": can_write, "queue": queue, "step": step}); + match q.window() { + Some(w) => { + let (messages, older, newer) = app.store.messages_window(&id, &w)?; + out["messages"] = json!(messages); + out["has_older"] = json!(older); + out["has_newer"] = json!(newer); + } + None => out["messages"] = json!(app.store.messages(&id)?), + } + Ok(Json(out)) +} + +/// More messages of a chat: before / after / around a message. +async fn session_messages(Extension(user): Extension, State(app): State, Path(id): Path, Query(q): Query) -> ApiResult { + readable(&app, &user, &id)?; + let w = q.window().ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "before, after, around or limit is required".into()))?; + let (messages, older, newer) = app.store.messages_window(&id, &w).map_err(|e| ApiError(StatusCode::NOT_FOUND, format!("{e:#}")))?; + Ok(Json(json!({"messages": messages, "has_older": older, "has_newer": newer}))) +} + +/// Images, videos and audio files shown in a chat (all of it, with subagents). +async fn session_media(Extension(user): Extension, State(app): State, Path(id): Path) -> ApiResult { + readable(&app, &user, &id)?; + let a = app.clone(); + let items = tokio::task::spawn_blocking(move || crate::media::collect(&a, &id)).await.map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))??; + Ok(Json(json!(items))) +} + +/// Search within one chat (all of it, not only the loaded part). +async fn search_in_session(Extension(user): Extension, State(app): State, Path(id): Path, Query(q): Query) -> ApiResult { + readable(&app, &user, &id)?; + if q.q.trim().chars().count() < 2 { + return Ok(Json(json!([]))); + } + let hits: Vec = app + .store + .search_messages(&id, q.q.trim(), 100)? + .into_iter() + .map(|(mid, role, at, snippet)| json!({"id": mid, "role": role, "created_at": at, "snippet": snippet})) + .collect(); + Ok(Json(json!(hits))) } #[derive(Deserialize)] diff --git a/server/src/main.rs b/server/src/main.rs index 375c862..825fa76 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -22,6 +22,7 @@ mod lsp; mod mentions; mod metrics; mod mcp; +mod media; mod permission; mod prompts; mod push; diff --git a/server/src/media.rs b/server/src/media.rs new file mode 100644 index 0000000..fb6c7a5 --- /dev/null +++ b/server/src/media.rs @@ -0,0 +1,200 @@ +//! Media overview of a chat: every image, video and audio file that was shown in it – +//! generated images, images in tool results (screenshots), media files the agent wrote, +//! media referenced in answers, and the user's own attachments. Includes subagents. + +use crate::app::App; +use crate::types::{Message, Part, Role}; +use serde::Serialize; +use serde_json::Value; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Serialize, PartialEq)] +pub struct MediaItem { + /// "image", "video" or "audio". + pub kind: &'static str, + pub url: String, + pub name: String, + /// "generated", "tool", "written", "answer" or "upload". + pub source: &'static str, + pub created_at: i64, + /// Message to jump to (only for messages of the chat itself, not of subagents). + #[serde(skip_serializing_if = "Option::is_none")] + pub message_id: Option, +} + +const IMAGE: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "avif"]; +const VIDEO: &[&str] = &["mp4", "webm", "mov", "m4v", "ogv"]; +const AUDIO: &[&str] = &["mp3", "wav", "ogg", "oga", "m4a", "flac", "opus", "aac"]; + +/// Kind of a media file by its extension. +pub fn kind_of(path: &str) -> Option<&'static str> { + let ext = Path::new(path.split(['?', '#']).next().unwrap_or(path)).extension()?.to_str()?.to_ascii_lowercase(); + if IMAGE.contains(&ext.as_str()) { + Some("image") + } else if VIDEO.contains(&ext.as_str()) { + Some("video") + } else if AUDIO.contains(&ext.as_str()) { + Some("audio") + } else { + None + } +} + +fn kind_of_mime(mime: &str) -> &'static str { + if mime.starts_with("video/") { + "video" + } else if mime.starts_with("audio/") { + "audio" + } else { + "image" + } +} + +/// Like JavaScript's encodeURIComponent. +fn encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + if b.is_ascii_alphanumeric() || b"-_.!~*'()".contains(&b) { + out.push(b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +/// Absolute, normalised path of `p` (relative to `cwd`). +fn resolve(p: &str, cwd: &Path) -> PathBuf { + let joined = if p.starts_with('/') { PathBuf::from(p) } else { cwd.join(p) }; + crate::tools::normalize(&joined) +} + +/// Media file paths in a text: Markdown images/links and plain paths with a media extension. +fn paths_in(text: &str) -> Vec { + let mut out = Vec::new(); + for token in text.split(|c: char| c.is_whitespace() || "()[]<>\"'`,;".contains(c)) { + let t = token.trim_end_matches(['.', ':', '!', '?']); + if t.is_empty() || t.contains("://") || t.starts_with("data:") { + continue; + } + if kind_of(t).is_some() && t.chars().any(|c| c == '/' || c == '.') { + out.push(t.to_string()); + } + } + out +} + +/// All media of a chat and its subagents, newest first. +pub fn collect(app: &App, session_id: &str) -> anyhow::Result> { + let mut sessions = vec![session_id.to_string()]; + // Subagents (and theirs). + let mut i = 0; + while i < sessions.len() && sessions.len() < 500 { + let children = app.store.child_sessions(&sessions[i])?; + sessions.extend(children); + i += 1; + } + let mut out: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + let mut push = |item: MediaItem, out: &mut Vec| { + if seen.insert(item.url.clone()) { + out.push(item); + } + }; + for sid in &sessions { + let Some(session) = app.store.get_session(sid)? else { continue }; + let cwd = PathBuf::from(&session.cwd); + let own = sid == session_id; + // Tool results are shown in the card of the call: jump to that message. + let mut call_msg: Option = None; + for m in app.store.messages(sid)? { + collect_message(&m, &cwd, own, call_msg.as_deref(), &mut |item| push(item, &mut out)); + if m.role == Role::Assistant { + call_msg = Some(m.id.clone()); + } + } + } + // Generated images whose tool result was not found (e.g. removed by a rewind). + for img in app.store.images_of_sessions(&sessions)? { + push( + MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None }, + &mut out, + ); + } + // Names of generated images (their files). + let names: std::collections::HashMap = app.store.images_of_sessions(&sessions)?.into_iter().map(|i| (format!("/api/images/{}/file", i.id), file_name(&i.path))).collect(); + for item in &mut out { + if let Some(n) = names.get(&item.url) { + item.name = n.clone(); + } + } + out.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + Ok(out) +} + +fn file_name(p: &str) -> String { + Path::new(p).file_name().and_then(|n| n.to_str()).unwrap_or(p).to_string() +} + +fn collect_message(m: &Message, cwd: &Path, own: bool, call_msg: Option<&str>, add: &mut dyn FnMut(MediaItem)) { + let mid = own.then(|| m.id.clone()); + let result_mid = own.then(|| call_msg.unwrap_or(&m.id).to_string()); + let at = m.created_at; + let file = |p: &str, source: &'static str| -> Option { + let kind = kind_of(p)?; + let abs = resolve(p, cwd); + abs.is_file().then(|| MediaItem { kind, url: format!("/api/files/raw?path={}", encode(&abs.display().to_string())), name: file_name(p), source, created_at: at, message_id: mid.clone() }) + }; + for p in &m.parts { + match p { + Part::Image { image } if m.role == Role::User => { + if let Some(h) = &image.blob { + add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone() }); + } + } + Part::ToolResult { images, meta, .. } => { + let ids: Vec<&str> = meta.as_ref().and_then(|v| v["image_ids"].as_array()).map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default(); + if !ids.is_empty() { + for id in ids { + add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone() }); + } + } else { + for img in images { + if let Some(h) = &img.blob { + add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone() }); + } + } + } + } + Part::ToolCall { name, input, .. } if matches!(name.as_str(), "write" | "edit") => { + if let Some(item) = input["path"].as_str().and_then(|p| file(p, "written")) { + add(item); + } + } + Part::Text { text } if m.role == Role::Assistant => { + for p in paths_in(text) { + if let Some(item) = file(&p, "answer") { + add(item); + } + } + } + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kinds_paths_and_encoding() { + assert_eq!(kind_of("a/b.PNG"), Some("image")); + assert_eq!(kind_of("clip.mp4?v=1"), Some("video")); + assert_eq!(kind_of("song.flac"), Some("audio")); + assert_eq!(kind_of("main.rs"), None); + assert_eq!(paths_in("Saved to `art/final/hero.png` and ![x](shots/a.webp). See https://x.org/y.png"), vec!["art/final/hero.png", "shots/a.webp"]); + assert_eq!(encode("/w/ä b.png"), "%2Fw%2F%C3%A4%20b.png"); + } +} diff --git a/server/src/store.rs b/server/src/store.rs index d292968..ae30a12 100644 --- a/server/src/store.rs +++ b/server/src/store.rs @@ -1,6 +1,6 @@ //! SQLite persistence for sessions and messages. -use crate::types::{ImageData, Message, Part, Session, now_ms}; +use crate::types::{ImageData, Message, Part, Role, Session, now_ms}; use serde::Serialize; use serde_json::{Value, json}; use anyhow::Result; @@ -244,28 +244,7 @@ impl Store { let (sdata, mdata) = row?; let session: Session = serde_json::from_str(&sdata)?; // Snippet from the visible text of the matching message (not JSON keys). - let mut snippet = String::new(); - if let Some(m) = mdata.and_then(|d| serde_json::from_str::(&d).ok()) { - for p in &m.parts { - let text = match p { - Part::Text { text } | Part::Summary { text, .. } | Part::Reasoning { text, .. } => text.as_str(), - Part::ToolResult { content, .. } => content.as_str(), - _ => continue, - }; - if let Some(i) = text.to_lowercase().find(&needle) { - let mut a = i.saturating_sub(60); - while !text.is_char_boundary(a) { - a -= 1; - } - let mut b = (i + needle.len() + 100).min(text.len()); - while !text.is_char_boundary(b) { - b += 1; - } - snippet = format!("{}{}{}", if a > 0 { "…" } else { "" }, text[a..b].replace('\n', " "), if b < text.len() { "…" } else { "" }); - break; - } - } - } + let snippet = mdata.and_then(|d| serde_json::from_str::(&d).ok()).and_then(|m| snippet_in(&m, &needle)).unwrap_or_default(); if snippet.is_empty() && !session.title.to_lowercase().contains(&needle) { continue; // matched only JSON structure (keys, ids) } @@ -274,6 +253,121 @@ impl Store { Ok(out) } + /// A part of a session's messages for the chat view (long chats are not loaded at + /// once). Returns the messages (oldest first) and whether older / newer ones exist. + pub fn messages_window(&self, session_id: &str, w: &MessageWindow) -> Result<(Vec, bool, bool)> { + let conn = self.conn.lock().unwrap(); + let seq_of = |id: &str| -> Result { + conn.query_row("SELECT seq FROM messages WHERE session_id = ?1 AND id = ?2", params![session_id, id], |r| r.get(0)) + .optional()? + .ok_or_else(|| anyhow::anyhow!("message {id} not found")) + }; + let (sql, a, limit): (&str, i64, i64) = match w { + // From the last summary (compaction) on, at most `limit`. + MessageWindow::Tail { limit } => { + let from: i64 = conn + .query_row( + "SELECT seq FROM messages WHERE session_id = ?1 AND data LIKE '%\"type\":\"summary\"%' ORDER BY seq DESC LIMIT 1", + [session_id], + |r| r.get(0), + ) + .optional()? + .unwrap_or(i64::MIN); + ("SELECT data FROM messages WHERE session_id = ?1 AND seq >= ?2 ORDER BY seq DESC LIMIT ?3", from, *limit) + } + // Scrolling up stops at a summary (compaction); the history before it on request. + MessageWindow::Before { id, limit, stop_at_summary } => { + let before = seq_of(id)?; + let summary: Option = if *stop_at_summary { + conn.query_row( + "SELECT seq FROM messages WHERE session_id = ?1 AND seq < ?2 AND data LIKE '%\"type\":\"summary\"%' ORDER BY seq DESC LIMIT 1", + params![session_id, before], + |r| r.get(0), + ) + .optional()? + } else { + None + }; + match summary { + Some(from) => { + let mut stmt = conn.prepare("SELECT seq, data FROM messages WHERE session_id = ?1 AND seq >= ?2 AND seq < ?3 ORDER BY seq DESC LIMIT ?4")?; + let mut rows: Vec<(i64, Message)> = Vec::new(); + for r in stmt.query_map(params![session_id, from, before, limit], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))? { + let (seq, data) = r?; + rows.push((seq, serde_json::from_str(&data)?)); + } + drop(stmt); + return Self::finish_window(&conn, session_id, rows, false); + } + None => ("SELECT data FROM messages WHERE session_id = ?1 AND seq < ?2 ORDER BY seq DESC LIMIT ?3", before, *limit), + } + } + MessageWindow::After { id, limit } => ("SELECT data FROM messages WHERE session_id = ?1 AND seq > ?2 ORDER BY seq ASC LIMIT ?3", seq_of(id)?, *limit), + // The message and about half the limit before it. + MessageWindow::Around { id, limit } => { + let seq = seq_of(id)?; + let before: i64 = conn.query_row( + "SELECT COALESCE(MIN(seq), ?2) FROM (SELECT seq FROM messages WHERE session_id = ?1 AND seq < ?2 ORDER BY seq DESC LIMIT ?3)", + params![session_id, seq, limit / 2], + |r| r.get(0), + )?; + ("SELECT data FROM messages WHERE session_id = ?1 AND seq >= ?2 ORDER BY seq ASC LIMIT ?3", before, *limit) + } + }; + let mut rows: Vec<(i64, Message)> = Vec::new(); + let mut stmt = conn.prepare(&sql.replace("SELECT data", "SELECT seq, data"))?; + for r in stmt.query_map(params![session_id, a, limit], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))? { + let (seq, data) = r?; + rows.push((seq, serde_json::from_str(&data)?)); + } + drop(stmt); + Self::finish_window(&conn, session_id, rows, matches!(w, MessageWindow::Tail { .. })) + } + + /// Sort a window and tell whether older / newer messages exist around it. + fn finish_window(conn: &Connection, session_id: &str, mut rows: Vec<(i64, Message)>, tail: bool) -> Result<(Vec, bool, bool)> { + rows.sort_by_key(|(seq, _)| *seq); + let exists = |q: &str, seq: i64| -> Result { Ok(conn.prepare(q)?.exists(params![session_id, seq])?) }; + let older = match rows.first() { + Some((first, _)) => exists("SELECT 1 FROM messages WHERE session_id = ?1 AND seq < ?2", *first)?, + None => tail && exists("SELECT 1 FROM messages WHERE session_id = ?1 AND seq > ?2", i64::MIN)?, + }; + let newer = match rows.last() { + Some((last, _)) => exists("SELECT 1 FROM messages WHERE session_id = ?1 AND seq > ?2", *last)?, + None => false, + }; + Ok((rows.into_iter().map(|(_, m)| m).collect(), older, newer)) + } + + /// Messages of one session containing `q` (case-insensitive), newest first: + /// (message id, role, created_at, snippet). + pub fn search_messages(&self, session_id: &str, q: &str, limit: usize) -> Result> { + let needle = q.to_lowercase(); + let like = format!("%{}%", q.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_")); + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare("SELECT data FROM messages WHERE session_id = ?1 AND data LIKE ?2 ESCAPE '\\' ORDER BY seq DESC")?; + let mut out = Vec::new(); + for r in stmt.query_map(params![session_id, like], |r| r.get::<_, String>(0))? { + let Ok(m) = serde_json::from_str::(&r?) else { continue }; + // Only visible text counts (not JSON keys or ids). + if let Some(snippet) = snippet_in(&m, &needle) { + out.push((m.id.clone(), m.role, m.created_at, snippet)); + if out.len() >= limit { + break; + } + } + } + Ok(out) + } + + /// Ids of the subagent sessions started by a session. + pub fn child_sessions(&self, id: &str) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM sessions WHERE parent_id = ?1")?; + let rows = stmt.query_map([id], |r| r.get(0))?.collect::>>()?; + Ok(rows) + } + pub fn delete_session(&self, id: &str) -> Result<()> { let conn = self.conn.lock().unwrap(); // Child sessions (subagents) go with their parent. @@ -1045,6 +1139,17 @@ impl Store { } /// Latest images; `user = None` lists everybody's (admins). + /// Generated images of these sessions. + pub fn images_of_sessions(&self, sessions: &[String]) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare(&format!("SELECT {IMAGE_COLS} FROM images WHERE session_id = ?1"))?; + let mut out = Vec::new(); + for s in sessions { + out.extend(stmt.query_map([s], image_row)?.collect::>>()?); + } + Ok(out) + } + pub fn images(&self, user: Option<&str>, limit: i64) -> Result> { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare(&format!( @@ -1392,3 +1497,45 @@ pub struct PushSub { pub p256dh: String, pub auth: String, } + +/// Which messages of a session the chat view loads (see Store::messages_window). +pub enum MessageWindow { + Tail { limit: i64 }, + Before { id: String, limit: i64, stop_at_summary: bool }, + After { id: String, limit: i64 }, + Around { id: String, limit: i64 }, +} + +/// Text around the first occurrence of `needle` (lowercase) in the visible text of a +/// message: its text, reasoning, summaries, tool calls and tool output. +fn snippet_in(m: &Message, needle: &str) -> Option { + for p in &m.parts { + let owned; + let text = match p { + Part::Text { text } | Part::Summary { text, .. } | Part::Reasoning { text, .. } => text.as_str(), + Part::ToolResult { content, .. } => content.as_str(), + Part::ToolCall { input, .. } => { + owned = input.to_string(); + owned.as_str() + } + _ => continue, + }; + let lower = text.to_lowercase(); + // Lowercasing can change byte lengths (rare): then cut the lowercase text. + let (src, i) = match lower.find(needle) { + Some(i) if lower.len() == text.len() => (text, i), + Some(i) => (lower.as_str(), i), + None => continue, + }; + let mut a = i.saturating_sub(60); + while !src.is_char_boundary(a) { + a -= 1; + } + let mut b = (i + needle.len() + 100).min(src.len()); + while !src.is_char_boundary(b) { + b += 1; + } + return Some(format!("{}{}{}", if a > 0 { "…" } else { "" }, src[a..b].replace('\n', " "), if b < src.len() { "…" } else { "" })); + } + None +} diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 3ec11dd..8f45859 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -1,12 +1,12 @@ import { takePrefill } from "../share"; import { Icon } from "./Icon"; import { FileChip, FileFigure, fileType, imageSize, imageSrc } from "./FileFigure"; -import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from "react"; import { del, get, patch, post } from "../api"; import { useFetch, useModels } from "../hooks"; import { fmtCompact, fmtCost, fmtNumber, fmtPercent, useT, type Key } from "../i18n"; import { navigate } from "../router"; -import { dismissError, loadSession, loadSessions, useStore } from "../store"; +import { dismissError, getState, loadAround, loadOlder, loadSession, loadSessions, useStore } from "../store"; import type { Aggregate, ImageData, Message, Part, Session, Todo } from "../types"; import { ActivityLine, Dots, Spinner, ThinkIcon, WorkBar } from "./Activity"; import { useReasoningMode, useTodosCollapsed } from "../prefs"; @@ -26,8 +26,10 @@ import { ShellsChip } from "./ShellsPanel"; import { TodoList, ToolCard } from "./ToolCard"; import { confirmDialog, promptDialog, toast } from "../dialogs"; import { ChatTabBar, ChatTabPanes, useChatTabs } from "./ChatTabs"; +import { MediaGallery } from "./MediaGallery"; type ToolResult = Extract; +type ToolCall = Extract; const EMPTY: Message[] = []; @@ -166,10 +168,299 @@ function RefineFeedback({ text }: { text: string }) { ); } +/** Search the whole chat on the server; a hit loads its part of the chat. */ +function ChatSearch({ sessionId, onPick, onClose }: { sessionId: string; onPick: (messageId: string) => void; onClose: () => void }) { + const t = useT(); + const [q, setQ] = useState(""); + const [hits, setHits] = useState<{ id: string; role: string; created_at: number; snippet: string }[]>(); + useEffect(() => { + const query = q.trim(); + if (query.length < 2) return setHits(undefined); + let alive = true; + const h = setTimeout(() => { + get(`/sessions/${sessionId}/search?q=${encodeURIComponent(query)}`) + .then((r) => alive && setHits(r ?? [])) + .catch(() => alive && setHits([])); + }, 250); + return () => { + alive = false; + clearTimeout(h); + }; + }, [q, sessionId]); + return ( +
+
+ + setQ(e.target.value)} onKeyDown={(e) => e.key === "Escape" && onClose()} aria-label={t("chat.search")} /> + +
+ {hits && ( +
+
{hits.length ? t("chat.searchHits", { n: hits.length }) : t("chat.searchNone")}
+ {hits.map((h) => ( + + ))} +
+ )} +
+ ); +} + +/** A part of the chat: a user message, a summary, or the work of the agent after it. */ +type Block = { kind: "user" | "summary" | "work"; messages: Message[] }; + +const hasUserText = (m: Message) => m.role === "user" && m.parts.some((p) => (p.type === "text" && !isHarnessNote(p.text)) || p.type === "image"); + +/** Group the messages into user messages and the agent's work between them. */ +function blocksOf(messages: Message[]): Block[] { + const out: Block[] = []; + for (const m of messages) { + const kind = m.parts.some((p) => p.type === "summary") ? "summary" : hasUserText(m) ? "user" : "work"; + const last = out[out.length - 1]; + if (kind === "work" && last?.kind === "work") last.messages.push(m); + else out.push({ kind, messages: [m] }); + } + return out; +} + +/** + * The messages of a chat. Memoised: typing, streaming and status changes of the + * chat view do not touch it – only new messages, tool results or the busy state. + */ +const MessageList = memo(function MessageList({ + messages, + results, + busy, + canEdit, + canRegenerate, + sessionId, + model, + baseDir, + onRewind, + highlight, +}: { + messages: Message[]; + results: Map; + busy: boolean; + canEdit: boolean; + canRegenerate: boolean; + sessionId: string; + model: string; + baseDir: string; + onRewind: (m: Message) => void; + highlight?: string; +}) { + const blocks = useMemo(() => blocksOf(messages), [messages]); + const lastWork = blocks.map((b) => b.kind).lastIndexOf("work"); + const lastAssistant = [...messages].reverse().find((m) => m.role === "assistant")?.id; + return ( + <> + {blocks.map((b, i) => { + // Stable while older messages are loaded above (they may join the first block): + // the last message; the block still growing at the end keeps its first one. + const key = b.kind === "work" && i !== lastWork ? b.messages[b.messages.length - 1].id : b.messages[0].id; + if (b.kind !== "work") return ; + const items = b.messages.map((m) => ( + + )); + // Finished turns with tool steps fold into one line (the answer stays visible). + const steps = b.messages.flatMap((m) => m.parts.filter((p) => p.type === "tool_call")) as ToolCall[]; + if (i === lastWork || steps.length < 2) return
{items}
; + return m.id === highlight)} baseDir={baseDir}>{items}; + })} + + ); +}); + +/** A finished turn: "42 steps · 3 files" and the answer; the steps on click. */ +function WorkBlock({ messages, steps, open: forced, baseDir, children }: { messages: Message[]; steps: ToolCall[]; open: boolean; baseDir: string; children: ReactNode }) { + const t = useT(); + const [open, setOpen] = useState(false); + const files = new Set(steps.filter((c) => ["write", "edit"].includes(c.name) && typeof c.input?.path === "string").map((c) => c.input.path as string)); + // The answer: text of the last assistant message of the turn. + const answer = [...messages].reverse().find((m) => m.role === "assistant" && m.parts.some((p) => p.type === "text" && p.text.trim())); + const answerText = answer ? (answer.parts.filter((p) => p.type === "text") as { text: string }[]).map((p) => p.text).join("\n\n") : ""; + if (open || forced) { + return ( +
+ + {children} +
+ ); + } + return ( +
+ + {answerText && ( + <> + + + + )} +
+ ); +} + +/** An assistant message, or a user message that only carries tool results / harness notes. */ +const AssistantOrNote = memo(function AssistantOrNote({ + m, + results, + busy, + baseDir, + highlight, + regenerate, +}: { + m: Message; + results: Map; + busy: boolean; + baseDir: string; + highlight?: string; + regenerate?: { sessionId: string; model: string }; +}) { + const cls = `msg ${highlight === m.id ? "highlight" : ""}`; + if (m.role === "assistant") { + return ( +
+ + {regenerate && } +
+ ); + } + const notes = (m.parts.filter((p) => p.type === "text") as { text: string }[]).filter((p) => isHarnessNote(p.text)); + if (!notes.length) return null; + return ( +
+ {notes.map((p, i) => ( + + ))} +
+ ); +}); + +/** A user message or a summary of the history before it. */ +const MessageItem = memo(function MessageItem({ m, canEdit, onRewind, highlight }: { m: Message; canEdit: boolean; onRewind: (m: Message) => void; highlight?: string }) { + const t = useT(); + const [summaryOpen, setSummaryOpen] = useState(false); + const summary = m.parts.find((p) => p.type === "summary"); + if (summary && summary.type === "summary") { + return ( +
setSummaryOpen((e.currentTarget as HTMLDetailsElement).open)}> + + {summary.auto + ? t("chat.compactedAuto", { tokens: summary.tokens ? fmtCompact(summary.tokens) : "?" }) + : summary.tokens + ? t("chat.compactedManual", { tokens: fmtCompact(summary.tokens) }) + : t("chat.compacted")} + + {summaryOpen && } +
+ ); + } + const allTexts = m.parts.filter((p) => p.type === "text") as { text: string }[]; + const background = allTexts.filter((p) => isHarnessNote(p.text)); + const texts = allTexts.filter((p) => !isHarnessNote(p.text)); + const imgs = m.parts.filter((p) => p.type === "image") as { image: ImageData }[]; + return ( +
+ {background.map((p, i) => ( + + ))} +
+ {canEdit && ( +
+ +
+ )} + p.text).join("\n\n")} /> + {imgs.length > 0 && ( +
+ {imgs.map((p, i) => ( + + + + ))} +
+ )} +
+
+ ); +}); + +/** + * Older messages: loaded automatically when scrolled to the top; the history before a + * summary only on request. Keeps the scroll position while the list grows upwards. + */ +function OlderLoader({ sessionId, afterSummary, scrollRef }: { sessionId: string; afterSummary?: boolean; scrollRef: RefObject }) { + const t = useT(); + const [loading, setLoading] = useState(false); + const sentinel = useRef(null); + const load = useCallback(async () => { + const el = scrollRef.current; + if (loading || !el) return; + setLoading(true); + const before = { height: el.scrollHeight, top: el.scrollTop }; + try { + // Scrolling stops at a summary; its button loads the history before it. + await loadOlder(sessionId, !afterSummary); + requestAnimationFrame(() => { + el.scrollTop = before.top + (el.scrollHeight - before.height); + }); + } finally { + setLoading(false); + } + }, [loading, sessionId, scrollRef, afterSummary]); + useEffect(() => { + const node = sentinel.current; + if (!node || afterSummary || typeof IntersectionObserver === "undefined") return; + const io = new IntersectionObserver((e) => e[0]?.isIntersecting && void load(), { root: scrollRef.current, rootMargin: "300px 0px 0px 0px" }); + io.observe(node); + return () => io.disconnect(); + }, [afterSummary, load, scrollRef]); + return ( +
+ {loading ? ( + + ) : ( + + )} +
+ ); +} + export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string; onMenu: () => void; onClosePane?: () => void }) { const t = useT(); const session = useStore((s) => s.sessionById[sessionId]); const messages = useStore((s) => s.messages[sessionId] ?? EMPTY); + const win = useStore((s) => s.windows[sessionId]); + // Message to show (search hit, media): scrolled into view and marked. + const [highlight, setHighlight] = useState(); + const [searching, setSearching] = useState(false); + const [gallery, setGallery] = useState(false); const draft = useStore((s) => s.drafts[sessionId]); const busy = useStore((s) => !!s.busy[sessionId]); const errors = useStore((s) => s.errors[sessionId]); @@ -226,11 +517,36 @@ export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string return m; }, [messages]); - // Keep scrolled to the bottom while the user hasn't scrolled up. + // Keep scrolled to the bottom while the user hasn't scrolled up (only when the + // content changes – reading the height forces a layout). + const permCount = permissions.length + questions.length; useLayoutEffect(() => { const el = scrollRef.current; if (el && stick.current) el.scrollTop = el.scrollHeight; - }); + }, [messages, draft, busy, waiting, compacting, permCount, errors]); + + // Stable callbacks for the (memoised) message list. + const rewindRef = useRef<(m: Message) => void>(() => undefined); + const onRewind = useCallback((m: Message) => rewindRef.current(m), []); + const showLatest = useCallback(async () => { + setHighlight(undefined); + await loadSession(sessionId); + follow(true); + }, [sessionId]); + /** Load the part of the chat around a message and scroll to it. */ + const jumpTo = useCallback( + async (messageId: string) => { + follow(false); + if (!getState().messages[sessionId]?.some((m) => m.id === messageId)) await loadAround(sessionId, messageId); + setHighlight(messageId); + }, + [sessionId], + ); + useEffect(() => { + if (!highlight) return; + const h = setTimeout(() => document.querySelector(`[data-mid="${highlight}"]`)?.scrollIntoView({ block: "center" }), 50); + return () => clearTimeout(h); + }, [highlight, messages]); // … also when the pane gets smaller (on-screen keyboard opens) without a render. const hasSession = !!session; useEffect(() => { @@ -280,9 +596,10 @@ export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string } }; - const lastAssistantIdx = messages.reduce((acc, m, i) => (m.role === "assistant" ? i : acc), -1); + const [rewinding, setRewinding] = useState(); const rewind = (m: Message) => setRewinding(m); + rewindRef.current = rewind; const rewindDone = async () => { const m = rewinding!; setRewinding(undefined); @@ -371,6 +688,9 @@ export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string > + @@ -394,6 +714,16 @@ export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string > {t("instructions.title")} + { + e.preventDefault(); + (e.currentTarget.closest("details") as HTMLDetailsElement).open = false; + setGallery(true); + }} + > + {t("media.title")} + {telegramPaired && !session.parent_id && ( )} + {gallery && ( + setGallery(false)} + onJump={(id) => { + setGallery(false); + void jumpTo(id); + }} + /> + )} + {searching && void jumpTo(id)} onClose={() => setSearching(false)} />}