diff --git a/CHANGELOG.md b/CHANGELOG.md index 793193d..0a9d74d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to CodeCraft. Versions follow [Semantic Versioning](https://semver.org/); until 1.0, minor versions may contain breaking changes. +## [Unreleased] + +### Changed + +- **Chats with many images load much faster:** images (screenshots, generated images, pasted + images) are stored separately instead of inside the messages and load on demand when they + scroll into view, cached by the browser. Large PNGs without transparency are stored as JPEG. + Older chats are converted once on the first start; the database usually shrinks a lot. + Example: a chat with 120 screenshots loads 1.5 MB instead of 66 MB. +- **Compressed responses:** the API and the web UI are sent with gzip/brotli (not images, + downloads or live streams). + ## [v202611-00009] – 2026-11-09 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 41448a7..2c339fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" + +[[package]] +name = "alloc-stdlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -153,6 +168,18 @@ dependencies = [ "password-hash", ] +[[package]] +name = "async-compression" +version = "0.4.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -363,6 +390,27 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "brotli" +version = "9.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "6.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.13.1" @@ -682,6 +730,24 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compression-codecs" +version = "0.4.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + [[package]] name = "const-oid" version = "0.10.2" @@ -4035,6 +4101,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ + "async-compression", "bitflags 2.13.2", "bytes", "futures-core", diff --git a/cli/src/app.rs b/cli/src/app.rs index 71fb0fe..7a8cf92 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -421,7 +421,7 @@ impl App { _ => None, }; if let (Some(mt), Ok(bytes)) = (mt, std::fs::read(&path)) { - self.images.push(ImageData { media_type: mt.into(), data: base64::engine::general_purpose::STANDARD.encode(bytes) }); + self.images.push(ImageData { blob: None, size: None, media_type: mt.into(), data: base64::engine::general_purpose::STANDARD.encode(bytes) }); } } } @@ -840,7 +840,7 @@ impl App { for (cmd, args) in tries { if let Ok(out) = std::process::Command::new(cmd).args(*args).stderr(std::process::Stdio::null()).output() { if out.status.success() && out.stdout.starts_with(b"\x89PNG") { - self.images.push(ImageData { media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(&out.stdout) }); + self.images.push(ImageData { blob: None, size: None, media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(&out.stdout) }); self.status = format!("image {} attached", self.images.len()); return; } diff --git a/cli/src/tools.rs b/cli/src/tools.rs index 0ea4dc7..8bcfffa 100644 --- a/cli/src/tools.rs +++ b/cli/src/tools.rs @@ -408,7 +408,7 @@ pub async fn run(ctx: &Ctx, name: &str, input: &Value) -> Result { return Ok(Output { content: format!("Image {} ({} bytes)", rel(&ctx.cwd, &path), bytes.len()), is_error: false, - images: vec![ImageData { media_type: mt.into(), data: base64::engine::general_purpose::STANDARD.encode(&bytes) }], + images: vec![ImageData { blob: None, size: None, media_type: mt.into(), data: base64::engine::general_purpose::STANDARD.encode(&bytes) }], diff: None, }); } diff --git a/core/src/types.rs b/core/src/types.rs index 027b57d..bc42d6a 100644 --- a/core/src/types.rs +++ b/core/src/types.rs @@ -10,11 +10,25 @@ pub enum Role { Assistant, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] pub struct ImageData { pub media_type: String, - /// Base64-encoded bytes. + /// Base64-encoded bytes. Empty when the image is stored separately (`blob`). + #[serde(default, skip_serializing_if = "String::is_empty")] pub data: String, + /// Server only: SHA-256 of the bytes in the image store; the UI loads + /// /api/blobs/ and the server fills `data` in before a model request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob: Option, + /// Size in bytes of a stored image (shown in the UI without fetching it). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, +} + +impl ImageData { + pub fn new(media_type: impl Into, data: String) -> Self { + Self { media_type: media_type.into(), data, blob: None, size: None } + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] diff --git a/server/Cargo.toml b/server/Cargo.toml index 6400910..ce53bb7 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -38,7 +38,7 @@ tokio-tungstenite = "0.29" tokio-util = "0.7" toml = "1" toml_edit = "0.25" -tower-http = { version = "0.7", features = ["fs", "cors", "trace"] } +tower-http = { version = "0.7", features = ["fs", "cors", "trace", "compression-gzip", "compression-br"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } uuid = { version = "1", features = ["v4", "v7"] } diff --git a/server/src/api.rs b/server/src/api.rs index 763573b..0e65e33 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -80,6 +80,7 @@ pub fn router(app: AppState) -> Router { .route("/sessions/{id}/worktree", get(worktree_status).post(worktree_action)) .route("/sessions/{id}/tabs", get(session_tabs).put(put_file_tabs)) .route("/sessions/{id}/export", get(export_session)) + .route("/blobs/{hash}", get(get_blob)) .route("/sessions/{id}", get(get_session).patch(update_session).delete(delete_session)) .route("/sessions/{id}/prompt", post(prompt)) .route("/sessions/{id}/abort", post(abort)) @@ -181,9 +182,21 @@ pub fn router(app: AppState) -> Router { .route("/cli/install.sh", get(cli_install_script)) .route("/cli/download/{target}", get(cli_download)) .fallback_service(static_files) + .layer(compression()) .with_state(app) } +/// gzip/brotli for JSON, HTML, JS and CSS. Images and SSE streams are left alone (default +/// predicate), and so are binary downloads (already compressed or large). +fn compression() -> tower_http::compression::CompressionLayer { + use tower_http::compression::predicate::{DefaultPredicate, NotForContentType, Predicate}; + let predicate = DefaultPredicate::new() + .and(NotForContentType::const_new("application/octet-stream")) + .and(NotForContentType::const_new("application/zip")) + .and(NotForContentType::const_new("application/gzip")); + tower_http::compression::CompressionLayer::new().compress_when(predicate) +} + fn cli_dir() -> PathBuf { std::env::var("CODECRAFT_CLI_DIR").map(PathBuf::from).unwrap_or_else(|_| PathBuf::from("/app/cli")) } @@ -744,7 +757,8 @@ struct ExportQ { /// Download a session as Markdown (default) or JSON. async fn export_session(Extension(user): Extension, State(app): State, Path(id): Path, Query(q): Query) -> Result { let s = readable(&app, &user, &id)?; - let messages = app.store.messages(&id)?; + // JSON exports are self-contained: images inline instead of store references. + let messages = if q.format == "json" { app.store.messages_for_model(&id)? } else { app.store.messages(&id)? }; let base: String = s.title.chars().map(|c| if c.is_alphanumeric() || c == '-' { c } else { '_' }).collect::().trim_matches('_').to_string(); let base = if base.is_empty() { format!("session-{}", &id[..8.min(id.len())]) } else { base }; let (body, mime, ext) = if q.format == "json" { @@ -879,9 +893,33 @@ async fn delete_session(Extension(user): Extension, State(app): State, State(app): State, Path(hash): Path) -> Result { + if !crate::blobs::is_hash(&hash) { + return Err(not_found("image")); + } + let (mime, bytes) = app.store.blob(&hash)?.ok_or_else(|| not_found("image"))?; + Ok(( + [ + (axum::http::header::CONTENT_TYPE, mime), + (axum::http::header::CACHE_CONTROL, "private, max-age=31536000, immutable".to_string()), + ], + bytes, + ) + .into_response()) +} + #[derive(Deserialize)] struct PromptBody { text: String, diff --git a/server/src/app.rs b/server/src/app.rs index f0ed2ec..217a70c 100644 --- a/server/src/app.rs +++ b/server/src/app.rs @@ -131,9 +131,32 @@ impl App { }); crate::auth::bootstrap(&app)?; app.clone().start_mcp(); + app.clone().start_image_migration(); Ok(app) } + /// Move inline images of older chats to the image store, drop unused images and + /// shrink the database file (in the background; chats load meanwhile as before). + fn start_image_migration(self: Arc) { + tokio::task::spawn_blocking(move || { + match self.store.migrate_inline_images() { + Ok(0) => {} + Ok(n) => tracing::info!("moved {n} images of older chats to the image store"), + Err(e) => tracing::warn!("moving images to the image store: {e:#}"), + } + match self.store.gc_blobs() { + Ok(0) => {} + Ok(n) => tracing::info!("removed {n} unused images"), + Err(e) => tracing::warn!("image cleanup: {e:#}"), + } + match self.store.vacuum_if_worthwhile() { + Ok(true) => tracing::info!("database compacted"), + Ok(false) => {} + Err(e) => tracing::warn!("database compaction: {e:#}"), + } + }); + } + fn start_mcp(self: Arc) { let servers = self.config().mcp.clone(); let mcp = self.mcp.clone(); diff --git a/server/src/blobs.rs b/server/src/blobs.rs new file mode 100644 index 0000000..ff5fc6b --- /dev/null +++ b/server/src/blobs.rs @@ -0,0 +1,134 @@ +//! Images of chat messages are stored in the `blobs` table instead of as base64 inside the +//! message JSON. A chat then loads without its images; the UI fetches them on demand from +//! /api/blobs/ (cached by the browser), and the runner puts the bytes back in before +//! a model request. Large PNGs without transparency are stored as JPEG. + +use crate::types::{ImageData, Message, Part}; +use base64::Engine; +use sha2::{Digest, Sha256}; + +const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + +/// PNGs above this size are re-encoded as JPEG when that is smaller and loses no transparency. +const RECOMPRESS_ABOVE: usize = 100 * 1024; +const JPEG_QUALITY: u8 = 85; + +/// An image to put into the store. +pub struct Blob { + pub hash: String, + pub media_type: String, + pub bytes: Vec, +} + +/// Call `f` for every image of a message (pasted images and tool results). +pub fn for_each_image(m: &mut Message, mut f: impl FnMut(&mut ImageData)) { + for part in &mut m.parts { + match part { + Part::Image { image } => f(image), + Part::ToolResult { images, .. } => images.iter_mut().for_each(&mut f), + _ => {} + } + } +} + +/// Move the inline images of `m` out: each becomes a reference (`blob`, `size`, no +/// `data`). Returns the blobs to store. Images that already are references stay as they are. +pub fn externalize(m: &mut Message) -> Vec { + let mut out = Vec::new(); + for_each_image(m, |img| { + if img.data.is_empty() { + return; + } + let Ok(raw) = B64.decode(img.data.as_bytes()) else { return }; + let (media_type, bytes) = compact(&img.media_type, raw); + let hash = hash(&bytes); + img.media_type = media_type.clone(); + img.data.clear(); + img.blob = Some(hash.clone()); + img.size = Some(bytes.len() as u64); + out.push(Blob { hash, media_type, bytes }); + }); + out +} + +/// Whether `s` looks like a blob hash (64 lowercase hex digits). +pub fn is_hash(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect() +} + +/// Re-encode a large opaque PNG as JPEG if that is smaller; anything else is kept. +fn compact(media_type: &str, raw: Vec) -> (String, Vec) { + if media_type != "image/png" || raw.len() <= RECOMPRESS_ABOVE { + return (media_type.to_string(), raw); + } + let Ok(img) = image::load_from_memory_with_format(&raw, image::ImageFormat::Png) else { + return (media_type.to_string(), raw); + }; + if img.color().has_alpha() && img.to_rgba8().pixels().any(|p| p.0[3] != 255) { + return (media_type.to_string(), raw); + } + let mut jpeg = Vec::new(); + let enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, JPEG_QUALITY); + if img.to_rgb8().write_with_encoder(enc).is_err() || jpeg.len() >= raw.len() { + return (media_type.to_string(), raw); + } + ("image/jpeg".to_string(), jpeg) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::Role; + + fn png(w: u32, h: u32, alpha: u8) -> Vec { + // Noise, so the PNG is large and JPEG wins. + let img = image::RgbaImage::from_fn(w, h, |x, y| { + let v = (x.wrapping_mul(7919) ^ y.wrapping_mul(104_729)).wrapping_mul(2_654_435_761) >> 24; + image::Rgba([v as u8, (v >> 1) as u8, (v >> 2) as u8, alpha]) + }); + let mut buf = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png).unwrap(); + buf + } + + fn message(img: ImageData) -> Message { + let mut m = Message { id: "m1".into(), role: Role::User, parts: vec![], model: None, usage: None, created_at: 0 }; + m.parts.push(Part::ToolResult { call_id: "c".into(), name: "browser".into(), content: String::new(), images: vec![img], is_error: false, meta: None }); + m + } + + #[test] + fn large_opaque_png_becomes_a_jpeg_reference() { + let raw = png(400, 400, 255); + let mut m = message(ImageData::new("image/png", B64.encode(&raw))); + let blobs = externalize(&mut m); + assert_eq!(blobs.len(), 1); + assert_eq!(blobs[0].media_type, "image/jpeg"); + assert!(blobs[0].bytes.len() < raw.len()); + let Part::ToolResult { images, .. } = &m.parts[0] else { panic!() }; + assert!(images[0].data.is_empty()); + assert_eq!(images[0].blob.as_deref(), Some(blobs[0].hash.as_str())); + assert_eq!(images[0].size, Some(blobs[0].bytes.len() as u64)); + assert!(is_hash(&blobs[0].hash)); + // Already a reference: nothing to do. + assert!(externalize(&mut m).is_empty()); + } + + #[test] + fn transparent_and_small_pngs_stay_png() { + let mut m = message(ImageData::new("image/png", B64.encode(png(400, 400, 128)))); + assert_eq!(externalize(&mut m)[0].media_type, "image/png"); + let mut m = message(ImageData::new("image/png", B64.encode(png(8, 8, 255)))); + assert_eq!(externalize(&mut m)[0].media_type, "image/png"); + } + + #[test] + fn serialized_reference_has_no_data() { + let img = ImageData { media_type: "image/jpeg".into(), data: String::new(), blob: Some("ab".into()), size: Some(3) }; + assert_eq!(serde_json::to_value(&img).unwrap(), serde_json::json!({"media_type": "image/jpeg", "blob": "ab", "size": 3})); + } +} diff --git a/server/src/docs.rs b/server/src/docs.rs index bf94e2f..95d0cbc 100644 --- a/server/src/docs.rs +++ b/server/src/docs.rs @@ -129,7 +129,7 @@ async fn render_page(pdf: &str, page: u32) -> Result { let img = img.resize(1568, 1568, image::imageops::FilterType::Triangle); let mut buf = Vec::new(); img.write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png)?; - Ok(ImageData { media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(buf) }) + Ok(ImageData { blob: None, size: None, media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(buf) }) } fn spreadsheet(path: &Path, sheet: Option<&str>) -> Result { diff --git a/server/src/images.rs b/server/src/images.rs index e407ab1..ac1ac1c 100644 --- a/server/src/images.rs +++ b/server/src/images.rs @@ -258,7 +258,7 @@ fn preview_for_model(path: &Path) -> Option { let img = img.resize(1024, 1024, image::imageops::FilterType::Triangle); let mut buf = Vec::new(); img.write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png).ok()?; - Some(ImageData { media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(buf) }) + Some(ImageData { blob: None, size: None, media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(buf) }) } pub struct GenerateImage; diff --git a/server/src/main.rs b/server/src/main.rs index 83dc115..375c862 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -4,6 +4,7 @@ mod api; mod app; mod auth; mod backup; +mod blobs; mod cli_api; mod budget; mod config; diff --git a/server/src/mcp.rs b/server/src/mcp.rs index 6663f5d..67b840e 100644 --- a/server/src/mcp.rs +++ b/server/src/mcp.rs @@ -212,6 +212,8 @@ impl McpServer { match c["type"].as_str().unwrap_or_default() { "text" => texts.push(c["text"].as_str().unwrap_or_default().to_string()), "image" => out.images.push(ImageData { + blob: None, + size: None, media_type: c["mimeType"].as_str().unwrap_or("image/png").into(), data: c["data"].as_str().unwrap_or_default().into(), }), diff --git a/server/src/runner.rs b/server/src/runner.rs index 41233d3..fdc4e23 100644 --- a/server/src/runner.rs +++ b/server/src/runner.rs @@ -291,7 +291,7 @@ pub async fn run_session(app: Arc, session_id: &str, cancel: CancellationTo let owner = session.owner_id.as_deref().and_then(|id| app.store.user(id).ok().flatten()); let tools = available_tools(&app, &agent, owner.as_ref()); - let history = active_history(app.store.messages(session_id)?); + let history = active_history(app.store.messages_for_model(session_id)?); let req = ChatRequest { system: prompts::with_chat_instructions(prompts::system_prompt(&app, &agent, &root, &cwd, &model_spec, &session.permission_mode, owner.as_ref()), &session.instructions), messages: history, @@ -1126,7 +1126,7 @@ async fn review_turn(app: &Arc, session: &Session, agent: &prompts::Agent, model.effort = Some("medium".into()); app.emit(Event::Waiting { session_id: session.id.clone(), reason: Some(format!("review:{round}/{rounds}")) }); let result = async { - let history = active_history(app.store.messages(&session.id)?); + let history = active_history(app.store.messages_for_model(&session.id)?); // The user's request: the last user message with text that is not feedback/tool output. let req_idx = history .iter() @@ -1251,7 +1251,7 @@ pub async fn compact(app: Arc, session_id: &str, cancel: CancellationToken, async fn compact_inner(app: Arc, session_id: &str, cancel: CancellationToken, auto: Option<(u64, u64)>) -> Result<()> { let session = app.session(session_id)?; - let history = active_history(app.store.messages(session_id)?); + let history = active_history(app.store.messages_for_model(session_id)?); if history.is_empty() { return Ok(()); } diff --git a/server/src/store.rs b/server/src/store.rs index 1a29ada..4dda493 100644 --- a/server/src/store.rs +++ b/server/src/store.rs @@ -1,6 +1,6 @@ //! SQLite persistence for sessions and messages. -use crate::types::{Message, Part, Session, now_ms}; +use crate::types::{ImageData, Message, Part, Session, now_ms}; use serde::Serialize; use serde_json::{Value, json}; use anyhow::Result; @@ -90,6 +90,13 @@ impl Store { created_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS images_user ON images(user_id, created_at); + -- Images of messages, referenced by hash from the message JSON (see blobs.rs). + CREATE TABLE IF NOT EXISTS blobs ( + hash TEXT PRIMARY KEY, + media_type TEXT NOT NULL, + data BLOB NOT NULL, + created_at INTEGER NOT NULL + ); CREATE TABLE IF NOT EXISTS queue ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, @@ -274,12 +281,21 @@ impl Store { } pub fn add_message(&self, session_id: &str, m: &Message) -> Result<()> { + // Images go to the blob table (encoded before taking the lock). + let mut m = m.clone(); + let blobs = crate::blobs::externalize(&mut m); let conn = self.conn.lock().unwrap(); + for b in &blobs { + conn.execute( + "INSERT OR IGNORE INTO blobs (hash, media_type, data, created_at) VALUES (?1, ?2, ?3, ?4)", + params![b.hash, b.media_type, b.bytes, now_ms()], + )?; + } conn.execute( "INSERT INTO messages (id, session_id, seq, data) VALUES (?1, ?2, (SELECT COALESCE(MAX(seq), 0) + 1 FROM messages WHERE session_id = ?2), ?3) ON CONFLICT(id) DO UPDATE SET data = excluded.data", - params![m.id, session_id, serde_json::to_string(m)?], + params![m.id, session_id, serde_json::to_string(&m)?], )?; conn.execute( "UPDATE sessions SET updated_at = ?2 WHERE id = ?1", @@ -300,6 +316,103 @@ impl Store { Ok(out) } + /// Messages with the bytes of their images filled in – for model requests and exports. + pub fn messages_for_model(&self, session_id: &str) -> Result> { + let mut out = self.messages(session_id)?; + let conn = self.conn.lock().unwrap(); + let mut cache: std::collections::HashMap = std::collections::HashMap::new(); + for m in &mut out { + crate::blobs::for_each_image(m, |img| fill_image(&conn, &mut cache, img)); + } + Ok(out) + } + + /// Fill in the bytes of one image reference (no-op for inline images). + pub fn resolve_image(&self, img: &ImageData) -> ImageData { + let mut img = img.clone(); + let conn = self.conn.lock().unwrap(); + fill_image(&conn, &mut std::collections::HashMap::new(), &mut img); + img + } + + /// One stored image: (media type, bytes). + pub fn blob(&self, hash: &str) -> Result)>> { + let conn = self.conn.lock().unwrap(); + Ok(conn + .query_row("SELECT media_type, data FROM blobs WHERE hash = ?1", [hash], |r| Ok((r.get(0)?, r.get(1)?))) + .optional()?) + } + + /// Move inline images of older messages to the blob table (once, after the update + /// that introduced it). Does not touch `updated_at` of the sessions. + pub fn migrate_inline_images(&self) -> Result { + let ids: Vec = { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM messages WHERE data LIKE '%\"media_type\"%\"data\":\"%'")?; + let rows = stmt.query_map([], |r| r.get(0))?.collect::>>()?; + rows + }; + let mut moved = 0; + for id in ids { + let data: Option = { + let conn = self.conn.lock().unwrap(); + conn.query_row("SELECT data FROM messages WHERE id = ?1", [&id], |r| r.get(0)).optional()? + }; + let Some(data) = data else { continue }; + let Ok(mut m) = serde_json::from_str::(&data) else { continue }; + let blobs = crate::blobs::externalize(&mut m); + if blobs.is_empty() { + continue; + } + let conn = self.conn.lock().unwrap(); + for b in &blobs { + conn.execute( + "INSERT OR IGNORE INTO blobs (hash, media_type, data, created_at) VALUES (?1, ?2, ?3, ?4)", + params![b.hash, b.media_type, b.bytes, now_ms()], + )?; + } + conn.execute("UPDATE messages SET data = ?2 WHERE id = ?1", params![id, serde_json::to_string(&m)?])?; + moved += blobs.len(); + } + Ok(moved) + } + + /// Delete images no message refers to any more (deleted or truncated sessions). + pub fn gc_blobs(&self) -> Result { + let conn = self.conn.lock().unwrap(); + let mut used = std::collections::HashSet::new(); + let mut stmt = conn.prepare("SELECT data FROM messages WHERE data LIKE '%\"blob\":\"%'")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let data: String = row.get(0)?; + for (i, _) in data.match_indices("\"blob\":\"") { + if let Some(h) = data.get(i + 8..i + 72) { + used.insert(h.to_string()); + } + } + } + drop(rows); + drop(stmt); + let all: Vec = conn.prepare("SELECT hash FROM blobs")?.query_map([], |r| r.get(0))?.collect::>()?; + let mut removed = 0; + for h in all.iter().filter(|h| !used.contains(*h)) { + removed += conn.execute("DELETE FROM blobs WHERE hash = ?1", [h])?; + } + Ok(removed) + } + + /// Give the space of deleted data back to the file system when it is worth it. + pub fn vacuum_if_worthwhile(&self) -> Result { + let conn = self.conn.lock().unwrap(); + let free: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?; + let page: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?; + if free * page < 32 * 1024 * 1024 { + return Ok(false); + } + conn.execute_batch("VACUUM")?; + Ok(true) + } + /// Remove all messages after (and including) the given one – used for "edit & resend". pub fn truncate_from(&self, session_id: &str, message_id: &str) -> Result<()> { let conn = self.conn.lock().unwrap(); @@ -312,6 +425,20 @@ impl Store { } } +/// Put the bytes of a stored image into `img.data`. +fn fill_image(conn: &Connection, cache: &mut std::collections::HashMap, img: &mut ImageData) { + use base64::Engine; + let Some(hash) = img.blob.clone().filter(|_| img.data.is_empty()) else { return }; + if !cache.contains_key(&hash) { + let Ok(bytes) = conn.query_row("SELECT data FROM blobs WHERE hash = ?1", [&hash], |r| r.get::<_, Vec>(0)) else { + tracing::warn!("image {hash} is missing in the store"); + return; + }; + cache.insert(hash.clone(), base64::engine::general_purpose::STANDARD.encode(bytes)); + } + img.data = cache[&hash].clone(); +} + /// One model request, as recorded for statistics. #[derive(Debug, Clone, Serialize)] pub struct UsageRecord { diff --git a/server/src/telegram.rs b/server/src/telegram.rs index 6f0d667..4f7d289 100644 --- a/server/src/telegram.rs +++ b/server/src/telegram.rs @@ -354,13 +354,13 @@ async fn handle_update(app: &Arc, user_id: &str, bot: &Bot, u: &Value) -> R let mut notes = Vec::new(); if let Some(photo) = m["photo"].as_array().and_then(|p| p.last()) { let (bytes, _) = bot.download(photo["file_id"].as_str().unwrap_or("")).await?; - images.push(ImageData { media_type: "image/jpeg".into(), data: base64::engine::general_purpose::STANDARD.encode(bytes) }); + images.push(ImageData { blob: None, size: None, media_type: "image/jpeg".into(), data: base64::engine::general_purpose::STANDARD.encode(bytes) }); } if let Some(doc) = m.get("document") { let mime = doc["mime_type"].as_str().unwrap_or(""); let (bytes, path) = bot.download(doc["file_id"].as_str().unwrap_or("")).await?; if mime.starts_with("image/") && bytes.len() < 10 * 1024 * 1024 { - images.push(ImageData { media_type: mime.into(), data: base64::engine::general_purpose::STANDARD.encode(bytes) }); + images.push(ImageData { blob: None, size: None, media_type: mime.into(), data: base64::engine::general_purpose::STANDARD.encode(bytes) }); } else { let name = doc["file_name"].as_str().map(String::from).unwrap_or_else(|| path.rsplit('/').next().unwrap_or("file").to_string()); let rel = save_upload(&session, &name, &bytes)?; @@ -876,7 +876,7 @@ async fn listen(app: Arc) { } #[allow(clippy::too_many_arguments)] -async fn on_event(_app: &App, bot: &Bot, chat: i64, de: bool, session: &Session, is_current: bool, ev: &Event, progress: &mut HashMap) -> Result<()> { +async fn on_event(app: &App, bot: &Bot, chat: i64, de: bool, session: &Session, is_current: bool, ev: &Event, progress: &mut HashMap) -> Result<()> { let sid = session.id.clone(); match ev { // Approvals and questions of every own session: the agent waits for them. @@ -947,7 +947,7 @@ async fn on_event(_app: &App, bot: &Bot, chat: i64, de: bool, session: &Session, for part in &message.parts { if let Part::ToolResult { images, name, .. } = part { for img in images { - let _ = bot.send_photo(chat, img, Some(name)).await; + let _ = bot.send_photo(chat, &app.store.resolve_image(img), Some(name)).await; } } } diff --git a/server/src/tools/cad.rs b/server/src/tools/cad.rs index 4bb9f4d..8e2fa23 100644 --- a/server/src/tools/cad.rs +++ b/server/src/tools/cad.rs @@ -186,7 +186,7 @@ impl Tool for OpenScad { continue; } if let Ok(bytes) = std::fs::read(&out) { - images.push(ImageData { media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(&bytes) }); + images.push(ImageData { blob: None, size: None, media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(&bytes) }); files.push(out); } } diff --git a/server/src/tools/craft.rs b/server/src/tools/craft.rs index 41fafa4..5e51908 100644 --- a/server/src/tools/craft.rs +++ b/server/src/tools/craft.rs @@ -27,6 +27,8 @@ pub(crate) fn png_for_model(img: &image::DynamicImage) -> Result { let mut buf = Vec::new(); img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Png)?; Ok(ImageData { + blob: None, + size: None, media_type: "image/png".into(), data: base64::engine::general_purpose::STANDARD.encode(&buf), }) diff --git a/server/src/tools/fs.rs b/server/src/tools/fs.rs index a16e408..e3d853b 100644 --- a/server/src/tools/fs.rs +++ b/server/src/tools/fs.rs @@ -107,6 +107,8 @@ impl Tool for Read { return Ok(ToolOutput { content: format!("Image {} ({} bytes)", rel(ctx, &path), bytes.len()), images: vec![ImageData { + blob: None, + size: None, media_type: mt.into(), data: base64::engine::general_purpose::STANDARD.encode(&bytes), }], diff --git a/web/src/components/ChatView.tsx b/web/src/components/ChatView.tsx index 606b258..870ebe0 100644 --- a/web/src/components/ChatView.tsx +++ b/web/src/components/ChatView.tsx @@ -1,6 +1,6 @@ import { takePrefill } from "../share"; import { Icon } from "./Icon"; -import { base64Size, FileChip, FileFigure, fileType } from "./FileFigure"; +import { FileChip, FileFigure, fileType, imageSize, imageSrc } from "./FileFigure"; import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { del, get, patch, post } from "../api"; import { useModels } from "../hooks"; @@ -471,8 +471,8 @@ export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string {imgs.length > 0 && (
{imgs.map((p, i) => ( - - + + ))}
diff --git a/web/src/components/Composer.tsx b/web/src/components/Composer.tsx index 787a75f..d77a77e 100644 --- a/web/src/components/Composer.tsx +++ b/web/src/components/Composer.tsx @@ -1,6 +1,6 @@ import { TemplatePicker } from "./TemplatePicker"; import { Icon } from "./Icon"; -import { formatSize } from "./FileFigure"; +import { formatSize, imageSrc } from "./FileFigure"; import { uploadFile } from "../api"; import { useEffect, useMemo, useRef, useState } from "react"; import { useAgents, useFetch, useModels } from "../hooks"; @@ -257,7 +257,7 @@ export function Composer({ )} {images.map((img, i) => (
- + diff --git a/web/src/components/FileFigure.tsx b/web/src/components/FileFigure.tsx index dd0f933..f3388e7 100644 --- a/web/src/components/FileFigure.tsx +++ b/web/src/components/FileFigure.tsx @@ -1,5 +1,6 @@ import { Icon } from "./Icon"; import { useEffect, useState, type ReactNode } from "react"; +import type { ImageData } from "../types"; /** Human-readable file size. */ export function formatSize(n: number) { @@ -32,6 +33,16 @@ export function basename(p: string) { } /** Bytes of a base64 payload. */ +/** URL of an image of a message: inline (data URL) or from the image store. */ +export function imageSrc(img: ImageData) { + return img.data ? `data:${img.media_type};base64,${img.data}` : `/api/blobs/${img.blob}`; +} + +/** Size in bytes of an image of a message. */ +export function imageSize(img: ImageData) { + return img.data ? base64Size(img.data) : img.size; +} + export function base64Size(b64: string) { const pad = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0; return Math.floor((b64.length * 3) / 4) - pad; diff --git a/web/src/components/NewSession.tsx b/web/src/components/NewSession.tsx index 810401d..5a8c70f 100644 --- a/web/src/components/NewSession.tsx +++ b/web/src/components/NewSession.tsx @@ -5,7 +5,7 @@ import { useAgents, useFetch } from "../hooks"; import { useT, type Key } from "../i18n"; import type { ImageData } from "../types"; import { readImage } from "./Composer"; -import { formatSize } from "./FileFigure"; +import { formatSize, imageSrc } from "./FileFigure"; import { ModelSelect } from "./ModelSelect"; import { takePrefill } from "../share"; @@ -478,7 +478,7 @@ export function NewSession({ onMenu, onCreated }: { onMenu: () => void; onCreate
{images.map((img, i) => (
- + diff --git a/web/src/components/QueueList.tsx b/web/src/components/QueueList.tsx index 2f3ab8b..4658aef 100644 --- a/web/src/components/QueueList.tsx +++ b/web/src/components/QueueList.tsx @@ -3,6 +3,7 @@ import { del, post } from "../api"; import { useT } from "../i18n"; import { useStore } from "../store"; import type { QueueItem } from "../types"; +import { imageSrc } from "./FileFigure"; import { confirmDialog, promptDialog, toast } from "../dialogs"; const EMPTY: QueueItem[] = []; @@ -30,7 +31,7 @@ export function QueueList({ sessionId, busy }: { sessionId: string; busy: boolea {it.source === "background" && {t("tool.background")}} {it.source === "background" ? t("queue.backgroundResult") : text || "—"} {images.map((p, i) => ( - + ))} {it.source !== "background" && (