diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fb45a8..ddafe27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to CodeCraft. Versions follow [Semantic Versioning](https://semver.org/); until 1.0, minor versions may contain breaking changes. +## [Unreleased] + +### Added + +- **3D models with fal.ai (`generate_3d`):** from an image (a project file or one pasted into the + chat, optional back/left/right views), from a text prompt, or from an image steered by a prompt + (the image is refined with the prompt first – the 3D models take no text with an image). + Styles textured, geometry (untextured, for 3D printing) and low-poly, optional PBR. Uses fal.ai's + queue (takes minutes, with progress), saves a GLB plus a preview image, books the cost. + Defaults: Hunyuan3D v3 image-to-3D / text-to-3D (`[images] model_3d`, `model_3d_text`); Tripo + and Trellis endpoints work too. +- The chat card offers "Open in the 3D view"; the media overview lists 3D models (also STL/3MF + written by the agent) with their previews. + ## [v202611-00023] – 2026-11-12 ### Added diff --git a/README.md b/README.md index 2900c58..7fcce33 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,10 @@ output limit, reasoning effort, extra request options and prices per million tok - **Images (fal.ai):** the agent drafts cheap previews (FLUX schnell), asks which one to refine, and creates the final image with the best model for the purpose – Nano Banana Pro refines the chosen draft, Ideogram handles lettering, Recraft design work. +- **3D models (fal.ai):** `generate_3d` turns an image (also one pasted into the chat, plus + optional back/left/right views), a text prompt, or an image steered by a prompt into a GLB + model – textured, untextured for 3D printing, or low-poly (Hunyuan3D v3 by default, + `[images] model_3d` / `model_3d_text`). Open it in the 3D view of the file manager. ## Terminal client diff --git a/server/src/config.rs b/server/src/config.rs index 85da1c6..23f7ab8 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -177,6 +177,19 @@ pub struct ImagesConfig { /// Optional fal.ai Admin key to query the account balance. #[serde(default, skip_serializing_if = "String::is_empty")] pub admin_key: String, + /// fal.ai model for 3D models from images (generate_3d). + #[serde(default = "default_model_3d")] + pub model_3d: String, + /// fal.ai model for 3D models from a text prompt only. + #[serde(default = "default_model_3d_text")] + pub model_3d_text: String, +} + +fn default_model_3d() -> String { + "fal-ai/hunyuan3d-v3/image-to-3d".into() +} +fn default_model_3d_text() -> String { + "fal-ai/hunyuan3d-v3/text-to-3d".into() } #[derive(Debug, Clone, Deserialize, Serialize, Default)] @@ -215,6 +228,8 @@ impl Default for ImagesConfig { models: vec![ImageModel { id: "fal-ai/flux/schnell".into(), name: Some("FLUX.1 schnell".into()), price: Some(0.003), ..Default::default() }], monthly_budget: None, admin_key: String::new(), + model_3d: default_model_3d(), + model_3d_text: default_model_3d_text(), } } } diff --git a/server/src/images.rs b/server/src/images.rs index e337145..3551303 100644 --- a/server/src/images.rs +++ b/server/src/images.rs @@ -38,7 +38,7 @@ pub struct GenerateRequest { pub references: Vec, } -fn slug(s: &str) -> String { +pub(crate) fn slug(s: &str) -> String { let mut out: String = s .chars() .map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' }) @@ -121,7 +121,7 @@ fn pick(all: &[ImageModel], stage: &str, purpose: Option<&str>, with_reference: } /// Read local reference images as data URIs (fal.ai accepts them in place of URLs). -async fn reference_uris(paths: &[PathBuf]) -> Result> { +pub(crate) async fn reference_uris(paths: &[PathBuf]) -> Result> { let mut out = Vec::new(); for p in paths { let bytes = tokio::fs::read(p).await.with_context(|| format!("reference image {}", p.display()))?; @@ -135,7 +135,7 @@ async fn reference_uris(paths: &[PathBuf]) -> Result> { Ok(out) } -async fn download(url: &str) -> Result<(Vec, String)> { +pub(crate) async fn download(url: &str) -> Result<(Vec, String)> { if let Some(rest) = url.strip_prefix("data:") { let (meta, data) = rest.split_once(',').context("malformed data URI")?; let ctype = meta.split(';').next().unwrap_or("image/png").to_string(); @@ -348,7 +348,7 @@ pub fn backfill_copies(app: &App) -> Result { } /// Downscaled copy for the model's context. -fn preview_for_model(path: &Path) -> Option { +pub(crate) fn preview_for_model(path: &Path) -> Option { let img = image::open(path).ok()?; let img = img.resize(1024, 1024, image::imageops::FilterType::Triangle); let mut buf = Vec::new(); diff --git a/server/src/main.rs b/server/src/main.rs index 825fa76..aa62f92 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -23,6 +23,7 @@ mod mentions; mod metrics; mod mcp; mod media; +mod mesh3d; mod permission; mod prompts; mod push; diff --git a/server/src/media.rs b/server/src/media.rs index fb6c7a5..1e597d6 100644 --- a/server/src/media.rs +++ b/server/src/media.rs @@ -11,7 +11,7 @@ use std::path::{Path, PathBuf}; #[derive(Debug, Serialize, PartialEq)] pub struct MediaItem { - /// "image", "video" or "audio". + /// "image", "video", "audio" or "model" (3D). pub kind: &'static str, pub url: String, pub name: String, @@ -21,11 +21,17 @@ pub struct MediaItem { /// Message to jump to (only for messages of the chat itself, not of subagents). #[serde(skip_serializing_if = "Option::is_none")] pub message_id: Option, + /// 3D models: absolute path (opened in the 3D view) and a preview image. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preview: Option, } const IMAGE: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "avif"]; const VIDEO: &[&str] = &["mp4", "webm", "mov", "m4v", "ogv"]; const AUDIO: &[&str] = &["mp3", "wav", "ogg", "oga", "m4a", "flac", "opus", "aac"]; +const MODEL: &[&str] = &["glb", "gltf", "stl", "3mf", "obj", "step", "stp"]; /// Kind of a media file by its extension. pub fn kind_of(path: &str) -> Option<&'static str> { @@ -36,6 +42,8 @@ pub fn kind_of(path: &str) -> Option<&'static str> { Some("video") } else if AUDIO.contains(&ext.as_str()) { Some("audio") + } else if MODEL.contains(&ext.as_str()) { + Some("model") } else { None } @@ -118,7 +126,7 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result> { // Generated images whose tool result was not found (e.g. removed by a rewind). for img in app.store.images_of_sessions(&sessions)? { push( - MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None }, + MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None, path: None, preview: None }, &mut out, ); } @@ -133,6 +141,10 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result> { Ok(out) } +fn raw_url(p: &Path) -> String { + format!("/api/files/raw?path={}", encode(&p.display().to_string())) +} + fn file_name(p: &str) -> String { Path::new(p).file_name().and_then(|n| n.to_str()).unwrap_or(p).to_string() } @@ -144,25 +156,33 @@ fn collect_message(m: &Message, cwd: &Path, own: bool, call_msg: Option<&str>, a let file = |p: &str, source: &'static str| -> Option { let kind = kind_of(p)?; let abs = resolve(p, cwd); - abs.is_file().then(|| MediaItem { kind, url: format!("/api/files/raw?path={}", encode(&abs.display().to_string())), name: file_name(p), source, created_at: at, message_id: mid.clone() }) + abs.is_file().then(|| MediaItem { kind, url: raw_url(&abs), name: file_name(p), source, created_at: at, message_id: mid.clone(), path: Some(abs.display().to_string()), preview: None }) }; for p in &m.parts { match p { Part::Image { image } if m.role == Role::User => { if let Some(h) = &image.blob { - add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone() }); + add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone(), path: None, preview: None }); + } + } + // generate_3d: the model with its preview (not the preview as a separate image). + Part::ToolResult { meta: Some(meta), .. } if meta["model_file"].is_string() => { + let file = PathBuf::from(meta["model_file"].as_str().unwrap_or_default()); + if file.is_file() { + let preview = meta["preview"].as_str().map(PathBuf::from).filter(|p| p.is_file()).map(|p| raw_url(&p)); + add(MediaItem { kind: "model", url: raw_url(&file), name: file_name(&file.display().to_string()), source: "generated", created_at: at, message_id: result_mid.clone(), path: Some(file.display().to_string()), preview }); } } Part::ToolResult { images, meta, .. } => { let ids: Vec<&str> = meta.as_ref().and_then(|v| v["image_ids"].as_array()).map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default(); if !ids.is_empty() { for id in ids { - add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone() }); + add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone(), path: None, preview: None }); } } else { for img in images { if let Some(h) = &img.blob { - add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone() }); + add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone(), path: None, preview: None }); } } } diff --git a/server/src/mesh3d.rs b/server/src/mesh3d.rs new file mode 100644 index 0000000..446d6da --- /dev/null +++ b/server/src/mesh3d.rs @@ -0,0 +1,430 @@ +//! 3D models with fal.ai (generate_3d): from an image (plus optional back/left/right +//! views), from a text prompt, or from an image steered by text – the 3D models take no +//! text with an image, so the image is first refined with the prompt (image edit model) +//! and then converted. The result is a GLB file (plus a preview image) in the project. + +use crate::app::App; +use crate::config::expand_env; +use crate::store::UsageRecord; +use crate::tools::{Tool, ToolCtx, ToolOutput}; +use crate::types::now_ms; +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const STYLES: &[&str] = &["textured", "geometry", "lowpoly"]; + +/// Request body for a fal.ai 3D endpoint (the families name their fields differently). +fn request_body(model: &str, prompt: Option<&str>, images: &Views, style: &str, pbr: bool, faces: Option) -> Value { + let mut b = json!({}); + if model.contains("tripo") { + // Tripo: image_url / prompt, texture + pbr flags, face_limit. + if let Some(front) = &images.front { + b["image_url"] = json!(front); + } + if let Some(p) = prompt { + b["prompt"] = json!(p); + } + b["texture"] = json!(style != "geometry"); + b["pbr"] = json!(style != "geometry" && pbr); + if let Some(f) = faces { + b["face_limit"] = json!(f); + } + if style == "lowpoly" { + b["face_limit"] = json!(faces.unwrap_or(20_000)); + } + } else if model.contains("trellis") { + if let Some(front) = &images.front { + b["image_url"] = json!(front); + } + } else { + // Hunyuan3D: front + optional views, generate_type, enable_pbr, face_count. + if let Some(front) = &images.front { + b["input_image_url"] = json!(front); + } + for (k, v) in [("back_image_url", &images.back), ("left_image_url", &images.left), ("right_image_url", &images.right)] { + if let Some(v) = v { + b[k] = json!(v); + } + } + if let Some(p) = prompt { + b["prompt"] = json!(p); + } + b["generate_type"] = json!(match style { + "geometry" => "Geometry", + "lowpoly" => "LowPoly", + _ => "Normal", + }); + b["enable_pbr"] = json!(style != "geometry" && pbr); + if let Some(f) = faces { + b["face_count"] = json!(f.clamp(40_000, 1_500_000)); + } + } + b +} + +/// Data URIs of the views (front required for image-to-3D). +#[derive(Default)] +struct Views { + front: Option, + back: Option, + left: Option, + right: Option, +} + +impl Views { + fn extra(&self) -> usize { + [&self.back, &self.left, &self.right].iter().filter(|v| v.is_some()).count() + } +} + +/// Price of one generation (USD) by the published fal.ai prices; a price configured for +/// the model under [images] models wins. +fn price(app: &App, model: &str, style: &str, pbr: bool, multiview: bool, faces: bool) -> f64 { + if let Some(p) = app.config().images.models.iter().find(|m| m.id == model).and_then(|m| m.price) { + return p; + } + if model.contains("tripo") { + return match style { + "geometry" => 0.20, + _ => 0.30, + }; + } + if model.contains("hunyuan") { + let base = match style { + "geometry" => 0.225, + "lowpoly" => 0.45, + _ => 0.375, + }; + let extras = [pbr && style != "geometry", multiview, faces].iter().filter(|x| **x).count() as f64; + return base + 0.15 * extras; + } + 0.0 +} + +/// URL of the queue API for fal.ai (requests that take minutes). +fn queue_base(app: &App) -> String { + let base = expand_env(&app.config().images.base_url); + if base.contains("://fal.run") { "https://queue.fal.run".into() } else { base.trim_end_matches('/').to_string() } +} + +/// Submit to fal.ai's queue and wait for the result, reporting progress. +async fn run_queued(app: &App, ctx: &ToolCtx, model: &str, body: &Value) -> Result { + let key = expand_env(&app.config().images.api_key); + let client = crate::provider::http_client(); + let submit = client + .post(format!("{}/{}", queue_base(app), model.trim_matches('/'))) + .header("Authorization", format!("Key {key}")) + .timeout(Duration::from_secs(120)) + .json(body) + .send() + .await + .context("fal.ai request failed")?; + if !submit.status().is_success() { + return Err(crate::provider::http_error(submit).await); + } + let sub: Value = submit.json().await?; + let status_url = sub["status_url"].as_str().context("fal.ai: no status_url")?.to_string(); + let response_url = sub["response_url"].as_str().context("fal.ai: no response_url")?.to_string(); + let started = std::time::Instant::now(); + loop { + if ctx.cancel.is_cancelled() { + bail!("cancelled"); + } + if started.elapsed() > Duration::from_secs(20 * 60) { + bail!("fal.ai did not finish within 20 minutes"); + } + tokio::time::sleep(Duration::from_secs(4)).await; + let st: Value = client.get(&status_url).header("Authorization", format!("Key {key}")).timeout(Duration::from_secs(30)).send().await?.json().await?; + match st["status"].as_str().unwrap_or("") { + "COMPLETED" => break, + "IN_QUEUE" => ctx.progress(format!("waiting in fal.ai's queue (position {}) – {} s", st["queue_position"].as_u64().unwrap_or(0), started.elapsed().as_secs())), + "IN_PROGRESS" => ctx.progress(format!("generating the 3D model – {} s", started.elapsed().as_secs())), + other => bail!("fal.ai: unexpected status {other}: {st}"), + } + } + let resp = client.get(&response_url).header("Authorization", format!("Key {key}")).timeout(Duration::from_secs(60)).send().await?; + if !resp.status().is_success() { + return Err(crate::provider::http_error(resp).await); + } + Ok(resp.json().await?) +} + +/// URL of the mesh and of the preview image in a fal.ai 3D result. +fn result_urls(out: &Value) -> (Option, Option) { + let s = |v: &Value| v.as_str().map(String::from); + let mesh = s(&out["model_glb"]["url"]) + .or_else(|| s(&out["model_mesh"]["url"])) + .or_else(|| s(&out["model_urls"]["glb"]["url"])) + .or_else(|| s(&out["model_file"]["url"])) + .or_else(|| s(&out["mesh"]["url"])); + let thumb = s(&out["thumbnail"]["url"]).or_else(|| s(&out["rendered_image"]["url"])); + (mesh, thumb) +} + +/// File extension of a downloaded mesh: from the URL, else GLB (what the models return). +fn ext_of(url: &str) -> &'static str { + let path = url.split(['?', '#']).next().unwrap_or(url).to_ascii_lowercase(); + ["glb", "gltf", "fbx", "obj", "usdz", "stl"].into_iter().find(|e| path.ends_with(&format!(".{e}"))).unwrap_or("glb") +} + +/// An image the user attached to the chat ("attached" = the first image of the latest +/// message with images, "attached:2" = its second), written to `dir` so it can be used +/// like a project file. +async fn attached_image(app: &App, session_id: &str, spec: &str, dir: &Path) -> Result { + let n: usize = spec.strip_prefix("attached").map(|r| r.trim_start_matches(':').trim()).filter(|r| !r.is_empty()).map(|r| r.parse()).transpose().map_err(|_| anyhow!("use \"attached\" or \"attached:2\""))?.unwrap_or(1); + let root = app.root_session_id(session_id); + let messages = app.store.messages(&root)?; + let images: Vec = messages + .iter() + .rev() + .filter(|m| m.role == crate::types::Role::User) + .map(|m| m.parts.iter().filter_map(|p| if let crate::types::Part::Image { image } = p { Some(image.clone()) } else { None }).collect::>()) + .find(|imgs| !imgs.is_empty()) + .ok_or_else(|| anyhow!("no image attached to this chat"))?; + let img = images.get(n.saturating_sub(1)).ok_or_else(|| anyhow!("the latest message has only {} image(s)", images.len()))?; + let (mime, bytes) = match &img.blob { + Some(h) => app.store.blob(h)?.ok_or_else(|| anyhow!("attached image not found"))?, + None => (img.media_type.clone(), base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img.data.as_bytes())?), + }; + let ext = match mime.as_str() { + "image/jpeg" => "jpg", + "image/webp" => "webp", + _ => "png", + }; + tokio::fs::create_dir_all(dir).await?; + let path = dir.join(format!("{}-attached-{n}.{ext}", chrono::Local::now().format("%Y%m%d-%H%M%S"))); + tokio::fs::write(&path, &bytes).await?; + Ok(path) +} + +/// An image argument: a project file, or an image attached to the chat. +async fn image_arg(ctx: &ToolCtx, spec: &str, dir: &Path) -> Result { + if spec.starts_with("attached") { + attached_image(&ctx.app, &ctx.session_id, spec, dir).await + } else { + ctx.resolve(spec) + } +} + +pub struct Generate3d; + +#[async_trait] +impl Tool for Generate3d { + fn name(&self) -> &str { + "generate_3d" + } + fn description(&self) -> String { + "Generate a 3D model (GLB) with fal.ai and save it in the project – from an image of the \ + object (a file, or image: \"attached\" for an image the user pasted into the chat; best: a \ + clean front view; optional back/left/right views improve the shape), from a \ + text prompt alone, or from an image steered by a prompt (the image is refined with the \ + prompt first, then converted). style: \"textured\" (default), \"geometry\" (untextured – for \ + 3D printing), \"lowpoly\" (games). Takes one to several minutes and costs money (about \ + $0.25–0.75): generate only what the user asked for. The user can open the model in the 3D \ + view of the file manager." + .into() + } + fn schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "prompt": {"type": "string", "description": "What the model should be (English). With an image: how to change/clean it up before the conversion."}, + "image": {"type": "string", "description": "Image of the object (front view): a file path, or \"attached\" for the image the user attached to the chat (\"attached:2\" = the second of that message)"}, + "back": {"type": "string", "description": "Optional back view image (path or \"attached:N\")"}, + "left": {"type": "string", "description": "Optional left view image (path or \"attached:N\")"}, + "right": {"type": "string", "description": "Optional right view image (path or \"attached:N\")"}, + "style": {"type": "string", "enum": STYLES}, + "pbr": {"type": "boolean", "description": "PBR materials (metal/roughness), textured models only; costs extra"}, + "face_count": {"type": "integer", "description": "Target number of faces (optional)"}, + "directory": {"type": "string", "description": "Target directory relative to the working directory (default: /3d)"}, + "model": {"type": "string", "description": "fal.ai model id – only if the user asks for a specific one"} + } + }) + } + fn title(&self, input: &Value) -> String { + let what = input["prompt"].as_str().map(|p| p.chars().take(70).collect::()).or_else(|| input["image"].as_str().map(String::from)).unwrap_or_default(); + format!("3D model: {what}") + } + fn read_only(&self) -> bool { + false + } + async fn run(&self, input: Value, ctx: &ToolCtx) -> Result { + let app = ctx.app.clone(); + let cfg = app.config(); + if !cfg.images.enabled { + bail!("fal.ai is disabled – enable image generation in the settings"); + } + if expand_env(&cfg.images.api_key).is_empty() { + bail!("no fal.ai API key configured"); + } + let owner = app.session(&ctx.session_id).ok().and_then(|s| s.owner_id); + crate::budget::check(&app, "fal", owner.as_deref())?; + let prompt = input["prompt"].as_str().map(str::trim).filter(|p| !p.is_empty()).map(String::from); + let dir_arg = input["directory"].as_str().map(String::from).unwrap_or_else(|| format!("{}/3d", cfg.images.output_dir.trim_end_matches('/'))); + let dir = ctx.resolve(&dir_arg)?; + let image = match input["image"].as_str() { + Some(spec) => Some(image_arg(ctx, spec, &dir).await?), + None => None, + }; + if prompt.is_none() && image.is_none() { + bail!("give an image of the object, a prompt, or both"); + } + let style = input["style"].as_str().filter(|s| STYLES.contains(s)).unwrap_or("textured"); + let pbr = input["pbr"].as_bool().unwrap_or(false); + let faces = input["face_count"].as_u64(); + let mut notes = Vec::new(); + + // Image + prompt: refine the image with the prompt first (3D models take no text). + let mut front = image.clone(); + if let (Some(img), Some(p)) = (&image, &prompt) { + ctx.progress("refining the image with the prompt"); + let edit = crate::images::choose_model(&app, "final", Some("illustration"), true); + if edit.reference_param.is_some() { + let records = crate::images::generate( + &app, + crate::images::GenerateRequest { + prompt: format!( + "{p}. Show exactly one object, complete and centered, as a clean product shot: plain light neutral background, soft even lighting, no text, no other objects." + ), + model: Some(edit.id.clone()), + size: None, + count: 1, + dir: dir.clone(), + user_id: owner.clone(), + session_id: Some(ctx.session_id.clone()), + references: vec![img.clone()], + }, + ) + .await + .map_err(|e| anyhow!("refining the image failed: {e:#}"))?; + if let Some(r) = records.first() { + front = Some(PathBuf::from(&r.path)); + notes.push(format!("Refined image: {}", rel(&r.path, &ctx.cwd))); + } + } else { + notes.push("No image edit model configured – the prompt was not applied to the image.".into()); + } + } + + let mut views = Views::default(); + if let Some(f) = &front { + views.front = crate::images::reference_uris(std::slice::from_ref(f)).await?.pop(); + for (field, slot) in [("back", &mut views.back), ("left", &mut views.left), ("right", &mut views.right)] { + if let Some(p) = input[field].as_str() { + *slot = crate::images::reference_uris(&[image_arg(ctx, p, &dir).await?]).await?.pop(); + } + } + } + let model = input["model"].as_str().filter(|m| !m.is_empty()).map(String::from).unwrap_or_else(|| if front.is_some() { cfg.images.model_3d.clone() } else { cfg.images.model_3d_text.clone() }); + if views.extra() > 0 && !model.contains("hunyuan") { + notes.push("This model takes only the front view; the other views were not used.".into()); + } + let text = if front.is_none() { prompt.as_deref() } else { None }; + let body = request_body(&model, text, &views, style, pbr, faces); + ctx.progress(format!("sending to {model}")); + let out = run_queued(&app, ctx, &model, &body).await?; + let (mesh_url, thumb_url) = result_urls(&out); + let mesh_url = mesh_url.ok_or_else(|| anyhow!("fal.ai returned no 3D model: {}", out.to_string().chars().take(300).collect::()))?; + + tokio::fs::create_dir_all(&dir).await?; + let stamp = chrono::Local::now().format("%Y%m%d-%H%M%S"); + // Named after the prompt, else the image (without a time stamp it already carries). + let stem = image.as_ref().and_then(|p| p.file_stem()).and_then(|s| s.to_str()).unwrap_or("model"); + let stem = if stem.len() > 16 && stem.as_bytes()[8] == b'-' && stem[..8].bytes().all(|b| b.is_ascii_digit()) { &stem[16..] } else { stem }; + let base = crate::images::slug(prompt.as_deref().unwrap_or(stem).trim_start_matches('-')); + let (bytes, _) = crate::images::download(&mesh_url).await?; + let mesh_path = dir.join(format!("{stamp}-{base}.{}", ext_of(&mesh_url))); + tokio::fs::write(&mesh_path, &bytes).await?; + let mut thumb_path = None; + if let Some(t) = thumb_url { + if let Ok((tb, _)) = crate::images::download(&t).await { + let p = dir.join(format!("{stamp}-{base}-preview.png")); + if tokio::fs::write(&p, &tb).await.is_ok() { + thumb_path = Some(p); + } + } + } + + // Cost into the statistics (provider "fal", like images). + let cost = price(&app, &model, style, pbr, views.extra() > 0 && model.contains("hunyuan"), faces.is_some()); + let session = ctx.session_id.clone(); + let _ = app.store.add_usage(&UsageRecord { + ts: now_ms(), + session_id: session.clone(), + root_session_id: app.root_session_id(&session), + provider: "fal".into(), + model: model.clone(), + agent: "3d".into(), + input_tokens: 0, + output_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + ttft_ms: 0, + duration_ms: 0, + cost, + error: false, + user_id: owner, + allowance: None, + }); + + let mesh_rel = rel(&mesh_path.display().to_string(), &ctx.cwd); + let mut text = format!("3D model ({model}, {style}{}): {mesh_rel} ({} KB)", if pbr && style != "geometry" { ", PBR" } else { "" }, bytes.len() / 1024); + if let Some(t) = &thumb_path { + text.push_str(&format!("\nPreview: {}", rel(&t.display().to_string(), &ctx.cwd))); + } + if cost > 0.0 { + text.push_str(&format!("\nCost: about ${cost:.2}")); + } + for n in ¬es { + text.push_str(&format!("\n{n}")); + } + text.push_str("\nThe user can open the model in the 3D view of the file manager."); + let images = thumb_path.as_deref().and_then(crate::images::preview_for_model).into_iter().collect(); + Ok(ToolOutput { + content: text, + images, + meta: Some(json!({ + "model_file": mesh_path.display().to_string(), + "preview": thumb_path.map(|p| p.display().to_string()), + })), + ..Default::default() + }) + } +} + +fn rel(path: &str, cwd: &Path) -> String { + Path::new(path).strip_prefix(cwd).map(|p| p.display().to_string()).unwrap_or_else(|_| path.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bodies_per_model_family() { + let views = Views { front: Some("data:a".into()), back: Some("data:b".into()), ..Default::default() }; + let h = request_body("fal-ai/hunyuan3d-v3/image-to-3d", None, &views, "geometry", true, Some(10)); + assert_eq!(h["input_image_url"], "data:a"); + assert_eq!(h["back_image_url"], "data:b"); + assert_eq!(h["generate_type"], "Geometry"); + assert_eq!(h["enable_pbr"], false); + assert_eq!(h["face_count"], 40_000); + let t = request_body("tripo3d/h3.1/image-to-3d", None, &views, "textured", true, None); + assert_eq!((t["image_url"].as_str(), t["texture"].as_bool(), t["pbr"].as_bool()), (Some("data:a"), Some(true), Some(true))); + let txt = request_body("fal-ai/hunyuan3d-v3/text-to-3d", Some("a chair"), &Views::default(), "textured", false, None); + assert_eq!((txt["prompt"].as_str(), txt["generate_type"].as_str()), (Some("a chair"), Some("Normal"))); + } + + #[test] + fn result_urls_of_the_families() { + let h = json!({"model_glb": {"url": "https://x/m.glb"}, "thumbnail": {"url": "https://x/t.png"}}); + assert_eq!(result_urls(&h), (Some("https://x/m.glb".into()), Some("https://x/t.png".into()))); + let t = json!({"model_mesh": {"url": "https://x/m.glb"}, "rendered_image": {"url": "https://x/r.webp"}}); + assert_eq!(result_urls(&t).1.as_deref(), Some("https://x/r.webp")); + assert_eq!(ext_of("https://x/a.FBX?sig=1"), "fbx"); + assert_eq!(ext_of("https://x/blob"), "glb"); + } +} diff --git a/server/src/runner.rs b/server/src/runner.rs index ebc2955..09de854 100644 --- a/server/src/runner.rs +++ b/server/src/runner.rs @@ -215,7 +215,7 @@ pub fn available_tools(app: &App, agent: &Agent, owner: Option<&crate::store::Us let cfg = app.config(); let mut all = tools::builtin_tools(); if !cfg.images.enabled { - all.retain(|t| t.name() != "generate_image"); + all.retain(|t| t.name() != "generate_image" && t.name() != "generate_3d"); } let (direct, deferred) = app.mcp.tools(&cfg, owner); all.extend(direct); diff --git a/server/src/tools/mod.rs b/server/src/tools/mod.rs index 06b38a3..5548f60 100644 --- a/server/src/tools/mod.rs +++ b/server/src/tools/mod.rs @@ -177,6 +177,7 @@ pub fn builtin_tools() -> Vec> { Arc::new(craft::ImageCompare), Arc::new(crate::prompts::SkillTool), Arc::new(crate::images::GenerateImage), + Arc::new(crate::mesh3d::Generate3d), Arc::new(crate::reverse::BinaryInfo), Arc::new(crate::reverse::Decompile), ] diff --git a/web/src/components/MediaGallery.tsx b/web/src/components/MediaGallery.tsx index 96ab0f5..d5f6822 100644 --- a/web/src/components/MediaGallery.tsx +++ b/web/src/components/MediaGallery.tsx @@ -3,14 +3,24 @@ import { get } from "../api"; import { useT, type Key } from "../i18n"; import { Spinner } from "./Activity"; import { Icon } from "./Icon"; +import { navigate } from "../router"; + +/** Open a 3D model in the 3D view of the file manager. */ +const open3d = (path: string) => { + sessionStorage.setItem("codecraft.openFile", path); + navigate("/files"); +}; interface MediaItem { - kind: "image" | "video" | "audio"; + kind: "image" | "video" | "audio" | "model"; url: string; name: string; source: "generated" | "tool" | "written" | "answer" | "upload"; created_at: number; message_id?: string; + /** 3D models: file (opened in the 3D view) and preview image. */ + path?: string; + preview?: string; } type Filter = "all" | MediaItem["kind"]; @@ -55,7 +65,7 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string {t("media.title")}
- {(["all", "image", "video", "audio"] as Filter[]).map((k) => ( + {(["all", "image", "video", "audio", "model"] as Filter[]).map((k) => ( @@ -78,6 +88,16 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string {current.kind === "image" && {current.name}} {current.kind === "video" &&
))}
diff --git a/web/src/components/ToolCard.tsx b/web/src/components/ToolCard.tsx index a41898d..c5baedf 100644 --- a/web/src/components/ToolCard.tsx +++ b/web/src/components/ToolCard.tsx @@ -121,7 +121,7 @@ export function ToolCard({ call, result, running, baseDir }: { call: ToolCall; r const diffLines = typeof meta?.diff === "string" ? meta.diff.split("\n").length : 0; const openByDefault = (status === "err" && !!result) || - (!!result && !result.is_error && ((diffLines > 0 && diffLines <= 40) || call.name === "generate_image")); + (!!result && !result.is_error && ((diffLines > 0 && diffLines <= 40) || call.name === "generate_image" || call.name === "generate_3d")); const open = toggled ?? openByDefault; return ( @@ -164,6 +164,18 @@ export function ToolCard({ call, result, running, baseDir }: { call: ToolCall; r 
{result.content}
)} + {typeof meta?.model_file === "string" && ( + + )} {Array.isArray(meta?.image_ids) ? (
{meta.image_ids.map((id: string, i: number) => { diff --git a/web/src/i18n/de.ts b/web/src/i18n/de.ts index ac8644e..6afb50a 100644 --- a/web/src/i18n/de.ts +++ b/web/src/i18n/de.ts @@ -356,6 +356,7 @@ export const de: Messages = { "media.filter.all": "Alle", "media.filter.image": "Bilder", "media.filter.video": "Videos", + "media.filter.model": "3D", "media.filter.audio": "Audio", "media.source.generated": "generiert", "media.source.tool": "Tool-Ausgabe", @@ -713,6 +714,7 @@ export const de: Messages = { "plan.implementPrompt": "Setze den Plan um.", "tool.input": "Eingabe", + "tool.open3d": "In 3D-Ansicht öffnen", "tool.output": "Ausgabe", "tool.todosUpdated": "Aufgaben aktualisiert ({done}/{total} erledigt)", "tool.todos": "Aufgaben", diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 75ae219..0fa263d 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -355,6 +355,7 @@ export const en = { "media.filter.all": "All", "media.filter.image": "Images", "media.filter.video": "Videos", + "media.filter.model": "3D", "media.filter.audio": "Audio", "media.source.generated": "generated", "media.source.tool": "tool output", @@ -712,6 +713,7 @@ export const en = { "plan.implementPrompt": "Implement the plan.", "tool.input": "Input", + "tool.open3d": "Open in the 3D view", "tool.output": "Output", "tool.todosUpdated": "Tasks updated ({done}/{total} done)", "tool.todos": "Tasks", diff --git a/web/src/styles.css b/web/src/styles.css index 63a9a70..acf84a5 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -4410,3 +4410,32 @@ span.chat-tab > button { color: var(--muted); font-variant-numeric: tabular-nums; } + +.media-badge { + position: absolute; + top: 6px; + left: 6px; + font-size: 11px; + font-weight: 600; + padding: 1px 6px; + border-radius: 4px; + background: var(--bg0); + color: var(--accent); +} +.media-model { + display: flex; + flex-direction: column; + align-items: center; + gap: 12px; + max-height: 100%; +} +.media-model img { + max-height: calc(100% - 50px); + max-width: 100%; + object-fit: contain; +} +.media-model > .icon { + width: 64px; + height: 64px; + color: var(--accent); +} diff --git a/web/src/whatsnew.ts b/web/src/whatsnew.ts index 094a6b8..69f700d 100644 --- a/web/src/whatsnew.ts +++ b/web/src/whatsnew.ts @@ -3,6 +3,10 @@ * under "unreleased"; scripts/release.sh renames the key to the release version. */ export const WHATS_NEW: Record = { + unreleased: { + en: ["3D models with fal.ai: from an image (also pasted into the chat), a text prompt or both – open them in the 3D view"], + de: ["3D-Modelle mit fal.ai: aus einem Bild (auch eingefügt), einem Text oder beidem – öffnen in der 3D-Ansicht"], + }, "202611-00023": { en: ["Tokens and cost under every answer; the total cost of chats with opencode Go is shown correctly again"], de: ["Tokens und Kosten unter jeder Antwort; der Gesamtpreis von Chats mit opencode Go stimmt wieder"],