generate_3d: 3D models with fal.ai (image, text, or image steered by text)

- New tool: image-to-3D (project file or an image attached to the chat,
  optional back/left/right views) or text-to-3D; image + prompt refines the
  image with the prompt first (the 3D models take no text with an image).
  Styles textured / geometry / lowpoly, optional PBR, face count. Runs through
  fal.ai's queue API with progress, saves GLB + preview, books the cost
  (published prices, overridable per model). Request bodies for the Hunyuan3D,
  Tripo and Trellis families; defaults [images] model_3d / model_3d_text.
- Chat card: "Open in the 3D view"; media overview: kind "model" (also
  STL/3MF/OBJ written by agents) with previews.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard ZinkandClaude Opus 5.5 committed 2026-11-12 11:38:24 +01:00
1 parent d7baa77922
commit 48621f21d6
15 files changed
+578 -14

No files matched your search

+26 -6
View File
@@ -11,7 +11,7 @@ use std::path::{Path, PathBuf};
#[derive(Debug, Serialize, PartialEq)]
pub struct MediaItem {
/// "image", "video" or "audio".
/// "image", "video", "audio" or "model" (3D).
pub kind: &'static str,
pub url: String,
pub name: String,
@@ -21,11 +21,17 @@ pub struct MediaItem {
/// Message to jump to (only for messages of the chat itself, not of subagents).
#[serde(skip_serializing_if = "Option::is_none")]
pub message_id: Option<String>,
/// 3D models: absolute path (opened in the 3D view) and a preview image.
#[serde(skip_serializing_if = "Option::is_none")]
pub path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub preview: Option<String>,
}
const IMAGE: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "avif"];
const VIDEO: &[&str] = &["mp4", "webm", "mov", "m4v", "ogv"];
const AUDIO: &[&str] = &["mp3", "wav", "ogg", "oga", "m4a", "flac", "opus", "aac"];
const MODEL: &[&str] = &["glb", "gltf", "stl", "3mf", "obj", "step", "stp"];
/// Kind of a media file by its extension.
pub fn kind_of(path: &str) -> Option<&'static str> {
@@ -36,6 +42,8 @@ pub fn kind_of(path: &str) -> Option<&'static str> {
Some("video")
} else if AUDIO.contains(&ext.as_str()) {
Some("audio")
} else if MODEL.contains(&ext.as_str()) {
Some("model")
} else {
None
}
@@ -118,7 +126,7 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result<Vec<MediaItem>> {
// Generated images whose tool result was not found (e.g. removed by a rewind).
for img in app.store.images_of_sessions(&sessions)? {
push(
MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None },
MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None, path: None, preview: None },
&mut out,
);
}
@@ -133,6 +141,10 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result<Vec<MediaItem>> {
Ok(out)
}
fn raw_url(p: &Path) -> String {
format!("/api/files/raw?path={}", encode(&p.display().to_string()))
}
fn file_name(p: &str) -> String {
Path::new(p).file_name().and_then(|n| n.to_str()).unwrap_or(p).to_string()
}
@@ -144,25 +156,33 @@ fn collect_message(m: &Message, cwd: &Path, own: bool, call_msg: Option<&str>, a
let file = |p: &str, source: &'static str| -> Option<MediaItem> {
let kind = kind_of(p)?;
let abs = resolve(p, cwd);
abs.is_file().then(|| MediaItem { kind, url: format!("/api/files/raw?path={}", encode(&abs.display().to_string())), name: file_name(p), source, created_at: at, message_id: mid.clone() })
abs.is_file().then(|| MediaItem { kind, url: raw_url(&abs), name: file_name(p), source, created_at: at, message_id: mid.clone(), path: Some(abs.display().to_string()), preview: None })
};
for p in &m.parts {
match p {
Part::Image { image } if m.role == Role::User => {
if let Some(h) = &image.blob {
add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone() });
add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone(), path: None, preview: None });
}
}
// generate_3d: the model with its preview (not the preview as a separate image).
Part::ToolResult { meta: Some(meta), .. } if meta["model_file"].is_string() => {
let file = PathBuf::from(meta["model_file"].as_str().unwrap_or_default());
if file.is_file() {
let preview = meta["preview"].as_str().map(PathBuf::from).filter(|p| p.is_file()).map(|p| raw_url(&p));
add(MediaItem { kind: "model", url: raw_url(&file), name: file_name(&file.display().to_string()), source: "generated", created_at: at, message_id: result_mid.clone(), path: Some(file.display().to_string()), preview });
}
}
Part::ToolResult { images, meta, .. } => {
let ids: Vec<&str> = meta.as_ref().and_then(|v| v["image_ids"].as_array()).map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default();
if !ids.is_empty() {
for id in ids {
add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone() });
add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone(), path: None, preview: None });
}
} else {
for img in images {
if let Some(h) = &img.blob {
add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone() });
add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone(), path: None, preview: None });
}
}
}