generate_3d: 3D models with fal.ai (image, text, or image steered by text)
- New tool: image-to-3D (project file or an image attached to the chat, optional back/left/right views) or text-to-3D; image + prompt refines the image with the prompt first (the 3D models take no text with an image). Styles textured / geometry / lowpoly, optional PBR, face count. Runs through fal.ai's queue API with progress, saves GLB + preview, books the cost (published prices, overridable per model). Request bodies for the Hunyuan3D, Tripo and Trellis families; defaults [images] model_3d / model_3d_text. - Chat card: "Open in the 3D view"; media overview: kind "model" (also STL/3MF/OBJ written by agents) with previews. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d7baa77922
commit
48621f21d6
15 files changed
+578
-14
No files matched your search
+26
-6
@@ -11,7 +11,7 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct MediaItem {
|
||||
/// "image", "video" or "audio".
|
||||
/// "image", "video", "audio" or "model" (3D).
|
||||
pub kind: &'static str,
|
||||
pub url: String,
|
||||
pub name: String,
|
||||
@@ -21,11 +21,17 @@ pub struct MediaItem {
|
||||
/// Message to jump to (only for messages of the chat itself, not of subagents).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub message_id: Option<String>,
|
||||
/// 3D models: absolute path (opened in the 3D view) and a preview image.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub path: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub preview: Option<String>,
|
||||
}
|
||||
|
||||
const IMAGE: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "avif"];
|
||||
const VIDEO: &[&str] = &["mp4", "webm", "mov", "m4v", "ogv"];
|
||||
const AUDIO: &[&str] = &["mp3", "wav", "ogg", "oga", "m4a", "flac", "opus", "aac"];
|
||||
const MODEL: &[&str] = &["glb", "gltf", "stl", "3mf", "obj", "step", "stp"];
|
||||
|
||||
/// Kind of a media file by its extension.
|
||||
pub fn kind_of(path: &str) -> Option<&'static str> {
|
||||
@@ -36,6 +42,8 @@ pub fn kind_of(path: &str) -> Option<&'static str> {
|
||||
Some("video")
|
||||
} else if AUDIO.contains(&ext.as_str()) {
|
||||
Some("audio")
|
||||
} else if MODEL.contains(&ext.as_str()) {
|
||||
Some("model")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -118,7 +126,7 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result<Vec<MediaItem>> {
|
||||
// Generated images whose tool result was not found (e.g. removed by a rewind).
|
||||
for img in app.store.images_of_sessions(&sessions)? {
|
||||
push(
|
||||
MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None },
|
||||
MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None, path: None, preview: None },
|
||||
&mut out,
|
||||
);
|
||||
}
|
||||
@@ -133,6 +141,10 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result<Vec<MediaItem>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn raw_url(p: &Path) -> String {
|
||||
format!("/api/files/raw?path={}", encode(&p.display().to_string()))
|
||||
}
|
||||
|
||||
fn file_name(p: &str) -> String {
|
||||
Path::new(p).file_name().and_then(|n| n.to_str()).unwrap_or(p).to_string()
|
||||
}
|
||||
@@ -144,25 +156,33 @@ fn collect_message(m: &Message, cwd: &Path, own: bool, call_msg: Option<&str>, a
|
||||
let file = |p: &str, source: &'static str| -> Option<MediaItem> {
|
||||
let kind = kind_of(p)?;
|
||||
let abs = resolve(p, cwd);
|
||||
abs.is_file().then(|| MediaItem { kind, url: format!("/api/files/raw?path={}", encode(&abs.display().to_string())), name: file_name(p), source, created_at: at, message_id: mid.clone() })
|
||||
abs.is_file().then(|| MediaItem { kind, url: raw_url(&abs), name: file_name(p), source, created_at: at, message_id: mid.clone(), path: Some(abs.display().to_string()), preview: None })
|
||||
};
|
||||
for p in &m.parts {
|
||||
match p {
|
||||
Part::Image { image } if m.role == Role::User => {
|
||||
if let Some(h) = &image.blob {
|
||||
add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone() });
|
||||
add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone(), path: None, preview: None });
|
||||
}
|
||||
}
|
||||
// generate_3d: the model with its preview (not the preview as a separate image).
|
||||
Part::ToolResult { meta: Some(meta), .. } if meta["model_file"].is_string() => {
|
||||
let file = PathBuf::from(meta["model_file"].as_str().unwrap_or_default());
|
||||
if file.is_file() {
|
||||
let preview = meta["preview"].as_str().map(PathBuf::from).filter(|p| p.is_file()).map(|p| raw_url(&p));
|
||||
add(MediaItem { kind: "model", url: raw_url(&file), name: file_name(&file.display().to_string()), source: "generated", created_at: at, message_id: result_mid.clone(), path: Some(file.display().to_string()), preview });
|
||||
}
|
||||
}
|
||||
Part::ToolResult { images, meta, .. } => {
|
||||
let ids: Vec<&str> = meta.as_ref().and_then(|v| v["image_ids"].as_array()).map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default();
|
||||
if !ids.is_empty() {
|
||||
for id in ids {
|
||||
add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone() });
|
||||
add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone(), path: None, preview: None });
|
||||
}
|
||||
} else {
|
||||
for img in images {
|
||||
if let Some(h) = &img.blob {
|
||||
add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone() });
|
||||
add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone(), path: None, preview: None });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user