generate_3d: 3D models with fal.ai (image, text, or image steered by text)

- New tool: image-to-3D (project file or an image attached to the chat,
  optional back/left/right views) or text-to-3D; image + prompt refines the
  image with the prompt first (the 3D models take no text with an image).
  Styles textured / geometry / lowpoly, optional PBR, face count. Runs through
  fal.ai's queue API with progress, saves GLB + preview, books the cost
  (published prices, overridable per model). Request bodies for the Hunyuan3D,
  Tripo and Trellis families; defaults [images] model_3d / model_3d_text.
- Chat card: "Open in the 3D view"; media overview: kind "model" (also
  STL/3MF/OBJ written by agents) with previews.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard ZinkandClaude Opus 5.5 committed 2026-11-12 11:38:24 +01:00
1 parent d7baa77922
commit 48621f21d6
15 files changed
+578 -14

No files matched your search

+32 -2
View File
@@ -3,14 +3,24 @@ import { get } from "../api";
import { useT, type Key } from "../i18n";
import { Spinner } from "./Activity";
import { Icon } from "./Icon";
import { navigate } from "../router";
/** Open a 3D model in the 3D view of the file manager. */
const open3d = (path: string) => {
sessionStorage.setItem("codecraft.openFile", path);
navigate("/files");
};
interface MediaItem {
kind: "image" | "video" | "audio";
kind: "image" | "video" | "audio" | "model";
url: string;
name: string;
source: "generated" | "tool" | "written" | "answer" | "upload";
created_at: number;
message_id?: string;
/** 3D models: file (opened in the 3D view) and preview image. */
path?: string;
preview?: string;
}
type Filter = "all" | MediaItem["kind"];
@@ -55,7 +65,7 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string
<Icon name="image" /> {t("media.title")}
</h3>
<div className="media-filters" role="radiogroup">
{(["all", "image", "video", "audio"] as Filter[]).map((k) => (
{(["all", "image", "video", "audio", "model"] as Filter[]).map((k) => (
<button key={k} role="radio" aria-checked={filter === k} className={`chip-btn ${filter === k ? "active" : ""}`} onClick={() => (setFilter(k), setOpen(undefined))} disabled={!!items && count(k) === 0 && k !== "all"}>
{t(`media.filter.${k}` as Key)} {items && <small>{count(k)}</small>}
</button>
@@ -78,6 +88,16 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string
{current.kind === "image" && <img src={current.url} alt={current.name} />}
{current.kind === "video" && <video src={current.url} controls autoPlay />}
{current.kind === "audio" && <audio src={current.url} controls autoPlay />}
{current.kind === "model" && (
<div className="media-model">
{current.preview ? <img src={current.preview} alt={current.name} /> : <Icon name="cube" />}
{current.path && (
<button className="primary" onClick={() => open3d(current.path!)}>
<Icon name="cube" /> {t("tool.open3d")}
</button>
)}
</div>
)}
</div>
<div className="media-bar">
<button className="small ghost" disabled={open === 0} onClick={() => setOpen((o) => (o ?? 1) - 1)} aria-label={t("media.prev")}>
@@ -115,6 +135,16 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string
</span>
)}
{m.kind === "video" && <Icon name="play" className="media-play" />}
{m.kind === "model" &&
(m.preview ? (
<img src={m.preview} alt="" loading="lazy" decoding="async" />
) : (
<span className="media-audio">
<Icon name="cube" />
<span>{m.name}</span>
</span>
))}
{m.kind === "model" && <span className="media-badge">3D</span>}
</button>
))}
</div>
+13 -1
View File
@@ -121,7 +121,7 @@ export function ToolCard({ call, result, running, baseDir }: { call: ToolCall; r
const diffLines = typeof meta?.diff === "string" ? meta.diff.split("\n").length : 0;
const openByDefault =
(status === "err" && !!result) ||
(!!result && !result.is_error && ((diffLines > 0 && diffLines <= 40) || call.name === "generate_image"));
(!!result && !result.is_error && ((diffLines > 0 && diffLines <= 40) || call.name === "generate_image" || call.name === "generate_3d"));
const open = toggled ?? openByDefault;
return (
@@ -164,6 +164,18 @@ export function ToolCard({ call, result, running, baseDir }: { call: ToolCall; r
<pre className="mono">{result.content}</pre>
</>
)}
{typeof meta?.model_file === "string" && (
<button
className="small"
style={{ marginBottom: 8 }}
onClick={() => {
sessionStorage.setItem("codecraft.openFile", meta.model_file);
navigate("/files");
}}
>
<Icon name="cube" /> {t("tool.open3d")}
</button>
)}
{Array.isArray(meta?.image_ids) ? (
<div className="gallery small">
{meta.image_ids.map((id: string, i: number) => {
+2
View File
@@ -356,6 +356,7 @@ export const de: Messages = {
"media.filter.all": "Alle",
"media.filter.image": "Bilder",
"media.filter.video": "Videos",
"media.filter.model": "3D",
"media.filter.audio": "Audio",
"media.source.generated": "generiert",
"media.source.tool": "Tool-Ausgabe",
@@ -713,6 +714,7 @@ export const de: Messages = {
"plan.implementPrompt": "Setze den Plan um.",
"tool.input": "Eingabe",
"tool.open3d": "In 3D-Ansicht öffnen",
"tool.output": "Ausgabe",
"tool.todosUpdated": "Aufgaben aktualisiert ({done}/{total} erledigt)",
"tool.todos": "Aufgaben",
+2
View File
@@ -355,6 +355,7 @@ export const en = {
"media.filter.all": "All",
"media.filter.image": "Images",
"media.filter.video": "Videos",
"media.filter.model": "3D",
"media.filter.audio": "Audio",
"media.source.generated": "generated",
"media.source.tool": "tool output",
@@ -712,6 +713,7 @@ export const en = {
"plan.implementPrompt": "Implement the plan.",
"tool.input": "Input",
"tool.open3d": "Open in the 3D view",
"tool.output": "Output",
"tool.todosUpdated": "Tasks updated ({done}/{total} done)",
"tool.todos": "Tasks",
+29
View File
@@ -4410,3 +4410,32 @@ span.chat-tab > button {
color: var(--muted);
font-variant-numeric: tabular-nums;
}
.media-badge {
position: absolute;
top: 6px;
left: 6px;
font-size: 11px;
font-weight: 600;
padding: 1px 6px;
border-radius: 4px;
background: var(--bg0);
color: var(--accent);
}
.media-model {
display: flex;
flex-direction: column;
align-items: center;
gap: 12px;
max-height: 100%;
}
.media-model img {
max-height: calc(100% - 50px);
max-width: 100%;
object-fit: contain;
}
.media-model > .icon {
width: 64px;
height: 64px;
color: var(--accent);
}
+4
View File
@@ -3,6 +3,10 @@
* under "unreleased"; scripts/release.sh renames the key to the release version.
*/
export const WHATS_NEW: Record<string, { en: string[]; de: string[] }> = {
unreleased: {
en: ["3D models with fal.ai: from an image (also pasted into the chat), a text prompt or both – open them in the 3D view"],
de: ["3D-Modelle mit fal.ai: aus einem Bild (auch eingefügt), einem Text oder beidem – öffnen in der 3D-Ansicht"],
},
"202611-00023": {
en: ["Tokens and cost under every answer; the total cost of chats with opencode Go is shown correctly again"],
de: ["Tokens und Kosten unter jeder Antwort; der Gesamtpreis von Chats mit opencode Go stimmt wieder"],