Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c3df4a23cc | ||
|
|
48621f21d6 | ||
|
|
d7baa77922 | ||
|
|
1e8418336d |
No files matched your search
@@ -3,6 +3,37 @@
|
||||
All notable changes to CodeCraft. Versions follow [Semantic Versioning](https://semver.org/);
|
||||
until 1.0, minor versions may contain breaking changes.
|
||||
|
||||
## [v202611-00024] – 2026-11-12
|
||||
|
||||
### Added
|
||||
|
||||
- **3D models with fal.ai (`generate_3d`):** from an image (a project file or one pasted into the
|
||||
chat, optional back/left/right views), from a text prompt, or from an image steered by a prompt
|
||||
(the image is refined with the prompt first – the 3D models take no text with an image).
|
||||
Styles textured, geometry (untextured, for 3D printing) and low-poly, optional PBR. Uses fal.ai's
|
||||
queue (takes minutes, with progress), saves a GLB plus a preview image, books the cost.
|
||||
Defaults: Hunyuan3D v3 image-to-3D / text-to-3D (`[images] model_3d`, `model_3d_text`); Tripo
|
||||
and Trellis endpoints work too.
|
||||
- The chat card offers "Open in the 3D view"; the media overview lists 3D models (also STL/3MF
|
||||
written by the agent) with their previews.
|
||||
|
||||
## [v202611-00023] – 2026-11-12
|
||||
|
||||
### Added
|
||||
|
||||
- **Tokens and cost under every answer:** a small line under each answer of the model shows the
|
||||
tokens sent/received and the cost of all requests of that turn.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Total cost of chats stopped growing (opencode Go):** opencode now reports plan requests as
|
||||
"nothing charged" (cost 0, usage in `allowance`), and CodeCraft took the 0. Plan requests are
|
||||
valued at the model's list prices again; earlier ones are corrected once on start.
|
||||
- **Saving opencode Go models failed after fetching prices** ("data did not match any variant of
|
||||
untagged enum ModelEntry"): opencode lists cache prices per duration for some models
|
||||
(`{"30m": …, "12h": …}`); the shortest duration is used now.
|
||||
- The chat title in the top bar no longer shrinks to a single letter when there are many chips.
|
||||
|
||||
## [v202611-00022] – 2026-11-11
|
||||
|
||||
### Added
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img alt="Version v202611-00022" src="https://img.shields.io/badge/version-v202611--00022-d69e2e?style=flat-square">
|
||||
<img alt="Version v202611-00024" src="https://img.shields.io/badge/version-v202611--00024-d69e2e?style=flat-square">
|
||||
<img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-689d6a?style=flat-square">
|
||||
<img alt="Rust backend" src="https://img.shields.io/badge/backend-Rust-b7410e?style=flat-square&logo=rust">
|
||||
<img alt="React PWA" src="https://img.shields.io/badge/frontend-React%20PWA-2d81ff?style=flat-square&logo=react">
|
||||
@@ -176,6 +176,10 @@ output limit, reasoning effort, extra request options and prices per million tok
|
||||
- **Images (fal.ai):** the agent drafts cheap previews (FLUX schnell), asks which one to
|
||||
refine, and creates the final image with the best model for the purpose – Nano Banana Pro
|
||||
refines the chosen draft, Ideogram handles lettering, Recraft design work.
|
||||
- **3D models (fal.ai):** `generate_3d` turns an image (also one pasted into the chat, plus
|
||||
optional back/left/right views), a text prompt, or an image steered by a prompt into a GLB
|
||||
model – textured, untextured for 3D printing, or low-poly (Hunyuan3D v3 by default,
|
||||
`[images] model_3d` / `model_3d_text`). Open it in the 3D view of the file manager.
|
||||
|
||||
## Terminal client
|
||||
|
||||
|
||||
+22
-1
@@ -159,7 +159,13 @@ impl ModelConfig {
|
||||
/// Cost of one request now. Cache prices default to the input price.
|
||||
/// The provider's own figure wins when it reports one (opencode).
|
||||
pub fn cost(&self, u: &crate::types::Usage) -> f64 {
|
||||
u.cost.unwrap_or_else(|| self.cost_at(u, chrono::Utc::now()))
|
||||
match u.cost {
|
||||
// A plan request (opencode Go): nothing charged, the usage counts toward the plan –
|
||||
// its value at the model's list prices.
|
||||
Some(c) if c == 0.0 && u.allowance.is_some_and(|a| a > 0.0) => self.cost_at(u, chrono::Utc::now()),
|
||||
Some(c) => c,
|
||||
None => self.cost_at(u, chrono::Utc::now()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cost of one request at a given time: context tiers and time factors applied.
|
||||
@@ -187,6 +193,21 @@ mod price_tests {
|
||||
Usage { input_tokens: input, output_tokens: output, cache_read_tokens: read, ..Default::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plan_requests_are_valued_at_list_prices() {
|
||||
let m = ModelConfig { input_price: Some(1.0), output_price: Some(2.0), ..Default::default() };
|
||||
let mut u = usage(1_000_000, 1_000_000, 0);
|
||||
u.cost = Some(0.0);
|
||||
u.allowance = Some(0.4);
|
||||
assert_eq!(m.cost(&u), 3.0);
|
||||
// Charged by the provider: that amount; free without a plan: free.
|
||||
u.cost = Some(0.7);
|
||||
assert_eq!(m.cost(&u), 0.7);
|
||||
u.cost = Some(0.0);
|
||||
u.allowance = None;
|
||||
assert_eq!(m.cost(&u), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiers_and_multipliers() {
|
||||
let m = ModelConfig {
|
||||
|
||||
+15
-2
@@ -843,6 +843,19 @@ impl WindowQ {
|
||||
}
|
||||
}
|
||||
|
||||
/// The cost of each answer for the chat view (usage.cost: charged, or valued at list prices).
|
||||
fn with_costs(app: &App, mut messages: Vec<crate::types::Message>) -> Vec<crate::types::Message> {
|
||||
let registry = app.registry();
|
||||
for m in &mut messages {
|
||||
if let (Some(u), Some(spec)) = (m.usage.as_mut(), m.model.as_deref()) {
|
||||
if let Ok((_, model)) = registry.resolve(spec) {
|
||||
u.cost = Some(model.cost(u));
|
||||
}
|
||||
}
|
||||
}
|
||||
messages
|
||||
}
|
||||
|
||||
async fn get_session(Extension(user): Extension<User>, State(app): State<AppState>, Path(id): Path<String>, Query(q): Query<WindowQ>) -> ApiResult {
|
||||
let s = readable(&app, &user, &id)?;
|
||||
let busy = app.is_busy(&id);
|
||||
@@ -853,7 +866,7 @@ async fn get_session(Extension(user): Extension<User>, State(app): State<AppStat
|
||||
match q.window() {
|
||||
Some(w) => {
|
||||
let (messages, older, newer) = app.store.messages_window(&id, &w)?;
|
||||
out["messages"] = json!(messages);
|
||||
out["messages"] = json!(with_costs(&app, messages));
|
||||
out["has_older"] = json!(older);
|
||||
out["has_newer"] = json!(newer);
|
||||
}
|
||||
@@ -867,7 +880,7 @@ async fn session_messages(Extension(user): Extension<User>, State(app): State<Ap
|
||||
readable(&app, &user, &id)?;
|
||||
let w = q.window().ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "before, after, around or limit is required".into()))?;
|
||||
let (messages, older, newer) = app.store.messages_window(&id, &w).map_err(|e| ApiError(StatusCode::NOT_FOUND, format!("{e:#}")))?;
|
||||
Ok(Json(json!({"messages": messages, "has_older": older, "has_newer": newer})))
|
||||
Ok(Json(json!({"messages": with_costs(&app, messages), "has_older": older, "has_newer": newer})))
|
||||
}
|
||||
|
||||
/// Images, videos and audio files shown in a chat (all of it, with subagents).
|
||||
|
||||
@@ -144,6 +144,11 @@ impl App {
|
||||
Ok(n) => tracing::info!("moved {n} images of older chats to the image store"),
|
||||
Err(e) => tracing::warn!("moving images to the image store: {e:#}"),
|
||||
}
|
||||
match crate::budget::value_plan_requests(&self) {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!("valued {n} plan requests (opencode Go) at list prices"),
|
||||
Err(e) => tracing::warn!("valuing plan requests: {e:#}"),
|
||||
}
|
||||
match crate::images::backfill_copies(&self) {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!("kept copies of {n} generated images in the image store"),
|
||||
|
||||
+71
-9
@@ -361,6 +361,49 @@ fn opencode_when(w: &Value) -> Option<When> {
|
||||
Some((days, hours, from, until))
|
||||
}
|
||||
|
||||
/// Plan requests (opencode Go) that were booked at 0: value them at the model's list
|
||||
/// prices of the time of the request. Idempotent – valued rows no longer match.
|
||||
pub fn value_plan_requests(app: &App) -> Result<usize> {
|
||||
let mut n = 0;
|
||||
for (row, ts, spec, usage) in app.store.plan_requests_without_cost()? {
|
||||
let Ok((_, model)) = app.registry().resolve(&spec) else { continue };
|
||||
let Some(at) = chrono::DateTime::from_timestamp_millis(ts) else { continue };
|
||||
let cost = model.cost_at(&usage, at);
|
||||
if cost > 0.0 {
|
||||
app.store.set_usage_cost(row, cost)?;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// A price from opencode's list: a number, or one price per cache duration
|
||||
/// (`{"30m": 0.125, "12h": 0.25}`) – then the shortest duration (the default caching).
|
||||
fn price_of(v: &Value) -> Value {
|
||||
match v {
|
||||
Value::Number(_) => v.clone(),
|
||||
Value::Object(o) => {
|
||||
let secs = |k: &str| -> Option<f64> {
|
||||
let (num, unit) = k.split_at(k.find(|c: char| c.is_ascii_alphabetic())?);
|
||||
let n: f64 = num.parse().ok()?;
|
||||
Some(n * match unit {
|
||||
"s" => 1.0,
|
||||
"m" => 60.0,
|
||||
"h" => 3600.0,
|
||||
"d" => 86400.0,
|
||||
_ => return None,
|
||||
})
|
||||
};
|
||||
o.iter()
|
||||
.filter(|(_, x)| x.is_number())
|
||||
.min_by(|a, b| secs(a.0).unwrap_or(f64::MAX).total_cmp(&secs(b.0).unwrap_or(f64::MAX)))
|
||||
.map(|(_, x)| x.clone())
|
||||
.unwrap_or(Value::Null)
|
||||
}
|
||||
_ => Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
/// Model row from opencode's model list (prices per 1M tokens, tiers, time factors).
|
||||
fn opencode_model(m: &Value) -> Value {
|
||||
let cost = &m["cost"]["text"];
|
||||
@@ -371,13 +414,13 @@ fn opencode_model(m: &Value) -> Value {
|
||||
for o in side["overrides"].as_array().into_iter().flatten() {
|
||||
let Some(gt) = o["when"]["context"]["gt"].as_u64() else { continue };
|
||||
let t = tiers.entry(gt).or_default();
|
||||
t.insert(field.into(), o["price"].clone());
|
||||
t.insert(field.into(), price_of(&o["price"]));
|
||||
if field == "input_price" {
|
||||
if o["cache"]["read"].is_number() {
|
||||
t.insert("cache_read_price".into(), o["cache"]["read"].clone());
|
||||
for (key, k) in [("read", "cache_read_price"), ("write", "cache_write_price")] {
|
||||
let p = price_of(&o["cache"][key]);
|
||||
if !p.is_null() {
|
||||
t.insert(k.into(), p);
|
||||
}
|
||||
if o["cache"]["write"].is_number() {
|
||||
t.insert("cache_write_price".into(), o["cache"]["write"].clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -412,10 +455,10 @@ fn opencode_model(m: &Value) -> Value {
|
||||
"max_input": m["limit"]["input"],
|
||||
"vision": images,
|
||||
"reasoning": m["capabilities"]["reasoning"]["supported"].as_bool().unwrap_or(false),
|
||||
"input_price": input["price"],
|
||||
"output_price": output["price"],
|
||||
"cache_read_price": input["cache"]["read"],
|
||||
"cache_write_price": input["cache"]["write"],
|
||||
"input_price": price_of(&input["price"]),
|
||||
"output_price": price_of(&output["price"]),
|
||||
"cache_read_price": price_of(&input["cache"]["read"]),
|
||||
"cache_write_price": price_of(&input["cache"]["write"]),
|
||||
"price_tiers": price_tiers,
|
||||
"price_multipliers": price_multipliers,
|
||||
"notes": notes,
|
||||
@@ -481,6 +524,25 @@ pub async fn discover_prices(app: &App, provider: &str) -> Result<Value> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn prices_per_cache_duration() {
|
||||
use super::{opencode_model, price_of};
|
||||
use serde_json::{Value, json};
|
||||
assert_eq!(price_of(&json!(0.5)), json!(0.5));
|
||||
assert_eq!(price_of(&json!({"12h": 0.25, "30m": 0.125})), json!(0.125));
|
||||
assert_eq!(price_of(&json!({"1h": 2.0, "5m": 1.25})), json!(1.25));
|
||||
assert_eq!(price_of(&Value::Null), Value::Null);
|
||||
// A model of opencode's list with per-duration cache write prices.
|
||||
let m = json!({"id": "gpt-6-luna", "limit": {"context": 1050000, "input": 922000, "output": 128000},
|
||||
"cost": {"text": {"input": {"price": 0.1, "cache": {"read": 0.01, "write": {"30m": 0.125, "12h": 0.25}},
|
||||
"overrides": [{"when": {"context": {"gt": 272000}}, "price": 0.2, "cache": {"read": 0.02, "write": {"30m": 0.25, "12h": 0.5}}}]},
|
||||
"output": {"price": 0.5, "overrides": [{"when": {"context": {"gt": 272000}}, "price": 0.75}]}}}});
|
||||
let row = opencode_model(&m);
|
||||
assert_eq!(row["cache_write_price"], json!(0.125));
|
||||
assert_eq!(row["price_tiers"][0]["cache_write_price"], json!(0.25));
|
||||
assert_eq!(row["max_input"], json!(922000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opencode_cost_conversion() {
|
||||
let m: serde_json::Value = serde_json::from_str(r#"{"id":"grok","limit":{"context":500000,"output":8000},
|
||||
|
||||
@@ -177,6 +177,19 @@ pub struct ImagesConfig {
|
||||
/// Optional fal.ai Admin key to query the account balance.
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub admin_key: String,
|
||||
/// fal.ai model for 3D models from images (generate_3d).
|
||||
#[serde(default = "default_model_3d")]
|
||||
pub model_3d: String,
|
||||
/// fal.ai model for 3D models from a text prompt only.
|
||||
#[serde(default = "default_model_3d_text")]
|
||||
pub model_3d_text: String,
|
||||
}
|
||||
|
||||
fn default_model_3d() -> String {
|
||||
"fal-ai/hunyuan3d-v3/image-to-3d".into()
|
||||
}
|
||||
fn default_model_3d_text() -> String {
|
||||
"fal-ai/hunyuan3d-v3/text-to-3d".into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default)]
|
||||
@@ -215,6 +228,8 @@ impl Default for ImagesConfig {
|
||||
models: vec![ImageModel { id: "fal-ai/flux/schnell".into(), name: Some("FLUX.1 schnell".into()), price: Some(0.003), ..Default::default() }],
|
||||
monthly_budget: None,
|
||||
admin_key: String::new(),
|
||||
model_3d: default_model_3d(),
|
||||
model_3d_text: default_model_3d_text(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ pub struct GenerateRequest {
|
||||
pub references: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
fn slug(s: &str) -> String {
|
||||
pub(crate) fn slug(s: &str) -> String {
|
||||
let mut out: String = s
|
||||
.chars()
|
||||
.map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' })
|
||||
@@ -121,7 +121,7 @@ fn pick(all: &[ImageModel], stage: &str, purpose: Option<&str>, with_reference:
|
||||
}
|
||||
|
||||
/// Read local reference images as data URIs (fal.ai accepts them in place of URLs).
|
||||
async fn reference_uris(paths: &[PathBuf]) -> Result<Vec<String>> {
|
||||
pub(crate) async fn reference_uris(paths: &[PathBuf]) -> Result<Vec<String>> {
|
||||
let mut out = Vec::new();
|
||||
for p in paths {
|
||||
let bytes = tokio::fs::read(p).await.with_context(|| format!("reference image {}", p.display()))?;
|
||||
@@ -135,7 +135,7 @@ async fn reference_uris(paths: &[PathBuf]) -> Result<Vec<String>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn download(url: &str) -> Result<(Vec<u8>, String)> {
|
||||
pub(crate) async fn download(url: &str) -> Result<(Vec<u8>, String)> {
|
||||
if let Some(rest) = url.strip_prefix("data:") {
|
||||
let (meta, data) = rest.split_once(',').context("malformed data URI")?;
|
||||
let ctype = meta.split(';').next().unwrap_or("image/png").to_string();
|
||||
@@ -348,7 +348,7 @@ pub fn backfill_copies(app: &App) -> Result<usize> {
|
||||
}
|
||||
|
||||
/// Downscaled copy for the model's context.
|
||||
fn preview_for_model(path: &Path) -> Option<ImageData> {
|
||||
pub(crate) fn preview_for_model(path: &Path) -> Option<ImageData> {
|
||||
let img = image::open(path).ok()?;
|
||||
let img = img.resize(1024, 1024, image::imageops::FilterType::Triangle);
|
||||
let mut buf = Vec::new();
|
||||
|
||||
@@ -23,6 +23,7 @@ mod mentions;
|
||||
mod metrics;
|
||||
mod mcp;
|
||||
mod media;
|
||||
mod mesh3d;
|
||||
mod permission;
|
||||
mod prompts;
|
||||
mod push;
|
||||
|
||||
+26
-6
@@ -11,7 +11,7 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
pub struct MediaItem {
|
||||
/// "image", "video" or "audio".
|
||||
/// "image", "video", "audio" or "model" (3D).
|
||||
pub kind: &'static str,
|
||||
pub url: String,
|
||||
pub name: String,
|
||||
@@ -21,11 +21,17 @@ pub struct MediaItem {
|
||||
/// Message to jump to (only for messages of the chat itself, not of subagents).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub message_id: Option<String>,
|
||||
/// 3D models: absolute path (opened in the 3D view) and a preview image.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub path: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub preview: Option<String>,
|
||||
}
|
||||
|
||||
const IMAGE: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "avif"];
|
||||
const VIDEO: &[&str] = &["mp4", "webm", "mov", "m4v", "ogv"];
|
||||
const AUDIO: &[&str] = &["mp3", "wav", "ogg", "oga", "m4a", "flac", "opus", "aac"];
|
||||
const MODEL: &[&str] = &["glb", "gltf", "stl", "3mf", "obj", "step", "stp"];
|
||||
|
||||
/// Kind of a media file by its extension.
|
||||
pub fn kind_of(path: &str) -> Option<&'static str> {
|
||||
@@ -36,6 +42,8 @@ pub fn kind_of(path: &str) -> Option<&'static str> {
|
||||
Some("video")
|
||||
} else if AUDIO.contains(&ext.as_str()) {
|
||||
Some("audio")
|
||||
} else if MODEL.contains(&ext.as_str()) {
|
||||
Some("model")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -118,7 +126,7 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result<Vec<MediaItem>> {
|
||||
// Generated images whose tool result was not found (e.g. removed by a rewind).
|
||||
for img in app.store.images_of_sessions(&sessions)? {
|
||||
push(
|
||||
MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None },
|
||||
MediaItem { kind: "image", url: format!("/api/images/{}/file", img.id), name: file_name(&img.path), source: "generated", created_at: img.created_at, message_id: None, path: None, preview: None },
|
||||
&mut out,
|
||||
);
|
||||
}
|
||||
@@ -133,6 +141,10 @@ pub fn collect(app: &App, session_id: &str) -> anyhow::Result<Vec<MediaItem>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn raw_url(p: &Path) -> String {
|
||||
format!("/api/files/raw?path={}", encode(&p.display().to_string()))
|
||||
}
|
||||
|
||||
fn file_name(p: &str) -> String {
|
||||
Path::new(p).file_name().and_then(|n| n.to_str()).unwrap_or(p).to_string()
|
||||
}
|
||||
@@ -144,25 +156,33 @@ fn collect_message(m: &Message, cwd: &Path, own: bool, call_msg: Option<&str>, a
|
||||
let file = |p: &str, source: &'static str| -> Option<MediaItem> {
|
||||
let kind = kind_of(p)?;
|
||||
let abs = resolve(p, cwd);
|
||||
abs.is_file().then(|| MediaItem { kind, url: format!("/api/files/raw?path={}", encode(&abs.display().to_string())), name: file_name(p), source, created_at: at, message_id: mid.clone() })
|
||||
abs.is_file().then(|| MediaItem { kind, url: raw_url(&abs), name: file_name(p), source, created_at: at, message_id: mid.clone(), path: Some(abs.display().to_string()), preview: None })
|
||||
};
|
||||
for p in &m.parts {
|
||||
match p {
|
||||
Part::Image { image } if m.role == Role::User => {
|
||||
if let Some(h) = &image.blob {
|
||||
add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone() });
|
||||
add(MediaItem { kind: kind_of_mime(&image.media_type), url: format!("/api/blobs/{h}"), name: "image".into(), source: "upload", created_at: at, message_id: mid.clone(), path: None, preview: None });
|
||||
}
|
||||
}
|
||||
// generate_3d: the model with its preview (not the preview as a separate image).
|
||||
Part::ToolResult { meta: Some(meta), .. } if meta["model_file"].is_string() => {
|
||||
let file = PathBuf::from(meta["model_file"].as_str().unwrap_or_default());
|
||||
if file.is_file() {
|
||||
let preview = meta["preview"].as_str().map(PathBuf::from).filter(|p| p.is_file()).map(|p| raw_url(&p));
|
||||
add(MediaItem { kind: "model", url: raw_url(&file), name: file_name(&file.display().to_string()), source: "generated", created_at: at, message_id: result_mid.clone(), path: Some(file.display().to_string()), preview });
|
||||
}
|
||||
}
|
||||
Part::ToolResult { images, meta, .. } => {
|
||||
let ids: Vec<&str> = meta.as_ref().and_then(|v| v["image_ids"].as_array()).map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default();
|
||||
if !ids.is_empty() {
|
||||
for id in ids {
|
||||
add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone() });
|
||||
add(MediaItem { kind: "image", url: format!("/api/images/{id}/file"), name: "image".into(), source: "generated", created_at: at, message_id: result_mid.clone(), path: None, preview: None });
|
||||
}
|
||||
} else {
|
||||
for img in images {
|
||||
if let Some(h) = &img.blob {
|
||||
add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone() });
|
||||
add(MediaItem { kind: kind_of_mime(&img.media_type), url: format!("/api/blobs/{h}"), name: "screenshot".into(), source: "tool", created_at: at, message_id: result_mid.clone(), path: None, preview: None });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
//! 3D models with fal.ai (generate_3d): from an image (plus optional back/left/right
|
||||
//! views), from a text prompt, or from an image steered by text – the 3D models take no
|
||||
//! text with an image, so the image is first refined with the prompt (image edit model)
|
||||
//! and then converted. The result is a GLB file (plus a preview image) in the project.
|
||||
|
||||
use crate::app::App;
|
||||
use crate::config::expand_env;
|
||||
use crate::store::UsageRecord;
|
||||
use crate::tools::{Tool, ToolCtx, ToolOutput};
|
||||
use crate::types::now_ms;
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
const STYLES: &[&str] = &["textured", "geometry", "lowpoly"];
|
||||
|
||||
/// Request body for a fal.ai 3D endpoint (the families name their fields differently).
|
||||
fn request_body(model: &str, prompt: Option<&str>, images: &Views, style: &str, pbr: bool, faces: Option<u64>) -> Value {
|
||||
let mut b = json!({});
|
||||
if model.contains("tripo") {
|
||||
// Tripo: image_url / prompt, texture + pbr flags, face_limit.
|
||||
if let Some(front) = &images.front {
|
||||
b["image_url"] = json!(front);
|
||||
}
|
||||
if let Some(p) = prompt {
|
||||
b["prompt"] = json!(p);
|
||||
}
|
||||
b["texture"] = json!(style != "geometry");
|
||||
b["pbr"] = json!(style != "geometry" && pbr);
|
||||
if let Some(f) = faces {
|
||||
b["face_limit"] = json!(f);
|
||||
}
|
||||
if style == "lowpoly" {
|
||||
b["face_limit"] = json!(faces.unwrap_or(20_000));
|
||||
}
|
||||
} else if model.contains("trellis") {
|
||||
if let Some(front) = &images.front {
|
||||
b["image_url"] = json!(front);
|
||||
}
|
||||
} else {
|
||||
// Hunyuan3D: front + optional views, generate_type, enable_pbr, face_count.
|
||||
if let Some(front) = &images.front {
|
||||
b["input_image_url"] = json!(front);
|
||||
}
|
||||
for (k, v) in [("back_image_url", &images.back), ("left_image_url", &images.left), ("right_image_url", &images.right)] {
|
||||
if let Some(v) = v {
|
||||
b[k] = json!(v);
|
||||
}
|
||||
}
|
||||
if let Some(p) = prompt {
|
||||
b["prompt"] = json!(p);
|
||||
}
|
||||
b["generate_type"] = json!(match style {
|
||||
"geometry" => "Geometry",
|
||||
"lowpoly" => "LowPoly",
|
||||
_ => "Normal",
|
||||
});
|
||||
b["enable_pbr"] = json!(style != "geometry" && pbr);
|
||||
if let Some(f) = faces {
|
||||
b["face_count"] = json!(f.clamp(40_000, 1_500_000));
|
||||
}
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// Data URIs of the views (front required for image-to-3D).
|
||||
#[derive(Default)]
|
||||
struct Views {
|
||||
front: Option<String>,
|
||||
back: Option<String>,
|
||||
left: Option<String>,
|
||||
right: Option<String>,
|
||||
}
|
||||
|
||||
impl Views {
|
||||
fn extra(&self) -> usize {
|
||||
[&self.back, &self.left, &self.right].iter().filter(|v| v.is_some()).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// Price of one generation (USD) by the published fal.ai prices; a price configured for
|
||||
/// the model under [images] models wins.
|
||||
fn price(app: &App, model: &str, style: &str, pbr: bool, multiview: bool, faces: bool) -> f64 {
|
||||
if let Some(p) = app.config().images.models.iter().find(|m| m.id == model).and_then(|m| m.price) {
|
||||
return p;
|
||||
}
|
||||
if model.contains("tripo") {
|
||||
return match style {
|
||||
"geometry" => 0.20,
|
||||
_ => 0.30,
|
||||
};
|
||||
}
|
||||
if model.contains("hunyuan") {
|
||||
let base = match style {
|
||||
"geometry" => 0.225,
|
||||
"lowpoly" => 0.45,
|
||||
_ => 0.375,
|
||||
};
|
||||
let extras = [pbr && style != "geometry", multiview, faces].iter().filter(|x| **x).count() as f64;
|
||||
return base + 0.15 * extras;
|
||||
}
|
||||
0.0
|
||||
}
|
||||
|
||||
/// URL of the queue API for fal.ai (requests that take minutes).
|
||||
fn queue_base(app: &App) -> String {
|
||||
let base = expand_env(&app.config().images.base_url);
|
||||
if base.contains("://fal.run") { "https://queue.fal.run".into() } else { base.trim_end_matches('/').to_string() }
|
||||
}
|
||||
|
||||
/// Submit to fal.ai's queue and wait for the result, reporting progress.
|
||||
async fn run_queued(app: &App, ctx: &ToolCtx, model: &str, body: &Value) -> Result<Value> {
|
||||
let key = expand_env(&app.config().images.api_key);
|
||||
let client = crate::provider::http_client();
|
||||
let submit = client
|
||||
.post(format!("{}/{}", queue_base(app), model.trim_matches('/')))
|
||||
.header("Authorization", format!("Key {key}"))
|
||||
.timeout(Duration::from_secs(120))
|
||||
.json(body)
|
||||
.send()
|
||||
.await
|
||||
.context("fal.ai request failed")?;
|
||||
if !submit.status().is_success() {
|
||||
return Err(crate::provider::http_error(submit).await);
|
||||
}
|
||||
let sub: Value = submit.json().await?;
|
||||
let status_url = sub["status_url"].as_str().context("fal.ai: no status_url")?.to_string();
|
||||
let response_url = sub["response_url"].as_str().context("fal.ai: no response_url")?.to_string();
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
if ctx.cancel.is_cancelled() {
|
||||
bail!("cancelled");
|
||||
}
|
||||
if started.elapsed() > Duration::from_secs(20 * 60) {
|
||||
bail!("fal.ai did not finish within 20 minutes");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(4)).await;
|
||||
let st: Value = client.get(&status_url).header("Authorization", format!("Key {key}")).timeout(Duration::from_secs(30)).send().await?.json().await?;
|
||||
match st["status"].as_str().unwrap_or("") {
|
||||
"COMPLETED" => break,
|
||||
"IN_QUEUE" => ctx.progress(format!("waiting in fal.ai's queue (position {}) – {} s", st["queue_position"].as_u64().unwrap_or(0), started.elapsed().as_secs())),
|
||||
"IN_PROGRESS" => ctx.progress(format!("generating the 3D model – {} s", started.elapsed().as_secs())),
|
||||
other => bail!("fal.ai: unexpected status {other}: {st}"),
|
||||
}
|
||||
}
|
||||
let resp = client.get(&response_url).header("Authorization", format!("Key {key}")).timeout(Duration::from_secs(60)).send().await?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(crate::provider::http_error(resp).await);
|
||||
}
|
||||
Ok(resp.json().await?)
|
||||
}
|
||||
|
||||
/// URL of the mesh and of the preview image in a fal.ai 3D result.
|
||||
fn result_urls(out: &Value) -> (Option<String>, Option<String>) {
|
||||
let s = |v: &Value| v.as_str().map(String::from);
|
||||
let mesh = s(&out["model_glb"]["url"])
|
||||
.or_else(|| s(&out["model_mesh"]["url"]))
|
||||
.or_else(|| s(&out["model_urls"]["glb"]["url"]))
|
||||
.or_else(|| s(&out["model_file"]["url"]))
|
||||
.or_else(|| s(&out["mesh"]["url"]));
|
||||
let thumb = s(&out["thumbnail"]["url"]).or_else(|| s(&out["rendered_image"]["url"]));
|
||||
(mesh, thumb)
|
||||
}
|
||||
|
||||
/// File extension of a downloaded mesh: from the URL, else GLB (what the models return).
|
||||
fn ext_of(url: &str) -> &'static str {
|
||||
let path = url.split(['?', '#']).next().unwrap_or(url).to_ascii_lowercase();
|
||||
["glb", "gltf", "fbx", "obj", "usdz", "stl"].into_iter().find(|e| path.ends_with(&format!(".{e}"))).unwrap_or("glb")
|
||||
}
|
||||
|
||||
/// An image the user attached to the chat ("attached" = the first image of the latest
|
||||
/// message with images, "attached:2" = its second), written to `dir` so it can be used
|
||||
/// like a project file.
|
||||
async fn attached_image(app: &App, session_id: &str, spec: &str, dir: &Path) -> Result<PathBuf> {
|
||||
let n: usize = spec.strip_prefix("attached").map(|r| r.trim_start_matches(':').trim()).filter(|r| !r.is_empty()).map(|r| r.parse()).transpose().map_err(|_| anyhow!("use \"attached\" or \"attached:2\""))?.unwrap_or(1);
|
||||
let root = app.root_session_id(session_id);
|
||||
let messages = app.store.messages(&root)?;
|
||||
let images: Vec<crate::types::ImageData> = messages
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|m| m.role == crate::types::Role::User)
|
||||
.map(|m| m.parts.iter().filter_map(|p| if let crate::types::Part::Image { image } = p { Some(image.clone()) } else { None }).collect::<Vec<_>>())
|
||||
.find(|imgs| !imgs.is_empty())
|
||||
.ok_or_else(|| anyhow!("no image attached to this chat"))?;
|
||||
let img = images.get(n.saturating_sub(1)).ok_or_else(|| anyhow!("the latest message has only {} image(s)", images.len()))?;
|
||||
let (mime, bytes) = match &img.blob {
|
||||
Some(h) => app.store.blob(h)?.ok_or_else(|| anyhow!("attached image not found"))?,
|
||||
None => (img.media_type.clone(), base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img.data.as_bytes())?),
|
||||
};
|
||||
let ext = match mime.as_str() {
|
||||
"image/jpeg" => "jpg",
|
||||
"image/webp" => "webp",
|
||||
_ => "png",
|
||||
};
|
||||
tokio::fs::create_dir_all(dir).await?;
|
||||
let path = dir.join(format!("{}-attached-{n}.{ext}", chrono::Local::now().format("%Y%m%d-%H%M%S")));
|
||||
tokio::fs::write(&path, &bytes).await?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// An image argument: a project file, or an image attached to the chat.
|
||||
async fn image_arg(ctx: &ToolCtx, spec: &str, dir: &Path) -> Result<PathBuf> {
|
||||
if spec.starts_with("attached") {
|
||||
attached_image(&ctx.app, &ctx.session_id, spec, dir).await
|
||||
} else {
|
||||
ctx.resolve(spec)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Generate3d;
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for Generate3d {
|
||||
fn name(&self) -> &str {
|
||||
"generate_3d"
|
||||
}
|
||||
fn description(&self) -> String {
|
||||
"Generate a 3D model (GLB) with fal.ai and save it in the project – from an image of the \
|
||||
object (a file, or image: \"attached\" for an image the user pasted into the chat; best: a \
|
||||
clean front view; optional back/left/right views improve the shape), from a \
|
||||
text prompt alone, or from an image steered by a prompt (the image is refined with the \
|
||||
prompt first, then converted). style: \"textured\" (default), \"geometry\" (untextured – for \
|
||||
3D printing), \"lowpoly\" (games). Takes one to several minutes and costs money (about \
|
||||
$0.25–0.75): generate only what the user asked for. The user can open the model in the 3D \
|
||||
view of the file manager."
|
||||
.into()
|
||||
}
|
||||
fn schema(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {"type": "string", "description": "What the model should be (English). With an image: how to change/clean it up before the conversion."},
|
||||
"image": {"type": "string", "description": "Image of the object (front view): a file path, or \"attached\" for the image the user attached to the chat (\"attached:2\" = the second of that message)"},
|
||||
"back": {"type": "string", "description": "Optional back view image (path or \"attached:N\")"},
|
||||
"left": {"type": "string", "description": "Optional left view image (path or \"attached:N\")"},
|
||||
"right": {"type": "string", "description": "Optional right view image (path or \"attached:N\")"},
|
||||
"style": {"type": "string", "enum": STYLES},
|
||||
"pbr": {"type": "boolean", "description": "PBR materials (metal/roughness), textured models only; costs extra"},
|
||||
"face_count": {"type": "integer", "description": "Target number of faces (optional)"},
|
||||
"directory": {"type": "string", "description": "Target directory relative to the working directory (default: <images.output_dir>/3d)"},
|
||||
"model": {"type": "string", "description": "fal.ai model id – only if the user asks for a specific one"}
|
||||
}
|
||||
})
|
||||
}
|
||||
fn title(&self, input: &Value) -> String {
|
||||
let what = input["prompt"].as_str().map(|p| p.chars().take(70).collect::<String>()).or_else(|| input["image"].as_str().map(String::from)).unwrap_or_default();
|
||||
format!("3D model: {what}")
|
||||
}
|
||||
fn read_only(&self) -> bool {
|
||||
false
|
||||
}
|
||||
async fn run(&self, input: Value, ctx: &ToolCtx) -> Result<ToolOutput> {
|
||||
let app = ctx.app.clone();
|
||||
let cfg = app.config();
|
||||
if !cfg.images.enabled {
|
||||
bail!("fal.ai is disabled – enable image generation in the settings");
|
||||
}
|
||||
if expand_env(&cfg.images.api_key).is_empty() {
|
||||
bail!("no fal.ai API key configured");
|
||||
}
|
||||
let owner = app.session(&ctx.session_id).ok().and_then(|s| s.owner_id);
|
||||
crate::budget::check(&app, "fal", owner.as_deref())?;
|
||||
let prompt = input["prompt"].as_str().map(str::trim).filter(|p| !p.is_empty()).map(String::from);
|
||||
let dir_arg = input["directory"].as_str().map(String::from).unwrap_or_else(|| format!("{}/3d", cfg.images.output_dir.trim_end_matches('/')));
|
||||
let dir = ctx.resolve(&dir_arg)?;
|
||||
let image = match input["image"].as_str() {
|
||||
Some(spec) => Some(image_arg(ctx, spec, &dir).await?),
|
||||
None => None,
|
||||
};
|
||||
if prompt.is_none() && image.is_none() {
|
||||
bail!("give an image of the object, a prompt, or both");
|
||||
}
|
||||
let style = input["style"].as_str().filter(|s| STYLES.contains(s)).unwrap_or("textured");
|
||||
let pbr = input["pbr"].as_bool().unwrap_or(false);
|
||||
let faces = input["face_count"].as_u64();
|
||||
let mut notes = Vec::new();
|
||||
|
||||
// Image + prompt: refine the image with the prompt first (3D models take no text).
|
||||
let mut front = image.clone();
|
||||
if let (Some(img), Some(p)) = (&image, &prompt) {
|
||||
ctx.progress("refining the image with the prompt");
|
||||
let edit = crate::images::choose_model(&app, "final", Some("illustration"), true);
|
||||
if edit.reference_param.is_some() {
|
||||
let records = crate::images::generate(
|
||||
&app,
|
||||
crate::images::GenerateRequest {
|
||||
prompt: format!(
|
||||
"{p}. Show exactly one object, complete and centered, as a clean product shot: plain light neutral background, soft even lighting, no text, no other objects."
|
||||
),
|
||||
model: Some(edit.id.clone()),
|
||||
size: None,
|
||||
count: 1,
|
||||
dir: dir.clone(),
|
||||
user_id: owner.clone(),
|
||||
session_id: Some(ctx.session_id.clone()),
|
||||
references: vec![img.clone()],
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow!("refining the image failed: {e:#}"))?;
|
||||
if let Some(r) = records.first() {
|
||||
front = Some(PathBuf::from(&r.path));
|
||||
notes.push(format!("Refined image: {}", rel(&r.path, &ctx.cwd)));
|
||||
}
|
||||
} else {
|
||||
notes.push("No image edit model configured – the prompt was not applied to the image.".into());
|
||||
}
|
||||
}
|
||||
|
||||
let mut views = Views::default();
|
||||
if let Some(f) = &front {
|
||||
views.front = crate::images::reference_uris(std::slice::from_ref(f)).await?.pop();
|
||||
for (field, slot) in [("back", &mut views.back), ("left", &mut views.left), ("right", &mut views.right)] {
|
||||
if let Some(p) = input[field].as_str() {
|
||||
*slot = crate::images::reference_uris(&[image_arg(ctx, p, &dir).await?]).await?.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
let model = input["model"].as_str().filter(|m| !m.is_empty()).map(String::from).unwrap_or_else(|| if front.is_some() { cfg.images.model_3d.clone() } else { cfg.images.model_3d_text.clone() });
|
||||
if views.extra() > 0 && !model.contains("hunyuan") {
|
||||
notes.push("This model takes only the front view; the other views were not used.".into());
|
||||
}
|
||||
let text = if front.is_none() { prompt.as_deref() } else { None };
|
||||
let body = request_body(&model, text, &views, style, pbr, faces);
|
||||
ctx.progress(format!("sending to {model}"));
|
||||
let out = run_queued(&app, ctx, &model, &body).await?;
|
||||
let (mesh_url, thumb_url) = result_urls(&out);
|
||||
let mesh_url = mesh_url.ok_or_else(|| anyhow!("fal.ai returned no 3D model: {}", out.to_string().chars().take(300).collect::<String>()))?;
|
||||
|
||||
tokio::fs::create_dir_all(&dir).await?;
|
||||
let stamp = chrono::Local::now().format("%Y%m%d-%H%M%S");
|
||||
// Named after the prompt, else the image (without a time stamp it already carries).
|
||||
let stem = image.as_ref().and_then(|p| p.file_stem()).and_then(|s| s.to_str()).unwrap_or("model");
|
||||
let stem = if stem.len() > 16 && stem.as_bytes()[8] == b'-' && stem[..8].bytes().all(|b| b.is_ascii_digit()) { &stem[16..] } else { stem };
|
||||
let base = crate::images::slug(prompt.as_deref().unwrap_or(stem).trim_start_matches('-'));
|
||||
let (bytes, _) = crate::images::download(&mesh_url).await?;
|
||||
let mesh_path = dir.join(format!("{stamp}-{base}.{}", ext_of(&mesh_url)));
|
||||
tokio::fs::write(&mesh_path, &bytes).await?;
|
||||
let mut thumb_path = None;
|
||||
if let Some(t) = thumb_url {
|
||||
if let Ok((tb, _)) = crate::images::download(&t).await {
|
||||
let p = dir.join(format!("{stamp}-{base}-preview.png"));
|
||||
if tokio::fs::write(&p, &tb).await.is_ok() {
|
||||
thumb_path = Some(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cost into the statistics (provider "fal", like images).
|
||||
let cost = price(&app, &model, style, pbr, views.extra() > 0 && model.contains("hunyuan"), faces.is_some());
|
||||
let session = ctx.session_id.clone();
|
||||
let _ = app.store.add_usage(&UsageRecord {
|
||||
ts: now_ms(),
|
||||
session_id: session.clone(),
|
||||
root_session_id: app.root_session_id(&session),
|
||||
provider: "fal".into(),
|
||||
model: model.clone(),
|
||||
agent: "3d".into(),
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
ttft_ms: 0,
|
||||
duration_ms: 0,
|
||||
cost,
|
||||
error: false,
|
||||
user_id: owner,
|
||||
allowance: None,
|
||||
});
|
||||
|
||||
let mesh_rel = rel(&mesh_path.display().to_string(), &ctx.cwd);
|
||||
let mut text = format!("3D model ({model}, {style}{}): {mesh_rel} ({} KB)", if pbr && style != "geometry" { ", PBR" } else { "" }, bytes.len() / 1024);
|
||||
if let Some(t) = &thumb_path {
|
||||
text.push_str(&format!("\nPreview: {}", rel(&t.display().to_string(), &ctx.cwd)));
|
||||
}
|
||||
if cost > 0.0 {
|
||||
text.push_str(&format!("\nCost: about ${cost:.2}"));
|
||||
}
|
||||
for n in ¬es {
|
||||
text.push_str(&format!("\n{n}"));
|
||||
}
|
||||
text.push_str("\nThe user can open the model in the 3D view of the file manager.");
|
||||
let images = thumb_path.as_deref().and_then(crate::images::preview_for_model).into_iter().collect();
|
||||
Ok(ToolOutput {
|
||||
content: text,
|
||||
images,
|
||||
meta: Some(json!({
|
||||
"model_file": mesh_path.display().to_string(),
|
||||
"preview": thumb_path.map(|p| p.display().to_string()),
|
||||
})),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn rel(path: &str, cwd: &Path) -> String {
|
||||
Path::new(path).strip_prefix(cwd).map(|p| p.display().to_string()).unwrap_or_else(|_| path.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bodies_per_model_family() {
|
||||
let views = Views { front: Some("data:a".into()), back: Some("data:b".into()), ..Default::default() };
|
||||
let h = request_body("fal-ai/hunyuan3d-v3/image-to-3d", None, &views, "geometry", true, Some(10));
|
||||
assert_eq!(h["input_image_url"], "data:a");
|
||||
assert_eq!(h["back_image_url"], "data:b");
|
||||
assert_eq!(h["generate_type"], "Geometry");
|
||||
assert_eq!(h["enable_pbr"], false);
|
||||
assert_eq!(h["face_count"], 40_000);
|
||||
let t = request_body("tripo3d/h3.1/image-to-3d", None, &views, "textured", true, None);
|
||||
assert_eq!((t["image_url"].as_str(), t["texture"].as_bool(), t["pbr"].as_bool()), (Some("data:a"), Some(true), Some(true)));
|
||||
let txt = request_body("fal-ai/hunyuan3d-v3/text-to-3d", Some("a chair"), &Views::default(), "textured", false, None);
|
||||
assert_eq!((txt["prompt"].as_str(), txt["generate_type"].as_str()), (Some("a chair"), Some("Normal")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_urls_of_the_families() {
|
||||
let h = json!({"model_glb": {"url": "https://x/m.glb"}, "thumbnail": {"url": "https://x/t.png"}});
|
||||
assert_eq!(result_urls(&h), (Some("https://x/m.glb".into()), Some("https://x/t.png".into())));
|
||||
let t = json!({"model_mesh": {"url": "https://x/m.glb"}, "rendered_image": {"url": "https://x/r.webp"}});
|
||||
assert_eq!(result_urls(&t).1.as_deref(), Some("https://x/r.webp"));
|
||||
assert_eq!(ext_of("https://x/a.FBX?sig=1"), "fbx");
|
||||
assert_eq!(ext_of("https://x/blob"), "glb");
|
||||
}
|
||||
}
|
||||
@@ -215,7 +215,7 @@ pub fn available_tools(app: &App, agent: &Agent, owner: Option<&crate::store::Us
|
||||
let cfg = app.config();
|
||||
let mut all = tools::builtin_tools();
|
||||
if !cfg.images.enabled {
|
||||
all.retain(|t| t.name() != "generate_image");
|
||||
all.retain(|t| t.name() != "generate_image" && t.name() != "generate_3d");
|
||||
}
|
||||
let (direct, deferred) = app.mcp.tools(&cfg, owner);
|
||||
all.extend(direct);
|
||||
@@ -1113,7 +1113,8 @@ async fn stream_step(
|
||||
let mut message = Message::new(Role::Assistant, parts);
|
||||
message.id = msg_id;
|
||||
message.model = Some(model_spec.to_string());
|
||||
message.usage = Some(usage.clone());
|
||||
// With its cost (charged, or valued at list prices) for the line under the answer.
|
||||
message.usage = Some(Usage { cost: Some(req.model.cost(&usage)), ..usage.clone() });
|
||||
if !message.parts.is_empty() {
|
||||
app.store.add_message(&session.id, &message)?;
|
||||
app.emit(Event::MessageAdded { session_id: session.id.clone(), message: message.clone() });
|
||||
|
||||
@@ -653,6 +653,39 @@ impl Store {
|
||||
Ok(json!({"total": total, "by_model": by_model, "by_agent": by_agent, "first": first, "last": last}))
|
||||
}
|
||||
|
||||
/// Plan requests booked at 0 (opencode Go reports nothing charged, only `allowance`):
|
||||
/// (row id, ts, "provider/model", usage).
|
||||
pub fn plan_requests_without_cost(&self) -> Result<Vec<(i64, i64, String, crate::types::Usage)>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, ts, provider || '/' || model, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, allowance
|
||||
FROM usage WHERE cost = 0 AND allowance > 0",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get(0)?,
|
||||
r.get(1)?,
|
||||
r.get(2)?,
|
||||
crate::types::Usage {
|
||||
input_tokens: r.get::<_, i64>(3)? as u64,
|
||||
output_tokens: r.get::<_, i64>(4)? as u64,
|
||||
cache_read_tokens: r.get::<_, i64>(5)? as u64,
|
||||
cache_write_tokens: r.get::<_, i64>(6)? as u64,
|
||||
allowance: r.get(7)?,
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
pub fn set_usage_cost(&self, row: i64, cost: f64) -> Result<()> {
|
||||
self.conn.lock().unwrap().execute("UPDATE usage SET cost = ?2 WHERE id = ?1", params![row, cost])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Global statistics in [from, to) (ms). `tz_offset_min` shifts day/hour buckets
|
||||
/// into the viewer's local time.
|
||||
pub fn global_stats(&self, from: i64, to: i64, tz_offset_min: i64, user: Option<&str>) -> Result<Value> {
|
||||
|
||||
@@ -177,6 +177,7 @@ pub fn builtin_tools() -> Vec<Arc<dyn Tool>> {
|
||||
Arc::new(craft::ImageCompare),
|
||||
Arc::new(crate::prompts::SkillTool),
|
||||
Arc::new(crate::images::GenerateImage),
|
||||
Arc::new(crate::mesh3d::Generate3d),
|
||||
Arc::new(crate::reverse::BinaryInfo),
|
||||
Arc::new(crate::reverse::Decompile),
|
||||
]
|
||||
|
||||
@@ -245,8 +245,10 @@ const MessageList = memo(function MessageList({
|
||||
baseDir,
|
||||
onRewind,
|
||||
highlight,
|
||||
currency,
|
||||
}: {
|
||||
messages: Message[];
|
||||
currency: string;
|
||||
results: Map<string, ToolResult>;
|
||||
busy: boolean;
|
||||
canEdit: boolean;
|
||||
@@ -280,13 +282,52 @@ const MessageList = memo(function MessageList({
|
||||
));
|
||||
// Finished turns with tool steps fold into one line (the answer stays visible).
|
||||
const steps = b.messages.flatMap((m) => m.parts.filter((p) => p.type === "tool_call")) as ToolCall[];
|
||||
if (i === lastWork || steps.length < 2) return <div key={key}>{items}</div>;
|
||||
return <WorkBlock key={key} messages={b.messages} steps={steps} open={!!highlight && b.messages.some((m) => m.id === highlight)} baseDir={baseDir}>{items}</WorkBlock>;
|
||||
const usage = !(busy && i === lastWork) && <TurnUsage messages={b.messages} currency={currency} />;
|
||||
if (i === lastWork || steps.length < 2)
|
||||
return (
|
||||
<div key={key}>
|
||||
{items}
|
||||
{usage}
|
||||
</div>
|
||||
);
|
||||
return (
|
||||
<div key={key}>
|
||||
<WorkBlock messages={b.messages} steps={steps} open={!!highlight && b.messages.some((m) => m.id === highlight)} baseDir={baseDir}>
|
||||
{items}
|
||||
</WorkBlock>
|
||||
{usage}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</>
|
||||
);
|
||||
});
|
||||
|
||||
/** Tokens (in/out) and cost of one answer: all requests of the turn together. */
|
||||
function TurnUsage({ messages, currency }: { messages: Message[]; currency: string }) {
|
||||
const t = useT();
|
||||
let inp = 0;
|
||||
let out = 0;
|
||||
let cost = 0;
|
||||
let requests = 0;
|
||||
for (const m of messages) {
|
||||
const u = m.role === "assistant" ? m.usage : undefined;
|
||||
if (!u) continue;
|
||||
requests++;
|
||||
inp += u.input_tokens + u.cache_read_tokens + u.cache_write_tokens;
|
||||
out += u.output_tokens;
|
||||
cost += u.cost ?? 0;
|
||||
}
|
||||
if (!requests) return null;
|
||||
return (
|
||||
<div className="turn-usage" title={t("chat.turnUsageHint", { n: requests })}>
|
||||
{fmtCompact(inp)}
|
||||
<Icon name="arrow-up" className="tiny" /> {fmtCompact(out)}
|
||||
<Icon name="arrow-down" className="tiny" /> · {fmtCost(cost, currency)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** A finished turn: "42 steps · 3 files" and the answer; the steps on click. */
|
||||
function WorkBlock({ messages, steps, open: forced, baseDir, children }: { messages: Message[]; steps: ToolCall[]; open: boolean; baseDir: string; children: ReactNode }) {
|
||||
const t = useT();
|
||||
@@ -825,6 +866,7 @@ export function ChatView({ sessionId, onMenu, onClosePane }: { sessionId: string
|
||||
baseDir={session.cwd}
|
||||
onRewind={onRewind}
|
||||
highlight={highlight}
|
||||
currency={stats?.currency ?? "USD"}
|
||||
/>
|
||||
{win?.newer && (
|
||||
<button className="back-to-latest" onClick={() => void showLatest()}>
|
||||
|
||||
@@ -3,14 +3,24 @@ import { get } from "../api";
|
||||
import { useT, type Key } from "../i18n";
|
||||
import { Spinner } from "./Activity";
|
||||
import { Icon } from "./Icon";
|
||||
import { navigate } from "../router";
|
||||
|
||||
/** Open a 3D model in the 3D view of the file manager. */
|
||||
const open3d = (path: string) => {
|
||||
sessionStorage.setItem("codecraft.openFile", path);
|
||||
navigate("/files");
|
||||
};
|
||||
|
||||
interface MediaItem {
|
||||
kind: "image" | "video" | "audio";
|
||||
kind: "image" | "video" | "audio" | "model";
|
||||
url: string;
|
||||
name: string;
|
||||
source: "generated" | "tool" | "written" | "answer" | "upload";
|
||||
created_at: number;
|
||||
message_id?: string;
|
||||
/** 3D models: file (opened in the 3D view) and preview image. */
|
||||
path?: string;
|
||||
preview?: string;
|
||||
}
|
||||
|
||||
type Filter = "all" | MediaItem["kind"];
|
||||
@@ -55,7 +65,7 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string
|
||||
<Icon name="image" /> {t("media.title")}
|
||||
</h3>
|
||||
<div className="media-filters" role="radiogroup">
|
||||
{(["all", "image", "video", "audio"] as Filter[]).map((k) => (
|
||||
{(["all", "image", "video", "audio", "model"] as Filter[]).map((k) => (
|
||||
<button key={k} role="radio" aria-checked={filter === k} className={`chip-btn ${filter === k ? "active" : ""}`} onClick={() => (setFilter(k), setOpen(undefined))} disabled={!!items && count(k) === 0 && k !== "all"}>
|
||||
{t(`media.filter.${k}` as Key)} {items && <small>{count(k)}</small>}
|
||||
</button>
|
||||
@@ -78,6 +88,16 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string
|
||||
{current.kind === "image" && <img src={current.url} alt={current.name} />}
|
||||
{current.kind === "video" && <video src={current.url} controls autoPlay />}
|
||||
{current.kind === "audio" && <audio src={current.url} controls autoPlay />}
|
||||
{current.kind === "model" && (
|
||||
<div className="media-model">
|
||||
{current.preview ? <img src={current.preview} alt={current.name} /> : <Icon name="cube" />}
|
||||
{current.path && (
|
||||
<button className="primary" onClick={() => open3d(current.path!)}>
|
||||
<Icon name="cube" /> {t("tool.open3d")}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="media-bar">
|
||||
<button className="small ghost" disabled={open === 0} onClick={() => setOpen((o) => (o ?? 1) - 1)} aria-label={t("media.prev")}>
|
||||
@@ -115,6 +135,16 @@ export function MediaGallery({ sessionId, onClose, onJump }: { sessionId: string
|
||||
</span>
|
||||
)}
|
||||
{m.kind === "video" && <Icon name="play" className="media-play" />}
|
||||
{m.kind === "model" &&
|
||||
(m.preview ? (
|
||||
<img src={m.preview} alt="" loading="lazy" decoding="async" />
|
||||
) : (
|
||||
<span className="media-audio">
|
||||
<Icon name="cube" />
|
||||
<span>{m.name}</span>
|
||||
</span>
|
||||
))}
|
||||
{m.kind === "model" && <span className="media-badge">3D</span>}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
@@ -121,7 +121,7 @@ export function ToolCard({ call, result, running, baseDir }: { call: ToolCall; r
|
||||
const diffLines = typeof meta?.diff === "string" ? meta.diff.split("\n").length : 0;
|
||||
const openByDefault =
|
||||
(status === "err" && !!result) ||
|
||||
(!!result && !result.is_error && ((diffLines > 0 && diffLines <= 40) || call.name === "generate_image"));
|
||||
(!!result && !result.is_error && ((diffLines > 0 && diffLines <= 40) || call.name === "generate_image" || call.name === "generate_3d"));
|
||||
const open = toggled ?? openByDefault;
|
||||
|
||||
return (
|
||||
@@ -164,6 +164,18 @@ export function ToolCard({ call, result, running, baseDir }: { call: ToolCall; r
|
||||
<pre className="mono">{result.content}</pre>
|
||||
</>
|
||||
)}
|
||||
{typeof meta?.model_file === "string" && (
|
||||
<button
|
||||
className="small"
|
||||
style={{ marginBottom: 8 }}
|
||||
onClick={() => {
|
||||
sessionStorage.setItem("codecraft.openFile", meta.model_file);
|
||||
navigate("/files");
|
||||
}}
|
||||
>
|
||||
<Icon name="cube" /> {t("tool.open3d")}
|
||||
</button>
|
||||
)}
|
||||
{Array.isArray(meta?.image_ids) ? (
|
||||
<div className="gallery small">
|
||||
{meta.image_ids.map((id: string, i: number) => {
|
||||
|
||||
@@ -356,6 +356,7 @@ export const de: Messages = {
|
||||
"media.filter.all": "Alle",
|
||||
"media.filter.image": "Bilder",
|
||||
"media.filter.video": "Videos",
|
||||
"media.filter.model": "3D",
|
||||
"media.filter.audio": "Audio",
|
||||
"media.source.generated": "generiert",
|
||||
"media.source.tool": "Tool-Ausgabe",
|
||||
@@ -367,6 +368,7 @@ export const de: Messages = {
|
||||
"media.prev": "Vorheriges",
|
||||
"media.next": "Nächstes",
|
||||
"media.back": "Zurück zur Übersicht",
|
||||
"chat.turnUsageHint": "Alle {n} Anfragen dieser Antwort (Tokens gesendet/empfangen, Kosten)",
|
||||
"chat.reviewRound": "Review {round}: Änderungen nötig",
|
||||
"agents.refine": "Review-Schleife",
|
||||
"agents.refineRounds": "Review-Runden pro Aufgabe",
|
||||
@@ -712,6 +714,7 @@ export const de: Messages = {
|
||||
"plan.implementPrompt": "Setze den Plan um.",
|
||||
|
||||
"tool.input": "Eingabe",
|
||||
"tool.open3d": "In 3D-Ansicht öffnen",
|
||||
"tool.output": "Ausgabe",
|
||||
"tool.todosUpdated": "Aufgaben aktualisiert ({done}/{total} erledigt)",
|
||||
"tool.todos": "Aufgaben",
|
||||
|
||||
@@ -355,6 +355,7 @@ export const en = {
|
||||
"media.filter.all": "All",
|
||||
"media.filter.image": "Images",
|
||||
"media.filter.video": "Videos",
|
||||
"media.filter.model": "3D",
|
||||
"media.filter.audio": "Audio",
|
||||
"media.source.generated": "generated",
|
||||
"media.source.tool": "tool output",
|
||||
@@ -366,6 +367,7 @@ export const en = {
|
||||
"media.prev": "Previous",
|
||||
"media.next": "Next",
|
||||
"media.back": "Back to the overview",
|
||||
"chat.turnUsageHint": "All {n} requests of this answer (tokens sent/received, cost)",
|
||||
"chat.reviewRound": "Review {round}: changes requested",
|
||||
"agents.refine": "Review loop",
|
||||
"agents.refineRounds": "Review rounds per task",
|
||||
@@ -711,6 +713,7 @@ export const en = {
|
||||
"plan.implementPrompt": "Implement the plan.",
|
||||
|
||||
"tool.input": "Input",
|
||||
"tool.open3d": "Open in the 3D view",
|
||||
"tool.output": "Output",
|
||||
"tool.todosUpdated": "Tasks updated ({done}/{total} done)",
|
||||
"tool.todos": "Tasks",
|
||||
|
||||
+42
-1
@@ -456,7 +456,8 @@ button.session-group:hover {
|
||||
}
|
||||
.topbar .grow {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
/* The title stays readable: with many chips they wrap into a second row instead. */
|
||||
min-width: min(160px, 40vw);
|
||||
}
|
||||
.topbar h1 {
|
||||
font-size: 15px;
|
||||
@@ -4398,3 +4399,43 @@ span.chat-tab > button {
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* Tokens and cost under each answer. */
|
||||
.turn-usage {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 2px;
|
||||
margin: -6px 0 12px;
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.media-badge {
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
left: 6px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
padding: 1px 6px;
|
||||
border-radius: 4px;
|
||||
background: var(--bg0);
|
||||
color: var(--accent);
|
||||
}
|
||||
.media-model {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
max-height: 100%;
|
||||
}
|
||||
.media-model img {
|
||||
max-height: calc(100% - 50px);
|
||||
max-width: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
.media-model > .icon {
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
color: var(--accent);
|
||||
}
|
||||
@@ -31,6 +31,8 @@ export interface Usage {
|
||||
output_tokens: number;
|
||||
cache_read_tokens: number;
|
||||
cache_write_tokens: number;
|
||||
/** Cost of the request (charged, or valued at list prices for plan requests). */
|
||||
cost?: number;
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
* under "unreleased"; scripts/release.sh renames the key to the release version.
|
||||
*/
|
||||
export const WHATS_NEW: Record<string, { en: string[]; de: string[] }> = {
|
||||
"202611-00024": {
|
||||
en: ["3D models with fal.ai: from an image (also pasted into the chat), a text prompt or both – open them in the 3D view"],
|
||||
de: ["3D-Modelle mit fal.ai: aus einem Bild (auch eingefügt), einem Text oder beidem – öffnen in der 3D-Ansicht"],
|
||||
},
|
||||
"202611-00023": {
|
||||
en: ["Tokens and cost under every answer; the total cost of chats with opencode Go is shown correctly again"],
|
||||
de: ["Tokens und Kosten unter jeder Antwort; der Gesamtpreis von Chats mit opencode Go stimmt wieder"],
|
||||
},
|
||||
"202611-00022": {
|
||||
en: ["New theme “Tintenblau” (Settings → General → Theme)"],
|
||||
de: ["Neues Theme „Tintenblau“ (Einstellungen → Allgemein → Theme)"],
|
||||
|
||||
Reference in new issue
Block a user