From 01c1484033b8c9d3884c4a1eff49061077aba7a7 Mon Sep 17 00:00:00 2001 From: Richard Zink Date: Mon, 9 Nov 2026 06:07:03 +0100 Subject: [PATCH] Add backups with S3, daily cost push, automation templates and a browser tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Daily backups (database + config) into /backups, keep N, optional upload to any S3-compatible bucket (SigV4, bucket pruning); settings panel with run-now, list and download - Daily cost summary push at 20:00 (opt-in per user, preview button) - Automations from templates (dependency check, security review, test health, daily summary, TODO report, docs freshness) via POST /jobs/create - browser tool: headless Chromium over CDP – click, type, keys, waits, JavaScript, navigation, screenshots, console errors; requests to private network addresses are blocked (sub-resources and redirects included) - Secrets masked in the config view: S3 secret key, metrics token - Replace the remaining text arrows/refresh glyphs with SVG icons Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + config.example.toml | 24 ++ server/Cargo.toml | 1 + server/src/api.rs | 111 ++++- server/src/app.rs | 3 + server/src/backup.rs | 287 +++++++++++++ server/src/config.rs | 45 ++- server/src/digest.rs | 80 ++++ server/src/jobs.rs | 83 +++- server/src/main.rs | 4 + server/src/tools/browser.rs | 490 +++++++++++++++++++++++ server/src/tools/craft.rs | 2 +- server/src/tools/mod.rs | 2 + web/src/components/JobTemplateDialog.tsx | 134 +++++++ web/src/components/Jobs.tsx | 8 + web/src/components/NewSession.tsx | 2 +- web/src/components/Palette.tsx | 4 +- web/src/components/Settings.tsx | 39 ++ web/src/components/settings/Audit.tsx | 14 +- web/src/components/settings/Backups.tsx | 190 +++++++++ web/src/i18n/de.ts | 31 ++ web/src/i18n/en.ts | 31 ++ web/src/jobTemplates.ts | 83 ++++ web/src/styles.css | 96 +++++ web/src/whatsnew.ts | 2 + 25 files changed, 1720 insertions(+), 47 deletions(-) create mode 100644 server/src/backup.rs create mode 100644 server/src/digest.rs create mode 100644 server/src/tools/browser.rs create mode 100644 web/src/components/JobTemplateDialog.tsx create mode 100644 web/src/components/settings/Backups.tsx create mode 100644 web/src/jobTemplates.ts diff --git a/Cargo.lock b/Cargo.lock index 008b75a..4ff411b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -586,6 +586,7 @@ dependencies = [ "shell-words", "similar", "tokio", + "tokio-tungstenite", "tokio-util", "toml", "toml_edit", diff --git a/config.example.toml b/config.example.toml index 3633fd8..4fde92d 100644 --- a/config.example.toml +++ b/config.example.toml @@ -46,6 +46,8 @@ currency = "USD" # Largest file upload in MB. Uploads are sent in 8 MB chunks, so the reverse # proxy's body limit (e.g. Nginx client_max_body_size) does not apply. max_upload_mb = 500 +# Bearer token for Prometheus at GET /metrics (empty = /metrics disabled). +# metrics_token = "${CODECRAFT_METRICS_TOKEN}" # ----------------------------------------------------------------------------- @@ -189,6 +191,7 @@ diagnostics = "allow" openscad = "allow" # renders .scad files to STL/3MF + previews remember = "allow" # project memory (.codecraft/memory.md) browser_snapshot = "allow" +browser = "ask" # interactive browser (clicks, forms); private LAN is blocked image_compare = "allow" generate_image = "ask" # costs money per image "mcp__*" = "ask" @@ -358,6 +361,27 @@ enabled = true mode = "auto" +# ----------------------------------------------------------------------------- +# Backups +# ----------------------------------------------------------------------------- +# A zip with the database and this config, daily into /backups. +# The zip contains API keys: keep the bucket private. +[backup] +enabled = true +hour = 3 # local time +keep = 30 # local backups to keep + +# Optional copy to any S3-compatible storage (AWS, MinIO, Garage, B2, …). +[backup.s3] +endpoint = "" # e.g. "https://s3.eu-central-1.amazonaws.com" +region = "" +bucket = "" +prefix = "codecraft/" +access_key = "" +secret_key = "" # "${S3_SECRET_KEY}" keeps the key out of this file +keep = 0 # backups to keep in the bucket (0 = all) + + # ----------------------------------------------------------------------------- # Craft mode # ----------------------------------------------------------------------------- diff --git a/server/Cargo.toml b/server/Cargo.toml index 3edd435..30fb7d1 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -32,6 +32,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" similar = "2" tokio = { version = "1", features = ["full"] } +tokio-tungstenite = "0.29" tokio-util = "0.7" toml = "1" toml_edit = "0.25" diff --git a/server/src/api.rs b/server/src/api.rs index fd3151e..6e88b6a 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -117,6 +117,8 @@ pub fn router(app: AppState) -> Router { .route("/budget", get(budget)) .route("/audit", get(audit_log)) .route("/backup", get(backup)) + .route("/backups", get(list_backups).post(run_backup)) + .route("/backups/{name}", get(download_backup)) .route("/health", get(health)) .route("/tools", get(crate::reverse::status)) .route("/tools/ghidra/install", post(crate::reverse::install_ghidra)) @@ -138,6 +140,7 @@ pub fn router(app: AppState) -> Router { .route("/account/tokens", get(auth::list_api_tokens).post(auth::create_api_token)) .route("/account/tokens/{id}", delete(auth::delete_api_token)) .route("/jobs", get(list_jobs).post(register_job)) + .route("/jobs/create", post(create_job)) .route("/jobs/{id}", axum::routing::patch(update_job).delete(delete_job)) .route("/jobs/{id}/run", post(run_job)) .route("/jobs/{id}/webhook", post(job_webhook_settings)) @@ -147,6 +150,7 @@ pub fn router(app: AppState) -> Router { .route("/push/key", get(push_key)) .route("/push/subscribe", post(push_subscribe).delete(push_unsubscribe)) .route("/push/test", post(push_test)) + .route("/push/digest", get(get_digest).put(put_digest).post(send_digest)) .route("/ws", get(ws)) .layer(middleware::from_fn_with_state(app.clone(), auth::require_user)) // Prompts may carry images (base64); the default limit is 2 MB. @@ -1069,26 +1073,7 @@ async fn audit_log(Extension(user): Extension, State(app): State /// ZIP with config.toml and a consistent database snapshot. async fn backup(Extension(user): Extension, State(app): State) -> Result { require_admin(&user)?; - let tmp = std::env::temp_dir().join(format!("cc-backup-{}.db", new_id())); - app.store.snapshot(&tmp)?; - let config = app.config_text()?; - let db = tokio::fs::read(&tmp).await?; - let _ = tokio::fs::remove_file(&tmp).await; - let zip_bytes = tokio::task::spawn_blocking(move || -> anyhow::Result> { - use std::io::Write; - let mut buf = std::io::Cursor::new(Vec::new()); - { - let mut z = zip::ZipWriter::new(&mut buf); - let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); - z.start_file("config.toml", opts)?; - z.write_all(config.as_bytes())?; - z.start_file("codecraft.db", opts)?; - z.write_all(&db)?; - z.finish()?; - } - Ok(buf.into_inner()) - }) - .await??; + let zip_bytes = crate::backup::create_zip(&app).await?; app.audit(Some(&user), "backup.download", format!("{} bytes", zip_bytes.len()), ""); let name = format!("codecraft-backup-{}.zip", chrono::Local::now().format("%Y%m%d-%H%M")); Ok(( @@ -1101,6 +1086,46 @@ async fn backup(Extension(user): Extension, State(app): State) - .into_response()) } +/// Automatic backups: stored files, last run, settings summary. +async fn list_backups(Extension(user): Extension, State(app): State) -> ApiResult { + require_admin(&user)?; + let cfg = app.config().backup.clone(); + Ok(Json(json!({ + "files": crate::backup::list(&app), + "dir": crate::backup::backup_dir(&app).display().to_string(), + "last": *app.backup.last.lock().unwrap(), + "enabled": cfg.enabled, + "hour": cfg.hour, + "keep": cfg.keep, + "s3": !cfg.s3.endpoint.trim().is_empty(), + }))) +} + +async fn run_backup(Extension(user): Extension, State(app): State) -> ApiResult { + require_admin(&user)?; + let res = crate::backup::run(&app).await?; + app.audit(Some(&user), "backup.run", &res.file, ""); + *app.backup.last.lock().unwrap() = Some(res.clone()); + Ok(Json(json!(res))) +} + +async fn download_backup(Extension(user): Extension, State(app): State, Path(name): Path) -> Result { + require_admin(&user)?; + if !crate::backup::list(&app).iter().any(|b| b.name == name) { + return Err(ApiError(StatusCode::NOT_FOUND, "no such backup".into())); + } + let bytes = tokio::fs::read(crate::backup::backup_dir(&app).join(&name)).await?; + app.audit(Some(&user), "backup.download", &name, ""); + Ok(( + [ + (axum::http::header::CONTENT_TYPE, "application/zip".to_string()), + (axum::http::header::CONTENT_DISPOSITION, format!("attachment; filename=\"{name}\"")), + ], + bytes, + ) + .into_response()) +} + /// Liveness for monitoring (no auth needed – see auth::is_public). async fn health(State(app): State) -> Json { Json(json!({"ok": true, "version": crate::update::VERSION, "commit": crate::update::commit(), "sandbox": crate::sandbox::available(), "providers": app.registry().providers.len()})) @@ -1278,6 +1303,30 @@ async fn register_job(Extension(user): Extension, State(app): State/.codecraft/jobs/.md. + cwd: String, + #[serde(flatten)] + job: crate::jobs::NewJob, +} + +async fn create_job(Extension(user): Extension, State(app): State, Json(b): Json) -> ApiResult { + let dir = crate::files::checked_path(&app, &user, &b.cwd, true)?; + if b.job.name.trim().is_empty() || b.job.prompt.trim().is_empty() { + return Err(ApiError(StatusCode::BAD_REQUEST, "name and prompt are required".into())); + } + if !crate::permission::MODES.contains(&b.job.permission_mode.as_str()) { + return Err(ApiError(StatusCode::BAD_REQUEST, "invalid permission mode".into())); + } + // Inside the checked folder: the file name is a slug of the name. + let path = crate::jobs::new_job_path(&dir, &b.job.name); + crate::jobs::write_file(&path, &b.job).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("{e:#}")))?; + let job = app.store.job_register(&user.id, &path.display().to_string())?; + app.audit(Some(&user), "job.create", &job.path, ""); + Ok(Json(crate::jobs::describe(&job))) +} + #[derive(Deserialize)] struct UpdateJob { enabled: Option, @@ -1445,6 +1494,28 @@ async fn push_unsubscribe(Extension(user): Extension, State(app): State, State(app): State) -> ApiResult { + let on: bool = app.store.user_setting(&user.id, "cost_push")?; + Ok(Json(json!({"enabled": on}))) +} + +#[derive(Deserialize)] +struct DigestBody { + enabled: bool, +} + +async fn put_digest(Extension(user): Extension, State(app): State, Json(b): Json) -> ApiResult { + app.store.set_user_setting(&user.id, "cost_push", &b.enabled)?; + Ok(Json(json!({"enabled": b.enabled}))) +} + +/// Send today's summary right away (preview). +async fn send_digest(Extension(user): Extension, State(app): State) -> ApiResult { + let payload = crate::digest::summary(&app, &user.id).unwrap_or_else(|| json!({"kind": "digest", "title": "CodeCraft", "body": "No model requests today.", "tag": "cost-digest", "url": "/#/stats"})); + let (sent, errors) = crate::push::send_to_user(&app, &user.id, &payload).await; + Ok(Json(json!({"sent": sent, "errors": errors, "body": payload["body"]}))) +} + async fn push_test(Extension(user): Extension, State(app): State) -> ApiResult { let (sent, errors) = crate::push::send_to_user(&app, &user.id, &json!({"kind": "test", "title": "CodeCraft", "body": "Notifications work.", "tag": "test", "url": "/"})).await; Ok(Json(json!({"sent": sent, "errors": errors}))) diff --git a/server/src/app.rs b/server/src/app.rs index 431b8ca..8c70751 100644 --- a/server/src/app.rs +++ b/server/src/app.rs @@ -81,6 +81,8 @@ pub struct App { pub update: crate::update::UpdateState, /// Provider reachability (checked every 5 minutes). pub health: crate::health::Health, + /// Automatic backups (last result, run lock). + pub backup: crate::backup::BackupState, } /// What a user may do with a session. @@ -121,6 +123,7 @@ impl App { push: crate::push::Push::new(&data), terminals: Default::default(), health: Default::default(), + backup: Default::default(), update: Default::default(), }); crate::auth::bootstrap(&app)?; diff --git a/server/src/backup.rs b/server/src/backup.rs new file mode 100644 index 0000000..771cb15 --- /dev/null +++ b/server/src/backup.rs @@ -0,0 +1,287 @@ +//! Backups: a zip with the database snapshot and the config. Created on +//! demand (download) and daily into /backups, optionally uploaded to +//! an S3-compatible bucket. + +use crate::app::App; +use crate::config::{S3Config, expand_env}; +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{Local, Timelike}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// Outcome of the last automatic run (shown in the settings). +#[derive(Debug, Clone, Default, Serialize)] +pub struct LastRun { + pub at: i64, + pub file: String, + pub size: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// None = no bucket configured. + #[serde(skip_serializing_if = "Option::is_none")] + pub s3: Option, +} + +#[derive(Default)] +pub struct BackupState { + pub last: Mutex>, + running: tokio::sync::Mutex<()>, +} + +/// Zip with config.toml and a consistent snapshot of codecraft.db. +pub async fn create_zip(app: &App) -> Result> { + let tmp = std::env::temp_dir().join(format!("cc-backup-{}.db", crate::types::new_id())); + app.store.snapshot(&tmp)?; + let config = app.config_text()?; + let db = tokio::fs::read(&tmp).await; + let _ = tokio::fs::remove_file(&tmp).await; + let db = db?; + tokio::task::spawn_blocking(move || -> Result> { + use std::io::Write; + let mut buf = std::io::Cursor::new(Vec::new()); + { + let mut z = zip::ZipWriter::new(&mut buf); + let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + z.start_file("config.toml", opts)?; + z.write_all(config.as_bytes())?; + z.start_file("codecraft.db", opts)?; + z.write_all(&db)?; + z.finish()?; + } + Ok(buf.into_inner()) + }) + .await? +} + +pub fn backup_dir(app: &App) -> PathBuf { + let cfg = app.config(); + if cfg.backup.dir.is_empty() { cfg.data_dir().join("backups") } else { PathBuf::from(expand_env(&cfg.backup.dir)) } +} + +#[derive(Debug, Serialize)] +pub struct BackupFile { + pub name: String, + pub size: u64, + pub modified: i64, +} + +pub fn list(app: &App) -> Vec { + let mut out: Vec = std::fs::read_dir(backup_dir(app)) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| { + let name = e.file_name().to_string_lossy().to_string(); + if !(name.starts_with("codecraft-backup-") && name.ends_with(".zip")) { + return None; + } + let meta = e.metadata().ok()?; + let modified = meta.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?.as_millis() as i64; + Some(BackupFile { name, size: meta.len(), modified }) + }) + .collect(); + out.sort_by(|a, b| b.name.cmp(&a.name)); + out +} + +/// Create a backup now: local file (+ pruning) and S3 upload if configured. +pub async fn run(app: &App) -> Result { + let _guard = app.backup.running.lock().await; + let cfg = app.config().backup.clone(); + let dir = backup_dir(app); + tokio::fs::create_dir_all(&dir).await.with_context(|| format!("create {}", dir.display()))?; + let bytes = create_zip(app).await?; + let name = format!("codecraft-backup-{}.zip", Local::now().format("%Y%m%d-%H%M%S")); + let path = dir.join(&name); + tokio::fs::write(&path, &bytes).await.with_context(|| format!("write {}", path.display()))?; + // Restrictive permissions: the zip contains the config with API keys. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); + } + for old in list(app).into_iter().skip(cfg.keep.max(1)) { + let _ = tokio::fs::remove_file(dir.join(&old.name)).await; + } + let mut result = LastRun { at: Local::now().timestamp_millis(), file: name.clone(), size: bytes.len() as u64, error: None, s3: None }; + if !cfg.s3.endpoint.trim().is_empty() { + result.s3 = Some(match upload(&cfg.s3, &name, bytes).await { + Ok(()) => { + if cfg.s3.keep > 0 { + if let Err(e) = prune(&cfg.s3).await { + tracing::warn!("backup: pruning the bucket failed: {e:#}"); + } + } + "ok".into() + } + Err(e) => { + tracing::warn!("backup: S3 upload failed: {e:#}"); + format!("{e:#}") + } + }); + } + tracing::info!(file = %name, size = result.size, s3 = ?result.s3, "backup created"); + Ok(result) +} + +pub fn spawn(app: Arc) { + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(60)).await; + let cfg = app.config().backup.clone(); + let now = Local::now(); + if !cfg.enabled || now.hour() != cfg.hour.min(23) { + continue; + } + let today = format!("codecraft-backup-{}", now.format("%Y%m%d")); + if list(&app).iter().any(|b| b.name.starts_with(&today)) { + continue; + } + let res = match run(&app).await { + Ok(r) => r, + Err(e) => { + tracing::error!("backup failed: {e:#}"); + LastRun { at: now.timestamp_millis(), error: Some(format!("{e:#}")), ..Default::default() } + } + }; + let failed = res.error.is_some(); + *app.backup.last.lock().unwrap() = Some(res); + if failed { + // Do not retry every minute. + tokio::time::sleep(Duration::from_secs(3600)).await; + } + } + }); +} + +// --- S3 (Signature V4, path-style URLs) ------------------------------------------ + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn sha256_hex(b: &[u8]) -> String { + hex(&Sha256::digest(b)) +} + +fn hmac(key: &[u8], msg: &str) -> Vec { + use hmac::{KeyInit, Mac}; + let mut mac = as KeyInit>::new_from_slice(key).expect("hmac key"); + mac.update(msg.as_bytes()); + mac.finalize().into_bytes().to_vec() +} + +/// RFC 3986 encoding as AWS wants it ('/' kept in paths). +fn enc(s: &str, keep_slash: bool) -> String { + let mut out = String::new(); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(b as char), + b'/' if keep_slash => out.push('/'), + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +struct Signed { + url: String, + headers: Vec<(String, String)>, +} + +/// Sign a request. `query` must not be encoded yet. +fn sign(s3: &S3Config, method: &str, key: &str, query: &[(&str, &str)], payload_hash: &str, now: chrono::DateTime) -> Result { + let endpoint = s3.endpoint.trim().trim_end_matches('/'); + let url = reqwest::Url::parse(endpoint).context("invalid S3 endpoint")?; + let host = match url.port() { + Some(p) => format!("{}:{p}", url.host_str().ok_or_else(|| anyhow!("S3 endpoint without host"))?), + None => url.host_str().ok_or_else(|| anyhow!("S3 endpoint without host"))?.to_string(), + }; + let region = if s3.region.trim().is_empty() { "us-east-1" } else { s3.region.trim() }; + let path = format!("{}/{}{}", url.path().trim_end_matches('/'), enc(&s3.bucket, false), if key.is_empty() { "/".to_string() } else { format!("/{}", enc(key, true)) }); + let mut q: Vec<(String, String)> = query.iter().map(|(k, v)| (enc(k, false), enc(v, false))).collect(); + q.sort(); + let qs = q.iter().map(|(k, v)| format!("{k}={v}")).collect::>().join("&"); + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date = now.format("%Y%m%d").to_string(); + let canonical = format!("{method}\n{path}\n{qs}\nhost:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{amz_date}\n\nhost;x-amz-content-sha256;x-amz-date\n{payload_hash}"); + let scope = format!("{date}/{region}/s3/aws4_request"); + let to_sign = format!("AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", sha256_hex(canonical.as_bytes())); + let secret = expand_env(&s3.secret_key); + let k = hmac(format!("AWS4{secret}").as_bytes(), &date); + let k = hmac(&k, region); + let k = hmac(&k, "s3"); + let k = hmac(&k, "aws4_request"); + let signature = hex(&hmac(&k, &to_sign)); + let auth = format!("AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={signature}", expand_env(&s3.access_key)); + let base = format!("{}://{host}{path}", url.scheme()); + Ok(Signed { + url: if qs.is_empty() { base } else { format!("{base}?{qs}") }, + headers: vec![("x-amz-date".into(), amz_date), ("x-amz-content-sha256".into(), payload_hash.into()), ("authorization".into(), auth)], + }) +} + +async fn send(s3: &S3Config, method: reqwest::Method, key: &str, query: &[(&str, &str)], body: Vec) -> Result { + let signed = sign(s3, method.as_str(), key, query, &sha256_hex(&body), chrono::Utc::now())?; + let mut req = crate::provider::http_client().request(method, &signed.url).timeout(Duration::from_secs(600)); + for (k, v) in &signed.headers { + req = req.header(k, v); + } + let resp = req.body(body).send().await?; + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + if !status.is_success() { + let code = text.split("").nth(1).and_then(|s| s.split("").next()).unwrap_or(""); + bail!("S3 returned HTTP {status}{}", if code.is_empty() { String::new() } else { format!(" ({code})") }); + } + Ok(text) +} + +fn object_key(s3: &S3Config, name: &str) -> String { + format!("{}{name}", s3.prefix.trim_start_matches('/')) +} + +pub async fn upload(s3: &S3Config, name: &str, bytes: Vec) -> Result<()> { + if s3.bucket.trim().is_empty() { + bail!("no S3 bucket configured"); + } + send(s3, reqwest::Method::PUT, &object_key(s3, name), &[], bytes).await.map(|_| ()) +} + +/// Delete the oldest backups in the bucket beyond `keep`. +async fn prune(s3: &S3Config) -> Result<()> { + let prefix = object_key(s3, "codecraft-backup-"); + let xml = send(s3, reqwest::Method::GET, "", &[("list-type", "2"), ("prefix", &prefix)], Vec::new()).await?; + let mut keys: Vec = xml.split("").skip(1).filter_map(|s| s.split("").next()).map(|s| s.replace("&", "&")).collect(); + keys.sort_by(|a, b| b.cmp(a)); + for k in keys.into_iter().skip(s3.keep) { + send(s3, reqwest::Method::DELETE, &k, &[], Vec::new()).await?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Example from the AWS SigV4 documentation (GET object with range omitted + /// is not reproducible here, so check the derived signing key instead). + #[test] + fn signing_key_matches_aws_example() { + // https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html + let k = hmac(b"AWS4wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20120215"); + let k = hmac(&k, "us-east-1"); + let k = hmac(&k, "iam"); + let k = hmac(&k, "aws4_request"); + assert_eq!(hex(&k), "f4780e2d9f65fa895f9c67b32ce1baf0b0d8a43505a000a1a9e090d414db404d"); + } + + #[test] + fn encodes_keys() { + assert_eq!(enc("a b/c+d", true), "a%20b/c%2Bd"); + assert_eq!(enc("a/b", false), "a%2Fb"); + } +} diff --git a/server/src/config.rs b/server/src/config.rs index 285d3dc..e25bb21 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -30,10 +30,49 @@ pub struct Config { pub sandbox: SandboxConfig, pub search: SearchConfig, pub lsp: LspConfig, + pub backup: BackupConfig, #[serde(skip_serializing_if = "Vec::is_empty")] pub hooks: Vec, } +/// Automatic daily backups (database + config) on disk and optionally in S3. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(default)] +pub struct BackupConfig { + pub enabled: bool, + /// Local hour (0-23) of the daily run. + pub hour: u32, + /// Number of local backups to keep. + pub keep: usize, + /// Directory for the zip files (default: /backups). + #[serde(skip_serializing_if = "String::is_empty")] + pub dir: String, + pub s3: S3Config, +} + +impl Default for BackupConfig { + fn default() -> Self { + Self { enabled: true, hour: 3, keep: 30, dir: String::new(), s3: S3Config::default() } + } +} + +/// S3-compatible bucket (AWS, MinIO, Garage, Backblaze B2, Hetzner, …). +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(default)] +pub struct S3Config { + /// e.g. https://s3.eu-central-1.amazonaws.com or https://minio.example.com; empty = no upload. + pub endpoint: String, + pub region: String, + pub bucket: String, + /// Key prefix, e.g. "codecraft/". + pub prefix: String, + pub access_key: String, + /// Supports ${ENV_VAR}. + pub secret_key: String, + /// Old backups beyond this number are deleted from the bucket (0 = keep all). + pub keep: usize, +} + /// A shell command run on an agent event (see hooks.rs). #[derive(Debug, Clone, Deserialize, Serialize)] pub struct HookConfig { @@ -560,8 +599,10 @@ pub fn redacted_json(cfg: &Config) -> serde_json::Value { if let Some(t) = v.pointer_mut("/images/admin_key") { mask(t); } - if let Some(t) = v.pointer_mut("/search/api_key") { - mask(t); + for p in ["/search/api_key", "/backup/s3/secret_key", "/server/metrics_token"] { + if let Some(t) = v.pointer_mut(p) { + mask(t); + } } v } diff --git a/server/src/digest.rs b/server/src/digest.rs new file mode 100644 index 0000000..bf86a6b --- /dev/null +++ b/server/src/digest.rs @@ -0,0 +1,80 @@ +//! Daily cost summary as a push notification (20:00 local time, opt-in per +//! user via the user setting "cost_push"). + +use crate::app::App; +use chrono::{Local, Timelike}; +use serde_json::{Value, json}; +use std::sync::Arc; +use std::time::Duration; + +const HOUR: u32 = 20; + +fn money(v: f64, currency: &str, de: bool) -> String { + let s = format!("{v:.2}"); + let s = if de { s.replace('.', ",") } else { s }; + match currency { + "" | "USD" => if de { format!("{s} $") } else { format!("${s}") }, + "EUR" => format!("{s} €"), + c => format!("{s} {c}"), + } +} + +/// Today's summary for one user (None when nothing was used). +pub fn summary(app: &App, user_id: &str) -> Option { + let now = Local::now(); + let start = now.with_hour(0)?.with_minute(0)?.with_second(0)?.timestamp_millis(); + let offset = now.offset().local_minus_utc() as i64 / 60; + let stats = app.store.global_stats(start, now.timestamp_millis() + 1, offset, Some(user_id)).ok()?; + let requests = stats["total"]["requests"].as_i64().unwrap_or(0); + if requests == 0 { + return None; + } + let prefs: Value = app.store.user_setting(user_id, "prefs").unwrap_or_default(); + let de = prefs["lang"].as_str().unwrap_or("").starts_with("de"); + let currency = app.config().server.currency.clone(); + let cost = stats["total"]["cost"].as_f64().unwrap_or(0.0); + let top = stats["by_model"] + .as_array() + .and_then(|m| m.iter().max_by(|a, b| a["cost"].as_f64().unwrap_or(0.0).total_cmp(&b["cost"].as_f64().unwrap_or(0.0)))) + .and_then(|m| m["model"].as_str()) + .unwrap_or("") + .to_string(); + let sessions = stats["top_sessions"].as_array().map(|a| a.len()).unwrap_or(0); + let body = if de { + format!("{} · {requests} Anfragen in {sessions} Chats{}", money(cost, ¤cy, true), if top.is_empty() { String::new() } else { format!(" · meist {top}") }) + } else { + format!("{} · {requests} requests in {sessions} chats{}", money(cost, ¤cy, false), if top.is_empty() { String::new() } else { format!(" · mostly {top}") }) + }; + Some(json!({ + "kind": "digest", + "title": if de { "CodeCraft – Kosten heute" } else { "CodeCraft – today's costs" }, + "body": body, + "tag": "cost-digest", + "url": "/#/stats", + })) +} + +pub fn spawn(app: Arc) { + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(60)).await; + let now = Local::now(); + if now.hour() != HOUR { + continue; + } + let today = now.format("%Y-%m-%d").to_string(); + for user in app.store.users().unwrap_or_default() { + let on: bool = app.store.user_setting(&user.id, "cost_push").unwrap_or(false); + let last: String = app.store.user_setting(&user.id, "cost_push_last").unwrap_or_default(); + if !on || user.disabled || last == today { + continue; + } + let _ = app.store.set_user_setting(&user.id, "cost_push_last", &today); + if let Some(payload) = summary(&app, &user.id) { + let (sent, errors) = crate::push::send_to_user(&app, &user.id, &payload).await; + tracing::info!(user = %user.username, sent, errors = errors.len(), "daily cost summary"); + } + } + } + }); +} diff --git a/server/src/jobs.rs b/server/src/jobs.rs index f2d52cc..2a29e56 100644 --- a/server/src/jobs.rs +++ b/server/src/jobs.rs @@ -233,8 +233,66 @@ pub fn spawn_scheduler(app: Arc) { /// Tool: create, list, run and delete automations. pub struct Schedule; +/// Fields of a job file written by the agent tool or the "from template" dialog. +#[derive(Debug, serde::Deserialize)] +pub struct NewJob { + pub name: String, + pub schedule: String, + #[serde(default = "default_agent")] + pub agent: String, + #[serde(default)] + pub model: Option, + #[serde(default = "default_mode")] + pub permission_mode: String, + #[serde(default = "default_timeout")] + pub timeout_minutes: u64, + pub prompt: String, +} + +fn default_agent() -> String { + "build".into() +} +fn default_mode() -> String { + "auto".into() +} +fn default_timeout() -> u64 { + 60 +} + +/// Write (and validate) a job file. +pub fn write_file(path: &Path, j: &NewJob) -> Result { + croner::Cron::from_str(&j.schedule).map_err(|e| anyhow!("invalid schedule '{}': {e}", j.schedule))?; + let q = |s: &str| format!("\"{}\"", s.replace('"', "'").replace('\n', " ")); + let plain = |s: &str| s.chars().filter(|c| c.is_alphanumeric() || "-_/.:".contains(*c)).collect::(); + let mut fm = format!("---\nname: {}\nschedule: {}\nagent: {}\n", q(&j.name), q(&j.schedule), plain(&j.agent)); + if let Some(m) = j.model.as_deref().filter(|m| !m.is_empty()) { + fm.push_str(&format!("model: {}\n", plain(m))); + } + fm.push_str(&format!("permission_mode: {}\n", plain(&j.permission_mode))); + fm.push_str(&format!("timeout_minutes: {}\n---\n", j.timeout_minutes.clamp(1, 24 * 60))); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + std::fs::write(path, format!("{fm}{}\n", j.prompt.trim()))?; + match parse(path) { + Ok(spec) => Ok(spec), + Err(e) => { + let _ = std::fs::remove_file(path); + Err(e) + } + } +} + +/// Free file name for a new job in `/.codecraft/jobs/`. +pub fn new_job_path(project: &Path, name: &str) -> PathBuf { + let dir = project.join(".codecraft/jobs"); + let base = slug(name); + (1..100).map(|i| dir.join(if i == 1 { format!("{base}.md") } else { format!("{base}-{i}.md") })).find(|p| !p.exists()).unwrap_or_else(|| dir.join(format!("{base}-{}.md", new_id()))) +} + fn slug(s: &str) -> String { - let out: String = s.to_lowercase().chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }).collect(); + let lower = s.to_lowercase().replace('ä', "ae").replace('ö', "oe").replace('ü', "ue").replace('ß', "ss"); + let out: String = lower.chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }).collect(); out.split('-').filter(|p| !p.is_empty()).collect::>().join("-").chars().take(50).collect() } @@ -288,19 +346,20 @@ impl Tool for Schedule { let name = arg_str(&input, "name")?; let schedule = arg_str(&input, "schedule")?; let prompt = arg_str(&input, "prompt")?; - croner::Cron::from_str(schedule).map_err(|e| anyhow!("invalid schedule '{schedule}': {e}"))?; - let q = |s: &str| format!("\"{}\"", s.replace('"', "'")); - let mut fm = format!("---\nname: {}\nschedule: {}\nagent: {}\n", q(name), q(schedule), input["agent"].as_str().unwrap_or("build")); - if let Some(m) = input["model"].as_str().filter(|m| !m.is_empty()) { - fm.push_str(&format!("model: {m}\n")); - } - fm.push_str(&format!("permission_mode: {}\n", input["permission_mode"].as_str().unwrap_or("auto"))); - fm.push_str(&format!("timeout_minutes: {}\n---\n", input["timeout_minutes"].as_u64().unwrap_or(60))); let rel = format!(".codecraft/jobs/{}.md", slug(name)); let path = ctx.resolve(&rel)?; - std::fs::create_dir_all(path.parent().unwrap())?; - std::fs::write(&path, format!("{fm}{}\n", prompt.trim()))?; - parse(&path)?; + write_file( + &path, + &NewJob { + name: name.into(), + schedule: schedule.into(), + agent: input["agent"].as_str().unwrap_or("build").into(), + model: input["model"].as_str().map(String::from), + permission_mode: input["permission_mode"].as_str().unwrap_or("auto").into(), + timeout_minutes: input["timeout_minutes"].as_u64().unwrap_or(60), + prompt: prompt.into(), + }, + )?; let job = ctx.app.store.job_register(&owner, &path.display().to_string())?; let d = describe(&job); Ok(ToolOutput::text(format!( diff --git a/server/src/main.rs b/server/src/main.rs index c3f5de1..419adea 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -3,9 +3,11 @@ mod api; mod app; mod auth; +mod backup; mod cli_api; mod budget; mod config; +mod digest; mod docs; mod export; mod files; @@ -64,6 +66,8 @@ async fn main() -> Result<()> { jobs::spawn_scheduler(app.clone()); update::spawn_checker(app.clone()); health::spawn_checker(app.clone()); + digest::spawn(app.clone()); + backup::spawn(app.clone()); let router = api::router(app); let listener = tokio::net::TcpListener::bind(&bind).await?; tracing::info!("listening on http://{bind}"); diff --git a/server/src/tools/browser.rs b/server/src/tools/browser.rs new file mode 100644 index 0000000..60e7ad2 --- /dev/null +++ b/server/src/tools/browser.rs @@ -0,0 +1,490 @@ +//! Interactive headless browser for agents (Chrome DevTools Protocol): +//! open a page, click, type, run JavaScript, then get a screenshot, the +//! visible text and console errors. Public internet and localhost are +//! allowed; private network addresses are blocked for every request the page +//! makes (sub-resources and redirects included). + +use super::{Tool, ToolCtx, ToolOutput, arg_str, truncate}; +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use base64::Engine; +use futures::{SinkExt, StreamExt}; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::net::IpAddr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use tokio::sync::{mpsc, oneshot}; +use tokio_tungstenite::tungstenite::Message as WsMessage; + +/// Private, link-local and other non-public ranges (loopback is allowed). +pub fn is_blocked_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let o = v4.octets(); + v4.is_private() + || v4.is_link_local() + || v4.is_unspecified() + || v4.is_broadcast() + || (o[0] == 100 && (64..128).contains(&o[1])) // CGNAT + || (o[0] == 192 && o[1] == 0 && o[2] == 0) + || o[0] >= 240 + } + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return !v4.is_loopback() && is_blocked_ip(IpAddr::V4(v4)); + } + let s = v6.segments(); + v6.is_unspecified() || (s[0] & 0xfe00) == 0xfc00 || (s[0] & 0xffc0) == 0xfe80 + } + } +} + +/// Is the URL's host allowed? Cached per host. +async fn host_allowed(url: &str, cache: &Mutex>) -> bool { + let Ok(u) = reqwest::Url::parse(url) else { return false }; + match u.scheme() { + "http" | "https" | "ws" | "wss" => {} + "data" | "blob" | "about" => return true, + _ => return false, + } + let Some(host) = u.host_str().map(|h| h.trim_matches(['[', ']']).to_string()) else { return false }; + if let Some(&ok) = cache.lock().unwrap().get(&host) { + return ok; + } + let port = u.port_or_known_default().unwrap_or(80); + let ok = match host.parse::() { + Ok(ip) => !is_blocked_ip(ip), + Err(_) => match tokio::time::timeout(Duration::from_secs(5), tokio::net::lookup_host((host.as_str(), port))).await { + Ok(Ok(addrs)) => { + let addrs: Vec<_> = addrs.collect(); + !addrs.is_empty() && addrs.iter().all(|a| !is_blocked_ip(a.ip())) + } + _ => false, + }, + }; + cache.lock().unwrap().insert(host, ok); + ok +} + +type Pending = Arc>>>; + +/// Minimal CDP client on one page target. +struct Cdp { + out: mpsc::UnboundedSender, + pending: Pending, + next: Arc, +} + +impl Cdp { + async fn call(&self, method: &str, params: Value) -> Result { + call(&self.out, &self.pending, &self.next, method, params).await + } +} + +async fn call(out: &mpsc::UnboundedSender, pending: &Pending, next: &AtomicU64, method: &str, params: Value) -> Result { + let id = next.fetch_add(1, Ordering::Relaxed); + let (tx, rx) = oneshot::channel(); + pending.lock().unwrap().insert(id, tx); + out.send(json!({"id": id, "method": method, "params": params}).to_string()).map_err(|_| anyhow!("browser connection closed"))?; + let v = tokio::time::timeout(Duration::from_secs(30), rx).await.map_err(|_| anyhow!("{method}: browser did not answer"))?.map_err(|_| anyhow!("browser connection closed"))?; + if let Some(e) = v.get("error") { + bail!("{method}: {}", e["message"].as_str().unwrap_or("error")); + } + Ok(v["result"].clone()) +} + +#[derive(Default)] +struct PageLog { + console: Vec, + blocked: Vec, + loads: u64, +} + +pub struct Browser; + +const MAX_STEPS: usize = 40; + +#[async_trait] +impl Tool for Browser { + fn name(&self) -> &str { + "browser" + } + fn description(&self) -> String { + "Use a headless Chromium like a user: open a URL (public site or http://localhost:PORT), run \ + steps (click, type, press keys, wait, run JavaScript, navigate), then get a screenshot, the \ + visible text and console errors. Each call starts a fresh browser – put the whole flow \ + (e.g. log in, open a page, check it) into one call. Private network addresses are blocked." + .into() + } + fn schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "url": {"type": "string", "description": "Start page (http, https or file:// inside your folders)"}, + "steps": { + "type": "array", + "description": "Executed in order after the page loaded", + "items": { + "type": "object", + "properties": { + "action": {"type": "string", "enum": ["click", "type", "press", "wait", "wait_for", "eval", "navigate", "screenshot"]}, + "selector": {"type": "string", "description": "CSS selector (click, type, wait_for)"}, + "text": {"type": "string", "description": "Text to type (type); with submit=true Enter is pressed afterwards"}, + "submit": {"type": "boolean"}, + "key": {"type": "string", "description": "Enter, Tab, Escape, ArrowDown … (press)"}, + "ms": {"type": "integer", "description": "Milliseconds (wait), max 15000"}, + "js": {"type": "string", "description": "JavaScript expression; the (awaited) result is returned (eval)"}, + "url": {"type": "string", "description": "navigate"} + }, + "required": ["action"] + } + }, + "width": {"type": "integer", "description": "Viewport width, default 1280"}, + "height": {"type": "integer", "description": "Viewport height, default 800"}, + "full_page": {"type": "boolean", "description": "Screenshot of the whole page instead of the viewport"}, + "text": {"type": "boolean", "description": "Include the visible text (default true)"} + }, + "required": ["url"] + }) + } + fn title(&self, input: &Value) -> String { + let n = input["steps"].as_array().map(|s| s.len()).unwrap_or(0); + format!("Browser {}{}", input["url"].as_str().unwrap_or("?"), if n > 0 { format!(" ({n} steps)") } else { String::new() }) + } + async fn run(&self, input: Value, ctx: &ToolCtx) -> Result { + let url = arg_str(&input, "url")?.to_string(); + let steps = input["steps"].as_array().cloned().unwrap_or_default(); + if steps.len() > MAX_STEPS { + bail!("at most {MAX_STEPS} steps per call"); + } + let clean_room = ctx.agent == "craft" && ctx.clean_room; + if clean_room && steps.iter().any(|s| s["action"] == "eval") { + return Ok(ToolOutput::error("Clean room mode: running JavaScript on the page is not allowed (it could read the source).")); + } + if let Some(path) = url.strip_prefix("file://") { + ctx.resolve(path)?; + } else if !(url.starts_with("http://") || url.starts_with("https://")) { + bail!("only http(s) and file:// URLs are supported"); + } + let cache = Arc::new(Mutex::new(HashMap::new())); + if !url.starts_with("file://") && !host_allowed(&url, &cache).await { + bail!("blocked: {url} points to a private network address (only public hosts and localhost are allowed)"); + } + let fut = session(ctx, &input, &url, &steps, cache); + tokio::select! { + r = tokio::time::timeout(Duration::from_secs(180), fut) => r.map_err(|_| anyhow!("the browser session took longer than 3 minutes"))?, + _ = ctx.cancel.cancelled() => bail!("aborted"), + } + } +} + +async fn session(ctx: &ToolCtx, input: &Value, url: &str, steps: &[Value], cache: Arc>>) -> Result { + let profile = std::env::temp_dir().join(format!("cc-browser-{}", crate::types::new_id())); + let bin = ctx.app.config().craft.chromium.clone(); + let mut child = Command::new(&bin) + .args(["--headless=new", "--no-sandbox", "--disable-gpu", "--hide-scrollbars", "--mute-audio", "--disable-dev-shm-usage", "--no-first-run", "--no-default-browser-check", "--remote-debugging-port=0"]) + .arg(format!("--user-data-dir={}", profile.display())) + .arg("about:blank") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("could not start '{bin}' (set craft.chromium in the config)"))?; + let result = drive(&mut child, input, url, steps, cache).await; + let _ = child.kill().await; + let _ = tokio::fs::remove_dir_all(&profile).await; + result +} + +async fn drive(child: &mut tokio::process::Child, input: &Value, url: &str, steps: &[Value], cache: Arc>>) -> Result { + // "DevTools listening on ws://127.0.0.1:PORT/devtools/browser/…" + let stderr = child.stderr.take().ok_or_else(|| anyhow!("no stderr"))?; + let mut lines = BufReader::new(stderr).lines(); + let port = tokio::time::timeout(Duration::from_secs(20), async { + while let Some(l) = lines.next_line().await? { + if let Some(rest) = l.split("ws://127.0.0.1:").nth(1) { + return Ok::<_, anyhow::Error>(rest.split('/').next().unwrap_or("").to_string()); + } + } + bail!("chromium exited before DevTools was ready") + }) + .await + .map_err(|_| anyhow!("chromium did not start"))??; + tokio::spawn(async move { while let Ok(Some(_)) = lines.next_line().await {} }); + + let list: Value = reqwest::get(format!("http://127.0.0.1:{port}/json/list")).await?.json().await?; + let ws_url = list.as_array().and_then(|a| a.iter().find(|t| t["type"] == "page")).and_then(|t| t["webSocketDebuggerUrl"].as_str()).ok_or_else(|| anyhow!("no browser page"))?.to_string(); + let (ws, _) = tokio_tungstenite::connect_async(&ws_url).await.context("DevTools connection")?; + let (mut sink, mut stream) = ws.split(); + let (out_tx, mut out_rx) = mpsc::unbounded_channel::(); + tokio::spawn(async move { + while let Some(m) = out_rx.recv().await { + if sink.send(WsMessage::Text(m.into())).await.is_err() { + break; + } + } + }); + let pending: Pending = Arc::default(); + let next = Arc::new(AtomicU64::new(1)); + let log = Arc::new(Mutex::new(PageLog::default())); + { + let pending = pending.clone(); + let out = out_tx.clone(); + let next = next.clone(); + let log = log.clone(); + let cache = cache.clone(); + tokio::spawn(async move { + while let Some(Ok(msg)) = stream.next().await { + let WsMessage::Text(text) = msg else { continue }; + let Ok(v) = serde_json::from_str::(&text) else { continue }; + if let Some(id) = v["id"].as_u64() { + if let Some(tx) = pending.lock().unwrap().remove(&id) { + let _ = tx.send(v); + } + continue; + } + let p = &v["params"]; + match v["method"].as_str().unwrap_or("") { + // Every request passes here: block private addresses. + "Fetch.requestPaused" => { + let (out, pending, next, cache, log) = (out.clone(), pending.clone(), next.clone(), cache.clone(), log.clone()); + let req_id = p["requestId"].as_str().unwrap_or("").to_string(); + let req_url = p["request"]["url"].as_str().unwrap_or("").to_string(); + tokio::spawn(async move { + if host_allowed(&req_url, &cache).await || req_url.starts_with("file://") { + let _ = call(&out, &pending, &next, "Fetch.continueRequest", json!({"requestId": req_id})).await; + } else { + log.lock().unwrap().blocked.push(req_url); + let _ = call(&out, &pending, &next, "Fetch.failRequest", json!({"requestId": req_id, "errorReason": "BlockedByClient"})).await; + } + }); + } + "Runtime.consoleAPICalled" => { + let kind = p["type"].as_str().unwrap_or(""); + if matches!(kind, "error" | "warning" | "assert") { + let args: Vec = p["args"].as_array().into_iter().flatten().map(|a| a["value"].as_str().map(String::from).unwrap_or_else(|| a["description"].as_str().unwrap_or("").to_string())).collect(); + log.lock().unwrap().console.push(format!("console.{kind}: {}", args.join(" "))); + } + } + "Runtime.exceptionThrown" => { + let d = &p["exceptionDetails"]; + let msg = d["exception"]["description"].as_str().or(d["text"].as_str()).unwrap_or("exception"); + log.lock().unwrap().console.push(format!("uncaught: {}", msg.lines().next().unwrap_or(msg))); + } + "Log.entryAdded" => { + let e = &p["entry"]; + if e["level"] == "error" { + log.lock().unwrap().console.push(format!("{}: {} {}", e["source"].as_str().unwrap_or("log"), e["text"].as_str().unwrap_or(""), e["url"].as_str().unwrap_or(""))); + } + } + "Page.loadEventFired" => log.lock().unwrap().loads += 1, + _ => {} + } + } + }); + } + let cdp = Cdp { out: out_tx, pending, next }; + let width = input["width"].as_u64().unwrap_or(1280).clamp(320, 2560); + let height = input["height"].as_u64().unwrap_or(800).clamp(320, 2400); + for (m, p) in [ + ("Page.enable", json!({})), + ("Runtime.enable", json!({})), + ("Log.enable", json!({})), + ("Fetch.enable", json!({"patterns": [{"urlPattern": "*"}]})), + ("Emulation.setDeviceMetricsOverride", json!({"width": width, "height": height, "deviceScaleFactor": 1, "mobile": width < 700})), + ] { + cdp.call(m, p).await?; + } + + let navigate = |target: String| { + let cdp = &cdp; + let log = log.clone(); + async move { + let before = log.lock().unwrap().loads; + let r = cdp.call("Page.navigate", json!({"url": target})).await?; + if let Some(e) = r["errorText"].as_str() { + bail!("could not open {target}: {e}"); + } + wait_load(&log, before, 20_000).await; + Ok::<_, anyhow::Error>(()) + } + }; + navigate(url.to_string()).await?; + + let eval = |js: String| { + let cdp = &cdp; + async move { + let r = cdp.call("Runtime.evaluate", json!({"expression": js, "awaitPromise": true, "returnByValue": true, "userGesture": true})).await?; + if let Some(d) = r["exceptionDetails"].as_object() { + bail!("{}", d.get("exception").and_then(|e| e["description"].as_str()).or(d.get("text").and_then(|t| t.as_str())).unwrap_or("JavaScript error")); + } + Ok::<_, anyhow::Error>(r["result"]["value"].clone()) + } + }; + + let mut report = Vec::new(); + let mut images = Vec::new(); + for (i, s) in steps.iter().enumerate() { + let n = i + 1; + let sel = s["selector"].as_str().unwrap_or(""); + let sel_js = serde_json::to_string(sel)?; + let before = log.lock().unwrap().loads; + let res: Result = async { + match s["action"].as_str().unwrap_or("") { + "click" => { + let r = eval(format!("(()=>{{const e=document.querySelector({sel_js});if(!e)return 'not found';e.scrollIntoView({{block:'center'}});e.click();return 'ok'}})()")).await?; + if r != "ok" { + bail!("no element matches {sel}"); + } + settle(&log, before).await; + Ok(format!("clicked {sel}")) + } + "type" => { + let r = eval(format!("(()=>{{const e=document.querySelector({sel_js});if(!e)return 'not found';e.focus();if('select' in e)e.select();return 'ok'}})()")).await?; + if r != "ok" { + bail!("no element matches {sel}"); + } + let text = s["text"].as_str().unwrap_or(""); + cdp.call("Input.insertText", json!({"text": text})).await?; + if s["submit"].as_bool().unwrap_or(false) { + press(&cdp, "Enter").await?; + settle(&log, before).await; + } + Ok(format!("typed {} characters into {sel}", text.chars().count())) + } + "press" => { + let key = s["key"].as_str().unwrap_or("Enter"); + press(&cdp, key).await?; + settle(&log, before).await; + Ok(format!("pressed {key}")) + } + "wait" => { + let ms = s["ms"].as_u64().unwrap_or(1000).min(15_000); + tokio::time::sleep(Duration::from_millis(ms)).await; + Ok(format!("waited {ms} ms")) + } + "wait_for" => { + for _ in 0..50 { + if eval(format!("!!document.querySelector({sel_js})")).await? == true { + return Ok(format!("{sel} is there")); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + bail!("{sel} did not appear within 10 s") + } + "eval" => { + let v = eval(s["js"].as_str().unwrap_or("undefined").to_string()).await?; + Ok(format!("eval → {}", truncate(&serde_json::to_string(&v).unwrap_or_default(), 4000))) + } + "navigate" => { + let target = s["url"].as_str().ok_or_else(|| anyhow!("navigate needs url"))?.to_string(); + if !host_allowed(&target, &cache).await { + bail!("blocked: {target} points to a private network address"); + } + navigate(target.clone()).await?; + Ok(format!("opened {target}")) + } + "screenshot" => { + images.push(screenshot(&cdp, false).await?); + Ok("screenshot taken".into()) + } + other => bail!("unknown action '{other}'"), + } + } + .await; + match res { + Ok(r) => report.push(format!("{n}. {r}")), + Err(e) => { + report.push(format!("{n}. FAILED: {e:#} – remaining steps skipped")); + break; + } + } + } + + let page = eval("({t:document.title,u:location.href,x:document.body?document.body.innerText:''})".into()).await.unwrap_or(Value::Null); + if images.len() < 4 { + images.push(screenshot(&cdp, input["full_page"].as_bool().unwrap_or(false)).await?); + } + let mut text = format!("Page: {} – {}\n", page["t"].as_str().unwrap_or(""), page["u"].as_str().unwrap_or(url)); + if !report.is_empty() { + text.push_str(&format!("\nSteps:\n{}\n", report.join("\n"))); + } + let log = std::mem::take(&mut *log.lock().unwrap()); + if !log.console.is_empty() { + text.push_str(&format!("\nConsole errors ({}):\n{}\n", log.console.len(), truncate(&log.console.join("\n"), 6000))); + } + if !log.blocked.is_empty() { + text.push_str(&format!("\nBlocked requests to private addresses: {}\n", log.blocked.iter().take(10).cloned().collect::>().join(", "))); + } + if input["text"].as_bool().unwrap_or(true) { + text.push_str(&format!("\nVisible text:\n{}", truncate(page["x"].as_str().unwrap_or(""), 15_000))); + } + Ok(ToolOutput { content: text, images, ..Default::default() }) +} + +/// Wait until a new load event fired (or the timeout passed). +async fn wait_load(log: &Mutex, before: u64, ms: u64) { + let deadline = tokio::time::Instant::now() + Duration::from_millis(ms); + while log.lock().unwrap().loads <= before && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + tokio::time::sleep(Duration::from_millis(400)).await; +} + +/// After an interaction: give a navigation the chance to start and finish. +async fn settle(log: &Mutex, before: u64) { + tokio::time::sleep(Duration::from_millis(400)).await; + if log.lock().unwrap().loads > before { + return wait_load(log, before, 15_000).await; + } + wait_load(log, before, 800).await; +} + +async fn press(cdp: &Cdp, key: &str) -> Result<()> { + let (code, vk, text) = match key { + "Enter" => ("Enter", 13, "\r"), + "Tab" => ("Tab", 9, ""), + "Escape" => ("Escape", 27, ""), + "Backspace" => ("Backspace", 8, ""), + "ArrowDown" => ("ArrowDown", 40, ""), + "ArrowUp" => ("ArrowUp", 38, ""), + "ArrowLeft" => ("ArrowLeft", 37, ""), + "ArrowRight" => ("ArrowRight", 39, ""), + "Space" | " " => ("Space", 32, " "), + other => bail!("unsupported key '{other}'"), + }; + let key_name = if code == "Space" { " " } else { code }; + let mut down = json!({"type": "keyDown", "key": key_name, "code": code, "windowsVirtualKeyCode": vk}); + if !text.is_empty() { + down["text"] = json!(text); + } + cdp.call("Input.dispatchKeyEvent", down).await?; + cdp.call("Input.dispatchKeyEvent", json!({"type": "keyUp", "key": key_name, "code": code, "windowsVirtualKeyCode": vk})).await?; + Ok(()) +} + +async fn screenshot(cdp: &Cdp, full: bool) -> Result { + let r = cdp.call("Page.captureScreenshot", json!({"format": "png", "captureBeyondViewport": full})).await?; + let bytes = base64::engine::general_purpose::STANDARD.decode(r["data"].as_str().unwrap_or(""))?; + let img = image::load_from_memory(&bytes)?; + super::craft::png_for_model(&img) +} + +#[cfg(test)] +mod tests { + use super::is_blocked_ip; + + #[test] + fn private_ranges() { + for ip in ["10.1.2.3", "192.168.1.10", "172.17.0.1", "169.254.169.254", "100.64.0.1", "fd00::1", "fe80::1", "::ffff:192.168.0.1"] { + assert!(is_blocked_ip(ip.parse().unwrap()), "{ip}"); + } + for ip in ["127.0.0.1", "::1", "8.8.8.8", "1.1.1.1", "2606:4700::1111"] { + assert!(!is_blocked_ip(ip.parse().unwrap()), "{ip}"); + } + } +} diff --git a/server/src/tools/craft.rs b/server/src/tools/craft.rs index f91d6aa..41fafa4 100644 --- a/server/src/tools/craft.rs +++ b/server/src/tools/craft.rs @@ -17,7 +17,7 @@ const SNAPSHOT_HEIGHT: u32 = 2000; /// Long-side limit for images sent to the model. const MAX_IMAGE_SIDE: u32 = 1568; -fn png_for_model(img: &image::DynamicImage) -> Result { +pub(crate) fn png_for_model(img: &image::DynamicImage) -> Result { let (w, h) = img.dimensions(); let img = if w.max(h) > MAX_IMAGE_SIDE { img.resize(MAX_IMAGE_SIDE, MAX_IMAGE_SIDE, image::imageops::FilterType::Triangle) diff --git a/server/src/tools/mod.rs b/server/src/tools/mod.rs index 9ac68c8..06b38a3 100644 --- a/server/src/tools/mod.rs +++ b/server/src/tools/mod.rs @@ -3,6 +3,7 @@ pub mod agent; pub mod bash; +pub mod browser; pub mod craft; pub mod cad; pub mod fs; @@ -161,6 +162,7 @@ pub fn builtin_tools() -> Vec> { Arc::new(crate::shells::BashOutput), Arc::new(crate::shells::KillShell), Arc::new(web::WebFetch), + Arc::new(browser::Browser), Arc::new(web::WebSearch), Arc::new(agent::CraftPhase), Arc::new(agent::ChangeDirectory), diff --git a/web/src/components/JobTemplateDialog.tsx b/web/src/components/JobTemplateDialog.tsx new file mode 100644 index 0000000..c7e395d --- /dev/null +++ b/web/src/components/JobTemplateDialog.tsx @@ -0,0 +1,134 @@ +import { useEffect, useState } from "react"; +import { useFetch } from "../hooks"; +import { post } from "../api"; +import { getLang, useT } from "../i18n"; +import { JOB_TEMPLATES, SCHEDULES, type JobTemplate } from "../jobTemplates"; +import { Icon } from "./Icon"; +import { ModelSelect } from "./ModelSelect"; +import { FolderPicker } from "./NewSession"; + +/** Create an automation from a template: pick it, choose the project, adjust, create. */ +export function JobTemplateDialog({ onClose, onCreated }: { onClose: () => void; onCreated: () => void }) { + const t = useT(); + const de = getLang() === "de"; + const [tpl, setTpl] = useState(); + const [name, setName] = useState(""); + const [schedule, setSchedule] = useState(""); + const [agent, setAgent] = useState("build"); + const [mode, setMode] = useState("auto"); + const [model, setModel] = useState(""); + const [prompt, setPrompt] = useState(""); + const [cwd, setCwd] = useState(localStorage.getItem("codecraft.lastDir") ?? ""); + const [error, setError] = useState(); + const [busy, setBusy] = useState(false); + const config = useFetch<{ default_model?: string }>("/config").data; + useEffect(() => { + if (!model && config?.default_model) setModel(config.default_model); + }, [config, model]); + + const choose = (x: JobTemplate) => { + setTpl(x); + setName(de ? x.name.de : x.name.en); + setSchedule(x.schedule); + setAgent(x.agent); + setPrompt(x.prompt); + }; + + const create = async () => { + setBusy(true); + setError(undefined); + try { + await post("/jobs/create", { cwd, name: name.trim(), schedule: schedule.trim(), agent, permission_mode: mode, model: model || undefined, prompt }); + onCreated(); + onClose(); + } catch (e: any) { + setError(e.message); + } finally { + setBusy(false); + } + }; + + return ( +
+
e.stopPropagation()}> + {!tpl ? ( + <> +

{t("jobs.fromTemplate")}

+
+ {JOB_TEMPLATES.map((x) => ( + + ))} +
+
+ +
+ + ) : ( + <> +

+ {de ? tpl.name.de : tpl.name.en} +

+ +
+ {t("jobs.tplProject")} + +
+
+ {t("jobs.tplSchedule")} +
+ {SCHEDULES.map((s) => ( + + ))} +
+ setSchedule(e.target.value)} placeholder="0 3 * * *" /> + {t("jobs.tplCronHint")} +
+
+ + + +
+