diff --git a/Cargo.lock b/Cargo.lock index 9fcdd3d..008b75a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4109,6 +4109,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -4119,12 +4129,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/server/Cargo.toml b/server/Cargo.toml index c190881..3edd435 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -37,7 +37,7 @@ toml = "1" toml_edit = "0.25" tower-http = { version = "0.7", features = ["fs", "cors", "trace"] } tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } uuid = { version = "1", features = ["v4", "v7"] } p256 = { version = "0.14", features = ["ecdh", "ecdsa"] } hkdf = "0.13" diff --git a/server/src/api.rs b/server/src/api.rs index 54dfa0d..604df6a 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -121,6 +121,7 @@ pub fn router(app: AppState) -> Router { .route("/tools", get(crate::reverse::status)) .route("/tools/ghidra/install", post(crate::reverse::install_ghidra)) .route("/providers/{name}/prices", get(provider_prices)) + .route("/providers/status", get(provider_status).post(recheck_providers)) .route("/images", get(list_images).post(create_images)) .route("/images/config", get(images_config)) .route("/images/{id}", delete(delete_image)) @@ -162,6 +163,7 @@ pub fn router(app: AppState) -> Router { Router::new() .nest("/api", api) .route("/hooks/{id}", post(incoming_webhook)) + .route("/metrics", get(crate::metrics::metrics)) .route("/cli/install.sh", get(cli_install_script)) .route("/cli/download/{target}", get(cli_download)) .fallback_service(static_files) @@ -1073,6 +1075,16 @@ async fn health(State(app): State) -> Json { Json(json!({"ok": true, "version": crate::update::VERSION, "commit": crate::update::commit(), "sandbox": crate::sandbox::available(), "providers": app.registry().providers.len()})) } +async fn provider_status(State(app): State) -> Json { + Json(json!(app.health.all())) +} + +async fn recheck_providers(Extension(user): Extension, State(app): State) -> ApiResult { + require_admin(&user)?; + crate::health::check_all(&app).await; + Ok(Json(json!(app.health.all()))) +} + // --- budget ------------------------------------------------------------------- /// Budgets concern the whole installation: admins only. diff --git a/server/src/app.rs b/server/src/app.rs index f0c8e07..431b8ca 100644 --- a/server/src/app.rs +++ b/server/src/app.rs @@ -79,6 +79,8 @@ pub struct App { pub terminals: crate::terminal::Terminals, /// Newer release found by the update check (admins see it). pub update: crate::update::UpdateState, + /// Provider reachability (checked every 5 minutes). + pub health: crate::health::Health, } /// What a user may do with a session. @@ -118,6 +120,7 @@ impl App { lsp: Default::default(), push: crate::push::Push::new(&data), terminals: Default::default(), + health: Default::default(), update: Default::default(), }); crate::auth::bootstrap(&app)?; diff --git a/server/src/config.rs b/server/src/config.rs index 1dfd45f..285d3dc 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -256,6 +256,9 @@ pub struct ServerConfig { pub max_upload_mb: u64, /// Where to look for new releases (Gitea/GitHub "latest release" API); empty = no check. pub update_url: String, + /// Bearer token for GET /metrics (Prometheus); empty = /metrics disabled. + #[serde(skip_serializing_if = "String::is_empty")] + pub metrics_token: String, } impl Default for ServerConfig { @@ -275,6 +278,7 @@ impl Default for ServerConfig { budget_action: "warn".into(), max_upload_mb: 500, update_url: "https://git.familie-zink.org/api/v1/repos/zink/codecraft/releases/latest".into(), + metrics_token: String::new(), } } } diff --git a/server/src/health.rs b/server/src/health.rs new file mode 100644 index 0000000..ce78708 --- /dev/null +++ b/server/src/health.rs @@ -0,0 +1,72 @@ +//! Provider status: every 5 minutes each provider's model list is fetched +//! (free, no tokens) to see whether the host answers. + +use crate::app::App; +use serde::Serialize; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +#[derive(Debug, Clone, Serialize)] +pub struct ProviderStatus { + pub up: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + pub latency_ms: u64, + pub checked_at: i64, +} + +#[derive(Default)] +pub struct Health { + status: Mutex>, +} + +impl Health { + pub fn snapshot(&self) -> Vec<(String, bool)> { + self.status.lock().unwrap().iter().map(|(k, v)| (k.clone(), v.up)).collect() + } + + pub fn all(&self) -> BTreeMap { + self.status.lock().unwrap().clone() + } +} + +pub async fn check_all(app: &App) { + let registry = app.registry(); + let mut found = BTreeMap::new(); + let checks = registry.providers.iter().map(|(name, entry)| { + let provider = entry.provider.clone(); + let name = name.clone(); + async move { + let start = Instant::now(); + let res = tokio::time::timeout(Duration::from_secs(15), provider.list_models()).await; + let error = match res { + Ok(Ok(_)) => None, + Ok(Err(e)) => Some(e.to_string().chars().take(300).collect()), + Err(_) => Some("timed out".to_string()), + }; + (name, ProviderStatus { up: error.is_none(), error, latency_ms: start.elapsed().as_millis() as u64, checked_at: chrono_now() }) + } + }); + for (name, st) in futures::future::join_all(checks).await { + if !st.up { + tracing::warn!(provider = %name, error = st.error.as_deref().unwrap_or(""), "provider unreachable"); + } + found.insert(name, st); + } + *app.health.status.lock().unwrap() = found; +} + +fn chrono_now() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0) +} + +pub fn spawn_checker(app: Arc) { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + loop { + check_all(&app).await; + tokio::time::sleep(Duration::from_secs(300)).await; + } + }); +} diff --git a/server/src/main.rs b/server/src/main.rs index 59f3617..c3f5de1 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -10,12 +10,14 @@ mod docs; mod export; mod files; mod git; +mod health; mod gitcreds; mod hooks; mod images; mod jobs; mod lsp; mod mentions; +mod metrics; mod mcp; mod permission; mod prompts; @@ -36,9 +38,13 @@ use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,codecraft=info"))) - .init(); + // CODECRAFT_LOG_FORMAT=json: one JSON object per line (Loki, Elastic, …). + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,codecraft=info")); + if std::env::var("CODECRAFT_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json")) { + tracing_subscriber::fmt().json().flatten_event(true).with_current_span(false).with_env_filter(filter).init(); + } else { + tracing_subscriber::fmt().with_env_filter(filter).init(); + } // In Docker, HOME lives on the data volume; make sure it exists for git/npm. if let Ok(home) = std::env::var("HOME") { @@ -57,6 +63,7 @@ async fn main() -> Result<()> { terminal::spawn_reaper(app.clone()); jobs::spawn_scheduler(app.clone()); update::spawn_checker(app.clone()); + health::spawn_checker(app.clone()); let router = api::router(app); let listener = tokio::net::TcpListener::bind(&bind).await?; tracing::info!("listening on http://{bind}"); diff --git a/server/src/metrics.rs b/server/src/metrics.rs new file mode 100644 index 0000000..1e50e5d --- /dev/null +++ b/server/src/metrics.rs @@ -0,0 +1,74 @@ +//! Prometheus metrics (GET /metrics with the configured bearer token). + +use crate::api::AppState; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use std::fmt::Write; + +fn esc(s: &str) -> String { + s.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', " ") +} + +pub async fn metrics(State(app): State, headers: HeaderMap) -> Response { + let token = app.config().server.metrics_token.clone(); + if token.is_empty() { + return (StatusCode::NOT_FOUND, "metrics are disabled (set server.metrics_token)").into_response(); + } + let given = headers.get(header::AUTHORIZATION).and_then(|v| v.to_str().ok()).and_then(|v| v.strip_prefix("Bearer ")).unwrap_or(""); + if given.len() != token.len() || given.bytes().zip(token.bytes()).fold(0u8, |a, (x, y)| a | (x ^ y)) != 0 { + return (StatusCode::UNAUTHORIZED, "invalid metrics token").into_response(); + } + let mut out = String::new(); + let _ = writeln!(out, "# HELP codecraft_info Build information.\n# TYPE codecraft_info gauge"); + let _ = writeln!(out, "codecraft_info{{version=\"{}\",commit=\"{}\"}} 1", crate::update::VERSION, crate::update::commit()); + let busy = app.store.list_sessions(None).map(|l| l.iter().filter(|s| app.is_busy(&s.id)).count()).unwrap_or(0); + let gauges = [ + ("codecraft_users", "Registered users.", app.store.count("users") as f64), + ("codecraft_sessions", "Chat sessions (incl. subagents).", app.store.count("sessions") as f64), + ("codecraft_sessions_busy", "Sessions with a running agent.", busy as f64), + ("codecraft_permissions_pending", "Open permission requests.", app.pending_events().iter().filter(|e| matches!(e, crate::types::Event::PermissionRequest { .. })).count() as f64), + ("codecraft_questions_pending", "Open questions to users.", app.pending_events().iter().filter(|e| matches!(e, crate::types::Event::QuestionRequest { .. })).count() as f64), + ("codecraft_automations", "Registered automations.", app.store.count("jobs") as f64), + ("codecraft_push_subscriptions", "Devices with push notifications.", app.store.count("push_subs") as f64), + ("codecraft_sandbox_available", "1 if bubblewrap works.", if crate::sandbox::available() { 1.0 } else { 0.0 }), + ]; + for (name, help, v) in gauges { + let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} gauge\n{name} {v}"); + } + let totals = app.store.usage_totals().unwrap_or_default(); + let series: [(&str, &str, &str); 9] = [ + ("codecraft_model_requests_total", "Model requests.", "counter"), + ("codecraft_model_errors_total", "Failed model requests.", "counter"), + ("codecraft_tokens_input_total", "Input tokens.", "counter"), + ("codecraft_tokens_output_total", "Output tokens.", "counter"), + ("codecraft_tokens_cache_read_total", "Cache-read tokens.", "counter"), + ("codecraft_tokens_cache_write_total", "Cache-write tokens.", "counter"), + ("codecraft_cost_total", "Recorded cost in the configured currency.", "counter"), + ("codecraft_model_duration_seconds_total", "Time spent in model requests.", "counter"), + ("codecraft_provider_up", "1 if the provider answered its last health check.", "gauge"), + ]; + for (i, (name, help, kind)) in series.iter().enumerate() { + let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}"); + if i == 8 { + for (p, up) in app.health.snapshot() { + let _ = writeln!(out, "{name}{{provider=\"{}\"}} {}", esc(&p), if up { 1 } else { 0 }); + } + continue; + } + for (p, m, req, err, inp, outp, cr, cw, cost, dur) in &totals { + let v = match i { + 0 => *req as f64, + 1 => *err as f64, + 2 => *inp as f64, + 3 => *outp as f64, + 4 => *cr as f64, + 5 => *cw as f64, + 6 => *cost, + _ => *dur as f64 / 1000.0, + }; + let _ = writeln!(out, "{name}{{provider=\"{}\",model=\"{}\"}} {v}", esc(p), esc(m)); + } + } + ([(header::CONTENT_TYPE, "text/plain; version=0.0.4; charset=utf-8")], out).into_response() +} diff --git a/server/src/store.rs b/server/src/store.rs index 2ee3a19..3d90731 100644 --- a/server/src/store.rs +++ b/server/src/store.rs @@ -964,6 +964,25 @@ impl Store { // --- audit log & backup ---------------------------------------------------------- impl Store { + // --- metrics --------------------------------------------------------------- + + /// All-time usage per provider/model: (provider, model, requests, errors, in, out, cache_read, cache_write, cost, duration_ms). + pub fn usage_totals(&self) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT provider, model, COUNT(*), COALESCE(SUM(error),0), COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0), + COALESCE(SUM(cache_read_tokens),0), COALESCE(SUM(cache_write_tokens),0), COALESCE(SUM(cost),0), COALESCE(SUM(duration_ms),0) + FROM usage GROUP BY provider, model", + )?; + let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?, r.get(7)?, r.get(8)?, r.get(9)?)))?; + Ok(rows.collect::>()?) + } + + pub fn count(&self, table: &str) -> i64 { + // Table names are fixed strings from the code, never user input. + self.conn.lock().unwrap().query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0)).unwrap_or(0) + } + // --- scheduled jobs ------------------------------------------------------- fn job_row(r: &rusqlite::Row) -> rusqlite::Result { diff --git a/web/src/components/Settings.tsx b/web/src/components/Settings.tsx index 2772791..81e98a1 100644 --- a/web/src/components/Settings.tsx +++ b/web/src/components/Settings.tsx @@ -371,7 +371,9 @@ function cleanModel(m: ModelRow) { return out; } -function ProviderEditor({ name, p, reload }: { name: string; p: Json; reload: () => void }) { +type ProviderStatus = { up: boolean; error?: string; latency_ms: number; checked_at: number }; + +function ProviderEditor({ name, p, reload, status }: { name: string; p: Json; reload: () => void; status?: ProviderStatus }) { const t = useT(); const [form, setForm] = useState({ ...p, api_key: p.api_key === MASK ? "" : p.api_key, admin_key: "" }); const [priceNote, setPriceNote] = useState(); @@ -436,6 +438,15 @@ function ProviderEditor({ name, p, reload }: { name: string; p: Json; reload: () {name} {form.kind} {t("settings.modelCount", { n: models.length || discovered.length })} + {status && ( + + + {status.up ? t("settings.providerUp", { ms: status.latency_ms }) : t("settings.providerDown")} + + )} + {Object.entries(providers).map(([n, p]) => ( - + ))}

{t("settings.addProvider")}

diff --git a/web/src/i18n/de.ts b/web/src/i18n/de.ts index da1672c..7e624e0 100644 --- a/web/src/i18n/de.ts +++ b/web/src/i18n/de.ts @@ -569,6 +569,10 @@ export const de: Messages = { "settings.priceCacheWrite": "Cache schreiben", "settings.addProvider": "Provider hinzufügen", "settings.providerName": "Name (optional)", + "settings.providerUp": "erreichbar · {ms} ms", + "settings.providerDown": "nicht erreichbar", + "settings.providerCheck": "Provider prüfen", + "settings.providerChecking": "Prüfe…", "settings.reloadProviders": "Provider & Modelle neu laden", "settings.deleteProviderConfirm": "Provider „{name}“ löschen?", "settings.invalidName": "Namen dürfen nur Buchstaben, Ziffern, - und _ enthalten.", diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 88fa93f..0f73909 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -568,6 +568,10 @@ export const en = { "settings.priceCacheWrite": "Cache write", "settings.addProvider": "Add provider", "settings.providerName": "Name (optional)", + "settings.providerUp": "reachable · {ms} ms", + "settings.providerDown": "unreachable", + "settings.providerCheck": "Check providers", + "settings.providerChecking": "Checking…", "settings.reloadProviders": "Reload providers & models", "settings.deleteProviderConfirm": "Delete provider \"{name}\"?", "settings.invalidName": "Names may only contain letters, digits, - and _.",