Add Prometheus metrics, JSON logs and provider health checks

- GET /metrics (bearer server.metrics_token): build info, users, sessions,
  pending requests, per-model request/token/cost counters, provider_up
- CODECRAFT_LOG_FORMAT=json for one JSON object per log line
- Providers are probed every 5 minutes via their model list; status shown
  in the provider settings with a manual re-check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/Cargo.toml

@@ -37,7 +37,7 @@ toml = "1"
 toml_edit = "0.25"
 tower-http = { version = "0.7", features = ["fs", "cors", "trace"] }
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 uuid = { version = "1", features = ["v4", "v7"] }
 p256 = { version = "0.14", features = ["ecdh", "ecdsa"] }
 hkdf = "0.13"

server/src/api.rs

@@ -121,6 +121,7 @@ pub fn router(app: AppState) -> Router {
         .route("/tools", get(crate::reverse::status))
         .route("/tools/ghidra/install", post(crate::reverse::install_ghidra))
         .route("/providers/{name}/prices", get(provider_prices))
+        .route("/providers/status", get(provider_status).post(recheck_providers))
         .route("/images", get(list_images).post(create_images))
         .route("/images/config", get(images_config))
         .route("/images/{id}", delete(delete_image))
@@ -162,6 +163,7 @@ pub fn router(app: AppState) -> Router {
     Router::new()
         .nest("/api", api)
         .route("/hooks/{id}", post(incoming_webhook))
+        .route("/metrics", get(crate::metrics::metrics))
         .route("/cli/install.sh", get(cli_install_script))
         .route("/cli/download/{target}", get(cli_download))
         .fallback_service(static_files)
@@ -1073,6 +1075,16 @@ async fn health(State(app): State<AppState>) -> Json<Value> {
     Json(json!({"ok": true, "version": crate::update::VERSION, "commit": crate::update::commit(), "sandbox": crate::sandbox::available(), "providers": app.registry().providers.len()}))
 }
 
+async fn provider_status(State(app): State<AppState>) -> Json<Value> {
+    Json(json!(app.health.all()))
+}
+
+async fn recheck_providers(Extension(user): Extension<User>, State(app): State<AppState>) -> ApiResult {
+    require_admin(&user)?;
+    crate::health::check_all(&app).await;
+    Ok(Json(json!(app.health.all())))
+}
+
 // --- budget -------------------------------------------------------------------
 
 /// Budgets concern the whole installation: admins only.

server/src/app.rs

@@ -79,6 +79,8 @@ pub struct App {
     pub terminals: crate::terminal::Terminals,
     /// Newer release found by the update check (admins see it).
     pub update: crate::update::UpdateState,
+    /// Provider reachability (checked every 5 minutes).
+    pub health: crate::health::Health,
 }
 
 /// What a user may do with a session.
@@ -118,6 +120,7 @@ impl App {
             lsp: Default::default(),
             push: crate::push::Push::new(&data),
             terminals: Default::default(),
+            health: Default::default(),
             update: Default::default(),
         });
         crate::auth::bootstrap(&app)?;

server/src/config.rs

@@ -256,6 +256,9 @@ pub struct ServerConfig {
     pub max_upload_mb: u64,
     /// Where to look for new releases (Gitea/GitHub "latest release" API); empty = no check.
     pub update_url: String,
+    /// Bearer token for GET /metrics (Prometheus); empty = /metrics disabled.
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub metrics_token: String,
 }
 
 impl Default for ServerConfig {
@@ -275,6 +278,7 @@ impl Default for ServerConfig {
             budget_action: "warn".into(),
             max_upload_mb: 500,
             update_url: "https://git.familie-zink.org/api/v1/repos/zink/codecraft/releases/latest".into(),
+            metrics_token: String::new(),
         }
     }
 }

server/src/health.rs

@@ -0,0 +1,72 @@
+//! Provider status: every 5 minutes each provider's model list is fetched
+//! (free, no tokens) to see whether the host answers.
+
+use crate::app::App;
+use serde::Serialize;
+use std::collections::BTreeMap;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct ProviderStatus {
+    pub up: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+    pub latency_ms: u64,
+    pub checked_at: i64,
+}
+
+#[derive(Default)]
+pub struct Health {
+    status: Mutex<BTreeMap<String, ProviderStatus>>,
+}
+
+impl Health {
+    pub fn snapshot(&self) -> Vec<(String, bool)> {
+        self.status.lock().unwrap().iter().map(|(k, v)| (k.clone(), v.up)).collect()
+    }
+
+    pub fn all(&self) -> BTreeMap<String, ProviderStatus> {
+        self.status.lock().unwrap().clone()
+    }
+}
+
+pub async fn check_all(app: &App) {
+    let registry = app.registry();
+    let mut found = BTreeMap::new();
+    let checks = registry.providers.iter().map(|(name, entry)| {
+        let provider = entry.provider.clone();
+        let name = name.clone();
+        async move {
+            let start = Instant::now();
+            let res = tokio::time::timeout(Duration::from_secs(15), provider.list_models()).await;
+            let error = match res {
+                Ok(Ok(_)) => None,
+                Ok(Err(e)) => Some(e.to_string().chars().take(300).collect()),
+                Err(_) => Some("timed out".to_string()),
+            };
+            (name, ProviderStatus { up: error.is_none(), error, latency_ms: start.elapsed().as_millis() as u64, checked_at: chrono_now() })
+        }
+    });
+    for (name, st) in futures::future::join_all(checks).await {
+        if !st.up {
+            tracing::warn!(provider = %name, error = st.error.as_deref().unwrap_or(""), "provider unreachable");
+        }
+        found.insert(name, st);
+    }
+    *app.health.status.lock().unwrap() = found;
+}
+
+fn chrono_now() -> i64 {
+    std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0)
+}
+
+pub fn spawn_checker(app: Arc<App>) {
+    tokio::spawn(async move {
+        tokio::time::sleep(Duration::from_secs(10)).await;
+        loop {
+            check_all(&app).await;
+            tokio::time::sleep(Duration::from_secs(300)).await;
+        }
+    });
+}

server/src/main.rs

@@ -10,12 +10,14 @@ mod docs;
 mod export;
 mod files;
 mod git;
+mod health;
 mod gitcreds;
 mod hooks;
 mod images;
 mod jobs;
 mod lsp;
 mod mentions;
+mod metrics;
 mod mcp;
 mod permission;
 mod prompts;
@@ -36,9 +38,13 @@ use tracing_subscriber::EnvFilter;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    tracing_subscriber::fmt()
-        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,codecraft=info")))
-        .init();
+    // CODECRAFT_LOG_FORMAT=json: one JSON object per line (Loki, Elastic, …).
+    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,codecraft=info"));
+    if std::env::var("CODECRAFT_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json")) {
+        tracing_subscriber::fmt().json().flatten_event(true).with_current_span(false).with_env_filter(filter).init();
+    } else {
+        tracing_subscriber::fmt().with_env_filter(filter).init();
+    }
 
     // In Docker, HOME lives on the data volume; make sure it exists for git/npm.
     if let Ok(home) = std::env::var("HOME") {
@@ -57,6 +63,7 @@ async fn main() -> Result<()> {
     terminal::spawn_reaper(app.clone());
     jobs::spawn_scheduler(app.clone());
     update::spawn_checker(app.clone());
+    health::spawn_checker(app.clone());
     let router = api::router(app);
     let listener = tokio::net::TcpListener::bind(&bind).await?;
     tracing::info!("listening on http://{bind}");

server/src/metrics.rs

@@ -0,0 +1,74 @@
+//! Prometheus metrics (GET /metrics with the configured bearer token).
+
+use crate::api::AppState;
+use axum::extract::State;
+use axum::http::{HeaderMap, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use std::fmt::Write;
+
+fn esc(s: &str) -> String {
+    s.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', " ")
+}
+
+pub async fn metrics(State(app): State<AppState>, headers: HeaderMap) -> Response {
+    let token = app.config().server.metrics_token.clone();
+    if token.is_empty() {
+        return (StatusCode::NOT_FOUND, "metrics are disabled (set server.metrics_token)").into_response();
+    }
+    let given = headers.get(header::AUTHORIZATION).and_then(|v| v.to_str().ok()).and_then(|v| v.strip_prefix("Bearer ")).unwrap_or("");
+    if given.len() != token.len() || given.bytes().zip(token.bytes()).fold(0u8, |a, (x, y)| a | (x ^ y)) != 0 {
+        return (StatusCode::UNAUTHORIZED, "invalid metrics token").into_response();
+    }
+    let mut out = String::new();
+    let _ = writeln!(out, "# HELP codecraft_info Build information.\n# TYPE codecraft_info gauge");
+    let _ = writeln!(out, "codecraft_info{{version=\"{}\",commit=\"{}\"}} 1", crate::update::VERSION, crate::update::commit());
+    let busy = app.store.list_sessions(None).map(|l| l.iter().filter(|s| app.is_busy(&s.id)).count()).unwrap_or(0);
+    let gauges = [
+        ("codecraft_users", "Registered users.", app.store.count("users") as f64),
+        ("codecraft_sessions", "Chat sessions (incl. subagents).", app.store.count("sessions") as f64),
+        ("codecraft_sessions_busy", "Sessions with a running agent.", busy as f64),
+        ("codecraft_permissions_pending", "Open permission requests.", app.pending_events().iter().filter(|e| matches!(e, crate::types::Event::PermissionRequest { .. })).count() as f64),
+        ("codecraft_questions_pending", "Open questions to users.", app.pending_events().iter().filter(|e| matches!(e, crate::types::Event::QuestionRequest { .. })).count() as f64),
+        ("codecraft_automations", "Registered automations.", app.store.count("jobs") as f64),
+        ("codecraft_push_subscriptions", "Devices with push notifications.", app.store.count("push_subs") as f64),
+        ("codecraft_sandbox_available", "1 if bubblewrap works.", if crate::sandbox::available() { 1.0 } else { 0.0 }),
+    ];
+    for (name, help, v) in gauges {
+        let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} gauge\n{name} {v}");
+    }
+    let totals = app.store.usage_totals().unwrap_or_default();
+    let series: [(&str, &str, &str); 9] = [
+        ("codecraft_model_requests_total", "Model requests.", "counter"),
+        ("codecraft_model_errors_total", "Failed model requests.", "counter"),
+        ("codecraft_tokens_input_total", "Input tokens.", "counter"),
+        ("codecraft_tokens_output_total", "Output tokens.", "counter"),
+        ("codecraft_tokens_cache_read_total", "Cache-read tokens.", "counter"),
+        ("codecraft_tokens_cache_write_total", "Cache-write tokens.", "counter"),
+        ("codecraft_cost_total", "Recorded cost in the configured currency.", "counter"),
+        ("codecraft_model_duration_seconds_total", "Time spent in model requests.", "counter"),
+        ("codecraft_provider_up", "1 if the provider answered its last health check.", "gauge"),
+    ];
+    for (i, (name, help, kind)) in series.iter().enumerate() {
+        let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}");
+        if i == 8 {
+            for (p, up) in app.health.snapshot() {
+                let _ = writeln!(out, "{name}{{provider=\"{}\"}} {}", esc(&p), if up { 1 } else { 0 });
+            }
+            continue;
+        }
+        for (p, m, req, err, inp, outp, cr, cw, cost, dur) in &totals {
+            let v = match i {
+                0 => *req as f64,
+                1 => *err as f64,
+                2 => *inp as f64,
+                3 => *outp as f64,
+                4 => *cr as f64,
+                5 => *cw as f64,
+                6 => *cost,
+                _ => *dur as f64 / 1000.0,
+            };
+            let _ = writeln!(out, "{name}{{provider=\"{}\",model=\"{}\"}} {v}", esc(p), esc(m));
+        }
+    }
+    ([(header::CONTENT_TYPE, "text/plain; version=0.0.4; charset=utf-8")], out).into_response()
+}

server/src/store.rs

@@ -964,6 +964,25 @@ impl Store {
 // --- audit log & backup ----------------------------------------------------------
 
 impl Store {
+    // --- metrics ---------------------------------------------------------------
+
+    /// All-time usage per provider/model: (provider, model, requests, errors, in, out, cache_read, cache_write, cost, duration_ms).
+    pub fn usage_totals(&self) -> Result<Vec<(String, String, i64, i64, i64, i64, i64, i64, f64, i64)>> {
+        let conn = self.conn.lock().unwrap();
+        let mut stmt = conn.prepare(
+            "SELECT provider, model, COUNT(*), COALESCE(SUM(error),0), COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
+                    COALESCE(SUM(cache_read_tokens),0), COALESCE(SUM(cache_write_tokens),0), COALESCE(SUM(cost),0), COALESCE(SUM(duration_ms),0)
+             FROM usage GROUP BY provider, model",
+        )?;
+        let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?, r.get(7)?, r.get(8)?, r.get(9)?)))?;
+        Ok(rows.collect::<std::result::Result<_, _>>()?)
+    }
+
+    pub fn count(&self, table: &str) -> i64 {
+        // Table names are fixed strings from the code, never user input.
+        self.conn.lock().unwrap().query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0)).unwrap_or(0)
+    }
+
     // --- scheduled jobs -------------------------------------------------------
 
     fn job_row(r: &rusqlite::Row) -> rusqlite::Result<crate::jobs::Job> {