Add Prometheus metrics, JSON logs and provider health checks

- GET /metrics (bearer server.metrics_token): build info, users, sessions,
  pending requests, per-model request/token/cost counters, provider_up
- CODECRAFT_LOG_FORMAT=json for one JSON object per log line
- Providers are probed every 5 minutes via their model list; status shown
  in the provider settings with a manual re-check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/src/metrics.rs

@@ -0,0 +1,74 @@
+//! Prometheus metrics (GET /metrics with the configured bearer token).
+
+use crate::api::AppState;
+use axum::extract::State;
+use axum::http::{HeaderMap, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use std::fmt::Write;
+
+fn esc(s: &str) -> String {
+    s.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', " ")
+}
+
+pub async fn metrics(State(app): State<AppState>, headers: HeaderMap) -> Response {
+    let token = app.config().server.metrics_token.clone();
+    if token.is_empty() {
+        return (StatusCode::NOT_FOUND, "metrics are disabled (set server.metrics_token)").into_response();
+    }
+    let given = headers.get(header::AUTHORIZATION).and_then(|v| v.to_str().ok()).and_then(|v| v.strip_prefix("Bearer ")).unwrap_or("");
+    if given.len() != token.len() || given.bytes().zip(token.bytes()).fold(0u8, |a, (x, y)| a | (x ^ y)) != 0 {
+        return (StatusCode::UNAUTHORIZED, "invalid metrics token").into_response();
+    }
+    let mut out = String::new();
+    let _ = writeln!(out, "# HELP codecraft_info Build information.\n# TYPE codecraft_info gauge");
+    let _ = writeln!(out, "codecraft_info{{version=\"{}\",commit=\"{}\"}} 1", crate::update::VERSION, crate::update::commit());
+    let busy = app.store.list_sessions(None).map(|l| l.iter().filter(|s| app.is_busy(&s.id)).count()).unwrap_or(0);
+    let gauges = [
+        ("codecraft_users", "Registered users.", app.store.count("users") as f64),
+        ("codecraft_sessions", "Chat sessions (incl. subagents).", app.store.count("sessions") as f64),
+        ("codecraft_sessions_busy", "Sessions with a running agent.", busy as f64),
+        ("codecraft_permissions_pending", "Open permission requests.", app.pending_events().iter().filter(|e| matches!(e, crate::types::Event::PermissionRequest { .. })).count() as f64),
+        ("codecraft_questions_pending", "Open questions to users.", app.pending_events().iter().filter(|e| matches!(e, crate::types::Event::QuestionRequest { .. })).count() as f64),
+        ("codecraft_automations", "Registered automations.", app.store.count("jobs") as f64),
+        ("codecraft_push_subscriptions", "Devices with push notifications.", app.store.count("push_subs") as f64),
+        ("codecraft_sandbox_available", "1 if bubblewrap works.", if crate::sandbox::available() { 1.0 } else { 0.0 }),
+    ];
+    for (name, help, v) in gauges {
+        let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} gauge\n{name} {v}");
+    }
+    let totals = app.store.usage_totals().unwrap_or_default();
+    let series: [(&str, &str, &str); 9] = [
+        ("codecraft_model_requests_total", "Model requests.", "counter"),
+        ("codecraft_model_errors_total", "Failed model requests.", "counter"),
+        ("codecraft_tokens_input_total", "Input tokens.", "counter"),
+        ("codecraft_tokens_output_total", "Output tokens.", "counter"),
+        ("codecraft_tokens_cache_read_total", "Cache-read tokens.", "counter"),
+        ("codecraft_tokens_cache_write_total", "Cache-write tokens.", "counter"),
+        ("codecraft_cost_total", "Recorded cost in the configured currency.", "counter"),
+        ("codecraft_model_duration_seconds_total", "Time spent in model requests.", "counter"),
+        ("codecraft_provider_up", "1 if the provider answered its last health check.", "gauge"),
+    ];
+    for (i, (name, help, kind)) in series.iter().enumerate() {
+        let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}");
+        if i == 8 {
+            for (p, up) in app.health.snapshot() {
+                let _ = writeln!(out, "{name}{{provider=\"{}\"}} {}", esc(&p), if up { 1 } else { 0 });
+            }
+            continue;
+        }
+        for (p, m, req, err, inp, outp, cr, cw, cost, dur) in &totals {
+            let v = match i {
+                0 => *req as f64,
+                1 => *err as f64,
+                2 => *inp as f64,
+                3 => *outp as f64,
+                4 => *cr as f64,
+                5 => *cw as f64,
+                6 => *cost,
+                _ => *dur as f64 / 1000.0,
+            };
+            let _ = writeln!(out, "{name}{{provider=\"{}\",model=\"{}\"}} {v}", esc(p), esc(m));
+        }
+    }
+    ([(header::CONTENT_TYPE, "text/plain; version=0.0.4; charset=utf-8")], out).into_response()
+}