Add Prometheus metrics, JSON logs and provider health checks
- GET /metrics (bearer server.metrics_token): build info, users, sessions, pending requests, per-model request/token/cost counters, provider_up - CODECRAFT_LOG_FORMAT=json for one JSON object per log line - Providers are probed every 5 minutes via their model list; status shown in the provider settings with a manual re-check Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
10d4b64289
commit
b5de531d2e
12 files changed
+244
-6
No files matched your search
@@ -0,0 +1,72 @@
|
||||
//! Provider status: every 5 minutes each provider's model list is fetched
|
||||
//! (free, no tokens) to see whether the host answers.
|
||||
|
||||
use crate::app::App;
|
||||
use serde::Serialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ProviderStatus {
|
||||
pub up: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
pub latency_ms: u64,
|
||||
pub checked_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Health {
|
||||
status: Mutex<BTreeMap<String, ProviderStatus>>,
|
||||
}
|
||||
|
||||
impl Health {
|
||||
pub fn snapshot(&self) -> Vec<(String, bool)> {
|
||||
self.status.lock().unwrap().iter().map(|(k, v)| (k.clone(), v.up)).collect()
|
||||
}
|
||||
|
||||
pub fn all(&self) -> BTreeMap<String, ProviderStatus> {
|
||||
self.status.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn check_all(app: &App) {
|
||||
let registry = app.registry();
|
||||
let mut found = BTreeMap::new();
|
||||
let checks = registry.providers.iter().map(|(name, entry)| {
|
||||
let provider = entry.provider.clone();
|
||||
let name = name.clone();
|
||||
async move {
|
||||
let start = Instant::now();
|
||||
let res = tokio::time::timeout(Duration::from_secs(15), provider.list_models()).await;
|
||||
let error = match res {
|
||||
Ok(Ok(_)) => None,
|
||||
Ok(Err(e)) => Some(e.to_string().chars().take(300).collect()),
|
||||
Err(_) => Some("timed out".to_string()),
|
||||
};
|
||||
(name, ProviderStatus { up: error.is_none(), error, latency_ms: start.elapsed().as_millis() as u64, checked_at: chrono_now() })
|
||||
}
|
||||
});
|
||||
for (name, st) in futures::future::join_all(checks).await {
|
||||
if !st.up {
|
||||
tracing::warn!(provider = %name, error = st.error.as_deref().unwrap_or(""), "provider unreachable");
|
||||
}
|
||||
found.insert(name, st);
|
||||
}
|
||||
*app.health.status.lock().unwrap() = found;
|
||||
}
|
||||
|
||||
fn chrono_now() -> i64 {
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn spawn_checker(app: Arc<App>) {
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
loop {
|
||||
check_all(&app).await;
|
||||
tokio::time::sleep(Duration::from_secs(300)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user