//! Provider status: every 5 minutes each provider's model list is fetched //! (free, no tokens) to see whether the host answers. use crate::app::App; use serde::Serialize; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; #[derive(Debug, Clone, Serialize)] pub struct ProviderStatus { pub up: bool, /// "ok", "auth" (key missing/rejected) or "down" (host does not answer properly). pub state: &'static str, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, pub latency_ms: u64, pub checked_at: i64, } #[derive(Default)] pub struct Health { status: Mutex>, } impl Health { pub fn snapshot(&self) -> Vec<(String, bool)> { self.status.lock().unwrap().iter().map(|(k, v)| (k.clone(), v.up)).collect() } pub fn all(&self) -> BTreeMap { self.status.lock().unwrap().clone() } } /// Fetch the model list once and classify the answer. pub async fn probe(provider: Arc) -> (ProviderStatus, usize) { let start = Instant::now(); let res = tokio::time::timeout(Duration::from_secs(15), provider.list_models()).await; let (error, count): (Option, usize) = match res { Ok(Ok(m)) => (None, m.len()), Ok(Err(e)) => (Some(e.to_string().chars().take(300).collect()), 0), Err(_) => (Some("timed out".to_string()), 0), }; // The host answered but has no model list: reachable. let error = error.filter(|e| !(e.contains("HTTP 404") || e.contains("HTTP 405"))); let state = match &error { None => "ok", Some(e) if e.contains("HTTP 401") || e.contains("HTTP 403") => "auth", Some(_) => "down", }; (ProviderStatus { up: error.is_none(), state, error, latency_ms: start.elapsed().as_millis() as u64, checked_at: chrono_now() }, count) } pub async fn check_all(app: &App) { let registry = app.registry(); let mut found = BTreeMap::new(); let checks = registry.providers.iter().map(|(name, entry)| { let provider = entry.provider.clone(); let name = name.clone(); async move { (name, probe(provider).await.0) } }); for (name, st) in futures::future::join_all(checks).await { if !st.up { let msg = if st.state == "auth" { "provider rejected the API key" } else { "provider unreachable" }; tracing::warn!(provider = %name, error = st.error.as_deref().unwrap_or(""), "{msg}"); } found.insert(name, st); } *app.health.status.lock().unwrap() = found; } fn chrono_now() -> i64 { std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0) } pub fn spawn_checker(app: Arc) { tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(10)).await; loop { check_all(&app).await; tokio::time::sleep(Duration::from_secs(300)).await; } }); }