//! Backups: a zip with the database snapshot and the config. Created on //! demand (download) and daily into /backups, optionally uploaded to //! an S3-compatible bucket. use crate::app::App; use crate::config::{S3Config, expand_env}; use anyhow::{Context, Result, anyhow, bail}; use chrono::{Local, Timelike}; use serde::Serialize; use sha2::{Digest, Sha256}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; /// Outcome of the last automatic run (shown in the settings). #[derive(Debug, Clone, Default, Serialize)] pub struct LastRun { pub at: i64, pub file: String, pub size: u64, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, /// None = no bucket configured. #[serde(skip_serializing_if = "Option::is_none")] pub s3: Option, } #[derive(Default)] pub struct BackupState { pub last: Mutex>, running: tokio::sync::Mutex<()>, } /// Zip with config.toml and a consistent snapshot of codecraft.db. pub async fn create_zip(app: &App) -> Result> { let tmp = std::env::temp_dir().join(format!("cc-backup-{}.db", crate::types::new_id())); app.store.snapshot(&tmp)?; let config = app.config_text()?; let db = tokio::fs::read(&tmp).await; let _ = tokio::fs::remove_file(&tmp).await; let db = db?; tokio::task::spawn_blocking(move || -> Result> { use std::io::Write; let mut buf = std::io::Cursor::new(Vec::new()); { let mut z = zip::ZipWriter::new(&mut buf); let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); z.start_file("config.toml", opts)?; z.write_all(config.as_bytes())?; z.start_file("codecraft.db", opts)?; z.write_all(&db)?; z.finish()?; } Ok(buf.into_inner()) }) .await? } /// Directory a restore is staged in; applied at the next start (see `apply_pending_restore`). fn pending_dir(data_dir: &std::path::Path) -> PathBuf { data_dir.join("restore-pending") } /// Read config.toml and codecraft.db from a backup zip and check both. fn unpack(bytes: &[u8]) -> Result<(String, Vec)> { use std::io::Read; let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).context("not a zip file")?; let mut config = String::new(); z.by_name("config.toml").context("config.toml missing in the backup")?.read_to_string(&mut config)?; let mut db = Vec::new(); z.by_name("codecraft.db").context("codecraft.db missing in the backup")?.read_to_end(&mut db)?; if !db.starts_with(b"SQLite format 3\0") { bail!("codecraft.db in the backup is not an SQLite database"); } crate::config::Config::parse(&config).context("config.toml in the backup is invalid")?; Ok((config, db)) } /// Check a backup, back up the current state, and stage the backup for the next start. pub async fn stage_restore(app: &App, bytes: Vec) -> Result<()> { let (config, db) = tokio::task::spawn_blocking(move || unpack(&bytes)).await??; // Safety net: the current state as a normal backup file. run_named(app, "-before-restore").await.context("could not back up the current state first")?; let dir = pending_dir(&app.config().data_dir()); let _ = tokio::fs::remove_dir_all(&dir).await; tokio::fs::create_dir_all(&dir).await?; tokio::fs::write(dir.join("codecraft.db"), db).await?; tokio::fs::write(dir.join("config.toml"), config).await?; Ok(()) } /// Called at startup before the database is opened: move a staged restore into place. pub fn apply_pending_restore(data_dir: &std::path::Path, config_path: &std::path::Path) -> Result { let dir = pending_dir(data_dir); let (db, cfg) = (dir.join("codecraft.db"), dir.join("config.toml")); if !db.is_file() || !cfg.is_file() { return Ok(false); } let target = data_dir.join("codecraft.db"); for ext in ["db-wal", "db-shm"] { let _ = std::fs::remove_file(target.with_extension(ext)); } std::fs::rename(&db, &target)?; std::fs::copy(&cfg, config_path)?; std::fs::remove_dir_all(&dir)?; Ok(true) } pub fn backup_dir(app: &App) -> PathBuf { let cfg = app.config(); if cfg.backup.dir.is_empty() { cfg.data_dir().join("backups") } else { PathBuf::from(expand_env(&cfg.backup.dir)) } } #[derive(Debug, Serialize)] pub struct BackupFile { pub name: String, pub size: u64, pub modified: i64, } pub fn list(app: &App) -> Vec { let mut out: Vec = std::fs::read_dir(backup_dir(app)) .into_iter() .flatten() .flatten() .filter_map(|e| { let name = e.file_name().to_string_lossy().to_string(); if !(name.starts_with("codecraft-backup-") && name.ends_with(".zip")) { return None; } let meta = e.metadata().ok()?; let modified = meta.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?.as_millis() as i64; Some(BackupFile { name, size: meta.len(), modified }) }) .collect(); out.sort_by(|a, b| b.name.cmp(&a.name)); out } /// Create a backup now: local file (+ pruning) and S3 upload if configured. pub async fn run(app: &App) -> Result { run_named(app, "").await } /// Like `run`, with a suffix in the file name (e.g. "-before-restore"). pub async fn run_named(app: &App, suffix: &str) -> Result { let _guard = app.backup.running.lock().await; let cfg = app.config().backup.clone(); let dir = backup_dir(app); tokio::fs::create_dir_all(&dir).await.with_context(|| format!("create {}", dir.display()))?; let bytes = create_zip(app).await?; // Names have second resolution: never overwrite a backup made in the same second. let stamp = Local::now().format("%Y%m%d-%H%M%S").to_string(); let mut name = format!("codecraft-backup-{stamp}{suffix}.zip"); let mut n = 1; while dir.join(&name).exists() { n += 1; name = format!("codecraft-backup-{stamp}{suffix}-{n}.zip"); } let path = dir.join(&name); tokio::fs::write(&path, &bytes).await.with_context(|| format!("write {}", path.display()))?; // Restrictive permissions: the zip contains the config with API keys. #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); } for old in list(app).into_iter().skip(cfg.keep.max(1)) { let _ = tokio::fs::remove_file(dir.join(&old.name)).await; } let mut result = LastRun { at: Local::now().timestamp_millis(), file: name.clone(), size: bytes.len() as u64, error: None, s3: None }; if !cfg.s3.endpoint.trim().is_empty() { result.s3 = Some(match upload(&cfg.s3, &name, bytes).await { Ok(()) => { if cfg.s3.keep > 0 { if let Err(e) = prune(&cfg.s3).await { tracing::warn!("backup: pruning the bucket failed: {e:#}"); } } "ok".into() } Err(e) => { tracing::warn!("backup: S3 upload failed: {e:#}"); format!("{e:#}") } }); } tracing::info!(file = %name, size = result.size, s3 = ?result.s3, "backup created"); Ok(result) } pub fn spawn(app: Arc) { tokio::spawn(async move { loop { tokio::time::sleep(Duration::from_secs(60)).await; let cfg = app.config().backup.clone(); let now = Local::now(); if !cfg.enabled || now.hour() != cfg.hour.min(23) { continue; } let today = format!("codecraft-backup-{}", now.format("%Y%m%d")); if list(&app).iter().any(|b| b.name.starts_with(&today)) { continue; } let res = match run(&app).await { Ok(r) => r, Err(e) => { tracing::error!("backup failed: {e:#}"); LastRun { at: now.timestamp_millis(), error: Some(format!("{e:#}")), ..Default::default() } } }; let failed = res.error.is_some(); *app.backup.last.lock().unwrap() = Some(res); if failed { // Do not retry every minute. tokio::time::sleep(Duration::from_secs(3600)).await; } } }); } // --- S3 (Signature V4, path-style URLs) ------------------------------------------ fn hex(b: &[u8]) -> String { b.iter().map(|x| format!("{x:02x}")).collect() } fn sha256_hex(b: &[u8]) -> String { hex(&Sha256::digest(b)) } fn hmac(key: &[u8], msg: &str) -> Vec { use hmac::{KeyInit, Mac}; let mut mac = as KeyInit>::new_from_slice(key).expect("hmac key"); mac.update(msg.as_bytes()); mac.finalize().into_bytes().to_vec() } /// RFC 3986 encoding as AWS wants it ('/' kept in paths). fn enc(s: &str, keep_slash: bool) -> String { let mut out = String::new(); for b in s.bytes() { match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(b as char), b'/' if keep_slash => out.push('/'), _ => out.push_str(&format!("%{b:02X}")), } } out } struct Signed { url: String, headers: Vec<(String, String)>, } /// Sign a request. `query` must not be encoded yet. fn sign(s3: &S3Config, method: &str, key: &str, query: &[(&str, &str)], payload_hash: &str, now: chrono::DateTime) -> Result { let endpoint = s3.endpoint.trim().trim_end_matches('/'); let url = reqwest::Url::parse(endpoint).context("invalid S3 endpoint")?; let host = match url.port() { Some(p) => format!("{}:{p}", url.host_str().ok_or_else(|| anyhow!("S3 endpoint without host"))?), None => url.host_str().ok_or_else(|| anyhow!("S3 endpoint without host"))?.to_string(), }; let region = if s3.region.trim().is_empty() { "us-east-1" } else { s3.region.trim() }; let path = format!("{}/{}{}", url.path().trim_end_matches('/'), enc(&s3.bucket, false), if key.is_empty() { "/".to_string() } else { format!("/{}", enc(key, true)) }); let mut q: Vec<(String, String)> = query.iter().map(|(k, v)| (enc(k, false), enc(v, false))).collect(); q.sort(); let qs = q.iter().map(|(k, v)| format!("{k}={v}")).collect::>().join("&"); let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); let date = now.format("%Y%m%d").to_string(); let canonical = format!("{method}\n{path}\n{qs}\nhost:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{amz_date}\n\nhost;x-amz-content-sha256;x-amz-date\n{payload_hash}"); let scope = format!("{date}/{region}/s3/aws4_request"); let to_sign = format!("AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", sha256_hex(canonical.as_bytes())); let secret = expand_env(&s3.secret_key); let k = hmac(format!("AWS4{secret}").as_bytes(), &date); let k = hmac(&k, region); let k = hmac(&k, "s3"); let k = hmac(&k, "aws4_request"); let signature = hex(&hmac(&k, &to_sign)); let auth = format!("AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={signature}", expand_env(&s3.access_key)); let base = format!("{}://{host}{path}", url.scheme()); Ok(Signed { url: if qs.is_empty() { base } else { format!("{base}?{qs}") }, headers: vec![("x-amz-date".into(), amz_date), ("x-amz-content-sha256".into(), payload_hash.into()), ("authorization".into(), auth)], }) } async fn send(s3: &S3Config, method: reqwest::Method, key: &str, query: &[(&str, &str)], body: Vec) -> Result { let signed = sign(s3, method.as_str(), key, query, &sha256_hex(&body), chrono::Utc::now())?; let mut req = crate::provider::http_client().request(method, &signed.url).timeout(Duration::from_secs(600)); for (k, v) in &signed.headers { req = req.header(k, v); } let resp = req.body(body).send().await?; let status = resp.status(); let text = resp.text().await.unwrap_or_default(); if !status.is_success() { let code = text.split("").nth(1).and_then(|s| s.split("").next()).unwrap_or(""); bail!("S3 returned HTTP {status}{}", if code.is_empty() { String::new() } else { format!(" ({code})") }); } Ok(text) } fn object_key(s3: &S3Config, name: &str) -> String { format!("{}{name}", s3.prefix.trim_start_matches('/')) } pub async fn upload(s3: &S3Config, name: &str, bytes: Vec) -> Result<()> { if s3.bucket.trim().is_empty() { bail!("no S3 bucket configured"); } send(s3, reqwest::Method::PUT, &object_key(s3, name), &[], bytes).await.map(|_| ()) } /// Delete the oldest backups in the bucket beyond `keep`. async fn prune(s3: &S3Config) -> Result<()> { let prefix = object_key(s3, "codecraft-backup-"); let xml = send(s3, reqwest::Method::GET, "", &[("list-type", "2"), ("prefix", &prefix)], Vec::new()).await?; let mut keys: Vec = xml.split("").skip(1).filter_map(|s| s.split("").next()).map(|s| s.replace("&", "&")).collect(); keys.sort_by(|a, b| b.cmp(a)); for k in keys.into_iter().skip(s3.keep) { send(s3, reqwest::Method::DELETE, &k, &[], Vec::new()).await?; } Ok(()) } #[cfg(test)] mod tests { use super::*; /// Example from the AWS SigV4 documentation (GET object with range omitted /// is not reproducible here, so check the derived signing key instead). #[test] fn signing_key_matches_aws_example() { // https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html let k = hmac(b"AWS4wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20120215"); let k = hmac(&k, "us-east-1"); let k = hmac(&k, "iam"); let k = hmac(&k, "aws4_request"); assert_eq!(hex(&k), "f4780e2d9f65fa895f9c67b32ce1baf0b0d8a43505a000a1a9e090d414db404d"); } #[test] fn encodes_keys() { assert_eq!(enc("a b/c+d", true), "a%20b/c%2Bd"); assert_eq!(enc("a/b", false), "a%2Fb"); } }