Add backups with S3, daily cost push, automation templates and a browser tool
- Daily backups (database + config) into <data_dir>/backups, keep N, optional upload to any S3-compatible bucket (SigV4, bucket pruning); settings panel with run-now, list and download - Daily cost summary push at 20:00 (opt-in per user, preview button) - Automations from templates (dependency check, security review, test health, daily summary, TODO report, docs freshness) via POST /jobs/create - browser tool: headless Chromium over CDP – click, type, keys, waits, JavaScript, navigation, screenshots, console errors; requests to private network addresses are blocked (sub-resources and redirects included) - Secrets masked in the config view: S3 secret key, metrics token - Replace the remaining text arrows/refresh glyphs with SVG icons Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard ZinkandClaude Opus 5.5 committed
1 parent
873447ed17
commit
01c1484033
25 files changed
+1720
-47
No files matched your search
@@ -0,0 +1,287 @@
|
||||
//! Backups: a zip with the database snapshot and the config. Created on
|
||||
//! demand (download) and daily into <data_dir>/backups, optionally uploaded to
|
||||
//! an S3-compatible bucket.
|
||||
|
||||
use crate::app::App;
|
||||
use crate::config::{S3Config, expand_env};
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use chrono::{Local, Timelike};
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Outcome of the last automatic run (shown in the settings).
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct LastRun {
|
||||
pub at: i64,
|
||||
pub file: String,
|
||||
pub size: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
/// None = no bucket configured.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub s3: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct BackupState {
|
||||
pub last: Mutex<Option<LastRun>>,
|
||||
running: tokio::sync::Mutex<()>,
|
||||
}
|
||||
|
||||
/// Zip with config.toml and a consistent snapshot of codecraft.db.
|
||||
pub async fn create_zip(app: &App) -> Result<Vec<u8>> {
|
||||
let tmp = std::env::temp_dir().join(format!("cc-backup-{}.db", crate::types::new_id()));
|
||||
app.store.snapshot(&tmp)?;
|
||||
let config = app.config_text()?;
|
||||
let db = tokio::fs::read(&tmp).await;
|
||||
let _ = tokio::fs::remove_file(&tmp).await;
|
||||
let db = db?;
|
||||
tokio::task::spawn_blocking(move || -> Result<Vec<u8>> {
|
||||
use std::io::Write;
|
||||
let mut buf = std::io::Cursor::new(Vec::new());
|
||||
{
|
||||
let mut z = zip::ZipWriter::new(&mut buf);
|
||||
let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
|
||||
z.start_file("config.toml", opts)?;
|
||||
z.write_all(config.as_bytes())?;
|
||||
z.start_file("codecraft.db", opts)?;
|
||||
z.write_all(&db)?;
|
||||
z.finish()?;
|
||||
}
|
||||
Ok(buf.into_inner())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
pub fn backup_dir(app: &App) -> PathBuf {
|
||||
let cfg = app.config();
|
||||
if cfg.backup.dir.is_empty() { cfg.data_dir().join("backups") } else { PathBuf::from(expand_env(&cfg.backup.dir)) }
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct BackupFile {
|
||||
pub name: String,
|
||||
pub size: u64,
|
||||
pub modified: i64,
|
||||
}
|
||||
|
||||
pub fn list(app: &App) -> Vec<BackupFile> {
|
||||
let mut out: Vec<BackupFile> = std::fs::read_dir(backup_dir(app))
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.flatten()
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name().to_string_lossy().to_string();
|
||||
if !(name.starts_with("codecraft-backup-") && name.ends_with(".zip")) {
|
||||
return None;
|
||||
}
|
||||
let meta = e.metadata().ok()?;
|
||||
let modified = meta.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?.as_millis() as i64;
|
||||
Some(BackupFile { name, size: meta.len(), modified })
|
||||
})
|
||||
.collect();
|
||||
out.sort_by(|a, b| b.name.cmp(&a.name));
|
||||
out
|
||||
}
|
||||
|
||||
/// Create a backup now: local file (+ pruning) and S3 upload if configured.
|
||||
pub async fn run(app: &App) -> Result<LastRun> {
|
||||
let _guard = app.backup.running.lock().await;
|
||||
let cfg = app.config().backup.clone();
|
||||
let dir = backup_dir(app);
|
||||
tokio::fs::create_dir_all(&dir).await.with_context(|| format!("create {}", dir.display()))?;
|
||||
let bytes = create_zip(app).await?;
|
||||
let name = format!("codecraft-backup-{}.zip", Local::now().format("%Y%m%d-%H%M%S"));
|
||||
let path = dir.join(&name);
|
||||
tokio::fs::write(&path, &bytes).await.with_context(|| format!("write {}", path.display()))?;
|
||||
// Restrictive permissions: the zip contains the config with API keys.
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600));
|
||||
}
|
||||
for old in list(app).into_iter().skip(cfg.keep.max(1)) {
|
||||
let _ = tokio::fs::remove_file(dir.join(&old.name)).await;
|
||||
}
|
||||
let mut result = LastRun { at: Local::now().timestamp_millis(), file: name.clone(), size: bytes.len() as u64, error: None, s3: None };
|
||||
if !cfg.s3.endpoint.trim().is_empty() {
|
||||
result.s3 = Some(match upload(&cfg.s3, &name, bytes).await {
|
||||
Ok(()) => {
|
||||
if cfg.s3.keep > 0 {
|
||||
if let Err(e) = prune(&cfg.s3).await {
|
||||
tracing::warn!("backup: pruning the bucket failed: {e:#}");
|
||||
}
|
||||
}
|
||||
"ok".into()
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("backup: S3 upload failed: {e:#}");
|
||||
format!("{e:#}")
|
||||
}
|
||||
});
|
||||
}
|
||||
tracing::info!(file = %name, size = result.size, s3 = ?result.s3, "backup created");
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn spawn(app: Arc<App>) {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
let cfg = app.config().backup.clone();
|
||||
let now = Local::now();
|
||||
if !cfg.enabled || now.hour() != cfg.hour.min(23) {
|
||||
continue;
|
||||
}
|
||||
let today = format!("codecraft-backup-{}", now.format("%Y%m%d"));
|
||||
if list(&app).iter().any(|b| b.name.starts_with(&today)) {
|
||||
continue;
|
||||
}
|
||||
let res = match run(&app).await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::error!("backup failed: {e:#}");
|
||||
LastRun { at: now.timestamp_millis(), error: Some(format!("{e:#}")), ..Default::default() }
|
||||
}
|
||||
};
|
||||
let failed = res.error.is_some();
|
||||
*app.backup.last.lock().unwrap() = Some(res);
|
||||
if failed {
|
||||
// Do not retry every minute.
|
||||
tokio::time::sleep(Duration::from_secs(3600)).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// --- S3 (Signature V4, path-style URLs) ------------------------------------------
|
||||
|
||||
fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
fn sha256_hex(b: &[u8]) -> String {
|
||||
hex(&Sha256::digest(b))
|
||||
}
|
||||
|
||||
fn hmac(key: &[u8], msg: &str) -> Vec<u8> {
|
||||
use hmac::{KeyInit, Mac};
|
||||
let mut mac = <hmac::Hmac<Sha256> as KeyInit>::new_from_slice(key).expect("hmac key");
|
||||
mac.update(msg.as_bytes());
|
||||
mac.finalize().into_bytes().to_vec()
|
||||
}
|
||||
|
||||
/// RFC 3986 encoding as AWS wants it ('/' kept in paths).
|
||||
fn enc(s: &str, keep_slash: bool) -> String {
|
||||
let mut out = String::new();
|
||||
for b in s.bytes() {
|
||||
match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(b as char),
|
||||
b'/' if keep_slash => out.push('/'),
|
||||
_ => out.push_str(&format!("%{b:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
struct Signed {
|
||||
url: String,
|
||||
headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Sign a request. `query` must not be encoded yet.
|
||||
fn sign(s3: &S3Config, method: &str, key: &str, query: &[(&str, &str)], payload_hash: &str, now: chrono::DateTime<chrono::Utc>) -> Result<Signed> {
|
||||
let endpoint = s3.endpoint.trim().trim_end_matches('/');
|
||||
let url = reqwest::Url::parse(endpoint).context("invalid S3 endpoint")?;
|
||||
let host = match url.port() {
|
||||
Some(p) => format!("{}:{p}", url.host_str().ok_or_else(|| anyhow!("S3 endpoint without host"))?),
|
||||
None => url.host_str().ok_or_else(|| anyhow!("S3 endpoint without host"))?.to_string(),
|
||||
};
|
||||
let region = if s3.region.trim().is_empty() { "us-east-1" } else { s3.region.trim() };
|
||||
let path = format!("{}/{}{}", url.path().trim_end_matches('/'), enc(&s3.bucket, false), if key.is_empty() { "/".to_string() } else { format!("/{}", enc(key, true)) });
|
||||
let mut q: Vec<(String, String)> = query.iter().map(|(k, v)| (enc(k, false), enc(v, false))).collect();
|
||||
q.sort();
|
||||
let qs = q.iter().map(|(k, v)| format!("{k}={v}")).collect::<Vec<_>>().join("&");
|
||||
let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string();
|
||||
let date = now.format("%Y%m%d").to_string();
|
||||
let canonical = format!("{method}\n{path}\n{qs}\nhost:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{amz_date}\n\nhost;x-amz-content-sha256;x-amz-date\n{payload_hash}");
|
||||
let scope = format!("{date}/{region}/s3/aws4_request");
|
||||
let to_sign = format!("AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", sha256_hex(canonical.as_bytes()));
|
||||
let secret = expand_env(&s3.secret_key);
|
||||
let k = hmac(format!("AWS4{secret}").as_bytes(), &date);
|
||||
let k = hmac(&k, region);
|
||||
let k = hmac(&k, "s3");
|
||||
let k = hmac(&k, "aws4_request");
|
||||
let signature = hex(&hmac(&k, &to_sign));
|
||||
let auth = format!("AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={signature}", expand_env(&s3.access_key));
|
||||
let base = format!("{}://{host}{path}", url.scheme());
|
||||
Ok(Signed {
|
||||
url: if qs.is_empty() { base } else { format!("{base}?{qs}") },
|
||||
headers: vec![("x-amz-date".into(), amz_date), ("x-amz-content-sha256".into(), payload_hash.into()), ("authorization".into(), auth)],
|
||||
})
|
||||
}
|
||||
|
||||
async fn send(s3: &S3Config, method: reqwest::Method, key: &str, query: &[(&str, &str)], body: Vec<u8>) -> Result<String> {
|
||||
let signed = sign(s3, method.as_str(), key, query, &sha256_hex(&body), chrono::Utc::now())?;
|
||||
let mut req = crate::provider::http_client().request(method, &signed.url).timeout(Duration::from_secs(600));
|
||||
for (k, v) in &signed.headers {
|
||||
req = req.header(k, v);
|
||||
}
|
||||
let resp = req.body(body).send().await?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
if !status.is_success() {
|
||||
let code = text.split("<Code>").nth(1).and_then(|s| s.split("</Code>").next()).unwrap_or("");
|
||||
bail!("S3 returned HTTP {status}{}", if code.is_empty() { String::new() } else { format!(" ({code})") });
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
fn object_key(s3: &S3Config, name: &str) -> String {
|
||||
format!("{}{name}", s3.prefix.trim_start_matches('/'))
|
||||
}
|
||||
|
||||
pub async fn upload(s3: &S3Config, name: &str, bytes: Vec<u8>) -> Result<()> {
|
||||
if s3.bucket.trim().is_empty() {
|
||||
bail!("no S3 bucket configured");
|
||||
}
|
||||
send(s3, reqwest::Method::PUT, &object_key(s3, name), &[], bytes).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Delete the oldest backups in the bucket beyond `keep`.
|
||||
async fn prune(s3: &S3Config) -> Result<()> {
|
||||
let prefix = object_key(s3, "codecraft-backup-");
|
||||
let xml = send(s3, reqwest::Method::GET, "", &[("list-type", "2"), ("prefix", &prefix)], Vec::new()).await?;
|
||||
let mut keys: Vec<String> = xml.split("<Key>").skip(1).filter_map(|s| s.split("</Key>").next()).map(|s| s.replace("&", "&")).collect();
|
||||
keys.sort_by(|a, b| b.cmp(a));
|
||||
for k in keys.into_iter().skip(s3.keep) {
|
||||
send(s3, reqwest::Method::DELETE, &k, &[], Vec::new()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Example from the AWS SigV4 documentation (GET object with range omitted
|
||||
/// is not reproducible here, so check the derived signing key instead).
|
||||
#[test]
|
||||
fn signing_key_matches_aws_example() {
|
||||
// https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html
|
||||
let k = hmac(b"AWS4wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20120215");
|
||||
let k = hmac(&k, "us-east-1");
|
||||
let k = hmac(&k, "iam");
|
||||
let k = hmac(&k, "aws4_request");
|
||||
assert_eq!(hex(&k), "f4780e2d9f65fa895f9c67b32ce1baf0b0d8a43505a000a1a9e090d414db404d");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_keys() {
|
||||
assert_eq!(enc("a b/c+d", true), "a%20b/c%2Bd");
|
||||
assert_eq!(enc("a/b", false), "a%2Fb");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user