Test runner: per-project cache (tool caches, shared dirs, seeds)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard ZinkandClaude Sonnet 5.5 committed 2026-11-20 13:15:07 +01:00
1 parent a083798d7f
commit b9df00422a
5 files changed
+220 -3

No files matched your search

+189 -1
View File
@@ -30,6 +30,24 @@ use tokio::process::Command;
use tokio::task::JoinHandle;
const LABEL: &str = "codecraft.testrunner=1";
const CACHE_LABEL: &str = "codecraft.testrunner.cache=1";
/// What every environment container runs: link the shared directories, restore a matching seed, run
/// `install`, store a new seed (only after a successful install, so it is pristine), start the app.
const ENTRY_SCRIPT: &str = r#"cd /app
if [ -d /cache ]; then
for d in $CC_CACHE_DIRS; do mkdir -p "/cache/dirs/$d" "$(dirname "$d")" && rm -rf "$d" && ln -s "/cache/dirs/$d" "$d"; done
if [ -d /usr/local/cargo ]; then for d in registry git; do mkdir -p "/cache/cargo/$d"; rm -rf "/usr/local/cargo/$d"; ln -s "/cache/cargo/$d" "/usr/local/cargo/$d"; done; fi
if [ -n "$CC_SEED_KEY" ]; then for p in $CC_SEED_PATHS; do if [ -e "/cache/seed/$CC_SEED_KEY/$p" ]; then mkdir -p "$(dirname "$p")" && cp -a "/cache/seed/$CC_SEED_KEY/$p" "$p" && echo "[runner] restored $p from the cache"; fi; done; fi
fi
if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi
if [ -d /cache ] && [ -n "$CC_SEED_KEY" ] && [ ! -d "/cache/seed/$CC_SEED_KEY" ]; then
t="/cache/seed/.tmp-$$"; rm -rf "$t"; mkdir -p "$t"; ok=1
for p in $CC_SEED_PATHS; do if [ -e "$p" ]; then mkdir -p "$t/$(dirname "$p")" && cp -a "$p" "$t/$p" || ok=0; fi; done
if [ "$ok" = 1 ]; then mv "$t" "/cache/seed/$CC_SEED_KEY" && echo "[runner] stored the seed $CC_SEED_KEY"; else rm -rf "$t"; fi
ls -1t /cache/seed 2>/dev/null | grep -v '^\.tmp' | tail -n +4 | while read -r o; do rm -rf "/cache/seed/$o"; done
fi
exec sh -c "$CC_START""#;
// --- what a repository asks for (.factory/setup.json) ------------------------------------
@@ -41,6 +59,29 @@ pub struct Service {
pub env: BTreeMap<String, String>,
}
/// What stays between runs of one project (kept in a Docker volume of the runner).
///
/// * `dirs`: directories of the sources that persist and are shared by all runs (downloads, caches –
/// inputs only: whatever lands there is visible to the next run).
/// * `seed`: expensive *results* of `install` (e.g. a built disk image). They are stored after a successful
/// `install` under a key made from the image, the install command, the seed paths and the contents of the
/// `key` files/directories, and copied into fresh containers whenever that key matches – so a run never
/// sees changes of an earlier run, and a change of the listed sources builds the seed anew.
///
/// Tool caches (npm, yarn, pip, Go, Cargo registry and target directory) are always shared per project
/// unless `"cache": false`.
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct CacheCfg {
#[serde(default)]
pub disabled: bool,
#[serde(default)]
pub dirs: Vec<String>,
#[serde(default)]
pub seed_paths: Vec<String>,
#[serde(default)]
pub seed_key: Vec<String>,
}
/// The request to start an environment.
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct StartReq {
@@ -63,6 +104,11 @@ pub struct StartReq {
/// How long to wait until the port answers (seconds, default 300).
#[serde(default)]
pub ready_secs: u64,
/// Stable name of the project: its cache lives in the volume `cc-cache-<cache_id>`. Empty = no cache.
#[serde(default)]
pub cache_id: String,
#[serde(default)]
pub cache: CacheCfg,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -110,9 +156,19 @@ pub fn parse_setup(text: &str) -> Option<StartReq> {
env: strs("env"),
services,
ready_secs: v["ready_seconds"].as_u64().unwrap_or(0),
cache_id: String::new(),
cache: parse_cache(&v["cache"]),
})
}
fn parse_cache(v: &Value) -> CacheCfg {
let list = |x: &Value| -> Vec<String> { x.as_array().map(|a| a.iter().filter_map(|s| s.as_str().map(|s| s.trim().trim_end_matches('/').to_string())).filter(|s| !s.is_empty()).collect()).unwrap_or_default() };
if v.as_bool() == Some(false) {
return CacheCfg { disabled: true, ..Default::default() };
}
CacheCfg { disabled: false, dirs: list(&v["dirs"]), seed_paths: list(&v["seed"]["paths"]), seed_key: list(&v["seed"]["key"]) }
}
/// Port of `http://host:1234/path` (80/443 by scheme when missing).
pub fn port_of(url: &str) -> Option<u16> {
let (scheme, rest) = url.trim().split_once("://")?;
@@ -132,6 +188,61 @@ fn image_ok(s: &str) -> bool {
ident_ok(s, "._/:@-") && !s.starts_with('-')
}
/// A path inside the sources: relative, no `..`, only plain characters (it ends up in a shell loop).
fn rel_ok(p: &str) -> bool {
ident_ok(p, "._/-") && !p.starts_with('/') && !p.starts_with('-') && !p.split('/').any(|c| c.is_empty() || c == "..") && !p.split('/').any(|c| c == ".git")
}
/// Key of a seed: image, install command, seed paths and the contents of the key files (directories: recursively).
pub fn seed_key(work: &Path, image: &str, req: &StartReq) -> Option<String> {
use sha2::{Digest, Sha256};
let c = &req.cache;
if c.disabled || c.seed_paths.is_empty() {
return None;
}
fn walk(base: &Path, rel: &Path, out: &mut Vec<std::path::PathBuf>) {
let full = base.join(rel);
if full.is_dir() {
let mut kids: Vec<_> = std::fs::read_dir(&full).map(|d| d.filter_map(|e| e.ok()).map(|e| e.file_name()).collect()).unwrap_or_default();
kids.sort();
for k in kids {
if k == ".git" || k == "node_modules" {
continue;
}
walk(base, &rel.join(k), out);
}
} else if full.is_file() {
out.push(rel.to_path_buf());
}
}
let mut h = Sha256::new();
for part in [image, &req.install, &c.seed_paths.join("\n"), &req.dockerfile] {
h.update(part.as_bytes());
h.update([0]);
}
let mut files = Vec::new();
let mut keys = c.seed_key.clone();
keys.sort();
for k in &keys {
h.update(k.as_bytes());
h.update([1]);
walk(work, Path::new(k), &mut files);
}
let mut total = 0u64;
for f in files {
let data = std::fs::read(work.join(&f)).ok()?;
total += data.len() as u64;
if total > 200 << 20 {
return None;
}
h.update(f.to_string_lossy().as_bytes());
h.update([2]);
h.update(&data);
}
let d = h.finalize();
Some(d.iter().take(10).map(|b| format!("{b:02x}")).collect())
}
fn env_ok(m: &BTreeMap<String, String>) -> bool {
m.len() <= 64 && m.iter().all(|(k, v)| k.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && ident_ok(k, "_") && v.len() <= 4096 && !v.contains('\0'))
}
@@ -158,6 +269,16 @@ fn validate(req: &StartReq, factory_dir: &Path) -> Result<PathBuf> {
if !env_ok(&req.env) {
bail!("invalid environment variable");
}
if !req.cache_id.is_empty() && !ident_ok(&req.cache_id, "-_.") {
bail!("invalid cache id");
}
let c = &req.cache;
if c.dirs.len() > 16 || c.seed_paths.len() > 8 || c.seed_key.len() > 32 || !c.dirs.iter().chain(&c.seed_paths).chain(&c.seed_key).all(|p| rel_ok(p)) {
bail!("invalid cache path (relative, letters, digits and ._-/ only)");
}
if !c.seed_paths.is_empty() && c.seed_key.is_empty() {
bail!("cache.seed needs a key (the sources the seed depends on)");
}
let root = factory_dir.canonicalize().with_context(|| format!("factory area {} is not mounted", factory_dir.display()))?;
let work = Path::new(&req.work).canonicalize().context("source directory not found")?;
if !work.starts_with(&root) || !work.is_dir() {
@@ -254,6 +375,7 @@ pub async fn serve() -> Result<()> {
.route("/v1/envs", post(start_env))
.route("/v1/envs/{id}", delete(stop_env))
.route("/v1/envs/{id}/log", get(log_env))
.route("/v1/cache/{id}", delete(clear_cache))
.with_state(runner);
let bind = std::env::var("RUNNER_BIND").unwrap_or_else(|_| "0.0.0.0:8788".into());
let l = TcpListener::bind(&bind).await?;
@@ -350,12 +472,42 @@ async fn create(r: &Arc<Runner>, id: &str, req: &StartReq, work: &Path) -> Resul
a.push("-e".into());
a.push(format!("{k}={v}"));
}
// Per-project cache volume: tool caches, shared download directories and seeds (see CacheCfg).
let cached = !req.cache_id.is_empty() && !req.cache.disabled;
if cached {
let vol = format!("cc-cache-{}", req.cache_id);
docker(&["volume", "create", "--label", CACHE_LABEL, &vol]).await.context("cache volume")?;
a.push("-v".into());
a.push(format!("{vol}:/cache"));
for (k, v) in [
("npm_config_cache", "/cache/npm"),
("YARN_CACHE_FOLDER", "/cache/yarn"),
("PIP_CACHE_DIR", "/cache/pip"),
("GOMODCACHE", "/cache/go/mod"),
("GOCACHE", "/cache/go/build"),
("CARGO_TARGET_DIR", "/cache/target"),
("XDG_CACHE_HOME", "/cache/xdg"),
] {
if !req.env.contains_key(k) {
a.push("-e".into());
a.push(format!("{k}={v}"));
}
}
a.push("-e".into());
a.push(format!("CC_CACHE_DIRS={}", req.cache.dirs.join(" ")));
if let Some(key) = seed_key(work, &image, req) {
a.push("-e".into());
a.push(format!("CC_SEED_KEY={key}"));
a.push("-e".into());
a.push(format!("CC_SEED_PATHS={}", req.cache.seed_paths.join(" ")));
}
}
a.push("-e".into());
a.push(format!("CC_INSTALL={}", req.install));
a.push("-e".into());
a.push(format!("CC_START={}", req.start));
a.push(image);
a.extend(["sh", "-c", r#"if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi; exec sh -c "$CC_START""#].map(String::from));
a.extend(["sh", "-c", ENTRY_SCRIPT].map(String::from));
let refs: Vec<&str> = a.iter().map(String::as_str).collect();
docker(&refs).await.context("docker create")?;
// Copy the sources in (without .git); the container never sees the host directory.
@@ -443,6 +595,16 @@ async fn stop_env(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath
Ok(Json(json!({"ok": true})))
}
/// Forget the cache of a project (fails while an environment of it is running).
async fn clear_cache(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>) -> Resp<Json<Value>> {
authed(&r, &h)?;
if !ident_ok(&id, "-_.") {
return Err((StatusCode::BAD_REQUEST, "invalid cache id".into()));
}
docker(&["volume", "rm", &format!("cc-cache-{id}")]).await.map_err(|e| (StatusCode::CONFLICT, format!("{e:#}")))?;
Ok(Json(json!({"ok": true})))
}
#[derive(Deserialize)]
struct LogQ {
tail: Option<usize>,
@@ -574,6 +736,32 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn cache_config_and_seed_key() {
let r = parse_setup(r#"{"start":"x","port":1,"cache":{"dirs":["build/downloads/"],"seed":{"paths":["data/ubuntu.ext4"],"key":["guest","package-lock.json"]}}}"#).unwrap();
assert_eq!(r.cache.dirs, vec!["build/downloads"]);
assert_eq!(r.cache.seed_key, vec!["guest", "package-lock.json"]);
assert!(parse_setup(r#"{"start":"x","port":1,"cache":false}"#).unwrap().cache.disabled);
assert!(!parse_setup(r#"{"start":"x","port":1}"#).unwrap().cache.disabled);
assert!(rel_ok("build/downloads") && !rel_ok("../x") && !rel_ok("/etc") && !rel_ok("a b") && !rel_ok("a/.git/b") && !rel_ok("a//b"));
// The key follows the contents of the key files, not of anything else.
let dir = std::env::temp_dir().join(format!("ccs-{}", std::process::id()));
std::fs::create_dir_all(dir.join("guest")).unwrap();
std::fs::write(dir.join("guest/a.c"), "one").unwrap();
std::fs::write(dir.join("other.txt"), "x").unwrap();
let mut req = r.clone();
req.install = "npm ci".into();
let k1 = seed_key(&dir, "node:24", &req).unwrap();
std::fs::write(dir.join("other.txt"), "changed").unwrap();
assert_eq!(seed_key(&dir, "node:24", &req).unwrap(), k1);
std::fs::write(dir.join("guest/a.c"), "two").unwrap();
assert_ne!(seed_key(&dir, "node:24", &req).unwrap(), k1);
assert_ne!(seed_key(&dir, "node:22", &req).unwrap(), seed_key(&dir, "node:24", &req).unwrap());
req.cache.seed_paths.clear();
assert!(seed_key(&dir, "node:24", &req).is_none());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn tokens_compare() {
assert!(eq_const("abc", "abc"));