Test runner: per-project cache (tool caches, shared dirs, seeds)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a083798d7f
commit
b9df00422a
5 files changed
+220
-3
No files matched your search
+189
-1
@@ -30,6 +30,24 @@ use tokio::process::Command;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
const LABEL: &str = "codecraft.testrunner=1";
|
||||
const CACHE_LABEL: &str = "codecraft.testrunner.cache=1";
|
||||
|
||||
/// What every environment container runs: link the shared directories, restore a matching seed, run
|
||||
/// `install`, store a new seed (only after a successful install, so it is pristine), start the app.
|
||||
const ENTRY_SCRIPT: &str = r#"cd /app
|
||||
if [ -d /cache ]; then
|
||||
for d in $CC_CACHE_DIRS; do mkdir -p "/cache/dirs/$d" "$(dirname "$d")" && rm -rf "$d" && ln -s "/cache/dirs/$d" "$d"; done
|
||||
if [ -d /usr/local/cargo ]; then for d in registry git; do mkdir -p "/cache/cargo/$d"; rm -rf "/usr/local/cargo/$d"; ln -s "/cache/cargo/$d" "/usr/local/cargo/$d"; done; fi
|
||||
if [ -n "$CC_SEED_KEY" ]; then for p in $CC_SEED_PATHS; do if [ -e "/cache/seed/$CC_SEED_KEY/$p" ]; then mkdir -p "$(dirname "$p")" && cp -a "/cache/seed/$CC_SEED_KEY/$p" "$p" && echo "[runner] restored $p from the cache"; fi; done; fi
|
||||
fi
|
||||
if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi
|
||||
if [ -d /cache ] && [ -n "$CC_SEED_KEY" ] && [ ! -d "/cache/seed/$CC_SEED_KEY" ]; then
|
||||
t="/cache/seed/.tmp-$$"; rm -rf "$t"; mkdir -p "$t"; ok=1
|
||||
for p in $CC_SEED_PATHS; do if [ -e "$p" ]; then mkdir -p "$t/$(dirname "$p")" && cp -a "$p" "$t/$p" || ok=0; fi; done
|
||||
if [ "$ok" = 1 ]; then mv "$t" "/cache/seed/$CC_SEED_KEY" && echo "[runner] stored the seed $CC_SEED_KEY"; else rm -rf "$t"; fi
|
||||
ls -1t /cache/seed 2>/dev/null | grep -v '^\.tmp' | tail -n +4 | while read -r o; do rm -rf "/cache/seed/$o"; done
|
||||
fi
|
||||
exec sh -c "$CC_START""#;
|
||||
|
||||
// --- what a repository asks for (.factory/setup.json) ------------------------------------
|
||||
|
||||
@@ -41,6 +59,29 @@ pub struct Service {
|
||||
pub env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
/// What stays between runs of one project (kept in a Docker volume of the runner).
|
||||
///
|
||||
/// * `dirs`: directories of the sources that persist and are shared by all runs (downloads, caches –
|
||||
/// inputs only: whatever lands there is visible to the next run).
|
||||
/// * `seed`: expensive *results* of `install` (e.g. a built disk image). They are stored after a successful
|
||||
/// `install` under a key made from the image, the install command, the seed paths and the contents of the
|
||||
/// `key` files/directories, and copied into fresh containers whenever that key matches – so a run never
|
||||
/// sees changes of an earlier run, and a change of the listed sources builds the seed anew.
|
||||
///
|
||||
/// Tool caches (npm, yarn, pip, Go, Cargo registry and target directory) are always shared per project
|
||||
/// unless `"cache": false`.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct CacheCfg {
|
||||
#[serde(default)]
|
||||
pub disabled: bool,
|
||||
#[serde(default)]
|
||||
pub dirs: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub seed_paths: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub seed_key: Vec<String>,
|
||||
}
|
||||
|
||||
/// The request to start an environment.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct StartReq {
|
||||
@@ -63,6 +104,11 @@ pub struct StartReq {
|
||||
/// How long to wait until the port answers (seconds, default 300).
|
||||
#[serde(default)]
|
||||
pub ready_secs: u64,
|
||||
/// Stable name of the project: its cache lives in the volume `cc-cache-<cache_id>`. Empty = no cache.
|
||||
#[serde(default)]
|
||||
pub cache_id: String,
|
||||
#[serde(default)]
|
||||
pub cache: CacheCfg,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -110,9 +156,19 @@ pub fn parse_setup(text: &str) -> Option<StartReq> {
|
||||
env: strs("env"),
|
||||
services,
|
||||
ready_secs: v["ready_seconds"].as_u64().unwrap_or(0),
|
||||
cache_id: String::new(),
|
||||
cache: parse_cache(&v["cache"]),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_cache(v: &Value) -> CacheCfg {
|
||||
let list = |x: &Value| -> Vec<String> { x.as_array().map(|a| a.iter().filter_map(|s| s.as_str().map(|s| s.trim().trim_end_matches('/').to_string())).filter(|s| !s.is_empty()).collect()).unwrap_or_default() };
|
||||
if v.as_bool() == Some(false) {
|
||||
return CacheCfg { disabled: true, ..Default::default() };
|
||||
}
|
||||
CacheCfg { disabled: false, dirs: list(&v["dirs"]), seed_paths: list(&v["seed"]["paths"]), seed_key: list(&v["seed"]["key"]) }
|
||||
}
|
||||
|
||||
/// Port of `http://host:1234/path` (80/443 by scheme when missing).
|
||||
pub fn port_of(url: &str) -> Option<u16> {
|
||||
let (scheme, rest) = url.trim().split_once("://")?;
|
||||
@@ -132,6 +188,61 @@ fn image_ok(s: &str) -> bool {
|
||||
ident_ok(s, "._/:@-") && !s.starts_with('-')
|
||||
}
|
||||
|
||||
/// A path inside the sources: relative, no `..`, only plain characters (it ends up in a shell loop).
|
||||
fn rel_ok(p: &str) -> bool {
|
||||
ident_ok(p, "._/-") && !p.starts_with('/') && !p.starts_with('-') && !p.split('/').any(|c| c.is_empty() || c == "..") && !p.split('/').any(|c| c == ".git")
|
||||
}
|
||||
|
||||
/// Key of a seed: image, install command, seed paths and the contents of the key files (directories: recursively).
|
||||
pub fn seed_key(work: &Path, image: &str, req: &StartReq) -> Option<String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let c = &req.cache;
|
||||
if c.disabled || c.seed_paths.is_empty() {
|
||||
return None;
|
||||
}
|
||||
fn walk(base: &Path, rel: &Path, out: &mut Vec<std::path::PathBuf>) {
|
||||
let full = base.join(rel);
|
||||
if full.is_dir() {
|
||||
let mut kids: Vec<_> = std::fs::read_dir(&full).map(|d| d.filter_map(|e| e.ok()).map(|e| e.file_name()).collect()).unwrap_or_default();
|
||||
kids.sort();
|
||||
for k in kids {
|
||||
if k == ".git" || k == "node_modules" {
|
||||
continue;
|
||||
}
|
||||
walk(base, &rel.join(k), out);
|
||||
}
|
||||
} else if full.is_file() {
|
||||
out.push(rel.to_path_buf());
|
||||
}
|
||||
}
|
||||
let mut h = Sha256::new();
|
||||
for part in [image, &req.install, &c.seed_paths.join("\n"), &req.dockerfile] {
|
||||
h.update(part.as_bytes());
|
||||
h.update([0]);
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
let mut keys = c.seed_key.clone();
|
||||
keys.sort();
|
||||
for k in &keys {
|
||||
h.update(k.as_bytes());
|
||||
h.update([1]);
|
||||
walk(work, Path::new(k), &mut files);
|
||||
}
|
||||
let mut total = 0u64;
|
||||
for f in files {
|
||||
let data = std::fs::read(work.join(&f)).ok()?;
|
||||
total += data.len() as u64;
|
||||
if total > 200 << 20 {
|
||||
return None;
|
||||
}
|
||||
h.update(f.to_string_lossy().as_bytes());
|
||||
h.update([2]);
|
||||
h.update(&data);
|
||||
}
|
||||
let d = h.finalize();
|
||||
Some(d.iter().take(10).map(|b| format!("{b:02x}")).collect())
|
||||
}
|
||||
|
||||
fn env_ok(m: &BTreeMap<String, String>) -> bool {
|
||||
m.len() <= 64 && m.iter().all(|(k, v)| k.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && ident_ok(k, "_") && v.len() <= 4096 && !v.contains('\0'))
|
||||
}
|
||||
@@ -158,6 +269,16 @@ fn validate(req: &StartReq, factory_dir: &Path) -> Result<PathBuf> {
|
||||
if !env_ok(&req.env) {
|
||||
bail!("invalid environment variable");
|
||||
}
|
||||
if !req.cache_id.is_empty() && !ident_ok(&req.cache_id, "-_.") {
|
||||
bail!("invalid cache id");
|
||||
}
|
||||
let c = &req.cache;
|
||||
if c.dirs.len() > 16 || c.seed_paths.len() > 8 || c.seed_key.len() > 32 || !c.dirs.iter().chain(&c.seed_paths).chain(&c.seed_key).all(|p| rel_ok(p)) {
|
||||
bail!("invalid cache path (relative, letters, digits and ._-/ only)");
|
||||
}
|
||||
if !c.seed_paths.is_empty() && c.seed_key.is_empty() {
|
||||
bail!("cache.seed needs a key (the sources the seed depends on)");
|
||||
}
|
||||
let root = factory_dir.canonicalize().with_context(|| format!("factory area {} is not mounted", factory_dir.display()))?;
|
||||
let work = Path::new(&req.work).canonicalize().context("source directory not found")?;
|
||||
if !work.starts_with(&root) || !work.is_dir() {
|
||||
@@ -254,6 +375,7 @@ pub async fn serve() -> Result<()> {
|
||||
.route("/v1/envs", post(start_env))
|
||||
.route("/v1/envs/{id}", delete(stop_env))
|
||||
.route("/v1/envs/{id}/log", get(log_env))
|
||||
.route("/v1/cache/{id}", delete(clear_cache))
|
||||
.with_state(runner);
|
||||
let bind = std::env::var("RUNNER_BIND").unwrap_or_else(|_| "0.0.0.0:8788".into());
|
||||
let l = TcpListener::bind(&bind).await?;
|
||||
@@ -350,12 +472,42 @@ async fn create(r: &Arc<Runner>, id: &str, req: &StartReq, work: &Path) -> Resul
|
||||
a.push("-e".into());
|
||||
a.push(format!("{k}={v}"));
|
||||
}
|
||||
// Per-project cache volume: tool caches, shared download directories and seeds (see CacheCfg).
|
||||
let cached = !req.cache_id.is_empty() && !req.cache.disabled;
|
||||
if cached {
|
||||
let vol = format!("cc-cache-{}", req.cache_id);
|
||||
docker(&["volume", "create", "--label", CACHE_LABEL, &vol]).await.context("cache volume")?;
|
||||
a.push("-v".into());
|
||||
a.push(format!("{vol}:/cache"));
|
||||
for (k, v) in [
|
||||
("npm_config_cache", "/cache/npm"),
|
||||
("YARN_CACHE_FOLDER", "/cache/yarn"),
|
||||
("PIP_CACHE_DIR", "/cache/pip"),
|
||||
("GOMODCACHE", "/cache/go/mod"),
|
||||
("GOCACHE", "/cache/go/build"),
|
||||
("CARGO_TARGET_DIR", "/cache/target"),
|
||||
("XDG_CACHE_HOME", "/cache/xdg"),
|
||||
] {
|
||||
if !req.env.contains_key(k) {
|
||||
a.push("-e".into());
|
||||
a.push(format!("{k}={v}"));
|
||||
}
|
||||
}
|
||||
a.push("-e".into());
|
||||
a.push(format!("CC_CACHE_DIRS={}", req.cache.dirs.join(" ")));
|
||||
if let Some(key) = seed_key(work, &image, req) {
|
||||
a.push("-e".into());
|
||||
a.push(format!("CC_SEED_KEY={key}"));
|
||||
a.push("-e".into());
|
||||
a.push(format!("CC_SEED_PATHS={}", req.cache.seed_paths.join(" ")));
|
||||
}
|
||||
}
|
||||
a.push("-e".into());
|
||||
a.push(format!("CC_INSTALL={}", req.install));
|
||||
a.push("-e".into());
|
||||
a.push(format!("CC_START={}", req.start));
|
||||
a.push(image);
|
||||
a.extend(["sh", "-c", r#"if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi; exec sh -c "$CC_START""#].map(String::from));
|
||||
a.extend(["sh", "-c", ENTRY_SCRIPT].map(String::from));
|
||||
let refs: Vec<&str> = a.iter().map(String::as_str).collect();
|
||||
docker(&refs).await.context("docker create")?;
|
||||
// Copy the sources in (without .git); the container never sees the host directory.
|
||||
@@ -443,6 +595,16 @@ async fn stop_env(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath
|
||||
Ok(Json(json!({"ok": true})))
|
||||
}
|
||||
|
||||
/// Forget the cache of a project (fails while an environment of it is running).
|
||||
async fn clear_cache(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>) -> Resp<Json<Value>> {
|
||||
authed(&r, &h)?;
|
||||
if !ident_ok(&id, "-_.") {
|
||||
return Err((StatusCode::BAD_REQUEST, "invalid cache id".into()));
|
||||
}
|
||||
docker(&["volume", "rm", &format!("cc-cache-{id}")]).await.map_err(|e| (StatusCode::CONFLICT, format!("{e:#}")))?;
|
||||
Ok(Json(json!({"ok": true})))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LogQ {
|
||||
tail: Option<usize>,
|
||||
@@ -574,6 +736,32 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_config_and_seed_key() {
|
||||
let r = parse_setup(r#"{"start":"x","port":1,"cache":{"dirs":["build/downloads/"],"seed":{"paths":["data/ubuntu.ext4"],"key":["guest","package-lock.json"]}}}"#).unwrap();
|
||||
assert_eq!(r.cache.dirs, vec!["build/downloads"]);
|
||||
assert_eq!(r.cache.seed_key, vec!["guest", "package-lock.json"]);
|
||||
assert!(parse_setup(r#"{"start":"x","port":1,"cache":false}"#).unwrap().cache.disabled);
|
||||
assert!(!parse_setup(r#"{"start":"x","port":1}"#).unwrap().cache.disabled);
|
||||
assert!(rel_ok("build/downloads") && !rel_ok("../x") && !rel_ok("/etc") && !rel_ok("a b") && !rel_ok("a/.git/b") && !rel_ok("a//b"));
|
||||
// The key follows the contents of the key files, not of anything else.
|
||||
let dir = std::env::temp_dir().join(format!("ccs-{}", std::process::id()));
|
||||
std::fs::create_dir_all(dir.join("guest")).unwrap();
|
||||
std::fs::write(dir.join("guest/a.c"), "one").unwrap();
|
||||
std::fs::write(dir.join("other.txt"), "x").unwrap();
|
||||
let mut req = r.clone();
|
||||
req.install = "npm ci".into();
|
||||
let k1 = seed_key(&dir, "node:24", &req).unwrap();
|
||||
std::fs::write(dir.join("other.txt"), "changed").unwrap();
|
||||
assert_eq!(seed_key(&dir, "node:24", &req).unwrap(), k1);
|
||||
std::fs::write(dir.join("guest/a.c"), "two").unwrap();
|
||||
assert_ne!(seed_key(&dir, "node:24", &req).unwrap(), k1);
|
||||
assert_ne!(seed_key(&dir, "node:22", &req).unwrap(), seed_key(&dir, "node:24", &req).unwrap());
|
||||
req.cache.seed_paths.clear();
|
||||
assert!(seed_key(&dir, "node:24", &req).is_none());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_compare() {
|
||||
assert!(eq_const("abc", "abc"));
|
||||
|
||||
Reference in new issue
Block a user