//! Throw-away test environments for the factory's acceptance runs. //! //! Two halves live here: //! //! * the **service** (`codecraft testrunner`): a small HTTP server that owns the //! Docker socket. For every acceptance run it creates a private network, starts //! the application (and optional helper services such as a database) in fresh //! containers, forwards one port, and removes everything again afterwards. //! It only ever runs a fixed, locked-down `docker` command line: no host mounts, //! no privileged mode, capabilities dropped, memory/CPU/process limits. //! * the **client** used by the factory in the main server: start an environment, //! read its log, stop it, and forward a local port to it so that the agent can //! use `http://localhost:PORT` as usual. use anyhow::{Context, Result, bail}; use axum::extract::{Path as AxPath, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{delete, get, post}; use axum::{Json, Router}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::io::AsyncWriteExt; use tokio::net::{TcpListener, TcpStream}; use tokio::process::Command; use tokio::task::JoinHandle; const LABEL: &str = "codecraft.testrunner=1"; // --- what a repository asks for (.factory/setup.json) ------------------------------------ /// A helper container next to the application, reachable under its name (e.g. `db`). #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] pub struct Service { pub image: String, #[serde(default)] pub env: BTreeMap, } /// The request to start an environment. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] pub struct StartReq { /// Checked-out source (below the factory area); copied into the container, never mounted. pub work: String, /// Image to run in (default: the runner's default image). Ignored when `dockerfile` is set. #[serde(default)] pub image: String, /// Dockerfile inside the source to build the image from. #[serde(default)] pub dockerfile: String, #[serde(default)] pub install: String, pub start: String, pub port: u16, #[serde(default)] pub env: BTreeMap, #[serde(default)] pub services: BTreeMap, /// How long to wait until the port answers (seconds, default 300). #[serde(default)] pub ready_secs: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Started { pub id: String, /// Port on the runner that forwards to the application. pub port: u16, } /// Parse `.factory/setup.json` into a request (without `work`). `None` when there is nothing to start. pub fn parse_setup(text: &str) -> Option { let v: Value = serde_json::from_str(text).ok()?; // `"isolated": false`: the app cannot run in a container (e.g. a native window); acceptance starts it itself. if v["isolated"].as_bool() == Some(false) { return None; } let start = v["start"].as_str()?.trim().to_string(); if start.is_empty() { return None; } let url = v["url"].as_str().unwrap_or(""); let port = v["port"].as_u64().map(|p| p as u16).or_else(|| port_of(url))?; let strs = |k: &str| -> BTreeMap { v[k].as_object().map(|o| o.iter().filter_map(|(a, b)| b.as_str().map(|s| (a.clone(), s.to_string()))).collect()).unwrap_or_default() }; let services = v["services"] .as_object() .map(|o| { o.iter() .filter_map(|(name, s)| { let image = s["image"].as_str()?.to_string(); let env = s["env"].as_object().map(|e| e.iter().filter_map(|(a, b)| b.as_str().map(|x| (a.clone(), x.to_string()))).collect()).unwrap_or_default(); Some((name.clone(), Service { image, env })) }) .collect() }) .unwrap_or_default(); Some(StartReq { work: String::new(), image: v["image"].as_str().unwrap_or("").trim().to_string(), dockerfile: v["dockerfile"].as_str().unwrap_or("").trim().to_string(), install: v["install"].as_str().unwrap_or("").trim().to_string(), start, port, env: strs("env"), services, ready_secs: v["ready_seconds"].as_u64().unwrap_or(0), }) } /// Port of `http://host:1234/path` (80/443 by scheme when missing). pub fn port_of(url: &str) -> Option { let (scheme, rest) = url.trim().split_once("://")?; // Free text after the URL ("http://localhost:8080 (trunk serve)") is ignored. let host = rest.split(|c: char| c == '/' || c.is_whitespace()).next()?; match host.rsplit_once(':') { Some((_, p)) if p.chars().all(|c| c.is_ascii_digit()) && !p.is_empty() => p.parse().ok(), _ => Some(if scheme == "https" { 443 } else { 80 }), } } fn ident_ok(s: &str, extra: &str) -> bool { !s.is_empty() && s.len() <= 200 && s.chars().all(|c| c.is_ascii_alphanumeric() || extra.contains(c)) } fn image_ok(s: &str) -> bool { ident_ok(s, "._/:@-") && !s.starts_with('-') } fn env_ok(m: &BTreeMap) -> bool { m.len() <= 64 && m.iter().all(|(k, v)| k.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && ident_ok(k, "_") && v.len() <= 4096 && !v.contains('\0')) } /// Check a request against the rules of the runner and return the canonical source directory. fn validate(req: &StartReq, factory_dir: &Path) -> Result { if req.start.trim().is_empty() || req.start.len() > 4096 || req.install.len() > 4096 { bail!("start/install command missing or too long"); } if req.port == 0 { bail!("port missing"); } if !req.image.is_empty() && !image_ok(&req.image) { bail!("invalid image name"); } if req.services.len() > 8 { bail!("at most 8 services"); } for (name, s) in &req.services { if !ident_ok(name, "_-") || !image_ok(&s.image) || !env_ok(&s.env) { bail!("invalid service {name}"); } } if !env_ok(&req.env) { bail!("invalid environment variable"); } let root = factory_dir.canonicalize().with_context(|| format!("factory area {} is not mounted", factory_dir.display()))?; let work = Path::new(&req.work).canonicalize().context("source directory not found")?; if !work.starts_with(&root) || !work.is_dir() { bail!("the source must be a directory inside the factory area"); } if !req.dockerfile.is_empty() { let df = work.join(&req.dockerfile).canonicalize().context("dockerfile not found")?; if !df.starts_with(&work) || !df.is_file() { bail!("the dockerfile must be a file inside the source"); } } Ok(work) } // --- the service ---------------------------------------------------------------------------- struct Env { containers: Vec, network: String, image: Option, forward: JoinHandle<()>, started: Instant, } struct Runner { token: String, factory_dir: PathBuf, ports: (u16, u16), memory: String, cpus: String, default_image: String, max_age: Duration, /// Own container name (to join the per-run networks); empty when not running in Docker. me: String, envs: Mutex>, } async fn docker(args: &[&str]) -> Result { let out = Command::new("docker").args(args).stdin(Stdio::null()).output().await.context("cannot run docker")?; if !out.status.success() { bail!("docker {}: {}", args.first().unwrap_or(&""), String::from_utf8_lossy(&out.stderr).trim()); } Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()) } fn eq_const(a: &str, b: &str) -> bool { a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |d, (x, y)| d | (x ^ y)) == 0 } type Resp = std::result::Result; fn authed(r: &Runner, h: &HeaderMap) -> Resp<()> { let got = h.get("authorization").and_then(|v| v.to_str().ok()).and_then(|v| v.strip_prefix("Bearer ")).unwrap_or(""); if r.token.is_empty() || !eq_const(got, &r.token) { return Err((StatusCode::UNAUTHORIZED, "bad token".into())); } Ok(()) } /// Run `codecraft testrunner`: serve until the process is stopped. pub async fn serve() -> Result<()> { let token = std::env::var("RUNNER_TOKEN").unwrap_or_default(); if token.len() < 16 { bail!("RUNNER_TOKEN must be set (at least 16 characters)"); } let (lo, hi) = std::env::var("RUNNER_PORTS").ok().and_then(|s| s.split_once('-').and_then(|(a, b)| Some((a.trim().parse().ok()?, b.trim().parse().ok()?)))).unwrap_or((9100u16, 9199u16)); let runner = Arc::new(Runner { token, factory_dir: PathBuf::from(std::env::var("RUNNER_FACTORY_DIR").unwrap_or_else(|_| "/factory".into())), ports: (lo, hi), memory: std::env::var("RUNNER_MEMORY").unwrap_or_else(|_| "2g".into()), cpus: std::env::var("RUNNER_CPUS").unwrap_or_else(|_| "2".into()), default_image: std::env::var("RUNNER_DEFAULT_IMAGE").unwrap_or_else(|_| "node:22-bookworm".into()), max_age: Duration::from_secs(60 * std::env::var("RUNNER_MAX_MINUTES").ok().and_then(|s| s.parse().ok()).unwrap_or(90)), me: std::env::var("RUNNER_SELF").or_else(|_| std::env::var("HOSTNAME")).unwrap_or_default(), envs: Mutex::new(HashMap::new()), }); let ver = docker(&["version", "--format", "{{.Server.Version}}"]).await.context("the Docker socket is not available to the test runner")?; tracing::info!("test runner: Docker {ver}, ports {lo}-{hi}, default image {}", runner.default_image); sweep_all(&runner).await; let r2 = runner.clone(); tokio::spawn(async move { loop { tokio::time::sleep(Duration::from_secs(60)).await; let old: Vec = r2.envs.lock().unwrap().iter().filter(|(_, e)| e.started.elapsed() > r2.max_age).map(|(k, _)| k.clone()).collect(); for id in old { tracing::warn!("test runner: removing {id} (older than the limit)"); stop(&r2, &id).await; } } }); let app = Router::new() .route("/v1/health", get(health)) .route("/v1/envs", post(start_env)) .route("/v1/envs/{id}", delete(stop_env)) .route("/v1/envs/{id}/log", get(log_env)) .with_state(runner); let bind = std::env::var("RUNNER_BIND").unwrap_or_else(|_| "0.0.0.0:8788".into()); let l = TcpListener::bind(&bind).await?; tracing::info!("test runner listening on {bind}"); axum::serve(l, app).with_graceful_shutdown(async { let _ = tokio::signal::ctrl_c().await; }).await?; Ok(()) } /// Remove everything this service ever created (after a crash or restart). async fn sweep_all(r: &Runner) { if let Ok(ids) = docker(&["ps", "-aq", "--filter", &format!("label={LABEL}")]).await { for id in ids.split_whitespace() { let _ = docker(&["rm", "-f", "-v", id]).await; } } if let Ok(nets) = docker(&["network", "ls", "-q", "--filter", &format!("label={LABEL}")]).await { for n in nets.split_whitespace() { if !r.me.is_empty() { let _ = docker(&["network", "disconnect", "-f", n, &r.me]).await; } let _ = docker(&["network", "rm", n]).await; } } } async fn health(State(r): State>, h: HeaderMap) -> Resp> { authed(&r, &h)?; let n = r.envs.lock().unwrap().len(); Ok(Json(json!({"ok": true, "environments": n}))) } async fn bind_forward(r: &Runner) -> Result<(TcpListener, u16)> { for p in r.ports.0..=r.ports.1 { if let Ok(l) = TcpListener::bind(("0.0.0.0", p)).await { return Ok((l, p)); } } bail!("no free forward port") } async fn start_env(State(r): State>, h: HeaderMap, Json(req): Json) -> Resp> { authed(&r, &h)?; let work = validate(&req, &r.factory_dir).map_err(|e| (StatusCode::BAD_REQUEST, format!("{e:#}")))?; if r.envs.lock().unwrap().len() >= 16 { return Err((StatusCode::TOO_MANY_REQUESTS, "too many running environments".into())); } let id = uuid::Uuid::new_v4().simple().to_string()[..12].to_string(); match create(&r, &id, &req, &work).await { Ok(s) => Ok(Json(s)), Err(e) => { let msg = format!("{e:#}"); stop(&r, &id).await; Err((StatusCode::BAD_GATEWAY, msg)) } } } async fn create(r: &Arc, id: &str, req: &StartReq, work: &Path) -> Result { let net = format!("cc-run-{id}"); let app = format!("cc-run-{id}"); // Register first, so a failure at any step is cleaned up by stop(). let forward = tokio::spawn(async {}); r.envs.lock().unwrap().insert(id.to_string(), Env { containers: vec![app.clone()], network: net.clone(), image: None, forward, started: Instant::now() }); docker(&["network", "create", "--label", LABEL, &net]).await?; for (name, s) in &req.services { let cname = format!("{app}-{name}"); let mut a: Vec = ["run", "-d", "--name", &cname, "--network", &net, "--network-alias", name, "--label", LABEL, "--memory", &r.memory, "--pids-limit", "1024", "--security-opt", "no-new-privileges"].map(String::from).to_vec(); for (k, v) in &s.env { a.push("-e".into()); a.push(format!("{k}={v}")); } a.push(s.image.clone()); let refs: Vec<&str> = a.iter().map(String::as_str).collect(); docker(&refs).await.with_context(|| format!("service {name}"))?; r.envs.lock().unwrap().get_mut(id).unwrap().containers.push(cname); } let image = if req.dockerfile.is_empty() { if req.image.is_empty() { r.default_image.clone() } else { req.image.clone() } } else { let tag = format!("cc-run-{id}"); let df = work.join(&req.dockerfile); docker(&["build", "-q", "--label", LABEL, "-t", &tag, "-f", &df.display().to_string(), &work.display().to_string()]).await.context("docker build")?; r.envs.lock().unwrap().get_mut(id).unwrap().image = Some(tag.clone()); tag }; let mut a: Vec = [ "create", "--name", &app, "--network", &net, "--label", LABEL, "--memory", &r.memory, "--cpus", &r.cpus, "--pids-limit", "1024", "--cap-drop", "ALL", "--cap-add", "CHOWN", "--cap-add", "DAC_OVERRIDE", "--cap-add", "FOWNER", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "KILL", "--cap-add", "NET_BIND_SERVICE", "--security-opt", "no-new-privileges", "-w", "/app", "-e", "CI=1", ] .map(String::from) .to_vec(); for (k, v) in &req.env { a.push("-e".into()); a.push(format!("{k}={v}")); } a.push("-e".into()); a.push(format!("CC_INSTALL={}", req.install)); a.push("-e".into()); a.push(format!("CC_START={}", req.start)); a.push(image); a.extend(["sh", "-c", r#"if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi; exec sh -c "$CC_START""#].map(String::from)); let refs: Vec<&str> = a.iter().map(String::as_str).collect(); docker(&refs).await.context("docker create")?; // Copy the sources in (without .git); the container never sees the host directory. let cp = Command::new("sh") .args(["-c", r#"tar -C "$1" --exclude=.git -cf - . | docker cp - "$2":/app"#, "sh", &work.display().to_string(), &app]) .stdin(Stdio::null()) .output() .await?; if !cp.status.success() { bail!("copying the sources failed: {}", String::from_utf8_lossy(&cp.stderr).trim()); } docker(&["start", &app]).await?; if !r.me.is_empty() { let _ = docker(&["network", "connect", &net, &r.me]).await; } let ip = docker(&["inspect", "-f", &format!("{{{{(index .NetworkSettings.Networks \"{net}\").IPAddress}}}}"), &app]).await?; if ip.is_empty() { bail!("the container got no address"); } // Wait until the application answers; give up early when it has exited. let deadline = Instant::now() + Duration::from_secs(if req.ready_secs == 0 { 300 } else { req.ready_secs.min(1800) }); loop { if TcpStream::connect((ip.as_str(), req.port)).await.is_ok() { break; } let running = docker(&["inspect", "-f", "{{.State.Running}}", &app]).await.unwrap_or_default(); if running != "true" { bail!("the application exited before it listened on port {}:\n{}", req.port, tail(&logs(&app).await, 40)); } if Instant::now() > deadline { bail!("the application did not listen on port {} in time:\n{}", req.port, tail(&logs(&app).await, 40)); } tokio::time::sleep(Duration::from_millis(700)).await; } let (listener, port) = bind_forward(r).await?; let (target, tport) = (ip, req.port); let fwd = tokio::spawn(async move { loop { let Ok((mut inbound, _)) = listener.accept().await else { break }; let t = target.clone(); tokio::spawn(async move { if let Ok(mut out) = TcpStream::connect((t.as_str(), tport)).await { let _ = tokio::io::copy_bidirectional(&mut inbound, &mut out).await; } }); } }); if let Some(e) = r.envs.lock().unwrap().get_mut(id) { e.forward.abort(); e.forward = fwd; } Ok(Started { id: id.to_string(), port }) } async fn logs(container: &str) -> String { match Command::new("docker").args(["logs", "--tail", "400", container]).output().await { Ok(o) => format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)), Err(_) => String::new(), } } fn tail(s: &str, n: usize) -> String { let lines: Vec<&str> = s.lines().collect(); lines[lines.len().saturating_sub(n)..].join("\n") } async fn stop(r: &Runner, id: &str) { let Some(e) = r.envs.lock().unwrap().remove(id) else { return }; e.forward.abort(); for c in e.containers.iter().rev() { let _ = docker(&["rm", "-f", "-v", c]).await; } if !r.me.is_empty() { let _ = docker(&["network", "disconnect", "-f", &e.network, &r.me]).await; } let _ = docker(&["network", "rm", &e.network]).await; if let Some(i) = e.image { let _ = docker(&["rmi", "-f", &i]).await; } } async fn stop_env(State(r): State>, h: HeaderMap, AxPath(id): AxPath) -> Resp> { authed(&r, &h)?; stop(&r, &id).await; Ok(Json(json!({"ok": true}))) } #[derive(Deserialize)] struct LogQ { tail: Option, } async fn log_env(State(r): State>, h: HeaderMap, AxPath(id): AxPath, Query(q): Query) -> Resp { authed(&r, &h)?; let mut out = String::new(); let names = r.envs.lock().unwrap().get(&id).map(|e| e.containers.clone()).ok_or((StatusCode::NOT_FOUND, "unknown environment".to_string()))?; for c in names { let l = logs(&c).await; if !l.is_empty() { let label = c.strip_prefix(&format!("cc-run-{id}")).unwrap_or("").trim_start_matches('-'); out.push_str(&format!("==> {} <==\n{}\n", if label.is_empty() { "app" } else { label }, tail(&l, q.tail.unwrap_or(200)))); } } Ok(out) } // --- the client (in the main server) --------------------------------------------------------- #[derive(Clone)] pub struct Client { url: String, token: String, http: reqwest::Client, } impl Client { /// Configured through `CODECRAFT_RUNNER_URL` (e.g. http://runner:8788) and `CODECRAFT_RUNNER_TOKEN`. pub fn from_env() -> Option { let url = std::env::var("CODECRAFT_RUNNER_URL").ok()?.trim().trim_end_matches('/').to_string(); let token = std::env::var("CODECRAFT_RUNNER_TOKEN").ok()?.trim().to_string(); if url.is_empty() || token.is_empty() { return None; } Some(Self { url, token, http: reqwest::Client::builder().timeout(Duration::from_secs(40 * 60)).build().ok()? }) } pub fn host(&self) -> String { let rest = self.url.split_once("://").map(|(_, r)| r).unwrap_or(&self.url); rest.split('/').next().unwrap_or("").rsplit_once(':').map(|(h, _)| h).unwrap_or(rest).to_string() } async fn check(r: reqwest::Response) -> Result { if r.status().is_success() { return Ok(r); } let s = r.status(); bail!("test runner: {} {}", s, r.text().await.unwrap_or_default().trim()) } pub async fn start(&self, req: &StartReq) -> Result { let r = self.http.post(format!("{}/v1/envs", self.url)).bearer_auth(&self.token).json(req).send().await.context("test runner unreachable")?; Ok(Self::check(r).await?.json().await?) } pub async fn log(&self, id: &str) -> Result { let r = self.http.get(format!("{}/v1/envs/{id}/log?tail=300", self.url)).bearer_auth(&self.token).send().await?; Ok(Self::check(r).await?.text().await?) } pub async fn stop(&self, id: &str) { let _ = self.http.delete(format!("{}/v1/envs/{id}", self.url)).bearer_auth(&self.token).send().await; } } /// Listen on 127.0.0.1 (the wanted port if it is free, otherwise any) and forward every /// connection to `host:port`. Returns the local port and the task to abort. pub async fn forward_local(want: u16, host: String, port: u16) -> Result<(u16, JoinHandle<()>)> { let l = match TcpListener::bind(("127.0.0.1", want)).await { Ok(l) => l, Err(_) => TcpListener::bind(("127.0.0.1", 0)).await?, }; let local = l.local_addr()?.port(); let h = tokio::spawn(async move { loop { let Ok((mut inbound, _)) = l.accept().await else { break }; let host = host.clone(); tokio::spawn(async move { if let Ok(mut out) = TcpStream::connect((host.as_str(), port)).await { let _ = tokio::io::copy_bidirectional(&mut inbound, &mut out).await; let _ = out.shutdown().await; } }); } }); Ok((local, h)) } #[cfg(test)] mod tests { use super::*; #[test] fn setup_is_parsed() { let r = parse_setup(r#"{"install":"npm ci","start":"npm start","url":"http://localhost:8080/app","image":"node:20","env":{"A":"1"},"services":{"db":{"image":"postgres:16","env":{"POSTGRES_PASSWORD":"x"}}}}"#).unwrap(); assert_eq!((r.port, r.image.as_str(), r.install.as_str()), (8080, "node:20", "npm ci")); assert_eq!(r.services["db"].image, "postgres:16"); assert!(parse_setup(r#"{"url":"http://localhost:1"}"#).is_none()); assert!(parse_setup(r#"{"start":"x"}"#).is_none()); assert_eq!(parse_setup(r#"{"start":"x","port":3000}"#).unwrap().port, 3000); } #[test] fn ports_of_urls() { assert_eq!(port_of("http://localhost:8080"), Some(8080)); assert_eq!(port_of("http://localhost"), Some(80)); assert_eq!(port_of("https://x/y"), Some(443)); assert_eq!(port_of("nonsense"), None); assert_eq!(port_of("http://localhost:8080 (web build: trunk serve)"), Some(8080)); assert!(parse_setup(r#"{"start":"x","port":1,"isolated":false}"#).is_none()); } #[test] fn requests_are_checked() { let dir = std::env::temp_dir().join(format!("ccr-{}", std::process::id())); let work = dir.join("a/b"); std::fs::create_dir_all(&work).unwrap(); let ok = StartReq { work: work.display().to_string(), start: "x".into(), port: 80, ..Default::default() }; assert!(validate(&ok, &dir).is_ok()); // outside of the factory area, bad image, bad env name, dockerfile escaping the source assert!(validate(&StartReq { work: "/etc".into(), ..ok.clone() }, &dir).is_err()); assert!(validate(&StartReq { image: "x;rm".into(), ..ok.clone() }, &dir).is_err()); assert!(validate(&StartReq { image: "--privileged".into(), ..ok.clone() }, &dir).is_err()); assert!(validate(&StartReq { env: [("A B".to_string(), "1".to_string())].into(), ..ok.clone() }, &dir).is_err()); assert!(validate(&StartReq { dockerfile: "../../etc/passwd".into(), ..ok.clone() }, &dir).is_err()); assert!(validate(&StartReq { work: format!("{}/../../..", work.display()), ..ok.clone() }, &dir).is_err()); let _ = std::fs::remove_dir_all(&dir); } #[test] fn tokens_compare() { assert!(eq_const("abc", "abc")); assert!(!eq_const("abc", "abd")); assert!(!eq_const("abc", "ab")); } }