Files
codecraft/server/src/testrunner.rs
T
Richard ZinkandClaude Sonnet 5.5 6804139be7 Test runner: log start time, seed hit/miss and install time per environment
Merges the v73 changelog section into v74.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-20 20:15:47 +01:00

782 lines
34 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Throw-away test environments for the factory's acceptance runs.
//!
//! Two halves live here:
//!
//! * the **service** (`codecraft testrunner`): a small HTTP server that owns the
//! Docker socket. For every acceptance run it creates a private network, starts
//! the application (and optional helper services such as a database) in fresh
//! containers, forwards one port, and removes everything again afterwards.
//! It only ever runs a fixed, locked-down `docker` command line: no host mounts,
//! no privileged mode, capabilities dropped, memory/CPU/process limits.
//! * the **client** used by the factory in the main server: start an environment,
//! read its log, stop it, and forward a local port to it so that the agent can
//! use `http://localhost:PORT` as usual.
use anyhow::{Context, Result, bail};
use axum::extract::{Path as AxPath, Query, State};
use axum::http::{HeaderMap, StatusCode};
use axum::routing::{delete, get, post};
use axum::{Json, Router};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::io::AsyncWriteExt;
use tokio::net::{TcpListener, TcpStream};
use tokio::process::Command;
use tokio::task::JoinHandle;
const LABEL: &str = "codecraft.testrunner=1";
const CACHE_LABEL: &str = "codecraft.testrunner.cache=1";
/// What every environment container runs: link the shared directories, restore a matching seed, run
/// `install`, store a new seed (only after a successful install, so it is pristine), start the app.
const ENTRY_SCRIPT: &str = r#"cd /app
hit=0
if [ -d /cache ]; then
for d in $CC_CACHE_DIRS; do mkdir -p "/cache/dirs/$d" "$(dirname "$d")" && rm -rf "$d" && ln -s "/cache/dirs/$d" "$d"; done
if [ -d /usr/local/cargo ]; then for d in registry git; do mkdir -p "/cache/cargo/$d"; rm -rf "/usr/local/cargo/$d"; ln -s "/cache/cargo/$d" "/usr/local/cargo/$d"; done; fi
if [ -n "$CC_SEED_KEY" ]; then for p in $CC_SEED_PATHS; do if [ -e "/cache/seed/$CC_SEED_KEY/$p" ]; then mkdir -p "$(dirname "$p")" && cp -a "/cache/seed/$CC_SEED_KEY/$p" "$p" && hit=1 && echo "[runner] restored $p from the cache"; fi; done; fi
fi
if [ -n "$CC_SEED_KEY" ]; then if [ "$hit" = 1 ]; then echo "[runner] seed $CC_SEED_KEY: hit"; else echo "[runner] seed $CC_SEED_KEY: miss"; fi; fi
t0=$(date +%s)
if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi
echo "[runner] install took $(( $(date +%s) - t0 )) s"
if [ -d /cache ] && [ -n "$CC_SEED_KEY" ] && [ ! -d "/cache/seed/$CC_SEED_KEY" ]; then
t="/cache/seed/.tmp-$$"; rm -rf "$t"; mkdir -p "$t"; ok=1
for p in $CC_SEED_PATHS; do if [ -e "$p" ]; then mkdir -p "$t/$(dirname "$p")" && cp -a "$p" "$t/$p" || ok=0; fi; done
if [ "$ok" = 1 ]; then mv "$t" "/cache/seed/$CC_SEED_KEY" && echo "[runner] stored the seed $CC_SEED_KEY"; else rm -rf "$t"; fi
ls -1t /cache/seed 2>/dev/null | grep -v '^\.tmp' | tail -n +4 | while read -r o; do rm -rf "/cache/seed/$o"; done
fi
exec sh -c "$CC_START""#;
// --- what a repository asks for (.factory/setup.json) ------------------------------------
/// A helper container next to the application, reachable under its name (e.g. `db`).
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct Service {
pub image: String,
#[serde(default)]
pub env: BTreeMap<String, String>,
}
/// What stays between runs of one project (kept in a Docker volume of the runner).
///
/// * `dirs`: directories of the sources that persist and are shared by all runs (downloads, caches –
/// inputs only: whatever lands there is visible to the next run).
/// * `seed`: expensive *results* of `install` (e.g. a built disk image). They are stored after a successful
/// `install` under a key made from the image, the install command, the seed paths and the contents of the
/// `key` files/directories, and copied into fresh containers whenever that key matches – so a run never
/// sees changes of an earlier run, and a change of the listed sources builds the seed anew.
///
/// Tool caches (npm, yarn, pip, Go, Cargo registry and target directory) are always shared per project
/// unless `"cache": false`.
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct CacheCfg {
#[serde(default)]
pub disabled: bool,
#[serde(default)]
pub dirs: Vec<String>,
#[serde(default)]
pub seed_paths: Vec<String>,
#[serde(default)]
pub seed_key: Vec<String>,
}
/// The request to start an environment.
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct StartReq {
/// Checked-out source (below the factory area); copied into the container, never mounted.
pub work: String,
/// Image to run in (default: the runner's default image). Ignored when `dockerfile` is set.
#[serde(default)]
pub image: String,
/// Dockerfile inside the source to build the image from.
#[serde(default)]
pub dockerfile: String,
#[serde(default)]
pub install: String,
pub start: String,
pub port: u16,
#[serde(default)]
pub env: BTreeMap<String, String>,
#[serde(default)]
pub services: BTreeMap<String, Service>,
/// How long to wait until the port answers (seconds, default 300).
#[serde(default)]
pub ready_secs: u64,
/// Stable name of the project: its cache lives in the volume `cc-cache-<cache_id>`. Empty = no cache.
#[serde(default)]
pub cache_id: String,
#[serde(default)]
pub cache: CacheCfg,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Started {
pub id: String,
/// Port on the runner that forwards to the application.
pub port: u16,
}
/// Parse `.factory/setup.json` into a request (without `work`). `None` when there is nothing to start.
pub fn parse_setup(text: &str) -> Option<StartReq> {
let v: Value = serde_json::from_str(text).ok()?;
// `"isolated": false`: the app cannot run in a container (e.g. a native window); acceptance starts it itself.
if v["isolated"].as_bool() == Some(false) {
return None;
}
let start = v["start"].as_str()?.trim().to_string();
if start.is_empty() {
return None;
}
let url = v["url"].as_str().unwrap_or("");
let port = v["port"].as_u64().map(|p| p as u16).or_else(|| port_of(url))?;
let strs = |k: &str| -> BTreeMap<String, String> {
v[k].as_object().map(|o| o.iter().filter_map(|(a, b)| b.as_str().map(|s| (a.clone(), s.to_string()))).collect()).unwrap_or_default()
};
let services = v["services"]
.as_object()
.map(|o| {
o.iter()
.filter_map(|(name, s)| {
let image = s["image"].as_str()?.to_string();
let env = s["env"].as_object().map(|e| e.iter().filter_map(|(a, b)| b.as_str().map(|x| (a.clone(), x.to_string()))).collect()).unwrap_or_default();
Some((name.clone(), Service { image, env }))
})
.collect()
})
.unwrap_or_default();
Some(StartReq {
work: String::new(),
image: v["image"].as_str().unwrap_or("").trim().to_string(),
dockerfile: v["dockerfile"].as_str().unwrap_or("").trim().to_string(),
install: v["install"].as_str().unwrap_or("").trim().to_string(),
start,
port,
env: strs("env"),
services,
ready_secs: v["ready_seconds"].as_u64().unwrap_or(0),
cache_id: String::new(),
cache: parse_cache(&v["cache"]),
})
}
fn parse_cache(v: &Value) -> CacheCfg {
let list = |x: &Value| -> Vec<String> { x.as_array().map(|a| a.iter().filter_map(|s| s.as_str().map(|s| s.trim().trim_end_matches('/').to_string())).filter(|s| !s.is_empty()).collect()).unwrap_or_default() };
if v.as_bool() == Some(false) {
return CacheCfg { disabled: true, ..Default::default() };
}
CacheCfg { disabled: false, dirs: list(&v["dirs"]), seed_paths: list(&v["seed"]["paths"]), seed_key: list(&v["seed"]["key"]) }
}
/// Port of `http://host:1234/path` (80/443 by scheme when missing).
pub fn port_of(url: &str) -> Option<u16> {
let (scheme, rest) = url.trim().split_once("://")?;
// Free text after the URL ("http://localhost:8080 (trunk serve)") is ignored.
let host = rest.split(|c: char| c == '/' || c.is_whitespace()).next()?;
match host.rsplit_once(':') {
Some((_, p)) if p.chars().all(|c| c.is_ascii_digit()) && !p.is_empty() => p.parse().ok(),
_ => Some(if scheme == "https" { 443 } else { 80 }),
}
}
fn ident_ok(s: &str, extra: &str) -> bool {
!s.is_empty() && s.len() <= 200 && s.chars().all(|c| c.is_ascii_alphanumeric() || extra.contains(c))
}
fn image_ok(s: &str) -> bool {
ident_ok(s, "._/:@-") && !s.starts_with('-')
}
/// A path inside the sources: relative, no `..`, only plain characters (it ends up in a shell loop).
fn rel_ok(p: &str) -> bool {
ident_ok(p, "._/-") && !p.starts_with('/') && !p.starts_with('-') && !p.split('/').any(|c| c.is_empty() || c == "..") && !p.split('/').any(|c| c == ".git")
}
/// Key of a seed: image, install command, seed paths and the contents of the key files (directories: recursively).
pub fn seed_key(work: &Path, image: &str, req: &StartReq) -> Option<String> {
use sha2::{Digest, Sha256};
let c = &req.cache;
if c.disabled || c.seed_paths.is_empty() {
return None;
}
fn walk(base: &Path, rel: &Path, out: &mut Vec<std::path::PathBuf>) {
let full = base.join(rel);
if full.is_dir() {
let mut kids: Vec<_> = std::fs::read_dir(&full).map(|d| d.filter_map(|e| e.ok()).map(|e| e.file_name()).collect()).unwrap_or_default();
kids.sort();
for k in kids {
if k == ".git" || k == "node_modules" {
continue;
}
walk(base, &rel.join(k), out);
}
} else if full.is_file() {
out.push(rel.to_path_buf());
}
}
let mut h = Sha256::new();
for part in [image, &req.install, &c.seed_paths.join("\n"), &req.dockerfile] {
h.update(part.as_bytes());
h.update([0]);
}
let mut files = Vec::new();
let mut keys = c.seed_key.clone();
keys.sort();
for k in &keys {
h.update(k.as_bytes());
h.update([1]);
walk(work, Path::new(k), &mut files);
}
let mut total = 0u64;
for f in files {
let data = std::fs::read(work.join(&f)).ok()?;
total += data.len() as u64;
if total > 200 << 20 {
return None;
}
h.update(f.to_string_lossy().as_bytes());
h.update([2]);
h.update(&data);
}
let d = h.finalize();
Some(d.iter().take(10).map(|b| format!("{b:02x}")).collect())
}
fn env_ok(m: &BTreeMap<String, String>) -> bool {
m.len() <= 64 && m.iter().all(|(k, v)| k.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && ident_ok(k, "_") && v.len() <= 4096 && !v.contains('\0'))
}
/// Check a request against the rules of the runner and return the canonical source directory.
fn validate(req: &StartReq, factory_dir: &Path) -> Result<PathBuf> {
if req.start.trim().is_empty() || req.start.len() > 4096 || req.install.len() > 4096 {
bail!("start/install command missing or too long");
}
if req.port == 0 {
bail!("port missing");
}
if !req.image.is_empty() && !image_ok(&req.image) {
bail!("invalid image name");
}
if req.services.len() > 8 {
bail!("at most 8 services");
}
for (name, s) in &req.services {
if !ident_ok(name, "_-") || !image_ok(&s.image) || !env_ok(&s.env) {
bail!("invalid service {name}");
}
}
if !env_ok(&req.env) {
bail!("invalid environment variable");
}
if !req.cache_id.is_empty() && !ident_ok(&req.cache_id, "-_.") {
bail!("invalid cache id");
}
let c = &req.cache;
if c.dirs.len() > 16 || c.seed_paths.len() > 8 || c.seed_key.len() > 32 || !c.dirs.iter().chain(&c.seed_paths).chain(&c.seed_key).all(|p| rel_ok(p)) {
bail!("invalid cache path (relative, letters, digits and ._-/ only)");
}
if !c.seed_paths.is_empty() && c.seed_key.is_empty() {
bail!("cache.seed needs a key (the sources the seed depends on)");
}
let root = factory_dir.canonicalize().with_context(|| format!("factory area {} is not mounted", factory_dir.display()))?;
let work = Path::new(&req.work).canonicalize().context("source directory not found")?;
if !work.starts_with(&root) || !work.is_dir() {
bail!("the source must be a directory inside the factory area");
}
if !req.dockerfile.is_empty() {
let df = work.join(&req.dockerfile).canonicalize().context("dockerfile not found")?;
if !df.starts_with(&work) || !df.is_file() {
bail!("the dockerfile must be a file inside the source");
}
}
Ok(work)
}
// --- the service ----------------------------------------------------------------------------
struct Env {
containers: Vec<String>,
network: String,
image: Option<String>,
forward: JoinHandle<()>,
started: Instant,
}
struct Runner {
token: String,
factory_dir: PathBuf,
ports: (u16, u16),
memory: String,
cpus: String,
default_image: String,
max_age: Duration,
/// Own container name (to join the per-run networks); empty when not running in Docker.
me: String,
envs: Mutex<HashMap<String, Env>>,
}
async fn docker(args: &[&str]) -> Result<String> {
let out = Command::new("docker").args(args).stdin(Stdio::null()).output().await.context("cannot run docker")?;
if !out.status.success() {
bail!("docker {}: {}", args.first().unwrap_or(&""), String::from_utf8_lossy(&out.stderr).trim());
}
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
}
fn eq_const(a: &str, b: &str) -> bool {
a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |d, (x, y)| d | (x ^ y)) == 0
}
type Resp<T> = std::result::Result<T, (StatusCode, String)>;
fn authed(r: &Runner, h: &HeaderMap) -> Resp<()> {
let got = h.get("authorization").and_then(|v| v.to_str().ok()).and_then(|v| v.strip_prefix("Bearer ")).unwrap_or("");
if r.token.is_empty() || !eq_const(got, &r.token) {
return Err((StatusCode::UNAUTHORIZED, "bad token".into()));
}
Ok(())
}
/// Run `codecraft testrunner`: serve until the process is stopped.
pub async fn serve() -> Result<()> {
let token = std::env::var("RUNNER_TOKEN").unwrap_or_default();
if token.len() < 16 {
bail!("RUNNER_TOKEN must be set (at least 16 characters)");
}
let (lo, hi) = std::env::var("RUNNER_PORTS").ok().and_then(|s| s.split_once('-').and_then(|(a, b)| Some((a.trim().parse().ok()?, b.trim().parse().ok()?)))).unwrap_or((9100u16, 9199u16));
let runner = Arc::new(Runner {
token,
factory_dir: PathBuf::from(std::env::var("RUNNER_FACTORY_DIR").unwrap_or_else(|_| "/factory".into())),
ports: (lo, hi),
memory: std::env::var("RUNNER_MEMORY").unwrap_or_else(|_| "2g".into()),
cpus: std::env::var("RUNNER_CPUS").unwrap_or_else(|_| "2".into()),
default_image: std::env::var("RUNNER_DEFAULT_IMAGE").unwrap_or_else(|_| "node:22-bookworm".into()),
max_age: Duration::from_secs(60 * std::env::var("RUNNER_MAX_MINUTES").ok().and_then(|s| s.parse().ok()).unwrap_or(90)),
me: std::env::var("RUNNER_SELF").or_else(|_| std::env::var("HOSTNAME")).unwrap_or_default(),
envs: Mutex::new(HashMap::new()),
});
let ver = docker(&["version", "--format", "{{.Server.Version}}"]).await.context("the Docker socket is not available to the test runner")?;
tracing::info!("test runner: Docker {ver}, ports {lo}-{hi}, default image {}", runner.default_image);
sweep_all(&runner).await;
let r2 = runner.clone();
tokio::spawn(async move {
loop {
tokio::time::sleep(Duration::from_secs(60)).await;
let old: Vec<String> = r2.envs.lock().unwrap().iter().filter(|(_, e)| e.started.elapsed() > r2.max_age).map(|(k, _)| k.clone()).collect();
for id in old {
tracing::warn!("test runner: removing {id} (older than the limit)");
stop(&r2, &id).await;
}
}
});
let app = Router::new()
.route("/v1/health", get(health))
.route("/v1/envs", post(start_env))
.route("/v1/envs/{id}", delete(stop_env))
.route("/v1/envs/{id}/log", get(log_env))
.route("/v1/cache/{id}", delete(clear_cache))
.with_state(runner);
let bind = std::env::var("RUNNER_BIND").unwrap_or_else(|_| "0.0.0.0:8788".into());
let l = TcpListener::bind(&bind).await?;
tracing::info!("test runner listening on {bind}");
axum::serve(l, app).with_graceful_shutdown(async { let _ = tokio::signal::ctrl_c().await; }).await?;
Ok(())
}
/// Remove everything this service ever created (after a crash or restart).
async fn sweep_all(r: &Runner) {
if let Ok(ids) = docker(&["ps", "-aq", "--filter", &format!("label={LABEL}")]).await {
for id in ids.split_whitespace() {
let _ = docker(&["rm", "-f", "-v", id]).await;
}
}
if let Ok(nets) = docker(&["network", "ls", "-q", "--filter", &format!("label={LABEL}")]).await {
for n in nets.split_whitespace() {
if !r.me.is_empty() {
let _ = docker(&["network", "disconnect", "-f", n, &r.me]).await;
}
let _ = docker(&["network", "rm", n]).await;
}
}
}
async fn health(State(r): State<Arc<Runner>>, h: HeaderMap) -> Resp<Json<Value>> {
authed(&r, &h)?;
let n = r.envs.lock().unwrap().len();
Ok(Json(json!({"ok": true, "environments": n})))
}
async fn bind_forward(r: &Runner) -> Result<(TcpListener, u16)> {
for p in r.ports.0..=r.ports.1 {
if let Ok(l) = TcpListener::bind(("0.0.0.0", p)).await {
return Ok((l, p));
}
}
bail!("no free forward port")
}
async fn start_env(State(r): State<Arc<Runner>>, h: HeaderMap, Json(req): Json<StartReq>) -> Resp<Json<Started>> {
authed(&r, &h)?;
let work = validate(&req, &r.factory_dir).map_err(|e| (StatusCode::BAD_REQUEST, format!("{e:#}")))?;
if r.envs.lock().unwrap().len() >= 16 {
return Err((StatusCode::TOO_MANY_REQUESTS, "too many running environments".into()));
}
let id = uuid::Uuid::new_v4().simple().to_string()[..12].to_string();
match create(&r, &id, &req, &work).await {
Ok(s) => Ok(Json(s)),
Err(e) => {
let msg = format!("{e:#}");
stop(&r, &id).await;
Err((StatusCode::BAD_GATEWAY, msg))
}
}
}
async fn create(r: &Arc<Runner>, id: &str, req: &StartReq, work: &Path) -> Result<Started> {
let began = Instant::now();
let net = format!("cc-run-{id}");
let app = format!("cc-run-{id}");
// Register first, so a failure at any step is cleaned up by stop().
let forward = tokio::spawn(async {});
r.envs.lock().unwrap().insert(id.to_string(), Env { containers: vec![app.clone()], network: net.clone(), image: None, forward, started: Instant::now() });
docker(&["network", "create", "--label", LABEL, &net]).await?;
for (name, s) in &req.services {
let cname = format!("{app}-{name}");
let mut a: Vec<String> = ["run", "-d", "--name", &cname, "--network", &net, "--network-alias", name, "--label", LABEL, "--memory", &r.memory, "--pids-limit", "1024", "--security-opt", "no-new-privileges"].map(String::from).to_vec();
for (k, v) in &s.env {
a.push("-e".into());
a.push(format!("{k}={v}"));
}
a.push(s.image.clone());
let refs: Vec<&str> = a.iter().map(String::as_str).collect();
docker(&refs).await.with_context(|| format!("service {name}"))?;
r.envs.lock().unwrap().get_mut(id).unwrap().containers.push(cname);
}
let image = if req.dockerfile.is_empty() {
if req.image.is_empty() { r.default_image.clone() } else { req.image.clone() }
} else {
let tag = format!("cc-run-{id}");
let df = work.join(&req.dockerfile);
docker(&["build", "-q", "--label", LABEL, "-t", &tag, "-f", &df.display().to_string(), &work.display().to_string()]).await.context("docker build")?;
r.envs.lock().unwrap().get_mut(id).unwrap().image = Some(tag.clone());
tag
};
let mut a: Vec<String> = [
"create", "--name", &app, "--network", &net, "--label", LABEL, "--memory", &r.memory, "--cpus", &r.cpus, "--pids-limit", "1024",
"--cap-drop", "ALL", "--cap-add", "CHOWN", "--cap-add", "DAC_OVERRIDE", "--cap-add", "FOWNER", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "KILL", "--cap-add", "NET_BIND_SERVICE",
"--security-opt", "no-new-privileges", "-w", "/app", "-e", "CI=1",
]
.map(String::from)
.to_vec();
for (k, v) in &req.env {
a.push("-e".into());
a.push(format!("{k}={v}"));
}
// Per-project cache volume: tool caches, shared download directories and seeds (see CacheCfg).
let cached = !req.cache_id.is_empty() && !req.cache.disabled;
if cached {
let vol = format!("cc-cache-{}", req.cache_id);
docker(&["volume", "create", "--label", CACHE_LABEL, &vol]).await.context("cache volume")?;
a.push("-v".into());
a.push(format!("{vol}:/cache"));
for (k, v) in [
("npm_config_cache", "/cache/npm"),
("YARN_CACHE_FOLDER", "/cache/yarn"),
("PIP_CACHE_DIR", "/cache/pip"),
("GOMODCACHE", "/cache/go/mod"),
("GOCACHE", "/cache/go/build"),
("CARGO_TARGET_DIR", "/cache/target"),
("XDG_CACHE_HOME", "/cache/xdg"),
] {
if !req.env.contains_key(k) {
a.push("-e".into());
a.push(format!("{k}={v}"));
}
}
a.push("-e".into());
a.push(format!("CC_CACHE_DIRS={}", req.cache.dirs.join(" ")));
if let Some(key) = seed_key(work, &image, req) {
a.push("-e".into());
a.push(format!("CC_SEED_KEY={key}"));
a.push("-e".into());
a.push(format!("CC_SEED_PATHS={}", req.cache.seed_paths.join(" ")));
}
}
a.push("-e".into());
a.push(format!("CC_INSTALL={}", req.install));
a.push("-e".into());
a.push(format!("CC_START={}", req.start));
a.push(image);
a.extend(["sh", "-c", ENTRY_SCRIPT].map(String::from));
let refs: Vec<&str> = a.iter().map(String::as_str).collect();
docker(&refs).await.context("docker create")?;
// Copy the sources in (without .git); the container never sees the host directory.
let cp = Command::new("sh")
.args(["-c", r#"tar -C "$1" --exclude=.git -cf - . | docker cp - "$2":/app"#, "sh", &work.display().to_string(), &app])
.stdin(Stdio::null())
.output()
.await?;
if !cp.status.success() {
bail!("copying the sources failed: {}", String::from_utf8_lossy(&cp.stderr).trim());
}
docker(&["start", &app]).await?;
if !r.me.is_empty() {
let _ = docker(&["network", "connect", &net, &r.me]).await;
}
let ip = docker(&["inspect", "-f", &format!("{{{{(index .NetworkSettings.Networks \"{net}\").IPAddress}}}}"), &app]).await?;
if ip.is_empty() {
bail!("the container got no address");
}
// Wait until the application answers; give up early when it has exited.
let deadline = Instant::now() + Duration::from_secs(if req.ready_secs == 0 { 300 } else { req.ready_secs.min(1800) });
loop {
if TcpStream::connect((ip.as_str(), req.port)).await.is_ok() {
break;
}
let running = docker(&["inspect", "-f", "{{.State.Running}}", &app]).await.unwrap_or_default();
if running != "true" {
bail!("the application exited before it listened on port {}:\n{}", req.port, tail(&logs(&app).await, 40));
}
if Instant::now() > deadline {
bail!("the application did not listen on port {} in time:\n{}", req.port, tail(&logs(&app).await, 40));
}
tokio::time::sleep(Duration::from_millis(700)).await;
}
// Report what the cache brought: seed hit/miss, install time, total time until the app answered.
let facts: Vec<String> = logs(&app).await.lines().filter(|l| l.starts_with("[runner] seed ") || l.starts_with("[runner] install took")).map(|l| l.trim_start_matches("[runner] ").to_string()).collect();
let took = began.elapsed().as_secs();
let cache = if cached { "on" } else { "off" };
tracing::info!("test env {id} ({}) ready after {took} s; cache {cache}{}{}", req.cache_id, if facts.is_empty() { "" } else { "; " }, facts.join("; "));
let (listener, port) = bind_forward(r).await?;
let (target, tport) = (ip, req.port);
let fwd = tokio::spawn(async move {
loop {
let Ok((mut inbound, _)) = listener.accept().await else { break };
let t = target.clone();
tokio::spawn(async move {
if let Ok(mut out) = TcpStream::connect((t.as_str(), tport)).await {
let _ = tokio::io::copy_bidirectional(&mut inbound, &mut out).await;
}
});
}
});
if let Some(e) = r.envs.lock().unwrap().get_mut(id) {
e.forward.abort();
e.forward = fwd;
}
Ok(Started { id: id.to_string(), port })
}
async fn logs(container: &str) -> String {
match Command::new("docker").args(["logs", "--tail", "400", container]).output().await {
Ok(o) => format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)),
Err(_) => String::new(),
}
}
fn tail(s: &str, n: usize) -> String {
let lines: Vec<&str> = s.lines().collect();
lines[lines.len().saturating_sub(n)..].join("\n")
}
async fn stop(r: &Runner, id: &str) {
let Some(e) = r.envs.lock().unwrap().remove(id) else { return };
e.forward.abort();
for c in e.containers.iter().rev() {
let _ = docker(&["rm", "-f", "-v", c]).await;
}
if !r.me.is_empty() {
let _ = docker(&["network", "disconnect", "-f", &e.network, &r.me]).await;
}
let _ = docker(&["network", "rm", &e.network]).await;
if let Some(i) = e.image {
let _ = docker(&["rmi", "-f", &i]).await;
}
}
async fn stop_env(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>) -> Resp<Json<Value>> {
authed(&r, &h)?;
stop(&r, &id).await;
Ok(Json(json!({"ok": true})))
}
/// Forget the cache of a project (fails while an environment of it is running).
async fn clear_cache(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>) -> Resp<Json<Value>> {
authed(&r, &h)?;
if !ident_ok(&id, "-_.") {
return Err((StatusCode::BAD_REQUEST, "invalid cache id".into()));
}
docker(&["volume", "rm", &format!("cc-cache-{id}")]).await.map_err(|e| (StatusCode::CONFLICT, format!("{e:#}")))?;
Ok(Json(json!({"ok": true})))
}
#[derive(Deserialize)]
struct LogQ {
tail: Option<usize>,
}
async fn log_env(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>, Query(q): Query<LogQ>) -> Resp<String> {
authed(&r, &h)?;
let mut out = String::new();
let names = r.envs.lock().unwrap().get(&id).map(|e| e.containers.clone()).ok_or((StatusCode::NOT_FOUND, "unknown environment".to_string()))?;
for c in names {
let l = logs(&c).await;
if !l.is_empty() {
let label = c.strip_prefix(&format!("cc-run-{id}")).unwrap_or("").trim_start_matches('-');
out.push_str(&format!("==> {} <==\n{}\n", if label.is_empty() { "app" } else { label }, tail(&l, q.tail.unwrap_or(200))));
}
}
Ok(out)
}
// --- the client (in the main server) ---------------------------------------------------------
#[derive(Clone)]
pub struct Client {
url: String,
token: String,
http: reqwest::Client,
}
impl Client {
/// Configured through `CODECRAFT_RUNNER_URL` (e.g. http://runner:8788) and `CODECRAFT_RUNNER_TOKEN`.
pub fn from_env() -> Option<Self> {
let url = std::env::var("CODECRAFT_RUNNER_URL").ok()?.trim().trim_end_matches('/').to_string();
let token = std::env::var("CODECRAFT_RUNNER_TOKEN").ok()?.trim().to_string();
if url.is_empty() || token.is_empty() {
return None;
}
Some(Self { url, token, http: reqwest::Client::builder().timeout(Duration::from_secs(40 * 60)).build().ok()? })
}
pub fn host(&self) -> String {
let rest = self.url.split_once("://").map(|(_, r)| r).unwrap_or(&self.url);
rest.split('/').next().unwrap_or("").rsplit_once(':').map(|(h, _)| h).unwrap_or(rest).to_string()
}
async fn check(r: reqwest::Response) -> Result<reqwest::Response> {
if r.status().is_success() {
return Ok(r);
}
let s = r.status();
bail!("test runner: {} {}", s, r.text().await.unwrap_or_default().trim())
}
pub async fn start(&self, req: &StartReq) -> Result<Started> {
let r = self.http.post(format!("{}/v1/envs", self.url)).bearer_auth(&self.token).json(req).send().await.context("test runner unreachable")?;
Ok(Self::check(r).await?.json().await?)
}
pub async fn log(&self, id: &str) -> Result<String> {
let r = self.http.get(format!("{}/v1/envs/{id}/log?tail=300", self.url)).bearer_auth(&self.token).send().await?;
Ok(Self::check(r).await?.text().await?)
}
pub async fn stop(&self, id: &str) {
let _ = self.http.delete(format!("{}/v1/envs/{id}", self.url)).bearer_auth(&self.token).send().await;
}
}
/// Listen on 127.0.0.1 (the wanted port if it is free, otherwise any) and forward every
/// connection to `host:port`. Returns the local port and the task to abort.
pub async fn forward_local(want: u16, host: String, port: u16) -> Result<(u16, JoinHandle<()>)> {
let l = match TcpListener::bind(("127.0.0.1", want)).await {
Ok(l) => l,
Err(_) => TcpListener::bind(("127.0.0.1", 0)).await?,
};
let local = l.local_addr()?.port();
let h = tokio::spawn(async move {
loop {
let Ok((mut inbound, _)) = l.accept().await else { break };
let host = host.clone();
tokio::spawn(async move {
if let Ok(mut out) = TcpStream::connect((host.as_str(), port)).await {
let _ = tokio::io::copy_bidirectional(&mut inbound, &mut out).await;
let _ = out.shutdown().await;
}
});
}
});
Ok((local, h))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn setup_is_parsed() {
let r = parse_setup(r#"{"install":"npm ci","start":"npm start","url":"http://localhost:8080/app","image":"node:20","env":{"A":"1"},"services":{"db":{"image":"postgres:16","env":{"POSTGRES_PASSWORD":"x"}}}}"#).unwrap();
assert_eq!((r.port, r.image.as_str(), r.install.as_str()), (8080, "node:20", "npm ci"));
assert_eq!(r.services["db"].image, "postgres:16");
assert!(parse_setup(r#"{"url":"http://localhost:1"}"#).is_none());
assert!(parse_setup(r#"{"start":"x"}"#).is_none());
assert_eq!(parse_setup(r#"{"start":"x","port":3000}"#).unwrap().port, 3000);
}
#[test]
fn ports_of_urls() {
assert_eq!(port_of("http://localhost:8080"), Some(8080));
assert_eq!(port_of("http://localhost"), Some(80));
assert_eq!(port_of("https://x/y"), Some(443));
assert_eq!(port_of("nonsense"), None);
assert_eq!(port_of("http://localhost:8080 (web build: trunk serve)"), Some(8080));
assert!(parse_setup(r#"{"start":"x","port":1,"isolated":false}"#).is_none());
}
#[test]
fn requests_are_checked() {
let dir = std::env::temp_dir().join(format!("ccr-{}", std::process::id()));
let work = dir.join("a/b");
std::fs::create_dir_all(&work).unwrap();
let ok = StartReq { work: work.display().to_string(), start: "x".into(), port: 80, ..Default::default() };
assert!(validate(&ok, &dir).is_ok());
// outside of the factory area, bad image, bad env name, dockerfile escaping the source
assert!(validate(&StartReq { work: "/etc".into(), ..ok.clone() }, &dir).is_err());
assert!(validate(&StartReq { image: "x;rm".into(), ..ok.clone() }, &dir).is_err());
assert!(validate(&StartReq { image: "--privileged".into(), ..ok.clone() }, &dir).is_err());
assert!(validate(&StartReq { env: [("A B".to_string(), "1".to_string())].into(), ..ok.clone() }, &dir).is_err());
assert!(validate(&StartReq { dockerfile: "../../etc/passwd".into(), ..ok.clone() }, &dir).is_err());
assert!(validate(&StartReq { work: format!("{}/../../..", work.display()), ..ok.clone() }, &dir).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn cache_config_and_seed_key() {
let r = parse_setup(r#"{"start":"x","port":1,"cache":{"dirs":["build/downloads/"],"seed":{"paths":["data/ubuntu.ext4"],"key":["guest","package-lock.json"]}}}"#).unwrap();
assert_eq!(r.cache.dirs, vec!["build/downloads"]);
assert_eq!(r.cache.seed_key, vec!["guest", "package-lock.json"]);
assert!(parse_setup(r#"{"start":"x","port":1,"cache":false}"#).unwrap().cache.disabled);
assert!(!parse_setup(r#"{"start":"x","port":1}"#).unwrap().cache.disabled);
assert!(rel_ok("build/downloads") && !rel_ok("../x") && !rel_ok("/etc") && !rel_ok("a b") && !rel_ok("a/.git/b") && !rel_ok("a//b"));
// The key follows the contents of the key files, not of anything else.
let dir = std::env::temp_dir().join(format!("ccs-{}", std::process::id()));
std::fs::create_dir_all(dir.join("guest")).unwrap();
std::fs::write(dir.join("guest/a.c"), "one").unwrap();
std::fs::write(dir.join("other.txt"), "x").unwrap();
let mut req = r.clone();
req.install = "npm ci".into();
let k1 = seed_key(&dir, "node:24", &req).unwrap();
std::fs::write(dir.join("other.txt"), "changed").unwrap();
assert_eq!(seed_key(&dir, "node:24", &req).unwrap(), k1);
std::fs::write(dir.join("guest/a.c"), "two").unwrap();
assert_ne!(seed_key(&dir, "node:24", &req).unwrap(), k1);
assert_ne!(seed_key(&dir, "node:22", &req).unwrap(), seed_key(&dir, "node:24", &req).unwrap());
req.cache.seed_paths.clear();
assert!(seed_key(&dir, "node:24", &req).is_none());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn tokens_compare() {
assert!(eq_const("abc", "abc"));
assert!(!eq_const("abc", "abd"));
assert!(!eq_const("abc", "ab"));
}
}