Merges the v73 changelog section into v74. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
782 lines
34 KiB
Rust
782 lines
34 KiB
Rust
//! Throw-away test environments for the factory's acceptance runs.
|
||
//!
|
||
//! Two halves live here:
|
||
//!
|
||
//! * the **service** (`codecraft testrunner`): a small HTTP server that owns the
|
||
//! Docker socket. For every acceptance run it creates a private network, starts
|
||
//! the application (and optional helper services such as a database) in fresh
|
||
//! containers, forwards one port, and removes everything again afterwards.
|
||
//! It only ever runs a fixed, locked-down `docker` command line: no host mounts,
|
||
//! no privileged mode, capabilities dropped, memory/CPU/process limits.
|
||
//! * the **client** used by the factory in the main server: start an environment,
|
||
//! read its log, stop it, and forward a local port to it so that the agent can
|
||
//! use `http://localhost:PORT` as usual.
|
||
|
||
use anyhow::{Context, Result, bail};
|
||
use axum::extract::{Path as AxPath, Query, State};
|
||
use axum::http::{HeaderMap, StatusCode};
|
||
use axum::routing::{delete, get, post};
|
||
use axum::{Json, Router};
|
||
use serde::{Deserialize, Serialize};
|
||
use serde_json::{Value, json};
|
||
use std::collections::{BTreeMap, HashMap};
|
||
use std::path::{Path, PathBuf};
|
||
use std::process::Stdio;
|
||
use std::sync::{Arc, Mutex};
|
||
use std::time::{Duration, Instant};
|
||
use tokio::io::AsyncWriteExt;
|
||
use tokio::net::{TcpListener, TcpStream};
|
||
use tokio::process::Command;
|
||
use tokio::task::JoinHandle;
|
||
|
||
const LABEL: &str = "codecraft.testrunner=1";
|
||
const CACHE_LABEL: &str = "codecraft.testrunner.cache=1";
|
||
|
||
/// What every environment container runs: link the shared directories, restore a matching seed, run
|
||
/// `install`, store a new seed (only after a successful install, so it is pristine), start the app.
|
||
const ENTRY_SCRIPT: &str = r#"cd /app
|
||
hit=0
|
||
if [ -d /cache ]; then
|
||
for d in $CC_CACHE_DIRS; do mkdir -p "/cache/dirs/$d" "$(dirname "$d")" && rm -rf "$d" && ln -s "/cache/dirs/$d" "$d"; done
|
||
if [ -d /usr/local/cargo ]; then for d in registry git; do mkdir -p "/cache/cargo/$d"; rm -rf "/usr/local/cargo/$d"; ln -s "/cache/cargo/$d" "/usr/local/cargo/$d"; done; fi
|
||
if [ -n "$CC_SEED_KEY" ]; then for p in $CC_SEED_PATHS; do if [ -e "/cache/seed/$CC_SEED_KEY/$p" ]; then mkdir -p "$(dirname "$p")" && cp -a "/cache/seed/$CC_SEED_KEY/$p" "$p" && hit=1 && echo "[runner] restored $p from the cache"; fi; done; fi
|
||
fi
|
||
if [ -n "$CC_SEED_KEY" ]; then if [ "$hit" = 1 ]; then echo "[runner] seed $CC_SEED_KEY: hit"; else echo "[runner] seed $CC_SEED_KEY: miss"; fi; fi
|
||
t0=$(date +%s)
|
||
if [ -n "$CC_INSTALL" ]; then sh -c "$CC_INSTALL" || exit 1; fi
|
||
echo "[runner] install took $(( $(date +%s) - t0 )) s"
|
||
if [ -d /cache ] && [ -n "$CC_SEED_KEY" ] && [ ! -d "/cache/seed/$CC_SEED_KEY" ]; then
|
||
t="/cache/seed/.tmp-$$"; rm -rf "$t"; mkdir -p "$t"; ok=1
|
||
for p in $CC_SEED_PATHS; do if [ -e "$p" ]; then mkdir -p "$t/$(dirname "$p")" && cp -a "$p" "$t/$p" || ok=0; fi; done
|
||
if [ "$ok" = 1 ]; then mv "$t" "/cache/seed/$CC_SEED_KEY" && echo "[runner] stored the seed $CC_SEED_KEY"; else rm -rf "$t"; fi
|
||
ls -1t /cache/seed 2>/dev/null | grep -v '^\.tmp' | tail -n +4 | while read -r o; do rm -rf "/cache/seed/$o"; done
|
||
fi
|
||
exec sh -c "$CC_START""#;
|
||
|
||
// --- what a repository asks for (.factory/setup.json) ------------------------------------
|
||
|
||
/// A helper container next to the application, reachable under its name (e.g. `db`).
|
||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||
pub struct Service {
|
||
pub image: String,
|
||
#[serde(default)]
|
||
pub env: BTreeMap<String, String>,
|
||
}
|
||
|
||
/// What stays between runs of one project (kept in a Docker volume of the runner).
|
||
///
|
||
/// * `dirs`: directories of the sources that persist and are shared by all runs (downloads, caches –
|
||
/// inputs only: whatever lands there is visible to the next run).
|
||
/// * `seed`: expensive *results* of `install` (e.g. a built disk image). They are stored after a successful
|
||
/// `install` under a key made from the image, the install command, the seed paths and the contents of the
|
||
/// `key` files/directories, and copied into fresh containers whenever that key matches – so a run never
|
||
/// sees changes of an earlier run, and a change of the listed sources builds the seed anew.
|
||
///
|
||
/// Tool caches (npm, yarn, pip, Go, Cargo registry and target directory) are always shared per project
|
||
/// unless `"cache": false`.
|
||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||
pub struct CacheCfg {
|
||
#[serde(default)]
|
||
pub disabled: bool,
|
||
#[serde(default)]
|
||
pub dirs: Vec<String>,
|
||
#[serde(default)]
|
||
pub seed_paths: Vec<String>,
|
||
#[serde(default)]
|
||
pub seed_key: Vec<String>,
|
||
}
|
||
|
||
/// The request to start an environment.
|
||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||
pub struct StartReq {
|
||
/// Checked-out source (below the factory area); copied into the container, never mounted.
|
||
pub work: String,
|
||
/// Image to run in (default: the runner's default image). Ignored when `dockerfile` is set.
|
||
#[serde(default)]
|
||
pub image: String,
|
||
/// Dockerfile inside the source to build the image from.
|
||
#[serde(default)]
|
||
pub dockerfile: String,
|
||
#[serde(default)]
|
||
pub install: String,
|
||
pub start: String,
|
||
pub port: u16,
|
||
#[serde(default)]
|
||
pub env: BTreeMap<String, String>,
|
||
#[serde(default)]
|
||
pub services: BTreeMap<String, Service>,
|
||
/// How long to wait until the port answers (seconds, default 300).
|
||
#[serde(default)]
|
||
pub ready_secs: u64,
|
||
/// Stable name of the project: its cache lives in the volume `cc-cache-<cache_id>`. Empty = no cache.
|
||
#[serde(default)]
|
||
pub cache_id: String,
|
||
#[serde(default)]
|
||
pub cache: CacheCfg,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct Started {
|
||
pub id: String,
|
||
/// Port on the runner that forwards to the application.
|
||
pub port: u16,
|
||
}
|
||
|
||
/// Parse `.factory/setup.json` into a request (without `work`). `None` when there is nothing to start.
|
||
pub fn parse_setup(text: &str) -> Option<StartReq> {
|
||
let v: Value = serde_json::from_str(text).ok()?;
|
||
// `"isolated": false`: the app cannot run in a container (e.g. a native window); acceptance starts it itself.
|
||
if v["isolated"].as_bool() == Some(false) {
|
||
return None;
|
||
}
|
||
let start = v["start"].as_str()?.trim().to_string();
|
||
if start.is_empty() {
|
||
return None;
|
||
}
|
||
let url = v["url"].as_str().unwrap_or("");
|
||
let port = v["port"].as_u64().map(|p| p as u16).or_else(|| port_of(url))?;
|
||
let strs = |k: &str| -> BTreeMap<String, String> {
|
||
v[k].as_object().map(|o| o.iter().filter_map(|(a, b)| b.as_str().map(|s| (a.clone(), s.to_string()))).collect()).unwrap_or_default()
|
||
};
|
||
let services = v["services"]
|
||
.as_object()
|
||
.map(|o| {
|
||
o.iter()
|
||
.filter_map(|(name, s)| {
|
||
let image = s["image"].as_str()?.to_string();
|
||
let env = s["env"].as_object().map(|e| e.iter().filter_map(|(a, b)| b.as_str().map(|x| (a.clone(), x.to_string()))).collect()).unwrap_or_default();
|
||
Some((name.clone(), Service { image, env }))
|
||
})
|
||
.collect()
|
||
})
|
||
.unwrap_or_default();
|
||
Some(StartReq {
|
||
work: String::new(),
|
||
image: v["image"].as_str().unwrap_or("").trim().to_string(),
|
||
dockerfile: v["dockerfile"].as_str().unwrap_or("").trim().to_string(),
|
||
install: v["install"].as_str().unwrap_or("").trim().to_string(),
|
||
start,
|
||
port,
|
||
env: strs("env"),
|
||
services,
|
||
ready_secs: v["ready_seconds"].as_u64().unwrap_or(0),
|
||
cache_id: String::new(),
|
||
cache: parse_cache(&v["cache"]),
|
||
})
|
||
}
|
||
|
||
fn parse_cache(v: &Value) -> CacheCfg {
|
||
let list = |x: &Value| -> Vec<String> { x.as_array().map(|a| a.iter().filter_map(|s| s.as_str().map(|s| s.trim().trim_end_matches('/').to_string())).filter(|s| !s.is_empty()).collect()).unwrap_or_default() };
|
||
if v.as_bool() == Some(false) {
|
||
return CacheCfg { disabled: true, ..Default::default() };
|
||
}
|
||
CacheCfg { disabled: false, dirs: list(&v["dirs"]), seed_paths: list(&v["seed"]["paths"]), seed_key: list(&v["seed"]["key"]) }
|
||
}
|
||
|
||
/// Port of `http://host:1234/path` (80/443 by scheme when missing).
|
||
pub fn port_of(url: &str) -> Option<u16> {
|
||
let (scheme, rest) = url.trim().split_once("://")?;
|
||
// Free text after the URL ("http://localhost:8080 (trunk serve)") is ignored.
|
||
let host = rest.split(|c: char| c == '/' || c.is_whitespace()).next()?;
|
||
match host.rsplit_once(':') {
|
||
Some((_, p)) if p.chars().all(|c| c.is_ascii_digit()) && !p.is_empty() => p.parse().ok(),
|
||
_ => Some(if scheme == "https" { 443 } else { 80 }),
|
||
}
|
||
}
|
||
|
||
fn ident_ok(s: &str, extra: &str) -> bool {
|
||
!s.is_empty() && s.len() <= 200 && s.chars().all(|c| c.is_ascii_alphanumeric() || extra.contains(c))
|
||
}
|
||
|
||
fn image_ok(s: &str) -> bool {
|
||
ident_ok(s, "._/:@-") && !s.starts_with('-')
|
||
}
|
||
|
||
/// A path inside the sources: relative, no `..`, only plain characters (it ends up in a shell loop).
|
||
fn rel_ok(p: &str) -> bool {
|
||
ident_ok(p, "._/-") && !p.starts_with('/') && !p.starts_with('-') && !p.split('/').any(|c| c.is_empty() || c == "..") && !p.split('/').any(|c| c == ".git")
|
||
}
|
||
|
||
/// Key of a seed: image, install command, seed paths and the contents of the key files (directories: recursively).
|
||
pub fn seed_key(work: &Path, image: &str, req: &StartReq) -> Option<String> {
|
||
use sha2::{Digest, Sha256};
|
||
let c = &req.cache;
|
||
if c.disabled || c.seed_paths.is_empty() {
|
||
return None;
|
||
}
|
||
fn walk(base: &Path, rel: &Path, out: &mut Vec<std::path::PathBuf>) {
|
||
let full = base.join(rel);
|
||
if full.is_dir() {
|
||
let mut kids: Vec<_> = std::fs::read_dir(&full).map(|d| d.filter_map(|e| e.ok()).map(|e| e.file_name()).collect()).unwrap_or_default();
|
||
kids.sort();
|
||
for k in kids {
|
||
if k == ".git" || k == "node_modules" {
|
||
continue;
|
||
}
|
||
walk(base, &rel.join(k), out);
|
||
}
|
||
} else if full.is_file() {
|
||
out.push(rel.to_path_buf());
|
||
}
|
||
}
|
||
let mut h = Sha256::new();
|
||
for part in [image, &req.install, &c.seed_paths.join("\n"), &req.dockerfile] {
|
||
h.update(part.as_bytes());
|
||
h.update([0]);
|
||
}
|
||
let mut files = Vec::new();
|
||
let mut keys = c.seed_key.clone();
|
||
keys.sort();
|
||
for k in &keys {
|
||
h.update(k.as_bytes());
|
||
h.update([1]);
|
||
walk(work, Path::new(k), &mut files);
|
||
}
|
||
let mut total = 0u64;
|
||
for f in files {
|
||
let data = std::fs::read(work.join(&f)).ok()?;
|
||
total += data.len() as u64;
|
||
if total > 200 << 20 {
|
||
return None;
|
||
}
|
||
h.update(f.to_string_lossy().as_bytes());
|
||
h.update([2]);
|
||
h.update(&data);
|
||
}
|
||
let d = h.finalize();
|
||
Some(d.iter().take(10).map(|b| format!("{b:02x}")).collect())
|
||
}
|
||
|
||
fn env_ok(m: &BTreeMap<String, String>) -> bool {
|
||
m.len() <= 64 && m.iter().all(|(k, v)| k.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && ident_ok(k, "_") && v.len() <= 4096 && !v.contains('\0'))
|
||
}
|
||
|
||
/// Check a request against the rules of the runner and return the canonical source directory.
|
||
fn validate(req: &StartReq, factory_dir: &Path) -> Result<PathBuf> {
|
||
if req.start.trim().is_empty() || req.start.len() > 4096 || req.install.len() > 4096 {
|
||
bail!("start/install command missing or too long");
|
||
}
|
||
if req.port == 0 {
|
||
bail!("port missing");
|
||
}
|
||
if !req.image.is_empty() && !image_ok(&req.image) {
|
||
bail!("invalid image name");
|
||
}
|
||
if req.services.len() > 8 {
|
||
bail!("at most 8 services");
|
||
}
|
||
for (name, s) in &req.services {
|
||
if !ident_ok(name, "_-") || !image_ok(&s.image) || !env_ok(&s.env) {
|
||
bail!("invalid service {name}");
|
||
}
|
||
}
|
||
if !env_ok(&req.env) {
|
||
bail!("invalid environment variable");
|
||
}
|
||
if !req.cache_id.is_empty() && !ident_ok(&req.cache_id, "-_.") {
|
||
bail!("invalid cache id");
|
||
}
|
||
let c = &req.cache;
|
||
if c.dirs.len() > 16 || c.seed_paths.len() > 8 || c.seed_key.len() > 32 || !c.dirs.iter().chain(&c.seed_paths).chain(&c.seed_key).all(|p| rel_ok(p)) {
|
||
bail!("invalid cache path (relative, letters, digits and ._-/ only)");
|
||
}
|
||
if !c.seed_paths.is_empty() && c.seed_key.is_empty() {
|
||
bail!("cache.seed needs a key (the sources the seed depends on)");
|
||
}
|
||
let root = factory_dir.canonicalize().with_context(|| format!("factory area {} is not mounted", factory_dir.display()))?;
|
||
let work = Path::new(&req.work).canonicalize().context("source directory not found")?;
|
||
if !work.starts_with(&root) || !work.is_dir() {
|
||
bail!("the source must be a directory inside the factory area");
|
||
}
|
||
if !req.dockerfile.is_empty() {
|
||
let df = work.join(&req.dockerfile).canonicalize().context("dockerfile not found")?;
|
||
if !df.starts_with(&work) || !df.is_file() {
|
||
bail!("the dockerfile must be a file inside the source");
|
||
}
|
||
}
|
||
Ok(work)
|
||
}
|
||
|
||
// --- the service ----------------------------------------------------------------------------
|
||
|
||
struct Env {
|
||
containers: Vec<String>,
|
||
network: String,
|
||
image: Option<String>,
|
||
forward: JoinHandle<()>,
|
||
started: Instant,
|
||
}
|
||
|
||
struct Runner {
|
||
token: String,
|
||
factory_dir: PathBuf,
|
||
ports: (u16, u16),
|
||
memory: String,
|
||
cpus: String,
|
||
default_image: String,
|
||
max_age: Duration,
|
||
/// Own container name (to join the per-run networks); empty when not running in Docker.
|
||
me: String,
|
||
envs: Mutex<HashMap<String, Env>>,
|
||
}
|
||
|
||
async fn docker(args: &[&str]) -> Result<String> {
|
||
let out = Command::new("docker").args(args).stdin(Stdio::null()).output().await.context("cannot run docker")?;
|
||
if !out.status.success() {
|
||
bail!("docker {}: {}", args.first().unwrap_or(&""), String::from_utf8_lossy(&out.stderr).trim());
|
||
}
|
||
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
|
||
}
|
||
|
||
fn eq_const(a: &str, b: &str) -> bool {
|
||
a.len() == b.len() && a.bytes().zip(b.bytes()).fold(0u8, |d, (x, y)| d | (x ^ y)) == 0
|
||
}
|
||
|
||
type Resp<T> = std::result::Result<T, (StatusCode, String)>;
|
||
|
||
fn authed(r: &Runner, h: &HeaderMap) -> Resp<()> {
|
||
let got = h.get("authorization").and_then(|v| v.to_str().ok()).and_then(|v| v.strip_prefix("Bearer ")).unwrap_or("");
|
||
if r.token.is_empty() || !eq_const(got, &r.token) {
|
||
return Err((StatusCode::UNAUTHORIZED, "bad token".into()));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Run `codecraft testrunner`: serve until the process is stopped.
|
||
pub async fn serve() -> Result<()> {
|
||
let token = std::env::var("RUNNER_TOKEN").unwrap_or_default();
|
||
if token.len() < 16 {
|
||
bail!("RUNNER_TOKEN must be set (at least 16 characters)");
|
||
}
|
||
let (lo, hi) = std::env::var("RUNNER_PORTS").ok().and_then(|s| s.split_once('-').and_then(|(a, b)| Some((a.trim().parse().ok()?, b.trim().parse().ok()?)))).unwrap_or((9100u16, 9199u16));
|
||
let runner = Arc::new(Runner {
|
||
token,
|
||
factory_dir: PathBuf::from(std::env::var("RUNNER_FACTORY_DIR").unwrap_or_else(|_| "/factory".into())),
|
||
ports: (lo, hi),
|
||
memory: std::env::var("RUNNER_MEMORY").unwrap_or_else(|_| "2g".into()),
|
||
cpus: std::env::var("RUNNER_CPUS").unwrap_or_else(|_| "2".into()),
|
||
default_image: std::env::var("RUNNER_DEFAULT_IMAGE").unwrap_or_else(|_| "node:22-bookworm".into()),
|
||
max_age: Duration::from_secs(60 * std::env::var("RUNNER_MAX_MINUTES").ok().and_then(|s| s.parse().ok()).unwrap_or(90)),
|
||
me: std::env::var("RUNNER_SELF").or_else(|_| std::env::var("HOSTNAME")).unwrap_or_default(),
|
||
envs: Mutex::new(HashMap::new()),
|
||
});
|
||
let ver = docker(&["version", "--format", "{{.Server.Version}}"]).await.context("the Docker socket is not available to the test runner")?;
|
||
tracing::info!("test runner: Docker {ver}, ports {lo}-{hi}, default image {}", runner.default_image);
|
||
sweep_all(&runner).await;
|
||
let r2 = runner.clone();
|
||
tokio::spawn(async move {
|
||
loop {
|
||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||
let old: Vec<String> = r2.envs.lock().unwrap().iter().filter(|(_, e)| e.started.elapsed() > r2.max_age).map(|(k, _)| k.clone()).collect();
|
||
for id in old {
|
||
tracing::warn!("test runner: removing {id} (older than the limit)");
|
||
stop(&r2, &id).await;
|
||
}
|
||
}
|
||
});
|
||
let app = Router::new()
|
||
.route("/v1/health", get(health))
|
||
.route("/v1/envs", post(start_env))
|
||
.route("/v1/envs/{id}", delete(stop_env))
|
||
.route("/v1/envs/{id}/log", get(log_env))
|
||
.route("/v1/cache/{id}", delete(clear_cache))
|
||
.with_state(runner);
|
||
let bind = std::env::var("RUNNER_BIND").unwrap_or_else(|_| "0.0.0.0:8788".into());
|
||
let l = TcpListener::bind(&bind).await?;
|
||
tracing::info!("test runner listening on {bind}");
|
||
axum::serve(l, app).with_graceful_shutdown(async { let _ = tokio::signal::ctrl_c().await; }).await?;
|
||
Ok(())
|
||
}
|
||
|
||
/// Remove everything this service ever created (after a crash or restart).
|
||
async fn sweep_all(r: &Runner) {
|
||
if let Ok(ids) = docker(&["ps", "-aq", "--filter", &format!("label={LABEL}")]).await {
|
||
for id in ids.split_whitespace() {
|
||
let _ = docker(&["rm", "-f", "-v", id]).await;
|
||
}
|
||
}
|
||
if let Ok(nets) = docker(&["network", "ls", "-q", "--filter", &format!("label={LABEL}")]).await {
|
||
for n in nets.split_whitespace() {
|
||
if !r.me.is_empty() {
|
||
let _ = docker(&["network", "disconnect", "-f", n, &r.me]).await;
|
||
}
|
||
let _ = docker(&["network", "rm", n]).await;
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn health(State(r): State<Arc<Runner>>, h: HeaderMap) -> Resp<Json<Value>> {
|
||
authed(&r, &h)?;
|
||
let n = r.envs.lock().unwrap().len();
|
||
Ok(Json(json!({"ok": true, "environments": n})))
|
||
}
|
||
|
||
async fn bind_forward(r: &Runner) -> Result<(TcpListener, u16)> {
|
||
for p in r.ports.0..=r.ports.1 {
|
||
if let Ok(l) = TcpListener::bind(("0.0.0.0", p)).await {
|
||
return Ok((l, p));
|
||
}
|
||
}
|
||
bail!("no free forward port")
|
||
}
|
||
|
||
async fn start_env(State(r): State<Arc<Runner>>, h: HeaderMap, Json(req): Json<StartReq>) -> Resp<Json<Started>> {
|
||
authed(&r, &h)?;
|
||
let work = validate(&req, &r.factory_dir).map_err(|e| (StatusCode::BAD_REQUEST, format!("{e:#}")))?;
|
||
if r.envs.lock().unwrap().len() >= 16 {
|
||
return Err((StatusCode::TOO_MANY_REQUESTS, "too many running environments".into()));
|
||
}
|
||
let id = uuid::Uuid::new_v4().simple().to_string()[..12].to_string();
|
||
match create(&r, &id, &req, &work).await {
|
||
Ok(s) => Ok(Json(s)),
|
||
Err(e) => {
|
||
let msg = format!("{e:#}");
|
||
stop(&r, &id).await;
|
||
Err((StatusCode::BAD_GATEWAY, msg))
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn create(r: &Arc<Runner>, id: &str, req: &StartReq, work: &Path) -> Result<Started> {
|
||
let began = Instant::now();
|
||
let net = format!("cc-run-{id}");
|
||
let app = format!("cc-run-{id}");
|
||
// Register first, so a failure at any step is cleaned up by stop().
|
||
let forward = tokio::spawn(async {});
|
||
r.envs.lock().unwrap().insert(id.to_string(), Env { containers: vec![app.clone()], network: net.clone(), image: None, forward, started: Instant::now() });
|
||
docker(&["network", "create", "--label", LABEL, &net]).await?;
|
||
for (name, s) in &req.services {
|
||
let cname = format!("{app}-{name}");
|
||
let mut a: Vec<String> = ["run", "-d", "--name", &cname, "--network", &net, "--network-alias", name, "--label", LABEL, "--memory", &r.memory, "--pids-limit", "1024", "--security-opt", "no-new-privileges"].map(String::from).to_vec();
|
||
for (k, v) in &s.env {
|
||
a.push("-e".into());
|
||
a.push(format!("{k}={v}"));
|
||
}
|
||
a.push(s.image.clone());
|
||
let refs: Vec<&str> = a.iter().map(String::as_str).collect();
|
||
docker(&refs).await.with_context(|| format!("service {name}"))?;
|
||
r.envs.lock().unwrap().get_mut(id).unwrap().containers.push(cname);
|
||
}
|
||
let image = if req.dockerfile.is_empty() {
|
||
if req.image.is_empty() { r.default_image.clone() } else { req.image.clone() }
|
||
} else {
|
||
let tag = format!("cc-run-{id}");
|
||
let df = work.join(&req.dockerfile);
|
||
docker(&["build", "-q", "--label", LABEL, "-t", &tag, "-f", &df.display().to_string(), &work.display().to_string()]).await.context("docker build")?;
|
||
r.envs.lock().unwrap().get_mut(id).unwrap().image = Some(tag.clone());
|
||
tag
|
||
};
|
||
let mut a: Vec<String> = [
|
||
"create", "--name", &app, "--network", &net, "--label", LABEL, "--memory", &r.memory, "--cpus", &r.cpus, "--pids-limit", "1024",
|
||
"--cap-drop", "ALL", "--cap-add", "CHOWN", "--cap-add", "DAC_OVERRIDE", "--cap-add", "FOWNER", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "KILL", "--cap-add", "NET_BIND_SERVICE",
|
||
"--security-opt", "no-new-privileges", "-w", "/app", "-e", "CI=1",
|
||
]
|
||
.map(String::from)
|
||
.to_vec();
|
||
for (k, v) in &req.env {
|
||
a.push("-e".into());
|
||
a.push(format!("{k}={v}"));
|
||
}
|
||
// Per-project cache volume: tool caches, shared download directories and seeds (see CacheCfg).
|
||
let cached = !req.cache_id.is_empty() && !req.cache.disabled;
|
||
if cached {
|
||
let vol = format!("cc-cache-{}", req.cache_id);
|
||
docker(&["volume", "create", "--label", CACHE_LABEL, &vol]).await.context("cache volume")?;
|
||
a.push("-v".into());
|
||
a.push(format!("{vol}:/cache"));
|
||
for (k, v) in [
|
||
("npm_config_cache", "/cache/npm"),
|
||
("YARN_CACHE_FOLDER", "/cache/yarn"),
|
||
("PIP_CACHE_DIR", "/cache/pip"),
|
||
("GOMODCACHE", "/cache/go/mod"),
|
||
("GOCACHE", "/cache/go/build"),
|
||
("CARGO_TARGET_DIR", "/cache/target"),
|
||
("XDG_CACHE_HOME", "/cache/xdg"),
|
||
] {
|
||
if !req.env.contains_key(k) {
|
||
a.push("-e".into());
|
||
a.push(format!("{k}={v}"));
|
||
}
|
||
}
|
||
a.push("-e".into());
|
||
a.push(format!("CC_CACHE_DIRS={}", req.cache.dirs.join(" ")));
|
||
if let Some(key) = seed_key(work, &image, req) {
|
||
a.push("-e".into());
|
||
a.push(format!("CC_SEED_KEY={key}"));
|
||
a.push("-e".into());
|
||
a.push(format!("CC_SEED_PATHS={}", req.cache.seed_paths.join(" ")));
|
||
}
|
||
}
|
||
a.push("-e".into());
|
||
a.push(format!("CC_INSTALL={}", req.install));
|
||
a.push("-e".into());
|
||
a.push(format!("CC_START={}", req.start));
|
||
a.push(image);
|
||
a.extend(["sh", "-c", ENTRY_SCRIPT].map(String::from));
|
||
let refs: Vec<&str> = a.iter().map(String::as_str).collect();
|
||
docker(&refs).await.context("docker create")?;
|
||
// Copy the sources in (without .git); the container never sees the host directory.
|
||
let cp = Command::new("sh")
|
||
.args(["-c", r#"tar -C "$1" --exclude=.git -cf - . | docker cp - "$2":/app"#, "sh", &work.display().to_string(), &app])
|
||
.stdin(Stdio::null())
|
||
.output()
|
||
.await?;
|
||
if !cp.status.success() {
|
||
bail!("copying the sources failed: {}", String::from_utf8_lossy(&cp.stderr).trim());
|
||
}
|
||
docker(&["start", &app]).await?;
|
||
if !r.me.is_empty() {
|
||
let _ = docker(&["network", "connect", &net, &r.me]).await;
|
||
}
|
||
let ip = docker(&["inspect", "-f", &format!("{{{{(index .NetworkSettings.Networks \"{net}\").IPAddress}}}}"), &app]).await?;
|
||
if ip.is_empty() {
|
||
bail!("the container got no address");
|
||
}
|
||
// Wait until the application answers; give up early when it has exited.
|
||
let deadline = Instant::now() + Duration::from_secs(if req.ready_secs == 0 { 300 } else { req.ready_secs.min(1800) });
|
||
loop {
|
||
if TcpStream::connect((ip.as_str(), req.port)).await.is_ok() {
|
||
break;
|
||
}
|
||
let running = docker(&["inspect", "-f", "{{.State.Running}}", &app]).await.unwrap_or_default();
|
||
if running != "true" {
|
||
bail!("the application exited before it listened on port {}:\n{}", req.port, tail(&logs(&app).await, 40));
|
||
}
|
||
if Instant::now() > deadline {
|
||
bail!("the application did not listen on port {} in time:\n{}", req.port, tail(&logs(&app).await, 40));
|
||
}
|
||
tokio::time::sleep(Duration::from_millis(700)).await;
|
||
}
|
||
// Report what the cache brought: seed hit/miss, install time, total time until the app answered.
|
||
let facts: Vec<String> = logs(&app).await.lines().filter(|l| l.starts_with("[runner] seed ") || l.starts_with("[runner] install took")).map(|l| l.trim_start_matches("[runner] ").to_string()).collect();
|
||
let took = began.elapsed().as_secs();
|
||
let cache = if cached { "on" } else { "off" };
|
||
tracing::info!("test env {id} ({}) ready after {took} s; cache {cache}{}{}", req.cache_id, if facts.is_empty() { "" } else { "; " }, facts.join("; "));
|
||
let (listener, port) = bind_forward(r).await?;
|
||
let (target, tport) = (ip, req.port);
|
||
let fwd = tokio::spawn(async move {
|
||
loop {
|
||
let Ok((mut inbound, _)) = listener.accept().await else { break };
|
||
let t = target.clone();
|
||
tokio::spawn(async move {
|
||
if let Ok(mut out) = TcpStream::connect((t.as_str(), tport)).await {
|
||
let _ = tokio::io::copy_bidirectional(&mut inbound, &mut out).await;
|
||
}
|
||
});
|
||
}
|
||
});
|
||
if let Some(e) = r.envs.lock().unwrap().get_mut(id) {
|
||
e.forward.abort();
|
||
e.forward = fwd;
|
||
}
|
||
Ok(Started { id: id.to_string(), port })
|
||
}
|
||
|
||
async fn logs(container: &str) -> String {
|
||
match Command::new("docker").args(["logs", "--tail", "400", container]).output().await {
|
||
Ok(o) => format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)),
|
||
Err(_) => String::new(),
|
||
}
|
||
}
|
||
|
||
fn tail(s: &str, n: usize) -> String {
|
||
let lines: Vec<&str> = s.lines().collect();
|
||
lines[lines.len().saturating_sub(n)..].join("\n")
|
||
}
|
||
|
||
async fn stop(r: &Runner, id: &str) {
|
||
let Some(e) = r.envs.lock().unwrap().remove(id) else { return };
|
||
e.forward.abort();
|
||
for c in e.containers.iter().rev() {
|
||
let _ = docker(&["rm", "-f", "-v", c]).await;
|
||
}
|
||
if !r.me.is_empty() {
|
||
let _ = docker(&["network", "disconnect", "-f", &e.network, &r.me]).await;
|
||
}
|
||
let _ = docker(&["network", "rm", &e.network]).await;
|
||
if let Some(i) = e.image {
|
||
let _ = docker(&["rmi", "-f", &i]).await;
|
||
}
|
||
}
|
||
|
||
async fn stop_env(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>) -> Resp<Json<Value>> {
|
||
authed(&r, &h)?;
|
||
stop(&r, &id).await;
|
||
Ok(Json(json!({"ok": true})))
|
||
}
|
||
|
||
/// Forget the cache of a project (fails while an environment of it is running).
|
||
async fn clear_cache(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>) -> Resp<Json<Value>> {
|
||
authed(&r, &h)?;
|
||
if !ident_ok(&id, "-_.") {
|
||
return Err((StatusCode::BAD_REQUEST, "invalid cache id".into()));
|
||
}
|
||
docker(&["volume", "rm", &format!("cc-cache-{id}")]).await.map_err(|e| (StatusCode::CONFLICT, format!("{e:#}")))?;
|
||
Ok(Json(json!({"ok": true})))
|
||
}
|
||
|
||
#[derive(Deserialize)]
|
||
struct LogQ {
|
||
tail: Option<usize>,
|
||
}
|
||
|
||
async fn log_env(State(r): State<Arc<Runner>>, h: HeaderMap, AxPath(id): AxPath<String>, Query(q): Query<LogQ>) -> Resp<String> {
|
||
authed(&r, &h)?;
|
||
let mut out = String::new();
|
||
let names = r.envs.lock().unwrap().get(&id).map(|e| e.containers.clone()).ok_or((StatusCode::NOT_FOUND, "unknown environment".to_string()))?;
|
||
for c in names {
|
||
let l = logs(&c).await;
|
||
if !l.is_empty() {
|
||
let label = c.strip_prefix(&format!("cc-run-{id}")).unwrap_or("").trim_start_matches('-');
|
||
out.push_str(&format!("==> {} <==\n{}\n", if label.is_empty() { "app" } else { label }, tail(&l, q.tail.unwrap_or(200))));
|
||
}
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
// --- the client (in the main server) ---------------------------------------------------------
|
||
|
||
#[derive(Clone)]
|
||
pub struct Client {
|
||
url: String,
|
||
token: String,
|
||
http: reqwest::Client,
|
||
}
|
||
|
||
impl Client {
|
||
/// Configured through `CODECRAFT_RUNNER_URL` (e.g. http://runner:8788) and `CODECRAFT_RUNNER_TOKEN`.
|
||
pub fn from_env() -> Option<Self> {
|
||
let url = std::env::var("CODECRAFT_RUNNER_URL").ok()?.trim().trim_end_matches('/').to_string();
|
||
let token = std::env::var("CODECRAFT_RUNNER_TOKEN").ok()?.trim().to_string();
|
||
if url.is_empty() || token.is_empty() {
|
||
return None;
|
||
}
|
||
Some(Self { url, token, http: reqwest::Client::builder().timeout(Duration::from_secs(40 * 60)).build().ok()? })
|
||
}
|
||
|
||
pub fn host(&self) -> String {
|
||
let rest = self.url.split_once("://").map(|(_, r)| r).unwrap_or(&self.url);
|
||
rest.split('/').next().unwrap_or("").rsplit_once(':').map(|(h, _)| h).unwrap_or(rest).to_string()
|
||
}
|
||
|
||
async fn check(r: reqwest::Response) -> Result<reqwest::Response> {
|
||
if r.status().is_success() {
|
||
return Ok(r);
|
||
}
|
||
let s = r.status();
|
||
bail!("test runner: {} {}", s, r.text().await.unwrap_or_default().trim())
|
||
}
|
||
|
||
pub async fn start(&self, req: &StartReq) -> Result<Started> {
|
||
let r = self.http.post(format!("{}/v1/envs", self.url)).bearer_auth(&self.token).json(req).send().await.context("test runner unreachable")?;
|
||
Ok(Self::check(r).await?.json().await?)
|
||
}
|
||
|
||
pub async fn log(&self, id: &str) -> Result<String> {
|
||
let r = self.http.get(format!("{}/v1/envs/{id}/log?tail=300", self.url)).bearer_auth(&self.token).send().await?;
|
||
Ok(Self::check(r).await?.text().await?)
|
||
}
|
||
|
||
pub async fn stop(&self, id: &str) {
|
||
let _ = self.http.delete(format!("{}/v1/envs/{id}", self.url)).bearer_auth(&self.token).send().await;
|
||
}
|
||
}
|
||
|
||
/// Listen on 127.0.0.1 (the wanted port if it is free, otherwise any) and forward every
|
||
/// connection to `host:port`. Returns the local port and the task to abort.
|
||
pub async fn forward_local(want: u16, host: String, port: u16) -> Result<(u16, JoinHandle<()>)> {
|
||
let l = match TcpListener::bind(("127.0.0.1", want)).await {
|
||
Ok(l) => l,
|
||
Err(_) => TcpListener::bind(("127.0.0.1", 0)).await?,
|
||
};
|
||
let local = l.local_addr()?.port();
|
||
let h = tokio::spawn(async move {
|
||
loop {
|
||
let Ok((mut inbound, _)) = l.accept().await else { break };
|
||
let host = host.clone();
|
||
tokio::spawn(async move {
|
||
if let Ok(mut out) = TcpStream::connect((host.as_str(), port)).await {
|
||
let _ = tokio::io::copy_bidirectional(&mut inbound, &mut out).await;
|
||
let _ = out.shutdown().await;
|
||
}
|
||
});
|
||
}
|
||
});
|
||
Ok((local, h))
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn setup_is_parsed() {
|
||
let r = parse_setup(r#"{"install":"npm ci","start":"npm start","url":"http://localhost:8080/app","image":"node:20","env":{"A":"1"},"services":{"db":{"image":"postgres:16","env":{"POSTGRES_PASSWORD":"x"}}}}"#).unwrap();
|
||
assert_eq!((r.port, r.image.as_str(), r.install.as_str()), (8080, "node:20", "npm ci"));
|
||
assert_eq!(r.services["db"].image, "postgres:16");
|
||
assert!(parse_setup(r#"{"url":"http://localhost:1"}"#).is_none());
|
||
assert!(parse_setup(r#"{"start":"x"}"#).is_none());
|
||
assert_eq!(parse_setup(r#"{"start":"x","port":3000}"#).unwrap().port, 3000);
|
||
}
|
||
|
||
#[test]
|
||
fn ports_of_urls() {
|
||
assert_eq!(port_of("http://localhost:8080"), Some(8080));
|
||
assert_eq!(port_of("http://localhost"), Some(80));
|
||
assert_eq!(port_of("https://x/y"), Some(443));
|
||
assert_eq!(port_of("nonsense"), None);
|
||
assert_eq!(port_of("http://localhost:8080 (web build: trunk serve)"), Some(8080));
|
||
assert!(parse_setup(r#"{"start":"x","port":1,"isolated":false}"#).is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn requests_are_checked() {
|
||
let dir = std::env::temp_dir().join(format!("ccr-{}", std::process::id()));
|
||
let work = dir.join("a/b");
|
||
std::fs::create_dir_all(&work).unwrap();
|
||
let ok = StartReq { work: work.display().to_string(), start: "x".into(), port: 80, ..Default::default() };
|
||
assert!(validate(&ok, &dir).is_ok());
|
||
// outside of the factory area, bad image, bad env name, dockerfile escaping the source
|
||
assert!(validate(&StartReq { work: "/etc".into(), ..ok.clone() }, &dir).is_err());
|
||
assert!(validate(&StartReq { image: "x;rm".into(), ..ok.clone() }, &dir).is_err());
|
||
assert!(validate(&StartReq { image: "--privileged".into(), ..ok.clone() }, &dir).is_err());
|
||
assert!(validate(&StartReq { env: [("A B".to_string(), "1".to_string())].into(), ..ok.clone() }, &dir).is_err());
|
||
assert!(validate(&StartReq { dockerfile: "../../etc/passwd".into(), ..ok.clone() }, &dir).is_err());
|
||
assert!(validate(&StartReq { work: format!("{}/../../..", work.display()), ..ok.clone() }, &dir).is_err());
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
#[test]
|
||
fn cache_config_and_seed_key() {
|
||
let r = parse_setup(r#"{"start":"x","port":1,"cache":{"dirs":["build/downloads/"],"seed":{"paths":["data/ubuntu.ext4"],"key":["guest","package-lock.json"]}}}"#).unwrap();
|
||
assert_eq!(r.cache.dirs, vec!["build/downloads"]);
|
||
assert_eq!(r.cache.seed_key, vec!["guest", "package-lock.json"]);
|
||
assert!(parse_setup(r#"{"start":"x","port":1,"cache":false}"#).unwrap().cache.disabled);
|
||
assert!(!parse_setup(r#"{"start":"x","port":1}"#).unwrap().cache.disabled);
|
||
assert!(rel_ok("build/downloads") && !rel_ok("../x") && !rel_ok("/etc") && !rel_ok("a b") && !rel_ok("a/.git/b") && !rel_ok("a//b"));
|
||
// The key follows the contents of the key files, not of anything else.
|
||
let dir = std::env::temp_dir().join(format!("ccs-{}", std::process::id()));
|
||
std::fs::create_dir_all(dir.join("guest")).unwrap();
|
||
std::fs::write(dir.join("guest/a.c"), "one").unwrap();
|
||
std::fs::write(dir.join("other.txt"), "x").unwrap();
|
||
let mut req = r.clone();
|
||
req.install = "npm ci".into();
|
||
let k1 = seed_key(&dir, "node:24", &req).unwrap();
|
||
std::fs::write(dir.join("other.txt"), "changed").unwrap();
|
||
assert_eq!(seed_key(&dir, "node:24", &req).unwrap(), k1);
|
||
std::fs::write(dir.join("guest/a.c"), "two").unwrap();
|
||
assert_ne!(seed_key(&dir, "node:24", &req).unwrap(), k1);
|
||
assert_ne!(seed_key(&dir, "node:22", &req).unwrap(), seed_key(&dir, "node:24", &req).unwrap());
|
||
req.cache.seed_paths.clear();
|
||
assert!(seed_key(&dir, "node:24", &req).is_none());
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
#[test]
|
||
fn tokens_compare() {
|
||
assert!(eq_const("abc", "abc"));
|
||
assert!(!eq_const("abc", "abd"));
|
||
assert!(!eq_const("abc", "ab"));
|
||
}
|
||
}
|