From 78c4eeaa28bc064bb9fa15556b203d8ff36dd462 Mon Sep 17 00:00:00 2001 From: Richard Zink Date: Mon, 16 Nov 2026 09:26:11 +0100 Subject: [PATCH] Factory: acceptance judges only listed scenarios, issue scope, setup.json fallback Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 12 ++++++++++++ server/src/factory.rs | 19 +++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1834ac..4e6d400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to CodeCraft. Versions follow [Semantic Versioning](https://semver.org/); until 1.0, minor versions may contain breaking changes. +## [Unreleased] + +### Fixed + +- Factory acceptance: when the built branch has no `.factory/setup.json` (it was added to the default + branch later), the one from the default branch is used. +- Factory acceptance judged things outside the issue (e.g. later milestones of a large spec) and failed + the issue for them. Now only the listed scenarios count: the verdict follows the scenario results + (one failed scenario fails, unverified or out-of-scope gaps do not), and the prompts of review, build + and acceptance restrict the work to the issue's scope. An issue can name it with a `Scope: ` + line (e.g. `Scope: M1`); without one the review picks the first milestone that is not built yet. + ## [v202611-00044] – 2026-11-16 ### Added diff --git a/server/src/factory.rs b/server/src/factory.rs index 0e17274..3ae8adf 100644 --- a/server/src/factory.rs +++ b/server/src/factory.rs @@ -626,7 +626,8 @@ pub fn review_prompt(cfg: &RepoCfg, issue: &Value, comments: &[Value], extra: &s How to work:\n\ - The issue names its spec as `Spec: ` (relative to the repository at your working directory). Read the spec and look at the existing code. Do NOT change any file.\n\ - decision `approved`: spec is clear, consistent with the code, buildable. `question`: something essential is missing – ask precise questions (max 5). `rejected`: impossible, contradictory or out of scope – say why.\n\ -- Also write acceptance scenarios (markdown, checkable by someone who sees only the running app, with exact steps and expected results). They are kept away from the builder.\n\ +- Scope: if the issue has a `Scope: ` line, only that part of the spec is to be built; judge and write scenarios for it alone. If the spec is split into milestones and the issue names no scope, the scope is the first milestone that does not exist in the code yet – say which one in `summary`. Everything outside the scope is out of scope for build and acceptance, however much the spec says about it. +- Also write acceptance scenarios (markdown, checkable by someone who sees only the running app, with exact steps and expected results), only for the scope. They are kept away from the builder.\n\ - Finish with exactly one fenced json block and nothing after it:\n\ ```json\n{{\"decision\": \"approved|question|rejected\", \"summary\": \"one or two sentences\", \"questions\": [\"…\"], \"reason\": \"…\", \"scenarios\": \"markdown\"}}\n```\n{extra}", url = cfg.url, @@ -834,6 +835,7 @@ pub fn build_prompt(cfg: &RepoCfg, issue: &Value, comments: &[Value], feedback: How to work:\n\ - The issue names its spec as `Spec: `. Read it and the existing code first, follow the conventions you find.\n\ - Do NOT edit the spec file and do NOT run `git commit`, `git push` or anything that talks to the remote – the factory commits and pushes your changes itself.\n\ +- Scope: build only what the issue's `Scope: ` line names; without one, the next milestone of the spec that does not exist in the code yet (state which in `summary`). Everything else in the spec is out of scope.\n\ - Build what is specified, nothing more. Add or update tests where the project has a test setup, then run the tests/linters/build that are available and fix what breaks.\n\ - If you cannot verify something (no browser, no network, missing tools), say so openly in `notes` instead of claiming it works.\n\ - If the request cannot be built as specified, make no changes and answer `blocked` with the reason.\n\ @@ -979,6 +981,7 @@ pub fn qa_prompt(cfg: &RepoCfg, issue: &Value, scenarios: &str, extra: &str) -> How to work:\n\ - The repository at your working directory is the built branch. Do NOT change, commit or push anything; you are the checker, not the builder.\n\ - Start the app as `.factory/setup.json` says (keys: `install` – optional setup command, `start` – command that starts the app, `url` – where it listens, e.g. http://localhost:8080). Without that file work out the start from the README and package files and say so.\n\ +- Judge ONLY the scenarios listed above. Everything else in the issue or the spec – other milestones, features no scenario mentions, things that are simply not built yet – is out of scope: do not test it, do not fail anything because of it. Mention such gaps at most in `summary`, never as `fail` and never in `findings`.\n\ - Go through every scenario with the real running app. Use the `browser` tool for anything visual or interactive, `curl` or the project's own test runner for the rest. Judge only what you saw, not what the code suggests.\n\ - Per scenario answer `pass`, `fail` (it behaves differently – say how) or `unverified` (you could not check it, e.g. a missing tool; say why). Never mark something `pass` you did not observe.\n\ - Stop everything you started (app server, background jobs) before you finish.\n\ @@ -1018,10 +1021,17 @@ async fn qa(app: Arc, user: User, cfg: RepoCfg, issue: Value) { Some(b) => b.to_string(), None => return qa_fail(&app, &g, &cfg, n, "the issue has no built branch").await, }; - let (work, _) = match prepare_worktree(&app, &user, &cfg, n, &branch, true).await { + let (work, base) = match prepare_worktree(&app, &user, &cfg, n, &branch, true).await { Ok(r) => r, Err(e) => return qa_fail(&app, &g, &cfg, n, &format!("could not check out the branch: {e:#}")).await, }; + // The branch may predate the setup file: fall back to the one on the default branch. + if !work.join(".factory/setup.json").exists() { + if let Ok(text) = git(&cfg, &work, &["show", &format!("origin/{base}:.factory/setup.json")]).await { + let _ = std::fs::create_dir_all(work.join(".factory")); + let _ = std::fs::write(work.join(".factory/setup.json"), text); + } + } let prompt = qa_prompt(&cfg, &issue, row["scenarios"].as_str().unwrap_or(""), &cfg.qa.extra_prompt); let title = format!("Factory #{n} – acceptance{} – {}", if round > 0 { format!(" (round {})", round + 1) } else { String::new() }, issue["title"].as_str().unwrap_or("")); let started = start_phase(&app, &user, &cfg, n, "qa", round, prompt, &title, Some(&work)); @@ -1052,8 +1062,9 @@ async fn qa(app: Arc, user: User, cfg: RepoCfg, issue: Value) { let cell = |k: &str| r[k].as_str().unwrap_or("").replace('|', "\\|").replace('\n', " "); table.push_str(&format!("| {} | {} | {} |\n", cell("scenario"), cell("result"), cell("evidence"))); } - // One failed scenario fails the acceptance, whatever the model's own verdict says. - if res["decision"].as_str() != Some("pass") || fail > 0 { + // The scenarios decide, not the model's overall verdict (which tends to count out-of-scope gaps). + let failed = if results.is_empty() { res["decision"].as_str() != Some("pass") } else { fail > 0 }; + if failed { let findings = res["findings"].as_str().filter(|f| !f.trim().is_empty()).unwrap_or(&summary).trim().to_string(); if round >= MAX_REWORK { set(&[("state", json!("failed")), ("note", json!(findings.clone()))]);