From b309778be85893ca1a7f4fd922331afa8b4f4de8 Mon Sep 17 00:00:00 2001 From: Richard Zink Date: Wed, 18 Nov 2026 13:49:16 +0100 Subject: [PATCH] Factory recreate mode; chunked uploads with progress everywhere Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 10 ++ server/src/api.rs | 52 +++++++++- server/src/factory.rs | 45 +++++++-- web/src/api.ts | 24 +---- web/src/components/Composer.tsx | 7 +- web/src/components/Factory.tsx | 10 ++ web/src/components/FactoryDraft.tsx | 61 +++++++++++- web/src/components/Files.tsx | 6 +- web/src/components/NewSession.tsx | 15 +-- web/src/components/UploadPanel.tsx | 51 ++++++++++ web/src/components/settings/Backups.tsx | 7 +- web/src/dialogs.tsx | 2 + web/src/i18n/de.ts | 14 +++ web/src/i18n/en.ts | 14 +++ web/src/styles.css | 77 +++++++++++++++ web/src/uploads.ts | 125 ++++++++++++++++++++++++ web/src/whatsnew.ts | 4 + 17 files changed, 477 insertions(+), 47 deletions(-) create mode 100644 web/src/components/UploadPanel.tsx create mode 100644 web/src/uploads.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 90bcfe0..2891366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to CodeCraft. Versions follow [Semantic Versioning](https://semver.org/); until 1.0, minor versions may contain breaking changes. +## [Unreleased] + +### Changed + +- **All uploads are chunked and show progress**: files page, chat attachments, new-session wizard, recreate dialog and backup restore upload in 8 MB chunks (no more reverse-proxy limits) with a progress panel – one bar per file and one for the total. + +### Added + +- **Factory: recreate mode** ("Recreate software" next to "New project"): upload a program or give a source URL; an analyst chat (craft agent, deep mode, "recreate" skill) unpacks, decompiles and documents the original exhaustively, audits itself against a coverage list and writes a clean-room `spec.md` with milestones. Analysis, originals and notes stay in the private draft folder; the new repository gets only the specification, and the milestones are cut into issues afterwards. + ## [v202611-00059] – 2026-11-17 ### Changed diff --git a/server/src/api.rs b/server/src/api.rs index 00cc640..a18cc48 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -150,6 +150,8 @@ pub fn router(app: AppState) -> Router { .route("/backups", get(list_backups).post(run_backup)) .route("/backups/{name}", get(download_backup)) .route("/backups/{name}/restore", post(restore_stored_backup)) + .route("/backups/upload", put(upload_backup_chunk)) + .route("/backups/restore-uploaded", post(restore_chunked_backup)) .route("/backups/restore", post(restore_uploaded_backup).layer(axum::extract::DefaultBodyLimit::max(4 * 1024 * 1024 * 1024))) .route("/health", get(health)) .route("/tools", get(crate::reverse::status)) @@ -1964,6 +1966,43 @@ async fn restore_uploaded_backup(Extension(user): Extension, State(app): S restore_and_restart(&app, &user, body.to_vec(), &format!("upload ({} bytes)", body.len())).await } +#[derive(Deserialize)] +struct ChunkQuery { + offset: u64, +} + +fn restore_upload_path(app: &App) -> std::path::PathBuf { + app.config().data_dir().join("restore-upload.tmp") +} + +/// One chunk of a backup to restore (offset 0 starts anew); `restore-uploaded` applies the whole file. +async fn upload_backup_chunk(Extension(user): Extension, State(app): State, Query(q): Query, body: axum::body::Bytes) -> ApiResult { + require_admin(&user)?; + use tokio::io::AsyncWriteExt; + let p = restore_upload_path(&app); + if q.offset == 0 { + tokio::fs::write(&p, &body).await?; + } else { + let current = tokio::fs::metadata(&p).await.map(|m| m.len()).unwrap_or(0); + if current != q.offset { + return Err(ApiError(StatusCode::CONFLICT, format!("chunk offset {} does not match the uploaded size {current}", q.offset))); + } + let mut f = tokio::fs::OpenOptions::new().append(true).open(&p).await?; + f.write_all(&body).await?; + f.flush().await?; + } + Ok(Json(json!({"ok": true, "size": q.offset + body.len() as u64}))) +} + +async fn restore_chunked_backup(Extension(user): Extension, State(app): State) -> ApiResult { + require_admin(&user)?; + let p = restore_upload_path(&app); + let bytes = tokio::fs::read(&p).await.map_err(|_| ApiError(StatusCode::BAD_REQUEST, "no uploaded backup".into()))?; + let _ = tokio::fs::remove_file(&p).await; + let source = format!("upload ({} bytes)", bytes.len()); + restore_and_restart(&app, &user, bytes, &source).await +} + async fn download_backup(Extension(user): Extension, State(app): State, Path(name): Path) -> Result { require_admin(&user)?; if !crate::backup::list(&app).iter().any(|b| b.name == name) { @@ -2914,13 +2953,20 @@ async fn test_factory_connection(Extension(user): Extension, State(app): S struct NewDraft { #[serde(default)] model: Option, + /// "recreate": analyse existing software first (clean-room). + #[serde(default)] + mode: Option, + #[serde(default)] + source: String, + #[serde(default)] + notes: String, } /// Start a chat in which the user writes a spec with a model. async fn factory_new_draft(Extension(user): Extension, State(app): State, Json(b): Json) -> ApiResult { let s: crate::factory::FactorySettings = app.store.user_setting(&user.id, "factory")?; let model = b.model.filter(|m| !m.is_empty()).unwrap_or(s.defaults.spec_model.clone()); - let session = crate::factory::start_spec_chat(&app, &user, &model).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("{e:#}")))?; + let session = crate::factory::start_spec_chat(&app, &user, &model, (b.mode.as_deref() == Some("recreate")).then_some((b.source.as_str(), b.notes.as_str()))).map_err(|e| ApiError(StatusCode::BAD_REQUEST, format!("{e:#}")))?; Ok(Json(json!(session))) } @@ -2937,7 +2983,7 @@ async fn factory_draft(Extension(user): Extension, State(app): State = s.connections.iter().map(|c| json!({"id": c.id, "name": c.name, "kind": c.kind, "owner": c.owner})).collect(); let setup = std::fs::read_to_string(dir.join("setup.json")).ok().filter(|t| crate::factory::valid_setup(t)); - Ok(Json(json!({"spec": spec, "setup": setup, "proposal": crate::factory::proposal(&app, &id), "connections": conns}))) + Ok(Json(json!({"spec": spec, "setup": setup, "proposal": crate::factory::proposal(&app, &id), "connections": conns, "recreate": dir.join("recreate.txt").exists()}))) } #[derive(Deserialize)] @@ -2957,7 +3003,7 @@ async fn factory_create_project(Extension(user): Extension, State(app): St let spec = std::fs::read_to_string(dir.join("spec.md")).unwrap_or_default(); let mut s: crate::factory::FactorySettings = app.store.user_setting(&user.id, "factory")?; let setup = std::fs::read_to_string(dir.join("setup.json")).unwrap_or_default(); - let res = crate::factory::create_project(&app, &user, &mut s, &spec, &setup, &b.connection, &b.name, &b.slug, &b.description, &b.owner).await.map_err(|e| ApiError(StatusCode::BAD_GATEWAY, format!("{e:#}")))?; + let res = crate::factory::create_project(&app, &user, &mut s, &spec, &setup, &b.connection, &b.name, &b.slug, &b.description, &b.owner, !dir.join("recreate.txt").exists()).await.map_err(|e| ApiError(StatusCode::BAD_GATEWAY, format!("{e:#}")))?; app.store.set_user_setting(&user.id, "factory", &s)?; app.audit(Some(&user), "factory.project.create", res["repo"].as_str().unwrap_or(""), &crate::auth::client_ip(&headers)); Ok(Json(res)) diff --git a/server/src/factory.rs b/server/src/factory.rs index 87144b8..fea0b90 100644 --- a/server/src/factory.rs +++ b/server/src/factory.rs @@ -1558,19 +1558,44 @@ fn is_planning_path(factory: &std::path::Path, cwd: &std::path::Path) -> bool { cwd.strip_prefix(factory).map(|rel| rel.components().any(|c| c.as_os_str() == "_drafts" || c.as_os_str() == "_issues")).unwrap_or(false) } +/// Instructions of a "recreate" draft: analyse existing software in depth, then write a clean-room spec. +pub const RECREATE_INSTRUCTIONS: &str = r#"You are the analyst of a software factory working in RECREATE / clean-room mode. The user wants an existing program rebuilt as a new, independent implementation. You analyse the original down to the last detail and write the specification; later, other agents build the new software from that specification alone – they never see the original, your notes or any decompiled code. Answer in the user's language. + +Your working directory is private and never published. It holds `original/` (the program: files the user uploaded and/or what you download), `analysis/` (tool output, notes, scripts), `docs/` (documentation of the original) and the two deliverables `spec.md` and `setup.json`. + +Load the skill "recreate" and follow its phases – but go deeper than its minimum and work through them on your own, without waiting for the user between phases (ask_user only when something essential cannot be found out otherwise): +1. Acquire and inventory: fetch the source named in `recreate.txt` (git clone / curl into `original/`, never run installers or the program unsandboxed), unpack archives recursively, identify every file, write `docs/01-inventory.md`. +2. Analyse exhaustively: read the source code if it is available, otherwise use binary_info and decompile on EVERY executable and library. Cover entry points and main loop, every screen/dialog/command/option, all rules and algorithms with their exact constants, tables and string/message texts, data and file formats (byte layouts), configuration, persistence, error handling, timing, network protocols, third-party libraries. Keep `analysis/coverage.md`: a list of every file, function group and resource with its status (understood, partly, unknown) – and keep working until nothing important stays open or unknown. +3. Document in your own words in `docs/` (templates of the skill). +4. Write `spec.md`: the only input for the builders. Structure: Goal · Users · Behaviour per area (numbered features, each with acceptance criteria that can be checked in the running app) · Data and file formats (byte layouts are facts and may be exact) · Texts and constants that must match · Non-functional requirements · Tech stack with reasons · Out of scope · Open questions · **Milestones** (M0 project setup and test harness; then M1…Mn in build order, the smallest useful first version first; each milestone names its features; the last one is wrap-up/performance/docs). + Clean-room rules for spec.md: behaviour and facts only – no source or decompiled code, no decompiler-generated names or addresses, no pasted code snippets; describe algorithms in prose or pseudocode of your own. +5. Audit: re-read spec.md against `analysis/coverage.md` and the originals, close gaps, and check that nothing in it is copied code. Write the result to `analysis/audit.md`. +6. Keep `setup.json` like the spec writers do (install/start/url of the NEW software). Ask the user for the technology only if the recreation plan leaves it open. +7. Finish with a short summary and tell the user to review `spec.md` (and `docs/`), correct things together with you, and then say when it is done. Only then propose name and slug and end your message with exactly one fenced json block and nothing after it: +```json +{"decision": "ready", "name": "Project name", "slug": "project-name", "description": "one sentence"} +``` +The repository is then created with spec.md only; the milestones are cut into issues afterwards."#; + /// Start a spec chat with the given model. -pub fn start_spec_chat(app: &Arc, user: &User, model: &str) -> Result { +pub fn start_spec_chat(app: &Arc, user: &User, model: &str, recreate: Option<(&str, &str)>) -> Result { let id = crate::types::new_id(); let dir = draft_dir(app, user, &id)?; std::fs::create_dir_all(&dir)?; + if let Some((source, notes)) = recreate { + for d in ["original", "analysis", "docs"] { + std::fs::create_dir_all(dir.join(d))?; + } + std::fs::write(dir.join("recreate.txt"), format!("Source: {}\n\nNotes from the user:\n{}\n", Some(source.trim()).filter(|s| !s.is_empty()).unwrap_or("(none given – the files are in original/)"), notes.trim()))?; + } let model = Some(model.to_string()).filter(|m| !m.is_empty()).or_else(|| Some(app.config().server.default_model.clone()).filter(|m| !m.is_empty())).or_else(|| app.registry().first_model()).context("no model configured")?; let now = crate::types::now_ms(); let session = crate::types::Session { id, - title: "Factory – new project".into(), + title: if recreate.is_some() { "Factory – recreate" } else { "Factory – new project" }.into(), cwd: dir.display().to_string(), model, - agent: "build".into(), + agent: if recreate.is_some() { "craft" } else { "build" }.into(), permission_mode: "auto".into(), parent_id: None, owner_id: Some(user.id.clone()), @@ -1578,17 +1603,18 @@ pub fn start_spec_chat(app: &Arc, user: &User, model: &str) -> Result bool { /// Create the repository, file README and spec, register the repository and open the issue /// that starts the workflow. Safe to repeat after a failure: an empty repository is reused. -pub async fn create_project(app: &Arc, user: &User, settings: &mut FactorySettings, spec: &str, setup: &str, connection: &str, name: &str, slug: &str, description: &str, owner: &str) -> Result { +pub async fn create_project(app: &Arc, user: &User, settings: &mut FactorySettings, spec: &str, setup: &str, connection: &str, name: &str, slug: &str, description: &str, owner: &str, start: bool) -> Result { let slug = slugify(slug); if slug.len() < 2 { bail!("the repository name is too short"); @@ -1678,6 +1704,10 @@ pub async fn create_project(app: &Arc, user: &User, settings: &mut FactoryS // The issue that starts the workflow. let g = Forge::new(&resolved)?; g.ensure_labels().await?; + if !start { + // Recreate projects: the milestones are cut into issues next (issue chat), nothing is built yet. + return Ok(json!({"url": cfg.url, "repo": created["full_name"], "issue": Value::Null, "issue_url": Value::Null, "spec": spec_path, "setup": setup_filed})); + } let issue = g.create_issue(name, &format!("{}\n\nSpec: {spec_path}", description.trim()), &[cfg.label.clone()]).await?; let n = issue_no(&issue); app.store.factory_issue_add(&cfg.url, n, &user.id, name, &login)?; @@ -1808,6 +1838,7 @@ mod tests { let v = parse_result(t).unwrap(); assert_eq!((v["decision"].as_str(), v["slug"].as_str()), (Some("ready"), Some("todo"))); assert!(SPEC_INSTRUCTIONS.contains("\"decision\": \"ready\"")); + assert!(RECREATE_INSTRUCTIONS.contains("\"decision\": \"ready\"")); } #[test] diff --git a/web/src/api.ts b/web/src/api.ts index 3efd6bb..09e3ab5 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -1,4 +1,5 @@ import type { ServerEvent } from "./types"; +import { chunkedUpload } from "./uploads"; // Authentication uses an HttpOnly session cookie set by /api/auth/login. @@ -101,27 +102,10 @@ export function connectEvents(handlers: { }; } -const CHUNK = 8 * 1024 * 1024; - /** - * Upload a file in 8 MB chunks (stays below reverse proxy body limits; up to the - * server's max_upload_mb). `onProgress` gets the uploaded fraction 0..1. + * Upload a single file in chunks (see uploads.ts). `onProgress` gets the uploaded fraction 0..1. + * Without the progress panel – use `uploadBatch` for user-visible uploads. */ export async function uploadFile(path: string, file: Blob, onProgress?: (fraction: number) => void): Promise { - let offset = 0; - do { - const chunk = file.slice(offset, offset + CHUNK); - const res = await fetch(`/api/files/upload?path=${encodeURIComponent(path)}&offset=${offset}`, { method: "PUT", body: chunk, credentials: "same-origin" }); - if (!res.ok) { - let msg = `HTTP ${res.status}`; - try { - msg = (await res.json()).error ?? msg; - } catch { - if (res.status === 413) msg = "too large for the reverse proxy"; - } - throw new Error(msg); - } - offset += chunk.size; - onProgress?.(file.size ? offset / file.size : 1); - } while (offset < file.size); + await chunkedUpload(`/api/files/upload?path=${encodeURIComponent(path)}`, file, (n) => onProgress?.(file.size ? n / file.size : 1)); } diff --git a/web/src/components/Composer.tsx b/web/src/components/Composer.tsx index ef3aeb5..055faf0 100644 --- a/web/src/components/Composer.tsx +++ b/web/src/components/Composer.tsx @@ -1,7 +1,7 @@ import { TemplatePicker } from "./TemplatePicker"; import { Icon } from "./Icon"; import { formatSize, imageSrc } from "./FileFigure"; -import { uploadFile } from "../api"; +import { uploadBatch } from "../uploads"; import { useEffect, useMemo, useRef, useState } from "react"; import { useAgents, useFetch, useModels } from "../hooks"; import { useT, type Key } from "../i18n"; @@ -201,9 +201,8 @@ export function Composer({ await fetch(`/api/files/create?`, { method: "POST", credentials: "same-origin", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ path: `${session.cwd}/.codecraft/uploads`, dir: true }) }); // Keep uploads out of git status without touching the project's .gitignore. await fetch(`/api/files/upload?path=${encodeURIComponent(`${session.cwd}/.codecraft/uploads/.gitignore`)}`, { method: "PUT", body: "*\n", credentials: "same-origin" }); - await uploadFile(path, f).catch((e) => { - throw new Error(`${f.name}: ${e.message}`); - }); + const sent = await uploadBatch([{ file: f, path }]); + if (!sent.length) throw new Error(f.name); setFiles((cur) => [...cur, { name: f.name, path, size: f.size }]); } catch (e: any) { toast(e.message ?? String(e)); diff --git a/web/src/components/Factory.tsx b/web/src/components/Factory.tsx index c0b8add..3848561 100644 --- a/web/src/components/Factory.tsx +++ b/web/src/components/Factory.tsx @@ -6,6 +6,7 @@ import { choiceDialog, confirmDialog, promptDialog, toast } from "../dialogs"; import { Icon } from "./Icon"; import { Spinner } from "./Activity"; import { ModelSelect } from "./ModelSelect"; +import { RecreateDialog } from "./FactoryDraft"; import { ReleaseDialog, ReleasesPanel, type ReleaseRow } from "./FactoryRelease"; interface PrInfo { @@ -173,6 +174,7 @@ function FactoryOverview({ onSettings }: { onSettings: () => void }) { const [sel, setSel] = useState(""); const [now, setNow] = useState(Date.now()); const [busy, setBusy] = useState(false); + const [recreate, setRecreate] = useState(false); const reload = () => get("/factory/overview") @@ -267,11 +269,15 @@ function FactoryOverview({ onSettings }: { onSettings: () => void }) { if (data.repos.length === 0) return (
+ {recreate && setRecreate(false)} />}

{t("factory.empty")}

+
@@ -400,6 +406,9 @@ function FactoryOverview({ onSettings }: { onSettings: () => void }) { + @@ -449,6 +458,7 @@ function FactoryOverview({ onSettings }: { onSettings: () => void }) { !repo || r.repo === repo)} /> + {recreate && setRecreate(false)} />} {releaseRepo && setReleaseRepo("")} onStarted={() => void reload()} />}
diff --git a/web/src/components/FactoryDraft.tsx b/web/src/components/FactoryDraft.tsx index df67c73..30de751 100644 --- a/web/src/components/FactoryDraft.tsx +++ b/web/src/components/FactoryDraft.tsx @@ -3,6 +3,7 @@ import { get, post } from "../api"; import { useT } from "../i18n"; import { navigate } from "../router"; import { toast } from "../dialogs"; +import { uploadBatch } from "../uploads"; import { Icon } from "./Icon"; import { Spinner } from "./Activity"; @@ -11,6 +12,7 @@ interface Draft { setup?: string | null; proposal: { name?: string; slug?: string; description?: string } | null; connections: { id: string; name: string; kind: string; owner: string }[]; + recreate?: boolean; } export const isDraftChat = (cwd: string) => /\/_drafts\/[^/]+$/.test(cwd); @@ -88,9 +90,9 @@ function CreateProjectDialog({ sessionId, onClose }: { sessionId: string; onClos const create = async () => { setBusy(true); try { - const r = await post<{ repo: string; issue: number }>(`/factory/drafts/${sessionId}/create`, { connection, name, slug, description, owner }); + const r = await post<{ repo: string; issue: number | null }>(`/factory/drafts/${sessionId}/create`, { connection, name, slug, description, owner }); await post("/factory/poll", {}).catch(() => undefined); - toast(t("draft.created", { repo: r.repo, n: r.issue }), "success"); + toast(r.issue == null ? t("draft.createdRecreate", { repo: r.repo }) : t("draft.created", { repo: r.repo, n: r.issue }), "success"); onClose(); navigate("/factory"); } catch (e: any) { @@ -312,3 +314,58 @@ function CreateIssuesDialog({ sessionId, onClose }: { sessionId: string; onClose
); } + +/** Start a recreate draft: the original (upload and/or URL) is analysed in depth, then a clean-room spec is written. */ +export function RecreateDialog({ onClose }: { onClose: () => void }) { + const t = useT(); + const [source, setSource] = useState(""); + const [notes, setNotes] = useState(""); + const [files, setFiles] = useState([]); + const [busy, setBusy] = useState(false); + + const start = async () => { + setBusy(true); + try { + const s = await post<{ id: string; cwd: string }>("/factory/drafts", { mode: "recreate", source, notes }); + const sent = await uploadBatch(files.map((f) => ({ file: f, path: `${s.cwd}/original/${f.name}` }))); + if (sent.length < files.length) throw new Error(t("recreate.uploadFailed")); + await post(`/sessions/${s.id}/prompt`, { text: t("recreate.firstMessage") }); + onClose(); + navigate(`/s/${s.id}`); + } catch (e: any) { + toast(e.message); + setBusy(false); + } + }; + + return ( +
+
e.stopPropagation()}> +

{t("recreate.title")}

+
{t("recreate.hint")}
+
+ + +