Files
codecraft/server/src/skills_api.rs
T

327 lines
13 KiB
Rust

//! The user's own skills: a folder per skill with `SKILL.md` and further text files, kept in
//! `<data>/user-skills/<user id>/<name>/`; created in the UI or imported from a GitHub repository.
use crate::api::{ApiError, ApiResult, AppState};
use crate::prompts::{frontmatter, skill_files, user_skills_dir};
use crate::store::User;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::{Extension, Json};
use serde::Deserialize;
use serde_json::{Value, json};
use std::path::PathBuf;
const MAX_FILES: usize = 100;
const MAX_FILE: usize = 200_000;
const MAX_TOTAL: usize = 2_000_000;
fn bad(m: impl Into<String>) -> ApiError {
ApiError(StatusCode::BAD_REQUEST, m.into())
}
/// A skill name is a folder name: lowercase letters, digits, `-` and `_`.
pub fn valid_name(n: &str) -> bool {
!n.is_empty() && n.len() <= 64 && n.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') && !n.starts_with(['-', '_'])
}
/// Turn any title into a valid skill name.
pub fn slug(s: &str) -> String {
let mut out = String::new();
for c in s.to_lowercase().chars() {
if c.is_ascii_alphanumeric() {
out.push(c);
} else if !out.ends_with('-') && !out.is_empty() {
out.push('-');
}
}
out.trim_matches('-').chars().take(64).collect()
}
fn valid_rel(p: &str) -> bool {
let rel = std::path::Path::new(p);
!p.is_empty() && p.len() <= 200 && !rel.is_absolute() && rel.components().all(|c| matches!(c, std::path::Component::Normal(_)))
}
fn dir_of(app: &AppState, user: &User, name: &str) -> Result<PathBuf, ApiError> {
if !valid_name(name) {
return Err(bad("invalid skill name (lowercase letters, digits, - and _)"));
}
Ok(user_skills_dir(app, &user.id).join(name))
}
#[derive(Deserialize, Clone)]
pub struct FileBody {
path: String,
content: String,
}
/// Replace the files of a skill by `files` (must contain `SKILL.md`).
fn write_skill(dir: &std::path::Path, files: &[FileBody]) -> Result<(), ApiError> {
if !files.iter().any(|f| f.path == "SKILL.md" && !f.content.trim().is_empty()) {
return Err(bad("SKILL.md is required"));
}
if files.len() > MAX_FILES {
return Err(bad(format!("at most {MAX_FILES} files")));
}
let mut total = 0;
for f in files {
if !valid_rel(&f.path) {
return Err(bad(format!("invalid file path '{}'", f.path)));
}
if f.content.len() > MAX_FILE {
return Err(bad(format!("'{}' is too large (max {} KB)", f.path, MAX_FILE / 1000)));
}
total += f.content.len();
}
if total > MAX_TOTAL {
return Err(bad("the skill is too large (max 2 MB)"));
}
let tmp = dir.with_extension("saving");
let _ = std::fs::remove_dir_all(&tmp);
for f in files {
let p = tmp.join(&f.path);
if let Some(parent) = p.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(&p, &f.content)?;
}
let _ = std::fs::remove_dir_all(dir);
std::fs::rename(&tmp, dir)?;
Ok(())
}
pub async fn list(Extension(user): Extension<User>, State(app): State<AppState>) -> ApiResult {
let base = user_skills_dir(&app, &user.id);
let mut out = Vec::new();
for e in std::fs::read_dir(&base).into_iter().flatten().flatten() {
let Ok(text) = std::fs::read_to_string(e.path().join("SKILL.md")) else { continue };
let (meta, _) = frontmatter(&text);
out.push(json!({
"name": e.file_name().to_string_lossy(),
"description": meta.get("description").cloned().unwrap_or_default(),
"files": skill_files(&e.path()),
}));
}
out.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str()));
Ok(Json(json!({"skills": out})))
}
pub async fn get(Extension(user): Extension<User>, State(app): State<AppState>, Path(name): Path<String>) -> ApiResult {
let dir = dir_of(&app, &user, &name)?;
let main = std::fs::read_to_string(dir.join("SKILL.md")).map_err(|_| ApiError(StatusCode::NOT_FOUND, "skill not found".into()))?;
let mut files = vec![json!({"path": "SKILL.md", "content": main})];
for f in skill_files(&dir) {
let content = std::fs::read_to_string(dir.join(&f)).unwrap_or_default();
files.push(json!({"path": f, "content": content}));
}
Ok(Json(json!({"name": name, "files": files})))
}
#[derive(Deserialize)]
pub struct PutBody {
files: Vec<FileBody>,
}
pub async fn put(Extension(user): Extension<User>, State(app): State<AppState>, Path(name): Path<String>, Json(b): Json<PutBody>) -> ApiResult {
let dir = dir_of(&app, &user, &name)?;
std::fs::create_dir_all(user_skills_dir(&app, &user.id))?;
write_skill(&dir, &b.files)?;
app.audit(Some(&user), "skill.save", name, "");
Ok(Json(json!({"ok": true})))
}
pub async fn delete(Extension(user): Extension<User>, State(app): State<AppState>, Path(name): Path<String>) -> ApiResult {
let dir = dir_of(&app, &user, &name)?;
if !dir.join("SKILL.md").exists() {
return Err(ApiError(StatusCode::NOT_FOUND, "skill not found".into()));
}
std::fs::remove_dir_all(&dir)?;
app.audit(Some(&user), "skill.delete", name, "");
Ok(Json(json!({"ok": true})))
}
// --- GitHub import -------------------------------------------------------------------------
struct Source {
owner: String,
repo: String,
branch: Option<String>,
sub: String,
}
/// `https://github.com/<owner>/<repo>[/tree|blob/<branch>[/<path>]]`
fn parse_github(url: &str) -> Result<Source, ApiError> {
let rest = url.trim().strip_prefix("https://github.com/").ok_or_else(|| bad("only https://github.com/… URLs are supported"))?;
let rest = rest.split(['?', '#']).next().unwrap_or("").trim_end_matches('/');
let mut it = rest.split('/');
let (Some(owner), Some(repo)) = (it.next().filter(|s| !s.is_empty()), it.next().map(|r| r.trim_end_matches(".git")).filter(|s| !s.is_empty())) else {
return Err(bad("not a repository URL"));
};
let (mut branch, mut sub) = (None, String::new());
if matches!(it.next(), Some("tree") | Some("blob")) {
branch = it.next().map(str::to_string);
sub = it.collect::<Vec<_>>().join("/");
if sub.ends_with("SKILL.md") {
sub = sub.trim_end_matches("SKILL.md").trim_end_matches('/').to_string();
}
}
let ok = |s: &str| s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
if !ok(owner) || !ok(repo) {
return Err(bad("not a repository URL"));
}
Ok(Source { owner: owner.into(), repo: repo.into(), branch, sub })
}
fn client() -> Result<reqwest::Client, ApiError> {
reqwest::Client::builder().timeout(std::time::Duration::from_secs(30)).user_agent("codecraft-skill-import").build().map_err(|e| bad(e.to_string()))
}
/// Blobs (path, size) of the repository below `src.sub`, and the resolved branch.
async fn tree(c: &reqwest::Client, src: &Source) -> Result<(String, Vec<(String, u64)>), ApiError> {
let branch = match &src.branch {
Some(b) => b.clone(),
None => {
let v: Value = c.get(format!("https://api.github.com/repos/{}/{}", src.owner, src.repo)).send().await.map_err(|e| bad(e.to_string()))?.json().await.map_err(|e| bad(e.to_string()))?;
v["default_branch"].as_str().ok_or_else(|| bad("repository not found (is it public?)"))?.to_string()
}
};
let r = c.get(format!("https://api.github.com/repos/{}/{}/git/trees/{branch}?recursive=1", src.owner, src.repo)).send().await.map_err(|e| bad(e.to_string()))?;
if !r.status().is_success() {
return Err(bad(format!("GitHub answered {} (repository or branch not found, or rate limit)", r.status())));
}
let v: Value = r.json().await.map_err(|e| bad(e.to_string()))?;
let prefix = if src.sub.is_empty() { String::new() } else { format!("{}/", src.sub) };
let blobs = v["tree"]
.as_array()
.into_iter()
.flatten()
.filter(|e| e["type"] == "blob")
.filter_map(|e| Some((e["path"].as_str()?.to_string(), e["size"].as_u64().unwrap_or(0))))
.filter(|(p, _)| prefix.is_empty() || p.starts_with(&prefix))
.collect();
Ok((branch, blobs))
}
/// Folders that contain a `SKILL.md` (path relative to the repository root; "" = the root itself).
fn skill_dirs(blobs: &[(String, u64)]) -> Vec<String> {
let mut v: Vec<String> = blobs
.iter()
.filter_map(|(p, _)| if p == "SKILL.md" { Some(String::new()) } else { p.strip_suffix("/SKILL.md").map(str::to_string) })
.collect();
v.sort();
v.truncate(1000);
v
}
#[derive(Deserialize)]
pub struct ScanBody {
url: String,
}
/// List the skills found at a GitHub URL (nothing is imported).
pub async fn scan(Extension(_user): Extension<User>, Json(b): Json<ScanBody>) -> ApiResult {
let src = parse_github(&b.url)?;
let c = client()?;
let (branch, blobs) = tree(&c, &src).await?;
let dirs = skill_dirs(&blobs);
let list: Vec<Value> = dirs
.iter()
.map(|d| {
let name = d.rsplit('/').next().filter(|n| !n.is_empty()).unwrap_or(&src.repo);
let n = blobs.iter().filter(|(p, _)| d.is_empty() || p.starts_with(&format!("{d}/"))).count();
json!({"path": d, "name": slug(name), "files": n})
})
.collect();
Ok(Json(json!({"branch": branch, "skills": list})))
}
#[derive(Deserialize)]
pub struct ImportBody {
url: String,
/// Folders (as listed by `scan`).
paths: Vec<String>,
}
const BINARY: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "ico", "pdf", "zip", "gz", "tar", "woff", "woff2", "ttf", "otf", "mp4", "mov", "mp3", "wav", "pyc", "so", "bin", "exe"];
/// Fetch the chosen skills of a GitHub repository and save them as the user's own skills.
pub async fn import(Extension(user): Extension<User>, State(app): State<AppState>, Json(b): Json<ImportBody>) -> ApiResult {
if b.paths.is_empty() || b.paths.len() > 100 {
return Err(bad("choose between 1 and 100 skills"));
}
let src = parse_github(&b.url)?;
let c = client()?;
let (branch, blobs) = tree(&c, &src).await?;
let known = skill_dirs(&blobs);
std::fs::create_dir_all(user_skills_dir(&app, &user.id))?;
let (mut saved, mut skipped) = (Vec::new(), Vec::new());
for d in &b.paths {
if !known.contains(d) {
skipped.push(json!({"path": d, "why": "no SKILL.md there"}));
continue;
}
let name = slug(d.rsplit('/').next().filter(|n| !n.is_empty()).unwrap_or(&src.repo));
if !valid_name(&name) {
skipped.push(json!({"path": d, "why": "no usable name"}));
continue;
}
let prefix = if d.is_empty() { String::new() } else { format!("{d}/") };
let mut files = Vec::new();
let mut dropped = 0;
for (p, size) in blobs.iter().filter(|(p, _)| p.starts_with(&prefix)) {
let rel = &p[prefix.len()..];
let ext = rel.rsplit('.').next().unwrap_or("").to_lowercase();
if rel.is_empty() || !valid_rel(rel) || *size as usize > MAX_FILE || BINARY.contains(&ext.as_str()) || files.len() >= MAX_FILES || rel.starts_with(".git") {
dropped += 1;
continue;
}
// A skill at the repository root: only Markdown files, not the whole repository.
if d.is_empty() && rel != "SKILL.md" && !rel.ends_with(".md") {
dropped += 1;
continue;
}
let url = format!("https://raw.githubusercontent.com/{}/{}/{branch}/{p}", src.owner, src.repo);
let Ok(r) = c.get(&url).send().await else { dropped += 1; continue };
if !r.status().is_success() {
dropped += 1;
continue;
}
match r.text().await {
Ok(content) => files.push(FileBody { path: rel.to_string(), content }),
Err(_) => dropped += 1,
}
}
match write_skill(&user_skills_dir(&app, &user.id).join(&name), &files) {
Ok(()) => saved.push(json!({"name": name, "files": files.len(), "dropped": dropped})),
Err(e) => skipped.push(json!({"path": d, "why": e.1})),
}
}
app.audit(Some(&user), "skill.import", format!("{}/{}: {} skills", src.owner, src.repo, saved.len()), "");
Ok(Json(json!({"saved": saved, "skipped": skipped})))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names_and_urls() {
assert!(valid_name("frontend-design"));
assert!(!valid_name("../x") && !valid_name("Front") && !valid_name(""));
assert_eq!(slug("Frontend Design!"), "frontend-design");
let s = parse_github("https://github.com/o/r/blob/main/.claude/skills/frontend/SKILL.md").ok().unwrap();
assert_eq!((s.owner.as_str(), s.repo.as_str(), s.branch.as_deref(), s.sub.as_str()), ("o", "r", Some("main"), ".claude/skills/frontend"));
let s = parse_github("https://github.com/o/r.git").ok().unwrap();
assert!(s.branch.is_none() && s.sub.is_empty());
assert!(parse_github("https://example.org/o/r").is_err());
}
#[test]
fn finds_skill_folders() {
let b = |p: &str| (p.to_string(), 10u64);
let dirs = skill_dirs(&[b("a/SKILL.md"), b("a/ref.md"), b("x/y/SKILL.md"), b("README.md")]);
assert_eq!(dirs, vec!["a".to_string(), "x/y".to_string()]);
assert!(valid_rel("ref/a.md") && !valid_rel("../a") && !valid_rel("/etc/passwd"));
}
}