Files
codecraft/server/src/tools/web.rs
T
Richard ZinkandClaude Opus 5.5 5b569b6835 Add security hardening, Ghidra, recreate skill, checkpoints, shells, web search and themes
- Sandbox (bubblewrap) for non-admin shell/git, MCP per-user rights, hard budgets and quotas,
  login rate limiting, audit log and backups
- Ghidra headless decompilation (ARM64 natives built from source), binary_info tool,
  clean room / deep workflow per session
- Recreate skill and commands for analysing and re-building old software
- File checkpoints with rewind/restore, background shells (bash_output, kill_shell)
- web_search tool (DuckDuckGo, SearXNG, Brave, Tavily)
- In-app dialogs and toasts instead of browser dialogs
- 15 base themes (trilium-themes palettes plus Nord, VS Code, GitHub, Catppuccin, One,
  Dracula, Rose Pine, Everforest, Kanagawa) with light/dark/auto switcher and license notes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 19:12:30 +01:00

226 lines
8.4 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Fetch a URL and return readable text.
use super::{Tool, ToolCtx, ToolOutput, arg_str, truncate};
use anyhow::{Result, bail};
use async_trait::async_trait;
use serde_json::{Value, json};
use std::time::Duration;
const MAX_BYTES: usize = 5 * 1024 * 1024;
const MAX_OUTPUT: usize = 80_000;
pub struct WebFetch;
#[async_trait]
impl Tool for WebFetch {
fn name(&self) -> &str {
"web_fetch"
}
fn description(&self) -> String {
"Fetch a URL (http/https). HTML is converted to readable text; JSON and plain text are \
returned as-is. Set raw=true to get the unconverted body (e.g. HTML source)."
.into()
}
fn schema(&self) -> Value {
json!({
"type": "object",
"properties": {
"url": {"type": "string"},
"raw": {"type": "boolean"}
},
"required": ["url"]
})
}
fn read_only(&self) -> bool {
true
}
fn title(&self, input: &Value) -> String {
format!("Fetch {}", input["url"].as_str().unwrap_or("?"))
}
async fn run(&self, input: Value, ctx: &ToolCtx) -> Result<ToolOutput> {
let url = arg_str(&input, "url")?;
if !(url.starts_with("http://") || url.starts_with("https://")) {
bail!("only http(s) URLs are supported");
}
let raw = input["raw"].as_bool().unwrap_or(false);
if raw && ctx.agent == "craft" && ctx.clean_room {
return Ok(ToolOutput::error(
"Clean room mode: page source may not be read. Use browser_snapshot, or ask the user \
to switch this session to Deep mode.",
));
}
let resp = crate::provider::http_client()
.get(url)
.timeout(Duration::from_secs(30))
.send()
.await?;
let status = resp.status();
let ctype = resp
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let bytes = resp.bytes().await?;
let bytes = &bytes[..bytes.len().min(MAX_BYTES)];
let body = String::from_utf8_lossy(bytes);
let text = if !raw && ctype.contains("html") {
html2text::from_read(body.as_bytes(), 120).unwrap_or_else(|_| body.to_string())
} else {
body.to_string()
};
let out = format!("HTTP {status} ({ctype})\n\n{}", truncate(&text, MAX_OUTPUT));
Ok(if status.is_success() { ToolOutput::text(out) } else { ToolOutput::error(out) })
}
}
// ---------------------------------------------------------------------------
pub struct WebSearch;
struct Hit {
title: String,
url: String,
snippet: String,
}
fn strip_tags(s: &str) -> String {
let re = regex::Regex::new(r"<[^>]+>").unwrap();
html2text::from_read(re.replace_all(s, "").as_bytes(), 10_000).unwrap_or_default().trim().replace('\n', " ")
}
async fn duckduckgo(q: &str, n: usize) -> Result<Vec<Hit>> {
let html = crate::provider::http_client()
.post("https://html.duckduckgo.com/html/")
.form(&[("q", q)])
.header("User-Agent", "Mozilla/5.0 (X11; Linux) CodeCraft")
.timeout(Duration::from_secs(20))
.send()
.await?
.text()
.await?;
let link = regex::Regex::new(r#"(?s)<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>(.*?)</a>"#).unwrap();
let snip = regex::Regex::new(r#"(?s)<a[^>]+class="result__snippet"[^>]*>(.*?)</a>"#).unwrap();
let snippets: Vec<String> = snip.captures_iter(&html).map(|c| strip_tags(&c[1])).collect();
let mut hits = Vec::new();
for (i, c) in link.captures_iter(&html).enumerate() {
let mut url = c[1].replace("&amp;", "&");
// DuckDuckGo wraps results in a redirect: //duckduckgo.com/l/?uddg=<encoded>&...
if let Some(enc) = url.split("uddg=").nth(1) {
let enc = enc.split('&').next().unwrap_or(enc);
url = urlencoding_decode(enc);
}
if url.contains("duckduckgo.com/y.js") {
continue; // ads
}
hits.push(Hit { title: strip_tags(&c[2]), url, snippet: snippets.get(i).cloned().unwrap_or_default() });
if hits.len() >= n {
break;
}
}
if hits.is_empty() && html.contains("anomaly") {
bail!("DuckDuckGo rejected the request (rate limit) – configure another search provider");
}
Ok(hits)
}
fn urlencoding_decode(s: &str) -> String {
let bytes = s.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
match bytes[i] {
b'%' if i + 2 < bytes.len() => {
if let Ok(b) = u8::from_str_radix(&s[i + 1..i + 3], 16) {
out.push(b);
i += 3;
continue;
}
out.push(b'%');
}
b'+' => out.push(b' '),
b => out.push(b),
}
i += 1;
}
String::from_utf8_lossy(&out).into_owned()
}
async fn search(cfg: &crate::config::SearchConfig, q: &str, n: usize) -> Result<Vec<Hit>> {
let client = crate::provider::http_client();
let key = crate::config::expand_env(&cfg.api_key);
let pick = |v: &Value, title: &str, url: &str, snippet: &str| Hit {
title: v[title].as_str().unwrap_or("").to_string(),
url: v[url].as_str().unwrap_or("").to_string(),
snippet: v[snippet].as_str().unwrap_or("").to_string(),
};
Ok(match cfg.provider.as_str() {
"searxng" => {
let base = crate::config::expand_env(&cfg.url);
let url = reqwest::Url::parse_with_params(&format!("{}/search", base.trim_end_matches('/')), &[("q", q), ("format", "json")])?;
let v: Value = client.get(url).timeout(Duration::from_secs(20)).send().await?.error_for_status()?.json().await?;
v["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "content")).collect()
}
"brave" => {
let v: Value = client
.get(reqwest::Url::parse_with_params("https://api.search.brave.com/res/v1/web/search", &[("q", q), ("count", &n.to_string())])?)
.header("X-Subscription-Token", key)
.header("Accept", "application/json")
.timeout(Duration::from_secs(20))
.send()
.await?
.error_for_status()?
.json()
.await?;
v["web"]["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "description")).collect()
}
"tavily" => {
let v: Value = client
.post("https://api.tavily.com/search")
.json(&json!({"api_key": key, "query": q, "max_results": n}))
.timeout(Duration::from_secs(30))
.send()
.await?
.error_for_status()?
.json()
.await?;
v["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "content")).collect()
}
_ => duckduckgo(q, n).await?,
})
}
#[async_trait]
impl Tool for WebSearch {
fn name(&self) -> &str {
"web_search"
}
fn description(&self) -> String {
"Search the web. Returns titles, URLs and snippets; read promising results with web_fetch. \
Use for current information, documentation, error messages and libraries."
.into()
}
fn schema(&self) -> Value {
json!({"type": "object", "properties": {"query": {"type": "string"}, "max_results": {"type": "integer", "description": "1-20, default 8"}}, "required": ["query"]})
}
fn read_only(&self) -> bool {
true
}
fn title(&self, input: &Value) -> String {
format!("Search: {}", input["query"].as_str().unwrap_or(""))
}
async fn run(&self, input: Value, ctx: &ToolCtx) -> Result<ToolOutput> {
let q = arg_str(&input, "query")?;
let n = input["max_results"].as_u64().unwrap_or(8).clamp(1, 20) as usize;
let hits = search(&ctx.app.config().search, q, n).await?;
if hits.is_empty() {
return Ok(ToolOutput::text("No results."));
}
let mut out = String::new();
for (i, h) in hits.iter().enumerate() {
out.push_str(&format!("{}. {}\n {}\n {}\n\n", i + 1, h.title, h.url, h.snippet));
}
Ok(ToolOutput::text(out))
}
}