- Sandbox (bubblewrap) for non-admin shell/git, MCP per-user rights, hard budgets and quotas, login rate limiting, audit log and backups - Ghidra headless decompilation (ARM64 natives built from source), binary_info tool, clean room / deep workflow per session - Recreate skill and commands for analysing and re-building old software - File checkpoints with rewind/restore, background shells (bash_output, kill_shell) - web_search tool (DuckDuckGo, SearXNG, Brave, Tavily) - In-app dialogs and toasts instead of browser dialogs - 15 base themes (trilium-themes palettes plus Nord, VS Code, GitHub, Catppuccin, One, Dracula, Rose Pine, Everforest, Kanagawa) with light/dark/auto switcher and license notes Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
226 lines
8.4 KiB
Rust
226 lines
8.4 KiB
Rust
//! Fetch a URL and return readable text.
|
||
|
||
use super::{Tool, ToolCtx, ToolOutput, arg_str, truncate};
|
||
use anyhow::{Result, bail};
|
||
use async_trait::async_trait;
|
||
use serde_json::{Value, json};
|
||
use std::time::Duration;
|
||
|
||
const MAX_BYTES: usize = 5 * 1024 * 1024;
|
||
const MAX_OUTPUT: usize = 80_000;
|
||
|
||
pub struct WebFetch;
|
||
|
||
#[async_trait]
|
||
impl Tool for WebFetch {
|
||
fn name(&self) -> &str {
|
||
"web_fetch"
|
||
}
|
||
fn description(&self) -> String {
|
||
"Fetch a URL (http/https). HTML is converted to readable text; JSON and plain text are \
|
||
returned as-is. Set raw=true to get the unconverted body (e.g. HTML source)."
|
||
.into()
|
||
}
|
||
fn schema(&self) -> Value {
|
||
json!({
|
||
"type": "object",
|
||
"properties": {
|
||
"url": {"type": "string"},
|
||
"raw": {"type": "boolean"}
|
||
},
|
||
"required": ["url"]
|
||
})
|
||
}
|
||
fn read_only(&self) -> bool {
|
||
true
|
||
}
|
||
fn title(&self, input: &Value) -> String {
|
||
format!("Fetch {}", input["url"].as_str().unwrap_or("?"))
|
||
}
|
||
async fn run(&self, input: Value, ctx: &ToolCtx) -> Result<ToolOutput> {
|
||
let url = arg_str(&input, "url")?;
|
||
if !(url.starts_with("http://") || url.starts_with("https://")) {
|
||
bail!("only http(s) URLs are supported");
|
||
}
|
||
let raw = input["raw"].as_bool().unwrap_or(false);
|
||
if raw && ctx.agent == "craft" && ctx.clean_room {
|
||
return Ok(ToolOutput::error(
|
||
"Clean room mode: page source may not be read. Use browser_snapshot, or ask the user \
|
||
to switch this session to Deep mode.",
|
||
));
|
||
}
|
||
let resp = crate::provider::http_client()
|
||
.get(url)
|
||
.timeout(Duration::from_secs(30))
|
||
.send()
|
||
.await?;
|
||
let status = resp.status();
|
||
let ctype = resp
|
||
.headers()
|
||
.get(reqwest::header::CONTENT_TYPE)
|
||
.and_then(|v| v.to_str().ok())
|
||
.unwrap_or("")
|
||
.to_string();
|
||
let bytes = resp.bytes().await?;
|
||
let bytes = &bytes[..bytes.len().min(MAX_BYTES)];
|
||
let body = String::from_utf8_lossy(bytes);
|
||
let text = if !raw && ctype.contains("html") {
|
||
html2text::from_read(body.as_bytes(), 120).unwrap_or_else(|_| body.to_string())
|
||
} else {
|
||
body.to_string()
|
||
};
|
||
let out = format!("HTTP {status} ({ctype})\n\n{}", truncate(&text, MAX_OUTPUT));
|
||
Ok(if status.is_success() { ToolOutput::text(out) } else { ToolOutput::error(out) })
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
|
||
pub struct WebSearch;
|
||
|
||
struct Hit {
|
||
title: String,
|
||
url: String,
|
||
snippet: String,
|
||
}
|
||
|
||
fn strip_tags(s: &str) -> String {
|
||
let re = regex::Regex::new(r"<[^>]+>").unwrap();
|
||
html2text::from_read(re.replace_all(s, "").as_bytes(), 10_000).unwrap_or_default().trim().replace('\n', " ")
|
||
}
|
||
|
||
async fn duckduckgo(q: &str, n: usize) -> Result<Vec<Hit>> {
|
||
let html = crate::provider::http_client()
|
||
.post("https://html.duckduckgo.com/html/")
|
||
.form(&[("q", q)])
|
||
.header("User-Agent", "Mozilla/5.0 (X11; Linux) CodeCraft")
|
||
.timeout(Duration::from_secs(20))
|
||
.send()
|
||
.await?
|
||
.text()
|
||
.await?;
|
||
let link = regex::Regex::new(r#"(?s)<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>(.*?)</a>"#).unwrap();
|
||
let snip = regex::Regex::new(r#"(?s)<a[^>]+class="result__snippet"[^>]*>(.*?)</a>"#).unwrap();
|
||
let snippets: Vec<String> = snip.captures_iter(&html).map(|c| strip_tags(&c[1])).collect();
|
||
let mut hits = Vec::new();
|
||
for (i, c) in link.captures_iter(&html).enumerate() {
|
||
let mut url = c[1].replace("&", "&");
|
||
// DuckDuckGo wraps results in a redirect: //duckduckgo.com/l/?uddg=<encoded>&...
|
||
if let Some(enc) = url.split("uddg=").nth(1) {
|
||
let enc = enc.split('&').next().unwrap_or(enc);
|
||
url = urlencoding_decode(enc);
|
||
}
|
||
if url.contains("duckduckgo.com/y.js") {
|
||
continue; // ads
|
||
}
|
||
hits.push(Hit { title: strip_tags(&c[2]), url, snippet: snippets.get(i).cloned().unwrap_or_default() });
|
||
if hits.len() >= n {
|
||
break;
|
||
}
|
||
}
|
||
if hits.is_empty() && html.contains("anomaly") {
|
||
bail!("DuckDuckGo rejected the request (rate limit) – configure another search provider");
|
||
}
|
||
Ok(hits)
|
||
}
|
||
|
||
fn urlencoding_decode(s: &str) -> String {
|
||
let bytes = s.as_bytes();
|
||
let mut out = Vec::with_capacity(bytes.len());
|
||
let mut i = 0;
|
||
while i < bytes.len() {
|
||
match bytes[i] {
|
||
b'%' if i + 2 < bytes.len() => {
|
||
if let Ok(b) = u8::from_str_radix(&s[i + 1..i + 3], 16) {
|
||
out.push(b);
|
||
i += 3;
|
||
continue;
|
||
}
|
||
out.push(b'%');
|
||
}
|
||
b'+' => out.push(b' '),
|
||
b => out.push(b),
|
||
}
|
||
i += 1;
|
||
}
|
||
String::from_utf8_lossy(&out).into_owned()
|
||
}
|
||
|
||
async fn search(cfg: &crate::config::SearchConfig, q: &str, n: usize) -> Result<Vec<Hit>> {
|
||
let client = crate::provider::http_client();
|
||
let key = crate::config::expand_env(&cfg.api_key);
|
||
let pick = |v: &Value, title: &str, url: &str, snippet: &str| Hit {
|
||
title: v[title].as_str().unwrap_or("").to_string(),
|
||
url: v[url].as_str().unwrap_or("").to_string(),
|
||
snippet: v[snippet].as_str().unwrap_or("").to_string(),
|
||
};
|
||
Ok(match cfg.provider.as_str() {
|
||
"searxng" => {
|
||
let base = crate::config::expand_env(&cfg.url);
|
||
let url = reqwest::Url::parse_with_params(&format!("{}/search", base.trim_end_matches('/')), &[("q", q), ("format", "json")])?;
|
||
let v: Value = client.get(url).timeout(Duration::from_secs(20)).send().await?.error_for_status()?.json().await?;
|
||
v["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "content")).collect()
|
||
}
|
||
"brave" => {
|
||
let v: Value = client
|
||
.get(reqwest::Url::parse_with_params("https://api.search.brave.com/res/v1/web/search", &[("q", q), ("count", &n.to_string())])?)
|
||
.header("X-Subscription-Token", key)
|
||
.header("Accept", "application/json")
|
||
.timeout(Duration::from_secs(20))
|
||
.send()
|
||
.await?
|
||
.error_for_status()?
|
||
.json()
|
||
.await?;
|
||
v["web"]["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "description")).collect()
|
||
}
|
||
"tavily" => {
|
||
let v: Value = client
|
||
.post("https://api.tavily.com/search")
|
||
.json(&json!({"api_key": key, "query": q, "max_results": n}))
|
||
.timeout(Duration::from_secs(30))
|
||
.send()
|
||
.await?
|
||
.error_for_status()?
|
||
.json()
|
||
.await?;
|
||
v["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "content")).collect()
|
||
}
|
||
_ => duckduckgo(q, n).await?,
|
||
})
|
||
}
|
||
|
||
#[async_trait]
|
||
impl Tool for WebSearch {
|
||
fn name(&self) -> &str {
|
||
"web_search"
|
||
}
|
||
fn description(&self) -> String {
|
||
"Search the web. Returns titles, URLs and snippets; read promising results with web_fetch. \
|
||
Use for current information, documentation, error messages and libraries."
|
||
.into()
|
||
}
|
||
fn schema(&self) -> Value {
|
||
json!({"type": "object", "properties": {"query": {"type": "string"}, "max_results": {"type": "integer", "description": "1-20, default 8"}}, "required": ["query"]})
|
||
}
|
||
fn read_only(&self) -> bool {
|
||
true
|
||
}
|
||
fn title(&self, input: &Value) -> String {
|
||
format!("Search: {}", input["query"].as_str().unwrap_or(""))
|
||
}
|
||
async fn run(&self, input: Value, ctx: &ToolCtx) -> Result<ToolOutput> {
|
||
let q = arg_str(&input, "query")?;
|
||
let n = input["max_results"].as_u64().unwrap_or(8).clamp(1, 20) as usize;
|
||
let hits = search(&ctx.app.config().search, q, n).await?;
|
||
if hits.is_empty() {
|
||
return Ok(ToolOutput::text("No results."));
|
||
}
|
||
let mut out = String::new();
|
||
for (i, h) in hits.iter().enumerate() {
|
||
out.push_str(&format!("{}. {}\n {}\n {}\n\n", i + 1, h.title, h.url, h.snippet));
|
||
}
|
||
Ok(ToolOutput::text(out))
|
||
}
|
||
}
|