//! Fetch a URL and return readable text. use super::{Tool, ToolCtx, ToolOutput, arg_str, truncate}; use anyhow::{Result, bail}; use async_trait::async_trait; use serde_json::{Value, json}; use std::time::Duration; const MAX_BYTES: usize = 5 * 1024 * 1024; const MAX_OUTPUT: usize = 80_000; pub struct WebFetch; #[async_trait] impl Tool for WebFetch { fn name(&self) -> &str { "web_fetch" } fn description(&self) -> String { "Fetch a URL (http/https). HTML is converted to readable text; JSON and plain text are \ returned as-is. Set raw=true to get the unconverted body (e.g. HTML source)." .into() } fn schema(&self) -> Value { json!({ "type": "object", "properties": { "url": {"type": "string"}, "raw": {"type": "boolean"} }, "required": ["url"] }) } fn read_only(&self) -> bool { true } fn title(&self, input: &Value) -> String { format!("Fetch {}", input["url"].as_str().unwrap_or("?")) } async fn run(&self, input: Value, ctx: &ToolCtx) -> Result { let url = arg_str(&input, "url")?; if !(url.starts_with("http://") || url.starts_with("https://")) { bail!("only http(s) URLs are supported"); } let raw = input["raw"].as_bool().unwrap_or(false); if raw && ctx.agent == "craft" && ctx.clean_room { return Ok(ToolOutput::error( "Clean room mode: page source may not be read. Use browser_snapshot, or ask the user \ to switch this session to Deep mode.", )); } let resp = crate::provider::http_client() .get(url) .timeout(Duration::from_secs(30)) .send() .await?; let status = resp.status(); let ctype = resp .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) .unwrap_or("") .to_string(); let bytes = resp.bytes().await?; let bytes = &bytes[..bytes.len().min(MAX_BYTES)]; let body = String::from_utf8_lossy(bytes); let text = if !raw && ctype.contains("html") { html2text::from_read(body.as_bytes(), 120).unwrap_or_else(|_| body.to_string()) } else { body.to_string() }; let out = format!("HTTP {status} ({ctype})\n\n{}", truncate(&text, MAX_OUTPUT)); Ok(if status.is_success() { ToolOutput::text(out) } else { ToolOutput::error(out) }) } } // --------------------------------------------------------------------------- pub struct WebSearch; struct Hit { title: String, url: String, snippet: String, } fn strip_tags(s: &str) -> String { let re = regex::Regex::new(r"<[^>]+>").unwrap(); html2text::from_read(re.replace_all(s, "").as_bytes(), 10_000).unwrap_or_default().trim().replace('\n', " ") } async fn duckduckgo(q: &str, n: usize) -> Result> { let html = crate::provider::http_client() .post("https://html.duckduckgo.com/html/") .form(&[("q", q)]) .header("User-Agent", "Mozilla/5.0 (X11; Linux) CodeCraft") .timeout(Duration::from_secs(20)) .send() .await? .text() .await?; let link = regex::Regex::new(r#"(?s)]+class="result__a"[^>]+href="([^"]+)"[^>]*>(.*?)"#).unwrap(); let snip = regex::Regex::new(r#"(?s)]+class="result__snippet"[^>]*>(.*?)"#).unwrap(); let snippets: Vec = snip.captures_iter(&html).map(|c| strip_tags(&c[1])).collect(); let mut hits = Vec::new(); for (i, c) in link.captures_iter(&html).enumerate() { let mut url = c[1].replace("&", "&"); // DuckDuckGo wraps results in a redirect: //duckduckgo.com/l/?uddg=&... if let Some(enc) = url.split("uddg=").nth(1) { let enc = enc.split('&').next().unwrap_or(enc); url = urlencoding_decode(enc); } if url.contains("duckduckgo.com/y.js") { continue; // ads } hits.push(Hit { title: strip_tags(&c[2]), url, snippet: snippets.get(i).cloned().unwrap_or_default() }); if hits.len() >= n { break; } } if hits.is_empty() && html.contains("anomaly") { bail!("DuckDuckGo rejected the request (rate limit) – configure another search provider"); } Ok(hits) } fn urlencoding_decode(s: &str) -> String { let bytes = s.as_bytes(); let mut out = Vec::with_capacity(bytes.len()); let mut i = 0; while i < bytes.len() { match bytes[i] { b'%' if i + 2 < bytes.len() => { if let Ok(b) = u8::from_str_radix(&s[i + 1..i + 3], 16) { out.push(b); i += 3; continue; } out.push(b'%'); } b'+' => out.push(b' '), b => out.push(b), } i += 1; } String::from_utf8_lossy(&out).into_owned() } async fn search(cfg: &crate::config::SearchConfig, q: &str, n: usize) -> Result> { let client = crate::provider::http_client(); let key = crate::config::expand_env(&cfg.api_key); let pick = |v: &Value, title: &str, url: &str, snippet: &str| Hit { title: v[title].as_str().unwrap_or("").to_string(), url: v[url].as_str().unwrap_or("").to_string(), snippet: v[snippet].as_str().unwrap_or("").to_string(), }; Ok(match cfg.provider.as_str() { "searxng" => { let base = crate::config::expand_env(&cfg.url); let url = reqwest::Url::parse_with_params(&format!("{}/search", base.trim_end_matches('/')), &[("q", q), ("format", "json")])?; let v: Value = client.get(url).timeout(Duration::from_secs(20)).send().await?.error_for_status()?.json().await?; v["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "content")).collect() } "brave" => { let v: Value = client .get(reqwest::Url::parse_with_params("https://api.search.brave.com/res/v1/web/search", &[("q", q), ("count", &n.to_string())])?) .header("X-Subscription-Token", key) .header("Accept", "application/json") .timeout(Duration::from_secs(20)) .send() .await? .error_for_status()? .json() .await?; v["web"]["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "description")).collect() } "tavily" => { let v: Value = client .post("https://api.tavily.com/search") .json(&json!({"api_key": key, "query": q, "max_results": n})) .timeout(Duration::from_secs(30)) .send() .await? .error_for_status()? .json() .await?; v["results"].as_array().cloned().unwrap_or_default().iter().take(n).map(|r| pick(r, "title", "url", "content")).collect() } _ => duckduckgo(q, n).await?, }) } #[async_trait] impl Tool for WebSearch { fn name(&self) -> &str { "web_search" } fn description(&self) -> String { "Search the web. Returns titles, URLs and snippets; read promising results with web_fetch. \ Use for current information, documentation, error messages and libraries." .into() } fn schema(&self) -> Value { json!({"type": "object", "properties": {"query": {"type": "string"}, "max_results": {"type": "integer", "description": "1-20, default 8"}}, "required": ["query"]}) } fn read_only(&self) -> bool { true } fn title(&self, input: &Value) -> String { format!("Search: {}", input["query"].as_str().unwrap_or("")) } async fn run(&self, input: Value, ctx: &ToolCtx) -> Result { let q = arg_str(&input, "query")?; let n = input["max_results"].as_u64().unwrap_or(8).clamp(1, 20) as usize; let hits = search(&ctx.app.config().search, q, n).await?; if hits.is_empty() { return Ok(ToolOutput::text("No results.")); } let mut out = String::new(); for (i, h) in hits.iter().enumerate() { out.push_str(&format!("{}. {}\n {}\n {}\n\n", i + 1, h.title, h.url, h.snippet)); } Ok(ToolOutput::text(out)) } }