mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-17 04:10:04 +02:00
Lot 42 : outils web de l'agent exécutés par la façade
Les outils web des tours de la façade (web_search, web_images, web_open,
web_read, web_grep) ne passent plus par /tool/exec du Go : src/webtools.rs
porte chat_internet_tools.go, chat_internet.go et chat_web_images.go. Le
cache des pages (10 min, clé MD5 des options comme le Go) est dans la
façade ; client Crawl4AI au corps près, réponses lues comme json.Unmarshal ;
résultats DuckDuckGo tirés du markdown (uddg, entités, extraits) ;
recherche d'images avec le jeton vqd et les cookies gardés par hôte ;
web_grep sur les regex du Go, erreurs au texte de regexp.
Le moteur intégré lit le HTML avec go-readability et html-to-markdown, que
rien en Rust ne reproduit au caractère près : la conversion d'une page et la
lecture des résultats restent au Go, derrière /internal/v1/web/fetch et
/internal/v1/web/search (sans état). web_screenshot reste au Go.
Le traducteur go_regex refuse la répétition d'une répétition et les comptes
au-delà de 1000, et rend littéral un « { » qui n'ouvre pas de répétition,
comme le Go (motifs retry_patterns compris).
Parité : journal de 74 appels tiré du Go (parity_web_tools_test.go →
tests/parity_web/outils.json : résultats, requêtes reçues par les faux
Crawl4AI et DuckDuckGo, pages demandées au moteur intégré), rejoué par
tests/webtools.rs ; un tour de la façade qui ouvre une page (tests/lot6.rs) ;
TestInternalMoteurWebIntegre côté Go.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016tcGfJWefW5kUGoC2wL8cv
This commit is contained in:
19 files changed
+4561
-7
No files matched your search
@@ -1322,6 +1322,7 @@ impl ChatShared {
|
||||
mcp: None,
|
||||
tasks: None,
|
||||
memory: None,
|
||||
web: Default::default(),
|
||||
go_version: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,6 +347,8 @@ pub struct ChatShared {
|
||||
/// La mémoire, quand la façade la tient (LOKI_MEM=rust) : les outils
|
||||
/// `mem_*` et `tracker` du tour s'exécutent ici (lot 40).
|
||||
pub memory: Option<Arc<crate::memory::Memory>>,
|
||||
/// Les outils web (lot 42) : cache des pages, Crawl4AI, DuckDuckGo.
|
||||
pub web: Arc<crate::webtools::Web>,
|
||||
/// La version du Go, annoncée aux serveurs MCP à une reconnexion ; lue
|
||||
/// une fois.
|
||||
pub go_version: tokio::sync::OnceCell<String>,
|
||||
@@ -698,6 +700,13 @@ impl TurnHost for GoTurnHost {
|
||||
};
|
||||
}
|
||||
}
|
||||
// Outils web (lot 42) : le cache des pages est ici.
|
||||
if let Some(result) = self.shared.web.exec(&self.shared.store, name, args).await {
|
||||
return ToolOutcome {
|
||||
result,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
// Outils task_*, quand la façade tient les tâches (lot 37).
|
||||
if crate::tasks::tools::is_tool(name)
|
||||
&& let Some(tasks) = &self.shared.tasks
|
||||
|
||||
@@ -968,8 +968,67 @@ pub fn go_regex(p: &str) -> Option<String> {
|
||||
let cs: Vec<char> = p.chars().collect();
|
||||
let mut out = String::with_capacity(p.len() + 16);
|
||||
let (mut i, mut class, mut class_start) = (0, false, false);
|
||||
// Répétition qui précède : 0 aucune, 1 une répétition (un « ? » la
|
||||
// rend paresseuse), 2 déjà paresseuse. Le Go refuse d'en répéter une.
|
||||
let mut rep = 0u8;
|
||||
while i < cs.len() {
|
||||
let c = cs[i];
|
||||
if !class {
|
||||
match c {
|
||||
'*' | '+' | '?' => {
|
||||
if rep == 1 && c == '?' {
|
||||
rep = 2;
|
||||
} else if rep != 0 {
|
||||
return None;
|
||||
} else {
|
||||
rep = 1;
|
||||
}
|
||||
out.push(c);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
'{' => {
|
||||
// `{n}`, `{n,}`, `{n,m}` (1000 au plus) ; sinon un « { »
|
||||
// littéral pour le Go.
|
||||
let rest: String = cs[i + 1..].iter().collect();
|
||||
let body = rest.split_once('}').map(|(b, _)| b);
|
||||
let parsed = body.and_then(|b| {
|
||||
let (lo, hi) = match b.split_once(',') {
|
||||
Some((lo, hi)) => (lo, Some(hi)),
|
||||
None => (b, None),
|
||||
};
|
||||
let num = |t: &str| {
|
||||
(!t.is_empty() && t.bytes().all(|d| d.is_ascii_digit()))
|
||||
.then(|| t.parse::<u64>().unwrap_or(u64::MAX))
|
||||
};
|
||||
let lo = num(lo)?;
|
||||
let hi = match hi {
|
||||
None => Some(lo),
|
||||
Some("") => None,
|
||||
Some(h) => Some(num(h)?),
|
||||
};
|
||||
Some((b.chars().count(), lo, hi))
|
||||
});
|
||||
match parsed {
|
||||
Some((len, lo, hi)) => {
|
||||
if lo > 1000 || hi.is_some_and(|h| h > 1000 || h < lo) || rep != 0 {
|
||||
return None;
|
||||
}
|
||||
rep = 1;
|
||||
out.extend(&cs[i..i + len + 2]);
|
||||
i += len + 2;
|
||||
}
|
||||
None => {
|
||||
rep = 0;
|
||||
out.push_str(r"\{");
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => rep = 0,
|
||||
}
|
||||
}
|
||||
if c == '\\' && i + 1 < cs.len() {
|
||||
let n = cs[i + 1];
|
||||
i += 2;
|
||||
@@ -990,6 +1049,18 @@ pub fn go_regex(p: &str) -> Option<String> {
|
||||
class_start = false;
|
||||
continue;
|
||||
}
|
||||
// `\p{…}`, `\P{…}`, `\x{…}` : les accolades sont à l'échappement.
|
||||
if matches!(n, 'p' | 'P' | 'x')
|
||||
&& cs.get(i) == Some(&'{')
|
||||
&& let Some(k) = cs[i..].iter().position(|&c| c == '}')
|
||||
{
|
||||
out.push('\\');
|
||||
out.push(n);
|
||||
out.extend(&cs[i..=i + k]);
|
||||
i += k + 1;
|
||||
class_start = false;
|
||||
continue;
|
||||
}
|
||||
// Un chiffre seul est une référence arrière pour le Go, qui la
|
||||
// refuse ; avec l'octal permis, `regex` l'accepterait.
|
||||
if ('1'..='7').contains(&n) && !cs.get(i).is_some_and(|d| ('0'..='7').contains(d)) {
|
||||
@@ -2872,6 +2943,19 @@ mod tests {
|
||||
assert_eq!(m(r"\Q(x", "a(x"), Some("(x".into()));
|
||||
// Le Go refuse \Q dans une classe.
|
||||
assert_eq!(go_regex(r"[\Q]-\E]+"), None);
|
||||
// Répétition d'une répétition : refusée par le Go ; « ? » paresseux
|
||||
// permis une fois ; « { » hors répétition littéral ; bornes ≤ 1000.
|
||||
for p in ["a**", "a+*", "a*??", "a{2}*", "a*{2}", "a{1001}", "a{3,2}"] {
|
||||
assert_eq!(go_regex(p), None, "{p}");
|
||||
}
|
||||
assert_eq!(go_regex("a*?b+?c??").as_deref(), Some("a*?b+?c??"));
|
||||
assert_eq!(go_regex("a{2,}{").as_deref(), Some(r"a{2,}\{"));
|
||||
assert_eq!(go_regex("x{a}").as_deref(), Some(r"x\{a}"));
|
||||
assert!(compile_go_regex("x{,3}").unwrap().is_match("x{,3}"));
|
||||
assert_eq!(
|
||||
go_regex(r"\p{Greek}+\x{41}{2}").as_deref(),
|
||||
Some(r"\p{Greek}+\x{41}{2}")
|
||||
);
|
||||
// Dans une classe, « [ », « && », « -- » et « ~~ » sont littéraux.
|
||||
assert_eq!(m(r"[a[]+", "x[a["), Some("[a[".into()));
|
||||
assert_eq!(m(r"[a&&b]+", "x&ab"), Some("&ab".into()));
|
||||
|
||||
@@ -304,7 +304,7 @@ fn kind_of(v: &Value) -> &'static str {
|
||||
}
|
||||
|
||||
/// `quoteChar` du scanner Go.
|
||||
fn quote_char(c: u8) -> String {
|
||||
pub(crate) fn quote_char(c: u8) -> String {
|
||||
match c {
|
||||
b'\'' => r"'\''".into(),
|
||||
b'"' => "'\"'".into(),
|
||||
|
||||
@@ -52,3 +52,4 @@ pub mod tune;
|
||||
pub mod tz;
|
||||
pub mod ui;
|
||||
pub mod update;
|
||||
pub mod webtools;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,192 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn md5_rfc1321() {
|
||||
for (input, want) in [
|
||||
("", "d41d8cd98f00b204e9800998ecf8427e"),
|
||||
("a", "0cc175b9c0f1b6a831c399e269772661"),
|
||||
("abc", "900150983cd24fb0d6963f7d28e17f72"),
|
||||
("message digest", "f96b697d7cb7938d525a2f31aaf161d0"),
|
||||
(
|
||||
"12345678901234567890123456789012345678901234567890123456789012345678901234567890",
|
||||
"57edf4a22be3c955ac49da2e2107b67a",
|
||||
),
|
||||
] {
|
||||
assert_eq!(hex::encode(md5(input.as_bytes())), want, "{input}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plafond_de_sortie() {
|
||||
let s = "é".repeat(MAX_OUTPUT);
|
||||
assert_eq!(cap_output(&s), s);
|
||||
let long = format!("{s}x");
|
||||
let out = cap_output(&long);
|
||||
assert!(out.starts_with(&s));
|
||||
assert!(out.ends_with("ou cible avec web_grep.]"));
|
||||
assert!(!out.contains('x'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cles_du_cache() {
|
||||
let base = FetchOpts {
|
||||
dismiss_popups: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(cache_key_for("u", &base), "u");
|
||||
let o = FetchOpts {
|
||||
dismiss_popups: false,
|
||||
..Default::default()
|
||||
};
|
||||
// Valeurs tirées de cacheKeyFor du Go.
|
||||
assert_eq!(cache_key_for("u", &o), "u#77fc370a");
|
||||
let o = FetchOpts {
|
||||
dismiss_popups: true,
|
||||
actions: vec!["a < b && c".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(cache_key_for("u", &o), "u#d1533b44");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn urls() {
|
||||
for (raw, want) in [
|
||||
(
|
||||
"https://github.com/a/b",
|
||||
"https://raw.githubusercontent.com/a/b/HEAD/README.md",
|
||||
),
|
||||
(
|
||||
"//github.com/a/b/",
|
||||
"https://raw.githubusercontent.com/a/b/HEAD/README.md",
|
||||
),
|
||||
(
|
||||
"https://github.com/a%20x/b?tab=1#f",
|
||||
"https://raw.githubusercontent.com/a x/b/HEAD/README.md",
|
||||
),
|
||||
("https://GitHub.com/a/b", "https://GitHub.com/a/b"),
|
||||
("https://github.com:443/a/b", "https://github.com:443/a/b"),
|
||||
("https://github.com/a", "https://github.com/a"),
|
||||
("https://github.com/a/b/c", "https://github.com/a/b/c"),
|
||||
("https://github.com/%zz/b", "https://github.com/%zz/b"),
|
||||
("github.com/a/b", "github.com/a/b"),
|
||||
(":github.com/a/b", ":github.com/a/b"),
|
||||
] {
|
||||
assert_eq!(normalize_crawl_url(raw), want, "{raw}");
|
||||
}
|
||||
for (raw, want) in [
|
||||
(
|
||||
"//duckduckgo.com/l/?uddg=https%3A%2F%2Fa.example%2F%3Fx%3D1%26y%3D2&rut=z",
|
||||
"https://a.example/?x=1&y=2",
|
||||
),
|
||||
("https://b.example/", "https://b.example/"),
|
||||
(
|
||||
"//duckduckgo.com/l/?uddg=%ZZ",
|
||||
"https://duckduckgo.com/l/?uddg=%ZZ",
|
||||
),
|
||||
("//duckduckgo.com/l/?x;y=1&uddg=a%2Bb+c", "a b c"),
|
||||
("/l/?uddg=", "/l/?uddg="),
|
||||
] {
|
||||
assert_eq!(decode_uddg(raw), want, "{raw}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entites() {
|
||||
assert_eq!(
|
||||
decode_html_entities(
|
||||
"a &lt; été � � � ""
|
||||
),
|
||||
"a < été � A � \""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nombres_du_go() {
|
||||
assert_eq!(go_int(3.9), 3);
|
||||
assert_eq!(go_int(-3.9), -3);
|
||||
if cfg!(target_arch = "x86_64") {
|
||||
assert_eq!(go_int(1e300), i64::MIN);
|
||||
assert_eq!(go_int(f64::NAN), i64::MIN);
|
||||
}
|
||||
assert_eq!(format_bytes(1023), "1023 B");
|
||||
assert_eq!(format_bytes(1536), "1.5 KB");
|
||||
assert_eq!(format_bytes(3 * 1024 * 1024 + 1), "3.00 MB");
|
||||
assert_eq!(tail_runes("abcdé", 2), "dé");
|
||||
assert_eq!(tail_runes("ab", 5), "ab");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn erreurs_de_regex() {
|
||||
for (expr, want) in [
|
||||
("(?i)(", "missing closing ): `(?i)(`"),
|
||||
("(?i)a)", "unexpected ): `(?i)a)`"),
|
||||
("(?i)[abc", "missing closing ]: `[abc`"),
|
||||
("(?i)[[:alpha:]", "missing closing ]: `[[:alpha:]`"),
|
||||
("(?i)*x", "missing argument to repetition operator: `*`"),
|
||||
("(?i)a|+", "missing argument to repetition operator: `+`"),
|
||||
("(?i)a**", "invalid nested repetition operator: `**`"),
|
||||
("(?i)a*??", "invalid nested repetition operator: `*??`"),
|
||||
("(?i)\\q", "invalid escape sequence: `\\q`"),
|
||||
("(?i)\\1", "invalid escape sequence: `\\1`"),
|
||||
("(?i)a\\", "trailing backslash at end of expression: ``"),
|
||||
] {
|
||||
assert_eq!(
|
||||
go_regexp_error(expr),
|
||||
format!("error parsing regexp: {want}"),
|
||||
"{expr}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reponses_de_crawl4ai() {
|
||||
let md = |raw: &str, pr: bool| crawl_markdown(raw.as_bytes(), pr);
|
||||
assert_eq!(
|
||||
md(
|
||||
r#"{"results":[{"success":true,"markdown":{"raw_markdown":"r","fit_markdown":"f"}}]}"#,
|
||||
true
|
||||
),
|
||||
Ok("r".into())
|
||||
);
|
||||
assert_eq!(
|
||||
md(
|
||||
r#"{"results":[{"success":true,"markdown":{"raw_markdown":"","fit_markdown":"f"}}]}"#,
|
||||
true
|
||||
),
|
||||
Ok("f".into())
|
||||
);
|
||||
assert_eq!(
|
||||
md(
|
||||
r#"{"results":[{"success":true,"markdown":{"raw_markdown":"r"}}]}"#,
|
||||
false
|
||||
),
|
||||
Ok("r".into())
|
||||
);
|
||||
assert_eq!(
|
||||
md(r#"{"Results":[{"Success":true,"Markdown":"x"}]}"#, false),
|
||||
Ok("x".into())
|
||||
);
|
||||
assert_eq!(md("null", false), Err("Crawl4AI : échec du crawl".into()));
|
||||
assert_eq!(md("5", false), Err("Crawl4AI : réponse vide".into()));
|
||||
assert_eq!(
|
||||
md(r#"{"results":[{"success":"x"}],"success":"y"}"#, false),
|
||||
Err("Crawl4AI : échec du crawl".into())
|
||||
);
|
||||
assert_eq!(
|
||||
md(r#"{"results":[null,{"success":true}]}"#, false),
|
||||
Err("Crawl4AI : échec du crawl".into())
|
||||
);
|
||||
assert_eq!(
|
||||
md(r#"{"results":[{"success":true,"markdown":null}]}"#, false),
|
||||
Ok(String::new())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lignes_normalisees() {
|
||||
assert_eq!(
|
||||
normalize_lines("\n\n a \r\nb \n\n\n\nb\nb\nc\t\n\n"),
|
||||
[" a", "b", "", "b", "c"]
|
||||
);
|
||||
assert!(normalize_lines(" \n\t\n").is_empty());
|
||||
}
|
||||
@@ -149,6 +149,12 @@ pub struct FakeGo {
|
||||
pub projsnaps: Mutex<std::collections::VecDeque<serde_json::Value>>,
|
||||
pub projsnap_refresh: Mutex<Option<serde_json::Value>>,
|
||||
pub projsnap_refreshes: Mutex<Vec<serde_json::Value>>,
|
||||
/// Moteur web intégré (lot 42) : réponses de /internal/v1/web/fetch par
|
||||
/// URL et de /internal/v1/web/search par « requête|limite » ; appels
|
||||
/// reçus (« fetch URL », « search requête|limite »).
|
||||
pub web_fetch: Mutex<HashMap<String, serde_json::Value>>,
|
||||
pub web_search: Mutex<HashMap<String, serde_json::Value>>,
|
||||
pub web_log: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl FakeGo {
|
||||
@@ -505,6 +511,28 @@ async fn handle(st: Arc<FakeGo>, req: Request<Incoming>) -> Response<Full<Bytes>
|
||||
200,
|
||||
serde_json::json!({ "generating": *st.generating.lock().unwrap() }).to_string(),
|
||||
),
|
||||
"/internal/v1/web/fetch" | "/internal/v1/web/search" => {
|
||||
let fetch = path == "/internal/v1/web/fetch";
|
||||
let body = req.into_body().collect().await.unwrap().to_bytes();
|
||||
let v: serde_json::Value = serde_json::from_slice(&body).unwrap_or_default();
|
||||
let key = if fetch {
|
||||
v["url"].as_str().unwrap_or("").to_string()
|
||||
} else {
|
||||
format!("{}|{}", v["query"].as_str().unwrap_or(""), v["limit"])
|
||||
};
|
||||
st.web_log
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(format!("{} {key}", if fetch { "fetch" } else { "search" }));
|
||||
let map = if fetch { &st.web_fetch } else { &st.web_search };
|
||||
let out = map
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&key)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| serde_json::json!({ "error": format!("inconnu : {key}") }));
|
||||
resp(200, out.to_string())
|
||||
}
|
||||
"/internal/v1/web/reach/reset" => {
|
||||
*st.reach_resets.lock().unwrap() += 1;
|
||||
resp(204, "")
|
||||
|
||||
@@ -671,6 +671,7 @@ async fn tour_complet_par_le_go() {
|
||||
mcp: None,
|
||||
tasks: None,
|
||||
memory: None,
|
||||
web: Default::default(),
|
||||
go_version: Default::default(),
|
||||
});
|
||||
let cfg = std::collections::HashMap::new();
|
||||
@@ -2746,3 +2747,44 @@ async fn outil_memoire_par_la_facade() {
|
||||
.clone();
|
||||
assert_eq!(tool, serde_json::json!("[ok] page 'chat' créée"));
|
||||
}
|
||||
|
||||
/// Lot 42 : un outil web d'un tour de la façade s'exécute chez elle (cache
|
||||
/// des pages compris) ; seule la conversion du moteur intégré est demandée
|
||||
/// au Go, jamais /tool/exec.
|
||||
#[tokio::test]
|
||||
async fn outil_web_par_la_facade() {
|
||||
let call = sse(&[
|
||||
serde_json::json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"w1","type":"function","function":{"name":"web_open","arguments":"{\"url\":\"https://exemple.test/doc\"}"}}]}}]}),
|
||||
serde_json::json!({"choices":[{"delta":{},"finish_reason":"tool_calls"}]}),
|
||||
]);
|
||||
let (url, seen) = fake_engine(vec![call, answer("lu")]).await;
|
||||
let (svc, go, _stop, _t) = service_with(Some(&url), |_| {}).await;
|
||||
go.set("config", "WEB_ENGINE", "go");
|
||||
go.web_fetch.lock().unwrap().insert(
|
||||
"https://exemple.test/doc".into(),
|
||||
serde_json::json!({"markdown": "# Doc\n\n## Partie\ntexte"}),
|
||||
);
|
||||
svc.send(serde_json::json!({"message": "lis la doc", "cid": "w"}))
|
||||
.await
|
||||
.unwrap();
|
||||
until_done(&svc, 1).await;
|
||||
assert!(go.tool_log.lock().unwrap().is_empty(), "{:?}", go.tool_log);
|
||||
assert_eq!(
|
||||
*go.web_log.lock().unwrap(),
|
||||
vec!["fetch https://exemple.test/doc".to_string()]
|
||||
);
|
||||
let seen = seen.lock().unwrap().clone();
|
||||
let tool = seen[1].1["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.rfind(|m| m["role"] == "tool")
|
||||
.unwrap()["content"]
|
||||
.clone();
|
||||
assert_eq!(
|
||||
tool,
|
||||
serde_json::json!(
|
||||
"# Ouvert : https://exemple.test/doc\nTotal : 4 lignes, 23 B (23 caractères)\nEn cache 10 min. Utilise web_read ou web_grep pour lire.\n\n## Plan (n° de ligne des titres)\n```\n 1 | Doc\n 3 | Partie\n```"
|
||||
)
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,291 @@
|
||||
//! Lot 42 : outils web de l'agent (`web_search`, `web_images`, `web_open`,
|
||||
//! `web_read`, `web_grep`) exécutés par la façade.
|
||||
//!
|
||||
//! Le journal tiré du Go (`internal/loki/parity_web_tools_test.go` →
|
||||
//! `tests/parity_web/outils.json`) est rejoué contre le port : mêmes
|
||||
//! réponses du faux Crawl4AI et du faux DuckDuckGo, ce que le moteur intégré
|
||||
//! du Go a tiré des pages servi par le faux Go ; chaque appel doit rendre le
|
||||
//! même résultat et faire les mêmes requêtes (corps envoyés à Crawl4AI,
|
||||
//! en-têtes et cookies envoyés à DuckDuckGo, pages et recherches demandées
|
||||
//! au Go).
|
||||
|
||||
mod common;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use common::fakego;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Request, Response};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use loki_rs::store::Store;
|
||||
use loki_rs::webtools::Web;
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
fn journal() -> Value {
|
||||
serde_json::from_str(include_str!("parity_web/outils.json")).expect("journal")
|
||||
}
|
||||
|
||||
fn s(v: &Value) -> &str {
|
||||
v.as_str().unwrap_or("")
|
||||
}
|
||||
|
||||
type Log = Arc<Mutex<Vec<Value>>>;
|
||||
|
||||
/// Un petit serveur HTTP : `f(méthode, chemin?requête, en-têtes, corps)`
|
||||
/// rend (statut, en-têtes, corps).
|
||||
async fn serve<F>(f: F) -> String
|
||||
where
|
||||
F: Fn(String, String, hyper::HeaderMap, Vec<u8>) -> (u16, Vec<(String, String)>, String)
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static,
|
||||
{
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = l.local_addr().unwrap();
|
||||
let f = Arc::new(f);
|
||||
tokio::spawn(async move {
|
||||
while let Ok((st, _)) = l.accept().await {
|
||||
let f = f.clone();
|
||||
tokio::spawn(async move {
|
||||
let svc = service_fn(move |req: Request<hyper::body::Incoming>| {
|
||||
let f = f.clone();
|
||||
async move {
|
||||
let method = req.method().to_string();
|
||||
let uri = req
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(|p| p.to_string())
|
||||
.unwrap_or_default();
|
||||
let headers = req.headers().clone();
|
||||
let body = req.into_body().collect().await.unwrap().to_bytes().to_vec();
|
||||
let (code, hs, out) = f(method, uri, headers, body);
|
||||
let mut r = Response::new(Full::new(Bytes::from(out)));
|
||||
*r.status_mut() = hyper::StatusCode::from_u16(code).unwrap();
|
||||
for (k, v) in hs {
|
||||
r.headers_mut().insert(
|
||||
hyper::header::HeaderName::from_bytes(k.as_bytes()).unwrap(),
|
||||
v.parse().unwrap(),
|
||||
);
|
||||
}
|
||||
Ok::<_, Infallible>(r)
|
||||
}
|
||||
});
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(st), svc)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
});
|
||||
format!("http://{addr}")
|
||||
}
|
||||
|
||||
fn header(h: &hyper::HeaderMap, k: &str) -> String {
|
||||
h.get(k)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Une requête reçue, champs vides omis (comme `omitempty`).
|
||||
fn req(fields: &[(&str, String)]) -> Value {
|
||||
let mut m = Map::new();
|
||||
for (k, v) in fields {
|
||||
if !v.is_empty() {
|
||||
m.insert((*k).into(), json!(v));
|
||||
}
|
||||
}
|
||||
Value::Object(m)
|
||||
}
|
||||
|
||||
/// Le faux Crawl4AI : la réponse rangée pour l'URL demandée (dans l'ordre,
|
||||
/// la dernière reste).
|
||||
async fn fake_crawl(fx: Value, log: Log) -> String {
|
||||
let served: Arc<Mutex<HashMap<String, usize>>> = Default::default();
|
||||
serve(move |method, uri, h, body| {
|
||||
let path = uri.split('?').next().unwrap_or("").to_string();
|
||||
log.lock().unwrap().push(req(&[
|
||||
("path", format!("{method} {path}")),
|
||||
("auth", header(&h, "authorization")),
|
||||
("ctype", header(&h, "content-type")),
|
||||
("body", String::from_utf8_lossy(&body).into_owned()),
|
||||
]));
|
||||
let v: Value = serde_json::from_slice(&body).unwrap_or_default();
|
||||
let target = s(&v["urls"][0]).to_string();
|
||||
let Some(list) = fx[&target].as_array().filter(|l| !l.is_empty()) else {
|
||||
return (404, vec![], format!("inconnu : {target}"));
|
||||
};
|
||||
let mut served = served.lock().unwrap();
|
||||
let n = served.entry(target).or_default();
|
||||
let r = &list[(*n).min(list.len() - 1)];
|
||||
*n += 1;
|
||||
(
|
||||
r["status"].as_u64().unwrap() as u16,
|
||||
vec![],
|
||||
s(&r["body"]).to_string(),
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Le faux DuckDuckGo des images.
|
||||
async fn fake_ddg(fx: Value, log: Log) -> String {
|
||||
serve(move |_, uri, h, _| {
|
||||
log.lock().unwrap().push(req(&[
|
||||
("path", uri.clone()),
|
||||
("referer", header(&h, "referer")),
|
||||
("accept", header(&h, "accept")),
|
||||
("lang", header(&h, "accept-language")),
|
||||
("ua", header(&h, "user-agent")),
|
||||
("cookie", header(&h, "cookie")),
|
||||
]));
|
||||
let (path, query) = uri.split_once('?').unwrap_or((&uri, ""));
|
||||
let q = url::form_urlencoded::parse(query.as_bytes())
|
||||
.find(|(k, _)| k == "q")
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.unwrap_or_default();
|
||||
let f = &fx[&q];
|
||||
if path == "/i.js" {
|
||||
return (
|
||||
f["api_status"].as_u64().unwrap() as u16,
|
||||
vec![],
|
||||
s(&f["api"]).to_string(),
|
||||
);
|
||||
}
|
||||
let mut hs = vec![];
|
||||
if !s(&f["cookie"]).is_empty() {
|
||||
hs.push(("set-cookie".to_string(), s(&f["cookie"]).to_string()));
|
||||
}
|
||||
(
|
||||
f["page_status"].as_u64().unwrap() as u16,
|
||||
hs,
|
||||
s(&f["page"]).to_string(),
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn outils_web_comme_le_go() {
|
||||
let fx = journal();
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (_, go, _stop) = fakego::start(tmp.path().to_path_buf()).await;
|
||||
let backend = go.backend.lock().unwrap().unwrap();
|
||||
let store = Store::new(backend, fakego::JETON.into());
|
||||
|
||||
let (crawl_log, img_log): (Log, Log) = Default::default();
|
||||
let crawl = fake_crawl(fx["crawl"].clone(), crawl_log.clone()).await;
|
||||
let ddg = fake_ddg(fx["images"].clone(), img_log.clone()).await;
|
||||
let dead = {
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
l.local_addr().unwrap().to_string()
|
||||
};
|
||||
let pages = "http://pages.invalid".to_string();
|
||||
let unmask = |v: &str| {
|
||||
v.replace("{{CRAWL}}", &crawl)
|
||||
.replace("{{DDG}}", &ddg)
|
||||
.replace("{{PAGES}}", &pages)
|
||||
.replace("{{DEADADDR}}", &dead)
|
||||
.replace("{{DEAD}}", &format!("http://{dead}"))
|
||||
};
|
||||
let mask = |v: &str| {
|
||||
v.replace(&crawl, "{{CRAWL}}")
|
||||
.replace(&ddg, "{{DDG}}")
|
||||
.replace(&pages, "{{PAGES}}")
|
||||
.replace(&format!("http://{dead}"), "{{DEAD}}")
|
||||
.replace(&dead, "{{DEADADDR}}")
|
||||
};
|
||||
for (k, v) in fx["go_fetch"].as_object().unwrap() {
|
||||
let v: Value = serde_json::from_str(&unmask(&v.to_string())).unwrap();
|
||||
go.web_fetch.lock().unwrap().insert(unmask(k), v);
|
||||
}
|
||||
for (k, v) in fx["go_search"].as_object().unwrap() {
|
||||
go.web_search.lock().unwrap().insert(k.clone(), v.clone());
|
||||
}
|
||||
|
||||
let web = Web::with_ddg_base(&ddg);
|
||||
for c in fx["calls"].as_array().unwrap() {
|
||||
let name = s(&c["name"]);
|
||||
for (k, v) in c["setup"].as_object().into_iter().flatten() {
|
||||
let (bucket, v) = if k == "crawl_key" {
|
||||
("state", s(v).to_string())
|
||||
} else {
|
||||
("config", unmask(s(v)))
|
||||
};
|
||||
if v == "-" {
|
||||
go.del(bucket, k);
|
||||
} else {
|
||||
go.set(bucket, k, &v);
|
||||
}
|
||||
}
|
||||
if let Some(n) = c["advance_s"].as_u64() {
|
||||
web.advance(Duration::from_secs(n));
|
||||
}
|
||||
let args: Map<String, Value> =
|
||||
serde_json::from_str(&unmask(&c["args"].to_string())).unwrap();
|
||||
let got = web
|
||||
.exec(&store, s(&c["tool"]), &args)
|
||||
.await
|
||||
.expect("un outil web");
|
||||
assert_eq!(mask(&got), s(&c["result"]), "{name}");
|
||||
|
||||
let masked = |log: &Log| -> Value {
|
||||
let v = Value::Array(std::mem::take(&mut *log.lock().unwrap()));
|
||||
serde_json::from_str(&mask(&v.to_string())).unwrap()
|
||||
};
|
||||
assert_eq!(masked(&crawl_log), c["crawls"], "{name} : Crawl4AI");
|
||||
// reqwest pose « Accept: */* » quand la requête n'en a pas ; le
|
||||
// client du Go n'en pose aucun (sans effet sur DuckDuckGo).
|
||||
let mut imgs = masked(&img_log);
|
||||
for r in imgs.as_array_mut().into_iter().flatten() {
|
||||
if r["accept"] == "*/*" {
|
||||
r.as_object_mut().unwrap().remove("accept");
|
||||
}
|
||||
}
|
||||
assert_eq!(imgs, c["images"], "{name} : DuckDuckGo");
|
||||
let calls = std::mem::take(&mut *go.web_log.lock().unwrap());
|
||||
let fetched: Vec<String> = calls
|
||||
.iter()
|
||||
.filter_map(|l| l.strip_prefix("fetch "))
|
||||
.map(&mask)
|
||||
.collect();
|
||||
let searched: Vec<String> = calls
|
||||
.iter()
|
||||
.filter_map(|l| l.strip_prefix("search "))
|
||||
.map(|q| q.rsplit_once('|').map_or(q, |(q, _)| q).to_string())
|
||||
.collect();
|
||||
assert_eq!(json!(fetched), c["go_fetch"], "{name} : pages du Go");
|
||||
assert_eq!(json!(searched), c["go_search"], "{name} : recherches du Go");
|
||||
}
|
||||
}
|
||||
|
||||
/// Le Go injoignable : le moteur intégré (par défaut sans configuration)
|
||||
/// le dit ; un autre outil n'est pas servi ici.
|
||||
#[tokio::test]
|
||||
async fn moteur_integre_injoignable() {
|
||||
let dead = std::net::TcpListener::bind("127.0.0.1:0")
|
||||
.unwrap()
|
||||
.local_addr()
|
||||
.unwrap();
|
||||
let store = Store::new(dead, fakego::JETON.into());
|
||||
let web = Web::new();
|
||||
let mut args = Map::new();
|
||||
args.insert("url".into(), json!("https://exemple.test/"));
|
||||
let got = web.exec(&store, "web_open", &args).await.unwrap();
|
||||
assert!(
|
||||
got.starts_with("❌ moteur web intégré injoignable : "),
|
||||
"{got}"
|
||||
);
|
||||
args.insert("query".into(), json!("x"));
|
||||
let got = web.exec(&store, "web_search", &args).await.unwrap();
|
||||
assert!(
|
||||
got.starts_with("❌ Recherche échouée : moteur web intégré injoignable : "),
|
||||
"{got}"
|
||||
);
|
||||
assert_eq!(web.exec(&store, "web_screenshot", &args).await, None);
|
||||
}
|
||||
Reference in new issue
Block a user