mirror of
https://github.com/R0m1k3/Loki.git
synced 2026-10-17 04:10:04 +02:00
Lot 44 : navigateur piloté tenu par la façade
La façade tient la session Chrome des outils browser_* (src/browser/, port de computer_cdp.go, computer_use.go, browser_grid.go et cu_autoshot.go) : lancement (profil temporaire, mêmes arguments, tué avec la façade), client CDP multiplexé sur websocket, attente du chargement et du calme réseau, scripts injectés copiés à l'octet près, les neuf outils aux textes du Go (snapshot dédoublonné, recherche, clics, saisie et listes déroulantes, touches, défilement, navigation et ses erreurs), capture rangée propre et quadrillée pour le modèle (mêmes pixels), aperçus JPEG en anneau servis par /api/chat/cushot. Un seul navigateur : quand la façade tient la conversation, le Go lui passe les outils de ses sous-agents et tâches IA, ses aperçus et la fermeture (POST /internal/facade/v1/browser) ; façade injoignable, le navigateur du Go prend le relais. Couper l'interrupteur ferme les deux. Parité : journal de 42 étapes tiré du Go contre un faux Chrome (script + faux serveur CDP qui note chaque commande, scripts résumés par leur empreinte, pixels de la grille), rejoué par tests/browser.rs ; un vrai Chromium de bout en bout avec LOKI_CU_SMOKE=1 ; un tour de la façade qui ouvre une page (tests/lot6.rs) ; TestFacadeNavigateur côté Go. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016tcGfJWefW5kUGoC2wL8cv
This commit is contained in:
24 files changed
+3910
-11
No files matched your search
@@ -0,0 +1,126 @@
|
||||
//! Grille de coordonnées sur la capture envoyée au modèle — port de
|
||||
//! `browser_grid.go`. Lignes graduées tous les 100 px et étiquettes en
|
||||
//! police bitmap 3×5 : un petit modèle y lit x/y pour `browser_click_xy`.
|
||||
//! Mêmes pixels que le Go (l'encodeur PNG, lui, diffère).
|
||||
|
||||
use image::{ImageFormat, Rgba, RgbaImage};
|
||||
|
||||
/// `gridStep`.
|
||||
const GRID_STEP: u32 = 100;
|
||||
|
||||
/// `digits5x3`.
|
||||
fn glyph(c: char) -> Option<[&'static str; 5]> {
|
||||
Some(match c {
|
||||
'0' => ["###", "# #", "# #", "# #", "###"],
|
||||
'1' => [" # ", "## ", " # ", " # ", "###"],
|
||||
'2' => ["###", " #", "###", "# ", "###"],
|
||||
'3' => ["###", " #", "###", " #", "###"],
|
||||
'4' => ["# #", "# #", "###", " #", " #"],
|
||||
'5' => ["###", "# ", "###", " #", "###"],
|
||||
'6' => ["###", "# ", "###", "# #", "###"],
|
||||
'7' => ["###", " #", " #", " #", " #"],
|
||||
'8' => ["###", "# #", "###", "# #", "###"],
|
||||
'9' => ["###", "# #", "###", " #", "###"],
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
const LINE: [u8; 4] = [255, 0, 200, 130];
|
||||
const LABEL_BG: [u8; 4] = [0, 0, 0, 200];
|
||||
const LABEL_FG: [u8; 4] = [255, 255, 0, 255];
|
||||
|
||||
/// `overlayGrid` : le PNG avec sa grille ; illisible, rendu tel quel.
|
||||
pub fn overlay_grid(raw: &[u8]) -> Vec<u8> {
|
||||
let Ok(src) = image::load_from_memory_with_format(raw, ImageFormat::Png) else {
|
||||
return raw.to_vec();
|
||||
};
|
||||
let mut img = src.to_rgba8();
|
||||
draw(&mut img);
|
||||
let mut out = std::io::Cursor::new(Vec::new());
|
||||
if img.write_to(&mut out, ImageFormat::Png).is_err() {
|
||||
return raw.to_vec();
|
||||
}
|
||||
out.into_inner()
|
||||
}
|
||||
|
||||
/// La grille et ses graduations, dessinées sur l'image.
|
||||
pub fn draw(img: &mut RgbaImage) {
|
||||
let (w, h) = img.dimensions();
|
||||
let mut x = GRID_STEP;
|
||||
while x < w {
|
||||
for y in 0..h {
|
||||
blend(img, x as i64, y as i64, LINE);
|
||||
}
|
||||
draw_label(img, x as i64 + 2, 2, &x.to_string());
|
||||
x += GRID_STEP;
|
||||
}
|
||||
let mut y = GRID_STEP;
|
||||
while y < h {
|
||||
for x in 0..w {
|
||||
blend(img, x as i64, y as i64, LINE);
|
||||
}
|
||||
draw_label(img, 2, y as i64 + 2, &y.to_string());
|
||||
y += GRID_STEP;
|
||||
}
|
||||
}
|
||||
|
||||
fn inside(img: &RgbaImage, x: i64, y: i64) -> bool {
|
||||
x >= 0 && y >= 0 && (x as u32) < img.width() && (y as u32) < img.height()
|
||||
}
|
||||
|
||||
/// `blend` : couleur translucide sur un pixel (opaque ensuite).
|
||||
fn blend(img: &mut RgbaImage, x: i64, y: i64, c: [u8; 4]) {
|
||||
if !inside(img, x, y) {
|
||||
return;
|
||||
}
|
||||
let a = f64::from(c[3]) / 255.0;
|
||||
let o = img.get_pixel(x as u32, y as u32).0;
|
||||
let mix = |n: u8, o: u8| (f64::from(n) * a + f64::from(o) * (1.0 - a)) as u8;
|
||||
img.put_pixel(
|
||||
x as u32,
|
||||
y as u32,
|
||||
Rgba([mix(c[0], o[0]), mix(c[1], o[1]), mix(c[2], o[2]), 255]),
|
||||
);
|
||||
}
|
||||
|
||||
fn set_opaque(img: &mut RgbaImage, x: i64, y: i64, c: [u8; 4]) {
|
||||
if inside(img, x, y) {
|
||||
img.put_pixel(x as u32, y as u32, Rgba([c[0], c[1], c[2], 255]));
|
||||
}
|
||||
}
|
||||
|
||||
/// `drawLabel` : un nombre en police 3×5 (×2) sur fond sombre.
|
||||
fn draw_label(img: &mut RgbaImage, x: i64, y: i64, s: &str) {
|
||||
const SC: i64 = 2;
|
||||
let (glyph_w, glyph_h, gap) = (3 * SC, 5 * SC, SC);
|
||||
let box_w = s.chars().count() as i64 * (glyph_w + gap) + gap;
|
||||
let box_h = glyph_h + 2 * gap;
|
||||
for dy in -gap..box_h - gap {
|
||||
for dx in -gap..box_w - gap {
|
||||
set_opaque(img, x + dx, y + dy, LABEL_BG);
|
||||
}
|
||||
}
|
||||
let mut cx = x;
|
||||
for c in s.chars() {
|
||||
if let Some(g) = glyph(c) {
|
||||
for (row, line) in g.iter().enumerate() {
|
||||
for (col, b) in line.bytes().enumerate() {
|
||||
if b != b'#' {
|
||||
continue;
|
||||
}
|
||||
for iy in 0..SC {
|
||||
for ix in 0..SC {
|
||||
set_opaque(
|
||||
img,
|
||||
cx + col as i64 * SC + ix,
|
||||
y + row as i64 * SC + iy,
|
||||
LABEL_FG,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cx += glyph_w + gap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//! Scripts injectés dans la page (copiés de `computer_cdp.go`, à l'octet près :
|
||||
//! le journal de parité compare les expressions envoyées).
|
||||
|
||||
/// `cdpSameTabJS` : les navigations « nouvel onglet » restent dans l'onglet.
|
||||
pub const SAME_TAB_JS: &str = r#"(function(){try{
|
||||
window.open=function(u){if(u)location.href=u;return window;};
|
||||
document.addEventListener('click',function(e){
|
||||
var a=e.target&&e.target.closest&&e.target.closest('a[target]');
|
||||
if(a)a.removeAttribute('target');
|
||||
},true);
|
||||
}catch(e){}})();"#;
|
||||
|
||||
/// `cdpHelpersJS` : rôle, nom, actionnable, visible, shadow DOM.
|
||||
pub const HELPERS_JS: &str = r#"
|
||||
function ajRole(el){var r=el.getAttribute('role');if(r)return r;var t=el.tagName.toLowerCase();if(t==='a')return 'link';if(t==='button')return 'button';if(t==='select')return 'combobox';if(t==='textarea')return 'textbox';if(t==='input'){var ty=(el.type||'text').toLowerCase();var m={checkbox:'checkbox',radio:'radio',button:'button',submit:'button',reset:'button',file:'button',range:'slider',password:'password',search:'searchbox'};return m[ty]||'textbox';}return t;}
|
||||
function ajName(el){var t=el.tagName.toLowerCase();var lbl=el.getAttribute('aria-label')||'';if(!lbl){var lb=el.getAttribute('aria-labelledby');if(lb){var e2=document.getElementById(lb.split(' ')[0]);if(e2)lbl=e2.innerText||'';}}if(!lbl&&el.labels&&el.labels.length)lbl=el.labels[0].innerText||'';if(!lbl)lbl=el.getAttribute('placeholder')||el.getAttribute('title')||el.getAttribute('alt')||'';var val='';if((t==='input'||t==='textarea')&&el.type!=='password'&&el.type!=='hidden')val=el.value||'';var n=val?(lbl?(lbl+' = '+val):val):lbl;if(!n&&t!=='input'&&t!=='select'&&t!=='textarea')n=el.innerText||'';if(!n){var c=el.querySelector('img[alt],[aria-label],[title],svg title');if(c)n=c.getAttribute&&(c.getAttribute('alt')||c.getAttribute('aria-label')||c.getAttribute('title'))||c.textContent||'';}if(!n)n=el.getAttribute('name')||el.getAttribute('value')||'';return (n||'').replace(/\s+/g,' ').trim().slice(0,100);}
|
||||
function ajActionable(el){var t=el.tagName.toLowerCase();if(t==='a')return !!el.getAttribute('href');if(t==='button'||t==='input'||t==='select'||t==='textarea'||t==='summary')return el.type!=='hidden';if(el.isContentEditable)return true;if(el.getAttribute('onclick'))return true;var r=(el.getAttribute('role')||'').toLowerCase();return {button:1,link:1,checkbox:1,radio:1,tab:1,menuitem:1,menuitemcheckbox:1,menuitemradio:1,option:1,'switch':1,combobox:1,textbox:1,searchbox:1,slider:1,spinbutton:1}[r]===1;}
|
||||
function ajVis(el){var r=el.getBoundingClientRect();if(r.width<3||r.height<3)return false;var s=getComputedStyle(el);if(s.visibility==='hidden'||s.display==='none'||s.opacity==='0')return false;if(r.bottom<0||r.top>innerHeight||r.right<0||r.left>innerWidth)return false;return true;}
|
||||
var ajSel='a[href],button,input,select,textarea,[role],[onclick],[tabindex],summary,[contenteditable="true"]';
|
||||
// Traversée du SHADOW DOM : les bandeaux cookies et composants web (usercentrics,
|
||||
// cookiebot…) vivent dans des shadow roots que querySelectorAll ne voit pas → le
|
||||
// modèle ne pouvait ni les lister ni les cliquer (test Lady Sushi). ajQueryAll /
|
||||
// ajByRef / ajClear descendent récursivement dans chaque shadowRoot ouvert.
|
||||
function ajQueryAll(){var out=[];function w(r){var e=r.querySelectorAll(ajSel);for(var i=0;i<e.length;i++)out.push(e[i]);var a=r.querySelectorAll('*');for(var j=0;j<a.length;j++)if(a[j].shadowRoot)w(a[j].shadowRoot);}w(document);return out;}
|
||||
function ajByRef(ref){function f(r){var e=r.querySelector('[data-loki-ref="'+ref+'"]');if(e)return e;var a=r.querySelectorAll('*');for(var i=0;i<a.length;i++)if(a[i].shadowRoot){var x=f(a[i].shadowRoot);if(x)return x;}return null;}return f(document);}
|
||||
function ajClear(){function w(r){var o=r.querySelectorAll('[data-loki-ref]');for(var i=0;i<o.length;i++)o[i].removeAttribute('data-loki-ref');var a=r.querySelectorAll('*');for(var j=0;j<a.length;j++)if(a[j].shadowRoot)w(a[j].shadowRoot);}w(document);}
|
||||
"#;
|
||||
|
||||
/// La fin de `cdpSnapshotJS` (préfixée de [`HELPERS_JS`]).
|
||||
pub const SNAPSHOT_TAIL_JS: &str = r#"(function(){
|
||||
ajClear();
|
||||
var els=ajQueryAll().filter(function(el){return ajActionable(el)&&ajVis(el);});
|
||||
var map={},refs=[],i=0;
|
||||
for(var k=0;k<els.length&&i<200;k++){var el=els[k];var nm=ajName(el),rl=ajRole(el);
|
||||
i++;var r=el.getBoundingClientRect();map[i]={x:r.left+r.width/2,y:r.top+r.height/2};
|
||||
el.setAttribute('data-loki-ref',String(i));
|
||||
refs.push('['+i+'] '+rl+(nm?' "'+nm+'"':''));
|
||||
}
|
||||
window.__lokiRefs=map;
|
||||
var de=document.documentElement;
|
||||
var sh=Math.max(de.scrollHeight,document.body?document.body.scrollHeight:0);
|
||||
return {title:document.title||'',url:location.href,refs:refs,
|
||||
above:Math.round(scrollY),below:Math.round(Math.max(0,sh-scrollY-innerHeight))};
|
||||
})()"#;
|
||||
|
||||
/// `cdpFindJS` : recherche non destructive sur toute la page.
|
||||
pub const FIND_JS: &str = r#"function(q){
|
||||
q=(q||'').toLowerCase();
|
||||
var pool=ajQueryAll();
|
||||
var maxRef=0;for(var m=0;m<pool.length;m++){var v=parseInt(pool[m].getAttribute('data-loki-ref'),10)||0;if(v>maxRef)maxRef=v;}
|
||||
var els=pool.filter(ajActionable);
|
||||
var refs=[],count=0,first=null;
|
||||
for(var k=0;k<els.length&&count<60;k++){var el=els[k];var nm=ajName(el),rl=ajRole(el);
|
||||
if((rl+' '+nm).toLowerCase().indexOf(q)<0)continue;
|
||||
var st=getComputedStyle(el);if(st.display==='none'||st.visibility==='hidden')continue;
|
||||
var r=el.getBoundingClientRect();if(r.width<1&&r.height<1)continue;
|
||||
var ref=el.getAttribute('data-loki-ref');
|
||||
if(!ref){maxRef++;ref=String(maxRef);el.setAttribute('data-loki-ref',ref);}
|
||||
if(!first)first=el;count++;
|
||||
refs.push('['+ref+'] '+rl+(nm?' "'+nm+'"':''));
|
||||
}
|
||||
if(first)first.scrollIntoView({block:'center'});
|
||||
return {refs:refs};
|
||||
}"#;
|
||||
|
||||
/// `cdpSnapshotJS`.
|
||||
pub fn snapshot_js() -> String {
|
||||
format!("{HELPERS_JS}{SNAPSHOT_TAIL_JS}")
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,103 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn outils_et_plafond() {
|
||||
for n in [
|
||||
"browser_open",
|
||||
"browser_snapshot",
|
||||
"browser_find",
|
||||
"browser_click",
|
||||
"browser_click_xy",
|
||||
"browser_type",
|
||||
"browser_key",
|
||||
"browser_scroll",
|
||||
"browser_screenshot",
|
||||
] {
|
||||
assert!(is_tool(n), "{n}");
|
||||
}
|
||||
assert!(!is_tool("web_screenshot"));
|
||||
assert!(!auto_shot_tool("browser_snapshot") && !auto_shot_tool("browser_screenshot"));
|
||||
assert!(auto_shot_tool("browser_find"));
|
||||
let s = "é".repeat(MAX_OUTPUT);
|
||||
assert_eq!(cap_output(&s), s);
|
||||
assert_eq!(cap_output(&format!("{s}x")), format!("{s}\n…[tronqué]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nombres_comme_le_go() {
|
||||
assert_eq!(go_num(640.0).to_string(), "640");
|
||||
assert_eq!(go_num(412.5).to_string(), "412.5");
|
||||
assert_eq!(go_num(-3.0).to_string(), "-3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arguments_de_lancement() {
|
||||
let a = launch_args(Path::new("/tmp/p"), false);
|
||||
assert_eq!(a[0], "--headless=new");
|
||||
assert!(a.contains(&"--user-data-dir=/tmp/p".to_string()));
|
||||
assert_eq!(a.last().unwrap(), "about:blank");
|
||||
let h = launch_args(Path::new("/tmp/p"), true);
|
||||
assert_eq!(h.len(), a.len() - 1);
|
||||
assert_eq!(h[0], "--remote-debugging-port=0");
|
||||
}
|
||||
|
||||
/// La grille : lignes translucides tous les 100 px, étiquettes opaques,
|
||||
/// rien d'autre de touché.
|
||||
#[test]
|
||||
fn grille() {
|
||||
let mut img = image::RgbaImage::from_pixel(250, 150, image::Rgba([40, 80, 120, 255]));
|
||||
grid::draw(&mut img);
|
||||
let px = |x, y| img.get_pixel(x, y).0;
|
||||
assert_eq!(px(0, 0), [40, 80, 120, 255]);
|
||||
// 255×130/255 + 40×125/255 → 149 ; 0 + 80×(1−a) → 39 ; 200a + 120(1−a) → 160.
|
||||
assert_eq!(px(100, 50), [149, 39, 160, 255]);
|
||||
assert_eq!(px(200, 140), [149, 39, 160, 255]);
|
||||
assert_eq!(px(50, 100), [149, 39, 160, 255]);
|
||||
assert_eq!(px(101, 50), [40, 80, 120, 255]);
|
||||
// « 100 » en haut de la ligne : fond sombre, chiffre jaune.
|
||||
assert_eq!(px(100, 1), [0, 0, 0, 255]);
|
||||
assert_eq!(px(104, 4), [255, 255, 0, 255]);
|
||||
// Une image illisible passe telle quelle.
|
||||
assert_eq!(grid::overlay_grid(b"pas une image"), b"pas une image");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profils_temporaires() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let p = mkdtemp(tmp.path()).unwrap();
|
||||
assert!(
|
||||
p.file_name()
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.starts_with("loki-cu-")
|
||||
);
|
||||
assert!(p.is_dir());
|
||||
// Récent : gardé ; un autre dossier : jamais touché.
|
||||
std::fs::create_dir(tmp.path().join("autre")).unwrap();
|
||||
sweep_stale(tmp.path());
|
||||
assert!(p.is_dir() && tmp.path().join("autre").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lancement_impossible() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let b = Browser::with("", tmp.path().to_path_buf());
|
||||
let mut a = Map::new();
|
||||
a.insert("url".into(), json!("x.test"));
|
||||
let o = b.exec("browser_open", &a, false, "").await.unwrap();
|
||||
// Sans binaire imposé ni trouvé, ou introuvable : l'erreur dit quoi faire.
|
||||
assert!(o.result.starts_with("[erreur] "), "{}", o.result);
|
||||
let b = Browser::with("/nulle/part/chrome", tmp.path().to_path_buf());
|
||||
let o = b
|
||||
.exec("browser_snapshot", &Map::new(), false, "")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
o.result,
|
||||
"[erreur] fork/exec /nulle/part/chrome: no such file or directory"
|
||||
);
|
||||
// Le profil du lancement raté est retiré.
|
||||
assert_eq!(std::fs::read_dir(tmp.path()).unwrap().count(), 0);
|
||||
assert_eq!(b.exec("web_open", &a, false, "").await, None);
|
||||
assert!(b.shot("rien").is_none());
|
||||
}
|
||||
@@ -1324,6 +1324,7 @@ impl ChatShared {
|
||||
memory: None,
|
||||
web: Default::default(),
|
||||
jobs: crate::tools::jobs::global(),
|
||||
browser: crate::browser::global(),
|
||||
go_version: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
//! d'écriture du mode Code (état du Go) : `cache_hint` et `preflight` absents.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
@@ -116,6 +116,11 @@ pub struct ImageCache {
|
||||
}
|
||||
|
||||
impl ImageCache {
|
||||
/// `LOKI_HOME` (le parent du dossier des images).
|
||||
pub fn home(&self) -> PathBuf {
|
||||
self.dir.parent().map(Path::to_path_buf).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// `dir` : `LOKI_HOME/chatimg`.
|
||||
pub fn new(dir: PathBuf) -> Self {
|
||||
ImageCache {
|
||||
@@ -351,6 +356,8 @@ pub struct ChatShared {
|
||||
pub web: Arc<crate::webtools::Web>,
|
||||
/// Les jobs d'arrière-plan du mode Code (lot 43).
|
||||
pub jobs: Arc<crate::tools::jobs::BgJobs>,
|
||||
/// Le navigateur piloté (lot 44).
|
||||
pub browser: Arc<crate::browser::Browser>,
|
||||
/// La version du Go, annoncée aux serveurs MCP à une reconnexion ; lue
|
||||
/// une fois.
|
||||
pub go_version: tokio::sync::OnceCell<String>,
|
||||
@@ -709,6 +716,34 @@ impl TurnHost for GoTurnHost {
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
// Navigateur piloté (lot 44) : la session est ici ; aperçu de la page
|
||||
// après une action qui la change (jamais après une erreur), capture
|
||||
// relayée au modèle.
|
||||
if crate::browser::is_tool(name) {
|
||||
let cfg = self.shared.store.config().await;
|
||||
let vision = crate::system::vision_enabled(&cfg);
|
||||
let home = self.shared.images.home();
|
||||
let ws = tokio::task::spawn_blocking(move || {
|
||||
let env = std::env::var(crate::paths::WORKSPACE_ENV).ok();
|
||||
crate::paths::agent_workspace(&home, env.as_deref())
|
||||
})
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
if let Some(o) = self.shared.browser.exec(name, args, vision, &ws).await {
|
||||
let live_shot =
|
||||
if !o.result.starts_with("[erreur]") && crate::browser::auto_shot_tool(name) {
|
||||
self.shared.browser.auto_shot().await
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
return ToolOutcome {
|
||||
relay: o.image.map(|img| see_image_message(label, img)),
|
||||
result: o.result,
|
||||
live_shot,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
}
|
||||
// Jobs d'arrière-plan du mode Code (lot 43), dans le dossier de la
|
||||
// discussion ; contexte indisponible : le Go s'en charge.
|
||||
if name == "bash_tail" {
|
||||
|
||||
@@ -52,6 +52,7 @@ pub fn handles(path: &str) -> bool {
|
||||
| "/api/conversations/rename"
|
||||
| "/api/conversations/delete"
|
||||
| "/api/perf/summary"
|
||||
| "/api/chat/cushot"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -72,6 +73,7 @@ pub fn is_internal(path: &str) -> bool {
|
||||
| "/internal/facade/v1/perf/record"
|
||||
| "/internal/facade/v1/perf/snapshot"
|
||||
| "/internal/facade/v1/bgjobs"
|
||||
| "/internal/facade/v1/browser"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -137,6 +139,49 @@ pub async fn serve_internal(chat: &Arc<ChatService>, req: Request<Incoming>) ->
|
||||
},
|
||||
// L'anneau, de la plus ancienne à la plus récente (poids de
|
||||
// l'optimiseur).
|
||||
// Navigateur piloté (lot 44) : les appels du Go (sous-agents, tâches
|
||||
// IA) passent par la session tenue ici — un seul navigateur.
|
||||
"/internal/facade/v1/browser" => {
|
||||
let v = read_json(req.into_body()).await.unwrap_or_default();
|
||||
let b = &chat.shared.browser;
|
||||
match str_of(&v, "op") {
|
||||
"tool" => {
|
||||
let cfg = chat.shared.store.config().await;
|
||||
let vision = crate::system::vision_enabled(&cfg);
|
||||
let home = chat.shared.images.home();
|
||||
let ws = tokio::task::spawn_blocking(move || {
|
||||
let env = std::env::var(crate::paths::WORKSPACE_ENV).ok();
|
||||
crate::paths::agent_workspace(&home, env.as_deref())
|
||||
})
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let args = v
|
||||
.get("args")
|
||||
.and_then(Value::as_object)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
match b.exec(str_of(&v, "name"), &args, vision, &ws).await {
|
||||
Some(o) => send_json(
|
||||
StatusCode::OK,
|
||||
&json!({ "result": o.result, "image": o.image }),
|
||||
),
|
||||
None => send_json(
|
||||
StatusCode::BAD_REQUEST,
|
||||
&json!({ "error": "outil du navigateur inconnu" }),
|
||||
),
|
||||
}
|
||||
}
|
||||
"shot" => send_json(StatusCode::OK, &json!({ "id": b.auto_shot().await })),
|
||||
"shutdown" => {
|
||||
b.shutdown().await;
|
||||
send_json(StatusCode::OK, &json!({}))
|
||||
}
|
||||
_ => send_json(
|
||||
StatusCode::BAD_REQUEST,
|
||||
&json!({ "error": "opération inconnue" }),
|
||||
),
|
||||
}
|
||||
}
|
||||
// Jobs d'arrière-plan encore en vie (lot 43) : l'optimiseur du Go
|
||||
// refuse de mesurer à côté.
|
||||
"/internal/facade/v1/bgjobs" => send_json(
|
||||
@@ -276,6 +321,22 @@ pub async fn serve(
|
||||
chat.stop().await;
|
||||
ok()
|
||||
}
|
||||
// `handleCUShot` (lot 44) : un aperçu du navigateur piloté, en base64
|
||||
// (JSON, pour traverser aussi le tunnel du relais).
|
||||
"/api/chat/cushot" => {
|
||||
let id = query_param(&parts.uri, "id").unwrap_or_default();
|
||||
match chat.shared.browser.shot(&id) {
|
||||
Some(b) => send_json(
|
||||
StatusCode::OK,
|
||||
&json!({ "ok": true, "mime": "image/jpeg",
|
||||
"data": base64::Engine::encode(&base64::engine::general_purpose::STANDARD, b) }),
|
||||
),
|
||||
None => send_json(
|
||||
StatusCode::NOT_FOUND,
|
||||
&json!({ "ok": false, "error": "capture expirée" }),
|
||||
),
|
||||
}
|
||||
}
|
||||
"/api/chat/reset" => {
|
||||
chat.reset().await;
|
||||
ok()
|
||||
|
||||
@@ -14,6 +14,7 @@ pub mod api;
|
||||
pub mod auth;
|
||||
pub mod backends;
|
||||
pub mod bench;
|
||||
pub mod browser;
|
||||
pub mod chat;
|
||||
pub mod chatapi;
|
||||
pub mod chatx;
|
||||
|
||||
@@ -13,9 +13,10 @@
|
||||
//!
|
||||
//! Les réglages restent des clés de la base. Ce qui vit dans le processus Go
|
||||
//! reste au Go : son cache de joignabilité (vidé par
|
||||
//! `/internal/v1/web/reach/reset`), le Chromium des outils `browser_*`
|
||||
//! (`/internal/v1/cdp/shutdown`) et le redémarrage du moteur quand la façade ne
|
||||
//! le pilote pas (`/internal/v1/service/action`).
|
||||
//! `/internal/v1/web/reach/reset`), le Chromium qu'il pilote quand la façade
|
||||
//! ne tient pas la conversation (`/internal/v1/cdp/shutdown` ; sinon le
|
||||
//! navigateur est celui de la façade, lot 44) et le redémarrage du moteur
|
||||
//! quand la façade ne le pilote pas (`/internal/v1/service/action`).
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -528,8 +529,11 @@ pub async fn serve(
|
||||
if let Err(e) = put_bool(store, "computer", on).await {
|
||||
return fail(StatusCode::INTERNAL_SERVER_ERROR, e);
|
||||
}
|
||||
// Couper l'interrupteur ferme le navigateur (au Go, qui le
|
||||
// pilote pour les outils browser_*).
|
||||
// Couper l'interrupteur ferme le navigateur : celui de la
|
||||
// façade (lot 44), et celui du Go s'il en a ouvert un.
|
||||
if !on {
|
||||
crate::browser::global().shutdown().await;
|
||||
}
|
||||
if !on
|
||||
&& let Err(e) = store
|
||||
.internal_post("/internal/v1/cdp/shutdown", &json!({}))
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
//! Lot 44 : navigateur piloté (outils `browser_*`) tenu par la façade,
|
||||
//! rejoué sur le journal du Go (`internal/loki/parity_browser_test.go` →
|
||||
//! `tests/parity_browser/browser.json`) contre le même faux Chrome : un
|
||||
//! script qui annonce le port d'un faux serveur CDP, qui répond selon le
|
||||
//! scénario de chaque étape et note chaque commande reçue.
|
||||
#![cfg(target_os = "linux")]
|
||||
|
||||
mod common;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use common::fakecdp::{self, Calls, Scene, image_facts};
|
||||
use loki_rs::browser::Browser;
|
||||
use regex::Regex;
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn navigateur_comme_le_go() {
|
||||
let fx: Value =
|
||||
serde_json::from_str(include_str!("parity_browser/browser.json")).expect("journal");
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let args_file = tmp.path().join("args");
|
||||
let ws = tmp.path().join("ws");
|
||||
let profiles = tmp.path().join("profils");
|
||||
std::fs::create_dir_all(&ws).unwrap();
|
||||
std::fs::create_dir_all(&profiles).unwrap();
|
||||
let scene: Scene = Arc::new(Mutex::new(json!({})));
|
||||
let calls: Calls = Default::default();
|
||||
let port = fakecdp::start(scene.clone(), calls.clone()).await;
|
||||
let chrome = fakecdp::chrome_script(tmp.path(), port, &args_file);
|
||||
let browser = Browser::with(&chrome.to_string_lossy(), profiles.clone());
|
||||
let cap = Regex::new(r"capture-[0-9]{8}-[0-9]{6}\.png").unwrap();
|
||||
for st in fx["steps"].as_array().unwrap() {
|
||||
let name = st["name"].as_str().unwrap();
|
||||
if let Some(sc) = st.get("scene") {
|
||||
*scene.lock().unwrap() = sc.clone();
|
||||
}
|
||||
calls.lock().unwrap().clear();
|
||||
let args: Map<String, Value> = serde_json::from_value(st["args"].clone()).unwrap();
|
||||
let o = browser
|
||||
.exec(
|
||||
st["tool"].as_str().unwrap(),
|
||||
&args,
|
||||
st["vision"] == true,
|
||||
&ws.to_string_lossy(),
|
||||
)
|
||||
.await
|
||||
.expect("un outil du navigateur");
|
||||
assert_eq!(
|
||||
cap.replace_all(&o.result, "capture-{{T}}.png"),
|
||||
st["result"].as_str().unwrap(),
|
||||
"{name}"
|
||||
);
|
||||
assert_eq!(
|
||||
o.image.as_ref().map(image_facts),
|
||||
st.get("image").cloned(),
|
||||
"{name} : image"
|
||||
);
|
||||
let mut saved: Vec<String> = std::fs::read_dir(&ws)
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.map(|e| {
|
||||
let n = e.file_name().to_string_lossy().into_owned();
|
||||
let _ = std::fs::remove_file(e.path());
|
||||
cap.replace_all(&n, "capture-{{T}}.png").into_owned()
|
||||
})
|
||||
.collect();
|
||||
saved.sort();
|
||||
assert_eq!(json!(saved), st["saved"], "{name} : captures rangées");
|
||||
assert_eq!(
|
||||
json!(*calls.lock().unwrap()),
|
||||
st["calls"],
|
||||
"{name} : commandes CDP"
|
||||
);
|
||||
}
|
||||
let launch: Vec<String> = std::fs::read_to_string(&args_file)
|
||||
.unwrap()
|
||||
.trim_end_matches('\n')
|
||||
.split('\n')
|
||||
.map(|l| {
|
||||
if l.starts_with("--user-data-dir=") {
|
||||
"--user-data-dir={{PROFIL}}".into()
|
||||
} else {
|
||||
l.to_string()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(json!(launch), fx["launch"], "lancement");
|
||||
|
||||
// Aperçu : une capture JPEG gardée dans l'anneau, servie par son id.
|
||||
let id = browser.auto_shot().await;
|
||||
assert!(!id.is_empty());
|
||||
let jpeg = browser.shot(&id).unwrap();
|
||||
assert_eq!(&jpeg[..2], &[0xff, 0xd8], "JPEG");
|
||||
assert_eq!(
|
||||
image::load_from_memory(&jpeg)
|
||||
.unwrap()
|
||||
.to_rgb8()
|
||||
.dimensions(),
|
||||
(1280, 800)
|
||||
);
|
||||
// Fermeture : le profil temporaire part avec le navigateur.
|
||||
assert_eq!(std::fs::read_dir(&profiles).unwrap().count(), 1);
|
||||
browser.shutdown().await;
|
||||
assert_eq!(std::fs::read_dir(&profiles).unwrap().count(), 0);
|
||||
assert!(!browser.running().await);
|
||||
}
|
||||
|
||||
/// Un vrai Chromium, de bout en bout (comme `TestCDPSmoke`) : page locale,
|
||||
/// éléments numérotés, recherche, saisie, clic qui change la page, capture
|
||||
/// quadrillée, aperçu. Seulement avec `LOKI_CU_SMOKE=1` et un navigateur.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn vrai_navigateur() {
|
||||
if std::env::var("LOKI_CU_SMOKE").as_deref() != Ok("1") {
|
||||
eprintln!("LOKI_CU_SMOKE=1 pour piloter un vrai navigateur");
|
||||
return;
|
||||
}
|
||||
let bin = loki_rs::system::chrome_path();
|
||||
if bin.is_empty() {
|
||||
eprintln!("aucun navigateur");
|
||||
return;
|
||||
}
|
||||
let page = r#"<!doctype html><html><head><title>Banc Loki</title></head><body>
|
||||
<h1>Bonjour</h1>
|
||||
<a href="/suite">Voir la suite</a>
|
||||
<label for="nom">Nom</label><input id="nom" value="">
|
||||
<select id="pays"><option>Belgique</option><option>France</option></select>
|
||||
<button onclick="document.title='Cliqué'">Valider</button>
|
||||
<div style="height:3000px"></div><a href="/bas">Tout en bas</a>
|
||||
</body></html>"#;
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = l.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut st, _)) = l.accept().await {
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let n = st.read(&mut buf).await.unwrap_or(0);
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let body = if req.starts_with("GET /suite") {
|
||||
"<html><head><title>Suite</title></head><body><a href=\"/\">Retour</a></body></html>"
|
||||
} else {
|
||||
page
|
||||
};
|
||||
let _ = st
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
});
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let b = Browser::with(&bin, tmp.path().to_path_buf());
|
||||
let ws = tmp.path().to_string_lossy().into_owned();
|
||||
let run = |name: &'static str, args: Value| {
|
||||
let b = &b;
|
||||
let ws = ws.clone();
|
||||
async move {
|
||||
let args: Map<String, Value> = serde_json::from_value(args).unwrap();
|
||||
b.exec(name, &args, true, &ws).await.unwrap()
|
||||
}
|
||||
};
|
||||
let open = run("browser_open", json!({ "url": format!("http://{addr}/") })).await;
|
||||
assert!(open.result.starts_with("# Banc Loki\n"), "{}", open.result);
|
||||
assert!(
|
||||
open.result.contains(r#"link "Voir la suite""#),
|
||||
"{}",
|
||||
open.result
|
||||
);
|
||||
assert!(open.result.contains("↓ encore ~"), "{}", open.result);
|
||||
let find = run("browser_find", json!({ "text": "tout en bas" })).await;
|
||||
assert!(
|
||||
find.result.contains(r#"link "Tout en bas""#),
|
||||
"{}",
|
||||
find.result
|
||||
);
|
||||
let snap = run("browser_snapshot", json!({})).await;
|
||||
let re = Regex::new(r#"\[(\d+)\] textbox "Nom""#).unwrap();
|
||||
let typed = if let Some(c) = re.captures(&snap.result) {
|
||||
run(
|
||||
"browser_type",
|
||||
json!({ "text": "Jean", "ref": c[1].parse::<f64>().unwrap() }),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
let open = run("browser_open", json!({ "url": format!("http://{addr}/") })).await;
|
||||
let c = re.captures(&open.result).expect("champ Nom");
|
||||
run(
|
||||
"browser_type",
|
||||
json!({ "text": "Jean", "ref": c[1].parse::<f64>().unwrap() }),
|
||||
)
|
||||
.await
|
||||
};
|
||||
assert!(
|
||||
typed.result.contains(r#"textbox "Nom = Jean""#),
|
||||
"{}",
|
||||
typed.result
|
||||
);
|
||||
let sel = Regex::new(r#"\[(\d+)\] combobox"#).unwrap();
|
||||
let c = sel.captures(&typed.result).expect("liste");
|
||||
let chosen = run(
|
||||
"browser_type",
|
||||
json!({ "text": "fra", "ref": c[1].parse::<f64>().unwrap() }),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
chosen.result.starts_with("[ok] saisi « fra »"),
|
||||
"{}",
|
||||
chosen.result
|
||||
);
|
||||
// Le lien est hors de la vue depuis le défilement de browser_find.
|
||||
let found = run("browser_find", json!({ "text": "voir la suite" })).await;
|
||||
let link = Regex::new(r#"\[(\d+)\] link "Voir la suite""#).unwrap();
|
||||
let c = link.captures(&found.result).expect("lien");
|
||||
let click = run(
|
||||
"browser_click",
|
||||
json!({ "ref": c[1].parse::<f64>().unwrap() }),
|
||||
)
|
||||
.await;
|
||||
assert!(click.result.contains("# Suite\n"), "{}", click.result);
|
||||
let back = run("browser_open", json!({ "url": "back" })).await;
|
||||
assert!(back.result.starts_with("# Banc Loki"), "{}", back.result);
|
||||
let shot = run("browser_screenshot", json!({})).await;
|
||||
assert!(
|
||||
shot.result.starts_with("[ok] capture prise"),
|
||||
"{}",
|
||||
shot.result
|
||||
);
|
||||
let facts = image_facts(shot.image.as_ref().unwrap());
|
||||
assert_eq!(
|
||||
(facts["w"].clone(), facts["h"].clone()),
|
||||
(json!(1280), json!(800))
|
||||
);
|
||||
assert!(
|
||||
std::fs::read_dir(tmp.path())
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.any(|e| e.file_name().to_string_lossy().starts_with("capture-"))
|
||||
);
|
||||
let id = b.auto_shot().await;
|
||||
assert!(b.shot(&id).is_some());
|
||||
let bad = run("browser_open", json!({ "url": "http://127.0.0.1:9/" })).await;
|
||||
assert!(
|
||||
bad.result
|
||||
.starts_with("[erreur] navigation : chargement de http://127.0.0.1:9/ échoué"),
|
||||
"{}",
|
||||
bad.result
|
||||
);
|
||||
b.shutdown().await;
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Faux Chrome pour les tests du navigateur piloté (lot 44) : un script qui
|
||||
//! annonce le port d'un faux serveur CDP dans DevToolsActivePort, et ce
|
||||
//! serveur, qui répond selon un scénario et note chaque commande reçue
|
||||
//! (comme `parity_browser_test.go`).
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use base64::Engine as _;
|
||||
use futures_util::{SinkExt as _, StreamExt as _};
|
||||
use serde_json::{Map, Value, json};
|
||||
use sha2::Digest as _;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
/// Le faux navigateur : il annonce `port`, range ses arguments dans
|
||||
/// `args`, et attend.
|
||||
pub fn chrome_script(dir: &Path, port: u16, args: &Path) -> PathBuf {
|
||||
let p = dir.join("chrome");
|
||||
let script = format!(
|
||||
r#"#!/bin/sh
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--user-data-dir=*) dir="${{a#--user-data-dir=}}" ;;
|
||||
esac
|
||||
done
|
||||
printf '%s\n/devtools/browser/faux\n' '{port}' > "$dir/DevToolsActivePort"
|
||||
printf '%s\n' "$@" > '{}'
|
||||
exec sleep 600
|
||||
"#,
|
||||
args.display()
|
||||
);
|
||||
std::fs::write(&p, script).unwrap();
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
/// `parityCDPAbbrev`.
|
||||
pub fn abbrev(s: &str) -> String {
|
||||
if s.len() <= 60 {
|
||||
return s.to_string();
|
||||
}
|
||||
let h = sha2::Sha256::digest(s.as_bytes());
|
||||
let head: String = s.chars().take(40).collect();
|
||||
format!("{head}…[{}:{}]", s.len(), hex::encode(&h[..6]))
|
||||
}
|
||||
|
||||
/// `parityCDPCall`.
|
||||
pub fn call_line(method: &str, params: Option<&Value>) -> String {
|
||||
let Some(Value::Object(p)) = params else {
|
||||
return method.to_string();
|
||||
};
|
||||
let mut q = Map::new();
|
||||
for (k, v) in p {
|
||||
let v = match v {
|
||||
Value::String(s) if k == "expression" || k == "source" => json!(abbrev(s)),
|
||||
v => v.clone(),
|
||||
};
|
||||
q.insert(k.clone(), v);
|
||||
}
|
||||
format!("{method} {}", loki_rs::gojson::marshal(&Value::Object(q)))
|
||||
}
|
||||
|
||||
/// La capture unie du faux CDP (`parityPNG`).
|
||||
pub fn png(w: u32, h: u32) -> String {
|
||||
let img = image::RgbaImage::from_pixel(w, h, image::Rgba([40, 80, 120, 255]));
|
||||
let mut out = std::io::Cursor::new(Vec::new());
|
||||
img.write_to(&mut out, image::ImageFormat::Png).unwrap();
|
||||
base64::engine::general_purpose::STANDARD.encode(out.into_inner())
|
||||
}
|
||||
|
||||
pub type Scene = Arc<Mutex<Value>>;
|
||||
pub type Calls = Arc<Mutex<Vec<String>>>;
|
||||
|
||||
/// La réponse du faux CDP à une commande (`reply` du journal Go).
|
||||
pub fn reply(scene: &Value, method: &str, params: &Value) -> Result<Value, String> {
|
||||
if let Some(msg) = scene["errors"].get(method).and_then(Value::as_str) {
|
||||
return Err(msg.to_string());
|
||||
}
|
||||
let val = |v: &Value| {
|
||||
if v == "<absent>" {
|
||||
json!({ "result": { "type": "undefined" } })
|
||||
} else {
|
||||
json!({ "result": { "type": "object", "value": v } })
|
||||
}
|
||||
};
|
||||
match method {
|
||||
"Runtime.evaluate" => {
|
||||
let expr = params["expression"].as_str().unwrap_or("");
|
||||
match expr {
|
||||
"1" => return Ok(val(&json!(1))),
|
||||
"document.readyState" => return Ok(val(&json!("complete"))),
|
||||
"location.href" => return Ok(val(&json!(scene["href"].as_str().unwrap_or("")))),
|
||||
_ => {}
|
||||
}
|
||||
if scene["throw"] == true {
|
||||
return Ok(json!({ "result": { "type": "object" },
|
||||
"exceptionDetails": { "text": "Uncaught TypeError: x is not a function" } }));
|
||||
}
|
||||
let pick = |k: &str| scene.get(k).cloned().unwrap_or(Value::Null);
|
||||
Ok(if expr.contains("window.__lokiRefs=map") {
|
||||
val(&pick("snap"))
|
||||
} else if expr.contains("var pool=ajQueryAll();") {
|
||||
val(&pick("find"))
|
||||
} else if expr.contains("inline:'center'") {
|
||||
val(&pick("center"))
|
||||
} else if expr.contains("return 'TEXT';") {
|
||||
val(&pick("typed"))
|
||||
} else if expr.starts_with("history.") || expr.starts_with("window.scrollBy(") {
|
||||
val(&json!("ok"))
|
||||
} else {
|
||||
val(&json!("<absent>"))
|
||||
})
|
||||
}
|
||||
"Page.navigate" => {
|
||||
let mut r = json!({ "frameId": "F1", "loaderId": "L1" });
|
||||
if let Some(e) = scene["nav_error"].as_str().filter(|e| !e.is_empty()) {
|
||||
r["errorText"] = json!(e);
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
"Page.captureScreenshot" => Ok(json!({ "data": png(1280, 800) })),
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Le faux serveur CDP : `/json` puis la cible « page » en websocket.
|
||||
pub async fn start(scene: Scene, calls: Calls) -> u16 {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut st, _)) = l.accept().await {
|
||||
let (scene, calls) = (scene.clone(), calls.clone());
|
||||
tokio::spawn(async move {
|
||||
let mut head = [0u8; 16];
|
||||
let n = st.peek(&mut head).await.unwrap_or(0);
|
||||
if head[..n].starts_with(b"GET /json") {
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let _ = st.read(&mut buf).await;
|
||||
let body = json!([
|
||||
{ "type": "service_worker", "webSocketDebuggerUrl": format!("ws://127.0.0.1:{port}/devtools/sw/0") },
|
||||
{ "type": "page", "webSocketDebuggerUrl": format!("ws://127.0.0.1:{port}/devtools/page/1") },
|
||||
])
|
||||
.to_string();
|
||||
let _ = st
|
||||
.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
let Ok(mut ws) = tokio_tungstenite::accept_async(st).await else {
|
||||
return;
|
||||
};
|
||||
while let Some(Ok(m)) = ws.next().await {
|
||||
let Message::Text(t) = m else { continue };
|
||||
let v: Value = serde_json::from_str(t.as_str()).unwrap_or_default();
|
||||
let method = v["method"].as_str().unwrap_or("").to_string();
|
||||
calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(call_line(&method, v.get("params")));
|
||||
let sc = scene.lock().unwrap().clone();
|
||||
let out = match reply(&sc, &method, &v["params"]) {
|
||||
Ok(r) => json!({ "id": v["id"], "result": r }),
|
||||
Err(e) => {
|
||||
json!({ "id": v["id"], "error": { "code": -32000, "message": e } })
|
||||
}
|
||||
};
|
||||
if ws
|
||||
.send(Message::Text(out.to_string().into()))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
port
|
||||
}
|
||||
|
||||
/// Dimensions et pixels témoins de l'image relayée (`parityImageFacts`).
|
||||
pub fn image_facts(part: &Value) -> Value {
|
||||
let u = part["image_url"]["url"].as_str().unwrap();
|
||||
let raw = base64::engine::general_purpose::STANDARD
|
||||
.decode(u.strip_prefix("data:image/png;base64,").unwrap())
|
||||
.unwrap();
|
||||
let img = image::load_from_memory(&raw).unwrap().to_rgba8();
|
||||
let (w, h) = img.dimensions();
|
||||
let mut px = Map::new();
|
||||
for (x, y) in [
|
||||
(0, 0),
|
||||
(100, 50),
|
||||
(101, 50),
|
||||
(50, 100),
|
||||
(2, 2),
|
||||
(104, 4),
|
||||
(110, 12),
|
||||
(300, 799),
|
||||
(1279, 700),
|
||||
(4, 204),
|
||||
] {
|
||||
if x >= w || y >= h {
|
||||
continue;
|
||||
}
|
||||
let p = img.get_pixel(x, y).0;
|
||||
px.insert(
|
||||
format!("{x},{y}"),
|
||||
json!(format!("{},{},{},{}", p[0], p[1], p[2], p[3])),
|
||||
);
|
||||
}
|
||||
json!({ "w": w, "h": h, "px": px })
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
//! journalise les chemins demandés.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod fakecdp;
|
||||
pub mod fakego;
|
||||
pub mod parity;
|
||||
pub mod toolsfx;
|
||||
|
||||
@@ -673,6 +673,7 @@ async fn tour_complet_par_le_go() {
|
||||
memory: None,
|
||||
web: Default::default(),
|
||||
jobs: Default::default(),
|
||||
browser: Default::default(),
|
||||
go_version: Default::default(),
|
||||
});
|
||||
let cfg = std::collections::HashMap::new();
|
||||
@@ -2840,3 +2841,120 @@ async fn job_arriere_plan_par_la_facade() {
|
||||
}
|
||||
assert_eq!(jobs.running(), 0);
|
||||
}
|
||||
|
||||
/// Lot 44 : le navigateur piloté est celui de la façade. Un tour qui ouvre
|
||||
/// une page l'exécute chez elle (rien par le Go), l'aperçu est servi par
|
||||
/// /api/chat/cushot, et le Go passe par /internal/facade/v1/browser.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn navigateur_par_la_facade() {
|
||||
use common::fakecdp;
|
||||
let t = tempfile::tempdir().unwrap();
|
||||
let scene: fakecdp::Scene = Arc::new(std::sync::Mutex::new(serde_json::json!({
|
||||
"snap": { "title": "Exemple", "url": "https://exemple.test/", "refs": ["[1] link \"Accueil\""] },
|
||||
"href": "https://exemple.test/",
|
||||
})));
|
||||
let calls: fakecdp::Calls = Default::default();
|
||||
let port = fakecdp::start(scene, calls.clone()).await;
|
||||
let chrome = fakecdp::chrome_script(t.path(), port, &t.path().join("args"));
|
||||
std::fs::create_dir_all(t.path().join("profils")).unwrap();
|
||||
|
||||
// Le tour.
|
||||
let call = sse(&[
|
||||
serde_json::json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"n1","type":"function","function":{"name":"browser_open","arguments":"{\"url\":\"exemple.test\"}"}}]}}]}),
|
||||
serde_json::json!({"choices":[{"delta":{},"finish_reason":"tool_calls"}]}),
|
||||
]);
|
||||
let (url, seen) = fake_engine(vec![call, answer("ouvert")]).await;
|
||||
let browser = Arc::new(loki_rs::browser::Browser::with(
|
||||
&chrome.to_string_lossy(),
|
||||
t.path().join("profils"),
|
||||
));
|
||||
let (svc, go, _stop, _t) = service_with(Some(&url), |s| s.browser = browser.clone()).await;
|
||||
svc.send(serde_json::json!({"message": "ouvre exemple", "cid": "n"}))
|
||||
.await
|
||||
.unwrap();
|
||||
until_done(&svc, 1).await;
|
||||
assert!(go.tool_log.lock().unwrap().is_empty(), "{:?}", go.tool_log);
|
||||
let seen = seen.lock().unwrap().clone();
|
||||
let tool = seen[1].1["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.rfind(|m| m["role"] == "tool")
|
||||
.unwrap()["content"]
|
||||
.clone();
|
||||
assert_eq!(
|
||||
tool,
|
||||
serde_json::json!(
|
||||
"# Exemple\nhttps://exemple.test/\n\n1 élément(s) interactif(s) VISIBLE(s) — clique par numéro avec browser_click(ref) :\n[1] link \"Accueil\""
|
||||
)
|
||||
);
|
||||
assert!(browser.last_shot().is_some(), "aperçu après l'ouverture");
|
||||
assert!(
|
||||
calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|c| c == "Page.navigate {\"url\":\"https://exemple.test\"}")
|
||||
);
|
||||
browser.shutdown().await;
|
||||
|
||||
// L'API du Go et l'aperçu servi, par une façade complète.
|
||||
let t2 = tempfile::tempdir().unwrap();
|
||||
let (facade, _go, _stop2, chat) = fakego::start_with(
|
||||
t2.path().into(),
|
||||
fakego::Opts {
|
||||
chat: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let chat = chat.unwrap();
|
||||
chat.shared
|
||||
.browser
|
||||
.impose(&chrome.to_string_lossy(), t.path().join("profils"));
|
||||
let tok = [(loki_rs::auth::INTERNAL_HEADER, fakego::JETON)];
|
||||
let post = |body: serde_json::Value| {
|
||||
let b = body.to_string();
|
||||
async move {
|
||||
let (code, out) =
|
||||
fakego::send(facade, "POST", "/internal/facade/v1/browser", &tok, &b).await;
|
||||
(
|
||||
code,
|
||||
serde_json::from_slice::<serde_json::Value>(&out).unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
};
|
||||
let (code, out) =
|
||||
post(serde_json::json!({ "op": "tool", "name": "browser_snapshot", "args": {} })).await;
|
||||
assert_eq!(code, 200, "{out}");
|
||||
assert!(
|
||||
out["result"].as_str().unwrap().starts_with("# Exemple\n"),
|
||||
"{out}"
|
||||
);
|
||||
assert_eq!(out["image"], serde_json::Value::Null);
|
||||
let (_, out) = post(serde_json::json!({ "op": "shot" })).await;
|
||||
let id = out["id"].as_str().unwrap().to_string();
|
||||
assert!(!id.is_empty());
|
||||
let (code, out) =
|
||||
fakego::send(facade, "GET", &format!("/api/chat/cushot?id={id}"), &[], "").await;
|
||||
assert_eq!(code, 200);
|
||||
let v: serde_json::Value = serde_json::from_slice(&out).unwrap();
|
||||
assert_eq!(
|
||||
(v["ok"].clone(), v["mime"].clone()),
|
||||
(serde_json::json!(true), serde_json::json!("image/jpeg"))
|
||||
);
|
||||
let (code, out) = fakego::send(facade, "GET", "/api/chat/cushot?id=perdu", &[], "").await;
|
||||
assert_eq!(code, 404);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&out).unwrap(),
|
||||
serde_json::json!({ "ok": false, "error": "capture expirée" })
|
||||
);
|
||||
let (code, _) = post(serde_json::json!({ "op": "outil", "name": "x" })).await;
|
||||
assert_eq!(code, 400);
|
||||
let (code, _) = post(serde_json::json!({ "op": "tool", "name": "browser_voler" })).await;
|
||||
assert_eq!(code, 400);
|
||||
let (code, _) = post(serde_json::json!({ "op": "shutdown" })).await;
|
||||
assert_eq!(code, 200);
|
||||
assert!(!chat.shared.browser.running().await);
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user