diff --git a/manifest.json b/manifest.json index fa20c4f88..46d542310 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "WebBrain", - "version": "1.6.8", + "version": "1.7.0", "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.", "permissions": [ "sidePanel", diff --git a/package.json b/package.json index a39d85add..05a87032c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webbrain", - "version": "1.6.8", + "version": "1.7.0", "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.", "private": true, "type": "module", diff --git a/src/chrome/manifest.json b/src/chrome/manifest.json index b2d9d778e..4dd017ec9 100644 --- a/src/chrome/manifest.json +++ b/src/chrome/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "WebBrain", - "version": "1.6.8", + "version": "1.7.0", "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.", "permissions": [ "sidePanel", diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 455e75d2d..e2c7b7d1b 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -1259,25 +1259,61 @@ export class Agent { }; } if (args.text) { - // Text-based click: find the first interactive element whose text - // contains the given string (case-insensitive). Resolves in JS via - // a simple walker over common interactive selectors. Then clicks - // via the same robust CDP path. + // Text-based click with auto-fallback matching. + // When textMatch is not specified (default), tries exact → prefix → + // contains in order. At each level, if multiple elements match, an + // ambiguity error is returned instead of clicking an arbitrary one. + // When textMatch IS specified, only that mode is used. const result = await cdpClient.evaluate(tabId, ` (() => { const needle = ${JSON.stringify(args.text.toLowerCase())}; - const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, [onclick], [data-action]'; + const explicit = ${JSON.stringify(args.textMatch || '')}; + const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, label, [onclick], [data-action]'; const all = Array.from(document.querySelectorAll(sels)); - // Prefer exact text match, then prefix, then substring. - const exact = all.find(el => (el.innerText || el.value || el.ariaLabel || '').trim().toLowerCase() === needle); - const prefix = all.find(el => (el.innerText || el.value || el.ariaLabel || '').trim().toLowerCase().startsWith(needle)); - const sub = all.find(el => (el.innerText || el.value || el.ariaLabel || '').toLowerCase().includes(needle)); - const el = exact || prefix || sub; - if (!el) return { found: false }; + const normalized = all.map(el => ({ + el, + txt: (el.innerText || el.value || el.ariaLabel || '').trim().toLowerCase(), + })).filter(x => !!x.txt); + + function tryMode(mode) { + if (mode === 'exact') return normalized.filter(x => x.txt === needle); + if (mode === 'prefix') return normalized.filter(x => x.txt.startsWith(needle)); + if (mode === 'contains') return normalized.filter(x => x.txt.includes(needle)); + return []; + } + + // Determine which modes to try. + const modes = explicit ? [explicit] : ['exact', 'prefix', 'contains']; + if (explicit && !['exact', 'prefix', 'contains'].includes(explicit)) { + return { found: false, error: 'Invalid textMatch. Use exact, prefix, or contains.' }; + } + + let matches = []; + let usedMode = modes[0]; + for (const m of modes) { + matches = tryMode(m); + usedMode = m; + if (matches.length === 1) break; // unique match — use it + if (matches.length > 1) break; // ambiguous — report it + // 0 matches — try next mode + } + + if (matches.length === 0) return { found: false, mode: usedMode }; + if (matches.length > 1) { + return { + found: false, + ambiguous: true, + mode: usedMode, + count: matches.length, + candidates: matches.slice(0, 5).map(m => m.txt.slice(0, 80)), + }; + } + const el = matches[0].el; try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch (e) {} const r = el.getBoundingClientRect(); return { found: true, + mode: usedMode, x: r.left + r.width / 2, y: r.top + r.height / 2, tag: el.tagName, @@ -1287,9 +1323,19 @@ export class Agent { `); const info = result?.result?.value; if (!info?.found) { + if (info?.error) { + return { success: false, error: info.error }; + } + if (info?.ambiguous) { + return { + success: false, + error: `Ambiguous text match for "${args.text}" (mode=${info.mode}, matches=${info.count}). Use a more specific text, click({index:N}) from get_interactive_elements, or selector/x,y.`, + candidates: info.candidates || [], + }; + } return { success: false, - error: `No clickable element found containing text "${args.text}". Try get_interactive_elements to see what's actually on the page, or take a screenshot.`, + error: `No clickable element found for text "${args.text}". Try get_interactive_elements to see what's on the page, or use a selector.`, }; } // Wait for scroll to settle, then dispatch a real click via CDP. @@ -1300,6 +1346,7 @@ export class Agent { return { success: true, method: 'cdp-by-text', + textMatch: info.mode || (args.textMatch || 'exact'), tag: info.tag, text: info.text, matched: args.text, diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index ce81e5e49..cc4d39907 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -44,11 +44,12 @@ export const AGENT_TOOLS = [ type: 'function', function: { name: 'click', - description: 'Click an element. FOUR ways to use it: (1) CSS selector, (2) visible text — `click({text: "Publish release"})` finds the first button/link/clickable element whose text contains the string (case-insensitive), (3) element index from get_interactive_elements, (4) x/y coordinates. PREFER text or index over selectors when possible — selectors are easy to get wrong. Note: jQuery/Playwright pseudo-classes like `:contains()` and `:has-text()` are NOT valid CSS and will fail; use the `text` parameter instead.', + description: 'Click an element. FOUR ways to use it: (1) CSS selector, (2) visible text, (3) element index from get_interactive_elements, (4) x/y coordinates. For text clicks, default matching is EXACT and case-insensitive. You can opt into broader matching with `textMatch: "prefix"` or `textMatch: "contains"`. Note: jQuery/Playwright pseudo-classes like `:contains()` and `:has-text()` are NOT valid CSS and will fail; use the `text` parameter instead.', parameters: { type: 'object', properties: { - text: { type: 'string', description: 'Visible text to match — finds the first button/link/clickable element whose text contains this string (case-insensitive). Use this for buttons you can see in a screenshot.' }, + text: { type: 'string', description: 'Visible text to match against clickable elements.' }, + textMatch: { type: 'string', enum: ['exact', 'prefix', 'contains'], description: 'Text matching mode for `text`. Default is `exact` (safest).' }, selector: { type: 'string', description: 'CSS selector for the element to click' }, index: { type: 'number', description: 'Index from get_interactive_elements result' }, x: { type: 'number', description: 'X coordinate to click' }, @@ -554,7 +555,9 @@ TYPING — read this: - If you're filling multiple fields, click each one before typing into it, even if it looks like Tab would work. CLICKING — read this: -- For buttons and links you can SEE (in a screenshot or in get_interactive_elements output), the BEST way to click them is by visible text: \`click({text: "Publish release"})\`. This finds the first matching button/link and clicks it. No selector guessing required. +- For buttons and links you can SEE, click by visible text: \`click({text: "Publish release"})\`. Default matching is EXACT (case-insensitive). If exact fails (no match), the system automatically tries prefix then substring matching — but if multiple elements match at any level, it returns an ambiguity error instead of guessing. +- If you get an ambiguity error, use a more specific text string, switch to \`click({index: N})\` from \`get_interactive_elements\`, or use a selector. +- You can explicitly control matching with \`textMatch\`: \`"exact"\` (default), \`"prefix"\`, or \`"contains"\`. - Order of preference: 1. \`click({text: "..."})\` — visible button/link text. Most reliable. 2. \`click({index: N})\` — index from a get_interactive_elements call MADE THIS SAME TURN. diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index d21c2b8b0..8ee1c9bf6 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -166,15 +166,46 @@ } if (params.text) { const needle = params.text.toLowerCase(); - const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, [onclick], [data-action]'; + const explicit = params.textMatch || ''; + const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, label, [onclick], [data-action]'; const all = Array.from(document.querySelectorAll(sels)); - const exact = all.find(e => (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase() === needle); - const prefix = all.find(e => (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase().startsWith(needle)); - const sub = all.find(e => (e.innerText || e.value || e.ariaLabel || '').toLowerCase().includes(needle)); - el = exact || prefix || sub; - if (!el) { - return { success: false, error: `No clickable element found containing text "${params.text}"` }; + const normalized = all.map(e => ({ + e, + txt: (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase(), + })).filter(x => !!x.txt); + + function tryMode(mode) { + if (mode === 'exact') return normalized.filter(x => x.txt === needle); + if (mode === 'prefix') return normalized.filter(x => x.txt.startsWith(needle)); + if (mode === 'contains') return normalized.filter(x => x.txt.includes(needle)); + return []; } + + const modes = explicit ? [explicit] : ['exact', 'prefix', 'contains']; + if (explicit && !['exact', 'prefix', 'contains'].includes(explicit)) { + return { success: false, error: `Invalid textMatch "${explicit}". Use exact, prefix, or contains.` }; + } + + let matches = []; + let usedMode = modes[0]; + for (const m of modes) { + matches = tryMode(m); + usedMode = m; + if (matches.length === 1) break; + if (matches.length > 1) break; + } + + if (matches.length === 0) { + return { success: false, error: `No clickable element found for text "${params.text}"` }; + } + if (matches.length > 1) { + return { + success: false, + error: `Ambiguous text match for "${params.text}" (mode=${usedMode}, matches=${matches.length}).`, + candidates: matches.slice(0, 5).map(m => m.txt.slice(0, 80)), + }; + } + el = matches[0].e; } else if (params.selector) { el = document.querySelector(params.selector); } else if (params.index != null) { diff --git a/src/chrome/src/ui/settings.html b/src/chrome/src/ui/settings.html index e95056385..52ed93108 100644 --- a/src/chrome/src/ui/settings.html +++ b/src/chrome/src/ui/settings.html @@ -220,7 +220,7 @@
Configure your LLM providers and display preferences · v1.6.8
+Configure your LLM providers and display preferences · v1.7.0
Configure your LLM providers and display preferences · v1.6.8
+Configure your LLM providers and display preferences · v1.7.0