Tighten text-click with ambiguity detection and auto-fallback

manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "WebBrain",
-  "version": "1.6.8",
+  "version": "1.7.0",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "permissions": [
     "sidePanel",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "webbrain",
-  "version": "1.6.8",
+  "version": "1.7.0",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "private": true,
   "type": "module",

src/chrome/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "WebBrain",
-  "version": "1.6.8",
+  "version": "1.7.0",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "permissions": [
     "sidePanel",

src/chrome/src/agent/agent.js

@@ -1259,25 +1259,61 @@ export class Agent {
           };
         }
         if (args.text) {
-          // Text-based click: find the first interactive element whose text
-          // contains the given string (case-insensitive). Resolves in JS via
-          // a simple walker over common interactive selectors. Then clicks
-          // via the same robust CDP path.
+          // Text-based click with auto-fallback matching.
+          // When textMatch is not specified (default), tries exact → prefix →
+          // contains in order. At each level, if multiple elements match, an
+          // ambiguity error is returned instead of clicking an arbitrary one.
+          // When textMatch IS specified, only that mode is used.
           const result = await cdpClient.evaluate(tabId, `
             (() => {
               const needle = ${JSON.stringify(args.text.toLowerCase())};
-              const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, [onclick], [data-action]';
+              const explicit = ${JSON.stringify(args.textMatch || '')};
+              const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, label, [onclick], [data-action]';
               const all = Array.from(document.querySelectorAll(sels));
-              // Prefer exact text match, then prefix, then substring.
-              const exact = all.find(el => (el.innerText || el.value || el.ariaLabel || '').trim().toLowerCase() === needle);
-              const prefix = all.find(el => (el.innerText || el.value || el.ariaLabel || '').trim().toLowerCase().startsWith(needle));
-              const sub = all.find(el => (el.innerText || el.value || el.ariaLabel || '').toLowerCase().includes(needle));
-              const el = exact || prefix || sub;
-              if (!el) return { found: false };
+              const normalized = all.map(el => ({
+                el,
+                txt: (el.innerText || el.value || el.ariaLabel || '').trim().toLowerCase(),
+              })).filter(x => !!x.txt);
+
+              function tryMode(mode) {
+                if (mode === 'exact') return normalized.filter(x => x.txt === needle);
+                if (mode === 'prefix') return normalized.filter(x => x.txt.startsWith(needle));
+                if (mode === 'contains') return normalized.filter(x => x.txt.includes(needle));
+                return [];
+              }
+
+              // Determine which modes to try.
+              const modes = explicit ? [explicit] : ['exact', 'prefix', 'contains'];
+              if (explicit && !['exact', 'prefix', 'contains'].includes(explicit)) {
+                return { found: false, error: 'Invalid textMatch. Use exact, prefix, or contains.' };
+              }
+
+              let matches = [];
+              let usedMode = modes[0];
+              for (const m of modes) {
+                matches = tryMode(m);
+                usedMode = m;
+                if (matches.length === 1) break; // unique match — use it
+                if (matches.length > 1) break;   // ambiguous — report it
+                // 0 matches — try next mode
+              }
+
+              if (matches.length === 0) return { found: false, mode: usedMode };
+              if (matches.length > 1) {
+                return {
+                  found: false,
+                  ambiguous: true,
+                  mode: usedMode,
+                  count: matches.length,
+                  candidates: matches.slice(0, 5).map(m => m.txt.slice(0, 80)),
+                };
+              }
+              const el = matches[0].el;
               try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch (e) {}
               const r = el.getBoundingClientRect();
               return {
                 found: true,
+                mode: usedMode,
                 x: r.left + r.width / 2,
                 y: r.top + r.height / 2,
                 tag: el.tagName,
@@ -1287,9 +1323,19 @@ export class Agent {
           `);
           const info = result?.result?.value;
           if (!info?.found) {
+            if (info?.error) {
+              return { success: false, error: info.error };
+            }
+            if (info?.ambiguous) {
+              return {
+                success: false,
+                error: `Ambiguous text match for "${args.text}" (mode=${info.mode}, matches=${info.count}). Use a more specific text, click({index:N}) from get_interactive_elements, or selector/x,y.`,
+                candidates: info.candidates || [],
+              };
+            }
             return {
               success: false,
-              error: `No clickable element found containing text "${args.text}". Try get_interactive_elements to see what's actually on the page, or take a screenshot.`,
+              error: `No clickable element found for text "${args.text}". Try get_interactive_elements to see what's on the page, or use a selector.`,
             };
           }
           // Wait for scroll to settle, then dispatch a real click via CDP.
@@ -1300,6 +1346,7 @@ export class Agent {
           return {
             success: true,
             method: 'cdp-by-text',
+            textMatch: info.mode || (args.textMatch || 'exact'),
             tag: info.tag,
             text: info.text,
             matched: args.text,

src/chrome/src/agent/tools.js

@@ -44,11 +44,12 @@ export const AGENT_TOOLS = [
     type: 'function',
     function: {
       name: 'click',
-      description: 'Click an element. FOUR ways to use it: (1) CSS selector, (2) visible text — `click({text: "Publish release"})` finds the first button/link/clickable element whose text contains the string (case-insensitive), (3) element index from get_interactive_elements, (4) x/y coordinates. PREFER text or index over selectors when possible — selectors are easy to get wrong. Note: jQuery/Playwright pseudo-classes like `:contains()` and `:has-text()` are NOT valid CSS and will fail; use the `text` parameter instead.',
+      description: 'Click an element. FOUR ways to use it: (1) CSS selector, (2) visible text, (3) element index from get_interactive_elements, (4) x/y coordinates. For text clicks, default matching is EXACT and case-insensitive. You can opt into broader matching with `textMatch: "prefix"` or `textMatch: "contains"`. Note: jQuery/Playwright pseudo-classes like `:contains()` and `:has-text()` are NOT valid CSS and will fail; use the `text` parameter instead.',
       parameters: {
         type: 'object',
         properties: {
-          text: { type: 'string', description: 'Visible text to match — finds the first button/link/clickable element whose text contains this string (case-insensitive). Use this for buttons you can see in a screenshot.' },
+          text: { type: 'string', description: 'Visible text to match against clickable elements.' },
+          textMatch: { type: 'string', enum: ['exact', 'prefix', 'contains'], description: 'Text matching mode for `text`. Default is `exact` (safest).' },
           selector: { type: 'string', description: 'CSS selector for the element to click' },
           index: { type: 'number', description: 'Index from get_interactive_elements result' },
           x: { type: 'number', description: 'X coordinate to click' },
@@ -554,7 +555,9 @@ TYPING — read this:
 - If you're filling multiple fields, click each one before typing into it, even if it looks like Tab would work.
 
 CLICKING — read this:
-- For buttons and links you can SEE (in a screenshot or in get_interactive_elements output), the BEST way to click them is by visible text: \`click({text: "Publish release"})\`. This finds the first matching button/link and clicks it. No selector guessing required.
+- For buttons and links you can SEE, click by visible text: \`click({text: "Publish release"})\`. Default matching is EXACT (case-insensitive). If exact fails (no match), the system automatically tries prefix then substring matching — but if multiple elements match at any level, it returns an ambiguity error instead of guessing.
+- If you get an ambiguity error, use a more specific text string, switch to \`click({index: N})\` from \`get_interactive_elements\`, or use a selector.
+- You can explicitly control matching with \`textMatch\`: \`"exact"\` (default), \`"prefix"\`, or \`"contains"\`.
 - Order of preference:
   1. \`click({text: "..."})\` — visible button/link text. Most reliable.
   2. \`click({index: N})\` — index from a get_interactive_elements call MADE THIS SAME TURN.

src/chrome/src/content/content.js

@@ -166,15 +166,46 @@
     }
     if (params.text) {
       const needle = params.text.toLowerCase();
-      const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, [onclick], [data-action]';
+      const explicit = params.textMatch || '';
+      const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, label, [onclick], [data-action]';
       const all = Array.from(document.querySelectorAll(sels));
-      const exact = all.find(e => (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase() === needle);
-      const prefix = all.find(e => (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase().startsWith(needle));
-      const sub = all.find(e => (e.innerText || e.value || e.ariaLabel || '').toLowerCase().includes(needle));
-      el = exact || prefix || sub;
-      if (!el) {
-        return { success: false, error: `No clickable element found containing text "${params.text}"` };
+      const normalized = all.map(e => ({
+        e,
+        txt: (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase(),
+      })).filter(x => !!x.txt);
+
+      function tryMode(mode) {
+        if (mode === 'exact') return normalized.filter(x => x.txt === needle);
+        if (mode === 'prefix') return normalized.filter(x => x.txt.startsWith(needle));
+        if (mode === 'contains') return normalized.filter(x => x.txt.includes(needle));
+        return [];
       }
+
+      const modes = explicit ? [explicit] : ['exact', 'prefix', 'contains'];
+      if (explicit && !['exact', 'prefix', 'contains'].includes(explicit)) {
+        return { success: false, error: `Invalid textMatch "${explicit}". Use exact, prefix, or contains.` };
+      }
+
+      let matches = [];
+      let usedMode = modes[0];
+      for (const m of modes) {
+        matches = tryMode(m);
+        usedMode = m;
+        if (matches.length === 1) break;
+        if (matches.length > 1) break;
+      }
+
+      if (matches.length === 0) {
+        return { success: false, error: `No clickable element found for text "${params.text}"` };
+      }
+      if (matches.length > 1) {
+        return {
+          success: false,
+          error: `Ambiguous text match for "${params.text}" (mode=${usedMode}, matches=${matches.length}).`,
+          candidates: matches.slice(0, 5).map(m => m.txt.slice(0, 80)),
+        };
+      }
+      el = matches[0].e;
     } else if (params.selector) {
       el = document.querySelector(params.selector);
     } else if (params.index != null) {

src/chrome/src/ui/settings.html

@@ -220,7 +220,7 @@
 </head>
 <body>
   <h1>WebBrain Settings</h1>
-  <p class="subtitle">Configure your LLM providers and display preferences · v1.6.8</p>
+  <p class="subtitle">Configure your LLM providers and display preferences · v1.7.0</p>
 
   <h2>Display</h2>
   <div id="display-settings">

src/firefox/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 2,
   "name": "WebBrain",
-  "version": "1.6.8",
+  "version": "1.7.0",
   "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
   "permissions": [
     "activeTab",

src/firefox/src/agent/tools.js

@@ -44,11 +44,12 @@ export const AGENT_TOOLS = [
     type: 'function',
     function: {
       name: 'click',
-      description: 'Click an element. FOUR ways to use it: (1) visible text — `click({text: "Publish release"})` finds the first button/link whose text contains the string (case-insensitive); (2) element index from get_interactive_elements; (3) CSS selector; (4) x/y coordinates. PREFER text or index over selectors. jQuery/Playwright pseudo-classes like `:contains()` and `:has-text()` are NOT valid CSS — use the text parameter instead.',
+      description: 'Click an element. FOUR ways to use it: (1) visible text, (2) element index from get_interactive_elements, (3) CSS selector, (4) x/y coordinates. For text clicks, default matching is EXACT and case-insensitive. You can opt into broader matching with `textMatch: "prefix"` or `textMatch: "contains"`. jQuery/Playwright pseudo-classes like `:contains()` and `:has-text()` are NOT valid CSS — use the text parameter instead.',
       parameters: {
         type: 'object',
         properties: {
-          text: { type: 'string', description: 'Visible text — finds first matching button/link/clickable.' },
+          text: { type: 'string', description: 'Visible text to match against clickable elements.' },
+          textMatch: { type: 'string', enum: ['exact', 'prefix', 'contains'], description: 'Text matching mode for `text`. Default is `exact` (safest).' },
           selector: { type: 'string', description: 'CSS selector for the element to click.' },
           index: { type: 'number', description: 'Index from get_interactive_elements result.' },
           x: { type: 'number', description: 'X coordinate to click.' },
@@ -491,7 +492,9 @@ TYPING — read this:
 - Click each field before typing into it, even if Tab seems like it would work.
 
 CLICKING — read this:
-- For buttons and links you can SEE, the BEST way is to click by visible text: \`click({text: "Publish release"})\`. No selector guessing.
+- For buttons and links you can SEE, click by visible text: \`click({text: "Publish release"})\`. Default matching is EXACT (case-insensitive). If exact fails (no match), the system automatically tries prefix then substring matching — but if multiple elements match at any level, it returns an ambiguity error instead of guessing.
+- If you get an ambiguity error, use a more specific text string, switch to \`click({index: N})\` from \`get_interactive_elements\`, or use a selector.
+- You can explicitly control matching with \`textMatch\`: \`"exact"\` (default), \`"prefix"\`, or \`"contains"\`.
 - Order of preference:
   1. \`click({text: "..."})\` — visible text. Most reliable.
   2. \`click({index: N})\` — index from get_interactive_elements MADE THIS SAME TURN.

src/firefox/src/content/content.js

@@ -280,15 +280,46 @@
     // substring.
     if (params.text) {
       const needle = params.text.toLowerCase();
-      const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, [onclick], [data-action]';
+      const explicit = params.textMatch || '';
+      const sels = 'a, button, [role="button"], [role="link"], [role="tab"], [role="menuitem"], input[type="button"], input[type="submit"], summary, label, [onclick], [data-action]';
       const all = Array.from(document.querySelectorAll(sels));
-      const exact = all.find(e => (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase() === needle);
-      const prefix = all.find(e => (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase().startsWith(needle));
-      const sub = all.find(e => (e.innerText || e.value || e.ariaLabel || '').toLowerCase().includes(needle));
-      el = exact || prefix || sub;
-      if (!el) {
-        return { success: false, error: `No clickable element found containing text "${params.text}"` };
+      const normalized = all.map(e => ({
+        e,
+        txt: (e.innerText || e.value || e.ariaLabel || '').trim().toLowerCase(),
+      })).filter(x => !!x.txt);
+
+      function tryMode(mode) {
+        if (mode === 'exact') return normalized.filter(x => x.txt === needle);
+        if (mode === 'prefix') return normalized.filter(x => x.txt.startsWith(needle));
+        if (mode === 'contains') return normalized.filter(x => x.txt.includes(needle));
+        return [];
       }
+
+      const modes = explicit ? [explicit] : ['exact', 'prefix', 'contains'];
+      if (explicit && !['exact', 'prefix', 'contains'].includes(explicit)) {
+        return { success: false, error: `Invalid textMatch "${explicit}". Use exact, prefix, or contains.` };
+      }
+
+      let matches = [];
+      let usedMode = modes[0];
+      for (const m of modes) {
+        matches = tryMode(m);
+        usedMode = m;
+        if (matches.length === 1) break;
+        if (matches.length > 1) break;
+      }
+
+      if (matches.length === 0) {
+        return { success: false, error: `No clickable element found for text "${params.text}"` };
+      }
+      if (matches.length > 1) {
+        return {
+          success: false,
+          error: `Ambiguous text match for "${params.text}" (mode=${usedMode}, matches=${matches.length}).`,
+          candidates: matches.slice(0, 5).map(m => m.txt.slice(0, 80)),
+        };
+      }
+      el = matches[0].e;
     } else if (params.selector) {
       el = document.querySelector(params.selector);
     } else if (params.index != null) {

src/firefox/src/ui/settings.html

@@ -220,7 +220,7 @@
 </head>
 <body>
   <h1>WebBrain Settings</h1>
-  <p class="subtitle">Configure your LLM providers and display preferences · v1.6.8</p>
+  <p class="subtitle">Configure your LLM providers and display preferences · v1.7.0</p>
 
   <h2>Display</h2>
   <div id="display-settings">