fix(tools): keep Chinese and Japanese lines in load_web_page

src/google/adk/tools/load_web_page.py

@@ -18,6 +18,7 @@
 
 import time
 from typing import Any
+import unicodedata
 from urllib.parse import ParseResult
 
 import requests
@@ -239,6 +240,53 @@ def _fetch_content(url: str) -> bytes:
   )
 
 
+# Scripts written without spaces between words. Each of their letters counts as
+# a word when deciding whether a line is long enough to keep.
+_UNSPACED_SCRIPT_RANGES = (
+    (0x0E00, 0x0EFF),  # Thai and Lao
+    (0x1000, 0x109F),  # Myanmar
+    (0x1780, 0x17FF),  # Khmer
+    (0x3040, 0x30FF),  # Hiragana and Katakana
+    (0x3400, 0x4DBF),  # CJK Unified Ideographs Extension A
+    (0x4E00, 0x9FFF),  # CJK Unified Ideographs
+    (0xF900, 0xFAFF),  # CJK Compatibility Ideographs
+)
+
+
+def _is_unspaced_letter(char: str) -> bool:
+  """Returns whether `char` is a letter of a script written without spaces."""
+  code = ord(char)
+  return any(
+      low <= code <= high for low, high in _UNSPACED_SCRIPT_RANGES
+  ) and unicodedata.category(char).startswith('L')
+
+
+def _count_words(line: str) -> int:
+  """Counts the words in a line of page text.
+
+  Words are separated by whitespace. In a script written without spaces, such
+  as Chinese, Japanese or Thai, each letter counts as a word, and the runs of
+  other characters around those letters count only if they hold a letter or a
+  digit, so that a full stop such as "。" is not counted as a word.
+  """
+  count = 0
+  for token in line.split():
+    if not any(_is_unspaced_letter(char) for char in token):
+      count += 1
+      continue
+    run = ''
+    for char in token + ' ':
+      if _is_unspaced_letter(char) or char == ' ':
+        if any(c.isalnum() for c in run):
+          count += 1
+        run = ''
+        if char != ' ':
+          count += 1
+      else:
+        run += char
+  return count
+
+
 def load_web_page(url: str) -> str:
   """Fetches the content in the url and returns the text in it.
 
@@ -269,4 +317,4 @@ def load_web_page(url: str) -> str:
 
   # Split the text into lines, filtering out very short lines
   # (e.g., single words or short subtitles)
-  return '\n'.join(line for line in text.splitlines() if len(line.split()) > 3)
+  return '\n'.join(line for line in text.splitlines() if _count_words(line) > 3)

tests/unittests/tools/test_load_web_page.py

@@ -793,3 +793,50 @@ def _send(
   assert load_web_page('https://example.com') == (
       'Failed to fetch url: https://example.com'
   )
+
+
+@pytest.mark.parametrize(
+    ('html', 'expected'),
+    [
+        (
+            (
+                '<html><body><nav>首页</nav>'
+                '<p>2024款Model Y补贴后起售价为24.99万元。</p></body></html>'
+            ),
+            '2024款Model Y补贴后起售价为24.99万元。',
+        ),
+        (
+            (
+                '<html><body><nav>ホーム</nav>'
+                '<p>今月の注文分は4週間後に納車が始まります。</p></body></html>'
+            ),
+            '今月の注文分は4週間後に納車が始まります。',
+        ),
+    ],
+    ids=['chinese', 'japanese'],
+)
+def test_load_web_page_keeps_lines_of_scripts_written_without_spaces(
+    monkeypatch, html, expected
+):
+  monkeypatch.setattr(
+      load_web_page_module,
+      '_fetch_content',
+      mock.Mock(return_value=html.encode()),
+  )
+
+  assert load_web_page('https://example.com/') == expected
+
+
+@pytest.mark.parametrize(
+    ('line', 'expected'),
+    [
+        ('This page has enough words to keep.', 7),
+        ('- Home -', 3),
+        ('首页', 2),
+        ('本月下单的车辆预计四周后开始交付。', 16),
+        ('2024款Model Y价格', 6),
+        ('ホーム', 3),
+    ],
+)
+def test_count_words_counts_each_letter_of_an_unspaced_script(line, expected):
+  assert load_web_page_module._count_words(line) == expected