diff --git a/src/google/adk/tools/load_web_page.py b/src/google/adk/tools/load_web_page.py index 011f53bdc15..e00b801b160 100644 --- a/src/google/adk/tools/load_web_page.py +++ b/src/google/adk/tools/load_web_page.py @@ -18,6 +18,7 @@ import time from typing import Any +import unicodedata from urllib.parse import ParseResult import requests @@ -239,6 +240,53 @@ def _fetch_content(url: str) -> bytes: ) +# Scripts written without spaces between words. Each of their letters counts as +# a word when deciding whether a line is long enough to keep. +_UNSPACED_SCRIPT_RANGES = ( + (0x0E00, 0x0EFF), # Thai and Lao + (0x1000, 0x109F), # Myanmar + (0x1780, 0x17FF), # Khmer + (0x3040, 0x30FF), # Hiragana and Katakana + (0x3400, 0x4DBF), # CJK Unified Ideographs Extension A + (0x4E00, 0x9FFF), # CJK Unified Ideographs + (0xF900, 0xFAFF), # CJK Compatibility Ideographs +) + + +def _is_unspaced_letter(char: str) -> bool: + """Returns whether `char` is a letter of a script written without spaces.""" + code = ord(char) + return any( + low <= code <= high for low, high in _UNSPACED_SCRIPT_RANGES + ) and unicodedata.category(char).startswith('L') + + +def _count_words(line: str) -> int: + """Counts the words in a line of page text. + + Words are separated by whitespace. In a script written without spaces, such + as Chinese, Japanese or Thai, each letter counts as a word, and the runs of + other characters around those letters count only if they hold a letter or a + digit, so that a full stop such as "。" is not counted as a word. + """ + count = 0 + for token in line.split(): + if not any(_is_unspaced_letter(char) for char in token): + count += 1 + continue + run = '' + for char in token + ' ': + if _is_unspaced_letter(char) or char == ' ': + if any(c.isalnum() for c in run): + count += 1 + run = '' + if char != ' ': + count += 1 + else: + run += char + return count + + def load_web_page(url: str) -> str: """Fetches the content in the url and returns the text in it. @@ -269,4 +317,4 @@ def load_web_page(url: str) -> str: # Split the text into lines, filtering out very short lines # (e.g., single words or short subtitles) - return '\n'.join(line for line in text.splitlines() if len(line.split()) > 3) + return '\n'.join(line for line in text.splitlines() if _count_words(line) > 3) diff --git a/tests/unittests/tools/test_load_web_page.py b/tests/unittests/tools/test_load_web_page.py index 73a0a952040..232bf9139cb 100644 --- a/tests/unittests/tools/test_load_web_page.py +++ b/tests/unittests/tools/test_load_web_page.py @@ -793,3 +793,50 @@ def _send( assert load_web_page('https://example.com') == ( 'Failed to fetch url: https://example.com' ) + + +@pytest.mark.parametrize( + ('html', 'expected'), + [ + ( + ( + '
' + '2024款Model Y补贴后起售价为24.99万元。
' + ), + '2024款Model Y补贴后起售价为24.99万元。', + ), + ( + ( + '' + '今月の注文分は4週間後に納車が始まります。
' + ), + '今月の注文分は4週間後に納車が始まります。', + ), + ], + ids=['chinese', 'japanese'], +) +def test_load_web_page_keeps_lines_of_scripts_written_without_spaces( + monkeypatch, html, expected +): + monkeypatch.setattr( + load_web_page_module, + '_fetch_content', + mock.Mock(return_value=html.encode()), + ) + + assert load_web_page('https://example.com/') == expected + + +@pytest.mark.parametrize( + ('line', 'expected'), + [ + ('This page has enough words to keep.', 7), + ('- Home -', 3), + ('首页', 2), + ('本月下单的车辆预计四周后开始交付。', 16), + ('2024款Model Y价格', 6), + ('ホーム', 3), + ], +) +def test_count_words_counts_each_letter_of_an_unspaced_script(line, expected): + assert load_web_page_module._count_words(line) == expected