diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 6d05ca19f..8aa8e5bad 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -230,13 +230,25 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, prepare single term return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_tsquery_term(word: str) -> str: + """Render one relaxed word as a tsquery-safe prefix expression. + + Mirrors the SQLite renderer: a word token can contain an apostrophe, and + tsquery reads that as lexeme-quoting syntax rather than text. Quoting the + lexeme and doubling any interior quote keeps it literal. + """ + if "'" in word: + return "'{}':*".format(word.replace("'", "''")) + return f"{word}:*" + @staticmethod def _relaxed_tsquery_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed tsquery expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " | ".join(f"{word}:*" for word in words) + return " | ".join(PostgresSearchRepository._relaxed_tsquery_term(word) for word in words) def _prepare_boolean_query(self, query: str) -> str: """Convert Boolean query to tsquery format. diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 8544bdb47..d73d70c0c 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -1,6 +1,7 @@ """Shared full-text query preparation rules.""" import re +import unicodedata # Interrogative/function words contribute lexical noise when a strict # full-text query is relaxed: "when OR did OR a" matches loud wrong documents @@ -25,8 +26,179 @@ r"\uff65-\uff9f" # Halfwidth Katakana r"]" ) -RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+") RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" +# Written inside a word (Persian U+200C, Indic conjuncts) rather than between words. +# Format characters (Unicode Cf) are invisible and, with one exception, sit +# inside a word: soft hyphen from copied text, the Persian and Indic joiners, +# bidi marks, the Mongolian vowel separator, word joiners. Counting any of them +# as a separator splits one word into several tokens, which is the direction +# that defeats the guards below — grouping can only lower a token count, never +# inflate it past them. +# +# U+200B is the exception: zero-width space marks word *boundaries* in Thai and +# Khmer, so it must keep splitting, or a whole phrase collapses into one token +# and relaxation switches off for the one form of those scripts that reaches +# the guard at all. +RELAXATION_WORD_SEPARATOR_FORMATS = "\u200b" +# Of the characters kept inside a token, only these two are orthography: the +# Persian and Indic joiners are written in the text and therefore sit in the +# index too, so a relaxed term must keep them or it stops matching. Every other +# format character is a rendering artifact — a soft hyphen from a paginated +# document, a bidi hint, a stray BOM — absent from the stored note, so carrying +# it into the term is what stops the term matching. +RELAXATION_ORTHOGRAPHIC_JOINERS = "\u200c\u200d" +# Punctuation written inside a word, joined only between two letters. This is +# the whole of UAX #29 MidLetter, MidNumLet and Single_Quote — plus U+05F3, which +# the standard classes as a letter — minus two families that carry structure in +# this project rather than inside words: +# +# colons U+003A U+FE13 U+FE55 U+FF1A "tag:example" is query syntax +# full stops U+002E U+2024 U+FE52 U+FF0E permalinks and file names +# +# Joining across those would merge a qualifier with its value, or a name with its +# extension, into one term. Everything else in the class is here, so a new +# member is a change to the standard rather than an oversight. +RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2018\u2019\uff07\u00b7\u0387\u055f\u05f3\u05f4\u2027" + + +def _is_word_internal_format(char: str) -> bool: + """Whether an invisible format character belongs to the word around it.""" + return unicodedata.category(char) == "Cf" and char not in RELAXATION_WORD_SEPARATOR_FORMATS + + +def _strip_trailing_formats(token: str) -> str: + """Drop format characters left at a token's end, where they separate rather than join.""" + # Indexed rather than sliced: trimming one character at a time copies the + # shrinking token at every step, which is quadratic in a run of trailing + # format characters. + end = len(token) + while end and _is_word_internal_format(token[end - 1]): + end -= 1 + return token[:end] + + +def _is_attached(char: str) -> bool: + """Whether a character hangs off the one before it rather than standing alone.""" + return _is_word_internal_format(char) or unicodedata.category(char).startswith("M") + + +def _base_before(current: list[str]) -> bool: + """Whether the token so far ends in a letter, looking past what hangs off it. + + Pointed Hebrew and decomposed Latin put a mark between the letter and the + punctuation, so reading only the last character sees the mark and splits a + word the joining rule is meant to keep whole. + """ + for char in reversed(current): + if _is_attached(char): + continue + return char.isalpha() + return False + + +def _base_after(text: str, index: int) -> bool: + """Whether a letter follows the punctuation, looking past what hangs off it.""" + # Indexed rather than sliced: a slice copies the rest of the query at every + # joiner, which makes tokenizing a long query quadratic in its length. + for position in range(index + 1, len(text)): + char = text[position] + if _is_attached(char): + continue + return char.isalpha() + return False + + +def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: + """Whether a non-alphanumeric character belongs to the word being read. + + Combining marks, invisible word-internal format characters, and apostrophes are written inside + a word but are not alphanumeric, so a naive scan treats them as separators + and splits one orthographic word into several tokens. + + An apostrophe counts only between two letters. That keeps "п’ять" whole while + leaving "SPEC 16's" split, so the digit stays a token of its own and the + numeric-identifier guard still rejects the query. + """ + char = text[index] + if _is_word_internal_format(char) or unicodedata.category(char).startswith("M"): + return True + if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: + return _base_before(current) and _base_after(text, index) + return False + + +def relaxation_word_tokens(text: str) -> list[str]: + """Split text into word tokens for the relaxation eligibility guards. + + A token is a run of alphanumeric characters together with the combining + marks, join controls, and apostrophes written inside it. Counting this way matters because + an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, + Armenian, and Georgian queries, so the three-token guard below rejected every + one of them and the hybrid FTS branch silently contributed nothing. + + Counting characters that live inside a word as separators is just as wrong in + the other direction: it cuts abugidas (Devanagari, Thai), decomposed text, + and Persian or Indic words joined by U+200C/U+200D into fragments. One word + then looks like several tokens, clears the three-token guard, and relaxes + into a broad OR of fragments — the opposite of what the guard is for. + + Scripts normally written without spaces between words — Thai, Lao, Khmer — + are counted, but a whole phrase arrives as a single token and so never + reaches the three-token guard. They are not in RELAXATION_CJK_PATTERN either, + so nothing relaxes for them. Fixing that needs real word segmentation. + """ + tokens: list[str] = [] + current: list[str] = [] + + def flush() -> None: + # A trailing format character is word-internal by definition, so a token + # that ends in one is really a word followed by a separator. + token = _strip_trailing_formats("".join(current)) + if token: + tokens.append(token) + current.clear() + + for index, char in enumerate(text): + # A leading mark, join control, or apostrophe has no base character to + # attach to, so it cannot open a token; that keeps stray punctuation from + # forming fragment-only terms. + if char.isalnum() or (current and _is_token_continuation(text, index, current)): + current.append(char) + elif current: + flush() + flush() + return tokens + + +def _token_core(token: str) -> str: + """The token without the characters that only ever attach to another one. + + Combining marks and format characters are kept inside a token so a word is + counted once, but they must not disguise what the token *is*: a keycap digit + is not `isnumeric()` as a whole string, which would walk an identifier-like + query straight past the numeric guard. + """ + return "".join( + char + for char in token + if not _is_word_internal_format(char) and not unicodedata.category(char).startswith("M") + ) + + +def _is_numeric_token(token: str) -> bool: + """Whether a token is a bare number, and so identifier-like rather than a word. + + Classified by Unicode category, not by `isdigit()`/`isnumeric()`. Both of + those answer True for Han numerals, which are category Lo — letters, and + ordinary content words in CJK prose. Rejecting them would switch relaxation + off for the queries #1022 turned it on for. + + Everything in category N is a number character: ASCII and Arabic-Indic + digits (Nd), Roman numerals (Nl), vulgar fractions (No). A token made only + of those is the "SPEC 16" shape the guard exists to catch, in any script. + """ + core = _token_core(token) + return bool(core) and all(unicodedata.category(char).startswith("N") for char in core) def _dedupe_relaxation_words(words: list[str]) -> list[str]: @@ -48,6 +220,35 @@ def _split_relaxation_words(search_text: str) -> list[str]: return [word for word in words if word] +def _relaxation_term_variants(word: str) -> list[str]: + """Every form of a word that could match how the note happens to be stored. + + A format character is invisible, so the same word may be stored with it or + without it, and the two index differently: a note holding "foo\u00adbar" is + indexed as "foo" and "bar", one holding "foobar" as a single token. Neither + term matches the other note, so both forms are emitted and the OR that + relaxation already builds covers whichever the note actually has. + + Orthographic joiners are not stripped: they are written in the text, so the + stored form has them and the cleaned variant would only add noise. + """ + cleaned = "".join( + char + for char in word + if char in RELAXATION_ORTHOGRAPHIC_JOINERS or not _is_word_internal_format(char) + ) + if not cleaned: + return [] + return [cleaned] if cleaned == word else [cleaned, word] + + +def _emit_relaxation_terms(words: list[str]) -> list[str]: + """Expand the words into backend-ready terms, then drop duplicates.""" + return _dedupe_relaxation_words( + [variant for word in words for variant in _relaxation_term_variants(word)] + ) + + def relaxed_query_words(search_text: str | None) -> list[str] | None: """Return content-bearing words for OR-relaxing a strict full-text query. @@ -57,13 +258,16 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: - empty / quoted / explicit-boolean queries (user intent is not second-guessed); - - fewer than three alphanumeric tokens (short queries like "New Feature" + - fewer than three word tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize - to 1.0 and can outrank the vector result the user wanted); + to 1.0 and can outrank the vector result the user wanted). Tokens are + counted with relaxation_word_tokens, so scripts other than Latin reach the + same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because - the ASCII token gate would otherwise suppress the fallback entirely; - - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries - over-broaden and create false positives under OR. + they are not whitespace-delimited the way the token guard assumes; + - any numeric token ("root note 1", "SPEC 16", "SPEC Ⅻ") — identifier-like + queries over-broaden and create false positives under OR. Numeric-ness is + Unicode-wide, so Nl/No characters such as Ⅻ and ½ are caught too. """ if not search_text: return None @@ -77,22 +281,22 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) if has_cjk_term: - if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): + if len(cjk_words) < 2 or any(_is_numeric_token(word) for word in cjk_words): return None pruned_words = [ word for word in cjk_words if word.isalnum() and word.lower() not in RELAXATION_STOPWORDS ] - relaxed_words = _dedupe_relaxation_words(pruned_words) + relaxed_words = _emit_relaxation_terms(pruned_words) # Trigger: punctuation/stopword pruning or deduplication leaves only one term. # Why: the raw whitespace count can make an identifier-like mixed query # appear multi-term even though only one backend-safe CJK prefix remains. # Outcome: preserve the short-query guard after pruning to avoid a broad retry. return relaxed_words if len(relaxed_words) >= 2 else None - tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower()) - if len(tokens) < 3 or any(token.isdigit() for token in tokens): + tokens = relaxation_word_tokens(stripped.lower()) + if len(tokens) < 3 or any(_is_numeric_token(token) for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] - return _dedupe_relaxation_words(pruned_words or tokens) or None + return _emit_relaxation_terms(pruned_words or tokens) or None diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 91bcae435..9dced81f9 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -397,13 +397,26 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, use the single term preparation logic return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_fts_term(word: str) -> str: + """Render one relaxed word as an FTS5-safe prefix expression. + + A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated + bare it is FTS5 syntax, not text: the whole expression fails to parse, the + caller swallows the syntax error, and the relaxed retry returns nothing — + the exact silent-empty-FTS failure this fallback exists to prevent. + """ + if "'" in word or '"' in word: + return '"{}"*'.format(word.replace('"', '""')) + return f"{word}*" + @staticmethod def _relaxed_fts_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed FTS5 expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " OR ".join(f"{word}*" for word in words) + return " OR ".join(SQLiteSearchRepository._relaxed_fts_term(word) for word in words) @override async def semantic_effectively_enabled(self) -> bool: diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9a7181d8b..0a21611a7 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -34,3 +34,303 @@ def test_relaxed_query_words_supports_whitespace_separated_cjk_scripts( def test_relaxed_query_words_preserves_short_query_guard_after_cjk_pruning(query: str) -> None: """Unsafe, duplicate, or stopword terms cannot pad a one-term CJK relaxation.""" assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("как отозвать выданный доступ", ["как", "отозвать", "выданный", "доступ"]), + ("як відкликати виданий доступ", ["як", "відкликати", "виданий", "доступ"]), + ("πώς να ανακαλέσετε πρόσβαση", ["πώς", "να", "ανακαλέσετε", "πρόσβαση"]), + ("כיצד לבטל גישה שניתנה", ["כיצד", "לבטל", "גישה", "שניתנה"]), + ("كيف تلغي الوصول الممنوح", ["كيف", "تلغي", "الوصول", "الممنوح"]), + ("ինչպես չեղարկել տրված մուտքը", ["ինչպես", "չեղարկել", "տրված", "մուտքը"]), + ], +) +def test_relaxed_query_words_supports_non_latin_alphabetic_scripts( + query: str, + expected: list[str], +) -> None: + """Non-Latin alphabetic queries reach the same guard as Latin ones. + + An ASCII-only token pattern found zero tokens in these queries, so the + three-token guard rejected every one of them and the hybrid FTS branch + contributed nothing — hybrid search silently became vector-only. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("पहुंच कैसे रद्द करें", ["पहुंच", "कैसे", "रद्द", "करें"]), # Devanagari + ("วิธี เพิกถอน การเข้าถึง", ["วิธี", "เพิกถอน", "การเข้าถึง"]), # Thai + ("como revogar acesso concedido", ["como", "revogar", "acesso", "concedido"]), + ], +) +def test_relaxed_query_words_keeps_combining_marks_with_their_base_character( + query: str, + expected: list[str], +) -> None: + """Vowel signs and diacritics stay inside the word they attach to. + + Combining marks are not alphanumeric, so treating them as separators splits + one abugida word into syllable fragments. The token count then inflates past + the three-token guard and relaxation ORs those fragments together. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "अंतर्राष्ट्रीयकरण", # one Devanagari word: 7 fragments if marks split it + "การเข้าถึง", # one Thai word + "pre\u0301sentation", # one word, NFD-decomposed acute accent + ], +) +def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str) -> None: + """A single word stays one token, so the short-query guard still rejects it.""" + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + "query", + [ + "отозвать доступ", # fewer than three tokens + "спека 16 доступ", # pure-digit token + '"точная фраза"', # quoted: user intent is explicit + "доступ OR токен", # explicit boolean: user intent is explicit + ], +) +def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None: + """Non-Latin queries gain no exemption from the short-query and identifier guards.""" + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("می‌روم خانه", None), # two Persian words, one joined by ZWNJ + ("نمی‌خواهم دسترسی را لغو", ["نمی‌خواهم", "دسترسی", "را", "لغو"]), + ("क‍ष विशेष पहुंच", ["क‍ष", "विशेष", "पहुंच"]), # explicit ZWJ conjunct + ], +) +def test_relaxed_query_words_keeps_join_controls_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """U+200C/U+200D are written inside a word, so they must not split its token. + + Splitting on them inflates the token count: a two-word Persian query looks + like three tokens, clears the three-token guard, and relaxes into fragments. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "SPEC Ⅻ design", # Nl: Roman numeral twelve + "spec ½ design", # No: vulgar fraction one half + "٣ ٤ ٥", # Arabic-Indic digits + ], +) +def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None: + """The identifier guard classifies numbers Unicode-wide, not just as ASCII digits. + + `isdigit()` is false for Nl/No characters, so admitting every alphanumeric + character would let identifier-like queries slip past the numeric guard. + """ + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("п’ять проектів", None), # two Ukrainian words, U+2019 + ("об'єкт доступу", None), # two Ukrainian words, ASCII apostrophe + ( + "скасувати п’ять виданих об'єктів", + ["скасувати", "п’ять", "виданих", "об'єктів"], + ), + ], +) +def test_relaxed_query_words_keeps_apostrophes_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """A word-internal apostrophe must not split one word into several tokens. + + Splitting on it turned a two-word Ukrainian query into three tokens, which + cleared the three-token guard and relaxed into one-letter fragments. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_apostrophe_does_not_shield_numeric_tokens() -> None: + """An apostrophe joins letters only, so a digit stays a token of its own. + + Were `16's` read as one token it would not be numeric, and the query would + escape the identifier guard that rejects `SPEC 16 design`. + """ + assert relaxed_query_words("SPEC 16's design") is None + + +def test_relaxed_query_words_keeps_ascii_contractions_whole() -> None: + """ASCII contractions become one token instead of a word plus a stray letter. + + This is the one place where relaxed terms differ from the previous ASCII + behaviour. It only ever lowers the token count, so no query that the guards + used to reject can start relaxing because of it. + """ + assert relaxed_query_words("don't touch this") == ["don't", "touch"] + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("数据 三 分析", ["数据", "三", "分析"]), + ("日本 十 経済 統計", ["日本", "十", "経済", "統計"]), + ("データ 二 分析 結果", ["データ", "二", "分析", "結果"]), + ], +) +def test_relaxed_query_words_treats_han_numerals_as_content_words( + query: str, + expected: list[str], +) -> None: + """The CJK guard stays on `isdigit()`, so a numeral word does not veto relaxation. + + 143 characters in U+3000–U+9FFF are `isnumeric()` without being `isdigit()`. + Classifying them as identifiers would reject ordinary CJK prose and switch + relaxation back off for the queries it was turned on for. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "SPEC 16 设计", # ASCII digits + "SPEC Ⅻ 设计", # Roman numeral: a number character, not a Han word + "SPEC ½ 设计", # vulgar fraction + ], +) +def test_relaxed_query_words_rejects_number_characters_in_cjk_queries(query: str) -> None: + """Adding a CJK term must not smuggle an identifier past the numeric guard. + + Han numerals are category Lo — letters — so they stay content words, while + every category-N character is caught in both branches alike. + """ + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + "query", + [ + "SPEC 1️⃣ design", # keycap digit: digit plus VS16 plus enclosing keycap + "spec 1́ design", # digit carrying a combining acute + ], +) +def test_relaxed_query_words_sees_numbers_through_combining_marks(query: str) -> None: + """A mark attached to a digit must not disguise it from the numeric guard. + + Marks stay inside the token so the word is counted once, but classification + looks at the token without them — otherwise a decorated digit walks an + identifier-like query straight past the guard. + """ + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("пере­вод доступа", None), # soft hyphen from copied formatted text + ("сло⁠во доступа", None), # word joiner + ("слово доступа", None), # zero-width no-break space + ("сло᠎во доступа", None), # Mongolian vowel separator + ("сло‏во доступа", None), # right-to-left mark + ], +) +def test_relaxed_query_words_ignores_word_internal_format_characters( + query: str, + expected: list[str] | None, +) -> None: + """Invisible format characters inside a word must not split its token. + + Text pasted from formatted documents carries them, and splitting there + inflates the token count exactly as the join-control case did. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_treats_zero_width_space_as_a_word_boundary() -> None: + """U+200B separates words in Thai and Khmer, so it must keep splitting. + + Grouping it into the word would collapse a whole phrase into one token and + switch relaxation off for the one form of those scripts that reaches the + guard at all. + """ + assert relaxed_query_words("ฉัน​จะลอง​ชำระเงิน") == [ + "ฉัน", + "จะลอง", + "ชำระเงิน", + ] + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("ג׳ון סמית כתב", ["ג׳ון", "סמית", "כתב"]), # geresh: foreign sounds + ("ר״ת של המשפט", ["ר״ת", "של", "המשפט"]), # gershayim: acronym + ("col·lecció de dades", ["col·lecció", "de", "dades"]), # Catalan middle dot + ], +) +def test_relaxed_query_words_keeps_letter_joining_punctuation( + query: str, + expected: list[str], +) -> None: + """The apostrophe rule covers every UAX #29 MidLetter character taken here. + + Hebrew writes geresh and gershayim inside words constantly, so splitting on + them turns a three-word query into fragments — the same failure the + apostrophe case had, in a script this change exists to support. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "spec 1.2 design", # full stop: UAX #29 would join "1.2" into one token + "spec 1:2 design", # colon + ], +) +def test_relaxed_query_words_splits_on_structural_punctuation(query: str) -> None: + """Colon and full stop are deliberately outside the MidLetter set taken here. + + UAX #29 joins digits across a full stop, and "1.2" is not a category-N token, + so it would slip past the numeric guard. Both also carry structure in + permalinks, paths and version strings. + """ + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("גּ׳ון סמית כתב", ["גּ׳ון", "סמית", "כתב"]), # dagesh between letter and geresh + ("pré d'accord test", ["pré", "d'accord", "test"]), # NFD before apostrophe + ], +) +def test_relaxed_query_words_finds_the_base_letter_through_marks( + query: str, + expected: list[str], +) -> None: + """The joining rule looks for a letter, not for the last character. + + Pointed Hebrew and decomposed Latin put a mark between the letter and the + punctuation, so reading only the character before it sees the mark and + splits a word the rule is meant to keep whole. + """ + assert relaxed_query_words(query) == expected diff --git a/tests/repository/test_search_relaxation_unicode_classes.py b/tests/repository/test_search_relaxation_unicode_classes.py new file mode 100644 index 000000000..5fb725b2a --- /dev/null +++ b/tests/repository/test_search_relaxation_unicode_classes.py @@ -0,0 +1,298 @@ +"""Exhaustive sweeps over the Unicode classes the relaxation guards depend on. + +The case-based tests next door pin individual queries. These pin the rules those +cases are instances of, by walking every character in the class rather than the +ones review happened to surface: a soft hyphen, a Mongolian vowel separator and +a keycap digit were each found one at a time, and each was one member of a class +already covered here. + +The rules match Unicode word segmentation (UAX #29) — WB4 ignores format and +combining characters inside a word, WB6/WB7 keep an apostrophe between letters — +with two deliberate departures, both pinned below: U+200B splits, because it +marks word boundaries in Thai and Khmer, and Han numerals stay content words +rather than identifiers. +""" + +import time +import unicodedata + +import pytest + +from basic_memory.repository.search_query import ( + RELAXATION_WORD_INTERNAL_PUNCTUATION, + RELAXATION_WORD_SEPARATOR_FORMATS, + relaxation_word_tokens, + relaxed_query_words, +) + + +def _characters_in_categories(*categories: str) -> list[str]: + """Every assigned code point in the given general categories.""" + wanted = set(categories) + return [ + char + for code_point in range(0x110000) + if unicodedata.category(char := chr(code_point)) in wanted + ] + + +FORMAT_CHARACTERS = _characters_in_categories("Cf") +COMBINING_MARKS = _characters_in_categories("Mn", "Mc", "Me") +NUMBER_CHARACTERS = _characters_in_categories("Nd", "Nl", "No") +PUNCTUATION_CHARACTERS = _characters_in_categories("Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po") + +# Numerals written as letters (category Lo). Unicode gives them a numeric value, +# so isdigit()/isnumeric() answer True, but they are ordinary words in CJK prose. +HAN_NUMERALS = ["三", "四", "五", "十", "百", "千", "万", "億", "零"] + + +def _describe(characters: list[str], limit: int = 8) -> str: + """Render failing characters as code points, since most are invisible.""" + shown = " ".join(f"U+{ord(char):04X}" for char in characters[:limit]) + return f"{len(characters)}: {shown}{' …' if len(characters) > limit else ''}" + + +def test_every_format_character_stays_inside_the_word() -> None: + """No format character may split a word, apart from the declared separators. + + Splitting inflates the token count, which is the direction that walks a + query past the three-token guard and relaxes it into fragments. + """ + splitting = [ + char + for char in FORMAT_CHARACTERS + if char not in RELAXATION_WORD_SEPARATOR_FORMATS + and len(relaxation_word_tokens(f"сло{char}во доступа")) != 2 + ] + assert not splitting, f"format characters that split a word — {_describe(splitting)}" + + +def test_zero_width_space_stays_a_word_separator() -> None: + """U+200B marks word boundaries in Thai and Khmer, so it must keep splitting. + + Grouping it into the word would collapse a whole phrase into one token and + switch relaxation off for the one form of those scripts that reaches the + guard at all. + + Written as a literal rather than read from the constant: a test parametrized + over the exception list disappears when the list is emptied, which is exactly + the change it exists to catch. + """ + assert len(relaxation_word_tokens("сло\u200bво доступа")) == 3 + + +def test_zero_width_space_is_the_only_declared_separator() -> None: + """The sweep above skips whatever this constant holds, so its contents are load-bearing. + + Adding a character here silently removes it from that sweep, so the addition + has to be a deliberate edit here rather than a side effect elsewhere. + """ + assert RELAXATION_WORD_SEPARATOR_FORMATS == "\u200b" + + +def test_no_combining_mark_splits_a_word() -> None: + """Marks attach to the character before them, so they cannot end a token. + + Counting them as separators cuts abugidas and decomposed text into syllable + fragments — one word then looks like several tokens. + """ + splitting = [ + char for char in COMBINING_MARKS if len(relaxation_word_tokens(f"сло{char}во доступа")) != 2 + ] + assert not splitting, f"combining marks that split a word — {_describe(splitting)}" + + +def test_every_number_character_is_caught_by_the_identifier_guard() -> None: + """A bare number term makes a query identifier-like, in any script. + + The guard exists for "SPEC 16"; Roman numerals, vulgar fractions and + non-ASCII digits are the same shape and must not slip through it. + """ + admitted = [ + char for char in NUMBER_CHARACTERS if relaxed_query_words(f"spec {char} design") is not None + ] + assert not admitted, f"number characters that cleared the guard — {_describe(admitted)}" + + +@pytest.mark.parametrize("numeral", HAN_NUMERALS) +def test_han_numerals_stay_content_words(numeral: str) -> None: + """Han numerals are letters (category Lo) and ordinary words in CJK prose. + + Classifying them as identifiers would reject "数据 三 分析" and switch + relaxation off for the queries it was turned on for. + """ + assert unicodedata.category(numeral) == "Lo" + assert relaxed_query_words(f"数据 {numeral} 分析") == ["数据", numeral, "分析"] + + +def test_only_the_declared_punctuation_joins_a_word() -> None: + """Nothing else in the punctuation classes may join two letters. + + The joining set is a chosen subset of UAX #29 MidLetter, so it has to stay a + subset: any other punctuation that started joining would merge two words into + one term and quietly change what the backend searches for. + """ + joining = [ + char + for char in PUNCTUATION_CHARACTERS + if char not in RELAXATION_WORD_INTERNAL_PUNCTUATION + and len(relaxation_word_tokens(f"сло{char}во доступа")) != 3 + ] + assert not joining, f"punctuation that joined a word — {_describe(joining)}" + + +def test_declared_punctuation_is_exactly_the_chosen_midletter_subset() -> None: + """Pin the set itself: the sweep above skips whatever it holds. + + Removing a character silently drops it from every sweep here, and adding one + silently exempts it, so both have to be a deliberate edit rather than a side + effect. The named cases in test_search_relaxation.py pin what each is for. + """ + assert ( + RELAXATION_WORD_INTERNAL_PUNCTUATION + == "'\u2018\u2019\uff07\u00b7\u0387\u055f\u05f3\u05f4\u2027" + ) + + +def test_declared_punctuation_joins_letters_only() -> None: + """Each joiner must join letters, and none may join digits. + + Joining digits is what makes the exclusions below necessary: a term like + "1.2" is not a category-N token, so it would walk an identifier-like query + past the numeric guard. + """ + not_joining = [ + char + for char in RELAXATION_WORD_INTERNAL_PUNCTUATION + if len(relaxation_word_tokens(f"сло{char}во доступа")) != 2 + ] + assert not not_joining, f"declared joiners that split letters — {_describe(not_joining)}" + + joining_digits = [ + char + for char in RELAXATION_WORD_INTERNAL_PUNCTUATION + if relaxed_query_words(f"spec 1{char}2 design") is not None + ] + assert not joining_digits, ( + f"declared joiners that shielded a digit — {_describe(joining_digits)}" + ) + + +@pytest.mark.parametrize("structural", [":", "."]) +def test_colon_and_full_stop_split_although_uax29_joins_them(structural: str) -> None: + """The two deliberate departures from MidLetter, written as literals. + + UAX #29 returns "1.2" as a single token. Such a token is not category N, so + it would slip past the numeric guard that rejects "SPEC 16". Both characters + also carry structure here — permalinks, paths, version strings — so both keep + splitting, and a query built on them stays ineligible for relaxation. + """ + assert len(relaxation_word_tokens(f"сло{structural}во доступа")) == 3 + assert relaxed_query_words(f"spec 1{structural}2 design") is None + + +# UAX #29 Word_Break values for the punctuation that joins words, transcribed +# from the standard. Python's unicodedata does not expose the property, so the +# class is pinned here as data: the partition below then has to account for every +# member, and a character cannot be forgotten, only deliberately excluded. +UAX29_WORD_JOINING_PUNCTUATION = { + "MidLetter": ":··՟״‧︓﹕：", + "MidNumLet": ".‘’․﹒＇．", + "Single_Quote": "'", +} +# Excluded on purpose: these carry structure in this project rather than sitting +# inside words — "tag:example" is documented query syntax, and permalinks and +# file names are built on the full stop. +STRUCTURAL_PUNCTUATION = ":︓﹕：.․﹒．" + + +def test_the_joining_set_accounts_for_every_word_joining_character() -> None: + """Every UAX #29 word-joining character is either taken or named structural. + + Review surfaced these one at a time — the Armenian abbreviation mark and the + fullwidth apostrophe were the last two. Partitioning the class means a + missing character fails here rather than in another round. + """ + standard = set("".join(UAX29_WORD_JOINING_PUNCTUATION.values())) + taken = set(RELAXATION_WORD_INTERNAL_PUNCTUATION) + excluded = set(STRUCTURAL_PUNCTUATION) + + unaccounted = standard - taken - excluded + assert not unaccounted, ( + f"word-joining characters neither taken nor excluded — {_describe(sorted(unaccounted))}" + ) + + contradictory = taken & excluded + assert not contradictory, ( + f"characters both taken and excluded — {_describe(sorted(contradictory))}" + ) + + +def test_taken_characters_outside_the_standard_are_justified() -> None: + """U+05F3 is the one addition: UAX #29 classes geresh as a letter, not punctuation. + + Python sees it as Po, so the joining rule has to name it explicitly to reach + the same result the standard does for Hebrew. + """ + standard = set("".join(UAX29_WORD_JOINING_PUNCTUATION.values())) + assert set(RELAXATION_WORD_INTERNAL_PUNCTUATION) - standard == {"׳"} + + +@pytest.mark.parametrize("structural", sorted(STRUCTURAL_PUNCTUATION)) +def test_structural_punctuation_keeps_splitting(structural: str) -> None: + """Joining across these would merge a qualifier with its value, or a name with its extension.""" + assert len(relaxation_word_tokens(f"сло{structural}во доступа")) == 3 + + +def test_tokenizing_scales_linearly_with_query_length() -> None: + """Doubling the query must roughly double the work, not more. + + The joining rule has to look at what follows the punctuation. Reading that + with a slice copies the rest of the query at every joiner, which is + quadratic: an unbounded query full of apostrophes then ties up the worker + that tokenizes it. The ratio is asserted rather than a duration, so the test + does not depend on how fast the machine is. + """ + query = "a'b " * (128 * 1024 // 4) + + def elapsed(text: str) -> float: + start = time.perf_counter() + relaxation_word_tokens(text) + return time.perf_counter() - start + + # Fastest of three runs each: the ratio, not the duration, is what is being + # asserted, so a slow or busy machine does not turn this red. Measured here, + # the indexed reader scales at ×2.0 and the sliced one at ×3.2. + single = min(elapsed(query) for _ in range(3)) + double = min(elapsed(query * 2) for _ in range(3)) + + assert double < single * 2.5, ( + f"tokenizing scaled worse than linearly: {single * 1000:.1f} ms then {double * 1000:.1f} ms" + ) + + +def test_trailing_format_trim_scales_linearly() -> None: + """A word ending in a long run of format characters must trim in one pass. + + Format characters are word-internal, so a token collects them all before the + trailing trim runs. Trimming one character at a time copies the shrinking + token at every step, which is quadratic: an unbounded query ending in enough + soft hyphens then ties up the worker that tokenizes it. As above, the ratio + is asserted rather than a duration. + """ + + def elapsed(count: int) -> float: + text = "a" + "\u00ad" * count + start = time.perf_counter() + relaxation_word_tokens(text) + return time.perf_counter() - start + + # Sized so the trim dominates the measurement: at shorter lengths the + # tokenizer's linear per-character work dilutes the quadratic term below + # the ratio threshold and a regression would pass unnoticed. + single = min(elapsed(256 * 1024) for _ in range(3)) + double = min(elapsed(512 * 1024) for _ in range(3)) + + assert double < single * 2.5, ( + f"trailing trim scaled worse than linearly: {single * 1000:.1f} ms then {double * 1000:.1f} ms" + ) diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py new file mode 100644 index 000000000..fc43d062c --- /dev/null +++ b/tests/repository/test_search_relaxed_rendering.py @@ -0,0 +1,142 @@ +"""Relaxed-fallback rendering must survive the tokens the eligibility helper emits.""" + +import sqlite3 + +import pytest + +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository + +CREATE_FTS = ( + "CREATE VIRTUAL TABLE t USING fts5(" + "body, tokenize='unicode61 tokenchars 0x2F', prefix='1,2,3,4')" +) +DOCUMENT = ( + "don't touch this п’ять проектів об'єкт доступу как отозвать выданный доступ पहुंच कैसे रद्द करें" +) + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", '"don\'t"* OR touch*'), + ("скасувати об'єкт виданий доступ", 'скасувати* OR "об\'єкт"* OR виданий* OR доступ*'), + ("п’ять виданих різних об’єктів", "п’ять* OR виданих* OR різних* OR об’єктів*"), + ("how to revoke granted access", "revoke* OR granted* OR access*"), + ], +) +def test_sqlite_relaxed_text_quotes_only_terms_that_need_it(query: str, expected: str) -> None: + """Apostrophe terms are quoted; every other term renders exactly as before.""" + assert SQLiteSearchRepository._relaxed_fts_text(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "don't touch this", + "скасувати об'єкт виданий доступ", + "п’ять виданих різних об’єктів", + "как отозвать выданный доступ", + "पहुंच कैसे रद्द करें", + ], +) +def test_sqlite_relaxed_text_is_accepted_by_fts5(query: str) -> None: + """The rendered expression must parse. + + An unquoted apostrophe raises `fts5: syntax error`, which the repository + catches and turns into an empty result — the relaxed retry then silently + contributes nothing, which is the failure this fallback exists to prevent. + """ + relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + assert relaxed is not None + + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression matched nothing: {relaxed}" + + +def test_sqlite_relaxed_text_bare_apostrophe_would_be_rejected() -> None: + """Pin why the quoting exists, so removing it fails loudly rather than silently.""" + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + with pytest.raises(sqlite3.OperationalError, match="fts5: syntax error"): + connection.execute("SELECT rowid FROM t WHERE t MATCH ?", ("don't* OR touch*",)) + finally: + connection.close() + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", "'don''t':* | touch:*"), + ("скасувати об'єкт виданий доступ", "скасувати:* | 'об''єкт':* | виданий:* | доступ:*"), + ("how to revoke granted access", "revoke:* | granted:* | access:*"), + ], +) +def test_postgres_relaxed_tsquery_quotes_apostrophe_lexemes(query: str, expected: str) -> None: + """Postgres carries the same token shapes, so it needs the same escaping.""" + assert PostgresSearchRepository._relaxed_tsquery_text(query) == expected + + +@pytest.mark.parametrize( + ("document", "query"), + [ + ("перевод права доступа", "пере­vод права доступа"), # soft hyphen + ("слово права доступа", "сло⁠во права доступа"), # word joiner + ("نمی‌خواهم دسترسی را لغو", "نمی‌خواهم دسترسی را لغو"), # ZWNJ kept + ], +) +def test_relaxed_terms_match_the_stored_note(document: str, query: str) -> None: + """A term must match the note as stored, not as the query happened to be pasted. + + Rendering artifacts — a soft hyphen from a paginated document, a stray word + joiner — are absent from the note, so carrying them into the term stops it + matching. The Persian joiner is the opposite case: it is written in the text + and indexed with it, so removing it would break the match instead. + """ + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (document,)) + relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + assert relaxed is not None + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression did not match the stored note: {relaxed!r}" + + +@pytest.mark.parametrize("document", ["foo­bar", "foobar"]) +def test_relaxed_terms_match_either_stored_form(document: str) -> None: + """A format character is invisible, so the note may hold it or not. + + The two forms index differently — "foo­bar" as two tokens, "foobar" as + one — and neither term matches the other note. Both forms are emitted, and + the OR relaxation already builds covers whichever the note actually has. + """ + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (document,)) + relaxed = SQLiteSearchRepository._relaxed_fts_text("foo­bar права доступа") + assert relaxed is not None + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression missed the note {document!r}: {relaxed!r}" + + +def test_orthographic_joiners_are_not_duplicated_into_a_second_variant() -> None: + """Joiners are written in the text, so the stored form has them. + + A stripped variant would only widen the OR with a term no note can hold. + """ + relaxed = SQLiteSearchRepository._relaxed_fts_text("نمی‌خواهم دسترسی را لغو") + assert relaxed == "نمی‌خواهم* OR دسترسی* OR را* OR لغو*"