diff --git a/docs/NOTE-FORMAT.md b/docs/NOTE-FORMAT.md index be1ebb718..2d666f08a 100644 --- a/docs/NOTE-FORMAT.md +++ b/docs/NOTE-FORMAT.md @@ -166,6 +166,17 @@ Explicit relations: - 'in response to' [[Incident Review]] ``` +When the text before `[[` is ordinary prose rather than a relation type, add +`#bm:links_to` to force the reference to use the implicit `links_to` relation: + +```markdown +- Mother [[Alice]] #bm:links_to +``` + +The directive is parser metadata and is not included in the observation or +relation context. It is especially useful when a single-token prose prefix +would otherwise be interpreted as an explicit relation type. + Bare wiki links and prose list items create implicit `links_to` relations: ```markdown diff --git a/justfile b/justfile index da1da1c12..c537bd4c7 100644 --- a/justfile +++ b/justfile @@ -521,7 +521,6 @@ doctor: TMP_HOME=$(mktemp -d) TMP_CONFIG=$(mktemp -d) HOME="$TMP_HOME" \ - BASIC_MEMORY_ENV=test \ BASIC_MEMORY_HOME="$TMP_HOME/basic-memory" \ BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG" \ ./.venv/bin/python -m basic_memory.cli.main doctor --local diff --git a/src/basic_memory/alembic/versions/7f6a2b8c9d10_index_full_postgres_note_content.py b/src/basic_memory/alembic/versions/7f6a2b8c9d10_index_full_postgres_note_content.py new file mode 100644 index 000000000..b9fe1b77f --- /dev/null +++ b/src/basic_memory/alembic/versions/7f6a2b8c9d10_index_full_postgres_note_content.py @@ -0,0 +1,77 @@ +"""Index complete PostgreSQL note content for full-text search. + +Revision ID: 7f6a2b8c9d10 +Revises: 2d26b287813b +Create Date: 2026-08-24 22:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "7f6a2b8c9d10" +down_revision: Union[str, None] = "2d26b287813b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Index complete note bodies as bounded PostgreSQL FTS chunks.""" + connection = op.get_bind() + if connection.dialect.name == "postgresql": + op.execute(""" + CREATE TABLE search_index_fts_chunks ( + project_id INTEGER NOT NULL, + search_index_id INTEGER NOT NULL, + search_index_type VARCHAR NOT NULL, + chunk_index INTEGER NOT NULL, + chunk_text TEXT NOT NULL, + textsearchable_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('english', chunk_text) + ) STORED, + PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index), + FOREIGN KEY (search_index_id, search_index_type, project_id) + REFERENCES search_index(id, type, project_id) + ON UPDATE CASCADE ON DELETE CASCADE + ) + """) + op.execute(""" + -- PostgreSQL ignores lexemes at 2 KiB and above. Stepping 5,952 + -- characters leaves a conservative 2,048-character overlap, so + -- every indexable lexeme split at one edge is complete in the next. + INSERT INTO search_index_fts_chunks ( + project_id, + search_index_id, + search_index_type, + chunk_index, + chunk_text + ) + SELECT + search_index.project_id, + search_index.id, + search_index.type, + (chunk_start - 1) / 5952, + substring(search_index.content_snippet FROM chunk_start FOR 8000) + FROM search_index + CROSS JOIN LATERAL generate_series( + 1, + length(search_index.content_snippet), + 5952 + ) AS chunk_start + WHERE search_index.content_snippet IS NOT NULL + AND search_index.content_snippet <> '' + """) + op.execute(""" + CREATE INDEX idx_search_index_fts_chunks_fts + ON search_index_fts_chunks USING gin(textsearchable_index_col) + """) + + +def downgrade() -> None: + """Remove the bounded full-content PostgreSQL FTS chunks.""" + connection = op.get_bind() + if connection.dialect.name == "postgresql": + op.execute("DROP TABLE IF EXISTS search_index_fts_chunks") diff --git a/src/basic_memory/cli/auto_update.py b/src/basic_memory/cli/auto_update.py index 80435da54..739457e95 100644 --- a/src/basic_memory/cli/auto_update.py +++ b/src/basic_memory/cli/auto_update.py @@ -221,6 +221,38 @@ def _preload_lazy_console_modules() -> None: """ import rich._emoji_codes # noqa: F401 import typer.rich_utils # noqa: F401 + from rich.cells import cell_len + + # Trigger: rich defers its Unicode cell-width table (`rich._unicode_data. + # unicode`) until the first character it cannot measure with the + # ASCII fast path in `_cell_len`. + # Why: status messages echo captured `brew`/`uv` output, which carries + # non-ASCII characters (curly quotes, em dashes, warning glyphs), so the + # deferred import lands after the upgrade removed our files. Importing the + # module by name would hard-code a table version; calling `cell_len` uses + # rich's own resolution and honors UNICODE_VERSION like the print path does. + # Outcome: the table rich will reach for is resolved and cached up front. + cell_len("\u2500\u2018\u2713") + + +def print_update_status(console: Console, text: str, style: str) -> None: + """Print an update status line that cannot fail the command. + + Trigger: the line is printed after an in-place upgrade may already have + replaced this installation on disk. + Why: `_preload_lazy_console_modules` can only preload the deferred imports + we know about today, and rich/typer are free to add more. By the time this + prints, the upgrade has already succeeded -- a status line must never be + what turns it into a traceback and a non-zero exit. + Outcome: fall back to a plain, unstyled write that needs no new imports. + """ + try: + console.print(f"[{style}]{text}[/{style}]") + except Exception as exc: + logger.warning( + f"Rich console print failed after update, falling back to plain output: {exc}" + ) + print(text) def _save_last_checked_timestamp(config_manager: ConfigManager, checked_at: datetime) -> None: @@ -468,11 +500,11 @@ def maybe_run_periodic_auto_update( }: out = console or Console() if result.status == AutoUpdateStatus.UPDATED: - out.print(f"[green]{result.message}[/green]") + print_update_status(out, f"{result.message}", "green") elif result.status == AutoUpdateStatus.FAILED: error_detail = f" {result.error}" if result.error else "" - out.print(f"[yellow]{result.message}{error_detail}[/yellow]") + print_update_status(out, f"{result.message}{error_detail}", "yellow") elif result.message: - out.print(f"[cyan]{result.message}[/cyan]") + print_update_status(out, f"{result.message}", "cyan") return result diff --git a/src/basic_memory/cli/commands/doctor.py b/src/basic_memory/cli/commands/doctor.py index 1d4539cd2..1b614db09 100644 --- a/src/basic_memory/cli/commands/doctor.py +++ b/src/basic_memory/cli/commands/doctor.py @@ -81,6 +81,23 @@ async def _delete_doctor_project( await _delete_doctor_project_locally(project_name, project_id) +async def _read_materialized_api_note(api_file: Path, file_path: str) -> str: + """Wait for the accepted local write, then read its canonical file.""" + # Deferred: importing the materialization runtime at CLI module load would slow every + # command, while doctor is the only command that needs to observe its write immediately. + from basic_memory.index.note_content_materialization import drain_pending_materializations + + # Trigger: production accepts note content before its markdown file is materialized. + # Why: doctor verifies the complete DB -> file contract, not only write acceptance. + # Outcome: wait for the same deferred work drained at normal CLI shutdown before checking. + await drain_pending_materializations() + + if not api_file.exists(): + raise ValueError(f"API note file missing: {file_path}") + + return api_file.read_text(encoding="utf-8") + + async def run_doctor() -> None: """Run local consistency checks for file <-> database flows.""" # Deferred: the markdown parsing stack is only needed while the checks run, @@ -132,10 +149,7 @@ async def run_doctor() -> None: api_result = await knowledge_client.create_entity(api_note.model_dump()) api_file = project_path / api_result.file_path - if not api_file.exists(): - raise ValueError(f"API note file missing: {api_result.file_path}") - - api_text = api_file.read_text(encoding="utf-8") + api_text = await _read_materialized_api_note(api_file, api_result.file_path) if api_note_title not in api_text: raise ValueError("API note content missing from file") diff --git a/src/basic_memory/cli/commands/update.py b/src/basic_memory/cli/commands/update.py index b0cff27c3..22dc52d32 100644 --- a/src/basic_memory/cli/commands/update.py +++ b/src/basic_memory/cli/commands/update.py @@ -4,7 +4,7 @@ from rich.console import Console from basic_memory.cli.app import app -from basic_memory.cli.auto_update import AutoUpdateStatus, run_auto_update +from basic_memory.cli.auto_update import AutoUpdateStatus, print_update_status, run_auto_update console = Console() @@ -22,19 +22,21 @@ def update( if result.status == AutoUpdateStatus.FAILED: detail = f" {result.error}" if result.error else "" - console.print(f"[red]{result.message or 'Update failed.'}{detail}[/red]") + print_update_status(console, f"{result.message or 'Update failed.'}{detail}", "red") raise typer.Exit(1) if result.status == AutoUpdateStatus.UPDATED: - console.print(f"[green]{result.message or 'Basic Memory updated successfully.'}[/green]") + print_update_status( + console, f"{result.message or 'Basic Memory updated successfully.'}", "green" + ) return if result.status == AutoUpdateStatus.UP_TO_DATE: - console.print(f"[green]{result.message or 'Basic Memory is up to date.'}[/green]") + print_update_status(console, f"{result.message or 'Basic Memory is up to date.'}", "green") return if result.status == AutoUpdateStatus.UPDATE_AVAILABLE: - console.print(f"[cyan]{result.message or 'Update available.'}[/cyan]") + print_update_status(console, f"{result.message or 'Update available.'}", "cyan") return - console.print(f"[dim]{result.message or 'No update action was performed.'}[/dim]") + print_update_status(console, f"{result.message or 'No update action was performed.'}", "dim") diff --git a/src/basic_memory/markdown/plugins.py b/src/basic_memory/markdown/plugins.py index 5dfd3536d..1a41ccc63 100644 --- a/src/basic_memory/markdown/plugins.py +++ b/src/basic_memory/markdown/plugins.py @@ -13,6 +13,15 @@ # those bracket prefixes stay ordinary content (issue #1219). _TIMESTAMP_VALUE = r"\d{1,3}:\d{2}(?::\d{2})?(?:[.,]\d{1,3})?" _TIMESTAMP_CATEGORY = re.compile(rf"^{_TIMESTAMP_VALUE}(?:\s+-\s+{_TIMESTAMP_VALUE})?$") +_LINKS_TO_DIRECTIVE = re.compile(r"\s+#bm:links_to\s*$") + + +def remove_links_to_directive(content: str) -> tuple[str, bool]: + """Remove an exact terminal ``#bm:links_to`` directive from content.""" + match = _LINKS_TO_DIRECTIVE.search(content) + if not match: + return content, False + return content[: match.start()].rstrip(), True def _is_task_marker_category(category: str) -> bool: @@ -47,6 +56,7 @@ def is_observation(token: Token) -> bool: return False # Use token.tag which contains the actual content for test tokens, fallback to content content = (token.tag or token.content).strip() + content, _ = remove_links_to_directive(content) if not content: # pragma: no cover return False # if it's a markdown_task, return false @@ -74,6 +84,7 @@ def parse_observation(token: Token) -> Dict[str, Any]: # Use token.tag which contains the actual content for test tokens, fallback to content content = (token.tag or token.content).strip() + content, _ = remove_links_to_directive(content) # Parse [category] with regex; a timestamp-shaped prefix is not a category, so a # hashtag-promoted transcript line keeps its timecode inside the content instead. @@ -325,17 +336,19 @@ def relation_rule(state: Any) -> None: # Only process inline tokens if token.type == "inline": + content = token.tag or token.content + content_without_directive, has_directive = remove_links_to_directive(content) + # Check for explicit relations in list items - if in_list_item and is_explicit_relation(token): + if in_list_item and not has_directive and is_explicit_relation(token): rel = parse_relation(token) if rel: token.meta["relations"] = [rel] # Always check for inline links in any text else: - content = token.tag or token.content if "[[" in content: - rels = parse_inline_relations(content) + rels = parse_inline_relations(content_without_directive) if rels: token.meta["relations"] = token.meta.get("relations", []) + rels diff --git a/src/basic_memory/mcp/tools/write_note.py b/src/basic_memory/mcp/tools/write_note.py index edd823635..7b521ccc8 100644 --- a/src/basic_memory/mcp/tools/write_note.py +++ b/src/basic_memory/mcp/tools/write_note.py @@ -109,12 +109,15 @@ async def write_note( - Explicit: `- relation_type [[Entity]] (optional context)` - Quoted: `- "multi word relation type" [[Entity]] (optional context)` - Quoted: `- 'multi word relation type' [[Entity]] (optional context)` + - Disambiguation: Add `#bm:links_to` when prose before `[[Entity]]` + must not be treated as a single-token relation type - Inline: Any other `[[Entity]]` reference creates a `links_to` relation Examples: `- depends_on [[Content Parser]] (Need for semantic extraction)` `- "based on" [[Design Notes]]` `- 'in response to' [[Incident Review]]` + `- Mother [[Alice]] #bm:links_to` `- implements [[Search Spec]] (Initial implementation)` `- This feature extends [[Base Design]] and uses [[Core Utils]]` diff --git a/src/basic_memory/models/search.py b/src/basic_memory/models/search.py index 1c9fff6d5..c9616d850 100644 --- a/src/basic_memory/models/search.py +++ b/src/basic_memory/models/search.py @@ -33,7 +33,11 @@ created_at TIMESTAMP WITH TIME ZONE, updated_at TIMESTAMP WITH TIME ZONE, textsearchable_index_col tsvector GENERATED ALWAYS AS ( - to_tsvector('english', coalesce(title, '') || ' ' || coalesce(content_stems, '')) + to_tsvector( + 'english', + coalesce(title, '') || ' ' || + coalesce(content_stems, '') + ) ) STORED, PRIMARY KEY (id, type, project_id), FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE @@ -44,6 +48,30 @@ CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col) """) +# Full note bodies are stored in bounded child rows so one unusually large note +# cannot exceed PostgreSQL's per-tsvector size limit. +CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL(""" +CREATE TABLE IF NOT EXISTS search_index_fts_chunks ( + project_id INTEGER NOT NULL, + search_index_id INTEGER NOT NULL, + search_index_type VARCHAR NOT NULL, + chunk_index INTEGER NOT NULL, + chunk_text TEXT NOT NULL, + textsearchable_index_col tsvector GENERATED ALWAYS AS ( + to_tsvector('english', chunk_text) + ) STORED, + PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index), + FOREIGN KEY (search_index_id, search_index_type, project_id) + REFERENCES search_index(id, type, project_id) + ON UPDATE CASCADE ON DELETE CASCADE +) +""") + +CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX = DDL(""" +CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_fts +ON search_index_fts_chunks USING gin(textsearchable_index_col) +""") + CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL(""" CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops) """) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 8aa8e5bad..d468267c1 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -45,6 +45,66 @@ from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +POSTGRES_FTS_CHUNK_SIZE = 8_000 +# PostgreSQL ignores lexemes at 2 KiB and above. A 2,048-character overlap is +# therefore conservative for every indexable lexeme, including multi-byte text: +# any token split at one 8,000-character edge is complete in the next chunk. +POSTGRES_FTS_CHUNK_OVERLAP = 2_048 +_TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") +_TSQUERY_WORD_PATTERN = re.compile(r"[^\W_]+(?:'[^\W_]+)?", re.UNICODE) + + +def _iter_fts_chunks(content: str | None) -> list[tuple[int, str]]: + """Split full note text without losing an indexable lexeme at a chunk edge.""" + if not content: + return [] + + step = POSTGRES_FTS_CHUNK_SIZE - POSTGRES_FTS_CHUNK_OVERLAP + return [ + (chunk_index, content[start : start + POSTGRES_FTS_CHUNK_SIZE]) + for chunk_index, start in enumerate(range(0, len(content), step)) + ] + + +def _tsquery_operands(processed_text: str) -> list[tuple[str, str]]: + """Return unique (query operand, representative text) pairs in source order.""" + operands: dict[str, str] = {} + for operand in _TSQUERY_OPERAND_PATTERN.findall(processed_text): + representative = operand.removesuffix(":*") + if representative.startswith("'") and representative.endswith("'"): + representative = representative[1:-1].replace("''", "'") + operands.setdefault(operand, representative) + continue + + # An unquoted apostrophe is invalid tsquery syntax. Keep the literal + # word for the synthetic document, but quote and escape its probe so a + # strict syntax failure can proceed to the relaxed retry. + if "'" in representative: + escaped = "'{}'".format(representative.replace("'", "''")) + safe_operand = f"{escaped}:*" if operand.endswith(":*") else escaped + operands.setdefault(safe_operand, representative) + continue + + # PostgreSQL legitimately parses punctuation inside operands such as + # ``v0.13.0b2:*`` and ``auth-service:*``. Preserve those bytes so the + # synthetic document is tokenized the same way as the original note. + if "<" not in representative and ">" not in representative: + operands.setdefault(operand, representative) + continue + + # A malformed strict operand (for example ``foo dict[str, Any]: """Strip NUL bytes from all string values in a row dict. @@ -199,9 +259,76 @@ async def index_item(self, search_index_row: SearchIndexRow) -> None: """), insert_data, ) + await self._replace_fts_chunks(session, [search_index_row]) logger.debug(f"indexed row {search_index_row}") await session.commit() + async def _replace_fts_chunks( + self, + session: AsyncSession, + search_index_rows: Sequence[SearchIndexRow], + ) -> None: + """Replace bounded full-content FTS rows for one indexing batch.""" + if not search_index_rows: + return + + await session.execute( + text(""" + DELETE FROM search_index_fts_chunks + WHERE project_id = :project_id + AND (search_index_id, search_index_type) IN ( + SELECT search_index_id, search_index_type + FROM unnest( + CAST(:search_index_ids AS INTEGER[]), + CAST(:search_index_types AS VARCHAR[]) + ) AS indexed_rows(search_index_id, search_index_type) + ) + """), + { + "project_id": self.project_id, + "search_index_ids": [row.id for row in search_index_rows], + "search_index_types": [row.type for row in search_index_rows], + }, + ) + + chunks = [ + { + "search_index_id": row.id, + "search_index_type": row.type, + "chunk_index": chunk_index, + "chunk_text": chunk_text.replace("\x00", ""), + } + for row in search_index_rows + for chunk_index, chunk_text in _iter_fts_chunks(row.content_snippet) + ] + if not chunks: + return + + await session.execute( + text(""" + INSERT INTO search_index_fts_chunks ( + project_id, + search_index_id, + search_index_type, + chunk_index, + chunk_text + ) + SELECT + :project_id, + chunk.search_index_id, + chunk.search_index_type, + chunk.chunk_index, + chunk.chunk_text + FROM jsonb_to_recordset(CAST(:chunks AS JSONB)) AS chunk( + search_index_id INTEGER, + search_index_type VARCHAR, + chunk_index INTEGER, + chunk_text TEXT + ) + """), + {"project_id": self.project_id, "chunks": json.dumps(chunks)}, + ) + # ------------------------------------------------------------------ # tsquery preparation (backend-specific) # ------------------------------------------------------------------ @@ -657,6 +784,7 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non """), insert_data_list, ) + await self._replace_fts_chunks(session, search_index_rows) logger.debug(f"Bulk indexed {len(search_index_rows)} rows") await session.commit() @@ -685,12 +813,14 @@ async def _build_fts_query_parts( search_item_types: Optional[List[SearchItemType]] = None, categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, + allow_relaxed: bool = False, ) -> tuple[str, str, dict[str, Any], str, str]: """Build Postgres FTS FROM/WHERE params shared by search and count.""" conditions = [] params = {} order_by_clause = "" from_clause = "search_index" + document_vector_sql: str | None = None # Handle text search for title and content using tsvector if search_text: @@ -701,10 +831,61 @@ async def _build_fts_query_parts( # Prepare search term for tsquery processed_text = self._prepare_search_term(search_text.strip()) params["text"] = processed_text - # Use @@ operator for tsvector matching - conditions.append( - "search_index.textsearchable_index_col @@ to_tsquery('english', :text)" - ) + probe_texts = [processed_text] + if allow_relaxed: + relaxed_text = self._relaxed_tsquery_text(search_text) + if relaxed_text: + probe_texts.append(relaxed_text) + + candidate_operands: dict[str, None] = {} + for probe_text in probe_texts: + for operand, _representative in _tsquery_operands(probe_text): + candidate_operands.setdefault(operand, None) + if candidate_operands: + params["text_candidate"] = " | ".join(candidate_operands) + + # Trigger: PostgreSQL can extract a required-positive query tree. + # Why: OR-ing its operands is a safe indexed superset even when + # terms live in different chunks. Pure/optional negation returns + # ``T`` and must retain all project rows for correct semantics. + # Outcome: ordinary and required-positive NOT queries use both + # GIN indexes; only genuinely unindexable negation scans the project. + from_clause = """ + search_index JOIN ( + SELECT + candidate_parent.project_id, + candidate_parent.id, + candidate_parent.type + FROM search_index AS candidate_parent + WHERE candidate_parent.project_id = :project_id + AND querytree(to_tsquery('english', :text)) <> 'T' + AND candidate_parent.textsearchable_index_col + @@ to_tsquery('english', :text_candidate) + UNION + SELECT + candidate_chunk.project_id, + candidate_chunk.search_index_id AS id, + candidate_chunk.search_index_type AS type + FROM search_index_fts_chunks AS candidate_chunk + WHERE candidate_chunk.project_id = :project_id + AND querytree(to_tsquery('english', :text)) <> 'T' + AND candidate_chunk.textsearchable_index_col + @@ to_tsquery('english', :text_candidate) + UNION + SELECT + candidate_all.project_id, + candidate_all.id, + candidate_all.type + FROM search_index AS candidate_all + WHERE candidate_all.project_id = :project_id + AND querytree(to_tsquery('english', :text)) = 'T' + ) AS fts_candidate + ON fts_candidate.project_id = search_index.project_id + AND fts_candidate.id = search_index.id + AND fts_candidate.type = search_index.type + """ + document_vector_sql = self._document_fts_vector_sql(probe_texts, params) + conditions.append(f"{document_vector_sql} @@ to_tsquery('english', :text)") # Handle title search if title: @@ -785,7 +966,7 @@ async def _build_fts_query_parts( # path parts instead of #>> / #> with interpolated paths. if metadata_filters: parsed_filters = parse_metadata_filters(metadata_filters) - from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id" + from_clause = f"{from_clause} JOIN entity ON search_index.entity_id = entity.id" metadata_expr = "entity.entity_metadata::jsonb" for idx, filt in enumerate(parsed_filters): @@ -863,14 +1044,59 @@ async def _build_fts_query_parts( # Build SQL with ts_rank() for scoring # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 if search_text and search_text.strip() and search_text.strip() != "*": + assert document_vector_sql is not None score_expr = ( - "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text))" + "GREATEST(" + f"ts_rank({document_vector_sql}, to_tsquery('english', :text)), " + "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text)), " + "COALESCE((SELECT MAX(ts_rank(" + "fts_chunk.textsearchable_index_col, to_tsquery('english', :text))) " + "FROM search_index_fts_chunks AS fts_chunk " + "WHERE fts_chunk.project_id = search_index.project_id " + "AND fts_chunk.search_index_id = search_index.id " + "AND fts_chunk.search_index_type = search_index.type " + "AND fts_chunk.textsearchable_index_col " + "@@ to_tsquery('english', :text)), 0))" ) else: score_expr = "0" return from_clause, where_clause, params, order_by_clause, score_expr + @staticmethod + def _document_fts_vector_sql(processed_texts: Sequence[str], params: dict[str, Any]) -> str: + """Build a query-sized vector representing lexemes found anywhere in one item.""" + operands: dict[str, str] = {} + for processed_text in processed_texts: + for operand, representative in _tsquery_operands(processed_text): + operands.setdefault(operand, representative) + + present_lexemes: list[str] = [] + for index, (operand, representative) in enumerate(operands.items()): + operand_param = f"text_operand_{index}" + representative_param = f"text_representative_{index}" + params[operand_param] = operand + params[representative_param] = representative + present_lexemes.append( + "CASE WHEN (search_index.textsearchable_index_col " + f"@@ to_tsquery('english', :{operand_param}) OR EXISTS (" + "SELECT 1 FROM search_index_fts_chunks AS operand_chunk " + "WHERE operand_chunk.project_id = search_index.project_id " + "AND operand_chunk.search_index_id = search_index.id " + "AND operand_chunk.search_index_type = search_index.type " + "AND operand_chunk.textsearchable_index_col " + f"@@ to_tsquery('english', :{operand_param}))) " + f"THEN :{representative_param} ELSE '' END" + ) + + if not present_lexemes: + return "search_index.textsearchable_index_col" + + # The synthesized text contains at most the query operands, never the note body. + # This preserves document-wide Boolean semantics without recreating an unbounded vector. + lexeme_array = f"ARRAY[{', '.join(present_lexemes)}]" + return f"to_tsvector('english', array_to_string({lexeme_array}, ' '))" + @override async def search( self, @@ -930,6 +1156,7 @@ async def search( search_item_types=search_item_types, categories=categories, metadata_filters=metadata_filters, + allow_relaxed=allow_relaxed, ) # set limit and offset @@ -1111,6 +1338,7 @@ async def count( search_item_types=search_item_types, categories=categories, metadata_filters=metadata_filters, + allow_relaxed=allow_relaxed, ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" logger.trace(f"Count {sql} params: {params}") diff --git a/test-int/conftest.py b/test-int/conftest.py index 6d3256a7c..32fb81975 100644 --- a/test-int/conftest.py +++ b/test-int/conftest.py @@ -223,6 +223,8 @@ async def _reset_postgres_integration_schema(engine: AsyncEngine, async_url: str """Restore the shared Postgres integration schema to a clean baseline.""" from basic_memory.models.search import ( CREATE_POSTGRES_SEARCH_INDEX_FTS, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE, CREATE_POSTGRES_SEARCH_INDEX_METADATA, CREATE_POSTGRES_SEARCH_INDEX_PERMALINK, CREATE_POSTGRES_SEARCH_INDEX_TABLE, @@ -236,6 +238,8 @@ async def _reset_postgres_integration_schema(engine: AsyncEngine, async_url: str await conn.run_sync(Base.metadata.create_all) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK) diff --git a/test-int/semantic/conftest.py b/test-int/semantic/conftest.py index 617265525..03fc2d599 100644 --- a/test-int/semantic/conftest.py +++ b/test-int/semantic/conftest.py @@ -33,6 +33,8 @@ from basic_memory.models.base import Base from basic_memory.models.search import ( CREATE_POSTGRES_SEARCH_INDEX_FTS, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE, CREATE_POSTGRES_SEARCH_INDEX_METADATA, CREATE_POSTGRES_SEARCH_INDEX_PERMALINK, CREATE_POSTGRES_SEARCH_INDEX_TABLE, @@ -168,6 +170,8 @@ async def _reset_postgres_semantic_schema(engine: AsyncEngine) -> None: await conn.run_sync(Base.metadata.create_all) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK) diff --git a/tests/cli/test_auto_update.py b/tests/cli/test_auto_update.py index e9f85cdab..65e3e2503 100644 --- a/tests/cli/test_auto_update.py +++ b/tests/cli/test_auto_update.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re import subprocess import sys import urllib.error @@ -21,6 +22,7 @@ _check_homebrew_update_available, _is_interactive_session, _preload_lazy_console_modules, + print_update_status, detect_install_source, maybe_run_periodic_auto_update, run_auto_update, @@ -387,6 +389,72 @@ def test_preload_lazy_console_modules_imports_deferred_modules(monkeypatch): assert "typer.rich_utils" in sys.modules +class _UpgradedAwayFinder: + """Stand-in for the deleted install prefix: nothing new can be imported.""" + + def find_spec(self, fullname, path=None, target=None): # noqa: D102 + raise ModuleNotFoundError(f"No module named {fullname!r}") + + +def _cool_deferred_width_table(monkeypatch) -> None: + """Return rich to the cold state a freshly started process is in. + + rich caches the Unicode cell-width table aggressively, and earlier tests in + this session will already have warmed it -- without this the regression test + below passes whether or not the table was preloaded. + """ + import rich.cells + + for cached in ("cached_cell_len", "get_character_cell_size"): + clear = getattr(getattr(rich.cells, cached, None), "cache_clear", None) + if clear is not None: + clear() + + # rich >= 14.2 splits the tables into rich._unicode_data.unicode; + # older versions inline them and have nothing to unload. + unicode_data = sys.modules.get("rich._unicode_data") + clear_load = getattr(getattr(unicode_data, "load", None), "cache_clear", None) + if clear_load is not None: + clear_load() + for name in [n for n in sys.modules if n.startswith("rich._unicode_data.unicode")]: + monkeypatch.delitem(sys.modules, name, raising=False) + + +def test_status_message_survives_upgraded_away_install(monkeypatch): + # Regression (#1316): `brew upgrade` removes the running install's files, so + # the status message printed afterwards must not need any new import. The + # message is long and non-ASCII on purpose -- that is what makes rich wrap + # the line and reach for the deferred cell-width table. + output = StringIO() + console = Console(width=40, file=output) + console.print("warm up the print path") + _cool_deferred_width_table(monkeypatch) + + _preload_lazy_console_modules() + monkeypatch.setattr(sys, "meta_path", [_UpgradedAwayFinder(), *sys.meta_path]) + + # Deliberately not print_update_status: its fallback would mask the bug. + console.print( + "[red]Automatic update failed. Error: could not link \u2018basic-memory\u2019 " + "\u2014 the files were replaced while running. " + "detail " * 12 + "[/red]" + ) + + # rich wrapped and styled the line; normalize before checking the content. + rendered = re.sub(r"\x1b\[[0-9;]*m", "", output.getvalue()) + assert "could not link" in " ".join(rendered.split()) + + +def test_print_update_status_falls_back_to_plain_output(capsys): + # A status line must never be what fails a command whose upgrade succeeded. + class ExplodingConsole: + def print(self, *args, **kwargs): + raise ModuleNotFoundError("No module named 'rich._unicode_data.unicode17-0-0'") + + print_update_status(cast(Console, ExplodingConsole()), "Basic Memory was updated.", "green") + + assert "Basic Memory was updated." in capsys.readouterr().out + + def test_homebrew_outdated_triggers_upgrade(monkeypatch, tmp_path): config = _base_config(tmp_path) manager = StubConfigManager(config) diff --git a/tests/cli/test_doctor_command.py b/tests/cli/test_doctor_command.py index 382cffc60..cb1de6848 100644 --- a/tests/cli/test_doctor_command.py +++ b/tests/cli/test_doctor_command.py @@ -6,10 +6,12 @@ from typing import Callable, NoReturn +import pytest from typer.testing import CliRunner from basic_memory.cli.app import app import basic_memory.cli.commands.doctor as doctor_cmd +from basic_memory.index import note_content_materialization runner = CliRunner() @@ -49,3 +51,43 @@ def test_doctor_unexpected_failure_message_never_blank(monkeypatch): assert result.exit_code == 1 assert "Doctor failed: RuntimeError()" in result.stderr + + +@pytest.mark.asyncio +async def test_doctor_waits_for_deferred_api_note_materialization(tmp_path, monkeypatch): + api_file = tmp_path / "doctor" / "Doctor API Note.md" + + async def materialize_note() -> None: + api_file.parent.mkdir(parents=True) + api_file.write_text("# Doctor API Note", encoding="utf-8") + + monkeypatch.setattr( + note_content_materialization, + "drain_pending_materializations", + materialize_note, + ) + + content = await doctor_cmd._read_materialized_api_note( + api_file, + "doctor/Doctor API Note.md", + ) + + assert content == "# Doctor API Note" + + +@pytest.mark.asyncio +async def test_doctor_reports_api_note_missing_after_materialization_drain(tmp_path, monkeypatch): + async def drain_without_writing() -> None: + pass + + monkeypatch.setattr( + note_content_materialization, + "drain_pending_materializations", + drain_without_writing, + ) + + with pytest.raises(ValueError, match="API note file missing: doctor/missing.md"): + await doctor_cmd._read_materialized_api_note( + tmp_path / "doctor" / "missing.md", + "doctor/missing.md", + ) diff --git a/tests/conftest.py b/tests/conftest.py index 7d92cd778..ef6cdb9d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -152,6 +152,8 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No """Restore the shared Postgres schema to a clean baseline.""" from basic_memory.models.search import ( CREATE_POSTGRES_SEARCH_INDEX_FTS, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX, + CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE, CREATE_POSTGRES_SEARCH_INDEX_METADATA, CREATE_POSTGRES_SEARCH_INDEX_PERMALINK, CREATE_POSTGRES_SEARCH_INDEX_TABLE, @@ -166,6 +168,8 @@ async def _reset_postgres_test_schema(engine: AsyncEngine, async_url: str) -> No await conn.run_sync(Base.metadata.create_all) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_TABLE) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE) + await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_METADATA) await conn.execute(CREATE_POSTGRES_SEARCH_INDEX_PERMALINK) await conn.execute(CREATE_POSTGRES_SEARCH_VECTOR_CHUNKS_TABLE) diff --git a/tests/markdown/test_relation_edge_cases.py b/tests/markdown/test_relation_edge_cases.py index e38112bcc..9bcc73835 100644 --- a/tests/markdown/test_relation_edge_cases.py +++ b/tests/markdown/test_relation_edge_cases.py @@ -2,7 +2,13 @@ from markdown_it import MarkdownIt -from basic_memory.markdown.plugins import relation_plugin, parse_relation, parse_inline_relations +from basic_memory.markdown.plugins import ( + observation_plugin, + relation_plugin, + parse_relation, + parse_inline_relations, +) +from basic_memory.markdown.entity_parser import parse from basic_memory.markdown.schemas import Relation @@ -247,3 +253,56 @@ def test_bare_list_wikilink_is_inline_link_not_default_explicit_relation(): assert token.meta["relations"] == [{"type": "links_to", "target": "Target", "context": None}] assert parse_relation(token) is None + + +def test_links_to_directive_forces_implicit_relations(): + """The terminal directive disambiguates a single-token relation prefix.""" + md = MarkdownIt().use(relation_plugin) + + tokens = md.parse("- Mother [[Alice]] #bm:links_to") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"] == [{"type": "links_to", "target": "Alice", "context": None}] + + tokens = md.parse("- Mentions [[Alice]] and [[Bob]] #bm:links_to") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"] == [ + {"type": "links_to", "target": "Alice", "context": None}, + {"type": "links_to", "target": "Bob", "context": None}, + ] + + +def test_links_to_directive_is_terminal_and_explicit_relations_are_unchanged(): + """Only the exact terminal directive changes relation interpretation.""" + md = MarkdownIt().use(relation_plugin) + + tokens = md.parse("- spouse_of [[Alice]]") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"][0]["type"] == "spouse_of" + + tokens = md.parse("- Mother [[Alice]] #bm:links_to later") + token = next(t for t in tokens if t.type == "inline") + assert token.meta["relations"][0]["type"] == "links_to" + + +def test_links_to_directive_is_not_an_observation_tag(): + """The directive is syntax, not a note tag or indexed observation text.""" + md = MarkdownIt().use(observation_plugin).use(relation_plugin) + + tokens = md.parse("- Mother [[Alice]] #bm:links_to") + token = next(t for t in tokens if t.type == "inline") + assert "observation" not in token.meta + + +def test_links_to_directive_preserves_source_and_observation_content(): + """The directive stays in source while remaining outside indexed semantics.""" + source = "- [note] Mother [[Alice]] #bm:links_to\n" + parsed = parse(source) + + assert parsed.content == source + assert len(parsed.observations) == 1 + assert parsed.observations[0].category == "note" + assert parsed.observations[0].content == "Mother [[Alice]]" + assert parsed.observations[0].tags is None + assert len(parsed.relations) == 1 + assert parsed.relations[0].type == "links_to" + assert parsed.relations[0].target == "Alice" diff --git a/tests/repository/test_postgres_search_repository.py b/tests/repository/test_postgres_search_repository.py index 61f571067..7682994da 100644 --- a/tests/repository/test_postgres_search_repository.py +++ b/tests/repository/test_postgres_search_repository.py @@ -167,6 +167,68 @@ async def test_postgres_search_repository_index_and_search(session_maker, test_p assert len(results) == 1 +@pytest.mark.asyncio +async def test_postgres_search_indexes_full_note_content(session_maker, test_project): + """An unbounded note is searchable without creating one unbounded tsvector.""" + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + deep_content = "shallowmarker " + ("padding " * 150_000) + "deepmarker" + + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=2, + title="Deep Search Note", + content_stems="deep search note shallowmarker", + content_snippet=deep_content, + permalink="docs/deep-search-note", + file_path="docs/deep-search-note.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + assert len(deep_content.encode()) > 1_048_575 + results = await repo.search(search_text="shallowmarker AND deepmarker") + assert [result.permalink for result in results] == ["docs/deep-search-note"] + assert await repo.count(search_text="shallowmarker AND deepmarker") == 1 + + +@pytest.mark.asyncio +async def test_postgres_search_preserves_long_lexeme_across_chunk_edge( + session_maker, + test_project, +): + """A PostgreSQL-indexable lexeme crossing 8,000 characters remains searchable.""" + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + long_identifier = "lexeme" + ("x" * 394) + content = (" " * 7_700) + long_identifier + (" " * 1_000) + + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=3, + title="Chunk Edge Note", + content_stems="chunk edge note", + content_snippet=content, + permalink="docs/chunk-edge-note", + file_path="docs/chunk-edge-note.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + results = await repo.search(search_text=long_identifier) + + assert [result.permalink for result in results] == ["docs/chunk-edge-note"] + assert await repo.count(search_text=long_identifier) == 1 + + @pytest.mark.asyncio async def test_postgres_search_repository_bulk_index_items_and_prepare_terms( session_maker, test_project @@ -184,6 +246,29 @@ async def test_postgres_search_repository_bulk_index_items_and_prepare_terms( assert repo._prepare_single_term(" ") == " " assert repo._prepare_single_term("coffee", is_prefix=False) == "coffee" + indexed_from, _where, indexed_params, _order, _score = await repo._build_fts_query_parts( + search_text="coffee brewing", + allow_relaxed=True, + ) + assert "FROM search_index AS candidate_parent" in indexed_from + assert "FROM search_index_fts_chunks AS candidate_chunk" in indexed_from + assert "querytree(to_tsquery('english', :text))" in indexed_from + assert indexed_params["text_candidate"] == "coffee:* | brewing:*" + + filtered_from, _where, _params, _order, _score = await repo._build_fts_query_parts( + search_text="coffee brewing", + metadata_filters={"status": "active"}, + ) + assert "AS fts_candidate" in filtered_from + assert "JOIN entity ON search_index.entity_id = entity.id" in filtered_from + + negated_from, _where, negated_params, _order, _score = await repo._build_fts_query_parts( + search_text="coffee NOT brewing", + ) + assert "AS fts_candidate" in negated_from + assert "FROM search_index AS candidate_all" in negated_from + assert negated_params["text_candidate"] == "coffee | brewing" + now = datetime.now(timezone.utc) rows = [ SearchIndexRow( @@ -221,6 +306,10 @@ async def test_postgres_search_repository_bulk_index_items_and_prepare_terms( assert "docs/pour-over" in permalinks assert "docs/french-press" in permalinks + negated_results = await repo.search(search_text="coffee NOT french") + assert [result.permalink for result in negated_results] == ["docs/pour-over"] + assert await repo.count(search_text="coffee NOT french") == 1 + @pytest.mark.asyncio async def test_postgres_search_repository_wildcard_text_and_permalink_match_exact( @@ -327,6 +416,7 @@ async def test_postgres_search_repository_reraises_non_tsquery_db_errors( from basic_memory import db async with db.scoped_session(session_maker) as session: + await session.execute(text("DROP TABLE search_index_fts_chunks")) await session.execute(text("DROP TABLE search_index")) await session.commit() @@ -1162,3 +1252,96 @@ def record_syntax_error(exc: Exception) -> bool: search_syntax_error_count = len(syntax_errors) assert await repo.count(search_text=query, allow_relaxed=True) == 1 assert len(syntax_errors) > search_syntax_error_count + + +@pytest.mark.asyncio +async def test_postgres_relaxed_retry_probes_joined_formatting_characters( + session_maker, + test_project, +): + """Relaxed probes include words joined after removing formatting characters.""" + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=79, + title="Joined word reference", + content_stems="a document containing foobar", + content_snippet="A document containing foobar.", + permalink="docs/joined-word-reference", + file_path="docs/joined-word-reference.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + query = "foo\u00adbar absentone absenttwo" + results = await repo.search(search_text=query, allow_relaxed=True) + + assert any(row.id == 79 for row in results) + assert await repo.count(search_text=query, allow_relaxed=True) == 1 + + +@pytest.mark.asyncio +async def test_postgres_relaxed_retry_quotes_apostrophe_operands( + session_maker, + test_project, +): + """An apostrophe syntax error does not poison relaxed candidate probes.""" + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=81, + title="Contraction reference", + content_stems="can't sunrise", + content_snippet="Can't miss the sunrise.", + permalink="docs/contraction-reference", + file_path="docs/contraction-reference.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + query = "can't find sunrise" + results = await repo.search(search_text=query, allow_relaxed=True) + + assert any(row.id == 81 for row in results) + assert await repo.count(search_text=query, allow_relaxed=True) == 1 + + +@pytest.mark.asyncio +async def test_postgres_search_supports_more_than_one_hundred_operands( + session_maker, + test_project, +): + """Synthetic document vectors do not exceed PostgreSQL's function argument limit.""" + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=80, + title="Long query reference", + content_stems="a document containing targetterm", + content_snippet="A document containing targetterm.", + permalink="docs/long-query-reference", + file_path="docs/long-query-reference.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + query = " ".join(["targetterm", *(f"absent{index}" for index in range(100))]) + results = await repo.search(search_text=query, allow_relaxed=True) + + assert any(row.id == 80 for row in results) + assert await repo.count(search_text=query, allow_relaxed=True) == 1 diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 52c2cae3c..8eb049033 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -804,6 +804,8 @@ async def test_fts5_error_handling_database_error(self, search_repository): # Force a real database error (not an FTS5 syntax error) by removing the search index. # The repository should re-raise the error rather than returning an empty list. async with db.scoped_session(search_repository.session_maker) as session: + if is_postgres_backend(search_repository): + await session.execute(text("DROP TABLE IF EXISTS search_index_fts_chunks")) await session.execute(text("DROP TABLE IF EXISTS search_index")) await session.commit() diff --git a/tests/test_postgres_full_content_search_migration.py b/tests/test_postgres_full_content_search_migration.py new file mode 100644 index 000000000..8ee664a31 --- /dev/null +++ b/tests/test_postgres_full_content_search_migration.py @@ -0,0 +1,54 @@ +"""Tests for the PostgreSQL full-content FTS migration.""" + +from importlib import import_module +from types import SimpleNamespace + + +migration = import_module( + "basic_memory.alembic.versions.7f6a2b8c9d10_index_full_postgres_note_content" +) + + +def _connection(dialect_name: str) -> SimpleNamespace: + return SimpleNamespace(dialect=SimpleNamespace(name=dialect_name)) + + +def test_upgrade_creates_and_backfills_bounded_postgres_vectors(monkeypatch) -> None: + statements: list[str] = [] + monkeypatch.setattr(migration.op, "get_bind", lambda: _connection("postgresql")) + monkeypatch.setattr( + migration.op, "execute", lambda statement: statements.append(str(statement)) + ) + + migration.upgrade() + + assert "CREATE TABLE search_index_fts_chunks" in statements[0] + assert "to_tsvector('english', chunk_text)" in statements[0] + assert "generate_series" in statements[1] + assert "FOR 8000" in statements[1] + assert "5952" in statements[1] + assert "2,048-character overlap" in statements[1] + assert "CREATE INDEX idx_search_index_fts_chunks_fts" in statements[2] + + +def test_downgrade_drops_postgres_chunk_table(monkeypatch) -> None: + statements: list[str] = [] + monkeypatch.setattr(migration.op, "get_bind", lambda: _connection("postgresql")) + monkeypatch.setattr( + migration.op, "execute", lambda statement: statements.append(str(statement)) + ) + + migration.downgrade() + + assert statements == ["DROP TABLE IF EXISTS search_index_fts_chunks"] + + +def test_migration_is_noop_for_sqlite(monkeypatch) -> None: + execute_calls: list[object] = [] + monkeypatch.setattr(migration.op, "get_bind", lambda: _connection("sqlite")) + monkeypatch.setattr(migration.op, "execute", execute_calls.append) + + migration.upgrade() + migration.downgrade() + + assert execute_calls == []