Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/NOTE-FORMAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,17 @@ Explicit relations:
- 'in response to' [[Incident Review]]
```

When the text before `[[` is ordinary prose rather than a relation type, add
`#bm:links_to` to force the reference to use the implicit `links_to` relation:

```markdown
- Mother [[Alice]] #bm:links_to
```

The directive is parser metadata and is not included in the observation or
relation context. It is especially useful when a single-token prose prefix
would otherwise be interpreted as an explicit relation type.

Bare wiki links and prose list items create implicit `links_to` relations:

```markdown
Expand Down
1 change: 0 additions & 1 deletion justfile
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,6 @@ doctor:
TMP_HOME=$(mktemp -d)
TMP_CONFIG=$(mktemp -d)
HOME="$TMP_HOME" \
BASIC_MEMORY_ENV=test \
BASIC_MEMORY_HOME="$TMP_HOME/basic-memory" \
BASIC_MEMORY_CONFIG_DIR="$TMP_CONFIG" \
./.venv/bin/python -m basic_memory.cli.main doctor --local
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
"""Index complete PostgreSQL note content for full-text search.

Revision ID: 7f6a2b8c9d10
Revises: 2d26b287813b
Create Date: 2026-08-24 22:00:00.000000

"""

from typing import Sequence, Union

from alembic import op


# revision identifiers, used by Alembic.
revision: str = "7f6a2b8c9d10"
down_revision: Union[str, None] = "2d26b287813b"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None


def upgrade() -> None:
"""Index complete note bodies as bounded PostgreSQL FTS chunks."""
connection = op.get_bind()
if connection.dialect.name == "postgresql":
op.execute("""
CREATE TABLE search_index_fts_chunks (
project_id INTEGER NOT NULL,
search_index_id INTEGER NOT NULL,
search_index_type VARCHAR NOT NULL,
chunk_index INTEGER NOT NULL,
chunk_text TEXT NOT NULL,
textsearchable_index_col tsvector GENERATED ALWAYS AS (
to_tsvector('english', chunk_text)
) STORED,
PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index),
FOREIGN KEY (search_index_id, search_index_type, project_id)
REFERENCES search_index(id, type, project_id)
ON UPDATE CASCADE ON DELETE CASCADE
)
""")
op.execute("""
-- PostgreSQL ignores lexemes at 2 KiB and above. Stepping 5,952
-- characters leaves a conservative 2,048-character overlap, so
-- every indexable lexeme split at one edge is complete in the next.
INSERT INTO search_index_fts_chunks (
project_id,
search_index_id,
search_index_type,
chunk_index,
chunk_text
)
SELECT
search_index.project_id,
search_index.id,
search_index.type,
(chunk_start - 1) / 5952,
substring(search_index.content_snippet FROM chunk_start FOR 8000)
FROM search_index
CROSS JOIN LATERAL generate_series(
1,
length(search_index.content_snippet),
5952
) AS chunk_start
WHERE search_index.content_snippet IS NOT NULL
AND search_index.content_snippet <> ''
""")
op.execute("""
CREATE INDEX idx_search_index_fts_chunks_fts
ON search_index_fts_chunks USING gin(textsearchable_index_col)
""")


def downgrade() -> None:
"""Remove the bounded full-content PostgreSQL FTS chunks."""
connection = op.get_bind()
if connection.dialect.name == "postgresql":
op.execute("DROP TABLE IF EXISTS search_index_fts_chunks")
38 changes: 35 additions & 3 deletions src/basic_memory/cli/auto_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,38 @@ def _preload_lazy_console_modules() -> None:
"""
import rich._emoji_codes # noqa: F401
import typer.rich_utils # noqa: F401
from rich.cells import cell_len

# Trigger: rich defers its Unicode cell-width table (`rich._unicode_data.
# unicode<version>`) until the first character it cannot measure with the
# ASCII fast path in `_cell_len`.
# Why: status messages echo captured `brew`/`uv` output, which carries
# non-ASCII characters (curly quotes, em dashes, warning glyphs), so the
# deferred import lands after the upgrade removed our files. Importing the
# module by name would hard-code a table version; calling `cell_len` uses
# rich's own resolution and honors UNICODE_VERSION like the print path does.
# Outcome: the table rich will reach for is resolved and cached up front.
cell_len("\u2500\u2018\u2713")


def print_update_status(console: Console, text: str, style: str) -> None:
"""Print an update status line that cannot fail the command.

Trigger: the line is printed after an in-place upgrade may already have
replaced this installation on disk.
Why: `_preload_lazy_console_modules` can only preload the deferred imports
we know about today, and rich/typer are free to add more. By the time this
prints, the upgrade has already succeeded -- a status line must never be
what turns it into a traceback and a non-zero exit.
Outcome: fall back to a plain, unstyled write that needs no new imports.
"""
try:
console.print(f"[{style}]{text}[/{style}]")
except Exception as exc:
logger.warning(
f"Rich console print failed after update, falling back to plain output: {exc}"
)
print(text)


def _save_last_checked_timestamp(config_manager: ConfigManager, checked_at: datetime) -> None:
Expand Down Expand Up @@ -468,11 +500,11 @@ def maybe_run_periodic_auto_update(
}:
out = console or Console()
if result.status == AutoUpdateStatus.UPDATED:
out.print(f"[green]{result.message}[/green]")
print_update_status(out, f"{result.message}", "green")
elif result.status == AutoUpdateStatus.FAILED:
error_detail = f" {result.error}" if result.error else ""
out.print(f"[yellow]{result.message}{error_detail}[/yellow]")
print_update_status(out, f"{result.message}{error_detail}", "yellow")
elif result.message:
out.print(f"[cyan]{result.message}[/cyan]")
print_update_status(out, f"{result.message}", "cyan")

return result
22 changes: 18 additions & 4 deletions src/basic_memory/cli/commands/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,23 @@ async def _delete_doctor_project(
await _delete_doctor_project_locally(project_name, project_id)


async def _read_materialized_api_note(api_file: Path, file_path: str) -> str:
"""Wait for the accepted local write, then read its canonical file."""
# Deferred: importing the materialization runtime at CLI module load would slow every
# command, while doctor is the only command that needs to observe its write immediately.
from basic_memory.index.note_content_materialization import drain_pending_materializations

# Trigger: production accepts note content before its markdown file is materialized.
# Why: doctor verifies the complete DB -> file contract, not only write acceptance.
# Outcome: wait for the same deferred work drained at normal CLI shutdown before checking.
await drain_pending_materializations()

if not api_file.exists():
raise ValueError(f"API note file missing: {file_path}")

return api_file.read_text(encoding="utf-8")


async def run_doctor() -> None:
"""Run local consistency checks for file <-> database flows."""
# Deferred: the markdown parsing stack is only needed while the checks run,
Expand Down Expand Up @@ -132,10 +149,7 @@ async def run_doctor() -> None:
api_result = await knowledge_client.create_entity(api_note.model_dump())

api_file = project_path / api_result.file_path
if not api_file.exists():
raise ValueError(f"API note file missing: {api_result.file_path}")

api_text = api_file.read_text(encoding="utf-8")
api_text = await _read_materialized_api_note(api_file, api_result.file_path)
if api_note_title not in api_text:
raise ValueError("API note content missing from file")

Expand Down
14 changes: 8 additions & 6 deletions src/basic_memory/cli/commands/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from rich.console import Console

from basic_memory.cli.app import app
from basic_memory.cli.auto_update import AutoUpdateStatus, run_auto_update
from basic_memory.cli.auto_update import AutoUpdateStatus, print_update_status, run_auto_update

console = Console()

Expand All @@ -22,19 +22,21 @@ def update(

if result.status == AutoUpdateStatus.FAILED:
detail = f" {result.error}" if result.error else ""
console.print(f"[red]{result.message or 'Update failed.'}{detail}[/red]")
print_update_status(console, f"{result.message or 'Update failed.'}{detail}", "red")
raise typer.Exit(1)

if result.status == AutoUpdateStatus.UPDATED:
console.print(f"[green]{result.message or 'Basic Memory updated successfully.'}[/green]")
print_update_status(
console, f"{result.message or 'Basic Memory updated successfully.'}", "green"
)
return

if result.status == AutoUpdateStatus.UP_TO_DATE:
console.print(f"[green]{result.message or 'Basic Memory is up to date.'}[/green]")
print_update_status(console, f"{result.message or 'Basic Memory is up to date.'}", "green")
return

if result.status == AutoUpdateStatus.UPDATE_AVAILABLE:
console.print(f"[cyan]{result.message or 'Update available.'}[/cyan]")
print_update_status(console, f"{result.message or 'Update available.'}", "cyan")
return

console.print(f"[dim]{result.message or 'No update action was performed.'}[/dim]")
print_update_status(console, f"{result.message or 'No update action was performed.'}", "dim")
19 changes: 16 additions & 3 deletions src/basic_memory/markdown/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@
# those bracket prefixes stay ordinary content (issue #1219).
_TIMESTAMP_VALUE = r"\d{1,3}:\d{2}(?::\d{2})?(?:[.,]\d{1,3})?"
_TIMESTAMP_CATEGORY = re.compile(rf"^{_TIMESTAMP_VALUE}(?:\s+-\s+{_TIMESTAMP_VALUE})?$")
_LINKS_TO_DIRECTIVE = re.compile(r"\s+#bm:links_to\s*$")


def remove_links_to_directive(content: str) -> tuple[str, bool]:
"""Remove an exact terminal ``#bm:links_to`` directive from content."""
match = _LINKS_TO_DIRECTIVE.search(content)
if not match:
return content, False
return content[: match.start()].rstrip(), True


def _is_task_marker_category(category: str) -> bool:
Expand Down Expand Up @@ -47,6 +56,7 @@ def is_observation(token: Token) -> bool:
return False
# Use token.tag which contains the actual content for test tokens, fallback to content
content = (token.tag or token.content).strip()
content, _ = remove_links_to_directive(content)
if not content: # pragma: no cover
return False
# if it's a markdown_task, return false
Expand Down Expand Up @@ -74,6 +84,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:

# Use token.tag which contains the actual content for test tokens, fallback to content
content = (token.tag or token.content).strip()
content, _ = remove_links_to_directive(content)

# Parse [category] with regex; a timestamp-shaped prefix is not a category, so a
# hashtag-promoted transcript line keeps its timecode inside the content instead.
Expand Down Expand Up @@ -325,17 +336,19 @@ def relation_rule(state: Any) -> None:

# Only process inline tokens
if token.type == "inline":
content = token.tag or token.content
content_without_directive, has_directive = remove_links_to_directive(content)

# Check for explicit relations in list items
if in_list_item and is_explicit_relation(token):
if in_list_item and not has_directive and is_explicit_relation(token):
rel = parse_relation(token)
if rel:
token.meta["relations"] = [rel]

# Always check for inline links in any text
else:
content = token.tag or token.content
if "[[" in content:
rels = parse_inline_relations(content)
rels = parse_inline_relations(content_without_directive)
if rels:
token.meta["relations"] = token.meta.get("relations", []) + rels

Expand Down
3 changes: 3 additions & 0 deletions src/basic_memory/mcp/tools/write_note.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,15 @@ async def write_note(
- Explicit: `- relation_type [[Entity]] (optional context)`
- Quoted: `- "multi word relation type" [[Entity]] (optional context)`
- Quoted: `- 'multi word relation type' [[Entity]] (optional context)`
- Disambiguation: Add `#bm:links_to` when prose before `[[Entity]]`
must not be treated as a single-token relation type
- Inline: Any other `[[Entity]]` reference creates a `links_to` relation

Examples:
`- depends_on [[Content Parser]] (Need for semantic extraction)`
`- "based on" [[Design Notes]]`
`- 'in response to' [[Incident Review]]`
`- Mother [[Alice]] #bm:links_to`
`- implements [[Search Spec]] (Initial implementation)`
`- This feature extends [[Base Design]] and uses [[Core Utils]]`

Expand Down
30 changes: 29 additions & 1 deletion src/basic_memory/models/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@
created_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE,
textsearchable_index_col tsvector GENERATED ALWAYS AS (
to_tsvector('english', coalesce(title, '') || ' ' || coalesce(content_stems, ''))
to_tsvector(
'english',
coalesce(title, '') || ' ' ||
coalesce(content_stems, '')
)
) STORED,
PRIMARY KEY (id, type, project_id),
FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE
Expand All @@ -44,6 +48,30 @@
CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col)
""")

# Full note bodies are stored in bounded child rows so one unusually large note
# cannot exceed PostgreSQL's per-tsvector size limit.
CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_TABLE = DDL("""
CREATE TABLE IF NOT EXISTS search_index_fts_chunks (
project_id INTEGER NOT NULL,
search_index_id INTEGER NOT NULL,
search_index_type VARCHAR NOT NULL,
chunk_index INTEGER NOT NULL,
chunk_text TEXT NOT NULL,
textsearchable_index_col tsvector GENERATED ALWAYS AS (
to_tsvector('english', chunk_text)
) STORED,
PRIMARY KEY (project_id, search_index_id, search_index_type, chunk_index),
FOREIGN KEY (search_index_id, search_index_type, project_id)
REFERENCES search_index(id, type, project_id)
ON UPDATE CASCADE ON DELETE CASCADE
)
""")

CREATE_POSTGRES_SEARCH_INDEX_FTS_CHUNKS_INDEX = DDL("""
CREATE INDEX IF NOT EXISTS idx_search_index_fts_chunks_fts
ON search_index_fts_chunks USING gin(textsearchable_index_col)
""")

CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL("""
CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops)
""")
Expand Down
Loading
Loading