Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion src/basic_memory/repository/entity_repository.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
"""Repository for managing entities in the knowledge graph."""

import unicodedata
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import override, List, Optional, Sequence, Union, Any


from loguru import logger
from sqlalchemy import case, exists, func, or_, select
from sqlalchemy.exc import IntegrityError
Expand All @@ -21,6 +21,12 @@
type EntityMetadata = dict[str, Any] | None


def file_path_alias(file_path: Union[Path, str]) -> str:
"""Return the forgiving filename alias used only after exact path lookup misses."""
normalized_path = unicodedata.normalize("NFC", Path(file_path).as_posix())
return normalized_path.casefold().replace("_", "-")


@dataclass(frozen=True, slots=True)
class AcceptedPendingEntityWrite:
"""Entity fields accepted by the database before the source file is materialized."""
Expand Down Expand Up @@ -195,6 +201,40 @@ async def get_by_file_path(
lock_for_update=lock_for_update,
)

async def get_unique_by_file_path_alias(
self,
session: AsyncSession,
file_path: Union[Path, str],
*,
load_relations: bool = True,
) -> Optional[Entity]:
"""Resolve one case-insensitive underscore/hyphen file-path alias.

Exact paths remain the canonical identity and are queried separately by callers. This
fallback only returns an entity when the forgiving alias is unique within the project;
colliding files such as ``alpha-note.md`` and ``alpha_note.md`` remain unresolved.
"""
normalized_alias = file_path_alias(file_path)

# SQLite's lower() only folds ASCII while Postgres follows its configured collation.
# Compare the lightweight identity rows in Python so both backends apply the same
# Unicode casefolding rules. This path runs only after exact semantic/path matches miss;
# bulk relation resolution builds the equivalent project-wide index once instead.
query = self.select(Entity.id, Entity.file_path)
result = await self.execute_query(session, query, use_query_options=False)
matching_ids = [
entity_id
for entity_id, stored_file_path in result.all()
if file_path_alias(stored_file_path) == normalized_alias
]
if len(matching_ids) != 1:
return None
return await self.get_by_id(
session,
matching_ids[0],
load_relations=load_relations,
)

# -------------------------------------------------------------------------
# Lightweight methods for permalink resolution (no eager loading)
# -------------------------------------------------------------------------
Expand Down
23 changes: 20 additions & 3 deletions src/basic_memory/services/bulk_link_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from basic_memory.config import BasicMemoryConfig
from basic_memory.models import Entity, Project
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.repository.entity_repository import EntityRepository, file_path_alias
from basic_memory.repository.project_repository import ProjectRepository
from basic_memory.services.link_resolver import normalize_link_text
from basic_memory.utils import (
Expand Down Expand Up @@ -100,6 +100,7 @@ class ProjectEntityIdentityIndex:
by_permalink: Mapping[str, Entity]
by_title: Mapping[str, tuple[Entity, ...]]
by_file_path: Mapping[str, Entity]
by_file_path_alias: Mapping[str, Entity]

@classmethod
def from_entities(
Expand All @@ -112,13 +113,15 @@ def from_entities(
by_external_id: dict[str, Entity] = {}
by_permalink: dict[str, Entity] = {}
by_file_path: dict[str, Entity] = {}
file_path_alias_matches: dict[str, list[Entity]] = {}

for entity in entities:
by_external_id[entity.external_id] = entity
if entity.permalink is not None:
by_permalink[entity.permalink] = entity
title_matches.setdefault(entity.title, []).append(entity)
by_file_path[entity.file_path] = entity
file_path_alias_matches.setdefault(file_path_alias(entity.file_path), []).append(entity)

return cls(
project=project,
Expand All @@ -131,6 +134,11 @@ def from_entities(
for title, matches in title_matches.items()
},
by_file_path=by_file_path,
by_file_path_alias={
alias: matches[0]
for alias, matches in file_path_alias_matches.items()
if len(matches) == 1
},
)

def resolve_strict(
Expand Down Expand Up @@ -167,11 +175,20 @@ def resolve_strict(
if path_match is not None:
return StrictProjectLinkMatch(path_match)

if not normalized_path.endswith(".md") and "/" in normalized_path:
path_match = self.by_file_path.get(f"{normalized_path}.md")
path_with_md = normalized_path
can_use_stem_path = "/" in normalized_path or not ambiguous_title
has_markdown_extension = normalized_path.casefold().endswith(".md")
if not has_markdown_extension and can_use_stem_path:
path_with_md = f"{normalized_path}.md"
path_match = self.by_file_path.get(path_with_md)
if path_match is not None:
return StrictProjectLinkMatch(path_match)

if has_markdown_extension or can_use_stem_path:
alias_match = self.by_file_path_alias.get(file_path_alias(path_with_md))
if alias_match is not None:
return StrictProjectLinkMatch(alias_match)

return StrictProjectLinkMatch(entity=None, ambiguous=ambiguous_title)


Expand Down
23 changes: 14 additions & 9 deletions src/basic_memory/services/entity_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -446,17 +446,22 @@ async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityMod
f"Creating or updating entity: {schema.file_path}, permalink: {schema.permalink}"
)

existing = await self.link_resolver.resolve_link(
schema.file_path,
strict=True,
load_relations=False,
)
if not existing and schema.permalink:
existing = await self.link_resolver.resolve_link(
schema.permalink,
strict=True,
# Canonical writes may update only the exact file path or permalink requested by the
# caller. Forgiving link aliases are intentionally excluded: treating ``alpha-note.md``
# as the existing ``alpha_note.md`` here would move/overwrite the canonical note instead
# of creating the distinct file the caller requested.
async with db.scoped_session(self.session_maker) as session:
existing = await self.repository.get_by_file_path(
session,
schema.file_path,
load_relations=False,
)
if not existing and schema.permalink:
existing = await self.repository.get_by_permalink(
session,
schema.permalink,
load_relations=False,
)

if existing:
logger.debug(f"Found existing entity: {existing.file_path}")
Expand Down
55 changes: 50 additions & 5 deletions src/basic_memory/services/link_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,9 @@ class LinkResolver:
1. Try exact permalink match (fastest)
2. Try exact title match
3. Try exact file path match
4. Try file path with .md extension (for folder/title patterns)
5. Fall back to search for fuzzy matching
4. Try file path with .md extension (for root or folder/title patterns)
5. Try a unique case-insensitive underscore/hyphen file-path alias
6. Fall back to search for fuzzy matching

When ``strict`` is set (the destructive edit/move paths) and a title matches more than one
note, resolution raises ``AmbiguousIdentifierError`` instead of guessing — the caller must pass
Expand Down Expand Up @@ -330,6 +331,7 @@ async def _resolve_in_project(
# Trigger: source_path is provided AND link contains "/"
# Why: Resolve paths like [[nested/deep-note]] relative to source folder first
# Outcome: [[nested/deep-note]] from testing/link-test.md → testing/nested/deep-note.md
deferred_relative_alias_path: Optional[str] = None
if source_path and "/" in clean_text:
if not (
include_project
Expand Down Expand Up @@ -361,6 +363,16 @@ async def _resolve_in_project(
if entity:
return entity

# Only exact relative paths may resolve this early. The forgiving
# alias spelling is deferred until every exact identity — permalink,
# title, file path — has missed, or a nearby alias would shadow
# another note's exact permalink.
deferred_relative_alias_path = (
relative_path
if relative_path.casefold().endswith(".md")
else f"{relative_path}.md"
)

# When source_path is provided, use context-aware resolution:
# Check both permalink and title matches, prefer closest to source.
# Example: [[testing]] from folder/note.md prefers folder/testing.md
Expand Down Expand Up @@ -463,8 +475,14 @@ async def _resolve_in_project(
logger.debug(f"Found entity with path: {found_path.file_path}")
return found_path

# 4. Try file path with .md extension if not already present
if not clean_text.endswith(".md") and "/" in clean_text:
# 4. Try file path with .md extension if not already present. Root-level
# filename links need the same fallback as nested paths (#1253). Under a
# strict resolve, a duplicated bare title remains ambiguous; callers can
# disambiguate it by supplying the exact ``.md`` file path as before.
file_path_with_md = clean_text
can_use_stem_path = "/" in clean_text or not ambiguous_title_candidates
has_markdown_extension = clean_text.casefold().endswith(".md")
if not has_markdown_extension and can_use_stem_path:
file_path_with_md = f"{clean_text}.md"
found_path_md = await entity_repository.get_by_file_path(
session,
Expand All @@ -475,6 +493,33 @@ async def _resolve_in_project(
logger.debug(f"Found entity with path (with .md): {found_path_md.file_path}")
return found_path_md

# 5. Direct-on-disk vaults commonly mix filename separators and case in
# wikilinks. Treat those spellings as aliases only after every exact semantic
# and path lookup, and only when the alias identifies one file uniquely.
# A source-relative alias is the closest forgiving spelling, so it is tried
# ahead of the project-wide one.
if deferred_relative_alias_path:
relative_alias_entity = await entity_repository.get_unique_by_file_path_alias(
session,
deferred_relative_alias_path,
load_relations=load_relations,
)
if relative_alias_entity:
logger.debug(
f"Found entity by relative file path alias: {relative_alias_entity.file_path}"
)
return relative_alias_entity

if has_markdown_extension or can_use_stem_path:
alias_path = await entity_repository.get_unique_by_file_path_alias(
session,
file_path_with_md,
load_relations=load_relations,
)
Comment thread
phernandez marked this conversation as resolved.
if alias_path:
logger.debug(f"Found entity by unique file path alias: {alias_path.file_path}")
return alias_path

# No exact permalink or file path matched. If the only thing that matched was a title
# shared by several notes under a strict resolve, refuse to guess (#1148).
if ambiguous_title_candidates:
Expand All @@ -487,7 +532,7 @@ async def _resolve_in_project(
if strict:
return None

# 5. Fall back to search for fuzzy matching (only if not in strict mode)
# 6. Fall back to search for fuzzy matching (only if not in strict mode)
if use_search and "*" not in clean_text:
results = await search_service.search(
query=SearchQuery(text=clean_text, entity_types=[SearchItemType.ENTITY]),
Expand Down
78 changes: 77 additions & 1 deletion tests/services/test_bulk_link_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,43 @@ async def test_bulk_resolution_normalizes_file_paths(
updated_at=now,
project_id=project_id,
)
root_filename_note = Entity(
title="Alpha — a descriptive human title",
note_type="note",
content_type="text/markdown",
file_path="alpha_note.md",
permalink="descriptive-alpha",
created_at=now,
updated_at=now,
project_id=project_id,
)
unicode_filename_note = Entity(
title="A descriptive French school title",
note_type="note",
content_type="text/markdown",
file_path="École_note.md",
permalink="descriptive-school",
created_at=now,
updated_at=now,
project_id=project_id,
)
async with db.scoped_session(session_maker) as session:
await entity_repository.add(session, custom_permalink_note)
await entity_repository.add(session, root_filename_note)
await entity_repository.add(session, unicode_filename_note)

resolver = BulkLinkResolver(entity_repository, app_config)

async with db.scoped_session(session_maker) as session:
results = await resolver.resolve_relation_targets(
["./assets//image.png", "docs/Guide"],
[
"./assets//image.png",
"docs/Guide",
"alpha_note",
"alpha-note",
"ALPHA-NOTE.MD",
"école-note",
],
session=session,
)

Expand All @@ -184,6 +213,53 @@ async def test_bulk_resolution_normalizes_file_paths(
resolved_guide = results["docs/Guide"]
assert resolved_guide is not None
assert resolved_guide.id == custom_permalink_note.id
for identifier in ("alpha_note", "alpha-note", "ALPHA-NOTE.MD"):
resolved_root_note = results[identifier]
assert resolved_root_note is not None
assert resolved_root_note.id == root_filename_note.id
resolved_unicode_note = results["école-note"]
assert resolved_unicode_note is not None
assert resolved_unicode_note.id == unicode_filename_note.id


def test_project_entity_index_declines_ambiguous_file_path_aliases(
test_project: Project,
bulk_entities: list[Entity],
) -> None:
"""Bulk resolution never chooses between colliding underscore/hyphen aliases."""
now = datetime.now(timezone.utc)
project_id = test_project.id
aliases = [
Entity(
title="Hyphenated target",
note_type="note",
content_type="text/markdown",
file_path="alpha-note.md",
permalink="hyphenated-target",
created_at=now,
updated_at=now,
project_id=project_id,
),
Entity(
title="Underscored target",
note_type="note",
content_type="text/markdown",
file_path="alpha_note.md",
permalink="underscored-target",
created_at=now,
updated_at=now,
project_id=project_id,
),
]
index = ProjectEntityIdentityIndex.from_entities(test_project, [*bulk_entities, *aliases])

match = index.resolve_strict(
"ALPHA-NOTE",
include_project_permalinks=False,
workspace_permalink=None,
)

assert match.entity is None


def test_project_entity_index_reports_title_derived_permalink_ambiguity(
Expand Down
Loading
Loading