diff --git a/src/basic_memory/repository/entity_repository.py b/src/basic_memory/repository/entity_repository.py index 5d117300b..a35663141 100644 --- a/src/basic_memory/repository/entity_repository.py +++ b/src/basic_memory/repository/entity_repository.py @@ -1,11 +1,11 @@ """Repository for managing entities in the knowledge graph.""" +import unicodedata from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import override, List, Optional, Sequence, Union, Any - from loguru import logger from sqlalchemy import case, exists, func, or_, select from sqlalchemy.exc import IntegrityError @@ -21,6 +21,12 @@ type EntityMetadata = dict[str, Any] | None +def file_path_alias(file_path: Union[Path, str]) -> str: + """Return the forgiving filename alias used only after exact path lookup misses.""" + normalized_path = unicodedata.normalize("NFC", Path(file_path).as_posix()) + return normalized_path.casefold().replace("_", "-") + + @dataclass(frozen=True, slots=True) class AcceptedPendingEntityWrite: """Entity fields accepted by the database before the source file is materialized.""" @@ -195,6 +201,40 @@ async def get_by_file_path( lock_for_update=lock_for_update, ) + async def get_unique_by_file_path_alias( + self, + session: AsyncSession, + file_path: Union[Path, str], + *, + load_relations: bool = True, + ) -> Optional[Entity]: + """Resolve one case-insensitive underscore/hyphen file-path alias. + + Exact paths remain the canonical identity and are queried separately by callers. This + fallback only returns an entity when the forgiving alias is unique within the project; + colliding files such as ``alpha-note.md`` and ``alpha_note.md`` remain unresolved. + """ + normalized_alias = file_path_alias(file_path) + + # SQLite's lower() only folds ASCII while Postgres follows its configured collation. + # Compare the lightweight identity rows in Python so both backends apply the same + # Unicode casefolding rules. This path runs only after exact semantic/path matches miss; + # bulk relation resolution builds the equivalent project-wide index once instead. + query = self.select(Entity.id, Entity.file_path) + result = await self.execute_query(session, query, use_query_options=False) + matching_ids = [ + entity_id + for entity_id, stored_file_path in result.all() + if file_path_alias(stored_file_path) == normalized_alias + ] + if len(matching_ids) != 1: + return None + return await self.get_by_id( + session, + matching_ids[0], + load_relations=load_relations, + ) + # ------------------------------------------------------------------------- # Lightweight methods for permalink resolution (no eager loading) # ------------------------------------------------------------------------- diff --git a/src/basic_memory/services/bulk_link_resolver.py b/src/basic_memory/services/bulk_link_resolver.py index 099bc22b0..abb2ac05a 100644 --- a/src/basic_memory/services/bulk_link_resolver.py +++ b/src/basic_memory/services/bulk_link_resolver.py @@ -9,7 +9,7 @@ from basic_memory.config import BasicMemoryConfig from basic_memory.models import Entity, Project -from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.entity_repository import EntityRepository, file_path_alias from basic_memory.repository.project_repository import ProjectRepository from basic_memory.services.link_resolver import normalize_link_text from basic_memory.utils import ( @@ -100,6 +100,7 @@ class ProjectEntityIdentityIndex: by_permalink: Mapping[str, Entity] by_title: Mapping[str, tuple[Entity, ...]] by_file_path: Mapping[str, Entity] + by_file_path_alias: Mapping[str, Entity] @classmethod def from_entities( @@ -112,6 +113,7 @@ def from_entities( by_external_id: dict[str, Entity] = {} by_permalink: dict[str, Entity] = {} by_file_path: dict[str, Entity] = {} + file_path_alias_matches: dict[str, list[Entity]] = {} for entity in entities: by_external_id[entity.external_id] = entity @@ -119,6 +121,7 @@ def from_entities( by_permalink[entity.permalink] = entity title_matches.setdefault(entity.title, []).append(entity) by_file_path[entity.file_path] = entity + file_path_alias_matches.setdefault(file_path_alias(entity.file_path), []).append(entity) return cls( project=project, @@ -131,6 +134,11 @@ def from_entities( for title, matches in title_matches.items() }, by_file_path=by_file_path, + by_file_path_alias={ + alias: matches[0] + for alias, matches in file_path_alias_matches.items() + if len(matches) == 1 + }, ) def resolve_strict( @@ -167,11 +175,20 @@ def resolve_strict( if path_match is not None: return StrictProjectLinkMatch(path_match) - if not normalized_path.endswith(".md") and "/" in normalized_path: - path_match = self.by_file_path.get(f"{normalized_path}.md") + path_with_md = normalized_path + can_use_stem_path = "/" in normalized_path or not ambiguous_title + has_markdown_extension = normalized_path.casefold().endswith(".md") + if not has_markdown_extension and can_use_stem_path: + path_with_md = f"{normalized_path}.md" + path_match = self.by_file_path.get(path_with_md) if path_match is not None: return StrictProjectLinkMatch(path_match) + if has_markdown_extension or can_use_stem_path: + alias_match = self.by_file_path_alias.get(file_path_alias(path_with_md)) + if alias_match is not None: + return StrictProjectLinkMatch(alias_match) + return StrictProjectLinkMatch(entity=None, ambiguous=ambiguous_title) diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 8451ecc30..0affcd1a1 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -446,17 +446,22 @@ async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityMod f"Creating or updating entity: {schema.file_path}, permalink: {schema.permalink}" ) - existing = await self.link_resolver.resolve_link( - schema.file_path, - strict=True, - load_relations=False, - ) - if not existing and schema.permalink: - existing = await self.link_resolver.resolve_link( - schema.permalink, - strict=True, + # Canonical writes may update only the exact file path or permalink requested by the + # caller. Forgiving link aliases are intentionally excluded: treating ``alpha-note.md`` + # as the existing ``alpha_note.md`` here would move/overwrite the canonical note instead + # of creating the distinct file the caller requested. + async with db.scoped_session(self.session_maker) as session: + existing = await self.repository.get_by_file_path( + session, + schema.file_path, load_relations=False, ) + if not existing and schema.permalink: + existing = await self.repository.get_by_permalink( + session, + schema.permalink, + load_relations=False, + ) if existing: logger.debug(f"Found existing entity: {existing.file_path}") diff --git a/src/basic_memory/services/link_resolver.py b/src/basic_memory/services/link_resolver.py index 17479e42a..6bb3cc313 100644 --- a/src/basic_memory/services/link_resolver.py +++ b/src/basic_memory/services/link_resolver.py @@ -92,8 +92,9 @@ class LinkResolver: 1. Try exact permalink match (fastest) 2. Try exact title match 3. Try exact file path match - 4. Try file path with .md extension (for folder/title patterns) - 5. Fall back to search for fuzzy matching + 4. Try file path with .md extension (for root or folder/title patterns) + 5. Try a unique case-insensitive underscore/hyphen file-path alias + 6. Fall back to search for fuzzy matching When ``strict`` is set (the destructive edit/move paths) and a title matches more than one note, resolution raises ``AmbiguousIdentifierError`` instead of guessing — the caller must pass @@ -330,6 +331,7 @@ async def _resolve_in_project( # Trigger: source_path is provided AND link contains "/" # Why: Resolve paths like [[nested/deep-note]] relative to source folder first # Outcome: [[nested/deep-note]] from testing/link-test.md → testing/nested/deep-note.md + deferred_relative_alias_path: Optional[str] = None if source_path and "/" in clean_text: if not ( include_project @@ -361,6 +363,16 @@ async def _resolve_in_project( if entity: return entity + # Only exact relative paths may resolve this early. The forgiving + # alias spelling is deferred until every exact identity — permalink, + # title, file path — has missed, or a nearby alias would shadow + # another note's exact permalink. + deferred_relative_alias_path = ( + relative_path + if relative_path.casefold().endswith(".md") + else f"{relative_path}.md" + ) + # When source_path is provided, use context-aware resolution: # Check both permalink and title matches, prefer closest to source. # Example: [[testing]] from folder/note.md prefers folder/testing.md @@ -463,8 +475,14 @@ async def _resolve_in_project( logger.debug(f"Found entity with path: {found_path.file_path}") return found_path - # 4. Try file path with .md extension if not already present - if not clean_text.endswith(".md") and "/" in clean_text: + # 4. Try file path with .md extension if not already present. Root-level + # filename links need the same fallback as nested paths (#1253). Under a + # strict resolve, a duplicated bare title remains ambiguous; callers can + # disambiguate it by supplying the exact ``.md`` file path as before. + file_path_with_md = clean_text + can_use_stem_path = "/" in clean_text or not ambiguous_title_candidates + has_markdown_extension = clean_text.casefold().endswith(".md") + if not has_markdown_extension and can_use_stem_path: file_path_with_md = f"{clean_text}.md" found_path_md = await entity_repository.get_by_file_path( session, @@ -475,6 +493,33 @@ async def _resolve_in_project( logger.debug(f"Found entity with path (with .md): {found_path_md.file_path}") return found_path_md + # 5. Direct-on-disk vaults commonly mix filename separators and case in + # wikilinks. Treat those spellings as aliases only after every exact semantic + # and path lookup, and only when the alias identifies one file uniquely. + # A source-relative alias is the closest forgiving spelling, so it is tried + # ahead of the project-wide one. + if deferred_relative_alias_path: + relative_alias_entity = await entity_repository.get_unique_by_file_path_alias( + session, + deferred_relative_alias_path, + load_relations=load_relations, + ) + if relative_alias_entity: + logger.debug( + f"Found entity by relative file path alias: {relative_alias_entity.file_path}" + ) + return relative_alias_entity + + if has_markdown_extension or can_use_stem_path: + alias_path = await entity_repository.get_unique_by_file_path_alias( + session, + file_path_with_md, + load_relations=load_relations, + ) + if alias_path: + logger.debug(f"Found entity by unique file path alias: {alias_path.file_path}") + return alias_path + # No exact permalink or file path matched. If the only thing that matched was a title # shared by several notes under a strict resolve, refuse to guess (#1148). if ambiguous_title_candidates: @@ -487,7 +532,7 @@ async def _resolve_in_project( if strict: return None - # 5. Fall back to search for fuzzy matching (only if not in strict mode) + # 6. Fall back to search for fuzzy matching (only if not in strict mode) if use_search and "*" not in clean_text: results = await search_service.search( query=SearchQuery(text=clean_text, entity_types=[SearchItemType.ENTITY]), diff --git a/tests/services/test_bulk_link_resolver.py b/tests/services/test_bulk_link_resolver.py index 573be428e..b33d419ff 100644 --- a/tests/services/test_bulk_link_resolver.py +++ b/tests/services/test_bulk_link_resolver.py @@ -167,14 +167,43 @@ async def test_bulk_resolution_normalizes_file_paths( updated_at=now, project_id=project_id, ) + root_filename_note = Entity( + title="Alpha — a descriptive human title", + note_type="note", + content_type="text/markdown", + file_path="alpha_note.md", + permalink="descriptive-alpha", + created_at=now, + updated_at=now, + project_id=project_id, + ) + unicode_filename_note = Entity( + title="A descriptive French school title", + note_type="note", + content_type="text/markdown", + file_path="École_note.md", + permalink="descriptive-school", + created_at=now, + updated_at=now, + project_id=project_id, + ) async with db.scoped_session(session_maker) as session: await entity_repository.add(session, custom_permalink_note) + await entity_repository.add(session, root_filename_note) + await entity_repository.add(session, unicode_filename_note) resolver = BulkLinkResolver(entity_repository, app_config) async with db.scoped_session(session_maker) as session: results = await resolver.resolve_relation_targets( - ["./assets//image.png", "docs/Guide"], + [ + "./assets//image.png", + "docs/Guide", + "alpha_note", + "alpha-note", + "ALPHA-NOTE.MD", + "école-note", + ], session=session, ) @@ -184,6 +213,53 @@ async def test_bulk_resolution_normalizes_file_paths( resolved_guide = results["docs/Guide"] assert resolved_guide is not None assert resolved_guide.id == custom_permalink_note.id + for identifier in ("alpha_note", "alpha-note", "ALPHA-NOTE.MD"): + resolved_root_note = results[identifier] + assert resolved_root_note is not None + assert resolved_root_note.id == root_filename_note.id + resolved_unicode_note = results["école-note"] + assert resolved_unicode_note is not None + assert resolved_unicode_note.id == unicode_filename_note.id + + +def test_project_entity_index_declines_ambiguous_file_path_aliases( + test_project: Project, + bulk_entities: list[Entity], +) -> None: + """Bulk resolution never chooses between colliding underscore/hyphen aliases.""" + now = datetime.now(timezone.utc) + project_id = test_project.id + aliases = [ + Entity( + title="Hyphenated target", + note_type="note", + content_type="text/markdown", + file_path="alpha-note.md", + permalink="hyphenated-target", + created_at=now, + updated_at=now, + project_id=project_id, + ), + Entity( + title="Underscored target", + note_type="note", + content_type="text/markdown", + file_path="alpha_note.md", + permalink="underscored-target", + created_at=now, + updated_at=now, + project_id=project_id, + ), + ] + index = ProjectEntityIdentityIndex.from_entities(test_project, [*bulk_entities, *aliases]) + + match = index.resolve_strict( + "ALPHA-NOTE", + include_project_permalinks=False, + workspace_permalink=None, + ) + + assert match.entity is None def test_project_entity_index_reports_title_derived_permalink_ambiguity( diff --git a/tests/services/test_entity_service_disable_permalinks.py b/tests/services/test_entity_service_disable_permalinks.py index 22aadc2d6..589c1e106 100644 --- a/tests/services/test_entity_service_disable_permalinks.py +++ b/tests/services/test_entity_service_disable_permalinks.py @@ -58,6 +58,54 @@ async def test_create_entity_with_permalinks_disabled( assert metadata["type"] == "note" +@pytest.mark.asyncio +async def test_create_or_update_keeps_aliasing_file_paths_distinct_without_permalinks( + entity_repository, + observation_repository, + relation_repository, + entity_parser, + file_service: FileService, + link_resolver, + session_maker, +): + """A forgiving read alias must never turn a distinct create into a canonical move.""" + entity_service = EntityService( + entity_parser=entity_parser, + entity_repository=entity_repository, + observation_repository=observation_repository, + relation_repository=relation_repository, + file_service=file_service, + link_resolver=link_resolver, + app_config=BasicMemoryConfig(disable_permalinks=True), + session_maker=session_maker, + ) + + underscored, underscored_created = await entity_service.create_or_update_entity( + EntitySchema( + title="alpha_note", + directory="", + note_type="note", + content="Underscored content", + ) + ) + hyphenated, hyphenated_created = await entity_service.create_or_update_entity( + EntitySchema( + title="alpha-note", + directory="", + note_type="note", + content="Hyphenated content", + ) + ) + + assert underscored_created is True + assert hyphenated_created is True + assert underscored.id != hyphenated.id + assert underscored.file_path == "alpha_note.md" + assert hyphenated.file_path == "alpha-note.md" + assert await file_service.exists(underscored.file_path) + assert await file_service.exists(hyphenated.file_path) + + @pytest.mark.asyncio async def test_update_entity_with_permalinks_disabled( entity_repository, diff --git a/tests/services/test_link_resolver.py b/tests/services/test_link_resolver.py index 6e6d46e6d..c451aa186 100644 --- a/tests/services/test_link_resolver.py +++ b/tests/services/test_link_resolver.py @@ -411,7 +411,7 @@ async def test_folder_title_pattern_with_md_extension(link_resolver, test_entiti assert entity is not None assert entity.permalink == f"{project_prefix}/components/core-service" - # Test that it doesn't try to add .md to single words (no slash) + # A nonexistent root filename still stays unresolved. entity = await link_resolver.resolve_link("NonExistent") assert entity is None @@ -421,6 +421,102 @@ async def test_folder_title_pattern_with_md_extension(link_resolver, test_entiti assert entity.permalink == f"{project_prefix}/components/core-service" +@pytest.mark.asyncio +async def test_root_filename_alias_resolves_descriptive_title( + entity_repository, session_maker, link_resolver +): + """Root filename stems resolve independently of title and permalink (#1253).""" + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + target = await entity_repository.add( + session, + EntityModel( + title="Alpha — a descriptive human title", + note_type="note", + content_type="text/markdown", + file_path="alpha_note.md", + permalink="descriptive-alpha", + created_at=now, + updated_at=now, + project_id=entity_repository.project_id, + ), + ) + + for identifier in ("alpha_note", "alpha-note", "ALPHA-NOTE", "ALPHA-NOTE.MD"): + result = await link_resolver.resolve_link(identifier, strict=True, use_search=False) + assert result is not None, identifier + assert result.id == target.id + + +@pytest.mark.asyncio +async def test_file_path_alias_uses_backend_independent_unicode_casefolding( + entity_repository, session_maker, link_resolver +): + """SQLite and Postgres resolve non-ASCII filename case with identical rules.""" + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + target = await entity_repository.add( + session, + EntityModel( + title="A descriptive French school title", + note_type="note", + content_type="text/markdown", + file_path="École_note.md", + permalink="descriptive-school", + created_at=now, + updated_at=now, + project_id=entity_repository.project_id, + ), + ) + + result = await link_resolver.resolve_link("école-note", strict=True, use_search=False) + + assert result is not None + assert result.id == target.id + + +@pytest.mark.asyncio +async def test_file_path_alias_collision_does_not_guess( + entity_repository, session_maker, link_resolver +): + """A forgiving alias must stay unresolved when two exact paths normalize to it.""" + now = datetime.now(timezone.utc) + async with db.scoped_session(session_maker) as session: + hyphenated = await entity_repository.add( + session, + EntityModel( + title="Hyphenated target", + note_type="note", + content_type="text/markdown", + file_path="alpha-note.md", + permalink="hyphenated-target", + created_at=now, + updated_at=now, + project_id=entity_repository.project_id, + ), + ) + await entity_repository.add( + session, + EntityModel( + title="Underscored target", + note_type="note", + content_type="text/markdown", + file_path="alpha_note.md", + permalink="underscored-target", + created_at=now, + updated_at=now, + project_id=entity_repository.project_id, + ), + ) + + ambiguous = await link_resolver.resolve_link("ALPHA-NOTE", use_search=False) + assert ambiguous is None + + exact = await link_resolver.resolve_link("alpha-note.md", use_search=False) + assert exact is not None + assert exact.id == hyphenated.id + + # Tests for strict mode parameter combinations @pytest.mark.asyncio async def test_strict_mode_parameter_combinations(link_resolver, test_entities, project_prefix): @@ -1099,7 +1195,8 @@ async def relative_path_entities(entity_repository, session_maker): ├── testing/ │ ├── link-test.md (source file for testing) │ └── nested/ - │ └── deep-note.md (target for relative path) + │ ├── deep-note.md (target for relative path) + │ └── under_score.md (target for filename alias) ├── nested/ │ └── deep-note.md (different deep-note at root level) └── other/ @@ -1174,6 +1271,56 @@ async def relative_path_entities(entity_repository, session_maker): ) entities.append(e4) + # testing/nested/under_score.md (relative filename-alias target) + e5 = await entity_repository.add( + session, + EntityModel( + title="Relative target with descriptive title", + note_type="note", + content_type="text/markdown", + file_path="testing/nested/under_score.md", + permalink="relative-descriptive-target", + created_at=now, + updated_at=now, + project_id=project_id, + ), + ) + entities.append(e5) + + # elsewhere/permalink-owner.md owns the exact permalink "nested/shadow-target", + # while testing/nested/shadow_target.md is only an alias spelling away from the + # same link text. Exact identities must win over forgiving alias spellings. + e6 = await entity_repository.add( + session, + EntityModel( + title="Exact permalink owner", + note_type="note", + content_type="text/markdown", + file_path="elsewhere/permalink-owner.md", + permalink="nested/shadow-target", + created_at=now, + updated_at=now, + project_id=project_id, + ), + ) + entities.append(e6) + + # testing/nested/shadow_target.md (relative filename-alias decoy for e6) + e7 = await entity_repository.add( + session, + EntityModel( + title="Relative alias decoy", + note_type="note", + content_type="text/markdown", + file_path="testing/nested/shadow_target.md", + permalink="relative-alias-decoy", + created_at=now, + updated_at=now, + project_id=project_id, + ), + ) + entities.append(e7) + return entities @@ -1196,6 +1343,28 @@ async def test_relative_path_resolution_from_subfolder(relative_path_resolver): assert result.file_path == "testing/nested/deep-note.md" +@pytest.mark.asyncio +async def test_relative_path_resolution_uses_unique_filename_alias(relative_path_resolver): + """Relative filename links apply the same underscore/hyphen alias as root links.""" + result = await relative_path_resolver.resolve_link( + "nested/under-score", source_path="testing/link-test.md", use_search=False + ) + + assert result is not None + assert result.file_path == "testing/nested/under_score.md" + + +@pytest.mark.asyncio +async def test_exact_permalink_wins_over_relative_filename_alias(relative_path_resolver): + """A note owning the exact permalink beats a source-relative alias spelling.""" + result = await relative_path_resolver.resolve_link( + "nested/shadow-target", source_path="testing/link-test.md", use_search=False + ) + + assert result is not None + assert result.file_path == "elsewhere/permalink-owner.md" + + @pytest.mark.asyncio async def test_relative_path_falls_back_to_absolute(relative_path_resolver): """Test that if relative path doesn't exist, falls back to absolute resolution.""" diff --git a/tests/services/test_upsert_entity_optimization.py b/tests/services/test_upsert_entity_optimization.py index 5e390850b..c8111cd53 100644 --- a/tests/services/test_upsert_entity_optimization.py +++ b/tests/services/test_upsert_entity_optimization.py @@ -28,24 +28,32 @@ async def test_create_or_update_entity_uses_lightweight_exact_resolution( content="# Create Or Update", ) sentinel_entity = SimpleNamespace(file_path="notes/existing.md") - resolve_calls: list[tuple[str, dict[str, Any]]] = [] + repository_calls: list[tuple[str, str, dict[str, Any]]] = [] - async def fake_resolve_link(link_text: str, **kwargs): - resolve_calls.append((link_text, kwargs)) - if link_text == schema.file_path: - return None + async def fake_get_by_file_path(session, file_path: str, **kwargs): + repository_calls.append(("file_path", file_path, kwargs)) + return None + + async def fake_get_by_permalink(session, permalink: str, **kwargs): + repository_calls.append(("permalink", permalink, kwargs)) return sentinel_entity - monkeypatch.setattr(entity_service.link_resolver, "resolve_link", fake_resolve_link) + monkeypatch.setattr(entity_service.repository, "get_by_file_path", fake_get_by_file_path) + monkeypatch.setattr(entity_service.repository, "get_by_permalink", fake_get_by_permalink) + monkeypatch.setattr( + entity_service.link_resolver, + "resolve_link", + AsyncMock(side_effect=AssertionError("canonical writes must not use link aliases")), + ) monkeypatch.setattr(entity_service, "update_entity", AsyncMock(return_value=sentinel_entity)) entity, is_new = await entity_service.create_or_update_entity(schema) assert entity is sentinel_entity assert is_new is False - assert resolve_calls == [ - (schema.file_path, {"strict": True, "load_relations": False}), - (schema.permalink, {"strict": True, "load_relations": False}), + assert repository_calls == [ + ("file_path", schema.file_path, {"load_relations": False}), + ("permalink", schema.permalink, {"load_relations": False}), ]