fix(core): disambiguate truncated observation permalinks to prevent index collisions

src/basic_memory/models/knowledge.py

@@ -1,5 +1,6 @@
 """Knowledge graph models."""
 
+import hashlib
 import uuid
 from datetime import datetime
 from basic_memory.utils import ensure_timezone_aware
@@ -252,8 +253,18 @@ def permalink(self) -> str:
         Content is truncated to 200 chars to stay under PostgreSQL's
         btree index limit of 2704 bytes.
         """
-        # Truncate content to avoid exceeding PostgreSQL's btree index limit
-        content_for_permalink = self.content[:200] if len(self.content) > 200 else self.content
+        if len(self.content) > 200:
+            # Trigger: content exceeds the 200-char budget imposed by PostgreSQL's
+            # 2704-byte btree index row limit, so the permalink can only carry a prefix.
+            # Why: two distinct observations with the same category and an identical
+            # 200-char prefix would collide on the same synthetic permalink, and the
+            # search index (permalink-keyed upsert) silently drops the second one.
+            # Outcome: a short stable digest of the FULL content disambiguates
+            # truncated permalinks while staying well under the index limit.
+            digest = hashlib.sha256(self.content.encode("utf-8")).hexdigest()[:12]
+            content_for_permalink = f"{self.content[:200]}-{digest}"
+        else:
+            content_for_permalink = self.content
         return generate_permalink(
             f"{self.entity.permalink}/observations/{self.category}/{content_for_permalink}"
         )

tests/repository/test_observation_repository.py

@@ -466,3 +466,64 @@ async def test_observation_permalink_short_content_unchanged(
         # Short content should be fully included (after permalink normalization)
         # The generate_permalink function normalizes the content
         assert "short-observation-content" in permalink.lower()
+
+
+@pytest.mark.asyncio
+async def test_observation_permalink_disambiguates_truncated_content(
+    session_maker: async_sessionmaker, repo, test_project: Project
+):
+    """Regression test for issue #909: shared 200-char prefixes must not collide.
+
+    Truncating content to 200 chars (PostgreSQL btree limit) made two distinct
+    observations with the same category and an identical 200-char prefix produce
+    the same synthetic permalink, silently dropping the second from the search
+    index. A digest of the full content now disambiguates them.
+    """
+    async with db.scoped_session(session_maker) as session:
+        entity = Entity(
+            project_id=test_project.id,
+            title="test_entity",
+            note_type="test",
+            permalink="test/test-entity",
+            file_path="test/test_entity.md",
+            content_type="text/markdown",
+            created_at=datetime.now(timezone.utc),
+            updated_at=datetime.now(timezone.utc),
+        )
+        session.add(entity)
+        await session.flush()
+
+        shared_prefix = "x" * 210  # identical beyond the 200-char truncation point
+        obs_alpha = Observation(
+            project_id=test_project.id,
+            entity_id=entity.id,
+            content=f"{shared_prefix} ALPHA_UNIQUE_MARKER",
+            category="note",
+        )
+        obs_beta = Observation(
+            project_id=test_project.id,
+            entity_id=entity.id,
+            content=f"{shared_prefix} BETA_UNIQUE_MARKER",
+            category="note",
+        )
+        session.add_all([obs_alpha, obs_beta])
+        await session.flush()
+
+        # Distinct content must yield distinct permalinks despite the shared prefix
+        assert obs_alpha.permalink != obs_beta.permalink
+
+        # The digest suffix must keep permalinks under the PostgreSQL btree budget
+        assert len(obs_alpha.permalink) < 300
+        assert len(obs_beta.permalink) < 300
+
+        # Truly identical long content must still produce the same permalink so
+        # exact duplicates continue to dedupe in the search index
+        obs_beta_dup = Observation(
+            project_id=test_project.id,
+            entity_id=entity.id,
+            content=f"{shared_prefix} BETA_UNIQUE_MARKER",
+            category="note",
+        )
+        session.add(obs_beta_dup)
+        await session.flush()
+        assert obs_beta_dup.permalink == obs_beta.permalink

tests/services/test_search_service.py

@@ -1248,6 +1248,78 @@ async def test_index_entity_multiple_categories_same_content(
     assert len(results) >= 2
 
 
+@pytest.mark.asyncio
+async def test_index_entity_long_observations_shared_prefix_both_searchable(
+    search_service, session_maker, test_project
+):
+    """Regression test for issue #909: truncated permalink collisions drop observations.
+
+    Observation permalinks truncate content to 200 chars (PostgreSQL btree limit),
+    so two distinct observations of the same category sharing a 200-char prefix
+    collided on the same synthetic permalink and the second was silently skipped
+    during indexing. Both must be independently searchable.
+    """
+    from basic_memory.repository import EntityRepository, ObservationRepository
+    from datetime import datetime
+
+    entity_repo = EntityRepository(session_maker, project_id=test_project.id)
+    obs_repo = ObservationRepository(session_maker, project_id=test_project.id)
+
+    entity_data = {
+        "title": "Long Observation Collision Entity",
+        "note_type": "note",
+        "entity_metadata": {},
+        "content_type": "text/markdown",
+        "file_path": "test/long-obs-collision.md",
+        "permalink": "test/long-obs-collision",
+        "project_id": test_project.id,
+        "created_at": datetime.now(),
+        "updated_at": datetime.now(),
+    }
+    entity = await entity_repo.create(entity_data)
+
+    # Identical for the first 210 chars (beyond the 200-char truncation point),
+    # differing only in the trailing unique marker
+    shared_prefix = "x" * 210
+    await obs_repo.create(
+        {
+            "entity_id": entity.id,
+            "category": "note",
+            "content": f"{shared_prefix} ALPHA_UNIQUE_MARKER",
+        }
+    )
+    await obs_repo.create(
+        {
+            "entity_id": entity.id,
+            "category": "note",
+            "content": f"{shared_prefix} BETA_UNIQUE_MARKER",
+        }
+    )
+
+    # Reload entity with observations (get_by_permalink eagerly loads observations)
+    entity = await entity_repo.get_by_permalink("test/long-obs-collision")
+    assert entity is not None
+    assert len(entity.observations) == 2
+
+    # Distinct content must produce distinct permalinks despite the shared prefix
+    permalinks = {obs.permalink for obs in entity.observations}
+    assert len(permalinks) == 2
+
+    await search_service.index_entity(entity, content="")
+
+    # The second observation must be findable by its own unique marker
+    results = await search_service.search(
+        SearchQuery(text="BETA_UNIQUE_MARKER", entity_types=[SearchItemType.OBSERVATION])
+    )
+    assert any("BETA_UNIQUE_MARKER" in (r.content_snippet or "") for r in results)
+
+    # The first observation must remain findable as well
+    results = await search_service.search(
+        SearchQuery(text="ALPHA_UNIQUE_MARKER", entity_types=[SearchItemType.OBSERVATION])
+    )
+    assert any("ALPHA_UNIQUE_MARKER" in (r.content_snippet or "") for r in results)
+
+
 # Tests for NUL byte stripping