From 4e63cb996cd087ee572c2b1810def3fb9de41d95 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 00:37:07 -0500 Subject: [PATCH 01/25] feat(core): add temporal qualifiers and time-aware search (SPEC-82) Observations may carry one authored temporal qualifier naming when a claim applies in the world, and search gains explicit valid-time filtering over it. Valid time and recorded time stay separate axes; recorded history waits for SPEC-59 stable identity rather than shipping a half-version that makes false historical claims. Authored forms accept what dateparser reads, canonicalized into one portable range model: @effective[2026-06-10,2026-07-27) range literal, agents and precision @effective:2026-07-27 point, role named @2026-07-27 point, files on valid A naive timestamp is read as UTC, matching the house convention rather than enforcing an offset rule that exists nowhere else in the system. Date-only input never acquires a time of day. Slash-formatted dates resolve under a new date_order config setting. The only diagnostic is an unknown role: everything else either reads as time or stays ordinary content, silently, because direct file editing is a supported path and does not deserve warnings. Storage is a portable projection table with scalar bounds, so SQLite and Postgres share one logical contract and pass the same containment, overlap, inclusivity, unbounded, and empty-range tests. Rows rebuild on every index pass and die with the entity. Undated notes are unchanged when no temporal filter is present, and excluded when one is. Closes the Phase 1 and Phase 2 MVP of SPEC-82. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- ...4t5e6m7p8o9_add_memory_time_index_table.py | 103 +++ .../api/v2/routers/search_router.py | 33 +- src/basic_memory/api/v2/utils.py | 94 ++- src/basic_memory/config_models.py | 9 + src/basic_memory/deps/__init__.py | 4 + src/basic_memory/deps/repositories.py | 16 + .../indexing/accepted_note_write_runner.py | 11 + src/basic_memory/indexing/batch_indexer.py | 8 + src/basic_memory/indexing/models.py | 4 + .../indexing/relation_persistence.py | 133 +++- src/basic_memory/man/man3/search-notes(3).md | 29 +- src/basic_memory/markdown/entity_parser.py | 12 + src/basic_memory/markdown/plugins.py | 11 + src/basic_memory/markdown/schemas.py | 15 +- .../markdown/temporal_qualifier.py | 170 +++++ src/basic_memory/mcp/clients/search.py | 22 + src/basic_memory/mcp/tools/search.py | 142 +++- src/basic_memory/models/__init__.py | 2 + src/basic_memory/models/knowledge.py | 111 +++ src/basic_memory/repository/__init__.py | 6 + .../repository/accepted_note_repositories.py | 4 + .../memory_time_index_repository.py | 147 ++++ .../repository/observation_repository.py | 22 +- .../repository/postgres_search_repository.py | 21 + .../repository/search_repository.py | 3 + .../repository/search_repository_base.py | 17 + .../repository/sqlite_search_repository.py | 21 + .../repository/temporal_filters.py | 138 ++++ src/basic_memory/schemas/search.py | 84 ++- src/basic_memory/services/entity_service.py | 3 + .../services/note_content_writes.py | 4 + src/basic_memory/services/note_preparation.py | 2 + src/basic_memory/services/search_service.py | 59 ++ src/basic_memory/temporal.py | 496 ++++++++++++++ tests/api/v2/test_search_router_telemetry.py | 6 +- tests/api/v2/test_search_router_temporal.py | 291 ++++++++ tests/cloud/test_cloud_services.py | 15 + tests/index/test_local_project_index.py | 7 +- .../test_accepted_note_mutation_runner.py | 24 + .../test_accepted_note_write_runner.py | 37 + tests/indexing/test_relation_persistence.py | 64 +- .../test_relation_persistence_temporal.py | 443 ++++++++++++ tests/markdown/test_entity_parser.py | 32 + tests/markdown/test_temporal_qualifier.py | 490 ++++++++++++++ .../clients/test_search_client_temporal.py | 97 +++ tests/mcp/test_tool_contracts.py | 3 + tests/mcp/test_tool_search_temporal.py | 332 +++++++++ tests/mcp/test_tool_telemetry.py | 1 + ...est_search_notes_multi_project_temporal.py | 162 +++++ tests/repository/test_hybrid_fusion.py | 3 + .../test_memory_time_index_contract.py | 631 ++++++++++++++++++ tests/repository/test_rerank_pipeline.py | 1 + tests/repository/test_search_trace.py | 1 + tests/repository/test_semantic_search_base.py | 2 + tests/repository/test_semantic_vector_sync.py | 2 + tests/repository/test_vector_pagination.py | 3 + .../repository/test_vector_temporal_filter.py | 139 ++++ tests/repository/test_vector_threshold.py | 3 + tests/schemas/test_document_agent_temporal.py | 64 ++ .../services/test_search_service_temporal.py | 230 +++++++ tests/test_config.py | 34 +- tests/test_memory_time_index_migration.py | 283 ++++++++ tests/test_note_section_migration.py | 12 +- tests/test_temporal.py | 514 ++++++++++++++ 64 files changed, 5832 insertions(+), 50 deletions(-) create mode 100644 src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py create mode 100644 src/basic_memory/markdown/temporal_qualifier.py create mode 100644 src/basic_memory/repository/memory_time_index_repository.py create mode 100644 src/basic_memory/repository/temporal_filters.py create mode 100644 src/basic_memory/temporal.py create mode 100644 tests/api/v2/test_search_router_temporal.py create mode 100644 tests/indexing/test_relation_persistence_temporal.py create mode 100644 tests/markdown/test_temporal_qualifier.py create mode 100644 tests/mcp/clients/test_search_client_temporal.py create mode 100644 tests/mcp/test_tool_search_temporal.py create mode 100644 tests/mcp/tools/test_search_notes_multi_project_temporal.py create mode 100644 tests/repository/test_memory_time_index_contract.py create mode 100644 tests/repository/test_vector_temporal_filter.py create mode 100644 tests/schemas/test_document_agent_temporal.py create mode 100644 tests/services/test_search_service_temporal.py create mode 100644 tests/test_memory_time_index_migration.py create mode 100644 tests/test_temporal.py diff --git a/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py b/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py new file mode 100644 index 000000000..90844d3f4 --- /dev/null +++ b/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py @@ -0,0 +1,103 @@ +"""Add memory_time_index table + +Revision ID: u4t5e6m7p8o9 +Revises: t3n4o5t6e7s8 +Create Date: 2026-08-31 12:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = "u4t5e6m7p8o9" +down_revision: Union[str, None] = "t3n4o5t6e7s8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create memory_time_index: the projection of authored valid time (SPEC-82). + + Rows derive from temporal qualifiers written in canonical markdown + (``@effective[2026-06-10,2026-07-27)``). Like observations and sections they are + rebuilt under the note_content generation fence on every (re)index and removed with + the entity, so the table is always reproducible from the notes. + + Every column is a portable scalar, and every type used here renders on both SQLite + and PostgreSQL, so this migration needs no dialect branching. Bounds are canonical + fixed-width text rather than DATE/TIMESTAMP: a date bound must never acquire a time + of day or a timezone (SQLAlchemy's SQLite DateTime silently drops an offset and + stores the wrong instant), and fixed-width canonical text makes byte-lexicographic + order chronological, so one identical predicate serves both backends. Native + PostgreSQL range columns remain a later addition generated from these columns. + + ``source_id`` carries no foreign key by design: it addresses whichever table + ``source_type`` names (``observation`` today). Referential lifecycle rides on + ``entity_id``'s cascade plus the fenced replace instead. + + Only ``ix_memory_time_index_lookup`` indexes the filter columns. The bound values + are deliberately unindexed, and this table is *not* always driven by a full-text + candidate set: a valid-time filter counts as criteria on its own + (``SearchQuery.no_criteria``), so a temporal-only search scans the bound columns + for every row matching project + role + kind. That is an acceptable scan at + expected sizes -- one row per authored qualifier, so thousands, not millions. If + temporal-only queries ever become a hot path, the answer is a native PostgreSQL + range column with a GiST index, not a btree over these text bounds. + """ + op.create_table( + "memory_time_index", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("project_id", sa.Integer(), nullable=False), + sa.Column("entity_id", sa.Integer(), nullable=False), + sa.Column("source_type", sa.String(length=32), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("time_role", sa.String(length=32), nullable=False), + sa.Column("range_kind", sa.String(length=16), nullable=False), + sa.Column("lower_value", sa.String(length=32), nullable=True), + sa.Column("upper_value", sa.String(length=32), nullable=True), + sa.Column("lower_inclusive", sa.Boolean(), nullable=False), + sa.Column("upper_inclusive", sa.Boolean(), nullable=False), + sa.Column("is_empty", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("extractor", sa.String(length=32), nullable=False), + sa.Column("source_text", sa.Text(), nullable=False), + sa.Column("assertion_metadata", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(["project_id"], ["project.id"]), + sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.CheckConstraint( + "range_kind IN ('date', 'instant')", + name="ck_memory_time_index_range_kind", + ), + sa.CheckConstraint( + "NOT is_empty OR (lower_value IS NULL AND upper_value IS NULL)", + name="ck_memory_time_index_empty_has_no_bounds", + ), + sa.CheckConstraint( + "(lower_value IS NOT NULL OR NOT lower_inclusive) " + "AND (upper_value IS NOT NULL OR NOT upper_inclusive)", + name="ck_memory_time_index_unbounded_is_exclusive", + ), + ) + op.create_index( + "ix_memory_time_index_lookup", + "memory_time_index", + ["project_id", "time_role", "range_kind", "source_type", "source_id"], + unique=False, + ) + op.create_index( + "ix_memory_time_index_entity_id", + "memory_time_index", + ["entity_id"], + unique=False, + ) + + +def downgrade() -> None: + """Drop memory_time_index and its supporting indexes.""" + op.drop_index("ix_memory_time_index_entity_id", table_name="memory_time_index") + op.drop_index("ix_memory_time_index_lookup", table_name="memory_time_index") + op.drop_table("memory_time_index") diff --git a/src/basic_memory/api/v2/routers/search_router.py b/src/basic_memory/api/v2/routers/search_router.py index eaf618b89..49a015868 100644 --- a/src/basic_memory/api/v2/routers/search_router.py +++ b/src/basic_memory/api/v2/routers/search_router.py @@ -12,13 +12,16 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Response import logfire -from basic_memory.api.v2.utils import to_search_results +from basic_memory import db +from basic_memory.api.v2.utils import load_temporal_metadata, to_search_results from basic_memory.deps import ( EntityServiceV2ExternalDep, + MemoryTimeIndexRepositoryV2ExternalDep, ProjectExternalIdPathDep, ReadCacheDep, SearchReindexSchedulerDep, SearchServiceV2ExternalDep, + SessionMakerDep, create_model_read_cache, ) from basic_memory.read_cache import ( @@ -83,6 +86,8 @@ async def search( query: SearchQuery, search_service: SearchServiceV2ExternalDep, entity_service: EntityServiceV2ExternalDep, + temporal_repository: MemoryTimeIndexRepositoryV2ExternalDep, + session_maker: SessionMakerDep, read_cache: SearchReadCacheDep, response: Response, project_id: str = Path(..., description="Project external UUID"), @@ -98,6 +103,8 @@ async def search( query: Search query parameters (text, filters, etc.) search_service: Search service scoped to project entity_service: Entity service scoped to project + temporal_repository: Valid-time projection, read to explain temporal matches + session_maker: Session factory for the temporal hydration read page: Page number for pagination page_size: Number of results per page @@ -105,6 +112,10 @@ async def search( SearchResponse with paginated search results """ response.headers["Accept-Query"] = "application/json" + # Read from the request rather than from the parsed filter: this is a plain field + # check that cannot raise, and reaching the hydration step below already proves + # the service accepted and executed the filter. + temporal_requested = query.has_temporal_filter() with logfire.span( "api.request.search", entrypoint="api", @@ -125,7 +136,9 @@ async def search( or query.categories or query.metadata_filters or query.file_path_prefix + or temporal_requested ), + has_temporal_filter=temporal_requested, ): cache_key = ReadCacheKey( project_id=project_id, @@ -202,7 +215,20 @@ async def search( phase="hydrate_results", result_count=len(results), ): - search_results = await to_search_results(entity_service, results) + # Trigger: the caller asked a valid-time question. + # Why: the assertions explain *why* each hit matched, but loading them + # costs a query, and a search with no temporal filter has nothing to + # explain -- so ordinary searches stay exactly as expensive as before. + # Outcome: temporal metadata rides along only on temporal searches. + temporal_by_source = {} + if temporal_requested: + async with db.scoped_session(session_maker) as session: + temporal_by_source = await load_temporal_metadata( + temporal_repository, session, results + ) + search_results = await to_search_results( + entity_service, results, temporal_by_source=temporal_by_source + ) with logfire.span( "api.search.search.build_response", domain="search", @@ -217,6 +243,9 @@ async def search( total=total, total_is_exact=exact_count_available, has_more=has_more, + # None, not False, when nothing was asked: an ordinary search + # payload stays exactly what it was before valid time existed. + temporal_applied=True if temporal_requested else None, ) cached.value = result return result diff --git a/src/basic_memory/api/v2/utils.py b/src/basic_memory/api/v2/utils.py index 853f2c76f..a7c93b8b6 100644 --- a/src/basic_memory/api/v2/utils.py +++ b/src/basic_memory/api/v2/utils.py @@ -1,7 +1,10 @@ +from collections import defaultdict +from collections.abc import Mapping from typing import Any, Protocol, Optional, List, Sequence import logfire from sqlalchemy.ext.asyncio import AsyncSession +from basic_memory.models import MemoryTimeIndex from basic_memory.repository.search_repository import SearchIndexRow from basic_memory.schemas.memory import ( EntitySummary, @@ -11,11 +14,17 @@ GraphContext, ContextResult, ) -from basic_memory.schemas.search import SearchItemType, SearchResult +from basic_memory.schemas.search import ( + SearchItemType, + SearchResult, + TemporalRangeValue, + TemporalResultMetadata, +) from basic_memory.services.context_service import ( ContextResultRow, ContextResult as ServiceContextResult, ) +from basic_memory.temporal import TemporalRange, TemporalRangeKind class EntityBatchLookup(Protocol): @@ -32,6 +41,19 @@ class EntityServiceBatchLookup(Protocol): async def get_entities_by_id(self, ids: List[int]) -> Sequence[Any]: ... +class TemporalAssertionLookup(Protocol): + async def find_for_sources( + self, + session: AsyncSession, + sources: Sequence[tuple[str, int]], + ) -> Sequence[MemoryTimeIndex]: ... + + +# One page of hits, keyed by the (search row type, search row id) pair the projection +# addresses. Empty means "no valid-time metadata was loaded", never "none exists". +type TemporalMetadataBySource = Mapping[tuple[str, int], list[TemporalResultMetadata]] + + async def get_entities_by_id_lookup( entity_service: EntityServiceBatchLookup, entity_ids: Sequence[int], @@ -217,8 +239,71 @@ def to_summary( ) +def _temporal_result_metadata(row: MemoryTimeIndex) -> TemporalResultMetadata: + """Shape one projected assertion into the value a caller sees. + + Rebuilding the domain range from the stored scalars re-runs its invariants, so a + row that somehow violated them surfaces here instead of being rendered as a + plausible-looking interval. + """ + valid_during = TemporalRange( + kind=TemporalRangeKind(row.range_kind), + lower=row.lower_value, + upper=row.upper_value, + lower_inclusive=row.lower_inclusive, + upper_inclusive=row.upper_inclusive, + is_empty=row.is_empty, + ) + return TemporalResultMetadata( + role=row.time_role, + valid_during=TemporalRangeValue( + kind=valid_during.kind.value, + literal=str(valid_during), + lower=valid_during.lower, + upper=valid_during.upper, + lower_inclusive=valid_during.lower_inclusive, + upper_inclusive=valid_during.upper_inclusive, + is_empty=valid_during.is_empty, + ), + source_text=row.source_text, + ) + + +async def load_temporal_metadata( + temporal_repository: TemporalAssertionLookup, + session: AsyncSession, + results: Sequence[SearchIndexRow], +) -> dict[tuple[str, int], list[TemporalResultMetadata]]: + """Load the authored valid-time assertions behind one page of search hits. + + Keyed on the search row's own ``(type, id)`` pair, which is exactly the address + the projection stores -- so an observation hit resolves to the assertions written + on that observation, not to its note's other assertions. + """ + sources = [(result.type, result.id) for result in results] + if not sources: + return {} + + with logfire.span( + "search.hydrate_results.fetch_temporal", + domain="search", + action="search", + phase="fetch_temporal", + result_count=len(sources), + ): + rows = await temporal_repository.find_for_sources(session, sources) + + by_source: defaultdict[tuple[str, int], list[TemporalResultMetadata]] = defaultdict(list) + for row in rows: + by_source[(row.source_type, row.source_id)].append(_temporal_result_metadata(row)) + return dict(by_source) + + async def to_search_results( - entity_service: EntityServiceBatchLookup, results: List[SearchIndexRow] + entity_service: EntityServiceBatchLookup, + results: List[SearchIndexRow], + *, + temporal_by_source: TemporalMetadataBySource | None = None, ) -> list[SearchResult]: with logfire.span( "search.hydrate_results", @@ -299,6 +384,11 @@ async def to_search_results( from_entity=from_entity.permalink if from_entity else None, to_entity=to_entity.permalink if to_entity else None, relation_type=result.relation_type, + temporal=( + temporal_by_source.get((result.type, result.id)) + if temporal_by_source + else None + ), ) ) return search_results diff --git a/src/basic_memory/config_models.py b/src/basic_memory/config_models.py index bcb9227ab..fb8b4991b 100644 --- a/src/basic_memory/config_models.py +++ b/src/basic_memory/config_models.py @@ -586,6 +586,15 @@ def __init__(self, **data: Any) -> None: ... gt=0, ) + # Spelled as a bare Literal rather than the `DateOrder` alias so `bm config set` + # keeps discovering it: CONFIGURABLE_FIELDS reads model_fields annotations, and a + # PEP 695 alias arrives there unresolved. `temporal.DateOrder` is the same union, + # and a test pins the two together. + date_order: Literal["YMD", "DMY", "MDY"] = Field( + default="YMD", + description="Component order used to read an ambiguous slash-formatted date in an authored temporal qualifier (e.g. '@10/07/2026'). YMD and DMY read that as 10 July 2026, MDY as 7 October 2026. ISO dates like '2026-07-10' are never re-guessed.", + ) + kebab_filenames: bool = Field( default=False, description="Format for generated filenames. False preserves spaces and special chars, True converts them to hyphens for consistency with permalinks", diff --git a/src/basic_memory/deps/__init__.py b/src/basic_memory/deps/__init__.py index d45c6e18f..f274c88b8 100644 --- a/src/basic_memory/deps/__init__.py +++ b/src/basic_memory/deps/__init__.py @@ -51,6 +51,8 @@ ObservationRepositoryV2ExternalDep, get_relation_repository_v2_external, RelationRepositoryV2ExternalDep, + get_memory_time_index_repository_v2_external, + MemoryTimeIndexRepositoryV2ExternalDep, get_search_repository_v2_external, SearchRepositoryV2ExternalDep, ) @@ -140,6 +142,8 @@ "ObservationRepositoryV2ExternalDep", "get_relation_repository_v2_external", "RelationRepositoryV2ExternalDep", + "get_memory_time_index_repository_v2_external", + "MemoryTimeIndexRepositoryV2ExternalDep", "get_search_repository_v2_external", "SearchRepositoryV2ExternalDep", # Services diff --git a/src/basic_memory/deps/repositories.py b/src/basic_memory/deps/repositories.py index ccc325880..7dd583318 100644 --- a/src/basic_memory/deps/repositories.py +++ b/src/basic_memory/deps/repositories.py @@ -18,6 +18,7 @@ from basic_memory.deps.db import SessionMakerDep from basic_memory.deps.projects import ProjectExternalIdPathDep from basic_memory.repository.entity_repository import EntityRepository +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository from basic_memory.repository.observation_repository import ObservationRepository from basic_memory.repository.relation_repository import RelationRepository from basic_memory.repository.search_repository import SearchRepository, create_search_repository @@ -68,6 +69,21 @@ async def get_relation_repository_v2_external( ] +# --- Temporal Projection Repository --- + + +async def get_memory_time_index_repository_v2_external( + project_id: ProjectExternalIdPathDep, +) -> MemoryTimeIndexRepository: + """Create a MemoryTimeIndexRepository instance for v2 API (uses external_id).""" + return MemoryTimeIndexRepository(project_id=project_id) + + +MemoryTimeIndexRepositoryV2ExternalDep = Annotated[ + MemoryTimeIndexRepository, Depends(get_memory_time_index_repository_v2_external) +] + + # --- Search Repository --- diff --git a/src/basic_memory/indexing/accepted_note_write_runner.py b/src/basic_memory/indexing/accepted_note_write_runner.py index 3eb6fbc79..b98dae8d8 100644 --- a/src/basic_memory/indexing/accepted_note_write_runner.py +++ b/src/basic_memory/indexing/accepted_note_write_runner.py @@ -21,6 +21,7 @@ RelationGenerationPublication, RelationGenerationStore, SectionGenerationStore, + TemporalGenerationStore, ) from basic_memory.models import Entity, NoteContent from basic_memory.repository import ( @@ -189,6 +190,10 @@ class AcceptedNoteSectionRepository(SectionGenerationStore, Protocol): """Generation-fenced section persistence for accepted note writes.""" +class AcceptedNoteTemporalRepository(TemporalGenerationStore, Protocol): + """Generation-fenced valid-time persistence for accepted note writes.""" + + class AcceptedNoteRelationRepository(RelationGenerationStore, Protocol): """Generation-fenced relation persistence for accepted note writes.""" @@ -221,6 +226,11 @@ def section_repository( project_id: ProjectId, ) -> AcceptedNoteSectionRepository: ... + def temporal_repository( + self, + project_id: ProjectId, + ) -> AcceptedNoteTemporalRepository: ... + def relation_repository( self, project_id: ProjectId, @@ -612,6 +622,7 @@ async def accepted_relation_generation_publication( category=observation.category, context=observation.context, tags=observation.tags, + temporal=observation.temporal, ) for observation in observations ) diff --git a/src/basic_memory/indexing/batch_indexer.py b/src/basic_memory/indexing/batch_indexer.py index b8dd1446e..2cff00399 100644 --- a/src/basic_memory/indexing/batch_indexer.py +++ b/src/basic_memory/indexing/batch_indexer.py @@ -44,6 +44,7 @@ from basic_memory.models import Entity, NoteContent, Relation, RelationSearchRefresh from basic_memory.repository import EntityRepository, ObservationRepository, RelationRepository from basic_memory.repository.note_content_repository import NoteContentRepository +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository from basic_memory.repository.note_section_repository import NoteSectionRepository from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.relation_repository import lock_note_content_before_entity_mutation @@ -163,6 +164,7 @@ def __init__( self.relation_repository = relation_repository self.note_content_repository = NoteContentRepository(project_id=project_id) self.section_repository = NoteSectionRepository(project_id=project_id) + self.temporal_repository = MemoryTimeIndexRepository(project_id=project_id) self.search_service = search_service self.file_writer = file_writer self.session_maker = session_maker @@ -170,6 +172,7 @@ def __init__( relation_repository=relation_repository, observation_repository=observation_repository, section_repository=self.section_repository, + temporal_repository=self.temporal_repository, session_maker=session_maker, ) self.relation_resolution = RepositoryRelationResolutionRuntime( @@ -773,6 +776,10 @@ async def _clear_note_only_state(self, session: AsyncSession, entity: Entity) -> relation.to_entity = None await self.observation_repository.delete_by_fields(session, entity_id=entity.id) + # Valid time is asserted by observations, so it retires with them: a resource + # that is no longer Markdown asserts nothing, and no later pass would repair + # rows left addressing observations that have just been deleted. + await self.temporal_repository.delete_by_fields(session, entity_id=entity.id) await self.section_repository.delete_by_fields(session, entity_id=entity.id) await self.relation_repository.delete_by_fields(session, from_id=entity.id) await self.note_content_repository.delete_by_entity_id(session, entity.id) @@ -1092,6 +1099,7 @@ async def _build_prepared_entity( category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in prepared.markdown.observations ) diff --git a/src/basic_memory/indexing/models.py b/src/basic_memory/indexing/models.py index a733fcea0..477cdb5d6 100644 --- a/src/basic_memory/indexing/models.py +++ b/src/basic_memory/indexing/models.py @@ -44,6 +44,7 @@ StorageEtag, normalize_storage_etag, ) +from basic_memory.temporal import TemporalAssertion if TYPE_CHECKING: # pragma: no cover from basic_memory.models import Entity @@ -123,6 +124,9 @@ class IndexedObservation: category: str | None context: str | None tags: list[str] | None + # Authored valid time (SPEC-82). Published as its own projection keyed on the + # observation row this write creates, never as observation columns. + temporal: tuple[TemporalAssertion, ...] = () @dataclass(frozen=True, slots=True) diff --git a/src/basic_memory/indexing/relation_persistence.py b/src/basic_memory/indexing/relation_persistence.py index eb260b5fe..0579f8b17 100644 --- a/src/basic_memory/indexing/relation_persistence.py +++ b/src/basic_memory/indexing/relation_persistence.py @@ -1,7 +1,7 @@ """Publish a note's derived graph under one accepted content generation. -The publication carries the complete observation, section, and relation projections -through one note_content fence lifecycle and one durable retry marker. +The publication carries the complete observation, temporal, section, and relation +projections through one note_content fence lifecycle and one durable retry marker. The graph tables are eventually consistent projections, not part of the accepted write's transaction. Publication runs after the content commit; a stale fence makes every statement @@ -23,6 +23,10 @@ from basic_memory import db from basic_memory.indexing.models import IndexedObservation, IndexedRelation, IndexedSection +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import ( AcceptedSectionWrite, SectionGenerationWriteResult, @@ -37,6 +41,7 @@ RelationGenerationWriteResult, ) from basic_memory.runtime.storage import ProjectId, RuntimeEntityId, RuntimeNoteContentVersion +from basic_memory.schemas.search import SearchItemType class RelationGenerationStore(Protocol): @@ -94,6 +99,19 @@ async def replace_sections_for_generation( ) -> SectionGenerationWriteResult: ... +class TemporalGenerationStore(Protocol): + """Repository operation needed to replace one temporal generation.""" + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: ... + + @dataclass(frozen=True, slots=True) class RelationGenerationPublication: """Derived graph intent authorized by one accepted note-content generation.""" @@ -108,11 +126,12 @@ class RelationGenerationPublication: @dataclass(frozen=True, slots=True) class RelationGenerationPublisher: - """Commit observations, sections, relation chunks, and cleanup under source fences.""" + """Commit observations, valid time, sections, relations, and cleanup under fences.""" relation_repository: RelationGenerationStore observation_repository: ObservationGenerationStore section_repository: SectionGenerationStore + temporal_repository: TemporalGenerationStore session_maker: async_sessionmaker[AsyncSession] async def publish( @@ -163,27 +182,11 @@ async def publish( if not publication.generation_is_current: return False - # Observations fit one statement batch, so their fenced replacement owns one short - # transaction. An empty desired set must still execute to wipe the prior projection. - accepted_observations = tuple( - AcceptedObservationWrite( - content=observation.content, - category=observation.category, - context=observation.context, - tags=observation.tags, - ) - for observation in observations - ) - async with db.scoped_session(self.session_maker) as session: - observation_result = ( - await self.observation_repository.replace_observations_for_generation( - session, - entity_id=entity_id, - generation=generation, - observations=accepted_observations, - ) - ) - if not observation_result.generation_is_current: + if not await self._publish_observations( + entity_id=entity_id, + generation=generation, + observations=observations, + ): return False # Sections mirror the observation projection: one fenced wipe-and-recreate in @@ -234,3 +237,85 @@ async def publish( generation=generation, ) return cleanup.generation_is_current + + async def _publish_observations( + self, + *, + entity_id: int, + generation: int, + observations: Sequence[IndexedObservation], + ) -> bool: + """Replace observations and their authored valid time under one fence. + + Observations fit one statement batch, so their fenced replacement owns one + short transaction. An empty desired set must still execute, for both writes, + to wipe the prior projections. + + Constraint: the temporal projection addresses observations by row id, and + those ids are minted by the insert here. Unlike sections and observations, + which key on the stable entity_id, this write cannot be deferred to a + transaction of its own: a same-generation republish landing between two + commits would wipe and re-mint the observation rows, leaving temporal rows + addressing ids that no longer exist and that no later pass repairs. One + transaction under one held fence keeps a row and its valid time atomic. That + is a narrow exception earned by the id dependency, not a licence to widen the + other statements. + """ + accepted_observations = tuple( + AcceptedObservationWrite( + content=observation.content, + category=observation.category, + context=observation.context, + tags=observation.tags, + temporal=observation.temporal, + ) + for observation in observations + ) + async with db.scoped_session(self.session_maker) as session: + observation_result = ( + await self.observation_repository.replace_observations_for_generation( + session, + entity_id=entity_id, + generation=generation, + observations=accepted_observations, + ) + ) + if not observation_result.generation_is_current: + return False + + temporal_result = await self.temporal_repository.replace_assertions_for_generation( + session, + entity_id=entity_id, + generation=generation, + assertions=_accepted_temporal_assertions( + observations, + observation_result.observation_ids, + ), + ) + return temporal_result.generation_is_current + + +def _accepted_temporal_assertions( + observations: Sequence[IndexedObservation], + observation_ids: Sequence[int], +) -> tuple[AcceptedTemporalAssertion, ...]: + """Pair each authored assertion with the observation row that now carries it. + + Both sequences are in document order, so position is the pairing. A length + mismatch means the observation write returned ids for a different set of rows + than it was given, which would silently attach valid time to the wrong statement. + """ + if len(observation_ids) != len(observations): + raise ValueError( + f"Observation publication returned {len(observation_ids)} row ids " + f"for {len(observations)} observations" + ) + return tuple( + AcceptedTemporalAssertion( + source_type=SearchItemType.OBSERVATION.value, + source_id=observation_id, + assertion=assertion, + ) + for observation, observation_id in zip(observations, observation_ids) + for assertion in observation.temporal + ) diff --git a/src/basic_memory/man/man3/search-notes(3).md b/src/basic_memory/man/man3/search-notes(3).md index d982154ee..fe75e1573 100644 --- a/src/basic_memory/man/man3/search-notes(3).md +++ b/src/basic_memory/man/man3/search-notes(3).md @@ -25,7 +25,8 @@ search_notes(query=None, project=None, project_id=None, search_type=None, output_format="text", note_types=None, entity_types=None, categories=None, after_date=None, metadata_filters=None, tags=None, status=None, - min_similarity=None) + min_similarity=None, valid_at=None, valid_overlaps=None, + time_role=None) ``` CLI: @@ -52,6 +53,20 @@ rows), `categories` (observation categories, paired with `metadata_filters` — equality matches against arbitrary frontmatter fields, which is how the manual implements apropos (see [[Manpage]]). +**Valid time** (`valid_at`, `valid_overlaps`, `time_role`) queries what a note +*says was true*, not when it was last edited. Observations can carry a +qualifier — a range, `- [decision] @effective[2026-06-10,2026-07-27) ...`, or +a point, `- [decision] @effective:2026-07-27 ...` / `- [decision] @2026-07-27 +...` — and these filters match against that authored interval. A point means +the span its precision covers: `@2026` is that year, `@2026-06` that month, +and `@2026-06-10` from that date onward. It is a separate axis from +`after_date`, which keeps filtering last-indexed time. Bounds follow +PostgreSQL range conventions, calendar dates and instants never convert into +one another, and a source with no qualifier is excluded from any valid-time +query. Because one note can carry several assertions that disagree, these +queries return observation-level results, each carrying the assertion that +matched. + ## PARAMETERS - **query** — search string; optional. Omit it for filter-only searches @@ -65,6 +80,14 @@ which is how the manual implements apropos (see [[Manpage]]). - **tags** — list or comma string, same convention as [[write-note(3)]] - **min_similarity** — float override for vector/hybrid threshold; `0.0` shows everything, `0.8` is high precision +- **valid_at** — date (`2026-07-28`) or RFC 3339 instant + (`2026-07-28T09:00:00Z`) that the authored range must contain; a timestamp + written without an offset is read as UTC (aliases: `as_of`, `valid_on`) +- **valid_overlaps** — range literal the authored range must overlap: + `[2026-06-10,2026-07-27)`, `(,2026-07-27]`, `[2026-06-10,)`. Mutually + exclusive with `valid_at` (aliases: `overlaps`, `valid_during`) +- **time_role** — valid-time axis: `effective`, `valid`, `occurred`, `due`, + or `mentioned`; usable on its own (aliases: `role`, `time_axis`) - **search_all_projects** — opt-in cross-project search; ignored when `project`/`project_id` is given - **page**, **page_size** — pagination (aliases: `page_number`, `limit`, @@ -103,6 +126,10 @@ bm tool search-notes "conflict error" --project manual --page-size 2 - [gotcha] Score semantics differ by mode: FTS rank scores in text mode, similarity scores in hybrid/vector — don't compare across modes #scoring - [gotcha] The CLI takes QUERY positionally; there is no --query flag #cli-parity - [gotcha] search_all_projects is silently ignored when a project is specified #routing +- [gotcha] A valid-time filter excludes every source without a temporal qualifier — an undated note makes no claim about when it holds, so drop the filter to search dated and undated content together #valid-time +- [gotcha] valid_at and valid_overlaps never mix calendar dates with instants: a date query matches only date ranges and an instant query only instant ranges, so `2026-07-27` and `2026-07-27T00:00:00Z` are different questions #valid-time +- [gotcha] A timestamp written without an offset is read as UTC, in an authored qualifier and in a filter alike — same convention as every other naive datetime in Basic Memory #valid-time +- [gotcha] An authored token that does not read as a date is left as ordinary observation content with no warning; only an unknown role (`@asserted:2026-06-10`) is reported #valid-time ## SEE ALSO diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 495b9702b..47474566f 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -335,6 +335,18 @@ async def parse_markdown_content( entity_content = ( parse(post.content) if parse_semantics else EntityContent(content=post.content) ) + + # The parser reports exactly one thing: a qualifier that reads as time but names + # an unknown role. That never reaches the index, so this warning is how an author + # learns the line needs fixing. Text that simply is not a date is ordinary content + # and says nothing here. The typed `temporal_error` field carries the same message + # to programmatic callers; this layer adds the path. + for observation in entity_content.observations: + if observation.temporal_error: + logger.warning( + f"Temporal qualifier ignored in {file_path}: {observation.temporal_error}" + ) + # Sections are structural, not semantic: they index the body for range reads, # so the bm_parse_semantics opt-out above must not suppress them. sections = scan_sections(post.content) diff --git a/src/basic_memory/markdown/plugins.py b/src/basic_memory/markdown/plugins.py index af00c0b0d..d88a4a7e2 100644 --- a/src/basic_memory/markdown/plugins.py +++ b/src/basic_memory/markdown/plugins.py @@ -3,6 +3,7 @@ import re from typing import List, Any, Dict +from basic_memory.markdown.temporal_qualifier import parse_temporal_qualifier from basic_memory.utils import normalize_project_reference from markdown_it import MarkdownIt from markdown_it.rules_inline.backticks import backtick @@ -100,6 +101,14 @@ def parse_observation(token: Token) -> Dict[str, Any]: if empty_match: content = empty_match.group(1).strip() + # Parse the temporal qualifier before the (context) rule below. An authored + # `@effective(2026-06-10,2026-07-27)` at end of line ends in ")", so the context + # rule would otherwise claim it and leave `@effective` as the whole observation. + # A qualifier that was not accepted is never peeled, so that line keeps its exact + # text (SPEC-82). + temporal = parse_temporal_qualifier(content) + content = temporal.content + # Parse (context) context = None if content.endswith(")"): @@ -124,6 +133,8 @@ def parse_observation(token: Token) -> Dict[str, Any]: "content": content, "tags": tags if tags else None, "context": context, + "temporal": list(temporal.assertions), + "temporal_error": temporal.error, } diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index 8788efd24..ab8c38a64 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, model_validator from basic_memory.markdown.sections import MarkdownSection +from basic_memory.temporal import TemporalAssertion class Observation(BaseModel): @@ -15,10 +16,22 @@ class Observation(BaseModel): content: str tags: Optional[List[str]] = None context: Optional[str] = None + # Collection-shaped from day one: the MVP parses at most one qualifier per + # observation, but carrying several later must not be a schema break (SPEC-82). + temporal: List[TemporalAssertion] = [] + # Set for the one reported case: a qualifier that reads as time but names an + # unknown role. Its text stays in `content`, so nothing is dropped -- only the + # derived temporal projection is withheld until the author fixes the line. Text + # that simply is not a date sets nothing here; it is ordinary content. + temporal_error: Optional[str] = None @override def __str__(self) -> str: - obs_string = f"- [{self.category}] {self.content}" + # Replaying `source_text` verbatim is what makes parse/serialize a byte-exact + # round trip: `valid_during` holds normalized bounds, the author's text does not. + qualifiers = " ".join(assertion.source_text for assertion in self.temporal) + prefix = f"{qualifiers} " if qualifiers else "" + obs_string = f"- [{self.category}] {prefix}{self.content}" if self.context: obs_string += f" ({self.context})" return obs_string diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py new file mode 100644 index 000000000..f357f8c21 --- /dev/null +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -0,0 +1,170 @@ +"""Peel SPEC-82 temporal qualifiers off observation content. + +An observation may carry one qualifier immediately after its category and before its +content. Two authored forms exist, and the role is optional in both: + + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective:2026-07-27 The cache layer will use Memcached. + - [decision] @2026-07-27 The cache layer will use Memcached. + +The bracket form carries a range literal and needs no separator, because no role name +can begin with `[` or `(`. The point form needs the `:` because a date can begin with a +letter (`yesterday`), so nothing else would tell `@occurred:yesterday` from a handle. +A role-less point must begin with a digit and be at least as wide as a year: a bare +`@word` is overwhelmingly a mention, a version, or a handle, and dateparser reads many +short tokens as dates (`@may` as May, `@v2` as February, `@1` as January). With a role +the author has said what they mean, so any text dateparser can read is accepted there, +relative dates included. + +One rule decides everything else: **if the payload reads as time, the token becomes a +qualifier; if it does not, the token stays ordinary observation content, silently.** +Prose is full of `@` -- email addresses, handles, `@todo:` markers -- and warning about +each one that is not a date would be noise, not help. + +The single exception is an **unknown role**. `@asserted:2026-06-10` parses as time and +names an axis, so the author is plainly reaching for this feature and a short list of +valid roles makes the diagnostic actionable. + +A refused or unread qualifier is never peeled. Its text stays in the observation +content, so the line indexes exactly as it does today and remains full-text searchable; +only the derived temporal projection is withheld. +""" + +import re +from dataclasses import dataclass + +from basic_memory.temporal import ( + DateOrder, + TemporalAssertion, + TemporalQualifierError, + TemporalRange, + TimeRole, + parse_authored_point, + parse_range_literal, +) + +_ROLE_NAMES = frozenset(role.value for role in TimeRole) + +# A point with no role is filed on the axis the feature is named for: the author said +# when the statement holds without narrowing *how* it holds. +DEFAULT_TIME_ROLE = TimeRole.VALID + +_ROLE_PATTERN = r"[A-Za-z][A-Za-z0-9_]*" + +# `@[role]` glued to one balanced bracket group carrying a range literal's comma. The +# lookahead stops `@effective[a,b)x` from half-matching, and the `^` anchor keeps +# `paul@basicmemory.com` and mid-sentence `@handles` out entirely. +_RANGE_QUALIFIER = re.compile(rf"^@({_ROLE_PATTERN})?([\[(][^\[\]()]*,[^\[\]()]*[\])])(?=\s|$)") + +# `@role:`. +_ROLE_POINT_QUALIFIER = re.compile(rf"^@({_ROLE_PATTERN}):(\S+)") + +# `@` -- the role-less point. At least four characters +# wide, the width of a year: dateparser reads `1` as January and `3.5` as March 5, and +# a token that short at the head of a line is a list marker or a version, not a date. +_BARE_POINT_QUALIFIER = re.compile(r"^@(\d\S{3,})") + + +@dataclass(frozen=True, slots=True) +class ObservationTemporalParse: + """What a qualifier scan found at the head of one observation's content. + + Exactly three shapes exist: a peel (content shortened, one assertion, no error), an + unknown-role refusal (content untouched, no assertions, an error message), and no + qualifier at all (content untouched, nothing found). + """ + + content: str + assertions: tuple[TemporalAssertion, ...] + error: str | None + + +def _no_qualifier(content: str) -> ObservationTemporalParse: + """Leave the line exactly as authored, with nothing to report.""" + return ObservationTemporalParse(content=content, assertions=(), error=None) + + +def _refuse(content: str, reason: str) -> ObservationTemporalParse: + """Keep the line exactly as authored and report why no assertion was derived.""" + return ObservationTemporalParse(content=content, assertions=(), error=reason) + + +@dataclass(frozen=True, slots=True) +class _ReadQualifier: + """One token that read as time: how much of the line it spans, and what it says.""" + + token: str + end: int + role_name: str | None + valid_during: TemporalRange + + +def _read_range_qualifier(content: str) -> _ReadQualifier | None: + """Match the bracket form and parse its literal, or report no usable qualifier.""" + match = _RANGE_QUALIFIER.match(content) + if match is None: + return None + try: + valid_during = parse_range_literal(match.group(2)) + except TemporalQualifierError: + # A literal we cannot read is not a qualifier. Saying *how* it is malformed + # would be a diagnostic about how someone wrote a date, which this feature + # deliberately does not issue. + return None + return _ReadQualifier(match.group(0), match.end(), match.group(1), valid_during) + + +def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _ReadQualifier | None: + """Match either point form and read its date, or report no usable qualifier.""" + roled = _ROLE_POINT_QUALIFIER.match(content) + bare = None if roled is not None else _BARE_POINT_QUALIFIER.match(content) + match = roled or bare + if match is None: + return None + + # Deferred, following utils.ensure_timezone_aware: the markdown parser is a + # low-level module that many entrypoints import, and pulling the config stack in at + # import time couples parsing to configuration load order for no benefit. Resolved + # here rather than at the top of the scan so only a token that already looks like a + # qualifier pays for reading the config -- or for loading dateparser. + from basic_memory.config import ConfigManager + + order = date_order if date_order is not None else ConfigManager().config.date_order + point = match.group(2) if roled is not None else match.group(1) + valid_during = parse_authored_point(point, date_order=order) + if valid_during is None: + return None + role_name = match.group(1) if roled is not None else None + return _ReadQualifier(match.group(0), match.end(), role_name, valid_during) + + +def parse_temporal_qualifier( + content: str, *, date_order: DateOrder | None = None +) -> ObservationTemporalParse: + """Split a leading temporal qualifier off observation content. + + The MVP reads at most one qualifier per observation, but the result is a collection + so supporting several later is not a schema break. `date_order` defaults to the + configured `date_order`; tests and callers that already hold the config pass it. + """ + read = _read_range_qualifier(content) or _read_point_qualifier(content, date_order) + if read is None: + return _no_qualifier(content) + + role_name = read.role_name + if role_name is not None and role_name not in _ROLE_NAMES: + known = ", ".join(sorted(_ROLE_NAMES)) + return _refuse(content, f"unknown temporal role {role_name!r} in {read.token!r} ({known})") + + remainder = content[read.end :].strip() + if not remainder: + # A qualifier with nothing to qualify would leave an empty observation, which + # the plugin drops outright. Keep the line whole instead. + return _no_qualifier(content) + + assertion = TemporalAssertion( + time_role=TimeRole(role_name) if role_name is not None else DEFAULT_TIME_ROLE, + valid_during=read.valid_during, + source_text=read.token, + ) + return ObservationTemporalParse(content=remainder, assertions=(assertion,), error=None) diff --git a/src/basic_memory/mcp/clients/search.py b/src/basic_memory/mcp/clients/search.py index 0d04fda2c..e3bdca068 100644 --- a/src/basic_memory/mcp/clients/search.py +++ b/src/basic_memory/mcp/clients/search.py @@ -14,6 +14,10 @@ # so each method defers the import to call time instead (#886). from basic_memory.schemas.search import SearchResponse, SearchRetrievalMode +# The valid-time fields SearchQuery carries. Named here so the skew check below stays +# in step with the schema without importing the model's internals. +_TEMPORAL_QUERY_FIELDS = ("valid_at", "valid_overlaps", "time_role") + class SearchClient: """Typed client for search operations. @@ -59,6 +63,7 @@ async def search( Raises: ToolError: If the request fails + ValueError: If a requested valid-time filter was not applied by the server """ from basic_memory.mcp.tools.utils import call_query @@ -87,4 +92,21 @@ async def search( retrieval_mode = query.get("retrieval_mode", SearchRetrievalMode.FTS) payload["total_is_exact"] = retrieval_mode == SearchRetrievalMode.FTS + # Trigger: this request carried a valid-time filter but the response does not + # confirm the server ran it. + # Why: SearchQuery ignores unknown fields, so a server predating SPEC-82 accepts + # the request and returns results that look filtered. A valid-time query + # excludes undated sources; unfiltered results include them, and the caller + # would have no way to tell. + # Outcome: fail loudly instead of returning a wrong answer that reads as right. + if any(query.get(field) for field in _TEMPORAL_QUERY_FIELDS) and ( + payload.get("temporal_applied") is not True + ): + raise ValueError( + "The search API did not apply the requested valid-time filter " + "(no temporal_applied confirmation in the response). The server is " + "likely older than this client; upgrade it or drop valid_at / " + "valid_overlaps / time_role from the query." + ) + return SearchResponse.model_validate(payload) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index a3db00834..208e4d6f8 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -395,6 +395,14 @@ def _format_search_markdown( parts.append(f"- score: {r.score:.4f}") if r.matched_chunk: parts.append(f"- match: {r.matched_chunk[:200]}") + # Name the axis and the units. A bare "2026-06-10" here would read as an edit + # date; "effective valid time ... (date)" says which time this is and that it + # is a calendar date carrying no timezone. + for assertion in r.temporal or []: + parts.append( + f"- {assertion.role} valid time: {assertion.valid_during.literal} " + f"({assertion.valid_during.kind})" + ) parts.append("") # --- Footer with pagination --- @@ -575,11 +583,19 @@ async def _search_all_projects( tags: list[str] | None, status: str | None, min_similarity: float | None, + valid_at: str | None, + valid_overlaps: str | None, + time_role: str | None, context: Context | None, ) -> dict[str, Any] | str: """Search every accessible project when the caller explicitly opts in.""" requested_page = max(page, 1) requested_page_size = max(page_size, 1) + # Each per-project call runs through search_notes -> SearchClient, which refuses a + # response that does not confirm the filter ran. So a project either honored the + # valid-time filter or was dropped with a warning below; the merged answer never + # silently mixes filtered and unfiltered rows. + temporal_requested = bool(valid_at or valid_overlaps or time_role) project_refs = await _load_search_project_refs(context=context) if not project_refs: response = SearchResponse( @@ -589,6 +605,7 @@ async def _search_all_projects( total=0, total_is_exact=True, has_more=False, + temporal_applied=True if temporal_requested else None, ) if output_format == "json": return response.model_dump(mode="json", exclude_none=True) @@ -636,6 +653,9 @@ async def _search_all_projects( tags=tags, status=status, min_similarity=min_similarity, + valid_at=valid_at, + valid_overlaps=valid_overlaps, + time_role=time_role, search_all_projects=False, context=context, ) @@ -679,6 +699,7 @@ async def _search_all_projects( "total": total, "total_is_exact": total_is_exact, "has_more": any_project_has_more or total > end or len(sorted_results) > end, + "temporal_applied": True if temporal_requested else None, } ) @@ -789,6 +810,41 @@ async def search_notes( validation_alias=AliasChoices("min_similarity", "threshold", "similarity_threshold"), ), ] = None, + # --- Valid-time filters (SPEC-82) --- + # A different axis from after_date: these ask what a note SAYS was true, not when + # the note was last touched. Appended at the end of the signature so no existing + # positional caller shifts. + valid_at: Annotated[ + Optional[str], + Field( + default=None, + validation_alias=AliasChoices("valid_at", "as_of", "valid_on"), + ), + "Return only sources whose authored valid range CONTAINS this date " + "('2026-07-28') or RFC 3339 instant ('2026-07-28T09:00:00Z'; a timestamp " + "with no offset is read as UTC). Sources with no temporal qualifier are " + "excluded.", + ] = None, + valid_overlaps: Annotated[ + Optional[str], + Field( + default=None, + validation_alias=AliasChoices("valid_overlaps", "overlaps", "valid_during"), + ), + "Return only sources whose authored valid range OVERLAPS this range literal, " + "written PostgreSQL-style: '[2026-06-10,2026-07-27)', '(,2026-07-27]', " + "'[2026-06-10,)'. Mutually exclusive with valid_at.", + ] = None, + time_role: Annotated[ + Optional[str], + Field( + default=None, + validation_alias=AliasChoices("time_role", "role", "time_axis"), + ), + "Narrow valid-time matching to one authored axis: 'effective', 'valid', " + "'occurred', 'due', or 'mentioned'. Usable on its own to find every source " + "carrying an assertion on that axis.", + ] = None, context: Context | None = None, ) -> dict[str, Any] | str: """Search across all content in the knowledge base with comprehensive syntax support. @@ -866,6 +922,44 @@ async def search_notes( `tags` and `status` are shorthand for metadata_filters. If the same key exists in metadata_filters, that value wins. + ### Valid-Time Filters (what a note says was true) + Notes can state when a fact holds, by writing a qualifier on an observation: + + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective:2026-07-27 The cache layer will use Memcached. + + The bracket form is an explicit range; the `@role:date` form is a point, meaning + the span its precision covers — `@2026` that year, `@2026-06` that month, and + `@2026-06-10` from that date onward. The role may be omitted (`@2026-07-27`), + which files the assertion on the `valid` axis; a role-less point has to start + with a digit and be at least as wide as a year, so `@v2` and `@may` stay prose. + + These filters query that authored time, which is a different axis from `after_date` + (last-indexed time) — `after_date` is never reinterpreted as valid time. + - `search_notes("cache layer", role="effective", valid_at="2026-07-28")` + - Returns the Memcached decision; the Redis decision expired at the cutover. + - `search_notes("cache layer", role="effective", valid_at="2026-07-01")` + - Returns the Redis decision; Memcached is not yet effective. + - `search_notes("cache layer", role="effective", valid_overlaps="[2026-06-01,2026-08-01)")` + - Returns both, since each overlaps that window. + - `search_notes("cache layer")` with no valid-time filter + - Both compete under ordinary relevance, exactly as before. + + **Sources with no temporal qualifier are excluded from any valid-time query.** + An undated note makes no claim about when it holds, so it cannot answer "what was + true on this date". Drop the valid-time filter to search dated and undated content + together. + + Because a single note can carry several assertions that disagree (as above), these + queries return observation-level results by default rather than whole notes, and + each result carries the assertion that matched so the answer can explain itself. + + Bounds follow PostgreSQL range conventions: `[` / `]` include an endpoint, `(` / `)` + exclude it, and an omitted side is unbounded. Calendar dates (`2026-07-27`) and + instants (`2026-07-27T16:42:00Z`) are separate axes that never convert into each + other: a date query matches only date ranges, an instant query only instant ranges. + An instant written without an offset is read as UTC. + ### Advanced Pattern Examples - `search_notes("project AND (meeting OR discussion)", project="work-project")` - Complex boolean logic - `search_notes('"exact phrase" AND keyword', project="research")` - Combine phrase and keyword search @@ -906,6 +1000,14 @@ async def search_notes( min_similarity: Optional float to override the global semantic_min_similarity threshold for this query. E.g., 0.0 to see all vector results, or 0.8 for high precision. Only applies to vector and hybrid search types. + valid_at: Optional date ("2026-07-28") or RFC 3339 instant ("2026-07-28T09:00:00Z"; + a timestamp with no offset is read as UTC). Returns sources whose authored + valid range contains it. Sources with no temporal qualifier are excluded. + valid_overlaps: Optional PostgreSQL-style range literal ("[2026-06-10,2026-07-27)", + "(,2026-07-27]", "[2026-06-10,)"). Returns sources whose authored valid range + overlaps it. Mutually exclusive with valid_at; also excludes undated sources. + time_role: Optional valid-time axis to narrow to: "effective", "valid", "occurred", + "due", or "mentioned". Valid on its own. context: Optional FastMCP context for performance caching. Returns: @@ -990,6 +1092,14 @@ async def search_notes( if page_size < 1: raise ValueError(f"page_size must be >= 1, got {page_size}") + # Trigger: both valid-time forms supplied. + # Why: SearchQuery rejects the pair too, but the tool assigns its fields after + # construction, so that validator never runs on this path — the caller would + # otherwise learn about it as an opaque 422 from the API. + # Outcome: one clear error naming the two mutually exclusive parameters. + if valid_at and valid_overlaps: + raise ValueError("Use either valid_at (containment) or valid_overlaps (overlap), not both.") + # Trigger: list params arrived via a direct function call instead of the MCP layer. # Why: the BeforeValidator annotations only run through MCP/Pydantic validation; direct # callers (e.g. `bm tool search-notes --type note,task` in cli/commands/tool.py, @@ -1065,6 +1175,9 @@ async def search_notes( tags=tags, status=status, min_similarity=min_similarity, + valid_at=valid_at, + valid_overlaps=valid_overlaps, + time_role=time_role, context=context, ) return all_projects_result @@ -1092,9 +1205,13 @@ async def search_notes( or entity_types or categories or after_date + or valid_at + or valid_overlaps + or time_role ), has_tags_filter=bool(tags), has_status_filter=bool(status), + has_temporal_filter=bool(valid_at or valid_overlaps or time_role), ): async with get_project_client(project, context=context, project_id=project_id) as ( client, @@ -1175,6 +1292,12 @@ async def search_notes( search_query.status = status if min_similarity is not None: search_query.min_similarity = min_similarity + if valid_at: + search_query.valid_at = valid_at + if valid_overlaps: + search_query.valid_overlaps = valid_overlaps + if time_role: + search_query.time_role = time_role # Reject searches with no criteria at all if search_query.no_criteria(): @@ -1182,7 +1305,7 @@ async def search_notes( "# No Search Criteria\n\n" "Please provide at least one of: `query`, `metadata_filters`, " "`tags`, `status`, `note_types`, `entity_types`, `categories`, " - "or `after_date`." + "`after_date`, `valid_at`, `valid_overlaps`, or `time_role`." ) # Default to entity-level results to avoid returning individual @@ -1190,14 +1313,17 @@ async def search_notes( # Applied after no_criteria() so that the implicit default doesn't # mask a truly empty search request. if not search_query.entity_types: - # Trigger: a category filter was supplied without an explicit - # entity_types. - # Why: categories only exist on observations — defaulting to "entity" - # (whose rows have NULL category) would AND a category filter against - # entity rows and return nothing, defeating a category-only search. + # Trigger: a category or valid-time filter was supplied without an + # explicit entity_types. + # Why: both only exist on observations — categories live on observation + # rows, and temporal assertions are projected against an + # observation's (type, id). Defaulting to "entity" would AND either + # filter against entity rows and return nothing, defeating the + # whole query. # Outcome: scope the implicit default to observations so - # search_notes(categories=[...]) returns the matching bullets. - if search_query.categories: + # search_notes(categories=[...]) and search_notes(valid_at=...) + # return the matching bullets. + if search_query.categories or search_query.has_temporal_filter(): search_query.entity_types = [SearchItemType("observation")] else: search_query.entity_types = [SearchItemType("entity")] diff --git a/src/basic_memory/models/__init__.py b/src/basic_memory/models/__init__.py index 6164aab74..f0377d8a3 100644 --- a/src/basic_memory/models/__init__.py +++ b/src/basic_memory/models/__init__.py @@ -4,6 +4,7 @@ from basic_memory.models.base import Base from basic_memory.models.knowledge import ( Entity, + MemoryTimeIndex, NoteContent, NoteFileVacate, NoteSection, @@ -17,6 +18,7 @@ "Base", "AcceptedProjectNoteChange", "Entity", + "MemoryTimeIndex", "NoteContent", "NoteFileVacate", "NoteSection", diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index f6693fc56..399127b24 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -8,6 +8,7 @@ from sqlalchemy import ( BigInteger, + Boolean, CheckConstraint, Integer, String, @@ -18,6 +19,7 @@ Index, JSON, Float, + false, text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship, validates @@ -132,6 +134,9 @@ class Entity(Base): uselist=False, ) sections = relationship("NoteSection", back_populates="entity", cascade="all, delete-orphan") + time_assertions = relationship( + "MemoryTimeIndex", back_populates="entity", cascade="all, delete-orphan" + ) @validates("created_at", "updated_at") def _normalize_semantic_timestamp(self, attribute_name: str, value: datetime) -> datetime: @@ -390,6 +395,112 @@ def __repr__(self) -> str: # pragma: no cover return f"Observation(id={self.id}, entity_id={self.entity_id}, content='{self.content}')" +class MemoryTimeIndex(Base): + """One authored temporal assertion, projected into queryable scalar columns. + + A note can say *when a statement is true of the world*, not merely when the file + was edited:: + + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + + That qualifier is canonical markdown. This table is its derived projection, + rebuilt under the note_content generation fence on every (re)index and removed + with the entity, exactly like observations and sections (SPEC-82). It is never a + second source of temporal truth: reindexing from the markdown reproduces it. + + The table is generic on purpose. ``source_type``/``source_id`` address whatever + carries the assertion -- observations in this MVP -- and match the ``(type, id)`` + pair of the corresponding search row, which is what lets a valid-time filter narrow + search results to the individual observation that was in force. ``source_id`` + deliberately carries no foreign key: it points into a different table per + ``source_type``. Lifecycle is carried instead by ``entity_id``'s cascade plus the + fenced replace, the same two mechanisms note_section relies on. + + Bounds are stored as canonical fixed-width text rather than DATE/TIMESTAMP columns: + + * A date bound is a calendar date and must never acquire a time of day or a + timezone. SQLAlchemy's SQLite ``DateTime`` silently discards an offset, storing + the wrong instant -- exactly the false precision the spec forbids. + * ``basic_memory.temporal`` canonicalizes every bound to a fixed-width form + (``YYYY-MM-DD``; ``YYYY-MM-DDTHH:MM:SS.ffffffZ`` in UTC), so byte-lexicographic + order *is* chronological order and one identical SQL predicate serves both + dialects. + + Native PostgreSQL ``daterange``/``tstzrange`` columns stay available as a later + addition: they would be generated from these columns, which remain the portable + source of truth. + """ + + __tablename__ = "memory_time_index" + __table_args__ = ( + # The valid-time predicate selects (source_type, source_id) after filtering on + # project, role, and axis, so this index both drives the scan and covers its + # projection. project_id leads it, which is why the column carries no separate + # index of its own the way sibling projection tables do. + Index( + "ix_memory_time_index_lookup", + "project_id", + "time_role", + "range_kind", + "source_type", + "source_id", + ), + # Fenced replace deletes by entity_id, and the cascade follows the same column. + Index("ix_memory_time_index_entity_id", "entity_id"), + CheckConstraint( + "range_kind IN ('date', 'instant')", + name="ck_memory_time_index_range_kind", + ), + # The empty range has no endpoints at all; representing it with bounds would + # make two rows describe the same interval two different ways. + CheckConstraint( + "NOT is_empty OR (lower_value IS NULL AND upper_value IS NULL)", + name="ck_memory_time_index_empty_has_no_bounds", + ), + # PostgreSQL's rule: an unbounded side cannot be inclusive, because there is no + # endpoint to include. Enforcing it here keeps the query predicates from having + # to defend against a bound state the domain value cannot produce. + CheckConstraint( + "(lower_value IS NOT NULL OR NOT lower_inclusive) " + "AND (upper_value IS NOT NULL OR NOT upper_inclusive)", + name="ck_memory_time_index_unbounded_is_exclusive", + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) # pyright: ignore [reportIncompatibleVariableOverride] + project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id")) + entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE")) + # Addresses the row that carried the qualifier, and equals the search row's + # (type, id) pair. No FK: the target table varies with source_type. + source_type: Mapped[str] = mapped_column(String(32)) + source_id: Mapped[int] = mapped_column(Integer) + time_role: Mapped[str] = mapped_column(String(32)) + range_kind: Mapped[str] = mapped_column(String(16)) + # Canonical lexical bounds; NULL means unbounded on that side. + lower_value: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + upper_value: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + lower_inclusive: Mapped[bool] = mapped_column(Boolean) + upper_inclusive: Mapped[bool] = mapped_column(Boolean) + is_empty: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false()) + extractor: Mapped[str] = mapped_column(String(32)) + # The qualifier exactly as authored, so a result can explain itself in the + # author's own precision rather than in the canonical form. + source_text: Mapped[str] = mapped_column(Text) + # `metadata` is reserved on the declarative base, so the column follows + # Entity.entity_metadata's naming convention for the same reason. + assertion_metadata: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON, nullable=True) + + entity = relationship("Entity", back_populates="time_assertions") + + @override + def __repr__(self) -> str: # pragma: no cover + return ( + f"MemoryTimeIndex(id={self.id}, entity_id={self.entity_id}, " + f"source={self.source_type}:{self.source_id}, role='{self.time_role}', " + f"range='{self.source_text}')" + ) + + class Relation(Base): """A directed relation between two entities.""" diff --git a/src/basic_memory/repository/__init__.py b/src/basic_memory/repository/__init__.py index 090e59e76..6280e34d5 100644 --- a/src/basic_memory/repository/__init__.py +++ b/src/basic_memory/repository/__init__.py @@ -1,4 +1,8 @@ from .entity_repository import EntityRepository +from .memory_time_index_repository import ( + AcceptedTemporalAssertion, + MemoryTimeIndexRepository, +) from .note_content_repository import ( AcceptedNoteContentWrite, NoteContentRepository, @@ -11,6 +15,8 @@ __all__ = [ "EntityRepository", + "AcceptedTemporalAssertion", + "MemoryTimeIndexRepository", "AcceptedNoteContentWrite", "NoteContentRepository", "NoteContentVersionConflict", diff --git a/src/basic_memory/repository/accepted_note_repositories.py b/src/basic_memory/repository/accepted_note_repositories.py index a219b50ac..35b55fccb 100644 --- a/src/basic_memory/repository/accepted_note_repositories.py +++ b/src/basic_memory/repository/accepted_note_repositories.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from basic_memory.repository import ( + MemoryTimeIndexRepository, NoteContentRepository, NoteSectionRepository, ObservationRepository, @@ -51,5 +52,8 @@ def observation_repository(self, project_id: ProjectId) -> ObservationRepository def section_repository(self, project_id: ProjectId) -> NoteSectionRepository: return NoteSectionRepository(project_id=project_id) + def temporal_repository(self, project_id: ProjectId) -> MemoryTimeIndexRepository: + return MemoryTimeIndexRepository(project_id=project_id) + def relation_repository(self, project_id: ProjectId) -> RelationRepository: return RelationRepository(project_id=project_id) diff --git a/src/basic_memory/repository/memory_time_index_repository.py b/src/basic_memory/repository/memory_time_index_repository.py new file mode 100644 index 000000000..a458bfb22 --- /dev/null +++ b/src/basic_memory/repository/memory_time_index_repository.py @@ -0,0 +1,147 @@ +"""Repository for managing MemoryTimeIndex rows.""" + +from collections import defaultdict +from dataclasses import dataclass +from typing import Iterable, Sequence + +from sqlalchemy.ext.asyncio import AsyncSession + +from basic_memory.models import MemoryTimeIndex +from basic_memory.repository.relation_repository import current_relation_generation_statement +from basic_memory.repository.repository import SELECT_BY_IDS_CHUNK_SIZE, Repository +from basic_memory.temporal import TemporalAssertion + + +@dataclass(frozen=True, slots=True) +class AcceptedTemporalAssertion: + """One authored assertion paired with the persisted row that carried it. + + The parser cannot supply `source_type`/`source_id`: it reads markdown, where the + projection's row identities do not exist yet. Publication mints them and pairs + them here. + """ + + source_type: str + source_id: int + assertion: TemporalAssertion + + +@dataclass(frozen=True, slots=True) +class TemporalGenerationWriteResult: + """Whether a guarded temporal replacement still owned its source generation.""" + + generation_is_current: bool + + +def _projection_row( + accepted: AcceptedTemporalAssertion, + *, + project_id: int, + entity_id: int, +) -> MemoryTimeIndex: + """Flatten one assertion into the portable scalar columns the table stores.""" + valid_during = accepted.assertion.valid_during + return MemoryTimeIndex( + project_id=project_id, + entity_id=entity_id, + source_type=accepted.source_type, + source_id=accepted.source_id, + time_role=accepted.assertion.time_role.value, + range_kind=valid_during.kind.value, + lower_value=valid_during.lower, + upper_value=valid_during.upper, + lower_inclusive=valid_during.lower_inclusive, + upper_inclusive=valid_during.upper_inclusive, + is_empty=valid_during.is_empty, + extractor=accepted.assertion.extractor, + source_text=accepted.assertion.source_text, + assertion_metadata=accepted.assertion.metadata, + ) + + +class MemoryTimeIndexRepository(Repository[MemoryTimeIndex]): + """Repository for the temporal projection of accepted note content.""" + + project_id: int + + def __init__(self, project_id: int): + """Initialize with project_id filter. + + Args: + project_id: Project ID to filter all operations by + """ + super().__init__(MemoryTimeIndex, project_id=project_id) + + async def find_by_entity( + self, session: AsyncSession, entity_id: int + ) -> Sequence[MemoryTimeIndex]: + """Find every temporal assertion projected from one entity.""" + query = ( + self.select() + .filter(MemoryTimeIndex.entity_id == entity_id) + .order_by(MemoryTimeIndex.source_id, MemoryTimeIndex.id) + ) + result = await self.execute_query(session, query) + return result.scalars().all() + + async def find_for_sources( + self, + session: AsyncSession, + sources: Iterable[tuple[str, int]], + ) -> Sequence[MemoryTimeIndex]: + """Find every assertion carried by the given ``(source_type, source_id)`` rows. + + Used to explain search hits, so it batches: search returns a page of rows and + this loads their assertions in one pass per source type rather than one query + per hit. Ids are chunked because SQLite caps bound parameters per statement. + """ + ids_by_type: defaultdict[str, list[int]] = defaultdict(list) + for source_type, source_id in sources: + ids_by_type[source_type].append(source_id) + if not ids_by_type: + return [] + + rows: list[MemoryTimeIndex] = [] + for source_type, source_ids in ids_by_type.items(): + for start in range(0, len(source_ids), SELECT_BY_IDS_CHUNK_SIZE): + chunk = source_ids[start : start + SELECT_BY_IDS_CHUNK_SIZE] + query = ( + self.select() + .filter(MemoryTimeIndex.source_type == source_type) + .filter(MemoryTimeIndex.source_id.in_(chunk)) + .order_by(MemoryTimeIndex.source_id, MemoryTimeIndex.id) + ) + result = await self.execute_query(session, query) + rows.extend(result.scalars().all()) + return rows + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + """Replace temporal rows only while the accepted content generation is current.""" + # This helper is a shared note_content fence despite its historical relation name. + current_generation = await session.scalar( + current_relation_generation_statement( + project_id=self.project_id, + entity_id=entity_id, + generation=generation, + ) + ) + # Trigger: a newer accepted note generation won before this transaction acquired the row. + # Why: replacing here would publish valid time the current markdown no longer asserts. + # Outcome: leave every existing row untouched and let the current writer publish. + if current_generation is None: + return TemporalGenerationWriteResult(generation_is_current=False) + + await self.delete_by_fields(session, entity_id=entity_id) + rows = [ + _projection_row(accepted, project_id=self.project_id, entity_id=entity_id) + for accepted in assertions + ] + await self.add_all_no_return(session, rows) + return TemporalGenerationWriteResult(generation_is_current=True) diff --git a/src/basic_memory/repository/observation_repository.py b/src/basic_memory/repository/observation_repository.py index 57e18c86a..3753184ec 100644 --- a/src/basic_memory/repository/observation_repository.py +++ b/src/basic_memory/repository/observation_repository.py @@ -11,6 +11,7 @@ from basic_memory.models import Observation from basic_memory.repository.relation_repository import current_relation_generation_statement from basic_memory.repository.repository import Repository +from basic_memory.temporal import TemporalAssertion @dataclass(frozen=True, slots=True) @@ -20,19 +21,30 @@ class AcceptedObservationWrite: Mirrors the markdown ``Observation`` fields so the accepted-write path can persist the graph without constructing ORM rows in the storage-neutral runner (issue #1076). + + ``temporal`` rides along rather than becoming observation columns: authored + valid time is its own projection keyed on the row this write mints, and an + observation may carry several assertions (SPEC-82). """ content: str category: str | None context: str | None tags: list[str] | None + temporal: tuple[TemporalAssertion, ...] = () @dataclass(frozen=True, slots=True) class ObservationGenerationWriteResult: - """Whether a guarded observation replacement still owned its source generation.""" + """Whether a guarded observation replacement still owned its source generation. + + ``observation_ids`` are the freshly minted row ids in document order, aligned + with the sequence that was written. The temporal projection addresses those + rows, and they only exist once the insert has flushed. + """ generation_is_current: bool + observation_ids: tuple[int, ...] = () class ObservationRepository(Repository[Observation]): @@ -142,4 +154,10 @@ async def replace_observations_for_generation( for obs in observations ] await self.add_all_no_return(session, rows) - return ObservationGenerationWriteResult(generation_is_current=True) + # add_all_no_return flushes, so every row now carries its database id. + # Reading them here, inside the same transaction, is what lets the temporal + # projection address these exact rows. + return ObservationGenerationWriteResult( + generation_is_current=True, + observation_ids=tuple(row.id for row in rows), + ) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index eb8a3f4e3..02cc5d7d2 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -35,6 +35,7 @@ build_fts_page_stage, ) from basic_memory.repository.metadata_filters import parse_metadata_filters +from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex from basic_memory.repository.semantic_vector_sync import ( @@ -48,6 +49,7 @@ from basic_memory.repository.pgvector_index import PgVectorIndex from basic_memory.repository.postgres_fts_chunks import split_postgres_fts_chunks from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter _TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") @@ -972,6 +974,7 @@ async def _build_fts_query_parts( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, allow_relaxed: bool = False, ) -> tuple[str, str, dict[str, Any], str, str]: """Build Postgres FTS FROM/WHERE params shared by search and count.""" @@ -1189,6 +1192,18 @@ async def _build_fts_query_parts( # order by most recent first order_by_clause = ", search_index.updated_at DESC" + # Handle authored valid time (SPEC-82). + # Trigger: caller asked when a statement was true of the world. + # Why: `after_date` above filters `updated_at`, which records when the note was + # last edited. That is bookkeeping, never a semantic claim; a decision + # effective through July says nothing about when its file was touched. + # Outcome: an independent predicate over the temporal projection, textually + # identical to the SQLite one because canonical bounds compare + # lexicographically on both backends. Undated sources carry no row and + # are therefore excluded whenever a valid-time filter is present. + if temporal is not None: + conditions.append(build_temporal_predicate(temporal, params)) + # Handle structured metadata filters (frontmatter) # Uses jsonb_extract_path_text() / jsonb_extract_path() with parameterized # path parts instead of #>> / #> with interpolated paths. @@ -1368,6 +1383,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -1390,6 +1406,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, limit=limit, @@ -1417,6 +1434,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, allow_relaxed=allow_relaxed, ) @@ -1564,6 +1582,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, @@ -1581,6 +1600,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, ) @@ -1602,6 +1622,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, allow_relaxed=allow_relaxed, ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index ace8e329e..4d7685128 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -27,6 +27,7 @@ from basic_memory.runtime.vector_sync import VectorSyncBatchResult from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter class SearchRepository(Protocol): @@ -80,6 +81,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -104,6 +106,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index daa561778..acc212f40 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -83,6 +83,7 @@ SearchRetrievalMode, normalize_file_path_prefix, ) +from basic_memory.temporal import TemporalFilter from basic_memory.utils import ensure_timezone_aware # --- Semantic search constants --- @@ -416,6 +417,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[Dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -437,6 +439,9 @@ async def search( categories: Filter observations by exact category (e.g. "requirement") metadata_filters: Structured frontmatter metadata filters file_path_prefix: Directory subtree scope, matched against file_path + temporal: Authored valid-time filter. Unlike after_date, which reads the + note's edit bookkeeping, this reads the time an observation claims to + be true of the world. Sources without such a claim are excluded. limit: Maximum results to return offset: Number of results to skip @@ -461,6 +466,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[Dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, @@ -2014,6 +2020,7 @@ async def _dispatch_retrieval_mode( categories: Optional[List[str]], metadata_filters: Optional[dict[str, Any]], file_path_prefix: Optional[str], + temporal: Optional[TemporalFilter], retrieval_mode: SearchRetrievalMode, min_similarity: Optional[float] = None, limit: int, @@ -2050,6 +2057,7 @@ async def _dispatch_retrieval_mode( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=limit, offset=offset, @@ -2072,6 +2080,7 @@ async def _dispatch_retrieval_mode( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=limit, offset=offset, @@ -2251,6 +2260,7 @@ async def _search_vector_only( categories: Optional[List[str]], metadata_filters: Optional[dict[str, Any]], file_path_prefix: Optional[str], + temporal: Optional[TemporalFilter], min_similarity: Optional[float] = None, limit: int, offset: int, @@ -2439,6 +2449,7 @@ def _log_vector_summary() -> None: categories, metadata_filters, file_path_prefix, + temporal, ] ) @@ -2454,6 +2465,7 @@ def _log_vector_summary() -> None: categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=SearchRetrievalMode.FTS, limit=VECTOR_FILTER_SCAN_LIMIT, offset=0, @@ -2518,6 +2530,7 @@ def _log_vector_summary() -> None: categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=stable_candidate_limit, offset=0, @@ -2591,6 +2604,7 @@ async def _search_hybrid( categories: Optional[List[str]], metadata_filters: Optional[dict[str, Any]], file_path_prefix: Optional[str], + temporal: Optional[TemporalFilter], min_similarity: Optional[float] = None, limit: int, offset: int, @@ -2630,6 +2644,7 @@ async def _search_hybrid( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=SearchRetrievalMode.FTS, limit=candidate_limit, offset=0, @@ -2649,6 +2664,7 @@ async def _search_hybrid( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=candidate_limit, offset=0, @@ -2800,6 +2816,7 @@ def _materialize(entry: tuple[SearchIndexKey, float]) -> SearchIndexRow: categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, min_similarity=min_similarity, limit=stable_candidate_limit, offset=0, diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 8f5983066..1e349e534 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -40,12 +40,14 @@ build_fts_page_stage, ) from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path +from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex from basic_memory.repository.semantic_vector_sync import StagedVectorDeletion from basic_memory.repository.semantic_vector_index_factory import build_vector_index_scope from basic_memory.repository.sqlite_vec_index import SQLiteVecIndex from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter SQLITE_WORD_COLUMNS = "{title content_stems content_snippet}" @@ -788,6 +790,7 @@ async def _build_fts_query_parts( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, ) -> tuple[str, str, dict[str, Any], str, str]: """Build SQLite FTS FROM/WHERE params shared by search and count.""" conditions = [] @@ -931,6 +934,18 @@ async def _build_fts_query_parts( # order by most recent first order_by_clause = ", search_index.updated_at DESC" + # Handle authored valid time (SPEC-82). + # Trigger: caller asked when a statement was true of the world. + # Why: `after_date` above filters `updated_at`, which records when the note was + # last edited. That is bookkeeping, never a semantic claim; a decision + # effective through July says nothing about when its file was touched. + # Outcome: an independent predicate over the temporal projection. It matches + # only sources carrying a structured qualifier, so undated sources are + # excluded whenever a valid-time filter is present, and no ordering + # changes -- relevance still decides the ranking. + if temporal is not None: + conditions.append(build_temporal_predicate(temporal, params)) + # Handle structured metadata filters (frontmatter) if metadata_filters: parsed_filters = parse_metadata_filters(metadata_filters) @@ -1085,6 +1100,7 @@ async def search( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -1113,6 +1129,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, limit=limit, @@ -1140,6 +1157,7 @@ async def search( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, ) # set limit on search query @@ -1272,6 +1290,7 @@ async def count( categories: Optional[List[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, allow_relaxed: bool = False, @@ -1289,6 +1308,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, retrieval_mode=retrieval_mode, min_similarity=min_similarity, ) @@ -1310,6 +1330,7 @@ async def count( categories=categories, metadata_filters=metadata_filters, file_path_prefix=file_path_prefix, + temporal=temporal, ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" logger.trace(f"Count {sql} params: {params}") diff --git a/src/basic_memory/repository/temporal_filters.py b/src/basic_memory/repository/temporal_filters.py new file mode 100644 index 000000000..ba27bd2c2 --- /dev/null +++ b/src/basic_memory/repository/temporal_filters.py @@ -0,0 +1,138 @@ +"""SQL for the valid-time search predicate (SPEC-82). + +One builder serves both dialects. That is not a coincidence to be maintained by +discipline -- it falls out of two decisions made upstream: + +* `basic_memory.temporal` canonicalizes every bound to a fixed-width lexical form, + so `<`, `>`, and `=` on plain text columns *are* chronological comparisons and no + typed date bind is needed on either side. +* Inclusivity on the query side is known while the SQL is being built, and + inclusivity on the stored side is a boolean column, so both fold into the SQL text. + The only bound parameters are the two bound values, the role, and the axis -- each + compared directly against a column, so PostgreSQL always infers their type and + asyncpg never sees a bare untyped parameter. + +The predicate is a *non-correlated* subquery, and that shape is load-bearing rather +than stylistic. SQLite's default word search emits an OR of per-column `MATCH` +predicates; adding a correlated `EXISTS` to that WHERE clause makes SQLite refuse the +statement outright ("unable to use function MATCH in the requested context"). A +non-correlated `IN` is evaluated independently and composes with every FTS shape in +this repository -- the OR-of-columns form, the table-level `MATCH` used for script +queries, the bm25-preserving derived table, and the rowid rewrite -- while leaving +bm25 ranking intact. It also needs no change to any `from_clause`. +""" + +from __future__ import annotations + +from typing import Any + +from basic_memory.temporal import TemporalFilter, TemporalRange + +TEMPORAL_INDEX_TABLE = "memory_time_index" + +# No stored assertion can match, and no subquery needs to run to prove it. +_MATCHES_NOTHING = "1 = 0" + + +def _not_source_ends_before_window(window: TemporalRange) -> str | None: + """Reject stored ranges that finish before the queried window begins. + + Returns None when the window is unbounded below, because then nothing can end + before it starts and the whole conjunct is vacuous. + """ + if window.lower is None: + return None + clauses = [ + # An unbounded stored upper end never terminates, so it can never be "before". + f"{TEMPORAL_INDEX_TABLE}.upper_value IS NULL", + f"{TEMPORAL_INDEX_TABLE}.upper_value > :tq_lower", + ] + if window.lower_inclusive: + # The window owns its lower endpoint, so a stored range that closes on that + # same endpoint still shares it. + clauses.append( + f"({TEMPORAL_INDEX_TABLE}.upper_value = :tq_lower " + f"AND {TEMPORAL_INDEX_TABLE}.upper_inclusive)" + ) + return f"({' OR '.join(clauses)})" + + +def _not_window_ends_before_source(window: TemporalRange) -> str | None: + """Reject stored ranges that begin after the queried window ends. + + The mirror image of `_not_source_ends_before_window`; None when the window is + unbounded above. + """ + if window.upper is None: + return None + clauses = [ + f"{TEMPORAL_INDEX_TABLE}.lower_value IS NULL", + f"{TEMPORAL_INDEX_TABLE}.lower_value < :tq_upper", + ] + if window.upper_inclusive: + clauses.append( + f"({TEMPORAL_INDEX_TABLE}.lower_value = :tq_upper " + f"AND {TEMPORAL_INDEX_TABLE}.lower_inclusive)" + ) + return f"({' OR '.join(clauses)})" + + +def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) -> str: + """Build the WHERE-clause fragment restricting search rows by authored valid time. + + Two intervals overlap exactly when neither lies entirely before the other, which + is what the two helpers above assert. Containment of a single date or instant is + the same question asked of the degenerate closed range `[p,p]`, so `valid_at` and + `valid_overlaps` share this one implementation and cannot drift apart. + + The result matches only sources carrying a structured assertion: a note without a + qualifier contributes no row here and is therefore excluded, which is the + documented default for a valid-time query. + + Binds are added to `params` in place, following the convention already used by the + surrounding FTS query builders. + """ + window = temporal.window + if window is not None and window.is_empty: + # PostgreSQL: nothing overlaps the empty range, not even itself. Emitting a + # false constant is both correct and cheaper than running the subquery. + return _MATCHES_NOTHING + + conditions = [f"{TEMPORAL_INDEX_TABLE}.project_id = :project_id"] + + if temporal.role is not None: + params["tq_role"] = temporal.role.value + conditions.append(f"{TEMPORAL_INDEX_TABLE}.time_role = :tq_role") + + if window is not None: + # Trigger: the caller asked about a specific date or a specific instant. + # Why: calendar dates and instants are different axes; converting between + # them would invent a timezone or a time of day the author never wrote. + # Outcome: a date query can never match an instant range, or the reverse. + params["tq_kind"] = window.kind.value + conditions.append(f"{TEMPORAL_INDEX_TABLE}.range_kind = :tq_kind") + # The empty stored range contains no points, so it overlaps nothing. + conditions.append(f"NOT {TEMPORAL_INDEX_TABLE}.is_empty") + + if window.lower is not None: + params["tq_lower"] = window.lower + if window.upper is not None: + params["tq_upper"] = window.upper + conditions.extend( + clause + for clause in ( + _not_source_ends_before_window(window), + _not_window_ends_before_source(window), + ) + if clause is not None + ) + + where_clause = "\n AND ".join(conditions) + # (type, id) is the search row's own identity and the address this projection + # stores, so the pair joins the two without a correlated reference. + return ( + "(search_index.type, search_index.id) IN (\n" + f" SELECT {TEMPORAL_INDEX_TABLE}.source_type, {TEMPORAL_INDEX_TABLE}.source_id\n" + f" FROM {TEMPORAL_INDEX_TABLE}\n" + f" WHERE {where_clause})" + ) diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 749d93243..ebf006e93 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -9,7 +9,7 @@ from typing import Optional, List, Union, Any from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from basic_memory.schemas.base import Permalink, normalize_note_type @@ -77,6 +77,12 @@ class SearchQuery(BaseModel): - file_path_prefix: Limit to one directory subtree of the project - tags: Convenience frontmatter tag filter - status: Convenience frontmatter status filter + - valid_at / valid_overlaps / time_role: Authored valid-time filters (SPEC-82) + + Valid time is what a note *says about the world*, written as a qualifier on an + observation (``- [decision] @effective[2026-06-10,2026-07-27) ...``). It is a + different axis from ``after_date``, which filters on when a row was last indexed + and is deliberately left untouched by these fields. Boolean search examples: - "python AND flask" - Find items with both terms @@ -106,6 +112,22 @@ class SearchQuery(BaseModel): retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS min_similarity: Optional[float] = None # Per-query override for semantic_min_similarity + # Authored valid-time filters. Kept as strings at the boundary so HTTP clients and + # MCP callers can pass one flat value; the service parses them into the portable + # domain values and rejects anything malformed with a visible diagnostic. + valid_at: Optional[str] = None # Date or RFC 3339 instant the range must contain + valid_overlaps: Optional[str] = None # Range literal, e.g. "[2026-06-10,2026-07-27)" + time_role: Optional[str] = None # effective | valid | occurred | due | mentioned + + @model_validator(mode="after") + def validate_temporal_filter(self) -> "SearchQuery": + """Refuse a query that asks two different valid-time questions at once.""" + if self.valid_at is not None and self.valid_overlaps is not None: + raise ValueError( + "Use either valid_at (containment) or valid_overlaps (overlap), not both." + ) + return self + @field_validator("after_date") @classmethod def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]: @@ -133,6 +155,15 @@ def normalize_scope(cls, value: Optional[str]) -> Optional[str]: """ return normalize_file_path_prefix(value) + def has_temporal_filter(self) -> bool: + """Whether this query asks a valid-time question at all. + + A role on its own is a legal filter: it asks for sources carrying any + assertion on that axis. Callers use this to decide whether valid time was + requested without parsing the values, which is why it never raises. + """ + return bool(self.valid_at or self.valid_overlaps or self.time_role) + def no_criteria(self) -> bool: text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip()) metadata_is_empty = not self.metadata_filters @@ -155,6 +186,7 @@ def no_criteria(self) -> bool: and self.file_path_prefix is None and tags_is_empty and status_is_empty + and not self.has_temporal_filter() ) def has_boolean_operators(self) -> bool: @@ -169,6 +201,37 @@ def has_boolean_operators(self) -> bool: return any(pattern in text for pattern in boolean_patterns) +class TemporalRangeValue(BaseModel): + """One authored interval, as a caller sees it. + + This is the single logical `valid_during` value the API and MCP boundary expose. + How the projection stores it -- which table, which columns, which indexes -- is + deliberately absent: `literal` is the canonical PostgreSQL range literal and the + decomposed bounds are the same interval, spelled out so a caller can compare + endpoints without parsing. + """ + + kind: str # "date" (calendar dates) or "instant" (UTC timestamps) + literal: str # e.g. "[2026-06-10,2026-07-27)", "(,2026-07-27]", "empty" + lower: Optional[str] = None # None means unbounded on that side + upper: Optional[str] = None + lower_inclusive: bool = False + upper_inclusive: bool = False + is_empty: bool = False + + +class TemporalResultMetadata(BaseModel): + """One authored valid-time assertion carried by a search result. + + Present so an agent can say *why* a source matched a valid-time query -- which + axis it was asserted on, over what interval, and in the author's own words. + """ + + role: str # effective | valid | occurred | due | mentioned + valid_during: TemporalRangeValue + source_text: str # the qualifier exactly as authored, e.g. "@effective[2026-06-10,)" + + class SearchResult(BaseModel): """Search result with score and metadata.""" @@ -199,6 +262,11 @@ class SearchResult(BaseModel): to_entity: Optional[Permalink] = None # For relations relation_type: Optional[str] = None # For relations + # Authored valid-time assertions carried by this row. Collection-shaped from day + # one: the MVP parser reads one qualifier per observation, but multiple assertions + # on multiple axes must not be a schema break later. + temporal: Optional[List[TemporalResultMetadata]] = None + class SearchResponse(BaseModel): """Wrapper for search results.""" @@ -215,3 +283,17 @@ class SearchResponse(BaseModel): description="Whether total is an exact count that clients can use for pagination", ) has_more: bool = False + # Version-skew guard. SearchQuery ignores unknown fields, so a client that sends a + # valid-time filter to a server predating SPEC-82 would receive unfiltered results + # that look filtered -- silently including the undated sources the filter excludes. + # + # Three states, all meaningful: True (asked and executed), None (never asked, so + # nothing to confirm), and -- only from a server that does not know this field -- + # missing, which parses as None while True was expected. Staying None rather than + # False when no filter was asked keeps every ordinary search payload byte-identical + # to what it was before valid time existed. + temporal_applied: Optional[bool] = Field( + default=None, + description="True when the server executed a requested valid-time filter; " + "absent when the request carried none", + ) diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 715c52cfc..d142b0cd6 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -30,6 +30,7 @@ from basic_memory.repository import ObservationRepository, RelationRepository from basic_memory.repository.entity_repository import EntityRepository from basic_memory.repository.note_content_repository import NoteContentRepository +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository from basic_memory.repository.note_section_repository import NoteSectionRepository from basic_memory.read_cache import ReadCache, invalidate_cache from basic_memory.runtime.note_move import normalize_note_move_destination_path @@ -233,6 +234,7 @@ async def _publish_markdown_graph( category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in markdown.observations ) @@ -252,6 +254,7 @@ async def _publish_markdown_graph( relation_repository=self.relation_repository, observation_repository=self.observation_repository, section_repository=NoteSectionRepository(project_id=self.repository.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=self.repository.project_id), session_maker=self.session_maker, ) published = await publisher.publish( diff --git a/src/basic_memory/services/note_content_writes.py b/src/basic_memory/services/note_content_writes.py index 1143ba745..4da33ab45 100644 --- a/src/basic_memory/services/note_content_writes.py +++ b/src/basic_memory/services/note_content_writes.py @@ -175,10 +175,14 @@ async def _publish_relation_generation( section_repository = self.mutation_dependencies.write_repositories.section_repository( publication.project_id ) + temporal_repository = self.mutation_dependencies.write_repositories.temporal_repository( + publication.project_id + ) publisher = RelationGenerationPublisher( relation_repository=repository, observation_repository=observation_repository, section_repository=section_repository, + temporal_repository=temporal_repository, session_maker=self.session_maker, ) await publisher.publish( diff --git a/src/basic_memory/services/note_preparation.py b/src/basic_memory/services/note_preparation.py index 23ed69e2a..e2508ed4d 100644 --- a/src/basic_memory/services/note_preparation.py +++ b/src/basic_memory/services/note_preparation.py @@ -95,6 +95,7 @@ def observations(self) -> list[AcceptedObservationWrite]: category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in self.entity_markdown.observations ] @@ -875,6 +876,7 @@ async def prepare_move_entity_content( category=observation.category, context=observation.context, tags=observation.tags, + temporal=tuple(observation.temporal), ) for observation in entity_markdown.observations ), diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index 588090365..acdaf920e 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -32,6 +32,13 @@ VectorSyncBatchResult, ) from basic_memory.services import FileService +from basic_memory.temporal import ( + TemporalFilter, + TemporalQualifierError, + TimeRole, + parse_point, + parse_range_literal, +) # Maximum size for content_stems field to stay under Postgres's 8KB index row limit. # We use 6000 characters to leave headroom for other indexed columns and overhead. @@ -52,6 +59,7 @@ class PreparedSearchQuery: after_date: datetime | None metadata_filters: dict[str, Any] | None file_path_prefix: str | None + temporal: TemporalFilter | None retrieval_mode: SearchRetrievalMode min_similarity: float | None @@ -84,6 +92,50 @@ def entity_embeddings_enabled(entity: Entity) -> bool: return True +def build_temporal_filter(query: SearchQuery) -> TemporalFilter | None: + """Parse the flat valid-time fields into one portable filter value. + + The boundary carries strings so HTTP and MCP callers can pass a single flat value + per axis. Every rejection here is deliberate and loud: an unknown role, a malformed + range literal, a range mixing calendar dates with instants, or an impossible range + raises rather than degrading into a filter that quietly matches something else. + Callers above map the error to a 400. A timestamp written without an offset is not + a rejection -- like every other naive datetime in the codebase, it is read as UTC. + """ + if not query.has_temporal_filter(): + return None + + role: TimeRole | None = None + if query.time_role: + try: + role = TimeRole(query.time_role) + except ValueError as exc: + raise TemporalQualifierError( + f"unknown time_role {query.time_role!r}; expected one of " + f"{', '.join(item.value for item in TimeRole)}" + ) from exc + + return TemporalFilter( + role=role, + at=parse_point(query.valid_at) if query.valid_at else None, + overlaps=parse_range_literal(query.valid_overlaps) if query.valid_overlaps else None, + ) + + +def _describe_temporal_criteria(temporal: TemporalFilter | None) -> str | None: + """Render the valid-time question that actually ran, for search traces.""" + if temporal is None: + return None + parts = [] + if temporal.role is not None: + parts.append(f"role={temporal.role.value}") + if temporal.at is not None: + parts.append(f"valid_at={temporal.at.value}") + elif temporal.overlaps is not None: + parts.append(f"valid_overlaps={temporal.overlaps}") + return ",".join(parts) + + def describe_search_criteria(prepared: PreparedSearchQuery) -> str: """Render the criteria the repository actually executed. @@ -111,6 +163,7 @@ def quoted(value: str | None) -> str | None: "categories": list(prepared.categories) if prepared.categories else None, "metadata_filters": dict(prepared.metadata_filters) if prepared.metadata_filters else None, "file_path_prefix": quoted(prepared.file_path_prefix), + "temporal": _describe_temporal_criteria(prepared.temporal), } return " ".join(f"{name}={value}" for name, value in criteria.items() if value is not None) @@ -227,6 +280,7 @@ def prepare_query(self, query: SearchQuery) -> PreparedSearchQuery | None: after_date=after_date, metadata_filters=metadata_filters, file_path_prefix=query.file_path_prefix, + temporal=build_temporal_filter(query), retrieval_mode=query.retrieval_mode or SearchRetrievalMode.FTS, min_similarity=query.min_similarity, ) @@ -243,6 +297,7 @@ def prepare_query(self, query: SearchQuery) -> PreparedSearchQuery | None: or prepared.metadata_filters # Normalized by SearchQuery, so only a real subtree reaches here. or prepared.file_path_prefix + or prepared.temporal ) if not has_criteria: logger.debug("no criteria passed to query") @@ -258,6 +313,7 @@ def _prepared_has_filters(prepared: PreparedSearchQuery) -> bool: or prepared.categories or prepared.after_date or prepared.file_path_prefix + or prepared.temporal ) async def _include_legacy_note_type_spellings( @@ -312,6 +368,7 @@ async def _search_repository( after_date=prepared.after_date, metadata_filters=prepared.metadata_filters, file_path_prefix=prepared.file_path_prefix, + temporal=prepared.temporal, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, limit=limit, @@ -330,6 +387,7 @@ async def _search_repository( after_date=prepared.after_date, metadata_filters=prepared.metadata_filters, file_path_prefix=prepared.file_path_prefix, + temporal=prepared.temporal, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, limit=limit, @@ -357,6 +415,7 @@ async def _count_repository( after_date=prepared.after_date, metadata_filters=prepared.metadata_filters, file_path_prefix=prepared.file_path_prefix, + temporal=prepared.temporal, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, allow_relaxed=allow_relaxed, diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py new file mode 100644 index 000000000..7009e634e --- /dev/null +++ b/src/basic_memory/temporal.py @@ -0,0 +1,496 @@ +"""Portable temporal value types for authored valid time (SPEC-82). + +Basic Memory authors time as *semantic* data. A `[decision]` that was effective from +June 10 until the July 27 cutover is a statement about the world, not a record of when +the note was edited. This module owns the values that carry such a statement and the +lexical grammar for the range literals authors write. + +PostgreSQL's range conventions are the language contract: `[lower,upper)` with explicit +inclusivity per side, unbounded ends, and a distinguished empty range. That is a +vocabulary choice, not a storage requirement -- these values reduce to portable scalars +so SQLite and Postgres can share one logical model. + +Two canonical lexical forms carry every bound: + + date ``YYYY-MM-DD`` (10 characters) + instant ``YYYY-MM-DDTHH:MM:SS.ffffffZ`` (27 characters, always UTC) + +Both are fixed width with ASCII digits in fixed positions, so byte-lexicographic order +is chronological order. That is what lets containment and overlap be plain string +comparisons with identical SQL text in either dialect. + +The two kinds never mix and never convert into one another. A date bound is a calendar +date: it acquires no time of day and no timezone, ever. An instant bound names a moment +and is normalized to UTC, so two instants written in different offsets compare as the +instants they name. A timestamp written without an offset is *read as UTC*, which is +the convention the rest of the codebase already uses for naive datetimes +(`utils.ensure_timezone_aware`, `recent_activity`). + +Two authored surfaces reach these values, and they trade precision for convenience in +opposite directions: + +* A **range literal** (`[2026-06-10,2026-07-27)`) is the precise form. Its bounds must + be written in the canonical lexical shapes above, to at most microsecond precision. +* A **point** (`2026-06-10`, `2026-06`, `2026`, `yesterday`) is the convenient form. + It is read with `dateparser` and denotes the span its precision covers, so an author + never has to spell out a range to say when something started. +""" + +import re +from dataclasses import dataclass +from datetime import UTC, date, datetime +from enum import StrEnum +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Literal, override + +if TYPE_CHECKING: # pragma: no cover - import exists only for the annotation below + from dateparser.date import DateDataParser + + +class TemporalQualifierError(ValueError): + """A temporal qualifier, range literal, or bound failed to parse or validate.""" + + +class TimeRole(StrEnum): + """Which time axis an assertion describes. + + `recorded` is deliberately absent: recorded time is never authored in markdown. + """ + + EFFECTIVE = "effective" + VALID = "valid" + OCCURRED = "occurred" + DUE = "due" + MENTIONED = "mentioned" + + +class TemporalRangeKind(StrEnum): + """Whether a range is measured in calendar dates or in instants.""" + + DATE = "date" + INSTANT = "instant" + + +EMPTY_RANGE_LITERAL = "empty" +OBSERVATION_EXTRACTOR = "observation" + +# Which component a slash-formatted date leads with. Only ambiguous forms consult it: +# `10/07/2026` is July 10 under YMD/DMY and October 7 under MDY, while `2026-06-10` is +# ISO and is never re-guessed. Mirrored by `BasicMemoryConfig.date_order`. +type DateOrder = Literal["YMD", "DMY", "MDY"] + +DEFAULT_DATE_ORDER: DateOrder = "YMD" + + +# --- Bound grammar --- + +# A date bound is exactly the canonical form, so authored and canonical text agree. +# The anchored pattern also rejects the compact `20260610` shape that +# `date.fromisoformat` accepts on 3.11+, which would break fixed-width ordering. +_DATE_BOUND = re.compile(r"^\d{4}-\d{2}-\d{2}$") + +# Sub-microsecond precision is refused rather than truncated: silently dropping digits +# would make the stored bound name a different instant than the author wrote. The +# offset is optional because a naive timestamp is read as UTC, not rejected. +_INSTANT_BOUND = re.compile( + r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:[Zz]|[+-]\d{2}:\d{2})?$" +) +_CANONICAL_INSTANT = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$") + +# Anything shaped like a date followed by a time separator is *meant* as a timestamp. +# Classifying it as an instant before validating it is what lets a broken timestamp +# report itself as one instead of as "not a calendar date". +_TIMESTAMP_SHAPE = re.compile(r"^\d{4}-\d{2}-\d{2}[Tt ]") + +# `[lower,upper)` and friends. Bounds carry no brackets and no comma, so one anchored +# pattern splits the literal without any nesting rules. +_RANGE_LITERAL = re.compile(r"^([\[(])([^,\[\]()]*),([^,\[\]()]*)([\])])$") + + +def _classify_bound(bound: str) -> TemporalRangeKind: + """Decide which axis an authored bound is written on.""" + if _TIMESTAMP_SHAPE.match(bound): + return TemporalRangeKind.INSTANT + return TemporalRangeKind.DATE + + +def _canonical_date(bound: str) -> str: + if not _DATE_BOUND.match(bound): + raise TemporalQualifierError(f"date bound must be YYYY-MM-DD: {bound!r}") + try: + return date.fromisoformat(bound).isoformat() + except ValueError as exc: + raise TemporalQualifierError(f"not a calendar date: {bound!r}") from exc + + +def _instant_value(moment: datetime) -> str: + """Render one moment as the canonical fixed-width UTC instant. + + A naive moment is read as UTC rather than refused. That is the house convention for + every other naive datetime in the codebase, and it is what lets an author write + `2026-07-27T18:42:00` without learning RFC 3339's offset syntax first. + """ + if moment.tzinfo is None: + moment = moment.replace(tzinfo=UTC) + return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" + + +def _canonical_instant(bound: str) -> str: + if not _INSTANT_BOUND.match(bound): + raise TemporalQualifierError( + f"timestamp bound must be RFC 3339 to microsecond precision, " + f"with an optional offset or Z: {bound!r}" + ) + # RFC 3339 allows lowercase `t`/`z`, which `datetime.fromisoformat` rejects. Every + # other character in a matched bound is a digit or punctuation, so upper-casing the + # whole bound only touches those two markers. + try: + moment = datetime.fromisoformat(bound.upper()) + except ValueError as exc: + raise TemporalQualifierError(f"not a valid timestamp: {bound!r}") from exc + return _instant_value(moment) + + +def canonical_bound(bound: str, kind: TemporalRangeKind) -> str: + """Normalize one authored bound to the canonical fixed-width form for its kind.""" + if kind is TemporalRangeKind.DATE: + return _canonical_date(bound) + return _canonical_instant(bound) + + +def _require_canonical(value: str, kind: TemporalRangeKind) -> None: + """Reject a value that skipped `canonical_bound` on its way into a domain value.""" + pattern = _DATE_BOUND if kind is TemporalRangeKind.DATE else _CANONICAL_INSTANT + if not pattern.match(value): + raise TemporalQualifierError(f"{kind.value} bound is not canonical: {value!r}") + + +# --- Values --- + + +@dataclass(frozen=True, slots=True) +class TemporalPoint: + """One calendar date or instant that a containment question is asked about.""" + + kind: TemporalRangeKind + value: str + + def __post_init__(self) -> None: + _require_canonical(self.value, self.kind) + + @override + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class TemporalRange: + """One authored interval on a single time axis. + + Bounds are canonical lexical strings; `None` means unbounded on that side. + Construction normalizes two PostgreSQL rules so no caller has to remember them: + an unbounded side is always exclusive, and a degenerate interval (`[a,a)`, + `(a,a]`, `(a,a)`) *is* the empty range. + + Unlike PostgreSQL's `daterange`, a discrete date range is not rewritten into the + canonical `[)` form -- `[a,b]` keeps the inclusivity the author wrote. Evaluating + the authored flags directly is set-equivalent for containment and overlap and needs + no date arithmetic; only the rendered literal differs. + """ + + kind: TemporalRangeKind + lower: str | None = None + upper: str | None = None + lower_inclusive: bool = False + upper_inclusive: bool = False + is_empty: bool = False + + def __post_init__(self) -> None: + if self.is_empty: + # The empty range has no endpoints at all, so inclusivity is meaningless + # for it; representing it two ways would make equality lie. + if ( + self.lower is not None + or self.upper is not None + or self.lower_inclusive + or self.upper_inclusive + ): + raise TemporalQualifierError("the empty range carries no bounds") + return + + for bound in (self.lower, self.upper): + if bound is not None: + _require_canonical(bound, self.kind) + + # Canonical bounds are fixed width, so string order is chronological order. + if self.lower is not None and self.upper is not None and self.lower > self.upper: + raise TemporalQualifierError( + f"range lower bound {self.lower} is after upper bound {self.upper}" + ) + + # PostgreSQL: an unbounded side cannot be inclusive; there is no endpoint. + if self.lower is None: + object.__setattr__(self, "lower_inclusive", False) + if self.upper is None: + object.__setattr__(self, "upper_inclusive", False) + + # PostgreSQL: an interval whose endpoints coincide without including both of + # them contains no points, and is therefore the empty range. + if ( + self.lower is not None + and self.lower == self.upper + and not (self.lower_inclusive and self.upper_inclusive) + ): + object.__setattr__(self, "lower", None) + object.__setattr__(self, "upper", None) + object.__setattr__(self, "lower_inclusive", False) + object.__setattr__(self, "upper_inclusive", False) + object.__setattr__(self, "is_empty", True) + + @classmethod + def empty(cls, kind: TemporalRangeKind) -> "TemporalRange": + """The empty range on one axis.""" + return cls(kind=kind, is_empty=True) + + @override + def __str__(self) -> str: + """Render the canonical PostgreSQL range literal.""" + if self.is_empty: + return EMPTY_RANGE_LITERAL + lower = "" if self.lower is None else self.lower + upper = "" if self.upper is None else self.upper + return ( + f"{'[' if self.lower_inclusive else '('}{lower},{upper}" + f"{']' if self.upper_inclusive else ')'}" + ) + + +@dataclass(frozen=True, slots=True) +class TemporalFilter: + """A valid-time question asked of the stored assertions. + + Exactly one of `at` (containment) or `overlaps` may be given, or neither -- a + role-only filter asks for sources that carry *any* assertion on that axis, which + is a legal and useful question. A filter that asks nothing at all is refused + rather than silently matching everything. + """ + + role: TimeRole | None = None + at: TemporalPoint | None = None + overlaps: TemporalRange | None = None + + def __post_init__(self) -> None: + if self.at is not None and self.overlaps is not None: + raise TemporalQualifierError( + "a temporal filter asks either 'at' or 'overlaps', never both" + ) + if self.role is None and self.at is None and self.overlaps is None: + raise TemporalQualifierError("a temporal filter must name a role, a point, or a range") + + @property + def window(self) -> TemporalRange | None: + """The interval this filter tests against, or None for a role-only filter. + + Containment of a point is overlap with the degenerate closed range `[p,p]`: + both ask whether the stored interval and the queried interval share at least + one point. Collapsing them here lets one predicate answer both questions, + which is also why the two can never disagree about inclusivity or bounds. + """ + if self.at is not None: + return TemporalRange( + kind=self.at.kind, + lower=self.at.value, + upper=self.at.value, + lower_inclusive=True, + upper_inclusive=True, + ) + return self.overlaps + + +@dataclass(frozen=True, slots=True) +class TemporalAssertion: + """One authored statement that a source is valid over a span of time. + + Source identity -- entity, source type, source row id -- is deliberately absent. + The parser reads markdown, where those ids do not exist yet; the projection layer + pairs this value with them when it writes derived rows. + + `source_text` is the exact authored token. Serialization replays it verbatim, so a + parse/serialize round trip reproduces the author's bounds and precision even though + `valid_during` holds the normalized form. + """ + + time_role: TimeRole + valid_during: TemporalRange + source_text: str + extractor: str = OBSERVATION_EXTRACTOR + metadata: dict[str, Any] | None = None + + +# --- Literal parsing --- + + +def parse_range_literal(literal: str, *, kind: TemporalRangeKind | None = None) -> TemporalRange: + """Parse a PostgreSQL-style range literal into a canonical `TemporalRange`. + + Accepts `[lower,upper)`, `(lower,upper]`, `[lower,)`, `(,upper)`, `(,)`, and the + bare token `empty`. `kind` asserts the expected axis; when omitted the axis is + inferred from the bounds, which is why the bound-less forms require it explicitly. + """ + text = literal.strip() + if text == EMPTY_RANGE_LITERAL: + if kind is None: + raise TemporalQualifierError( + "the 'empty' range literal has no bounds, so its kind must be given" + ) + return TemporalRange.empty(kind) + + match = _RANGE_LITERAL.match(text) + if match is None: + raise TemporalQualifierError( + f"range literal must be [lower,upper), (lower,upper], or 'empty': {literal!r}" + ) + open_bracket, lower_text, upper_text, close_bracket = match.groups() + lower_text = lower_text.strip() + upper_text = upper_text.strip() + + written_kinds = {_classify_bound(bound) for bound in (lower_text, upper_text) if bound} + if len(written_kinds) > 1: + raise TemporalQualifierError( + f"a range must not mix date-only and timestamp bounds: {literal!r}" + ) + if not written_kinds: + if kind is None: + raise TemporalQualifierError( + f"a fully unbounded range has no bounds to classify: {literal!r}" + ) + range_kind = kind + else: + range_kind = written_kinds.pop() + if kind is not None and range_kind is not kind: + raise TemporalQualifierError( + f"expected {kind.value} bounds but found {range_kind.value} bounds: {literal!r}" + ) + + return TemporalRange( + kind=range_kind, + lower=canonical_bound(lower_text, range_kind) if lower_text else None, + upper=canonical_bound(upper_text, range_kind) if upper_text else None, + lower_inclusive=open_bracket == "[", + upper_inclusive=close_bracket == "]", + ) + + +def parse_point(text: str) -> TemporalPoint: + """Parse one authored date or timestamp into a canonical `TemporalPoint`.""" + bound = text.strip() + if not bound: + raise TemporalQualifierError("a temporal point must not be empty") + kind = _classify_bound(bound) + return TemporalPoint(kind=kind, value=canonical_bound(bound, kind)) + + +# --- Flexible authored points --- + + +@lru_cache(maxsize=8) +def _date_data_parser(date_order: DateOrder) -> "DateDataParser": + """The flexible reader for authored points, built once per configured date order. + + Deferred import: dateparser costs ~0.13s and loads locale data, and the modules + that carry these values are imported on every CLI start (#886). Only an + observation that already looks like a qualifier ever reaches this function. + """ + from dateparser.date import DateDataParser + + return DateDataParser( + settings={ + "DATE_ORDER": date_order, + # Makes `period` report "time" when the author wrote a clock reading, + # which is exactly the date-vs-instant distinction this module keeps. + "RETURN_TIME_AS_PERIOD": True, + } + ) + + +def _calendar_span(lower: date, upper: date) -> TemporalRange: + """The half-open calendar period `[lower,upper)`.""" + return TemporalRange( + kind=TemporalRangeKind.DATE, + lower=lower.isoformat(), + upper=upper.isoformat(), + lower_inclusive=True, + ) + + +def parse_authored_point( + text: str, *, date_order: DateOrder = DEFAULT_DATE_ORDER +) -> TemporalRange | None: + """Read one authored point into the interval its precision denotes. + + The precision the author wrote is the meaning: + + 2026 -> [2026-01-01,2027-01-01) the year + 2026-06 -> [2026-06-01,2026-07-01) the month + 2026-06-10 -> [2026-06-10,) from that date onward + 2026-06-10T14:00:00 -> [that instant,) from that moment onward + + A year or a month is a period the author delimited by writing it. A date or a + moment is not: `@effective 2026-06-10` means the decision took effect that day and + still holds, so closing the range at midnight would expire it overnight. Callers + that need a closed interval write the range literal instead. + + Returns None when the text names no date. That is not an error -- the caller leaves + such a token as ordinary observation content. + """ + point = text.strip() + if _DATE_BOUND.match(point): + # Trigger: the text is already in the canonical ISO date shape. + # Why: dateparser is lenient with impossible components -- it reads + # "2026-13-01" as the 13th of January -- and a silently wrong date is worse + # than an unread token. + # Outcome: ISO dates are parsed as ISO, or refused. + try: + return TemporalRange( + kind=TemporalRangeKind.DATE, + lower=date.fromisoformat(point).isoformat(), + lower_inclusive=True, + ) + except ValueError: + return None + + date_data = _date_data_parser(date_order).get_date_data(point) + moment = date_data.date_obj + if moment is None: + return None + + # dateparser fills components the author did not write from today's date, so only + # the components `period` vouches for may be read off `moment`. + match date_data.period: + case "time": + return TemporalRange( + kind=TemporalRangeKind.INSTANT, + lower=_instant_value(moment), + lower_inclusive=True, + ) + case "year": + if moment.year >= date.max.year: + # There is no January 1 after year 9999 to close the span with. + return None + return _calendar_span(date(moment.year, 1, 1), date(moment.year + 1, 1, 1)) + case "month": + first = date(moment.year, moment.month, 1) + next_month = ( + date(first.year + 1, 1, 1) + if first.month == 12 + else date(first.year, first.month + 1, 1) + ) + return _calendar_span(first, next_month) + case _: + # Day precision, and any coarser calendar period dateparser resolves to a + # specific day ("last week"): the day it named, onward. + return TemporalRange( + kind=TemporalRangeKind.DATE, + lower=moment.date().isoformat(), + lower_inclusive=True, + ) diff --git a/tests/api/v2/test_search_router_telemetry.py b/tests/api/v2/test_search_router_telemetry.py index a3c33a0f1..d1eec98ba 100644 --- a/tests/api/v2/test_search_router_telemetry.py +++ b/tests/api/v2/test_search_router_telemetry.py @@ -32,7 +32,7 @@ def fake_span(name: str, **attrs): operations.append((name, attrs)) yield - async def fake_to_search_results(entity_service, results): + async def fake_to_search_results(entity_service, results, *, temporal_by_source=None): return [] monkeypatch.setattr(logfire, "span", fake_span) @@ -43,6 +43,9 @@ async def fake_to_search_results(entity_service, results): query=SearchQuery(text="hello world"), search_service=FakeSearchService(), entity_service=object(), + # This query carries no valid-time filter, so neither is touched. + temporal_repository=object(), + session_maker=object(), read_cache=None, response=http_response, project_id="11111111-1111-1111-1111-111111111111", @@ -64,5 +67,6 @@ async def fake_to_search_results(entity_service, results): "retrieval_mode": "fts", "has_query": True, "has_filters": False, + "has_temporal_filter": False, }, ) diff --git a/tests/api/v2/test_search_router_temporal.py b/tests/api/v2/test_search_router_temporal.py new file mode 100644 index 000000000..a85d2b38b --- /dev/null +++ b/tests/api/v2/test_search_router_temporal.py @@ -0,0 +1,291 @@ +"""Valid-time filters over the v2 search endpoint (SPEC-82). + +The router is where three things have to line up: the filter reaches the service, the +matched assertions come back with the results, and the response says the filter actually +ran. That last one is not decoration -- `SearchQuery` ignores unknown fields, so without +an explicit confirmation an older server would answer a valid-time query with unfiltered +results that look filtered. +""" + +from textwrap import dedent +from typing import Any + +import pytest +from httpx import AsyncClient + +from basic_memory.models import Project +from basic_memory.schemas import Entity as EntitySchema + +CACHE_LAYER_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective[2026-07-27,) The cache layer will use Memcached. + """) + +UNDATED_MARKDOWN = dedent(""" + # Queue Layer + + ## Observations + - [decision] The queue layer will use RabbitMQ. + """) + + +async def _index_note(entity_service, search_service, title: str, content: str): + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title=title, + note_type="note", + directory="decisions", + content=content, + ) + ) + await search_service.index_entity(entity) + return entity + + +async def _search(client: AsyncClient, v2_project_url: str, **query: Any) -> dict[str, Any]: + response = await client.post(f"{v2_project_url}/search/", json=query) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.asyncio +async def test_temporal_filter_round_trips_through_v2_search( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """A valid-time query narrows to the observation in force and explains why.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + time_role="effective", + valid_at="2026-07-28", + ) + + assert payload["temporal_applied"] is True + contents = [result["content"] for result in payload["results"]] + assert any("Memcached" in (content or "") for content in contents), contents + assert not any("Redis" in (content or "") for content in contents), contents + + [result] = payload["results"] + [assertion] = result["temporal"] + assert assertion["role"] == "effective" + assert assertion["source_text"] == "@effective[2026-07-27,)" + assert assertion["valid_during"] == { + "kind": "date", + "literal": "[2026-07-27,)", + "lower": "2026-07-27", + "upper": None, + "lower_inclusive": True, + "upper_inclusive": False, + "is_empty": False, + } + + +@pytest.mark.asyncio +async def test_overlap_filter_returns_both_competing_decisions( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """A window spanning the cutover overlaps both effective periods.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_overlaps="[2026-06-01,2026-08-01)", + ) + + assert payload["temporal_applied"] is True + contents = " ".join(result["content"] or "" for result in payload["results"]) + assert "Redis" in contents and "Memcached" in contents + + +@pytest.mark.asyncio +async def test_search_without_a_temporal_filter_is_unchanged( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """An ordinary search payload is byte-for-byte what it was before valid time. + + `temporal_applied` stays null rather than false, and no result carries a temporal + block, so nothing about an existing client's parsing changes. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, v2_project_url, text="cache layer", entity_types=["observation"] + ) + + assert payload["temporal_applied"] is None + assert payload["results"] + assert all(result["temporal"] is None for result in payload["results"]) + + +@pytest.mark.asyncio +async def test_undated_note_is_excluded_and_the_exclusion_is_confirmed( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """Acceptance 8 over HTTP: undated sources drop out, and the server says so.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + await _index_note(entity_service, search_service, "Queue Layer", UNDATED_MARKDOWN) + + unfiltered = await _search(client, v2_project_url, text="layer", entity_types=["observation"]) + assert any("RabbitMQ" in (r["content"] or "") for r in unfiltered["results"]) + + filtered = await _search( + client, + v2_project_url, + text="layer", + entity_types=["observation"], + valid_at="2026-07-28", + ) + assert filtered["temporal_applied"] is True + assert not any("RabbitMQ" in (r["content"] or "") for r in filtered["results"]) + + +@pytest.mark.asyncio +async def test_pagination_totals_respect_the_temporal_filter( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """`total` comes from a separate count query; it must run the same predicate. + + The router derives `has_more` from that total, so a count that ignored valid time + would advertise pages that do not exist. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + await _index_note(entity_service, search_service, "Queue Layer", UNDATED_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="layer", + entity_types=["observation"], + valid_at="2026-07-28", + ) + + assert payload["total"] == len(payload["results"]) == 1 + assert payload["has_more"] is False + + +@pytest.mark.asyncio +async def test_a_valid_time_query_with_no_matches_still_confirms_the_filter( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """An empty answer to a valid-time question is different from an unfiltered one. + + Nothing was in force in 2020, so there is nothing to hydrate -- but the caller still + needs to know the filter ran, or it cannot tell this apart from a stale server. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_at="2020-01-01", + ) + + assert payload["results"] == [] + assert payload["total"] == 0 + assert payload["temporal_applied"] is True + + +@pytest.mark.asyncio +async def test_valid_at_and_valid_overlaps_together_are_rejected( + client: AsyncClient, + test_project: Project, + v2_project_url: str, +): + """The schema refuses the contradictory pair, so it never reaches the service.""" + response = await client.post( + f"{v2_project_url}/search/", + json={"text": "cache", "valid_at": "2026-07-28", "valid_overlaps": "[2026-06-10,)"}, + ) + + assert response.status_code == 422 + assert "not both" in response.text + + +@pytest.mark.asyncio +async def test_temporal_only_query_is_accepted_as_criteria( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """A valid-time filter alone is a complete search request.""" + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + payload = await _search( + client, v2_project_url, entity_types=["observation"], time_role="effective" + ) + + assert payload["temporal_applied"] is True + assert len(payload["results"]) == 2 + + +@pytest.mark.asyncio +async def test_read_cache_distinguishes_two_valid_time_questions( + client: AsyncClient, + test_project: Project, + v2_project_url: str, + entity_service, + search_service, +): + """The response cache keys on the whole query, so two dates cannot share an entry. + + The digest hashes `SearchQuery.model_dump()`, which now includes the valid-time + fields; without that, the second question would be answered with the first's cached + results. + """ + await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) + + after = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_at="2026-07-28", + ) + before = await _search( + client, + v2_project_url, + text="cache layer", + entity_types=["observation"], + valid_at="2026-07-01", + ) + + assert "Memcached" in (after["results"][0]["content"] or "") + assert "Redis" in (before["results"][0]["content"] or "") diff --git a/tests/cloud/test_cloud_services.py b/tests/cloud/test_cloud_services.py index a4097dfac..2cfd4395f 100644 --- a/tests/cloud/test_cloud_services.py +++ b/tests/cloud/test_cloud_services.py @@ -523,6 +523,7 @@ def __call__(self) -> RecordingSession: observation_repository = object() section_repository = object() + temporal_repository = object() relation_repository = object() class WriteRepositories: @@ -536,6 +537,11 @@ def section_repository(self, project_id: int) -> object: events.append("section_repository") return section_repository + def temporal_repository(self, project_id: int) -> object: + assert project_id == publication.project_id + events.append("temporal_repository") + return temporal_repository + def relation_repository(self, project_id: int) -> object: assert project_id == publication.project_id events.append("relation_repository") @@ -565,11 +571,13 @@ def __init__( *, observation_repository: object, section_repository: object, + temporal_repository: object, relation_repository: object, session_maker: object, ) -> None: assert observation_repository is not None assert section_repository is not None + assert temporal_repository is not None assert relation_repository is not None assert session_maker is not None @@ -615,6 +623,7 @@ async def publish( "relation_repository", "observation_repository", "section_repository", + "temporal_repository", "publish", ] @@ -641,6 +650,10 @@ def section_repository(self, project_id: int) -> object: assert project_id == publication.project_id return object() + def temporal_repository(self, project_id: int) -> object: + assert project_id == publication.project_id + return object() + def relation_repository(self, project_id: int) -> object: assert project_id == publication.project_id return object() @@ -663,11 +676,13 @@ def __init__( *, observation_repository: object, section_repository: object, + temporal_repository: object, relation_repository: object, session_maker: object, ) -> None: assert observation_repository is not None assert section_repository is not None + assert temporal_repository is not None assert relation_repository is not None assert session_maker is not None diff --git a/tests/index/test_local_project_index.py b/tests/index/test_local_project_index.py index 5ea943fdd..5d3acc2c0 100644 --- a/tests/index/test_local_project_index.py +++ b/tests/index/test_local_project_index.py @@ -61,7 +61,11 @@ ) from basic_memory.indexing.relation_persistence import RelationGenerationPublisher from basic_memory.models import Entity, Project, Relation -from basic_memory.repository import EntityRepository, NoteSectionRepository +from basic_memory.repository import ( + EntityRepository, + MemoryTimeIndexRepository, + NoteSectionRepository, +) from basic_memory.repository.note_content_repository import ( AcceptedNoteContentWrite, NoteContentRepository, @@ -1803,6 +1807,7 @@ async def test_local_relation_resolution_refreshes_pending_source_without_markdo observation_repository=observation_repository, relation_repository=relation_repository, section_repository=NoteSectionRepository(project_id=observation_repository.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=observation_repository.project_id), session_maker=session_maker, ).publish( entity_id=source_id, diff --git a/tests/indexing/test_accepted_note_mutation_runner.py b/tests/indexing/test_accepted_note_mutation_runner.py index 7efa8fe52..31e9a9fe9 100644 --- a/tests/indexing/test_accepted_note_mutation_runner.py +++ b/tests/indexing/test_accepted_note_mutation_runner.py @@ -49,6 +49,10 @@ AcceptedSectionWrite, ) from basic_memory.repository.entity_repository import AcceptedPendingEntityWrite +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import SectionGenerationWriteResult from basic_memory.repository.observation_repository import ObservationGenerationWriteResult from basic_memory.repository.relation_repository import RelationGenerationWriteResult @@ -551,6 +555,20 @@ async def replace_sections_for_generation( raise AssertionError("section publication was not expected inside the accepted transaction") +class _TemporalRepository: + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + raise AssertionError( + "temporal publication was not expected inside the accepted transaction" + ) + + class _RelationRepository: def __init__(self) -> None: self.calls: list[tuple[int, Sequence[AcceptedRelationWrite]]] = [] @@ -595,6 +613,7 @@ class _MutationWriteRepositories: search_repository_result: _SearchRepository observation_repository_result: _ObservationRepository section_repository_result: _SectionRepository + temporal_repository_result: _TemporalRepository relation_repository_result: _RelationRepository def pending_entity_repository(self, project_id: int) -> _PendingEntityRepository: @@ -617,6 +636,10 @@ def section_repository(self, project_id: int) -> _SectionRepository: _ = project_id return self.section_repository_result + def temporal_repository(self, project_id: int) -> _TemporalRepository: + _ = project_id + return self.temporal_repository_result + def relation_repository(self, project_id: int) -> _RelationRepository: _ = project_id return self.relation_repository_result @@ -763,6 +786,7 @@ def _dependencies( search_repository_result=search_repository, observation_repository_result=observation_repository or _ObservationRepository(), section_repository_result=_SectionRepository(), + temporal_repository_result=_TemporalRepository(), relation_repository_result=relation_repository or _RelationRepository(), ), move_policy=move_policy diff --git a/tests/indexing/test_accepted_note_write_runner.py b/tests/indexing/test_accepted_note_write_runner.py index afb1b6bfe..b4276e19c 100644 --- a/tests/indexing/test_accepted_note_write_runner.py +++ b/tests/indexing/test_accepted_note_write_runner.py @@ -45,6 +45,10 @@ AcceptedRelationWrite, AcceptedSectionWrite, ) +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import SectionGenerationWriteResult from basic_memory.repository.observation_repository import ObservationGenerationWriteResult from basic_memory.repository.relation_repository import RelationGenerationWriteResult @@ -166,6 +170,22 @@ async def replace_sections_for_generation( return SectionGenerationWriteResult(generation_is_current=True) +class _TemporalRepository: + def __init__(self) -> None: + self.calls: list[tuple[int, Sequence[AcceptedTemporalAssertion]]] = [] + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + self.calls.append((entity_id, assertions)) + return TemporalGenerationWriteResult(generation_is_current=True) + + class _RelationRepository: def __init__(self) -> None: self.calls: list[tuple[int, Sequence[AcceptedRelationWrite]]] = [] @@ -227,6 +247,10 @@ def section_repository(self, project_id: int) -> _SectionRepository: assert project_id == 7 return _SectionRepository() + def temporal_repository(self, project_id: int) -> _TemporalRepository: + assert project_id == 7 + return _TemporalRepository() + def relation_repository(self, project_id: int) -> _RelationRepository: assert project_id == 7 return _RelationRepository() @@ -238,6 +262,7 @@ def relation_repository(self, project_id: int) -> _RelationRepository: assert isinstance(repositories.search_repository(7), _SearchRepository) assert isinstance(repositories.observation_repository(7), _ObservationRepository) assert isinstance(repositories.section_repository(7), _SectionRepository) + assert isinstance(repositories.temporal_repository(7), _TemporalRepository) assert isinstance(repositories.relation_repository(7), _RelationRepository) @@ -434,6 +459,10 @@ def _unexpected_section_repository(_project_id: int) -> _SectionRepository: raise AssertionError("section repository was not expected") +def _unexpected_temporal_repository(_project_id: int) -> _TemporalRepository: + raise AssertionError("temporal repository was not expected") + + @dataclass(frozen=True, slots=True) class _RepositoryProvider: pending_entity_repository_result: _PendingEntityRepository | None = None @@ -441,6 +470,7 @@ class _RepositoryProvider: search_repository_result: _SearchRepository | None = None observation_repository_result: _ObservationRepository | None = None section_repository_result: _SectionRepository | None = None + temporal_repository_result: _TemporalRepository | None = None relation_repository_result: _RelationRepository | None = None def pending_entity_repository(self, project_id: int) -> _PendingEntityRepository: @@ -468,6 +498,11 @@ def section_repository(self, project_id: int) -> _SectionRepository: return _unexpected_section_repository(project_id) return self.section_repository_result + def temporal_repository(self, project_id: int) -> _TemporalRepository: + if self.temporal_repository_result is None: + return _unexpected_temporal_repository(project_id) + return self.temporal_repository_result + def relation_repository(self, project_id: int) -> _RelationRepository: if self.relation_repository_result is None: return _unexpected_relation_repository(project_id) @@ -481,6 +516,7 @@ def _repository_provider( search_repository: _SearchRepository | None = None, observation_repository: _ObservationRepository | None = None, section_repository: _SectionRepository | None = None, + temporal_repository: _TemporalRepository | None = None, relation_repository: _RelationRepository | None = None, ) -> AcceptedNoteWriteRepositories: """Build a fail-fast fake repository provider for one focused test.""" @@ -488,6 +524,7 @@ def _repository_provider( pending_entity_repository_result=pending_entity_repository, observation_repository_result=observation_repository, section_repository_result=section_repository, + temporal_repository_result=temporal_repository, relation_repository_result=relation_repository, note_content_repository_result=note_content_repository, search_repository_result=search_repository, diff --git a/tests/indexing/test_relation_persistence.py b/tests/indexing/test_relation_persistence.py index d6b8dfe28..144305f9a 100644 --- a/tests/indexing/test_relation_persistence.py +++ b/tests/indexing/test_relation_persistence.py @@ -24,6 +24,11 @@ AcceptedNoteContentWrite, NoteContentRepository, ) +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + MemoryTimeIndexRepository, + TemporalGenerationWriteResult, +) from basic_memory.repository.note_section_repository import ( AcceptedSectionWrite, NoteSectionRepository, @@ -107,7 +112,11 @@ async def replace_observations_for_generation( assert session is not None self.events.append("observations") self.calls.append((generation, tuple(observations))) - return ObservationGenerationWriteResult(generation_is_current=self.generation_is_current) + return ObservationGenerationWriteResult( + generation_is_current=self.generation_is_current, + # The real repository returns one freshly minted row id per observation. + observation_ids=tuple(range(1, len(observations) + 1)), + ) @dataclass(slots=True) @@ -132,6 +141,28 @@ async def replace_sections_for_generation( return SectionGenerationWriteResult(generation_is_current=self.generation_is_current) +@dataclass(slots=True) +class RecordingTemporalGenerationStore: + """Record the fenced temporal replacement produced by the publisher.""" + + generation_is_current: bool = True + events: list[str] = field(default_factory=list) + calls: list[tuple[int, tuple[AcceptedTemporalAssertion, ...]]] = field(default_factory=list) + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: + assert session is not None + self.events.append("temporal") + self.calls.append((generation, tuple(assertions))) + return TemporalGenerationWriteResult(generation_is_current=self.generation_is_current) + + @pytest.mark.asyncio async def test_relation_generation_publisher_commits_sorted_chunks_before_cleanup( monkeypatch: pytest.MonkeyPatch, @@ -156,10 +187,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) relations = [ @@ -191,7 +224,15 @@ async def fake_scoped_session( ) assert generation_is_current - assert store.events == ["begin", "observations", "sections", "upsert", "upsert", "cleanup"] + assert store.events == [ + "begin", + "observations", + "temporal", + "sections", + "upsert", + "upsert", + "cleanup", + ] assert observation_store.calls == [ (7, (AcceptedObservationWrite("Observed", "note", None, ["graph"]),)) ] @@ -242,10 +283,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore(generation_is_current=False) observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -256,7 +299,7 @@ async def fake_scoped_session( ) assert not generation_is_current - assert store.events == ["begin", "observations", "sections", "upsert"] + assert store.events == ["begin", "observations", "temporal", "sections", "upsert"] assert [call[0] for call in store.calls] == ["begin", "upsert"] assert transaction_count == 4 @@ -285,10 +328,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore(begin_is_current=False) observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -330,10 +375,12 @@ async def fake_scoped_session( events=store.events, ) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -373,6 +420,7 @@ async def fake_scoped_session( ) store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore( generation_is_current=False, events=store.events, @@ -381,6 +429,7 @@ async def fake_scoped_session( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -390,7 +439,7 @@ async def fake_scoped_session( relations=[IndexedRelation("links_to", "Target", None)], sections=[], ) - assert store.events == ["begin", "observations", "sections"] + assert store.events == ["begin", "observations", "temporal", "sections"] assert [call[0] for call in store.calls] == ["begin"] assert section_store.calls == [(6, ())] assert transaction_count == 3 @@ -417,10 +466,12 @@ async def fake_scoped_session( store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -457,10 +508,12 @@ async def test_relation_generation_publisher_rejects_non_self_pre_resolved_targe store = RecordingRelationGenerationStore() observation_store = RecordingObservationGenerationStore(events=store.events) section_store = RecordingSectionGenerationStore(events=store.events) + temporal_store = RecordingTemporalGenerationStore(events=store.events) publisher = RelationGenerationPublisher( relation_repository=store, observation_repository=observation_store, section_repository=section_store, + temporal_repository=temporal_store, session_maker=cast(async_sessionmaker[AsyncSession], object()), ) @@ -545,6 +598,7 @@ async def cleanup_relation_generations( relation_repository=_FailAfterPublicationBegins(), observation_repository=observation_repository, section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=sample_entity.project_id), session_maker=session_maker, ) with pytest.raises(OSError, match="relation chunk write failed"): @@ -574,6 +628,7 @@ async def cleanup_relation_generations( relation_repository=relation_repository, observation_repository=observation_repository, section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=sample_entity.project_id), session_maker=session_maker, ) assert await retry_publisher.publish( @@ -642,6 +697,7 @@ async def test_generation_zero_relation_forces_generation_publication( relation_repository=relation_repository, observation_repository=observation_repository, section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=MemoryTimeIndexRepository(project_id=sample_entity.project_id), session_maker=session_maker, ) assert await publisher.publish( diff --git a/tests/indexing/test_relation_persistence_temporal.py b/tests/indexing/test_relation_persistence_temporal.py new file mode 100644 index 000000000..15b26c4df --- /dev/null +++ b/tests/indexing/test_relation_persistence_temporal.py @@ -0,0 +1,443 @@ +"""Publishing the valid-time projection under the note_content generation fence. + +Valid time is derived state: the markdown is the claim, these rows are its queryable +shadow, and every (re)index rebuilds them. Two properties keep that safe without adding +locks: + +* A stale writer no-ops. It never deletes the current rows and never inserts its own. +* Observations and their valid time move together. The projection addresses observation + rows by the ids the observation insert mints, so the two writes share one transaction + under one held fence -- the narrow exception the publisher documents. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.indexing.models import IndexedObservation +from basic_memory.indexing.relation_persistence import RelationGenerationPublisher +from basic_memory.models import Entity, NoteContent +from basic_memory.repository.memory_time_index_repository import ( + AcceptedTemporalAssertion, + MemoryTimeIndexRepository, + TemporalGenerationWriteResult, +) +from basic_memory.repository.note_section_repository import NoteSectionRepository +from basic_memory.repository.observation_repository import ( + AcceptedObservationWrite, + ObservationGenerationWriteResult, + ObservationRepository, +) +from basic_memory.repository.relation_repository import RelationRepository +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import ( + TemporalAssertion, + TemporalRangeKind, + TimeRole, + parse_range_literal, +) + +DATE = TemporalRangeKind.DATE + + +def _assertion(literal: str, role: TimeRole = TimeRole.EFFECTIVE) -> TemporalAssertion: + return TemporalAssertion( + time_role=role, + valid_during=parse_range_literal(literal, kind=DATE), + source_text=f"@{role.value}{literal}", + ) + + +def _accepted( + source_id: int, literal: str, role: TimeRole = TimeRole.EFFECTIVE +) -> AcceptedTemporalAssertion: + return AcceptedTemporalAssertion( + source_type=SearchItemType.OBSERVATION.value, + source_id=source_id, + assertion=_assertion(literal, role), + ) + + +async def _add_note_content_generation( + session_maker: async_sessionmaker[AsyncSession], + entity: Entity, + *, + generation: int, +) -> None: + """Give the entity a note_content row at `generation`, which is the fence.""" + async with db.scoped_session(session_maker) as session: + session.add( + NoteContent( + entity_id=entity.id, + project_id=entity.project_id, + external_id=f"content-{entity.external_id}", + file_path=entity.file_path, + markdown_content="# Current\n", + db_version=generation, + db_checksum=f"checksum-{generation}", + file_write_status="synced", + ) + ) + + +# --- Repository-level fence behavior --- + + +@pytest.mark.asyncio +async def test_temporal_projection_replaced_under_the_generation_fence( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """A later replace under the same fence discards every prior assertion.""" + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + await _add_note_content_generation(session_maker, sample_entity, generation=3) + + async with db.scoped_session(session_maker) as session: + await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=3, + assertions=[_accepted(1, "[2026-06-10,2026-07-27)")], + ) + async with db.scoped_session(session_maker) as session: + result = await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=3, + assertions=[ + _accepted(2, "[2026-07-27,)"), + _accepted(3, "[2026-01-01,2026-06-10)", TimeRole.DUE), + ], + ) + + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + rows = await repository.find_by_entity(session, sample_entity.id) + + assert [(row.source_id, row.time_role) for row in rows] == [(2, "effective"), (3, "due")] + assert rows[0].lower_value == "2026-07-27" + assert rows[0].upper_value is None + + +@pytest.mark.asyncio +async def test_stale_generation_leaves_temporal_rows_untouched( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """A stale writer no-ops instead of blocking: the current rows survive intact. + + This is the whole reason the fence exists rather than a lock. The loser of the race + writes nothing and reports it, and the winner's projection stands. + """ + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + await _add_note_content_generation(session_maker, sample_entity, generation=8) + async with db.scoped_session(session_maker) as session: + await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=8, + assertions=[_accepted(1, "[2026-07-27,)")], + ) + + async with db.scoped_session(session_maker) as session: + result = await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=7, + assertions=[_accepted(2, "[2026-01-01,2026-02-01)")], + ) + + assert not result.generation_is_current + async with db.scoped_session(session_maker) as session: + rows = await repository.find_by_entity(session, sample_entity.id) + assert [(row.source_id, row.lower_value) for row in rows] == [(1, "2026-07-27")] + + +@pytest.mark.asyncio +async def test_empty_assertion_set_wipes_prior_rows( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """Removing every qualifier from a note must remove its valid time, not keep it.""" + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + await _add_note_content_generation(session_maker, sample_entity, generation=5) + async with db.scoped_session(session_maker) as session: + await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=5, + assertions=[_accepted(1, "[2026-06-10,2026-07-27)")], + ) + + async with db.scoped_session(session_maker) as session: + result = await repository.replace_assertions_for_generation( + session, + entity_id=sample_entity.id, + generation=5, + assertions=[], + ) + + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + assert await repository.find_by_entity(session, sample_entity.id) == [] + + +@pytest.mark.asyncio +async def test_find_for_sources_returns_nothing_for_an_empty_request( + sample_entity: Entity, + session_maker: async_sessionmaker[AsyncSession], +): + """Hydrating an empty result page must not issue a query at all.""" + repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + + async with db.scoped_session(session_maker) as session: + assert await repository.find_for_sources(session, []) == [] + + +# --- Publisher-level: observations and their valid time move together --- + + +@pytest.mark.asyncio +async def test_publisher_addresses_the_observation_rows_it_just_minted( + sample_entity: Entity, + observation_repository: ObservationRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """Each assertion lands on the id of the observation that carried the qualifier. + + Observation rows are wiped and recreated on every publication, so their ids only + exist after the insert. Pairing by document order is what connects a qualifier back + to its own statement rather than to its neighbour. + """ + temporal_repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=observation_repository, + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=temporal_repository, + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=1) + + published = await publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[], + observations=[ + IndexedObservation( + content="The cache layer will use Redis.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-06-10,2026-07-27)"),), + ), + IndexedObservation( + content="The cache layer will use Memcached.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-07-27,)"),), + ), + IndexedObservation( + content="The queue layer will use RabbitMQ.", + category="decision", + context=None, + tags=None, + ), + ], + ) + + assert published + async with db.scoped_session(session_maker) as session: + observations = await observation_repository.find_by_entity(session, sample_entity.id) + rows = await temporal_repository.find_by_entity(session, sample_entity.id) + + ids_by_content = {observation.content: observation.id for observation in observations} + assert {row.source_id: row.source_text for row in rows} == { + ids_by_content["The cache layer will use Redis."]: "@effective[2026-06-10,2026-07-27)", + ids_by_content["The cache layer will use Memcached."]: "@effective[2026-07-27,)", + } + # The undated observation contributes no row: it makes no claim. + assert ids_by_content["The queue layer will use RabbitMQ."] not in { + row.source_id for row in rows + } + assert all(row.source_type == SearchItemType.OBSERVATION.value for row in rows) + + +@pytest.mark.asyncio +async def test_republishing_rebuilds_the_projection_against_the_new_row_ids( + sample_entity: Entity, + observation_repository: ObservationRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """Re-indexing re-mints observation ids, and the projection follows them. + + This is the invariant that makes the shared transaction necessary: if the temporal + write ran later, it would address ids the observation wipe had already discarded. + """ + temporal_repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=observation_repository, + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=temporal_repository, + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=1) + observation = IndexedObservation( + content="The cache layer will use Redis.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-06-10,2026-07-27)"),), + ) + + assert await publisher.publish( + entity_id=sample_entity.id, generation=1, relations=[], observations=[observation] + ) + assert await publisher.publish( + entity_id=sample_entity.id, generation=1, relations=[], observations=[observation] + ) + + async with db.scoped_session(session_maker) as session: + observations = await observation_repository.find_by_entity(session, sample_entity.id) + rows = await temporal_repository.find_by_entity(session, sample_entity.id) + + assert [row.source_id for row in rows] == [observation.id for observation in observations] + + +@dataclass(slots=True) +class _MisalignedObservationStore: + """An observation store that returns the wrong number of row ids.""" + + calls: list[int] = field(default_factory=list) + + async def replace_observations_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + observations: Sequence[AcceptedObservationWrite], + ) -> ObservationGenerationWriteResult: + del session, entity_id, generation + self.calls.append(len(observations)) + return ObservationGenerationWriteResult(generation_is_current=True, observation_ids=(1,)) + + +@dataclass(slots=True) +class _UnreachableTemporalStore: + """A temporal store that must never be called.""" + + async def replace_assertions_for_generation( + self, + session: AsyncSession, + *, + entity_id: int, + generation: int, + assertions: Sequence[AcceptedTemporalAssertion], + ) -> TemporalGenerationWriteResult: # pragma: no cover - reaching this is the failure + raise AssertionError("misaligned observation ids must be caught before publication") + + +@pytest.mark.asyncio +async def test_misaligned_observation_ids_fail_loudly( + sample_entity: Entity, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """Pairing by position is only safe while the two sequences agree in length. + + A mismatch would silently attach one statement's valid time to another, which is a + wrong answer rather than a stale one -- so it raises instead of publishing. + """ + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=_MisalignedObservationStore(), + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=_UnreachableTemporalStore(), + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=1) + + with pytest.raises(ValueError, match="returned 1 row ids for 2 observations"): + await publisher.publish( + entity_id=sample_entity.id, + generation=1, + relations=[], + observations=[ + IndexedObservation("First", "decision", None, None), + IndexedObservation("Second", "decision", None, None), + ], + ) + + +@pytest.mark.asyncio +async def test_observation_write_returns_the_ids_it_minted( + sample_entity: Entity, + observation_repository: ObservationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """The observation replace reports its new row ids in document order.""" + await _add_note_content_generation(session_maker, sample_entity, generation=2) + + async with db.scoped_session(session_maker) as session: + result = await observation_repository.replace_observations_for_generation( + session, + entity_id=sample_entity.id, + generation=2, + observations=[ + AcceptedObservationWrite("First", "decision", None, None), + AcceptedObservationWrite("Second", "decision", None, None), + ], + ) + + assert result.generation_is_current + async with db.scoped_session(session_maker) as session: + observations = await observation_repository.find_by_entity(session, sample_entity.id) + assert result.observation_ids == tuple(observation.id for observation in observations) + + +@pytest.mark.asyncio +async def test_stale_observation_fence_publishes_no_valid_time( + sample_entity: Entity, + observation_repository: ObservationRepository, + relation_repository: RelationRepository, + session_maker: async_sessionmaker[AsyncSession], +): + """A publication that lost its fence leaves both projections as they were.""" + temporal_repository = MemoryTimeIndexRepository(project_id=sample_entity.project_id) + publisher = RelationGenerationPublisher( + relation_repository=relation_repository, + observation_repository=observation_repository, + section_repository=NoteSectionRepository(project_id=sample_entity.project_id), + temporal_repository=temporal_repository, + session_maker=session_maker, + ) + await _add_note_content_generation(session_maker, sample_entity, generation=9) + + published = await publisher.publish( + entity_id=sample_entity.id, + generation=4, + relations=[], + observations=[ + IndexedObservation( + content="The cache layer will use Redis.", + category="decision", + context=None, + tags=None, + temporal=(_assertion("[2026-06-10,2026-07-27)"),), + ) + ], + ) + + assert not published + async with db.scoped_session(session_maker) as session: + assert await temporal_repository.find_by_entity(session, sample_entity.id) == [] diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index df55066fa..e8d5960ed 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -3,8 +3,10 @@ from datetime import UTC, datetime from pathlib import Path from textwrap import dedent +from typing import Any import pytest +from loguru import logger from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Relation from basic_memory.markdown.entity_parser import parse @@ -428,6 +430,36 @@ async def test_graph_silent_note_still_gets_sections(entity_parser): assert [section.heading for section in entity.sections] == ["Extracted"] +@pytest.mark.asyncio +async def test_malformed_qualifier_logs_diagnostic_with_file_path(entity_parser): + """A refused temporal qualifier warns with the path, so the author can fix the line. + + The typed `temporal_error` field carries the same message to programmatic callers; + this layer is the only one that knows which file the observation came from. + """ + records: list[Any] = [] + sink_id = logger.add(lambda message: records.append(message.record), level="WARNING") + try: + entity = await entity_parser.parse_markdown_content( + Path("decisions/cache-layer.md"), + "# Cache\n- [decision] @asserted[2026-06-10,) The cache layer will use Redis.\n", + ) + finally: + logger.remove(sink_id) + + [observation] = entity.observations + assert observation.temporal == [] + assert "unknown temporal role 'asserted'" in (observation.temporal_error or "") + # The qualifier is never dropped from the content, only from the projection. + assert observation.content.startswith("@asserted[2026-06-10,)") + + messages = [record["message"] for record in records] + assert any( + "decisions/cache-layer.md" in message and "Temporal qualifier ignored" in message + for message in messages + ), messages + + # @pytest.mark.asyncio # async def test_parse_file_invalid_yaml(test_config, entity_parser): # """Test parsing file with invalid YAML frontmatter.""" diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py new file mode 100644 index 000000000..a6b676197 --- /dev/null +++ b/tests/markdown/test_temporal_qualifier.py @@ -0,0 +1,490 @@ +"""Parsing and round-tripping SPEC-82 temporal qualifiers on observations. + +Three rules shape every test here: + +* **One grammar.** `@[role]` for a precise interval, + `@[role:]` for a point. The role is optional in both; a role-less point must + begin with a digit. +* **Silent when it is not time.** If the payload does not read as a date, the token is + ordinary content and nothing is reported. Prose is full of `@`, and diagnosing every + one of them would be noise. +* **One diagnostic.** A payload that *does* read as time but names an unknown role is + reported, because a short list of valid roles makes that actionable. + +And in every case a qualifier that was not accepted is **never dropped**: its text +stays in the observation content, so the line indexes and round-trips exactly as it did +before valid time existed. +""" + +from datetime import datetime, timedelta + +import pytest + +from basic_memory import config as config_module +from basic_memory.config import ConfigManager +from basic_memory.markdown.entity_parser import parse +from basic_memory.markdown.schemas import Observation +from basic_memory.markdown.temporal_qualifier import parse_temporal_qualifier +from basic_memory.temporal import TemporalRangeKind, TimeRole + + +@pytest.fixture(autouse=True) +def isolated_config(config_home, monkeypatch): + """Point config resolution at a temp HOME for the whole module. + + The point form consults `date_order`, so parsing a qualifier reads configuration. + `config_home` patches HOME; resetting the process cache keeps one test's config + from leaking into the next. + """ + monkeypatch.setattr(config_module, "_CONFIG_CACHE", None) + monkeypatch.setattr(config_module, "_CONFIG_MTIME", None) + monkeypatch.setattr(config_module, "_CONFIG_SIZE", None) + return config_home + + +def _observation(line: str) -> Observation: + """Parse a single observation line through the real markdown pipeline.""" + [observation] = parse(line).observations + return observation + + +# --- Acceptance 1: undated notes are untouched --- + + +def test_observation_without_qualifier_parses_byte_identically(): + """A note that asserts no valid time behaves exactly as it did before SPEC-82.""" + observation = _observation("- [decision] The cache layer will use Redis. #infra (agreed)") + + assert observation.category == "decision" + assert observation.content == "The cache layer will use Redis. #infra" + assert observation.tags == ["infra"] + assert observation.context == "agreed" + assert observation.temporal == [] + assert observation.temporal_error is None + assert str(observation) == "- [decision] The cache layer will use Redis. #infra (agreed)" + + +# --- Acceptance 2: round trip preserves role and bounds --- + + +@pytest.mark.parametrize( + "qualifier", + [ + # The range literal: the precise form, unchanged by the point form's arrival. + "@effective[2026-06-10,2026-07-27)", + "@effective(2026-06-10,2026-07-27]", + "@effective[2026-06-10,2026-07-27]", + "@effective(2026-06-10,2026-07-27)", + "@effective[2026-06-10,)", + "@effective(,2026-07-27)", + "@valid[2026-01-01,2026-12-31)", + "@occurred[2026-07-27T18:42:00Z,2026-07-27T19:00:00Z)", + "@due[2026-07-27T18:42:00+02:00,)", + "@mentioned[2026-07-27T18:42:00.123456Z,2026-07-28T00:00:00Z)", + "@[2026-06-10,2026-07-27)", + # The point: the convenient form, with and without a role. + "@effective:2026-07-27", + "@occurred:2026-07-27T18:42:00Z", + "@due:2026-07", + "@2026-07-27", + "@2026-07", + "@2026", + "@10/07/2026", + ], +) +def test_qualifier_round_trips_verbatim(qualifier: str): + """Serializing a parsed observation replays the author's exact qualifier text. + + `valid_during` holds normalized bounds -- UTC, microsecond precision, a canonical + interval -- but the author's own spelling is what gets written back, so a + parse/serialize cycle never rewrites their file. + """ + line = f"- [decision] {qualifier} The cache layer will use Redis." + + observation = _observation(line) + + [assertion] = observation.temporal + assert observation.temporal_error is None + assert observation.content == "The cache layer will use Redis." + assert assertion.source_text == qualifier + assert assertion.extractor == "observation" + assert str(observation) == line + + +def test_qualifier_carries_its_role_and_bounds(): + """The parsed assertion is the interval the author wrote, on the axis they named.""" + observation = _observation( + "- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis." + ) + + [assertion] = observation.temporal + assert assertion.time_role is TimeRole.EFFECTIVE + assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.valid_during.lower == "2026-06-10" + assert assertion.valid_during.upper == "2026-07-27" + assert assertion.valid_during.lower_inclusive is True + assert assertion.valid_during.upper_inclusive is False + assert str(assertion.valid_during) == "[2026-06-10,2026-07-27)" + + +def test_qualifier_is_peeled_before_context_and_tags(): + """Peel order matters: the context rule would otherwise steal a `)` qualifier. + + An exclusive-upper qualifier ends in `)`, and the context rule is a bare + suffix match, so parsing context first would claim the qualifier and leave the + observation content empty -- which the plugin then drops outright. + """ + observation = _observation( + "- [decision] @effective(2026-06-10,2026-07-27] Use Redis #infra (agreed)" + ) + + [assertion] = observation.temporal + assert assertion.source_text == "@effective(2026-06-10,2026-07-27]" + assert observation.content == "Use Redis #infra" + assert observation.context == "agreed" + # Qualifier digits are not tags: the peel happens before the tag scan. + assert observation.tags == ["infra"] + + +def test_qualifier_alone_on_the_line_still_parses(): + """A qualifier with no trailing context is the common case, not an edge case.""" + observation = _observation("- [decision] @effective(2026-06-10,2026-07-27] Use Redis") + + [assertion] = observation.temporal + assert assertion.source_text == "@effective(2026-06-10,2026-07-27]" + assert observation.content == "Use Redis" + assert observation.context is None + + +# --- The point form: what each precision means --- + + +@pytest.mark.parametrize( + ("qualifier", "literal", "kind"), + [ + # A year and a month are periods the author delimited by writing them. + ("@2026", "[2026-01-01,2027-01-01)", TemporalRangeKind.DATE), + ("@2026-06", "[2026-06-01,2026-07-01)", TemporalRangeKind.DATE), + # A date says when something started and leaves it open. + ("@2026-06-10", "[2026-06-10,)", TemporalRangeKind.DATE), + # So does a moment, on the instant axis. + ( + "@2026-06-10T14:00:00", + "[2026-06-10T14:00:00.000000Z,)", + TemporalRangeKind.INSTANT, + ), + ( + "@2026-06-10T14:00:00Z", + "[2026-06-10T14:00:00.000000Z,)", + TemporalRangeKind.INSTANT, + ), + ( + "@2026-06-10T14:00:00+02:00", + "[2026-06-10T12:00:00.000000Z,)", + TemporalRangeKind.INSTANT, + ), + ], +) +def test_point_qualifier_canonicalizes_to_the_span_its_precision_covers( + qualifier: str, literal: str, kind: TemporalRangeKind +): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert str(assertion.valid_during) == literal + assert assertion.valid_during.kind is kind + + +def test_role_less_point_is_filed_on_the_valid_axis(): + """`@2026-06-10` says when the statement holds, without narrowing how.""" + observation = _observation("- [decision] @2026-06-10 The cache layer will use Redis.") + + [assertion] = observation.temporal + assert assertion.time_role is TimeRole.VALID + + +def test_role_less_range_literal_is_filed_on_the_valid_axis(): + """The role is optional in both forms, and defaults the same way in both.""" + observation = _observation("- [decision] @[2026-06-10,2026-07-27) Use Redis.") + + [assertion] = observation.temporal + assert assertion.time_role is TimeRole.VALID + assert str(assertion.valid_during) == "[2026-06-10,2026-07-27)" + + +@pytest.mark.parametrize( + ("qualifier", "role"), + [ + ("@effective:2026-06-10", TimeRole.EFFECTIVE), + ("@occurred:2026-06-10", TimeRole.OCCURRED), + ("@due:2026-06-10", TimeRole.DUE), + ("@mentioned:2026-06-10", TimeRole.MENTIONED), + ("@valid:2026-06-10", TimeRole.VALID), + ], +) +def test_point_qualifier_names_its_axis_with_a_colon(qualifier: str, role: TimeRole): + """`:` separates role from date; a date can start with a letter, so it is needed.""" + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert assertion.time_role is role + assert str(assertion.valid_during) == "[2026-06-10,)" + + +def test_a_roled_point_accepts_a_relative_date(): + """With a role the author has said what they mean, so any readable date is taken. + + Relative wording resolves at parse time and is re-resolved on every index pass. + That is documented behavior, not a mistake to warn about. + """ + observation = _observation("- [decision] @occurred:yesterday The cutover ran.") + + [assertion] = observation.temporal + yesterday = datetime.now().date() - timedelta(days=1) + assert assertion.valid_during.lower == yesterday.isoformat() + assert observation.content == "The cutover ran." + + +@pytest.mark.parametrize( + "qualifier", + [ + # Words: dateparser reads several of these as months or years. + "@yesterday", + "@may", + "@v2", + "@june", + # Too short to be a year: list markers and version numbers, which dateparser + # would otherwise read as January, 2012, and March 5. + "@1", + "@12", + "@3.5", + "@5-3", + ], +) +def test_a_role_less_point_must_be_digit_led_and_year_wide(qualifier: str): + """A bare `@token` that short is a mention, a version, or a list marker. + + Accepting what dateparser makes of these would silently file wrong valid time on + ordinary prose. An author who really means one writes the role: `@occurred:may`. + """ + observation = _observation(f"- [decision] {qualifier} shipped the cutover.") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith(qualifier) + + +def test_a_short_point_is_still_accepted_when_the_role_is_named(): + """The width rule guards the *bare* form only; a role removes the ambiguity.""" + observation = _observation("- [decision] @occurred:may The cutover ran.") + + [assertion] = observation.temporal + assert assertion.time_role is TimeRole.OCCURRED + assert observation.content == "The cutover ran." + + +# --- Date order comes from configuration --- + + +def test_configured_date_order_decides_an_ambiguous_slash_date(monkeypatch): + """`@10/07/2026` is July 10 by default and October 7 under MDY.""" + default = _observation("- [decision] @10/07/2026 The cutover ran.") + [assertion] = default.temporal + assert assertion.valid_during.lower == "2026-07-10" + + monkeypatch.setenv("BASIC_MEMORY_DATE_ORDER", "MDY") + monkeypatch.setattr(config_module, "_CONFIG_CACHE", None) + assert ConfigManager().config.date_order == "MDY" + + reordered = _observation("- [decision] @10/07/2026 The cutover ran.") + [assertion] = reordered.temporal + assert assertion.valid_during.lower == "2026-10-07" + + +def test_configured_date_order_never_reinterprets_an_iso_date(monkeypatch): + """An ISO date is unambiguous, so the preference must not touch it.""" + monkeypatch.setenv("BASIC_MEMORY_DATE_ORDER", "MDY") + monkeypatch.setattr(config_module, "_CONFIG_CACHE", None) + + observation = _observation("- [decision] @2026-07-10 The cutover ran.") + + [assertion] = observation.temporal + assert assertion.valid_during.lower == "2026-07-10" + + +# --- The one diagnostic: an unknown role --- + + +def _refusal(line: str) -> Observation: + """Parse a line whose qualifier must be refused, and assert the shared contract.""" + observation = _observation(line) + assert observation.temporal == [] + assert observation.temporal_error is not None + return observation + + +def test_unknown_role_in_a_range_literal_reports_diagnostic_and_keeps_text(): + """`@asserted` is well-formed but names no axis this system understands.""" + observation = _refusal("- [decision] @asserted[2026-06-10,) The cache layer will use Redis.") + + assert "unknown temporal role 'asserted'" in (observation.temporal_error or "") + # The diagnostic names the roles that would have worked. + assert "effective" in (observation.temporal_error or "") + # Never silently dropped: the text is still searchable content. + assert observation.content.startswith("@asserted[2026-06-10,)") + + +def test_unknown_role_in_a_point_reports_diagnostic_and_keeps_text(): + """The payload reads as a date, so the author is plainly naming an axis.""" + observation = _refusal("- [decision] @asserted:2026-06-10 The cache layer will use Redis.") + + assert "unknown temporal role 'asserted'" in (observation.temporal_error or "") + assert observation.content.startswith("@asserted:2026-06-10") + + +def test_an_unknown_role_with_an_unreadable_payload_is_left_alone(): + """`@todo:fix the thing` is prose, not a broken qualifier. + + The diagnostic is reserved for a payload that actually reads as time; without that, + reporting would fire on ordinary `@word:` markers. + """ + observation = _observation("- [decision] @todo:fix the cache layer") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith("@todo:fix") + + +# --- Everything else is content, silently --- + + +@pytest.mark.parametrize( + ("line", "kept"), + [ + # A known role glued to something that is not a range literal. + ("- [decision] @effective[2026-06-10 Use Redis.", "@effective[2026-06-10"), + # A range mixing the two axes. + ("- [decision] @effective[2026-06-10,2026-07-27T00:00:00Z) Use Redis.", "@effective["), + # A range that ends before it begins. + ("- [decision] @effective[2026-08-01,2026-06-10) Use Redis.", "@effective["), + # A date that the calendar does not have. + ("- [decision] @effective[2026-02-30,) Use Redis.", "@effective[2026-02-30,)"), + ("- [decision] @2026-02-30 Use Redis.", "@2026-02-30"), + # Trailing junk: one broken token, not a qualifier plus content. + ("- [decision] @effective[2026-06-10,2026-07-27)x Use Redis.", "@effective["), + ], +) +def test_a_payload_that_does_not_read_as_time_stays_content(line: str, kept: str): + """No warning about how someone wrote a date -- the token is simply not a qualifier.""" + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith(kept) + + +def test_qualifier_with_nothing_to_qualify_stays_content(): + """Peeling it would leave an empty observation, which the plugin drops outright.""" + observation = _observation("- [decision] @effective[2026-06-10,2026-07-27)") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == "@effective[2026-06-10,2026-07-27)" + + +@pytest.mark.parametrize( + "line", + [ + "- [note] Contact paul@basicmemory.com about the cutover", + "- [note] Ping @paul before the cutover", + "- [note] @basicmemory.com is great", + "- [note] @someone(2026) filed the ticket", + "- [note] Email me at ops@example.com (urgent)", + "- [note] @ops@example.com owns the runbook", + "- [note] @paul reviewed the cutover", + ], +) +def test_non_qualifier_at_tokens_are_ordinary_content(line: str): + """`@` is common prose, and none of it may become a valid-time assertion.""" + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + + +# --- Acceptance 9 and 10: the two axes never convert into one another --- + + +def test_date_only_bounds_never_acquire_time_or_zone(): + """Acceptance 9: a calendar date stays a calendar date, with no false precision.""" + observation = _observation("- [decision] @effective[2026-06-10,2026-07-27) Use Redis.") + + [assertion] = observation.temporal + assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.valid_during.lower == "2026-06-10" + assert assertion.valid_during.upper == "2026-07-27" + assert "T" not in (assertion.valid_during.lower or "") + assert "Z" not in (assertion.valid_during.upper or "") + + +def test_a_date_point_never_becomes_midnight_utc(): + """The point form must not promote a date onto the instant axis either. + + Midnight in *which* zone is a question the author never answered, and answering it + for them would make a date query and an instant query disagree about this note. + """ + observation = _observation("- [decision] @effective:2026-06-10 Use Redis.") + + [assertion] = observation.temporal + assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.valid_during.lower == "2026-06-10" + assert "T00:00" not in str(assertion.valid_during) + + +def test_naive_timestamp_bounds_are_read_as_utc(): + """A timestamp with no offset is UTC, not a refusal. + + Both spellings of the same moment must produce the same stored bound, or a search + would answer differently depending on how the author punctuated it. + """ + naive = _observation("- [decision] @occurred[2026-07-27T18:42:00,) Cutover ran.") + explicit = _observation("- [decision] @occurred[2026-07-27T18:42:00Z,) Cutover ran.") + + [from_naive] = naive.temporal + [from_explicit] = explicit.temporal + assert naive.temporal_error is None + assert from_naive.valid_during == from_explicit.valid_during + assert from_naive.valid_during.lower == "2026-07-27T18:42:00.000000Z" + + +def test_instant_bounds_normalize_to_utc(): + """An offset bound names an instant, and is stored as that instant in UTC.""" + observation = _observation( + "- [decision] @occurred[2026-07-27T18:42:00+02:00,2026-07-28T00:00:00Z) Cutover ran." + ) + + [assertion] = observation.temporal + assert assertion.valid_during.lower == "2026-07-27T16:42:00.000000Z" + # The author's own text is what round-trips, offset and all. + assert assertion.source_text.startswith("@occurred[2026-07-27T18:42:00+02:00") + + +# --- Direct scanner contract --- + + +def test_scanner_returns_content_unchanged_when_nothing_is_attempted(): + """The scanner is a peel, not a rewrite: untouched content is returned as-is.""" + result = parse_temporal_qualifier("Plain observation content") + + assert result.content == "Plain observation content" + assert result.assertions == () + assert result.error is None + + +def test_scanner_takes_an_explicit_date_order(): + """A caller that already holds the config passes it instead of re-reading it.""" + result = parse_temporal_qualifier("@10/07/2026 The cutover ran.", date_order="MDY") + + [assertion] = result.assertions + assert assertion.valid_during.lower == "2026-10-07" + assert result.content == "The cutover ran." diff --git a/tests/mcp/clients/test_search_client_temporal.py b/tests/mcp/clients/test_search_client_temporal.py new file mode 100644 index 000000000..e91b69028 --- /dev/null +++ b/tests/mcp/clients/test_search_client_temporal.py @@ -0,0 +1,97 @@ +"""The valid-time version-skew guard in SearchClient (SPEC-82). + +`SearchQuery` ignores unknown fields, which is normally a harmless forward-compatibility +choice. For a valid-time filter it is not: a server predating SPEC-82 accepts the request +and returns *unfiltered* results, which include exactly the undated sources the filter +was asked to exclude. The caller cannot tell the difference by looking at them. + +The server therefore confirms explicitly that it ran the filter, and the client refuses a +response that does not carry that confirmation. +""" + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from basic_memory.mcp.clients import SearchClient + +# A response body from a server that knows nothing about valid time. +LEGACY_PAYLOAD: dict[str, Any] = { + "results": [], + "current_page": 1, + "page_size": 10, + "total": 0, + "total_is_exact": True, + "has_more": False, +} + + +def _stub_call_query(monkeypatch, payload: dict[str, Any]) -> None: + mock_response = MagicMock() + mock_response.json.return_value = payload + + async def mock_call_query(client, url, **kwargs): + return mock_response + + monkeypatch.setattr("basic_memory.mcp.tools.utils.call_query", mock_call_query) + + +@pytest.mark.parametrize("field", ["valid_at", "valid_overlaps", "time_role"]) +@pytest.mark.asyncio +async def test_unconfirmed_valid_time_filter_is_refused(monkeypatch, field: str): + """Every valid-time field triggers the check; none of them may pass unconfirmed.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD)) + client = SearchClient(MagicMock(), "proj-123") + + with pytest.raises(ValueError, match="did not apply the requested valid-time filter"): + await client.search({"text": "cache", field: "effective"}, page=1, page_size=10) + + +@pytest.mark.asyncio +async def test_explicitly_false_confirmation_is_also_refused(monkeypatch): + """A server that answers "no" is as unusable as one that answers nothing.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD, temporal_applied=False)) + client = SearchClient(MagicMock(), "proj-123") + + with pytest.raises(ValueError, match="did not apply the requested valid-time filter"): + await client.search({"text": "cache", "valid_at": "2026-07-28"}, page=1, page_size=10) + + +@pytest.mark.asyncio +async def test_confirmed_valid_time_filter_is_accepted(monkeypatch): + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD, temporal_applied=True)) + client = SearchClient(MagicMock(), "proj-123") + + response = await client.search( + {"text": "cache", "valid_at": "2026-07-28"}, page=1, page_size=10 + ) + + assert response.temporal_applied is True + + +@pytest.mark.asyncio +async def test_search_without_a_valid_time_filter_is_unaffected(monkeypatch): + """The guard must not touch ordinary searches against any server version.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD)) + client = SearchClient(MagicMock(), "proj-123") + + response = await client.search({"text": "cache"}, page=1, page_size=10) + + assert response.temporal_applied is None + assert response.total == 0 + + +@pytest.mark.asyncio +async def test_empty_valid_time_values_do_not_trigger_the_guard(monkeypatch): + """Fields present but unset are not a request, so nothing needs confirming.""" + _stub_call_query(monkeypatch, dict(LEGACY_PAYLOAD)) + client = SearchClient(MagicMock(), "proj-123") + + response = await client.search( + {"text": "cache", "valid_at": None, "valid_overlaps": None, "time_role": None}, + page=1, + page_size=10, + ) + + assert response.temporal_applied is None diff --git a/tests/mcp/test_tool_contracts.py b/tests/mcp/test_tool_contracts.py index 758d75d1b..7ade678c9 100644 --- a/tests/mcp/test_tool_contracts.py +++ b/tests/mcp/test_tool_contracts.py @@ -136,6 +136,9 @@ "tags", "status", "min_similarity", + "valid_at", + "valid_overlaps", + "time_role", ], "tail": ["timeframe", "lines", "project", "project_id"], "view_note": ["identifier", "project", "project_id"], diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py new file mode 100644 index 000000000..d19e99dc2 --- /dev/null +++ b/tests/mcp/test_tool_search_temporal.py @@ -0,0 +1,332 @@ +"""End-to-end valid-time search through the MCP `search_notes` tool (SPEC-82). + +These tests exercise the whole chain the spec's acceptance cases describe: markdown +carrying temporal qualifiers is written through `write_note`, indexed, projected, and +then queried by authored valid time through `search_notes`. + +The scenario is the spec's own: one note holding two `[decision]` observations that +disagree about the cache layer, each qualified with the window it was effective over. +Both live in a *single* note on purpose -- that is what makes entity-granular filtering +insufficient and forces the projection to address individual observations. +""" + +import inspect +from typing import Any + +import pytest + +from basic_memory.mcp.tools import write_note +from basic_memory.mcp.tools.search import search_notes + +# The spec's worked example, verbatim: one note, two decisions, adjacent half-open +# effective windows meeting at the July 27 cutover. +CACHE_LAYER_NOTE = """\ +# Cache Layer + +## Observations +- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. +- [decision] @effective[2026-07-27,) The cache layer will use Memcached. +""" + +# The same two decisions, written the convenient way. `@effective:2026-07-27` denotes +# `[2026-07-27,)` -- from the cutover onward -- so the cutover answers must not change. +CACHE_LAYER_POINT_NOTE = """\ +# Cache Layer + +## Observations +- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. +- [decision] @effective:2026-07-27 The cache layer will use Memcached. +""" + +UNDATED_NOTE = """\ +# Queue Layer + +## Observations +- [decision] The queue layer will use RabbitMQ. +""" + + +async def _write_cache_layer_note(project_name: str) -> None: + await write_note( + project=project_name, + title="Cache Layer", + directory="decisions", + content=CACHE_LAYER_NOTE, + ) + + +def _contents(response: dict[str, Any]) -> list[str]: + """The matched observation text of every result, for readable assertions.""" + return [result["content"] or "" for result in response["results"]] + + +@pytest.mark.asyncio +async def test_valid_at_after_cutover_returns_memcached_excludes_redis(client, test_project): + """Acceptance 5: `valid_at=2026-07-28` returns Memcached and not Redis. + + July 28 falls inside `[2026-07-27,)` and outside `[2026-06-10,2026-07-27)`, whose + exclusive upper bound expires it exactly at the cutover. + """ + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + time_role="effective", + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Memcached" in content for content in contents), contents + assert not any("Redis" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_valid_at_before_cutover_returns_redis_excludes_memcached(client, test_project): + """Acceptance 6: `valid_at=2026-07-01` returns Redis and not Memcached.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + time_role="effective", + valid_at="2026-07-01", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert not any("Memcached" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_point_qualifier_answers_the_cutover_like_a_range(client, test_project): + """The convenient form reaches the index and the predicate unchanged. + + `@effective:2026-07-27` means "from the cutover onward", so it must answer the + spec's two questions exactly as the explicit `[2026-07-27,)` range does -- and it + must not expire at midnight, which is what a closed single-day range would do. + """ + await write_note( + project=test_project.name, + title="Cache Layer", + directory="decisions", + content=CACHE_LAYER_POINT_NOTE, + ) + + after = await search_notes( + project=test_project.name, + query="cache layer", + time_role="effective", + valid_at="2026-07-28", + output_format="json", + ) + before = await search_notes( + project=test_project.name, + query="cache layer", + time_role="effective", + valid_at="2026-07-01", + output_format="json", + ) + + assert isinstance(after, dict) and isinstance(before, dict) + after_contents = _contents(after) + assert any("Memcached" in content for content in after_contents), after_contents + assert not any("Redis" in content for content in after_contents), after_contents + + before_contents = _contents(before) + assert any("Redis" in content for content in before_contents), before_contents + assert not any("Memcached" in content for content in before_contents), before_contents + + +@pytest.mark.asyncio +async def test_no_temporal_filter_lets_both_decisions_compete(client, test_project): + """Acceptance 7: with no valid-time filter both decisions are candidates again.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + entity_types=["observation"], + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert any("Memcached" in content for content in contents), contents + # Ranking, not filtering, decides between them -- and nothing claims a filter ran. + assert response.get("temporal_applied") is None + + +@pytest.mark.asyncio +async def test_valid_overlaps_returns_both_decisions(client, test_project): + """A window spanning the cutover overlaps both effective ranges.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + time_role="effective", + valid_overlaps="[2026-06-01,2026-08-01)", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert any("Memcached" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_undated_note_search_is_unchanged(client, test_project): + """Acceptance 1: a note with no qualifier searches exactly as it always did.""" + await write_note( + project=test_project.name, + title="Queue Layer", + directory="decisions", + content=UNDATED_NOTE, + ) + + response = await search_notes( + project=test_project.name, + query="RabbitMQ", + output_format="json", + ) + + assert isinstance(response, dict), response + assert response["results"], response + assert response.get("temporal_applied") is None + + +@pytest.mark.asyncio +async def test_valid_at_excludes_undated_observations(client, test_project): + """Acceptance 8: an undated statement cannot answer "what was true then".""" + await _write_cache_layer_note(test_project.name) + await write_note( + project=test_project.name, + title="Queue Layer", + directory="decisions", + content=UNDATED_NOTE, + ) + + unfiltered = await search_notes( + project=test_project.name, + query="layer", + entity_types=["observation"], + output_format="json", + ) + assert isinstance(unfiltered, dict), unfiltered + assert any("RabbitMQ" in content for content in _contents(unfiltered)) + + filtered = await search_notes( + project=test_project.name, + query="layer", + valid_at="2026-07-28", + output_format="json", + ) + assert isinstance(filtered, dict), filtered + assert not any("RabbitMQ" in content for content in _contents(filtered)) + assert filtered["temporal_applied"] is True + + +@pytest.mark.asyncio +async def test_results_carry_the_assertion_that_matched(client, test_project): + """A valid-time hit explains itself: role, canonical range, and authored text.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(response, dict), response + [result] = [r for r in response["results"] if "Memcached" in (r["content"] or "")] + [assertion] = result["temporal"] + assert assertion["role"] == "effective" + assert assertion["source_text"] == "@effective[2026-07-27,)" + assert assertion["valid_during"]["literal"] == "[2026-07-27,)" + assert assertion["valid_during"]["kind"] == "date" + assert assertion["valid_during"]["lower"] == "2026-07-27" + assert assertion["valid_during"]["lower_inclusive"] is True + # JSON output drops null fields, so an unbounded end shows up as an absent key. + assert assertion["valid_during"].get("upper") is None + + +@pytest.mark.asyncio +async def test_markdown_output_labels_the_time_axis(client, test_project): + """Human-readable output names the axis instead of printing a bare date.""" + await _write_cache_layer_note(test_project.name) + + rendered = await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-07-28", + ) + + assert isinstance(rendered, str), rendered + assert "effective valid time: [2026-07-27,) (date)" in rendered + + +@pytest.mark.asyncio +async def test_role_only_filter_finds_every_source_on_that_axis(client, test_project): + """A role with no point or range is a legal question: who asserts on this axis?""" + await _write_cache_layer_note(test_project.name) + await write_note( + project=test_project.name, + title="Queue Layer", + directory="decisions", + content=UNDATED_NOTE, + ) + + response = await search_notes( + project=test_project.name, + query="layer", + time_role="effective", + output_format="json", + ) + + assert isinstance(response, dict), response + contents = _contents(response) + assert any("Redis" in content for content in contents), contents + assert any("Memcached" in content for content in contents), contents + assert not any("RabbitMQ" in content for content in contents), contents + + +@pytest.mark.asyncio +async def test_valid_at_and_valid_overlaps_together_are_refused(client, test_project): + """The two forms ask different questions; supplying both is an authoring error.""" + with pytest.raises(ValueError, match="not both"): + await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-07-28", + valid_overlaps="[2026-06-01,2026-08-01)", + ) + + +@pytest.mark.asyncio +async def test_time_role_alone_is_enough_search_criteria(client, test_project): + """A valid-time filter is real criteria, so it must not trip the empty-query guard.""" + await _write_cache_layer_note(test_project.name) + + response = await search_notes( + project=test_project.name, + time_role="effective", + output_format="json", + ) + + assert isinstance(response, dict), response + assert len(response["results"]) == 2 + + +def test_tool_help_documents_undated_exclusion(): + """Acceptance 8: the exclusion is documented where a caller will read it.""" + doc = inspect.getdoc(search_notes) or "" + assert "Sources with no temporal qualifier are excluded" in doc + assert "valid_at" in doc and "valid_overlaps" in doc and "time_role" in doc diff --git a/tests/mcp/test_tool_telemetry.py b/tests/mcp/test_tool_telemetry.py index 2d8cf83d7..6338c5dce 100644 --- a/tests/mcp/test_tool_telemetry.py +++ b/tests/mcp/test_tool_telemetry.py @@ -174,6 +174,7 @@ async def test_search_notes_emits_root_operation_and_project_context( "has_filters": True, "has_tags_filter": True, "has_status_filter": False, + "has_temporal_filter": False, }, ) span_names = [name for name, _ in spans] diff --git a/tests/mcp/tools/test_search_notes_multi_project_temporal.py b/tests/mcp/tools/test_search_notes_multi_project_temporal.py new file mode 100644 index 000000000..8c69fe3c9 --- /dev/null +++ b/tests/mcp/tools/test_search_notes_multi_project_temporal.py @@ -0,0 +1,162 @@ +"""All-projects search must carry the valid-time filter into every project (SPEC-82). + +`_search_all_projects` re-declares the whole filter surface in its own signature and then +calls `search_notes` once per project. A filter that is not repeated there is dropped for +every project at once, and the merged answer would quietly mix filtered and unfiltered +rows -- the worst shape this failure can take, because the result still looks like an +answer. +""" + +import importlib +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from basic_memory.schemas.search import SearchItemType, SearchResponse, SearchResult + +PROJECT_REFS = [ + {"project": "personal/main", "project_id": "11111111-1111-1111-1111-111111111111"}, + {"project": "team-paul/main", "project_id": "22222222-2222-2222-2222-222222222222"}, +] + + +@pytest.fixture +def cloud_routing(monkeypatch): + """Pin the routing signals so project ids are forwarded deterministically.""" + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + monkeypatch.setattr(search_mod, "is_factory_mode", lambda: False) + monkeypatch.setattr(search_mod, "_explicit_routing", lambda: True) + monkeypatch.setattr(search_mod, "_force_local_mode", lambda: False) + monkeypatch.setattr(search_mod, "has_cloud_credentials", lambda config: True) + + +def _install_stub_client(monkeypatch, payloads: list[dict[str, Any]], refs) -> None: + """Route every per-project search into a stub that records its query payload.""" + clients_mod = importlib.import_module("basic_memory.mcp.clients") + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + class StubProject: + def __init__(self, name: str | None, external_id: str | None): + self.name = name or "main" + self.external_id = external_id or "local-main" + + @asynccontextmanager + async def fake_get_project_client(project=None, context=None, project_id=None): + yield object(), StubProject(project, project_id) + + async def fake_resolve_project_and_path(client, identifier, project=None, context=None): + return StubProject(project, None), identifier, False + + async def fake_load_search_project_refs(context=None): + return refs + + class MockSearchClient: + def __init__(self, client, project_id): + self.project_id = project_id + + async def search(self, payload, page, page_size): + payloads.append(payload) + return SearchResponse( + results=[ + SearchResult( + title="Cache Layer", + permalink="main/decisions/cache-layer", + content="The cache layer will use Memcached.", + type=SearchItemType.OBSERVATION, + score=0.5, + file_path="/main/decisions/cache-layer.md", + ) + ], + current_page=page, + page_size=page_size, + total=1, + temporal_applied=True, + ) + + monkeypatch.setattr(search_mod, "_load_search_project_refs", fake_load_search_project_refs) + monkeypatch.setattr(search_mod, "get_project_client", fake_get_project_client) + monkeypatch.setattr(search_mod, "resolve_project_and_path", fake_resolve_project_and_path) + monkeypatch.setattr(clients_mod, "SearchClient", MockSearchClient) + + +@pytest.mark.asyncio +async def test_all_projects_search_forwards_the_valid_time_filter(monkeypatch, cloud_routing): + """Every project is asked the same valid-time question, not just the first.""" + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + + result = await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + time_role="effective", + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(result, dict) + assert len(payloads) == len(PROJECT_REFS) + for payload in payloads: + assert payload["valid_at"] == "2026-07-28" + assert payload["time_role"] == "effective" + assert payload["valid_overlaps"] is None + # Every leg confirmed it ran the filter, so the merged answer confirms it too. + assert result["temporal_applied"] is True + + +@pytest.mark.asyncio +async def test_all_projects_search_forwards_an_overlap_filter(monkeypatch, cloud_routing): + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + valid_overlaps="[2026-06-01,2026-08-01)", + output_format="json", + ) + + assert [payload["valid_overlaps"] for payload in payloads] == [ + "[2026-06-01,2026-08-01)", + "[2026-06-01,2026-08-01)", + ] + + +@pytest.mark.asyncio +async def test_all_projects_search_without_a_filter_claims_nothing(monkeypatch, cloud_routing): + """An ordinary all-projects search stays exactly the payload it always was.""" + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + result = await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + ) + + assert isinstance(result, dict) + assert "temporal_applied" not in result + + +@pytest.mark.asyncio +async def test_all_projects_search_with_no_projects_still_confirms_the_filter( + monkeypatch, cloud_routing +): + """Zero projects is an empty answer to the valid-time question, not an unfiltered one.""" + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, []) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + result = await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + valid_at="2026-07-28", + output_format="json", + ) + + assert isinstance(result, dict) + assert result["results"] == [] + assert result["temporal_applied"] is True diff --git a/tests/repository/test_hybrid_fusion.py b/tests/repository/test_hybrid_fusion.py index 4a438ae6b..05aa2e7d7 100644 --- a/tests/repository/test_hybrid_fusion.py +++ b/tests/repository/test_hybrid_fusion.py @@ -18,6 +18,7 @@ from basic_memory.repository.search_repository_base import FUSION_BONUS, SearchRepositoryBase from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter @dataclass @@ -82,6 +83,7 @@ async def search( categories: Optional[list[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -162,6 +164,7 @@ def _fake_embedding_provider() -> EmbeddingProvider: categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=10, offset=0, ) diff --git a/tests/repository/test_memory_time_index_contract.py b/tests/repository/test_memory_time_index_contract.py new file mode 100644 index 000000000..3e839c2d6 --- /dev/null +++ b/tests/repository/test_memory_time_index_contract.py @@ -0,0 +1,631 @@ +"""The valid-time search predicate, as one contract over both dialects (SPEC-82). + +Acceptance case 4 requires SQLite and PostgreSQL to answer identically. This module is +that contract, expressed once: every test here uses only dialect-neutral fixtures +(`search_repository`, `session_maker`, `test_project`), and the repo runs the whole +`tests/` tree twice -- plain for SQLite, and under `BASIC_MEMORY_TEST_POSTGRES=1` for +PostgreSQL via testcontainers. A divergence therefore fails this same suite on one of +the two runs rather than hiding in a backend-specific file. + +The stored ranges below cover the dimensions PostgreSQL's range operators distinguish: +each inclusivity combination, each unbounded side, the fully unbounded range, the empty +range, and a separate instant axis that must never mix with the date axis. Timestamps +are written as explicit constants, never as "now", so the answers are the same on every +run and on every machine. +""" + +from dataclasses import dataclass +from datetime import datetime, timezone + +import pytest +import pytest_asyncio +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from basic_memory import db +from basic_memory.models import Entity, MemoryTimeIndex, Observation +from basic_memory.models.project import Project +from basic_memory.repository.memory_time_index_repository import MemoryTimeIndexRepository +from basic_memory.repository.search_repository import SearchIndexRow +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import ( + TemporalFilter, + TemporalPoint, + TemporalRange, + TemporalRangeKind, + TimeRole, + parse_point, + parse_range_literal, +) + +DATE = TemporalRangeKind.DATE +INSTANT = TemporalRangeKind.INSTANT + +# Every observation shares this word so one FTS query returns the whole population and +# the temporal predicate is the only thing that narrows it. That also proves the +# predicate composes with a MATCH/bm25 query rather than only with a bare scan. +SHARED_TERM = "cachelayer" + + +@dataclass(frozen=True, slots=True) +class StoredAssertion: + """One authored assertion, and the label the expectations refer to it by.""" + + label: str + valid_during: TemporalRange + role: TimeRole = TimeRole.EFFECTIVE + + +def _date_range(literal: str) -> TemporalRange: + return parse_range_literal(literal, kind=DATE) + + +def _instant_range(literal: str) -> TemporalRange: + return parse_range_literal(literal, kind=INSTANT) + + +# The population under test. Labels are the vocabulary of every expectation below. +STORED_ASSERTIONS: tuple[StoredAssertion, ...] = ( + StoredAssertion("closed_open", _date_range("[2026-06-10,2026-07-27)")), + StoredAssertion("open_closed", _date_range("(2026-06-10,2026-07-27]")), + StoredAssertion("closed_closed", _date_range("[2026-06-10,2026-07-27]")), + StoredAssertion("open_open", _date_range("(2026-06-10,2026-07-27)")), + StoredAssertion("from_cutover", _date_range("[2026-07-27,)")), + StoredAssertion("before_june", _date_range("(,2026-06-10)")), + StoredAssertion("always", TemporalRange(kind=DATE)), + StoredAssertion("empty", TemporalRange.empty(DATE)), + StoredAssertion( + "instant_window", + _instant_range("[2026-07-27T16:00:00Z,2026-07-27T18:00:00Z)"), + ), + StoredAssertion( + "instant_offset", + # Authored in +02:00; normalization must make it the UTC window [14:00,15:00). + _instant_range("[2026-07-27T16:00:00+02:00,2026-07-27T17:00:00+02:00)"), + ), + StoredAssertion("due_window", _date_range("[2026-06-10,2026-07-27)"), role=TimeRole.DUE), +) + +DATE_LABELS = frozenset( + stored.label + for stored in STORED_ASSERTIONS + if stored.valid_during.kind is DATE and stored.role is TimeRole.EFFECTIVE +) +NON_EMPTY_DATE_LABELS = DATE_LABELS - {"empty"} + + +@pytest_asyncio.fixture +async def temporal_population( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, +) -> dict[int, str]: + """Index one observation per stored assertion and project its valid time. + + Returns the observation id -> label map the assertions read results through, so a + test never has to know which row id the database happened to mint. + """ + indexed_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + labels_by_id: dict[int, str] = {} + + async with db.scoped_session(session_maker) as session: + entity = Entity( + project_id=test_project.id, + title="Cache Layer", + note_type="note", + permalink="decisions/cache-layer", + file_path="decisions/cache-layer.md", + content_type="text/markdown", + created_at=indexed_at, + updated_at=indexed_at, + ) + session.add(entity) + await session.flush() + entity_id = entity.id + + for stored in STORED_ASSERTIONS: + observation = Observation( + project_id=test_project.id, + entity_id=entity_id, + content=f"{SHARED_TERM} decision {stored.label}", + category="decision", + ) + session.add(observation) + await session.flush() + labels_by_id[observation.id] = stored.label + + session.add( + MemoryTimeIndex( + project_id=test_project.id, + entity_id=entity_id, + source_type=SearchItemType.OBSERVATION.value, + source_id=observation.id, + time_role=stored.role.value, + range_kind=stored.valid_during.kind.value, + lower_value=stored.valid_during.lower, + upper_value=stored.valid_during.upper, + lower_inclusive=stored.valid_during.lower_inclusive, + upper_inclusive=stored.valid_during.upper_inclusive, + is_empty=stored.valid_during.is_empty, + extractor="observation", + source_text=str(stored.valid_during), + ) + ) + + for observation_id, label in labels_by_id.items(): + await search_repository.index_item( + SearchIndexRow( + id=observation_id, + type=SearchItemType.OBSERVATION.value, + title=f"decision: {SHARED_TERM} {label}", + content_stems=f"{SHARED_TERM} decision {label}", + content_snippet=f"{SHARED_TERM} decision {label}", + permalink=f"decisions/cache-layer/observations/decision/{label}", + file_path="decisions/cache-layer.md", + category="decision", + entity_id=entity_id, + metadata={"tags": None}, + created_at=indexed_at, + updated_at=indexed_at, + project_id=test_project.id, + ) + ) + return labels_by_id + + +async def _matching_labels( + search_repository, + labels_by_id: dict[int, str], + temporal: TemporalFilter, + *, + search_text: str | None = SHARED_TERM, +) -> set[str]: + """Run one valid-time search and translate the hits back into labels.""" + results = await search_repository.search( + search_text=search_text, + search_item_types=[SearchItemType.OBSERVATION], + temporal=temporal, + limit=50, + ) + return {labels_by_id[result.id] for result in results} + + +# --- Acceptance 4: containment answers identically on both backends --- + + +@pytest.mark.parametrize( + ("at", "expected"), + [ + # Inclusive lower endpoints are owned; exclusive ones are not. + ("2026-06-10", {"closed_open", "closed_closed", "always"}), + # Interior points belong to every interval that spans them. + ("2026-07-01", {"closed_open", "open_closed", "closed_closed", "open_open", "always"}), + # The cutover: exclusive upper ends have already expired, inclusive ones have not, + # and the next period's inclusive lower end has begun. + ("2026-07-27", {"open_closed", "closed_closed", "from_cutover", "always"}), + # Before every bounded lower end: only the unbounded-below ranges remain. + ("2026-06-01", {"before_june", "always"}), + # After every bounded upper end: only the unbounded-above ranges remain. + ("2026-08-01", {"from_cutover", "always"}), + ], + ids=["inclusive-lower", "interior", "cutover", "before-all", "after-all"], +) +@pytest.mark.asyncio +async def test_containment_contract(search_repository, temporal_population, at, expected): + """`valid_at` returns exactly the ranges containing that date, on either backend.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point(at)), + ) + + assert matched == expected + # The empty range contains no point, ever. + assert "empty" not in matched + + +# --- Acceptance 4: overlap answers identically on both backends --- + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + # Adjacent half-open periods do not overlap: this is why `[a,b)` is the right + # shape for a sequence of effective windows. + ("[2026-07-27,2026-08-01)", {"open_closed", "closed_closed", "from_cutover", "always"}), + # A window spanning the whole timeline meets every non-empty range. + ("[2026-06-01,2026-08-01)", NON_EMPTY_DATE_LABELS), + # An exclusive query lower end does not own the shared endpoint either. + ("(2026-07-27,2026-08-01)", {"from_cutover", "always"}), + # Unbounded below: only ranges that start before the exclusive upper end. + ("(,2026-06-10)", {"before_june", "always"}), + # Unbounded above: only ranges that have not already ended. + ("[2026-08-01,)", {"from_cutover", "always"}), + # A single closed point behaves exactly like containment of that point. + ("[2026-06-10,2026-06-10]", {"closed_open", "closed_closed", "always"}), + ], + ids=[ + "adjacent-half-open", + "spanning-window", + "exclusive-lower", + "unbounded-lower", + "unbounded-upper", + "degenerate-point", + ], +) +@pytest.mark.asyncio +async def test_overlap_contract(search_repository, temporal_population, literal, expected): + """`valid_overlaps` returns exactly the ranges sharing a point with the window.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, overlaps=_date_range(literal)), + ) + + assert matched == expected + + +@pytest.mark.asyncio +async def test_overlap_with_fully_unbounded_window_matches_every_non_empty_range( + search_repository, temporal_population +): + """A window with no endpoints separates nothing, so only `empty` is excluded.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, overlaps=TemporalRange(kind=DATE)), + ) + + assert matched == NON_EMPTY_DATE_LABELS + + +@pytest.mark.asyncio +async def test_overlap_with_empty_window_matches_nothing(search_repository, temporal_population): + """PostgreSQL: nothing overlaps the empty range, not even the empty range.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, overlaps=TemporalRange.empty(DATE)), + ) + + assert matched == set() + + +@pytest.mark.asyncio +async def test_stored_empty_range_matches_no_query(search_repository, temporal_population): + """An empty stored range contains no points, so no containment query finds it.""" + for at in ("2026-06-10", "2026-07-01", "2026-08-01"): + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point(at)), + ) + assert "empty" not in matched, at + + +# --- Acceptance 9 and 10: the two axes are never confused --- + + +@pytest.mark.asyncio +async def test_date_query_does_not_match_instant_range(search_repository, temporal_population): + """Acceptance 9: a calendar-date question never reaches an instant range. + + Converting one into the other would have to invent a time of day or a timezone the + author never wrote, so the axes are simply disjoint. + """ + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27")), + ) + + assert "instant_window" not in matched + assert "instant_offset" not in matched + + +@pytest.mark.asyncio +async def test_instant_query_does_not_match_date_range(search_repository, temporal_population): + """The mirror image: an instant question never reaches a calendar-date range.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter( + role=TimeRole.EFFECTIVE, + at=parse_point("2026-07-27T17:00:00Z"), + ), + ) + + assert matched == {"instant_window"} + assert not matched & DATE_LABELS + + +@pytest.mark.asyncio +async def test_instant_ranges_compare_as_instants_across_offsets( + search_repository, temporal_population +): + """Acceptance 10: an offset bound names an instant and is compared as one. + + `instant_offset` was authored as `[16:00+02:00,17:00+02:00)`, which is the UTC + window `[14:00Z,15:00Z)`. A UTC query point inside that window matches it; the same + clock reading interpreted naively would not. + """ + inside = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T14:30:00Z")), + ) + assert inside == {"instant_offset"} + + # 16:00 in +02:00 is 14:00Z, so the naive reading of the same digits is outside it. + outside = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T16:30:00Z")), + ) + assert outside == {"instant_window"} + + +@pytest.mark.asyncio +async def test_instant_endpoints_respect_inclusivity(search_repository, temporal_population): + """Instant bounds obey the same endpoint rules as dates, to the microsecond.""" + at_lower = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T16:00:00Z")), + ) + assert at_lower == {"instant_window"} + + at_upper = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T18:00:00Z")), + ) + assert at_upper == set() + + +# --- Role narrowing --- + + +@pytest.mark.asyncio +async def test_role_filter_narrows_to_one_axis(search_repository, temporal_population): + """Two roles can assert the same interval; a role filter separates them.""" + due = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.DUE, at=parse_point("2026-07-01")), + ) + + assert due == {"due_window"} + + +@pytest.mark.asyncio +async def test_filter_without_role_spans_every_axis(search_repository, temporal_population): + """Omitting the role asks the question of every axis at once.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(at=parse_point("2026-07-01")), + ) + + assert matched == { + "closed_open", + "open_closed", + "closed_closed", + "open_open", + "always", + "due_window", + } + + +@pytest.mark.asyncio +async def test_role_only_filter_selects_every_source_on_that_axis( + search_repository, temporal_population +): + """A role with no window is a legal question, and the empty range still answers it. + + Without a window there is no axis to compare on and no interval to intersect, so + the filter asks only "does this source assert anything on this role" -- which the + empty range does. + """ + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE), + ) + + assert matched == DATE_LABELS | {"instant_window", "instant_offset"} + + +# --- Acceptance 1 and 11: only authored bounds ever participate --- + + +@pytest.mark.asyncio +async def test_note_without_qualifier_writes_no_temporal_rows( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + test_project: Project, + temporal_population, +): + """Acceptance 1: an undated observation is indexed, and projects no valid time.""" + indexed_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + async with db.scoped_session(session_maker) as session: + entity = Entity( + project_id=test_project.id, + title="Queue Layer", + note_type="note", + permalink="decisions/queue-layer", + file_path="decisions/queue-layer.md", + content_type="text/markdown", + created_at=indexed_at, + updated_at=indexed_at, + ) + session.add(entity) + await session.flush() + observation = Observation( + project_id=test_project.id, + entity_id=entity.id, + content=f"{SHARED_TERM} undated decision", + category="decision", + ) + session.add(observation) + await session.flush() + undated_id = observation.id + undated_entity_id = entity.id + + await search_repository.index_item( + SearchIndexRow( + id=undated_id, + type=SearchItemType.OBSERVATION.value, + title="decision: undated", + content_stems=f"{SHARED_TERM} undated decision", + content_snippet=f"{SHARED_TERM} undated decision", + permalink="decisions/queue-layer/observations/decision/undated", + file_path="decisions/queue-layer.md", + category="decision", + entity_id=undated_entity_id, + metadata={"tags": None}, + created_at=indexed_at, + updated_at=indexed_at, + project_id=test_project.id, + ) + ) + + # Unfiltered, the undated observation is an ordinary hit. + unfiltered = await search_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + limit=50, + ) + assert undated_id in {result.id for result in unfiltered} + + # Under any valid-time filter it is absent: it makes no claim to answer with. + filtered = await search_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=TemporalFilter(at=parse_point("2026-07-01")), + limit=50, + ) + assert undated_id not in {result.id for result in filtered} + + +@pytest.mark.asyncio +async def test_projection_rows_carry_only_authored_bounds( + session_maker: async_sessionmaker[AsyncSession], + temporal_population, + test_project: Project, +): + """Acceptance 11: nothing but the authored qualifier reaches the stored bounds. + + Entity `created_at`/`updated_at` are deliberately January 1 while every authored + window is in June/July. If edit bookkeeping ever leaked into the projection, one of + these bounds would carry a January value. + """ + repository = MemoryTimeIndexRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + rows = await repository.find_for_sources( + session, + [(SearchItemType.OBSERVATION.value, source_id) for source_id in temporal_population], + ) + + by_label = {temporal_population[row.source_id]: row for row in rows} + assert by_label["closed_open"].lower_value == "2026-06-10" + assert by_label["closed_open"].upper_value == "2026-07-27" + assert by_label["from_cutover"].upper_value is None + assert by_label["always"].lower_value is None and by_label["always"].upper_value is None + assert by_label["empty"].is_empty is True + for row in rows: + for bound in (row.lower_value, row.upper_value): + assert bound is None or not bound.startswith("2026-01"), row.source_text + + +# --- Pagination parity: filter and count must agree --- + + +@pytest.mark.asyncio +async def test_temporal_filter_count_matches_search(search_repository, temporal_population): + """`count()` runs the same predicate as `search()`, or pagination lies. + + The router gathers the two concurrently and derives `has_more` from the count, so a + count that ignored the filter would report pages that do not exist. + """ + temporal = TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-01")) + results = await search_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=temporal, + limit=50, + ) + total = await search_repository.count( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=temporal, + ) + + assert total == len(results) == 5 + + +@pytest.mark.asyncio +async def test_temporal_filter_applies_without_search_text(search_repository, temporal_population): + """A valid-time filter is criteria on its own; no MATCH is required to use it.""" + matched = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-08-01")), + search_text=None, + ) + + assert matched == {"from_cutover", "always"} + + +@pytest.mark.asyncio +async def test_temporal_filter_is_scoped_to_its_project( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + project_repository, + temporal_population, +): + """Assertions belong to a project; another project's rows can never match here.""" + async with db.scoped_session(session_maker) as session: + other_project = await project_repository.create( + session, + { + "name": "other-project", + "description": "Isolation check", + "path": "/other/project", + "is_active": True, + "is_default": None, + }, + ) + + other_repository = type(search_repository)( + search_repository.session_maker, project_id=other_project.id + ) + results = await other_repository.search( + search_text=SHARED_TERM, + search_item_types=[SearchItemType.OBSERVATION], + temporal=TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-01")), + limit=50, + ) + + assert results == [] + + +@pytest.mark.asyncio +async def test_temporal_point_and_range_agree_on_containment( + search_repository, temporal_population +): + """A point question is the degenerate closed range, so the two cannot disagree.""" + point = TemporalFilter(role=TimeRole.EFFECTIVE, at=TemporalPoint(kind=DATE, value="2026-07-27")) + window = TemporalFilter( + role=TimeRole.EFFECTIVE, + overlaps=TemporalRange( + kind=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=True, + upper_inclusive=True, + ), + ) + + assert await _matching_labels( + search_repository, temporal_population, point + ) == await _matching_labels(search_repository, temporal_population, window) diff --git a/tests/repository/test_rerank_pipeline.py b/tests/repository/test_rerank_pipeline.py index fcef636fa..5a496ea8b 100644 --- a/tests/repository/test_rerank_pipeline.py +++ b/tests/repository/test_rerank_pipeline.py @@ -794,6 +794,7 @@ async def deep_page(offset: int) -> list[SearchIndexRow]: categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=1, offset=offset, ) diff --git a/tests/repository/test_search_trace.py b/tests/repository/test_search_trace.py index 88cd3772a..aa6f95a89 100644 --- a/tests/repository/test_search_trace.py +++ b/tests/repository/test_search_trace.py @@ -1392,6 +1392,7 @@ def test_non_text_criteria_and_null_owner_rows_stay_inspectable(): after_date=None, metadata_filters=None, file_path_prefix=None, + temporal=None, retrieval_mode=SearchRetrievalMode.FTS, min_similarity=None, ) diff --git a/tests/repository/test_semantic_search_base.py b/tests/repository/test_semantic_search_base.py index d6bc9aef0..e8154a4e8 100644 --- a/tests/repository/test_semantic_search_base.py +++ b/tests/repository/test_semantic_search_base.py @@ -33,6 +33,7 @@ ) from basic_memory.repository.semantic_vector_sync import PendingEmbeddingJob from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter # --- Helpers --- @@ -88,6 +89,7 @@ async def search( categories: list[str] | None = None, metadata_filters: dict[str, Any] | None = None, file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: float | None = None, limit: int = 10, diff --git a/tests/repository/test_semantic_vector_sync.py b/tests/repository/test_semantic_vector_sync.py index c14e4cbd1..85c9ec721 100644 --- a/tests/repository/test_semantic_vector_sync.py +++ b/tests/repository/test_semantic_vector_sync.py @@ -16,6 +16,7 @@ from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter class _TestRepository(SearchRepositoryBase): @@ -57,6 +58,7 @@ async def search( categories: list[str] | None = None, metadata_filters: dict[str, Any] | None = None, file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: float | None = None, limit: int = 10, diff --git a/tests/repository/test_vector_pagination.py b/tests/repository/test_vector_pagination.py index 9e16a154a..8609324d3 100644 --- a/tests/repository/test_vector_pagination.py +++ b/tests/repository/test_vector_pagination.py @@ -16,6 +16,7 @@ from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter @dataclass @@ -67,6 +68,7 @@ async def search( categories: list[str] | None = None, metadata_filters: dict[str, Any] | None = None, file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: float | None = None, limit: int = 10, @@ -194,6 +196,7 @@ async def run_page(offset, limit): categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=limit, offset=offset, ) diff --git a/tests/repository/test_vector_temporal_filter.py b/tests/repository/test_vector_temporal_filter.py new file mode 100644 index 000000000..904768c0c --- /dev/null +++ b/tests/repository/test_vector_temporal_filter.py @@ -0,0 +1,139 @@ +"""Valid-time filters reach the semantic retrieval modes too (SPEC-82). + +Vector and hybrid search do not evaluate the temporal predicate themselves: they build a +candidate set from embeddings and then intersect it with an FTS-mode pass that carries +every structured filter. That means a filter is only honored in those modes if it is +both *counted* as a requested filter and *forwarded* to the intersecting search. + +Missing either half fails silently -- semantic search would answer a valid-time question +with unfiltered results, including the undated sources the filter excludes. These tests +pin both halves at the seam rather than trusting the call sites to stay in step. +""" + +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from basic_memory.temporal import TemporalFilter, TimeRole, parse_point +from tests.repository.test_hybrid_fusion import ( + HYBRID_KWARGS, + ConcreteSearchRepo as HybridSearchRepo, + FakeRow as HybridFakeRow, +) +from tests.repository.test_vector_threshold import ( + COMMON_SEARCH_KWARGS, + ConcreteSearchRepo as VectorSearchRepo, + FakeRow, + _fake_embedding_provider, + _make_vector_rows, + fake_scoped_session, +) + +TEMPORAL = TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-28")) + + +def _vector_kwargs(**overrides: Any) -> dict[str, Any]: + return {**COMMON_SEARCH_KWARGS, **overrides} + + +def _hybrid_kwargs(**overrides: Any) -> dict[str, Any]: + return {**HYBRID_KWARGS, **overrides} + + +def _forwarded_temporal(leg: AsyncMock) -> Any: + """The `temporal` argument one retrieval leg was actually called with.""" + assert leg.await_args is not None, "leg was never awaited" + return leg.await_args.kwargs["temporal"] + + +@pytest.mark.asyncio +async def test_temporal_filter_applies_in_vector_mode(): + """A valid-time filter narrows the vector candidate set, and is forwarded verbatim.""" + repo = VectorSearchRepo() + repo._semantic_min_similarity = 0.0 + repo._embedding_provider = _fake_embedding_provider(AsyncMock(return_value=[0.0] * 384)) + + # The embedding neighbourhood offers three entities; only entity 1 asserts a range + # covering the queried date, so the FTS intersection pass returns just that one. + filter_pass = AsyncMock(return_value=[FakeRow(id=1)]) + + with ( + patch( + "basic_memory.repository.search_repository_base.db.scoped_session", fake_scoped_session + ), + patch.object(repo, "_ensure_vector_tables", new_callable=AsyncMock), + patch.object(repo, "_prepare_vector_session", new_callable=AsyncMock), + patch.object( + repo, + "_run_vector_query", + new_callable=AsyncMock, + return_value=_make_vector_rows([0.9, 0.8, 0.7]), + ), + patch.object( + repo, + "_fetch_search_index_rows_by_ids", + new_callable=AsyncMock, + return_value={("entity", i): FakeRow(id=i) for i in range(3)}, + ), + patch.object(repo, "search", filter_pass), + ): + results = await repo._search_vector_only(**_vector_kwargs(temporal=TEMPORAL)) + + assert [row.id for row in results] == [1] + # Counted as a requested filter... + filter_pass.assert_awaited_once() + # ...and forwarded unchanged, so the intersection asks the same question. + assert _forwarded_temporal(filter_pass) is TEMPORAL + + +@pytest.mark.asyncio +async def test_vector_mode_without_a_temporal_filter_runs_no_intersection_pass(): + """An unfiltered semantic search must not pay for a filter pass it does not need.""" + repo = VectorSearchRepo() + repo._semantic_min_similarity = 0.0 + repo._embedding_provider = _fake_embedding_provider(AsyncMock(return_value=[0.0] * 384)) + filter_pass = AsyncMock(return_value=[]) + + with ( + patch( + "basic_memory.repository.search_repository_base.db.scoped_session", fake_scoped_session + ), + patch.object(repo, "_ensure_vector_tables", new_callable=AsyncMock), + patch.object(repo, "_prepare_vector_session", new_callable=AsyncMock), + patch.object( + repo, + "_run_vector_query", + new_callable=AsyncMock, + return_value=_make_vector_rows([0.9]), + ), + patch.object( + repo, + "_fetch_search_index_rows_by_ids", + new_callable=AsyncMock, + return_value={("entity", 0): FakeRow(id=0)}, + ), + patch.object(repo, "search", filter_pass), + ): + results = await repo._search_vector_only(**_vector_kwargs()) + + assert [row.id for row in results] == [0] + filter_pass.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_temporal_filter_applies_in_hybrid_mode(): + """Hybrid fuses two legs; both must ask the same valid-time question.""" + repo = HybridSearchRepo() + fts_leg = AsyncMock(return_value=[HybridFakeRow(id=1, score=5.0, title="dated")]) + vector_leg = AsyncMock(return_value=[HybridFakeRow(id=1, score=0.9, title="dated")]) + + with ( + patch.object(repo, "search", fts_leg), + patch.object(repo, "_search_vector_only", vector_leg), + ): + results = await repo._search_hybrid(**_hybrid_kwargs(temporal=TEMPORAL)) + + assert [row.id for row in results] == [1] + assert _forwarded_temporal(fts_leg) is TEMPORAL + assert _forwarded_temporal(vector_leg) is TEMPORAL diff --git a/tests/repository/test_vector_threshold.py b/tests/repository/test_vector_threshold.py index 96b39549b..711f09c22 100644 --- a/tests/repository/test_vector_threshold.py +++ b/tests/repository/test_vector_threshold.py @@ -17,6 +17,7 @@ ) from basic_memory.repository.search_trace import SearchTraceCollector from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode +from basic_memory.temporal import TemporalFilter @dataclass @@ -71,6 +72,7 @@ async def search( categories: Optional[list[str]] = None, metadata_filters: Optional[dict[str, Any]] = None, file_path_prefix: Optional[str] = None, + temporal: Optional[TemporalFilter] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, limit: int = 10, @@ -166,6 +168,7 @@ async def fake_scoped_session(session_maker): categories=None, metadata_filters=None, file_path_prefix=None, + temporal=None, limit=10, offset=0, ) diff --git a/tests/schemas/test_document_agent_temporal.py b/tests/schemas/test_document_agent_temporal.py new file mode 100644 index 000000000..5460b5382 --- /dev/null +++ b/tests/schemas/test_document_agent_temporal.py @@ -0,0 +1,64 @@ +"""How the agent observation contract meets temporal qualifiers (SPEC-82). + +`DocumentAgentObservationV1` re-parses its own formatted markdown and requires the +parsed fields to match what it was given. Now that the parser peels a valid-time +qualifier off content, an untrusted agent that puts one inside `content` no longer +round-trips -- and is rejected. + +That is the intended MVP behavior, not an oversight: the agent contract has no temporal +field, so the alternative would be an agent silently minting valid-time assertions +through a text channel. Rejection is loud, and this test pins it so that adding +`temporal` to the agent contract later is a deliberate decision rather than an accident. +""" + +import pytest +from pydantic import ValidationError + +from basic_memory.schemas.document import DocumentAgentObservationV1 + + +def test_agent_observation_content_with_qualifier_is_rejected(): + """An agent cannot smuggle authored valid time through the content field.""" + with pytest.raises(ValidationError, match="must match parsed Markdown semantics"): + DocumentAgentObservationV1( + category="summary", + content="@effective[2026-06-10,2026-07-27) The cache layer will use Redis.", + ) + + +def test_agent_observation_with_a_malformed_qualifier_is_accepted_as_plain_text(): + """A refused qualifier is never peeled, so the line still round-trips exactly. + + This is the other half of "never silently dropped": text that only looks like a + qualifier stays content, and the agent contract keeps accepting it. + """ + observation = DocumentAgentObservationV1( + category="summary", + content="@asserted[2026-06-10,) The cache layer will use Redis.", + ) + + assert observation.content.startswith("@asserted[2026-06-10,)") + + +def test_ordinary_agent_observations_are_unaffected(): + """Acceptance 1 at the agent boundary: undated content behaves as it always did.""" + observation = DocumentAgentObservationV1( + category="summary", + content="The cache layer will use Redis.", + tags=("infra",), + context="agreed", + ) + + assert observation.content == "The cache layer will use Redis." + assert observation.tags == ("infra",) + assert observation.context == "agreed" + + +def test_email_addresses_in_agent_content_are_not_qualifiers(): + """`@` is common prose, and the contract must not start rejecting it.""" + observation = DocumentAgentObservationV1( + category="summary", + content="Contact paul@basicmemory.com about the cutover.", + ) + + assert "paul@basicmemory.com" in observation.content diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py new file mode 100644 index 000000000..6b1e49dcc --- /dev/null +++ b/tests/services/test_search_service_temporal.py @@ -0,0 +1,230 @@ +"""Valid-time filtering through the search service (SPEC-82). + +The service is where the flat boundary strings become domain values, so this is where a +malformed filter must be refused loudly rather than degraded into a filter that quietly +matches something else. It is also the layer that proves acceptance case 11: a note's +edit bookkeeping is never reinterpreted as the time it claims to be true. +""" + +from datetime import datetime, timezone +from textwrap import dedent + +import pytest + +from basic_memory.schemas import Entity as EntitySchema +from basic_memory.schemas.search import SearchQuery +from basic_memory.services.search_service import ( + build_temporal_filter, + describe_search_criteria, +) +from basic_memory.temporal import TemporalQualifierError, TimeRole + +# The entity is created "now"; the qualifier claims June-July 2026. Keeping the two +# ranges disjoint is what makes acceptance case 11 testable at all. +EFFECTIVE_WINDOW_START = "2026-06-10" +EFFECTIVE_WINDOW_INSIDE = "2026-07-01" +EFFECTIVE_WINDOW_END = "2026-07-27" + +CACHE_LAYER_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + """) + + +async def _index_cache_layer_note(entity_service, search_service): + """Create the dated note through the real write path and index it for search.""" + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Cache Layer", + note_type="note", + directory="decisions", + content=CACHE_LAYER_MARKDOWN, + ) + ) + await search_service.index_entity(entity) + return entity + + +# --- Acceptance 11: entity time is never valid time --- + + +@pytest.mark.asyncio +async def test_entity_timestamps_are_never_used_as_observation_valid_time( + entity_service, search_service +): + """The note was written today; it claims to hold in June and July. + + Asking `valid_at` on the day the file was written must return nothing, because no + observation asserts that day. Asking inside the authored window returns the + observation. If edit bookkeeping ever leaked into the valid-time axis, the first + query would match and the distinction the spec draws would be gone. + """ + entity = await _index_cache_layer_note(entity_service, search_service) + written_on = entity.updated_at.date().isoformat() + assert written_on > EFFECTIVE_WINDOW_END, "fixture assumes the note is written after the window" + + at_write_time = await search_service.search( + SearchQuery(text="cache layer", valid_at=written_on) + ) + assert at_write_time == [] + + inside_window = await search_service.search( + SearchQuery(text="cache layer", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + assert [result.type for result in inside_window] == ["observation"] + assert "Redis" in (inside_window[0].content_snippet or "") + + +@pytest.mark.asyncio +async def test_after_date_still_filters_indexed_time_not_valid_time(entity_service, search_service): + """`after_date` keeps its meaning: it is the note's bookkeeping, not its claim. + + The note was indexed today and asserts a window that ended in July, so a filter on + each axis answers differently -- which is only possible because they stay separate. + """ + await _index_cache_layer_note(entity_service, search_service) + long_ago = datetime(2020, 1, 1, tzinfo=timezone.utc) + + recently_indexed = await search_service.search( + SearchQuery(text="cache layer", after_date=long_ago) + ) + assert recently_indexed + + still_effective_today = await search_service.search( + SearchQuery(text="cache layer", valid_at="2026-12-31") + ) + assert still_effective_today == [] + + +@pytest.mark.asyncio +async def test_valid_time_filter_narrows_to_the_asserting_observation( + entity_service, search_service +): + """A valid-time hit is the observation that carried the claim, not the whole note.""" + await _index_cache_layer_note(entity_service, search_service) + + results = await search_service.search( + SearchQuery(text="cache layer", time_role="effective", valid_at=EFFECTIVE_WINDOW_START) + ) + + assert [result.type for result in results] == ["observation"] + + +@pytest.mark.asyncio +async def test_undated_note_is_excluded_by_a_valid_time_filter(entity_service, search_service): + """Acceptance 8, at the service layer: no claim means no answer.""" + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Queue Layer", + note_type="note", + directory="decisions", + content="# Queue Layer\n\n## Observations\n- [decision] The queue layer uses RabbitMQ.\n", + ) + ) + await search_service.index_entity(entity) + + unfiltered = await search_service.search(SearchQuery(text="RabbitMQ")) + assert unfiltered + + filtered = await search_service.search( + SearchQuery(text="RabbitMQ", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + assert filtered == [] + + +# --- Diagnostics: the boundary refuses every malformed filter --- + + +def test_unknown_time_role_is_refused_with_the_known_roles(): + with pytest.raises(TemporalQualifierError, match="unknown time_role 'asserted'") as exc_info: + build_temporal_filter(SearchQuery(text="cache", time_role="asserted")) + + assert "effective" in str(exc_info.value) + + +def test_malformed_range_literal_is_refused(): + with pytest.raises(TemporalQualifierError, match="range literal must be"): + build_temporal_filter(SearchQuery(text="cache", valid_overlaps="2026-06-10..2026-07-27")) + + +def test_mixed_bound_kinds_are_refused(): + with pytest.raises(TemporalQualifierError, match="mix date-only and timestamp bounds"): + build_temporal_filter( + SearchQuery(text="cache", valid_overlaps="[2026-06-10,2026-07-27T00:00:00Z)") + ) + + +def test_timestamp_without_offset_is_read_as_utc(): + """A naive timestamp is not a rejection: it names the same instant as its `Z` form.""" + naive = build_temporal_filter(SearchQuery(text="cache", valid_at="2026-07-27T18:42:00")) + explicit = build_temporal_filter(SearchQuery(text="cache", valid_at="2026-07-27T18:42:00Z")) + + assert naive == explicit + assert naive is not None and naive.at is not None + assert naive.at.value == "2026-07-27T18:42:00.000000Z" + + +def test_impossible_range_is_refused(): + with pytest.raises(TemporalQualifierError, match="after upper bound"): + build_temporal_filter(SearchQuery(text="cache", valid_overlaps="[2026-08-01,2026-06-10)")) + + +def test_query_without_valid_time_fields_builds_no_filter(): + assert build_temporal_filter(SearchQuery(text="cache")) is None + + +def test_role_only_query_builds_a_role_filter(): + temporal = build_temporal_filter(SearchQuery(text="cache", time_role="effective")) + + assert temporal is not None + assert temporal.role is TimeRole.EFFECTIVE + assert temporal.at is None and temporal.overlaps is None + + +def test_valid_at_and_valid_overlaps_are_mutually_exclusive_at_the_schema(): + """The schema refuses the pair before any parsing or SQL can happen.""" + with pytest.raises(ValueError, match="not both"): + SearchQuery(text="cache", valid_at="2026-07-28", valid_overlaps="[2026-06-10,)") + + +# --- Query gating and traces --- + + +def test_a_valid_time_filter_alone_is_enough_criteria(): + """A temporal filter is real criteria; the empty-query guard must not swallow it.""" + assert SearchQuery(valid_at="2026-07-28").no_criteria() is False + assert SearchQuery(time_role="effective").no_criteria() is False + assert SearchQuery(valid_overlaps="[2026-06-10,)").no_criteria() is False + assert SearchQuery().no_criteria() is True + + +@pytest.mark.asyncio +async def test_prepared_query_carries_the_parsed_filter(search_service): + prepared = search_service.prepare_query( + SearchQuery(text="cache", time_role="effective", valid_at="2026-07-28") + ) + + assert prepared is not None + assert prepared.temporal is not None + assert prepared.temporal.role is TimeRole.EFFECTIVE + assert prepared.temporal.at is not None + assert prepared.temporal.at.value == "2026-07-28" + + +@pytest.mark.asyncio +async def test_search_trace_describes_the_valid_time_question(search_service): + """A trace must show the question that actually ran, valid time included.""" + containment = search_service.prepare_query( + SearchQuery(text="cache", time_role="effective", valid_at="2026-07-28") + ) + overlap = search_service.prepare_query( + SearchQuery(text="cache", valid_overlaps="[2026-06-10,2026-07-27)") + ) + plain = search_service.prepare_query(SearchQuery(text="cache")) + + assert containment is not None and overlap is not None and plain is not None + assert "temporal=role=effective,valid_at=2026-07-28" in describe_search_criteria(containment) + assert "temporal=valid_overlaps=[2026-06-10,2026-07-27)" in describe_search_criteria(overlap) + assert "temporal=" not in describe_search_criteria(plain) diff --git a/tests/test_config.py b/tests/test_config.py index c2f1588ca..c1e761287 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,8 +5,9 @@ import tempfile import pytest from datetime import datetime -from typing import Any, cast +from typing import Any, cast, get_args +from basic_memory.cli.commands.config import CONFIGURABLE_FIELDS from basic_memory.config import ( BasicMemoryConfig, ConfigManager, @@ -15,6 +16,7 @@ default_fastembed_cache_dir, resolve_data_dir, ) +from basic_memory.temporal import DEFAULT_DATE_ORDER, DateOrder from pathlib import Path @@ -1261,6 +1263,36 @@ def test_default_search_type_rejects_invalid_values(self): BasicMemoryConfig(default_search_type="invalid") +class TestDateOrderConfig: + """The preference used to read an ambiguous authored date (SPEC-82).""" + + def test_date_order_defaults_to_iso(self): + assert BasicMemoryConfig().date_order == "YMD" + + def test_date_order_accepts_the_three_component_orders(self): + for date_order in ("YMD", "DMY", "MDY"): + assert BasicMemoryConfig(date_order=date_order).date_order == date_order + + def test_date_order_rejects_anything_else(self): + with pytest.raises(Exception): + BasicMemoryConfig(date_order="ISO") + + def test_date_order_matches_the_domain_alias(self): + """The field is spelled as a bare Literal so `bm config set` can discover it. + + `temporal.DateOrder` is the same union used in function signatures; this pins + the two spellings together so neither can drift. + """ + assert set(get_args(BasicMemoryConfig.model_fields["date_order"].annotation)) == set( + get_args(DateOrder.__value__) + ) + assert BasicMemoryConfig().date_order == DEFAULT_DATE_ORDER + + def test_date_order_is_settable_from_the_cli(self): + """A user-facing preference is worth nothing if `bm config set` cannot reach it.""" + assert "date_order" in CONFIGURABLE_FIELDS + + class TestFormattingConfig: """Test file formatting configuration options.""" diff --git a/tests/test_memory_time_index_migration.py b/tests/test_memory_time_index_migration.py new file mode 100644 index 000000000..004ed4698 --- /dev/null +++ b/tests/test_memory_time_index_migration.py @@ -0,0 +1,283 @@ +"""Migration coverage for the memory_time_index table (SPEC-82). + +The migration is deliberately dialect-neutral: every type it uses renders on SQLite and +PostgreSQL alike, so there is no branch to test per backend. What must be proven is +that the *same* definition arrives intact on both -- the columns, the cascade, the +lookup index, and the three CHECK constraints that keep an impossible range out of the +projection in the first place. + +Two halves, following the repo's established split: a real SQLite upgrade/downgrade +round trip, and an offline render of the same migration against the PostgreSQL dialect. +""" + +import io +import sqlite3 +from importlib import import_module +from typing import Any + +import pytest +from alembic import command +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from tests.test_note_content_migration import sqlite_alembic_config + +migration = import_module("basic_memory.alembic.versions.u4t5e6m7p8o9_add_memory_time_index_table") + +# Pin the downgrade target to this migration's own parent. A relative "-1" would instead +# undo whichever migration currently sits at head, so the test would break every time a +# later revision lands. +DOWN_REVISION: str = str(migration.down_revision) + +EXPECTED_COLUMNS = { + "id", + "project_id", + "entity_id", + "source_type", + "source_id", + "time_role", + "range_kind", + "lower_value", + "upper_value", + "lower_inclusive", + "upper_inclusive", + "is_empty", + "extractor", + "source_text", + "assertion_metadata", +} + +# One row per column, in the table's declared order, for the constraint probes below. +VALID_ROW = ( + 1, # project_id + 1, # entity_id + "observation", + 1, # source_id + "effective", + "date", + "2026-06-10", + "2026-07-27", + 1, # lower_inclusive + 0, # upper_inclusive + 0, # is_empty + "observation", + "@effective[2026-06-10,2026-07-27)", +) +INSERT_SQL = """ + INSERT INTO memory_time_index ( + project_id, entity_id, source_type, source_id, time_role, range_kind, + lower_value, upper_value, lower_inclusive, upper_inclusive, is_empty, + extractor, source_text + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" + + +def _upgraded_database(tmp_path, monkeypatch, name: str): + """Run Alembic to head against a fresh temporary SQLite database.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) + database_path = tmp_path / name + config = sqlite_alembic_config(database_path) + command.upgrade(config, "head") + return database_path, config + + +def _row_with(**overrides: Any) -> tuple[Any, ...]: + """One valid row with named columns replaced, for the constraint probes.""" + columns = [ + "project_id", + "entity_id", + "source_type", + "source_id", + "time_role", + "range_kind", + "lower_value", + "upper_value", + "lower_inclusive", + "upper_inclusive", + "is_empty", + "extractor", + "source_text", + ] + values = dict(zip(columns, VALID_ROW)) + values.update(overrides) + return tuple(values[column] for column in columns) + + +def _seed_parent_rows(connection: sqlite3.Connection) -> None: + """Insert the project and entity the projection rows below hang off.""" + connection.execute( + "INSERT INTO project (id, external_id, name, permalink, path, is_active," + " created_at, updated_at)" + " VALUES (1, 'project-1', 'p', 'p', '/p', 1, '2026-01-01', '2026-01-01')" + ) + connection.execute( + "INSERT INTO entity (id, external_id, project_id, title, note_type, permalink," + " file_path, content_type, created_at, updated_at)" + " VALUES (1, 'entity-1', 1, 't', 'note', 'p/t', 't.md', 'text/markdown'," + " '2026-01-01', '2026-01-01')" + ) + + +def test_alembic_upgrade_creates_memory_time_index_table(tmp_path, monkeypatch): + """Upgrading to head creates the projection table with its full contract.""" + database_path, _ = _upgraded_database(tmp_path, monkeypatch, "memory-time-index.db") + + connection = sqlite3.connect(database_path) + try: + columns = { + row[1] for row in connection.execute("PRAGMA table_info(memory_time_index)").fetchall() + } + assert columns == EXPECTED_COLUMNS + + foreign_keys = connection.execute("PRAGMA foreign_key_list(memory_time_index)").fetchall() + entity_fk = next(row for row in foreign_keys if row[3] == "entity_id") + project_fk = next(row for row in foreign_keys if row[3] == "project_id") + assert (entity_fk[2], entity_fk[4]) == ("entity", "id") + # Valid time is removed with the entity it was asserted about. + assert entity_fk[6].upper() == "CASCADE" + assert (project_fk[2], project_fk[4]) == ("project", "id") + # source_id addresses whichever table source_type names, so it carries no FK. + assert {row[3] for row in foreign_keys} == {"entity_id", "project_id"} + + indexes = { + row[1] for row in connection.execute("PRAGMA index_list(memory_time_index)").fetchall() + } + assert "ix_memory_time_index_lookup" in indexes + assert "ix_memory_time_index_entity_id" in indexes + + lookup_columns = [ + row[2] + for row in connection.execute( + "PRAGMA index_info(ix_memory_time_index_lookup)" + ).fetchall() + ] + # The predicate filters on project/role/axis and projects (source_type, source_id), + # so this one index both drives the scan and covers its output. + assert lookup_columns == [ + "project_id", + "time_role", + "range_kind", + "source_type", + "source_id", + ] + + # Bound values are deliberately unindexed: the full-text candidate set drives. + assert not any(index.startswith("ix_memory_time_index_lower") for index in indexes) + assert not any(index.startswith("ix_memory_time_index_upper") for index in indexes) + finally: + connection.close() + + +def test_upgraded_table_accepts_a_well_formed_assertion(tmp_path, monkeypatch): + """The CHECK constraints must not reject the rows the projection actually writes.""" + database_path, _ = _upgraded_database(tmp_path, monkeypatch, "memory-time-index-insert.db") + + connection = sqlite3.connect(database_path) + try: + _seed_parent_rows(connection) + connection.execute(INSERT_SQL, VALID_ROW) + # Unbounded and empty ranges are legal shapes, not edge cases. + connection.execute( + INSERT_SQL, + _row_with(source_id=2, lower_value=None, lower_inclusive=0), + ) + connection.execute( + INSERT_SQL, + _row_with( + source_id=3, + lower_value=None, + upper_value=None, + lower_inclusive=0, + upper_inclusive=0, + is_empty=1, + ), + ) + connection.commit() + + assert connection.execute("SELECT COUNT(*) FROM memory_time_index").fetchone()[0] == 3 + finally: + connection.close() + + +@pytest.mark.parametrize( + ("overrides", "constraint"), + [ + ({"range_kind": "week"}, "ck_memory_time_index_range_kind"), + # An empty range with endpoints would describe the same interval two ways. + ({"is_empty": 1}, "ck_memory_time_index_empty_has_no_bounds"), + # PostgreSQL's rule: there is no endpoint to include on an unbounded side. + ( + {"lower_value": None, "lower_inclusive": 1}, + "ck_memory_time_index_unbounded_is_exclusive", + ), + ], + ids=["unknown-axis", "empty-with-bounds", "unbounded-but-inclusive"], +) +def test_check_constraints_reject_impossible_rows(tmp_path, monkeypatch, overrides, constraint): + """An interval the domain cannot produce must not be storable either.""" + database_path, _ = _upgraded_database( + tmp_path, monkeypatch, f"memory-time-index-{constraint}.db" + ) + + connection = sqlite3.connect(database_path) + try: + _seed_parent_rows(connection) + with pytest.raises(sqlite3.IntegrityError, match=constraint): + connection.execute(INSERT_SQL, _row_with(**overrides)) + finally: + connection.close() + + +def test_alembic_downgrade_drops_memory_time_index_table(tmp_path, monkeypatch): + """Downgrading past this revision removes the table and both of its indexes.""" + database_path, config = _upgraded_database(tmp_path, monkeypatch, "memory-time-index-down.db") + command.downgrade(config, DOWN_REVISION) + + connection = sqlite3.connect(database_path) + try: + table_exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_time_index'" + ).fetchone() + remaining_indexes = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'index'" + " AND name LIKE 'ix_memory_time_index%'" + ).fetchall() + finally: + connection.close() + + assert table_exists is None + assert remaining_indexes == [] + + +def test_postgres_render_carries_the_same_definition(monkeypatch): + """The identical migration renders on PostgreSQL with no dialect branching. + + Rendering offline is what proves it: if any type, default, or constraint needed a + backend-specific spelling, this would fail here rather than on a deploy. + """ + buffer = io.StringIO() + context = MigrationContext.configure( + dialect_name="postgresql", + opts={"as_sql": True, "output_buffer": buffer}, + ) + monkeypatch.setattr(migration, "op", Operations(context)) + + migration.upgrade() + migration.downgrade() + + sql = buffer.getvalue() + assert "CREATE TABLE memory_time_index" in sql + assert "FOREIGN KEY(entity_id) REFERENCES entity (id) ON DELETE CASCADE" in sql + assert "ck_memory_time_index_range_kind" in sql + assert "ck_memory_time_index_empty_has_no_bounds" in sql + assert "ck_memory_time_index_unbounded_is_exclusive" in sql + assert ( + "CREATE INDEX ix_memory_time_index_lookup ON memory_time_index " + "(project_id, time_role, range_kind, source_type, source_id)" in sql + ) + # Bounds stay portable text on both backends; a native range column would be a + # later, generated addition rather than a change to this definition. + assert "lower_value VARCHAR(32)" in sql + assert "upper_value VARCHAR(32)" in sql + assert "DROP TABLE memory_time_index" in sql diff --git a/tests/test_note_section_migration.py b/tests/test_note_section_migration.py index bc1ba0d3e..7370575ad 100644 --- a/tests/test_note_section_migration.py +++ b/tests/test_note_section_migration.py @@ -4,8 +4,16 @@ from alembic import command +from basic_memory.alembic.versions import ( # type: ignore[attr-defined] + t3n4o5t6e7s8_add_note_section_table as migration, +) from tests.test_note_content_migration import sqlite_alembic_config +# Pin the downgrade target to this migration's own parent. A relative "-1" would +# instead undo whichever migration currently sits at head, so the test would break +# every time a later revision lands. +DOWN_REVISION: str = str(migration.down_revision) + def test_alembic_upgrade_creates_note_section_table(tmp_path, monkeypatch): """Running Alembic head creates note_section with its expected contract.""" @@ -70,14 +78,14 @@ def test_alembic_upgrade_creates_note_section_table(tmp_path, monkeypatch): def test_alembic_downgrade_drops_note_section_table(tmp_path, monkeypatch): - """Downgrading one revision removes the table and its indexes.""" + """Downgrading past this revision removes the table and its indexes.""" monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("BASIC_MEMORY_HOME", str(tmp_path / "basic-memory")) database_path = tmp_path / "note-section-downgrade.db" config = sqlite_alembic_config(database_path) command.upgrade(config, "head") - command.downgrade(config, "-1") + command.downgrade(config, DOWN_REVISION) connection = sqlite3.connect(database_path) try: diff --git a/tests/test_temporal.py b/tests/test_temporal.py new file mode 100644 index 000000000..66860fea7 --- /dev/null +++ b/tests/test_temporal.py @@ -0,0 +1,514 @@ +"""The portable temporal value types and their lexical grammar (SPEC-82). + +These values are the shared vocabulary between the markdown parser, the projection, and +both search dialects. Two properties carry the whole design and are pinned here: + +* Canonical bounds are fixed width, so byte-lexicographic order *is* chronological + order -- which is what lets one SQL predicate serve SQLite and PostgreSQL alike. +* Dates and instants are separate axes. A date never gains a time of day or a zone, and + an instant written without an offset is read as UTC -- the same convention the rest + of the codebase applies to naive datetimes. +""" + +from datetime import date, datetime, timedelta + +import pytest + +from basic_memory.temporal import ( + DEFAULT_DATE_ORDER, + TemporalAssertion, + TemporalFilter, + TemporalPoint, + TemporalQualifierError, + TemporalRange, + TemporalRangeKind, + TimeRole, + canonical_bound, + parse_authored_point, + parse_point, + parse_range_literal, +) + +DATE = TemporalRangeKind.DATE +INSTANT = TemporalRangeKind.INSTANT + + +# --- Canonical bounds --- + + +@pytest.mark.parametrize( + ("written", "canonical"), + [ + ("2026-07-27T18:42:00Z", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27t18:42:00z", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27T18:42:00+02:00", "2026-07-27T16:42:00.000000Z"), + ("2026-07-27T18:42:00-05:00", "2026-07-27T23:42:00.000000Z"), + ("2026-07-27T18:42:00.5Z", "2026-07-27T18:42:00.500000Z"), + ("2026-07-27T18:42:00.123456Z", "2026-07-27T18:42:00.123456Z"), + ], +) +def test_instant_bounds_normalize_to_fixed_width_utc(written: str, canonical: str): + """Every instant lands on the same 27-character UTC form, whatever it was written as.""" + assert canonical_bound(written, INSTANT) == canonical + assert len(canonical) == 27 + + +def test_canonical_instants_sort_chronologically_as_plain_strings(): + """Fixed width plus fixed separator positions makes string order time order. + + This is the property the SQL predicate relies on: comparing canonical text columns + with `<` and `>` is comparing moments, on either backend, with no typed date bind. + """ + written = [ + "2026-07-27T18:42:00+02:00", # 16:42Z + "2026-07-27T17:00:00Z", + "2026-07-26T23:59:59Z", + "2026-07-27T18:42:00Z", + ] + canonical = [canonical_bound(bound, INSTANT) for bound in written] + + assert sorted(canonical) == [ + "2026-07-26T23:59:59.000000Z", + "2026-07-27T16:42:00.000000Z", + "2026-07-27T17:00:00.000000Z", + "2026-07-27T18:42:00.000000Z", + ] + + +def test_date_bounds_are_already_canonical(): + assert canonical_bound("2026-07-27", DATE) == "2026-07-27" + + +@pytest.mark.parametrize( + "bound", + [ + "20260727", # compact ISO: accepted by date.fromisoformat, breaks fixed width + "2026-7-27", + "27-07-2026", + "2026-02-30", + "not-a-date", + ], +) +def test_malformed_date_bounds_are_refused(bound: str): + with pytest.raises(TemporalQualifierError): + canonical_bound(bound, DATE) + + +@pytest.mark.parametrize( + ("written", "canonical"), + [ + ("2026-07-27T18:42:00", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27t18:42:00", "2026-07-27T18:42:00.000000Z"), + ("2026-07-27T18:42:00.5", "2026-07-27T18:42:00.500000Z"), + ], +) +def test_naive_timestamp_bounds_are_read_as_utc(written: str, canonical: str): + """A timestamp with no offset is UTC, not an error. + + This is the house convention for every other naive datetime in the codebase, and + it is what lets an author write a timestamp without learning RFC 3339's offset + syntax first. + """ + assert canonical_bound(written, INSTANT) == canonical + + +@pytest.mark.parametrize( + "bound", + [ + "2026-07-27 18:42:00", # space separator: not the canonical bound shape + "2026-07-27T18:42", # no seconds + "2026-07-27T18:42:00.1234567Z", # finer than microseconds: would be truncated + "2026-07-27", + ], +) +def test_malformed_instant_bounds_are_refused(bound: str): + with pytest.raises(TemporalQualifierError): + canonical_bound(bound, INSTANT) + + +def test_sub_microsecond_precision_is_refused_rather_than_truncated(): + """Dropping digits would make the stored bound name a different instant.""" + with pytest.raises(TemporalQualifierError, match="microsecond precision"): + canonical_bound("2026-07-27T18:42:00.1234567Z", INSTANT) + + +def test_timestamp_shaped_bound_on_a_date_that_does_not_exist_is_refused(): + """The lexical shape admits `2026-02-30T...`; the calendar does not.""" + with pytest.raises(TemporalQualifierError, match="not a valid timestamp"): + canonical_bound("2026-02-30T10:00:00Z", INSTANT) + + +# --- TemporalPoint --- + + +def test_point_rejects_a_non_canonical_value(): + """A value that skipped canonicalization must not enter the domain.""" + with pytest.raises(TemporalQualifierError, match="not canonical"): + TemporalPoint(kind=INSTANT, value="2026-07-27T18:42:00Z") + + +def test_point_renders_its_canonical_value(): + assert str(TemporalPoint(kind=DATE, value="2026-07-27")) == "2026-07-27" + + +def test_parse_point_infers_the_axis_from_what_was_written(): + assert parse_point("2026-07-27") == TemporalPoint(kind=DATE, value="2026-07-27") + assert parse_point(" 2026-07-27T18:42:00+02:00 ") == TemporalPoint( + kind=INSTANT, value="2026-07-27T16:42:00.000000Z" + ) + + +def test_parse_point_refuses_an_empty_string(): + with pytest.raises(TemporalQualifierError, match="must not be empty"): + parse_point(" ") + + +def test_parse_point_reads_a_naive_timestamp_as_utc(): + """The search boundary follows the same naive-is-UTC rule as authored bounds.""" + assert parse_point("2026-07-27T18:42:00") == TemporalPoint( + kind=INSTANT, value="2026-07-27T18:42:00.000000Z" + ) + + +# --- Flexible authored points --- +# +# The convenient form. `parse_authored_point` reads whatever dateparser reads and +# canonicalizes it into a TemporalRange, so an author never has to spell out a range +# literal to say when something started. + + +@pytest.mark.parametrize( + ("written", "literal", "kind"), + [ + # A year and a month are periods the author delimited by writing them. + ("2026", "[2026-01-01,2027-01-01)", DATE), + ("2026-06", "[2026-06-01,2026-07-01)", DATE), + ("2026-12", "[2026-12-01,2027-01-01)", DATE), + ("June 2026", "[2026-06-01,2026-07-01)", DATE), + # A date or a moment is not: it says when something started and left it open. + ("2026-06-10", "[2026-06-10,)", DATE), + ("Jan 15, 2024", "[2024-01-15,)", DATE), + ("2026-06-10T14:00:00", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10T14:00:00Z", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10T14:00:00+02:00", "[2026-06-10T12:00:00.000000Z,)", INSTANT), + (" 2026-06-10 ", "[2026-06-10,)", DATE), + ], +) +def test_authored_point_denotes_the_span_its_precision_covers(written, literal, kind): + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.kind is kind + assert span.lower_inclusive is True + + +def test_authored_date_never_acquires_a_time_of_day(): + """A calendar date must not become midnight UTC on the way in. + + Midnight in *which* zone is a question the author never answered, and answering it + for them would make a date query and an instant query disagree about the same note. + """ + span = parse_authored_point("2026-06-10") + + assert span is not None + assert span.kind is DATE + assert span.lower == "2026-06-10" + assert "T" not in span.lower and "Z" not in span.lower + + +def test_authored_naive_timestamp_is_read_as_utc_not_local_time(): + """The two spellings of the same moment produce the same stored bound.""" + naive = parse_authored_point("2026-06-10T14:00:00") + explicit = parse_authored_point("2026-06-10T14:00:00Z") + + assert naive is not None and explicit is not None + assert naive == explicit + assert naive.kind is INSTANT + assert naive.lower == "2026-06-10T14:00:00.000000Z" + + +def test_authored_relative_dates_resolve_at_parse_time(): + """`yesterday` is read against the clock now, and re-read on every index pass. + + That is documented behavior rather than a diagnostic: a file edited by hand keeps + its relative wording, and each pass resolves it fresh. + """ + span = parse_authored_point("yesterday") + + assert span is not None + assert span.kind is DATE + yesterday = datetime.now().date() - timedelta(days=1) + assert span.lower == yesterday.isoformat() + + +@pytest.mark.parametrize( + ("date_order", "expected_lower"), + [("YMD", "2026-07-10"), ("DMY", "2026-07-10"), ("MDY", "2026-10-07")], +) +def test_date_order_decides_an_ambiguous_slash_date(date_order, expected_lower): + """`10/07/2026` is July 10 or October 7 depending on the configured preference.""" + span = parse_authored_point("10/07/2026", date_order=date_order) + + assert span is not None + assert span.lower == expected_lower + + +def test_iso_dates_are_never_re_guessed_by_date_order(): + """An ISO date is unambiguous, so no preference may reinterpret it.""" + for date_order in ("YMD", "DMY", "MDY"): + span = parse_authored_point("2026-07-10", date_order=date_order) + assert span is not None + assert span.lower == "2026-07-10", date_order + + +def test_the_default_date_order_is_iso(): + assert DEFAULT_DATE_ORDER == "YMD" + assert parse_authored_point("10/07/2026") == parse_authored_point( + "10/07/2026", date_order="YMD" + ) + + +@pytest.mark.parametrize( + "written", + [ + "2026-02-30", # ISO-shaped, but February has no 30th + "2026-13-01", # ISO-shaped, but there is no 13th month + ], +) +def test_impossible_iso_dates_are_unread_rather_than_re_interpreted(written: str): + """dateparser reads `2026-13-01` as the 13th of January; a wrong date is worse. + + The canonical ISO shape takes the strict path precisely so leniency cannot invent + a date the author did not write. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + ["paul", "basicmemory.com", "ops@example.com", "someone(2026)", "Redis.", "Q3"], +) +def test_text_that_names_no_date_reads_as_nothing(written: str): + """No error and no assertion: the caller leaves such a token as content.""" + assert parse_authored_point(written) is None + + +def test_a_year_with_no_successor_is_unread(): + """Year 9999 has no January 1 after it to close the span with.""" + assert parse_authored_point("9999") is None + # The year before it still resolves, so the guard is the calendar edge, not 4 digits. + assert parse_authored_point("9998") == TemporalRange( + kind=DATE, + lower=date(9998, 1, 1).isoformat(), + upper=date(9999, 1, 1).isoformat(), + lower_inclusive=True, + ) + + +# --- TemporalRange normalization --- + + +def test_unbounded_sides_are_forced_exclusive(): + """PostgreSQL's rule: there is no endpoint to include, so inclusivity is meaningless.""" + span = TemporalRange( + kind=DATE, lower=None, upper="2026-07-27", lower_inclusive=True, upper_inclusive=True + ) + + assert span.lower_inclusive is False + assert span.upper_inclusive is True + assert str(span) == "(,2026-07-27]" + + +def test_fully_unbounded_range_is_exclusive_on_both_sides(): + span = TemporalRange(kind=DATE, lower_inclusive=True, upper_inclusive=True) + + assert (span.lower_inclusive, span.upper_inclusive) == (False, False) + assert str(span) == "(,)" + + +@pytest.mark.parametrize( + ("lower_inclusive", "upper_inclusive"), + [(True, False), (False, True), (False, False)], +) +def test_degenerate_range_collapses_to_empty(lower_inclusive: bool, upper_inclusive: bool): + """`[a,a)`, `(a,a]`, and `(a,a)` contain no points, so they *are* the empty range.""" + span = TemporalRange( + kind=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=lower_inclusive, + upper_inclusive=upper_inclusive, + ) + + assert span.is_empty is True + assert span.lower is None and span.upper is None + assert str(span) == "empty" + + +def test_closed_single_point_range_is_not_empty(): + """`[a,a]` contains exactly one point, which is a real interval.""" + span = TemporalRange( + kind=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert span.is_empty is False + assert str(span) == "[2026-07-27,2026-07-27]" + + +def test_inverted_range_is_refused(): + with pytest.raises(TemporalQualifierError, match="after upper bound"): + TemporalRange(kind=DATE, lower="2026-08-01", upper="2026-06-10") + + +def test_empty_range_cannot_carry_bounds(): + """Two representations of the same interval would make equality lie.""" + with pytest.raises(TemporalQualifierError, match="carries no bounds"): + TemporalRange(kind=DATE, lower="2026-07-27", is_empty=True) + with pytest.raises(TemporalQualifierError, match="carries no bounds"): + TemporalRange(kind=DATE, is_empty=True, upper_inclusive=True) + + +def test_range_rejects_non_canonical_bounds(): + with pytest.raises(TemporalQualifierError, match="not canonical"): + TemporalRange(kind=INSTANT, lower="2026-07-27T18:42:00Z") + + +def test_empty_constructor_builds_the_empty_range_on_one_axis(): + span = TemporalRange.empty(INSTANT) + + assert (span.kind, span.is_empty, span.lower, span.upper) == (INSTANT, True, None, None) + + +# --- Range literals --- + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + ("[2026-06-10,2026-07-27)", (True, False, "2026-06-10", "2026-07-27")), + ("(2026-06-10,2026-07-27]", (False, True, "2026-06-10", "2026-07-27")), + ("[2026-06-10,2026-07-27]", (True, True, "2026-06-10", "2026-07-27")), + ("(2026-06-10,2026-07-27)", (False, False, "2026-06-10", "2026-07-27")), + ("[2026-06-10,)", (True, False, "2026-06-10", None)), + ("(,2026-07-27]", (False, True, None, "2026-07-27")), + ], +) +def test_range_literal_round_trips_through_its_canonical_rendering(literal, expected): + span = parse_range_literal(literal) + + assert (span.lower_inclusive, span.upper_inclusive, span.lower, span.upper) == expected + assert str(span) == literal + + +def test_range_literal_tolerates_surrounding_whitespace(): + assert str(parse_range_literal(" [2026-06-10, 2026-07-27) ")) == "[2026-06-10,2026-07-27)" + + +def test_empty_literal_requires_an_explicit_axis(): + """`empty` carries no bounds to classify, so the caller must name the axis.""" + assert parse_range_literal("empty", kind=DATE).is_empty is True + with pytest.raises(TemporalQualifierError, match="kind must be given"): + parse_range_literal("empty") + + +def test_fully_unbounded_literal_requires_an_explicit_axis(): + assert parse_range_literal("(,)", kind=INSTANT).kind is INSTANT + with pytest.raises(TemporalQualifierError, match="no bounds to classify"): + parse_range_literal("(,)") + + +def test_range_literal_refuses_mixed_axes(): + with pytest.raises(TemporalQualifierError, match="mix date-only and timestamp bounds"): + parse_range_literal("[2026-06-10,2026-07-27T00:00:00Z)") + + +def test_range_literal_refuses_an_axis_it_was_not_asked_for(): + with pytest.raises(TemporalQualifierError, match="expected instant bounds"): + parse_range_literal("[2026-06-10,2026-07-27)", kind=INSTANT) + + +@pytest.mark.parametrize( + "literal", + [ + "2026-06-10,2026-07-27", # no brackets + "[2026-06-10]", # no comma + "[2026-06-10,2026-07-27", # unbalanced + "[2026-06-10,2026-07-27,2026-08-01)", # three bounds + "", + ], +) +def test_malformed_range_literals_are_refused(literal: str): + with pytest.raises(TemporalQualifierError, match="range literal must be"): + parse_range_literal(literal) + + +# --- TemporalFilter --- + + +def test_filter_refuses_asking_two_questions_at_once(): + with pytest.raises(TemporalQualifierError, match="never both"): + TemporalFilter( + at=parse_point("2026-07-27"), + overlaps=parse_range_literal("[2026-06-10,2026-07-27)"), + ) + + +def test_filter_refuses_asking_nothing_at_all(): + """A filter that names no role, point, or range would match everything silently.""" + with pytest.raises(TemporalQualifierError, match="must name a role"): + TemporalFilter() + + +def test_point_filter_window_is_the_degenerate_closed_range(): + """Containment is overlap with `[p,p]`, which is why one predicate answers both.""" + window = TemporalFilter(at=parse_point("2026-07-27")).window + + assert window == TemporalRange( + kind=DATE, + lower="2026-07-27", + upper="2026-07-27", + lower_inclusive=True, + upper_inclusive=True, + ) + + +def test_overlap_filter_window_is_the_range_itself(): + span = parse_range_literal("[2026-06-10,2026-07-27)") + + assert TemporalFilter(overlaps=span).window == span + + +def test_role_only_filter_has_no_window(): + """Nothing to intersect: the question is only "does this axis carry an assertion".""" + assert TemporalFilter(role=TimeRole.EFFECTIVE).window is None + + +# --- TemporalAssertion --- + + +def test_assertion_defaults_to_the_observation_extractor(): + assertion = TemporalAssertion( + time_role=TimeRole.EFFECTIVE, + valid_during=parse_range_literal("[2026-06-10,2026-07-27)"), + source_text="@effective[2026-06-10,2026-07-27)", + ) + + assert assertion.extractor == "observation" + assert assertion.metadata is None + + +def test_recorded_time_is_not_an_authorable_role(): + """Recorded time is never written in markdown, so no role names it.""" + assert "recorded" not in {role.value for role in TimeRole} + assert {role.value for role in TimeRole} == { + "effective", + "valid", + "occurred", + "due", + "mentioned", + } From ea44776a605f99e07b12544dab8fd33f31f95b9e Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 09:33:13 -0500 Subject: [PATCH 02/25] fix(core): canonicalize date ranges and fix the Windows diagnostic Calendar dates are a discrete domain, so preserving authored bounds made the overlap predicate report false positives: (2026-01-01,2026-01-03) holds only Jan 2 and (2026-01-02,2026-01-04) holds only Jan 3, yet each raw endpoint lies inside the other. Date ranges now canonicalize to half-open [lower,upper) at construction, the way PostgreSQL normalizes daterange, so the existing predicate becomes correct without special cases. Instants are a continuous domain and are left alone. The authored token is preserved separately in source_text, so files still round-trip byte-exact. Also: the malformed-qualifier diagnostic interpolated a Path, which renders with backslashes on Windows and failed the assertion there. It now uses as_posix(), matching how entity.file_path, permalinks, and search rows all name files. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/man/man3/search-notes(3).md | 2 + src/basic_memory/markdown/entity_parser.py | 7 +- .../markdown/temporal_qualifier.py | 60 +++- src/basic_memory/mcp/tools/search.py | 6 + .../repository/temporal_filters.py | 7 + src/basic_memory/schemas/search.py | 6 +- src/basic_memory/temporal.py | 119 ++++++-- tests/markdown/test_entity_parser.py | 5 + tests/markdown/test_temporal_qualifier.py | 135 ++++++++- .../test_memory_time_index_contract.py | 248 +++++++++++++++++ tests/test_temporal.py | 262 +++++++++++++++++- 11 files changed, 799 insertions(+), 58 deletions(-) diff --git a/src/basic_memory/man/man3/search-notes(3).md b/src/basic_memory/man/man3/search-notes(3).md index fe75e1573..87930aa1c 100644 --- a/src/basic_memory/man/man3/search-notes(3).md +++ b/src/basic_memory/man/man3/search-notes(3).md @@ -130,6 +130,8 @@ bm tool search-notes "conflict error" --project manual --page-size 2 - [gotcha] valid_at and valid_overlaps never mix calendar dates with instants: a date query matches only date ranges and an instant query only instant ranges, so `2026-07-27` and `2026-07-27T00:00:00Z` are different questions #valid-time - [gotcha] A timestamp written without an offset is read as UTC, in an authored qualifier and in a filter alike — same convention as every other naive datetime in Basic Memory #valid-time - [gotcha] An authored token that does not read as a date is left as ordinary observation content with no warning; only an unknown role (`@asserted:2026-06-10`) is reported #valid-time +- [gotcha] An authored point is one whitespace-delimited token: `@occurred:2026-06-10`, `@occurred:03/04/2026` and `@occurred:yesterday` work, but a multi-word date like `@occurred:June 10, 2026` is left as content because nothing can tell where it ends #valid-time +- [gotcha] `@occurred:03/04/2026` resolves by the `date_order` setting (YMD/DMY read it as 3 April, MDY as 4 March); ISO dates are never re-guessed #valid-time ## SEE ALSO diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 47474566f..3dd68f64d 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -341,10 +341,15 @@ async def parse_markdown_content( # learns the line needs fixing. Text that simply is not a date is ordinary content # and says nothing here. The typed `temporal_error` field carries the same message # to programmatic callers; this layer adds the path. + # `as_posix()` rather than the Path itself: Basic Memory names files with + # forward slashes everywhere (entity.file_path, permalinks, search rows), so a + # Windows `WindowsPath` rendering `decisions\cache-layer.md` would print a path + # the author cannot find in any other surface. for observation in entity_content.observations: if observation.temporal_error: logger.warning( - f"Temporal qualifier ignored in {file_path}: {observation.temporal_error}" + f"Temporal qualifier ignored in {file_path.as_posix()}: " + f"{observation.temporal_error}" ) # Sections are structural, not semantic: they index the body for range reads, diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py index f357f8c21..cb9ce3d5e 100644 --- a/src/basic_memory/markdown/temporal_qualifier.py +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -10,16 +10,29 @@ The bracket form carries a range literal and needs no separator, because no role name can begin with `[` or `(`. The point form needs the `:` because a date can begin with a letter (`yesterday`), so nothing else would tell `@occurred:yesterday` from a handle. -A role-less point must begin with a digit and be at least as wide as a year: a bare -`@word` is overwhelmingly a mention, a version, or a handle, and dateparser reads many -short tokens as dates (`@may` as May, `@v2` as February, `@1` as January). With a role -the author has said what they mean, so any text dateparser can read is accepted there, -relative dates included. -One rule decides everything else: **if the payload reads as time, the token becomes a -qualifier; if it does not, the token stays ordinary observation content, silently.** -Prose is full of `@` -- email addresses, handles, `@todo:` markers -- and warning about -each one that is not a date would be noise, not help. +**A point is one whitespace-delimited token**, in both forms. dateparser reads far more +than one token -- `June 10, 2026`, `2 days ago`, `2026-06-10 10:00 AM` all resolve, and +`parse_authored_point` accepts them -- but nothing here can tell where such a date ends: +dateparser also reads `June 10, 2026 The` and `2026-06-10 The`, so growing the token +until parsing fails would swallow the author's prose. A multi-word date therefore stays +content; `@occurred:2026-06-10` says the same thing in one token. + +That token boundary is also why two shapes that *do* parse are refused, so a truncated +read never becomes a plausible-looking assertion: + +* **A short number.** dateparser reads `1` as January and `3.5` as March 5, but at the + head of a line those are list markers and version numbers. A numeric point must be at + least as wide as a year. +* **A word naming only a month or a year** (`June`, `may`, `v2`). Alone it is usually + prose; as the first token of `June 10, 2026` reading it would file June 2026 and leave + `10, 2026` in the content. A word is taken only when it names a specific day + (`yesterday`, `today`), in whatever language dateparser resolves it. + +Beyond those, one rule decides everything: **if the payload reads as time, the token +becomes a qualifier; if it does not, the token stays ordinary observation content, +silently.** Prose is full of `@` -- email addresses, handles, `@todo:` markers -- and +warning about each one that is not a date would be noise, not help. The single exception is an **unknown role**. `@asserted:2026-06-10` parses as time and names an axis, so the author is plainly reaching for this feature and a short list of @@ -56,13 +69,15 @@ # `paul@basicmemory.com` and mid-sentence `@handles` out entirely. _RANGE_QUALIFIER = re.compile(rf"^@({_ROLE_PATTERN})?([\[(][^\[\]()]*,[^\[\]()]*[\])])(?=\s|$)") -# `@role:`. +# `@role:`. _ROLE_POINT_QUALIFIER = re.compile(rf"^@({_ROLE_PATTERN}):(\S+)") -# `@` -- the role-less point. At least four characters -# wide, the width of a year: dateparser reads `1` as January and `3.5` as March 5, and -# a token that short at the head of a line is a list marker or a version, not a date. -_BARE_POINT_QUALIFIER = re.compile(r"^@(\d\S{3,})") +# `@` -- the role-less point. Without a role there is nothing to +# distinguish a word from a handle, so only digits open the form at all. +_BARE_POINT_QUALIFIER = re.compile(r"^@(\d\S*)") + +# The width of a year, and the shortest numeric token worth reading as one. +_MIN_NUMERIC_POINT_WIDTH = 4 @dataclass(frozen=True, slots=True) @@ -114,6 +129,21 @@ def _read_range_qualifier(content: str) -> _ReadQualifier | None: return _ReadQualifier(match.group(0), match.end(), match.group(1), valid_during) +def _names_a_deliberate_date(point: str, valid_during: TemporalRange) -> bool: + """Whether a one-token point is specific enough to be an assertion rather than prose. + + The two shapes refused here both parse, which is exactly why they need refusing -- + see the module docstring for what each one costs if it is read. + + A bounded span is how a coarse point announces itself: `parse_authored_point` closes + a year or a month at its successor and leaves a day or a moment open, so + `upper is None` *is* "this names a specific day". + """ + if point[0].isdigit(): + return len(point) >= _MIN_NUMERIC_POINT_WIDTH + return valid_during.upper is None + + def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _ReadQualifier | None: """Match either point form and read its date, or report no usable qualifier.""" roled = _ROLE_POINT_QUALIFIER.match(content) @@ -132,7 +162,7 @@ def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _ReadQu order = date_order if date_order is not None else ConfigManager().config.date_order point = match.group(2) if roled is not None else match.group(1) valid_during = parse_authored_point(point, date_order=order) - if valid_during is None: + if valid_during is None or not _names_a_deliberate_date(point, valid_during): return None role_name = match.group(1) if roled is not None else None return _ReadQualifier(match.group(0), match.end(), role_name, valid_during) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 208e4d6f8..9022d7af0 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -934,6 +934,12 @@ async def search_notes( which files the assertion on the `valid` axis; a role-less point has to start with a digit and be at least as wide as a year, so `@v2` and `@may` stay prose. + A point is **one whitespace-delimited token**. Slash dates (`@occurred:03/04/2026`, + read by the `date_order` setting) and single-word relative dates + (`@occurred:yesterday`) work; multi-word dates like `@occurred:June 10, 2026` do + not, because nothing can tell where such a date ends — write `@occurred:2026-06-10` + instead. An unreadable token is left as ordinary content, never half-read. + These filters query that authored time, which is a different axis from `after_date` (last-indexed time) — `after_date` is never reinterpreted as valid time. - `search_notes("cache layer", role="effective", valid_at="2026-07-28")` diff --git a/src/basic_memory/repository/temporal_filters.py b/src/basic_memory/repository/temporal_filters.py index ba27bd2c2..6219bb504 100644 --- a/src/basic_memory/repository/temporal_filters.py +++ b/src/basic_memory/repository/temporal_filters.py @@ -12,6 +12,13 @@ compared directly against a column, so PostgreSQL always infers their type and asyncpg never sees a bare untyped parameter. +Comparing endpoint *values* like this is only equivalent to comparing the sets of times +they delimit because `TemporalRange` has already canonicalized every date range to the +half-open `[lower,upper)` form. Calendar dates are discrete, so two date ranges can hold +each other's raw endpoints while sharing no actual day; the canonical form is what rules +that out. The inclusivity branches below therefore only ever fire on the instant axis in +practice, and they stay because instants are continuous and keep the authored form. + The predicate is a *non-correlated* subquery, and that shape is load-bearing rather than stylistic. SQLite's default word search emits an OR of per-column `MATCH` predicates; adding a correlated `EXISTS` to that WHERE clause makes SQLite refuse the diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index ebf006e93..e023582b7 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -209,10 +209,14 @@ class TemporalRangeValue(BaseModel): deliberately absent: `literal` is the canonical PostgreSQL range literal and the decomposed bounds are the same interval, spelled out so a caller can compare endpoints without parsing. + + Canonical means canonical: a *date* range always reads half-open, whatever brackets + the author typed, because calendar dates are discrete. `source_text` on the + enclosing `TemporalResultMetadata` is where the author's own spelling survives. """ kind: str # "date" (calendar dates) or "instant" (UTC timestamps) - literal: str # e.g. "[2026-06-10,2026-07-27)", "(,2026-07-27]", "empty" + literal: str # e.g. "[2026-06-10,2026-07-28)", "(,2026-07-27)", "empty" lower: Optional[str] = None # None means unbounded on that side upper: Optional[str] = None lower_inclusive: bool = False diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 7009e634e..bb222e953 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -8,7 +8,10 @@ PostgreSQL's range conventions are the language contract: `[lower,upper)` with explicit inclusivity per side, unbounded ends, and a distinguished empty range. That is a vocabulary choice, not a storage requirement -- these values reduce to portable scalars -so SQLite and Postgres can share one logical model. +so SQLite and Postgres can share one logical model. Its *discrete* canonicalization is +part of the contract too: a date range is stored as `[lower,upper)`, for the reason +`TemporalRange` documents. The author's own spelling is not lost -- it is kept verbatim +on `TemporalAssertion.source_text`. Two canonical lexical forms carry every bound: @@ -38,7 +41,7 @@ import re from dataclasses import dataclass -from datetime import UTC, date, datetime +from datetime import UTC, date, datetime, timedelta from enum import StrEnum from functools import lru_cache from typing import TYPE_CHECKING, Any, Literal, override @@ -165,6 +168,19 @@ def _require_canonical(value: str, kind: TemporalRangeKind) -> None: raise TemporalQualifierError(f"{kind.value} bound is not canonical: {value!r}") +def _next_calendar_day(bound: str) -> str | None: + """The canonical date after `bound`, or None when the calendar has none. + + Only `9999-12-31` has no successor. Reporting that as None rather than raising lets + each side of a range decide what running off the end of the calendar means for it: + an upper end there covers every remaining day, a lower end past it covers none. + """ + day = date.fromisoformat(bound) + if day == date.max: + return None + return (day + timedelta(days=1)).isoformat() + + # --- Values --- @@ -188,14 +204,28 @@ class TemporalRange: """One authored interval on a single time axis. Bounds are canonical lexical strings; `None` means unbounded on that side. - Construction normalizes two PostgreSQL rules so no caller has to remember them: - an unbounded side is always exclusive, and a degenerate interval (`[a,a)`, - `(a,a]`, `(a,a)`) *is* the empty range. - - Unlike PostgreSQL's `daterange`, a discrete date range is not rewritten into the - canonical `[)` form -- `[a,b]` keeps the inclusivity the author wrote. Evaluating - the authored flags directly is set-equivalent for containment and overlap and needs - no date arithmetic; only the rendered literal differs. + Construction normalizes three PostgreSQL rules so no caller has to remember them: + an unbounded side is always exclusive, an interval containing no points *is* the + empty range, and -- exactly as `daterange` does -- a **date** range is rewritten + into the half-open `[lower,upper)` form. + + That last rule is what makes the scalar SQL predicate correct rather than merely + tidy. Calendar dates are a *discrete* domain, so `[a,b]` and `[a,b+1)` denote the + same set of days, but only the half-open spelling lets endpoint comparisons decide + membership. Left as authored, `(2026-01-01,2026-01-03)` holds only January 2 and + `(2026-01-02,2026-01-04)` holds only January 3 -- disjoint sets -- yet each raw + endpoint lies inside the other's bounds, so a comparison of raw endpoints reports + an overlap that does not exist. Canonicalized to `[2026-01-02,2026-01-03)` and + `[2026-01-03,2026-01-04)`, the same comparison is right. + + Instants are a continuous domain -- no moment is "the next one" -- so an instant + range keeps the inclusivity the author wrote and is never rewritten this way. + + Canonicalization changes the *stored* spelling, never the set of times: `[a,a]` + becomes `[a,a+1)`, the one day `a`. What the author typed is not lost; it is kept + verbatim on `TemporalAssertion.source_text`, which is what serialization replays + and what a search result quotes back. `__str__` renders the canonical form, and + re-parsing that rendering yields this same value. """ kind: TemporalRangeKind @@ -223,6 +253,8 @@ def __post_init__(self) -> None: _require_canonical(bound, self.kind) # Canonical bounds are fixed width, so string order is chronological order. + # Judged on the bounds as authored: an interval written backwards is an author + # error to report, not an empty range to accept silently. if self.lower is not None and self.upper is not None and self.lower > self.upper: raise TemporalQualifierError( f"range lower bound {self.lower} is after upper bound {self.upper}" @@ -234,18 +266,44 @@ def __post_init__(self) -> None: if self.upper is None: object.__setattr__(self, "upper_inclusive", False) - # PostgreSQL: an interval whose endpoints coincide without including both of - # them contains no points, and is therefore the empty range. - if ( - self.lower is not None - and self.lower == self.upper - and not (self.lower_inclusive and self.upper_inclusive) - ): - object.__setattr__(self, "lower", None) - object.__setattr__(self, "upper", None) - object.__setattr__(self, "lower_inclusive", False) - object.__setattr__(self, "upper_inclusive", False) - object.__setattr__(self, "is_empty", True) + # --- Discrete canonical form --- + # + # Rewrite a date range to `[lower,upper)`. See the class docstring for why the + # scalar overlap predicate needs this and why instants must not get it. + if self.kind is TemporalRangeKind.DATE: + if self.lower is not None and not self.lower_inclusive: + after_lower = _next_calendar_day(self.lower) + if after_lower is None: + # Nothing follows 9999-12-31, so a range starting strictly after it + # admits no date at all. + self._become_empty() + return + object.__setattr__(self, "lower", after_lower) + object.__setattr__(self, "lower_inclusive", True) + if self.upper is not None and self.upper_inclusive: + # None here loses no days: 9999-12-31 is the last date there is, so + # "through 9999-12-31 inclusive" and "unbounded above" hold the same + # set, and only the latter is representable in the canonical form. + object.__setattr__(self, "upper", _next_calendar_day(self.upper)) + object.__setattr__(self, "upper_inclusive", False) + + # PostgreSQL: an interval that admits no point at all *is* the empty range. The + # endpoints coincide without both being owned (`[a,a)`), or -- only reachable + # after the rewrite above, from `(a,a)` -- the lower end has overshot the upper. + if self.lower is not None and self.upper is not None: + admits_no_date = self.lower > self.upper or ( + self.lower == self.upper and not (self.lower_inclusive and self.upper_inclusive) + ) + if admits_no_date: + self._become_empty() + + def _become_empty(self) -> None: + """Collapse to the one empty representation, whatever bounds were written.""" + object.__setattr__(self, "lower", None) + object.__setattr__(self, "upper", None) + object.__setattr__(self, "lower_inclusive", False) + object.__setattr__(self, "upper_inclusive", False) + object.__setattr__(self, "is_empty", True) @classmethod def empty(cls, kind: TemporalRangeKind) -> "TemporalRange": @@ -254,7 +312,12 @@ def empty(cls, kind: TemporalRangeKind) -> "TemporalRange": @override def __str__(self) -> str: - """Render the canonical PostgreSQL range literal.""" + """Render the canonical PostgreSQL range literal. + + This is the normalized interval, not the author's text -- a date range always + renders half-open. Feeding the result back to `parse_range_literal` reproduces + this same value, so the rendering is a fixed point rather than a lossy view. + """ if self.is_empty: return EMPTY_RANGE_LITERAL lower = "" if self.lower is None else self.lower @@ -291,10 +354,12 @@ def __post_init__(self) -> None: def window(self) -> TemporalRange | None: """The interval this filter tests against, or None for a role-only filter. - Containment of a point is overlap with the degenerate closed range `[p,p]`: - both ask whether the stored interval and the queried interval share at least - one point. Collapsing them here lets one predicate answer both questions, - which is also why the two can never disagree about inclusivity or bounds. + Containment of a point is overlap with the closed range `[p,p]`: both ask + whether the stored interval and the queried interval share at least one point. + Collapsing them here lets one predicate answer both questions, which is also + why the two can never disagree about inclusivity or bounds. On the date axis + `TemporalRange` canonicalizes that window to `[p,p+1)` -- still the single day + `p`, now in the half-open form the predicate compares correctly. """ if self.at is not None: return TemporalRange( diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index e8d5960ed..e932e654f 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -436,6 +436,11 @@ async def test_malformed_qualifier_logs_diagnostic_with_file_path(entity_parser) The typed `temporal_error` field carries the same message to programmatic callers; this layer is the only one that knows which file the observation came from. + + The path is asserted in its forward-slash form on every platform. That is not a + convenience for the test: it is the same spelling `entity.file_path`, permalinks, + and search rows use, so the name in the warning is one the author can search for. + A `WindowsPath` interpolated directly would print `decisions\\cache-layer.md`. """ records: list[Any] = [] sink_id = logger.add(lambda message: records.append(message.record), level="WARNING") diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index a6b676197..b843e37f8 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -25,7 +25,7 @@ from basic_memory.markdown.entity_parser import parse from basic_memory.markdown.schemas import Observation from basic_memory.markdown.temporal_qualifier import parse_temporal_qualifier -from basic_memory.temporal import TemporalRangeKind, TimeRole +from basic_memory.temporal import DateOrder, TemporalRangeKind, TimeRole @pytest.fixture(autouse=True) @@ -127,6 +127,25 @@ def test_qualifier_carries_its_role_and_bounds(): assert str(assertion.valid_during) == "[2026-06-10,2026-07-27)" +def test_a_closed_qualifier_is_stored_half_open_without_rewriting_the_line(): + """The two forms coexist: canonical bounds for the index, the author's text on disk. + + `[2026-06-10,2026-07-27]` means "through July 27", which the discrete canonical form + spells `[2026-06-10,2026-07-28)`. That normalization is the projection's business -- + `source_text` keeps the author's words, so serializing the note writes the file back + exactly as they wrote it. + """ + line = "- [decision] @effective[2026-06-10,2026-07-27] The cache layer will use Redis." + + observation = _observation(line) + + [assertion] = observation.temporal + assert assertion.source_text == "@effective[2026-06-10,2026-07-27]" + assert str(assertion.valid_during) == "[2026-06-10,2026-07-28)" + assert assertion.valid_during.upper_inclusive is False + assert str(observation) == line + + def test_qualifier_is_peeled_before_context_and_tags(): """Peel order matters: the context rule would otherwise steal a `)` qualifier. @@ -274,12 +293,118 @@ def test_a_role_less_point_must_be_digit_led_and_year_wide(qualifier: str): assert observation.content.startswith(qualifier) -def test_a_short_point_is_still_accepted_when_the_role_is_named(): - """The width rule guards the *bare* form only; a role removes the ambiguity.""" - observation = _observation("- [decision] @occurred:may The cutover ran.") +def test_a_word_point_is_read_only_when_it_names_a_specific_day(): + """A role opens the form to words, but not to words that name only a period. - [assertion] = observation.temporal + `yesterday` resolves to one day and is taken. `may` resolves to a whole month, and + a bare month name at the head of a line is either prose or -- worse -- the first + token of `May 10, 2026`, where reading it would file May 2026 and leave `10, 2026` + behind as content. + """ + day = _observation("- [decision] @occurred:yesterday The cutover ran.") + [assertion] = day.temporal assert assertion.time_role is TimeRole.OCCURRED + assert day.content == "The cutover ran." + + period = _observation("- [decision] @occurred:may The cutover ran.") + assert period.temporal == [] + assert period.temporal_error is None + assert period.content.startswith("@occurred:may") + + +# --- The flexible vocabulary, as the qualifier grammar sees it --- +# +# `parse_authored_point` reads far more spellings than these (tests/test_temporal.py +# pins that vocabulary). The grammar is narrower on purpose, and this section is the +# boundary between the two: a qualifier is one whitespace-delimited token, because +# dateparser also reads `June 10, 2026 The` and `2026-06-10 The`, so there is no way to +# tell where a multi-word date stops without swallowing the author's prose. + + +@pytest.mark.parametrize( + ("qualifier", "literal", "kind"), + [ + # Single-token absolute dates, with a role and without. + ("@occurred:2026-06-10", "[2026-06-10,)", TemporalRangeKind.DATE), + ("@occurred:03/04/2026", "[2026-04-03,)", TemporalRangeKind.DATE), + ( + "@occurred:2026-06-10T10:00:00", + "[2026-06-10T10:00:00.000000Z,)", + TemporalRangeKind.INSTANT, + ), + # A role admits a word, as long as it names one day. + ("@occurred:today", None, TemporalRangeKind.DATE), + ("@occurred:yesterday", None, TemporalRangeKind.DATE), + ], +) +def test_single_token_points_are_accepted(qualifier: str, literal: str | None, kind): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert observation.content == "The cutover ran." + assert assertion.valid_during.kind is kind + if literal is not None: + assert str(assertion.valid_during) == literal + + +@pytest.mark.parametrize( + "qualifier", + [ + # Multi-word dates: only the first token reaches the reader, and each of these + # first tokens is refused, so the whole line stays content rather than being + # half-read. `@occurred:2026-06-10` says the same thing in one token. + "@occurred:June 10, 2026", + "@occurred:10 June 2026", + "@occurred:Jan 15, 2024", + "@occurred:2 days ago", + "@occurred:last week", + ], +) +def test_multi_word_dates_stay_content_whole(qualifier: str): + """The reader understands these; the grammar cannot delimit them. + + What matters is that an undelimitable date is left *entirely* alone: no coarse + assertion filed from its first token, and no words eaten out of the content. + """ + line = f"- [decision] {qualifier} The cutover ran." + + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == f"{qualifier} The cutover ran." + assert str(observation) == line + + +def test_a_multi_word_date_is_read_up_to_its_first_token_when_that_token_stands_alone(): + """The one partial read the token rule allows, pinned so it is a known boundary. + + `2026-06-10` is a complete date by itself, so the qualifier claims it and the clock + reading stays in the content. The assertion is coarser than the author meant -- a + date, not an instant -- but it is not wrong, and nothing is lost from the line. + """ + observation = _observation("- [decision] @occurred:2026-06-10 10:00 AM The cutover ran.") + + [assertion] = observation.temporal + assert str(assertion.valid_during) == "[2026-06-10,)" + assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert observation.content == "10:00 AM The cutover ran." + + +@pytest.mark.parametrize( + ("date_order", "expected_lower"), + [("YMD", "2026-04-03"), ("DMY", "2026-04-03"), ("MDY", "2026-03-04")], +) +def test_a_roled_slash_date_follows_the_configured_order( + date_order: DateOrder, expected_lower: str +): + """`@occurred:03/04/2026` resolves by preference, through the real parse path.""" + observation = parse_temporal_qualifier( + "@occurred:03/04/2026 The cutover ran.", date_order=date_order + ) + + [assertion] = observation.assertions + assert assertion.valid_during.lower == expected_lower assert observation.content == "The cutover ran." diff --git a/tests/repository/test_memory_time_index_contract.py b/tests/repository/test_memory_time_index_contract.py index 3e839c2d6..aaeeb4d72 100644 --- a/tests/repository/test_memory_time_index_contract.py +++ b/tests/repository/test_memory_time_index_contract.py @@ -12,6 +12,11 @@ range, and a separate instant axis that must never mix with the date axis. Timestamps are written as explicit constants, never as "now", so the answers are the same on every run and on every machine. + +A second population covers the dimension a *discrete* domain adds: date ranges whose +authored bounds and whose sets of days come apart. Those cases are what the half-open +canonicalization in `basic_memory.temporal` exists for, and both dialects must agree +about them too. """ from dataclasses import dataclass @@ -83,6 +88,39 @@ def _instant_range(literal: str) -> TemporalRange: _instant_range("[2026-07-27T16:00:00+02:00,2026-07-27T17:00:00+02:00)"), ), StoredAssertion("due_window", _date_range("[2026-06-10,2026-07-27)"), role=TimeRole.DUE), + # --- The discrete population --- + # + # Filed on the `occurred` axis and dated in March so it never widens an expectation + # above, and so no bound collides with the January bookkeeping timestamps that + # `test_projection_rows_carry_only_authored_bounds` watches for. + # + # Only March 2, and only March 3: adjacent as authored bounds, disjoint as days. + # This is the pair the half-open canonical form exists to tell apart. + StoredAssertion("only_mar_02", _date_range("(2026-03-01,2026-03-03)"), role=TimeRole.OCCURRED), + StoredAssertion("only_mar_03", _date_range("(2026-03-02,2026-03-04)"), role=TimeRole.OCCURRED), + # Back-to-back half-open periods, the shape a sequence of effective windows takes. + StoredAssertion( + "half_open_first", _date_range("[2026-03-10,2026-03-12)"), role=TimeRole.OCCURRED + ), + StoredAssertion( + "half_open_second", _date_range("[2026-03-12,2026-03-14)"), role=TimeRole.OCCURRED + ), + # Closed periods written by an author who means "through the 22nd": they share it. + StoredAssertion("closed_first", _date_range("[2026-03-20,2026-03-22]"), role=TimeRole.OCCURRED), + StoredAssertion( + "closed_second", _date_range("[2026-03-22,2026-03-24]"), role=TimeRole.OCCURRED + ), + StoredAssertion("one_day", _date_range("[2026-03-30,2026-03-30]"), role=TimeRole.OCCURRED), + # After the 5th and before the 6th there is no day, so this authored range is the + # empty range -- something only the discrete reading can see. + StoredAssertion("no_such_day", _date_range("(2026-03-05,2026-03-06)"), role=TimeRole.OCCURRED), + # An instant range with a closed upper end, so the date rewrite is proven to stop + # at the date axis rather than pushing this endpoint forward by a day. + StoredAssertion( + "instant_closed", + _instant_range("[2026-07-27T20:00:00Z,2026-07-27T21:00:00Z]"), + role=TimeRole.OCCURRED, + ), ) DATE_LABELS = frozenset( @@ -302,6 +340,216 @@ async def test_stored_empty_range_matches_no_query(search_repository, temporal_p assert "empty" not in matched, at +# --- The discrete domain: authored bounds vs. the days they denote --- +# +# Calendar dates are discrete, so an authored bound is not the boundary of the set of +# days it delimits. `basic_memory.temporal` closes that gap by storing every date range +# half-open; these tests are what proves the SQL predicate then answers about *days* +# rather than about endpoint strings -- identically on both backends. + + +async def _occurred_overlaps(search_repository, labels_by_id, literal: str) -> set[str]: + return await _matching_labels( + search_repository, + labels_by_id, + TemporalFilter(role=TimeRole.OCCURRED, overlaps=_date_range(literal)), + ) + + +async def _occurred_at(search_repository, labels_by_id, at: str) -> set[str]: + return await _matching_labels( + search_repository, + labels_by_id, + TemporalFilter(role=TimeRole.OCCURRED, at=parse_point(at)), + ) + + +@pytest.mark.asyncio +async def test_date_ranges_that_share_no_day_do_not_overlap(search_repository, temporal_population): + """The case the half-open canonical form exists to get right. + + `(2026-03-01,2026-03-03)` holds only March 2 and `(2026-03-02,2026-03-04)` holds + only March 3, so the two share nothing. Yet each range's raw endpoints lie inside + the other's raw bounds, so comparing the bounds *as authored* reports an overlap + that does not exist. Canonicalized to `[2026-03-02,2026-03-03)` and + `[2026-03-03,2026-03-04)`, the same scalar comparison is right. + """ + assert await _occurred_overlaps( + search_repository, temporal_population, "(2026-03-01,2026-03-03)" + ) == {"only_mar_02"} + + assert await _occurred_overlaps( + search_repository, temporal_population, "(2026-03-02,2026-03-04)" + ) == {"only_mar_03"} + + # And each holds exactly the one day it names. + assert await _occurred_at(search_repository, temporal_population, "2026-03-02") == { + "only_mar_02" + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-03") == { + "only_mar_03" + } + + +@pytest.mark.asyncio +async def test_adjacent_half_open_ranges_share_no_day(search_repository, temporal_population): + """`[a,b)` and `[b,c)` meet at b without sharing it -- the point of the shape.""" + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-10,2026-03-12)" + ) == {"half_open_first"} + + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-12,2026-03-14)" + ) == {"half_open_second"} + + # March 12 belongs to the second period alone. + assert await _occurred_at(search_repository, temporal_population, "2026-03-11") == { + "half_open_first" + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-12") == { + "half_open_second" + } + + +@pytest.mark.asyncio +async def test_closed_ranges_sharing_an_endpoint_do_overlap(search_repository, temporal_population): + """`[a,b]` and `[b,c]` both contain b, so they overlap on that one day. + + Canonicalization must preserve that: `[a,b+1)` and `[b,c+1)` still meet on b. An + author who writes closed bounds means the endpoint day is included, and the stored + form may not quietly take it away. + """ + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-20,2026-03-22]" + ) == {"closed_first", "closed_second"} + + # Narrowed to the shared day alone, both are still there. + assert await _occurred_at(search_repository, temporal_population, "2026-03-22") == { + "closed_first", + "closed_second", + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-21") == { + "closed_first" + } + assert await _occurred_at(search_repository, temporal_population, "2026-03-23") == { + "closed_second" + } + + +@pytest.mark.asyncio +async def test_a_single_day_range_holds_exactly_that_day(search_repository, temporal_population): + """`[a,a]` is one day: neither empty, nor wider than the day the author wrote.""" + assert await _occurred_at(search_repository, temporal_population, "2026-03-30") == {"one_day"} + assert await _occurred_at(search_repository, temporal_population, "2026-03-29") == set() + assert await _occurred_at(search_repository, temporal_population, "2026-03-31") == set() + + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-30,2026-03-30]" + ) == {"one_day"} + + +@pytest.mark.asyncio +async def test_a_date_range_spanning_no_day_is_stored_empty( + search_repository, + session_maker: async_sessionmaker[AsyncSession], + temporal_population, + test_project: Project, +): + """`(2026-03-05,2026-03-06)` reads as an interval but names no day. + + Only the discrete reading can tell: as a continuous interval it looks like an + ordinary bounded range. Stored empty, it answers no question -- not even one about + the days on either side of it. + """ + for at in ("2026-03-05", "2026-03-06"): + assert "no_such_day" not in await _occurred_at( + search_repository, temporal_population, at + ), at + + assert await _occurred_overlaps( + search_repository, temporal_population, "[2026-03-01,2026-03-09)" + ) == {"only_mar_02", "only_mar_03"} + + repository = MemoryTimeIndexRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + rows = await repository.find_for_sources( + session, + [(SearchItemType.OBSERVATION.value, source_id) for source_id in temporal_population], + ) + row = {temporal_population[row.source_id]: row for row in rows}["no_such_day"] + assert (row.is_empty, row.lower_value, row.upper_value) == (True, None, None) + + +@pytest.mark.asyncio +async def test_instant_ranges_are_untouched_by_the_date_canonicalization( + search_repository, temporal_population +): + """`instant_closed` is `[20:00Z,21:00Z]`, and stays exactly that. + + Instants are continuous: there is no next moment to close at, so the endpoint stays + owned and is emphatically not pushed forward by a day the way an inclusive date end + is. A query one microsecond past it, and one a whole day past it, both miss. + """ + at_upper = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.OCCURRED, at=parse_point("2026-07-27T21:00:00Z")), + ) + assert at_upper == {"instant_closed"} + + for outside in ("2026-07-27T21:00:00.000001Z", "2026-07-28T21:00:00Z"): + missed = await _matching_labels( + search_repository, + temporal_population, + TemporalFilter(role=TimeRole.OCCURRED, at=parse_point(outside)), + ) + assert missed == set(), outside + + +@pytest.mark.asyncio +async def test_projection_stores_date_bounds_in_the_canonical_half_open_form( + session_maker: async_sessionmaker[AsyncSession], + temporal_population, + test_project: Project, +): + """What actually lands in the columns the SQL predicate reads. + + The predicate compares bound values and inclusivity flags directly, so the + canonical form has to be in the rows -- not merely in the domain value that built + them. + """ + repository = MemoryTimeIndexRepository(project_id=test_project.id) + async with db.scoped_session(session_maker) as session: + rows = await repository.find_for_sources( + session, + [(SearchItemType.OBSERVATION.value, source_id) for source_id in temporal_population], + ) + by_label = {temporal_population[row.source_id]: row for row in rows} + + # Authored `(2026-03-01,2026-03-03)`: the exclusive lower end moved to the next day. + assert (by_label["only_mar_02"].lower_value, by_label["only_mar_02"].upper_value) == ( + "2026-03-02", + "2026-03-03", + ) + # Authored `[2026-03-20,2026-03-22]`: the inclusive upper end moved to the next day. + assert (by_label["closed_first"].lower_value, by_label["closed_first"].upper_value) == ( + "2026-03-20", + "2026-03-23", + ) + # Authored `[2026-03-30,2026-03-30]`: one day, spelled half-open. + assert (by_label["one_day"].lower_value, by_label["one_day"].upper_value) == ( + "2026-03-30", + "2026-03-31", + ) + for label in ("only_mar_02", "only_mar_03", "half_open_first", "closed_first", "one_day"): + row = by_label[label] + assert (row.lower_inclusive, row.upper_inclusive) == (True, False), label + + # The instant axis keeps the endpoint the author wrote, inclusivity and all. + instant = by_label["instant_closed"] + assert (instant.upper_value, instant.upper_inclusive) == ("2026-07-27T21:00:00.000000Z", True) + + # --- Acceptance 9 and 10: the two axes are never confused --- diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 66860fea7..1044621a0 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -242,6 +242,71 @@ def test_authored_relative_dates_resolve_at_parse_time(): assert span.lower == yesterday.isoformat() +# The written vocabulary. These are what the *reader* accepts; the qualifier grammar +# then decides how much of a line it can safely claim (see +# tests/markdown/test_temporal_qualifier.py), which is a narrower question. + + +@pytest.mark.parametrize( + ("written", "literal", "kind"), + [ + # Month names, in the orders English writes them. + ("June 10, 2026", "[2026-06-10,)", DATE), + ("10 June 2026", "[2026-06-10,)", DATE), + # The exact forms entity_parser.parse_date already advertises. + ("Jan 15, 2024", "[2024-01-15,)", DATE), + ("2024-01-15", "[2024-01-15,)", DATE), + # A clock reading moves the point onto the instant axis, read as UTC. + ("2024-01-15 10:00 AM", "[2024-01-15T10:00:00.000000Z,)", INSTANT), + ("2026-06-10 10:00 AM", "[2026-06-10T10:00:00.000000Z,)", INSTANT), + ], +) +def test_written_dates_read_on_the_axis_their_precision_names(written, literal, kind): + """A written date stays a date; adding a clock reading is what makes it an instant. + + `June 10, 2026` must never acquire a time of day -- midnight in which zone is a + question the author never answered -- while `10:00 AM` with no offset is UTC, the + same convention every other naive datetime here follows. + """ + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.kind is kind + + +def test_written_relative_dates_resolve_against_now(): + """dateparser's relative vocabulary is read whole when it is handed a whole phrase.""" + span = parse_authored_point("2 days ago") + + assert span is not None + assert span.kind is DATE + assert span.lower == (datetime.now().date() - timedelta(days=2)).isoformat() + + +@pytest.mark.parametrize( + ("written", "date_order", "expected_lower"), + [ + # Year last: YMD cannot apply, so dateparser falls back to day-first and only + # MDY reads it differently. + ("03/04/2026", "YMD", "2026-04-03"), + ("03/04/2026", "DMY", "2026-04-03"), + ("03/04/2026", "MDY", "2026-03-04"), + # Year first: now YMD and DMY disagree, so the three orders are pinned pairwise + # across the two forms and no setting is left unproven. + ("2026/03/04", "YMD", "2026-03-04"), + ("2026/03/04", "DMY", "2026-04-03"), + ("2026/03/04", "MDY", "2026-03-04"), + ], +) +def test_slash_dates_resolve_by_the_configured_order(written, date_order, expected_lower): + span = parse_authored_point(written, date_order=date_order) + + assert span is not None + assert span.lower == expected_lower + assert span.kind is DATE + + @pytest.mark.parametrize( ("date_order", "expected_lower"), [("YMD", "2026-07-10"), ("DMY", "2026-07-10"), ("MDY", "2026-10-07")], @@ -310,14 +375,23 @@ def test_a_year_with_no_successor_is_unread(): def test_unbounded_sides_are_forced_exclusive(): - """PostgreSQL's rule: there is no endpoint to include, so inclusivity is meaningless.""" + """PostgreSQL's rule: there is no endpoint to include, so inclusivity is meaningless. + + Asserted on the instant axis so this rule is the only one moving: a date range + would also be rewritten to `[)`, which is a separate normalization with its own + tests below. + """ span = TemporalRange( - kind=DATE, lower=None, upper="2026-07-27", lower_inclusive=True, upper_inclusive=True + kind=INSTANT, + lower=None, + upper="2026-07-27T00:00:00.000000Z", + lower_inclusive=True, + upper_inclusive=True, ) assert span.lower_inclusive is False assert span.upper_inclusive is True - assert str(span) == "(,2026-07-27]" + assert str(span) == "(,2026-07-27T00:00:00.000000Z]" def test_fully_unbounded_range_is_exclusive_on_both_sides(): @@ -347,7 +421,11 @@ def test_degenerate_range_collapses_to_empty(lower_inclusive: bool, upper_inclus def test_closed_single_point_range_is_not_empty(): - """`[a,a]` contains exactly one point, which is a real interval.""" + """`[a,a]` contains exactly one point, which is a real interval. + + On the date axis that one point is one day, and the canonical form says so by + closing at the following day rather than by owning both endpoints. + """ span = TemporalRange( kind=DATE, lower="2026-07-27", @@ -357,7 +435,7 @@ def test_closed_single_point_range_is_not_empty(): ) assert span.is_empty is False - assert str(span) == "[2026-07-27,2026-07-27]" + assert str(span) == "[2026-07-27,2026-07-28)" def test_inverted_range_is_refused(): @@ -384,21 +462,177 @@ def test_empty_constructor_builds_the_empty_range_on_one_axis(): assert (span.kind, span.is_empty, span.lower, span.upper) == (INSTANT, True, None, None) +# --- The discrete canonical form --- +# +# Calendar dates are a discrete domain, so every date range is stored half-open, the +# way PostgreSQL canonicalizes `daterange`. Without it the scalar endpoint comparisons +# in `repository.temporal_filters` do not decide membership -- see +# `test_date_ranges_that_share_no_day_do_not_overlap` for the case that proves it. + + +@pytest.mark.parametrize( + ("authored", "canonical"), + [ + # Already half-open: nothing moves. + ("[2026-06-10,2026-07-27)", "[2026-06-10,2026-07-27)"), + # An exclusive lower end starts on the following day. + ("(2026-06-10,2026-07-27)", "[2026-06-11,2026-07-27)"), + # An inclusive upper end closes at the start of the following day. + ("[2026-06-10,2026-07-27]", "[2026-06-10,2026-07-28)"), + ("(2026-06-10,2026-07-27]", "[2026-06-11,2026-07-28)"), + # An unbounded side has no endpoint to move, whichever side it is. + ("[2026-06-10,)", "[2026-06-10,)"), + ("(2026-06-10,)", "[2026-06-11,)"), + ("(,2026-07-27)", "(,2026-07-27)"), + ("(,2026-07-27]", "(,2026-07-28)"), + ("(,)", "(,)"), + # One authored day is one canonical day. + ("[2026-07-27,2026-07-27]", "[2026-07-27,2026-07-28)"), + ], +) +def test_date_ranges_are_stored_half_open(authored: str, canonical: str): + """Whatever the author wrote, the stored date range is `[lower,upper)`.""" + span = parse_range_literal(authored, kind=DATE) + + assert str(span) == canonical + # A bounded lower end is always owned, a bounded upper end never is. + assert span.lower_inclusive is (span.lower is not None) + assert span.upper_inclusive is False + + +def test_the_canonical_date_rendering_is_a_fixed_point(): + """Re-parsing what `__str__` produced yields this same value, not a third form.""" + for authored in ("(2026-06-10,2026-07-27]", "[2026-07-27,2026-07-27]", "(,2026-07-27]"): + span = parse_range_literal(authored, kind=DATE) + + assert parse_range_literal(str(span), kind=DATE) == span, authored + + +@pytest.mark.parametrize( + "literal", + [ + "[2026-07-27,2026-07-27)", # opens and closes on the same day + "(2026-07-27,2026-07-27]", # starts the 28th, ends the 27th + "(2026-07-27,2026-07-27)", + # After the 27th and before the 28th there is no day at all. Read as a + # continuous interval this looks non-empty, which is exactly the confusion + # the discrete canonical form removes. + "(2026-07-27,2026-07-28)", + ], +) +def test_date_ranges_that_admit_no_day_are_the_empty_range(literal: str): + span = parse_range_literal(literal, kind=DATE) + + assert span.is_empty is True + assert str(span) == "empty" + + +def test_an_inclusive_upper_end_on_the_last_date_becomes_unbounded(): + """`9999-12-31` has no successor to close against, and no later day to exclude.""" + span = TemporalRange( + kind=DATE, + lower="2026-06-10", + upper="9999-12-31", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert (span.upper, span.upper_inclusive) == (None, False) + assert str(span) == "[2026-06-10,)" + + +def test_the_last_date_alone_is_still_one_day_not_the_empty_range(): + """`[9999-12-31,9999-12-31]` survives the rewrite that drops its upper end.""" + span = TemporalRange( + kind=DATE, + lower="9999-12-31", + upper="9999-12-31", + lower_inclusive=True, + upper_inclusive=True, + ) + + assert span.is_empty is False + assert str(span) == "[9999-12-31,)" + + +def test_an_exclusive_lower_end_on_the_last_date_is_empty(): + """A range beginning strictly after the last date admits no date at all.""" + span = TemporalRange(kind=DATE, lower="9999-12-31") + + assert span.is_empty is True + assert str(span) == "empty" + + +@pytest.mark.parametrize( + ("literal", "expected"), + [ + ( + "(2026-07-27T18:42:00Z,2026-07-27T19:00:00Z]", + (False, True, "2026-07-27T18:42:00.000000Z", "2026-07-27T19:00:00.000000Z"), + ), + ( + "[2026-07-27T18:42:00Z,2026-07-27T19:00:00Z]", + (True, True, "2026-07-27T18:42:00.000000Z", "2026-07-27T19:00:00.000000Z"), + ), + ("(,2026-07-27T19:00:00Z]", (False, True, None, "2026-07-27T19:00:00.000000Z")), + ], +) +def test_instant_ranges_keep_the_inclusivity_they_were_written_with(literal, expected): + """Instants are continuous: there is no "next instant" to shift a bound onto. + + Adding a microsecond would be an invented precision, and rewriting an instant the + way a date is rewritten would move the endpoint to a moment nobody wrote. + """ + span = parse_range_literal(literal, kind=INSTANT) + + assert (span.lower_inclusive, span.upper_inclusive, span.lower, span.upper) == expected + + +def test_an_instant_range_over_one_day_is_not_widened_by_a_day(): + """The date rewrite must not reach the instant axis: `+1 day` there is a bug.""" + span = parse_range_literal("[2026-07-27T00:00:00Z,2026-07-27T23:59:59Z]", kind=INSTANT) + + assert span.upper == "2026-07-27T23:59:59.000000Z" + assert span.upper_inclusive is True + + +def test_a_degenerate_instant_range_still_holds_exactly_one_moment(): + """`[t,t]` on a continuous axis stays `[t,t]`; there is no successor to close at.""" + span = parse_range_literal("[2026-07-27T18:42:00Z,2026-07-27T18:42:00Z]", kind=INSTANT) + + assert span.is_empty is False + assert str(span) == "[2026-07-27T18:42:00.000000Z,2026-07-27T18:42:00.000000Z]" + + # --- Range literals --- @pytest.mark.parametrize( ("literal", "expected"), [ + # Date literals already in the canonical half-open form. ("[2026-06-10,2026-07-27)", (True, False, "2026-06-10", "2026-07-27")), - ("(2026-06-10,2026-07-27]", (False, True, "2026-06-10", "2026-07-27")), - ("[2026-06-10,2026-07-27]", (True, True, "2026-06-10", "2026-07-27")), - ("(2026-06-10,2026-07-27)", (False, False, "2026-06-10", "2026-07-27")), ("[2026-06-10,)", (True, False, "2026-06-10", None)), - ("(,2026-07-27]", (False, True, None, "2026-07-27")), + ("(,2026-07-27)", (False, False, None, "2026-07-27")), + # Instant literals, which are stored exactly as written whatever the brackets. + ( + "(2026-06-10T00:00:00.000000Z,2026-07-27T00:00:00.000000Z]", + (False, True, "2026-06-10T00:00:00.000000Z", "2026-07-27T00:00:00.000000Z"), + ), + ( + "[2026-06-10T00:00:00.000000Z,2026-07-27T00:00:00.000000Z]", + (True, True, "2026-06-10T00:00:00.000000Z", "2026-07-27T00:00:00.000000Z"), + ), + ("(,2026-07-27T00:00:00.000000Z]", (False, True, None, "2026-07-27T00:00:00.000000Z")), ], ) def test_range_literal_round_trips_through_its_canonical_rendering(literal, expected): + """A literal already in canonical form parses and renders back to itself. + + Date literals written some other way still round trip -- through their canonical + spelling rather than their authored one -- which + `test_the_canonical_date_rendering_is_a_fixed_point` pins separately. + """ span = parse_range_literal(literal) assert (span.lower_inclusive, span.upper_inclusive, span.lower, span.upper) == expected @@ -475,6 +709,16 @@ def test_point_filter_window_is_the_degenerate_closed_range(): lower_inclusive=True, upper_inclusive=True, ) + # Canonicalized like any other date range: still the single day 2026-07-27, now in + # the half-open form the SQL predicate compares correctly. + assert str(window) == "[2026-07-27,2026-07-28)" + + +def test_instant_point_filter_window_stays_a_closed_moment(): + """The instant axis has no successor to close at, so `[t,t]` is the window.""" + window = TemporalFilter(at=parse_point("2026-07-27T18:42:00Z")).window + + assert str(window) == "[2026-07-27T18:42:00.000000Z,2026-07-27T18:42:00.000000Z]" def test_overlap_filter_window_is_the_range_itself(): From 127986aeee6ce14bafdfdb7d39b775f3e8c475c2 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 11:55:34 -0500 Subject: [PATCH 03/25] feat(core): quote multi-word temporal dates, rename role to kind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-word dates now work by delimiting them explicitly: - [decision] @occurred:"June 10, 2026" The cutover ran. Inside quotes the author has marked where the date ends, so the specific-day guard does not apply and dateparser's full vocabulary is reachable — including relative and month-only forms whose unquoted spellings are refused. Unquoted behavior is unchanged: one token, same guards. The quoted opener must be tried before the bare-point pattern, whose \S+ would otherwise capture the opening quote and read '"June' as June. Only the double quote opens a value, matching the one existing quote-aware scanner; an unterminated quote is reported rather than raised, so one typo cannot fail a note's whole index. A refused token now names the fix when the line looks like a truncated date — a known kind and a following digit — and stays silent otherwise, so prose is never nagged. Naming: the effective/valid/occurred/due/mentioned family is now 'kind', because that is what it says — what kind of time this is. Date-vs-instant becomes 'axis', which is what the code already called it in prose. Both names now describe what they hold. The migration is edited in place rather than stacked, since SPEC-82 has not shipped. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- ...4t5e6m7p8o9_add_memory_time_index_table.py | 12 +- src/basic_memory/api/v2/utils.py | 8 +- src/basic_memory/man/man3/search-notes(3).md | 22 +- src/basic_memory/markdown/entity_parser.py | 11 +- src/basic_memory/markdown/schemas.py | 9 +- .../markdown/temporal_qualifier.py | 240 +++++++++--- src/basic_memory/mcp/clients/search.py | 4 +- src/basic_memory/mcp/tools/search.py | 69 ++-- src/basic_memory/models/knowledge.py | 16 +- .../memory_time_index_repository.py | 4 +- .../repository/temporal_filters.py | 12 +- src/basic_memory/schemas/search.py | 18 +- src/basic_memory/services/search_service.py | 20 +- src/basic_memory/temporal.py | 98 ++--- tests/api/v2/test_search_router_temporal.py | 8 +- .../test_relation_persistence_temporal.py | 22 +- tests/markdown/test_entity_parser.py | 2 +- tests/markdown/test_temporal_qualifier.py | 370 ++++++++++++++---- .../clients/test_search_client_temporal.py | 4 +- tests/mcp/test_tool_contracts.py | 2 +- tests/mcp/test_tool_search_temporal.py | 32 +- ...est_search_notes_multi_project_temporal.py | 4 +- .../test_memory_time_index_contract.py | 106 ++--- .../repository/test_vector_temporal_filter.py | 4 +- .../services/test_search_service_temporal.py | 26 +- tests/test_memory_time_index_migration.py | 22 +- tests/test_temporal.py | 108 ++--- 27 files changed, 806 insertions(+), 447 deletions(-) diff --git a/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py b/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py index 90844d3f4..b445520d1 100644 --- a/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py +++ b/src/basic_memory/alembic/versions/u4t5e6m7p8o9_add_memory_time_index_table.py @@ -43,7 +43,7 @@ def upgrade() -> None: are deliberately unindexed, and this table is *not* always driven by a full-text candidate set: a valid-time filter counts as criteria on its own (``SearchQuery.no_criteria``), so a temporal-only search scans the bound columns - for every row matching project + role + kind. That is an acceptable scan at + for every row matching project + kind + axis. That is an acceptable scan at expected sizes -- one row per authored qualifier, so thousands, not millions. If temporal-only queries ever become a hot path, the answer is a native PostgreSQL range column with a GiST index, not a btree over these text bounds. @@ -55,8 +55,8 @@ def upgrade() -> None: sa.Column("entity_id", sa.Integer(), nullable=False), sa.Column("source_type", sa.String(length=32), nullable=False), sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("time_role", sa.String(length=32), nullable=False), - sa.Column("range_kind", sa.String(length=16), nullable=False), + sa.Column("time_kind", sa.String(length=32), nullable=False), + sa.Column("range_axis", sa.String(length=16), nullable=False), sa.Column("lower_value", sa.String(length=32), nullable=True), sa.Column("upper_value", sa.String(length=32), nullable=True), sa.Column("lower_inclusive", sa.Boolean(), nullable=False), @@ -69,8 +69,8 @@ def upgrade() -> None: sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.CheckConstraint( - "range_kind IN ('date', 'instant')", - name="ck_memory_time_index_range_kind", + "range_axis IN ('date', 'instant')", + name="ck_memory_time_index_range_axis", ), sa.CheckConstraint( "NOT is_empty OR (lower_value IS NULL AND upper_value IS NULL)", @@ -85,7 +85,7 @@ def upgrade() -> None: op.create_index( "ix_memory_time_index_lookup", "memory_time_index", - ["project_id", "time_role", "range_kind", "source_type", "source_id"], + ["project_id", "time_kind", "range_axis", "source_type", "source_id"], unique=False, ) op.create_index( diff --git a/src/basic_memory/api/v2/utils.py b/src/basic_memory/api/v2/utils.py index a7c93b8b6..7ed3bfde1 100644 --- a/src/basic_memory/api/v2/utils.py +++ b/src/basic_memory/api/v2/utils.py @@ -24,7 +24,7 @@ ContextResultRow, ContextResult as ServiceContextResult, ) -from basic_memory.temporal import TemporalRange, TemporalRangeKind +from basic_memory.temporal import TemporalRange, TemporalRangeAxis class EntityBatchLookup(Protocol): @@ -247,7 +247,7 @@ def _temporal_result_metadata(row: MemoryTimeIndex) -> TemporalResultMetadata: plausible-looking interval. """ valid_during = TemporalRange( - kind=TemporalRangeKind(row.range_kind), + axis=TemporalRangeAxis(row.range_axis), lower=row.lower_value, upper=row.upper_value, lower_inclusive=row.lower_inclusive, @@ -255,9 +255,9 @@ def _temporal_result_metadata(row: MemoryTimeIndex) -> TemporalResultMetadata: is_empty=row.is_empty, ) return TemporalResultMetadata( - role=row.time_role, + kind=row.time_kind, valid_during=TemporalRangeValue( - kind=valid_during.kind.value, + axis=valid_during.axis.value, literal=str(valid_during), lower=valid_during.lower, upper=valid_during.upper, diff --git a/src/basic_memory/man/man3/search-notes(3).md b/src/basic_memory/man/man3/search-notes(3).md index 87930aa1c..41d81a8b5 100644 --- a/src/basic_memory/man/man3/search-notes(3).md +++ b/src/basic_memory/man/man3/search-notes(3).md @@ -26,7 +26,7 @@ search_notes(query=None, project=None, project_id=None, entity_types=None, categories=None, after_date=None, metadata_filters=None, tags=None, status=None, min_similarity=None, valid_at=None, valid_overlaps=None, - time_role=None) + time_kind=None) ``` CLI: @@ -53,14 +53,18 @@ rows), `categories` (observation categories, paired with `metadata_filters` — equality matches against arbitrary frontmatter fields, which is how the manual implements apropos (see [[Manpage]]). -**Valid time** (`valid_at`, `valid_overlaps`, `time_role`) queries what a note +**Valid time** (`valid_at`, `valid_overlaps`, `time_kind`) queries what a note *says was true*, not when it was last edited. Observations can carry a qualifier — a range, `- [decision] @effective[2026-06-10,2026-07-27) ...`, or a point, `- [decision] @effective:2026-07-27 ...` / `- [decision] @2026-07-27 ...` — and these filters match against that authored interval. A point means the span its precision covers: `@2026` is that year, `@2026-06` that month, -and `@2026-06-10` from that date onward. It is a separate axis from -`after_date`, which keeps filtering last-indexed time. Bounds follow +and `@2026-06-10` from that date onward. An unquoted point is one +whitespace-delimited token; a multi-word, relative, or month-only date goes in +double quotes, which end the token at the closing quote: +`@occurred:"June 10, 2026"`, `@occurred:"2 days ago"`, `@"June 2026"`. It is a +separate axis from `after_date`, which keeps filtering last-indexed time. +Bounds follow PostgreSQL range conventions, calendar dates and instants never convert into one another, and a source with no qualifier is excluded from any valid-time query. Because one note can carry several assertions that disagree, these @@ -86,8 +90,8 @@ matched. - **valid_overlaps** — range literal the authored range must overlap: `[2026-06-10,2026-07-27)`, `(,2026-07-27]`, `[2026-06-10,)`. Mutually exclusive with `valid_at` (aliases: `overlaps`, `valid_during`) -- **time_role** — valid-time axis: `effective`, `valid`, `occurred`, `due`, - or `mentioned`; usable on its own (aliases: `role`, `time_axis`) +- **time_kind** — kind of valid time: `effective`, `valid`, `occurred`, `due`, + or `mentioned`; usable on its own (alias: `kind`) - **search_all_projects** — opt-in cross-project search; ignored when `project`/`project_id` is given - **page**, **page_size** — pagination (aliases: `page_number`, `limit`, @@ -129,8 +133,10 @@ bm tool search-notes "conflict error" --project manual --page-size 2 - [gotcha] A valid-time filter excludes every source without a temporal qualifier — an undated note makes no claim about when it holds, so drop the filter to search dated and undated content together #valid-time - [gotcha] valid_at and valid_overlaps never mix calendar dates with instants: a date query matches only date ranges and an instant query only instant ranges, so `2026-07-27` and `2026-07-27T00:00:00Z` are different questions #valid-time - [gotcha] A timestamp written without an offset is read as UTC, in an authored qualifier and in a filter alike — same convention as every other naive datetime in Basic Memory #valid-time -- [gotcha] An authored token that does not read as a date is left as ordinary observation content with no warning; only an unknown role (`@asserted:2026-06-10`) is reported #valid-time -- [gotcha] An authored point is one whitespace-delimited token: `@occurred:2026-06-10`, `@occurred:03/04/2026` and `@occurred:yesterday` work, but a multi-word date like `@occurred:June 10, 2026` is left as content because nothing can tell where it ends #valid-time +- [gotcha] An authored token that does not read as a date is left as ordinary observation content with no warning; only a qualifier the author plainly meant is reported — an unknown kind (`@asserted:2026-06-10`), an unterminated quote, or a date the one-token rule truncated #valid-time +- [gotcha] An unquoted authored point is one whitespace-delimited token: `@occurred:2026-06-10`, `@occurred:03/04/2026` and `@occurred:yesterday` work, but a multi-word date like `@occurred:June 10, 2026` is left as content because nothing can tell where it ends #valid-time +- [gotcha] Double quotes lift the one-token rule and end the point at the closing quote, so `@occurred:"June 10, 2026"`, `@occurred:"2 days ago"` and `@"June 2026"` all file — inside quotes even a month-only or year-only date is taken, since the author delimited it #valid-time +- [gotcha] Only `"` opens a quoted point, never `'`, and an unterminated quote is reported rather than swallowing the rest of the line #valid-time - [gotcha] `@occurred:03/04/2026` resolves by the `date_order` setting (YMD/DMY read it as 3 April, MDY as 4 March); ISO dates are never re-guessed #valid-time ## SEE ALSO diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 3dd68f64d..d0caf6fa7 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -336,11 +336,12 @@ async def parse_markdown_content( parse(post.content) if parse_semantics else EntityContent(content=post.content) ) - # The parser reports exactly one thing: a qualifier that reads as time but names - # an unknown role. That never reaches the index, so this warning is how an author - # learns the line needs fixing. Text that simply is not a date is ordinary content - # and says nothing here. The typed `temporal_error` field carries the same message - # to programmatic callers; this layer adds the path. + # The parser reports only a qualifier the author plainly meant: an unknown kind, + # an unterminated quote, or a date the one-token rule truncated. None of those + # reach the index, so this warning is how an author learns the line needs fixing. + # Text that simply is not a date is ordinary content and says nothing here. The + # typed `temporal_error` field carries the same message to programmatic callers; + # this layer adds the path. # `as_posix()` rather than the Path itself: Basic Memory names files with # forward slashes everywhere (entity.file_path, permalinks, search rows), so a # Windows `WindowsPath` rendering `decisions\cache-layer.md` would print a path diff --git a/src/basic_memory/markdown/schemas.py b/src/basic_memory/markdown/schemas.py index ab8c38a64..4f9c30d71 100644 --- a/src/basic_memory/markdown/schemas.py +++ b/src/basic_memory/markdown/schemas.py @@ -19,10 +19,11 @@ class Observation(BaseModel): # Collection-shaped from day one: the MVP parses at most one qualifier per # observation, but carrying several later must not be a schema break (SPEC-82). temporal: List[TemporalAssertion] = [] - # Set for the one reported case: a qualifier that reads as time but names an - # unknown role. Its text stays in `content`, so nothing is dropped -- only the - # derived temporal projection is withheld until the author fixes the line. Text - # that simply is not a date sets nothing here; it is ordinary content. + # Set for the three reported cases: an unknown kind, an unterminated quote, and an + # unquoted point the one-token rule truncated. Its text stays in `content`, so + # nothing is dropped -- only the derived temporal projection is withheld until the + # author fixes the line. Text that simply is not a date sets nothing here; it is + # ordinary content. temporal_error: Optional[str] = None @override diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py index cb9ce3d5e..6334c3b46 100644 --- a/src/basic_memory/markdown/temporal_qualifier.py +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -1,25 +1,36 @@ """Peel SPEC-82 temporal qualifiers off observation content. An observation may carry one qualifier immediately after its category and before its -content. Two authored forms exist, and the role is optional in both: +content. Three authored forms exist, and the kind is optional in all of them: - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. - [decision] @effective:2026-07-27 The cache layer will use Memcached. + - [decision] @effective:"June 10, 2026" The cache layer will use Memcached. - [decision] @2026-07-27 The cache layer will use Memcached. -The bracket form carries a range literal and needs no separator, because no role name -can begin with `[` or `(`. The point form needs the `:` because a date can begin with a +The bracket form carries a range literal and needs no separator, because no kind name +can begin with `[` or `(`. The point forms need the `:` because a date can begin with a letter (`yesterday`), so nothing else would tell `@occurred:yesterday` from a handle. -**A point is one whitespace-delimited token**, in both forms. dateparser reads far more -than one token -- `June 10, 2026`, `2 days ago`, `2026-06-10 10:00 AM` all resolve, and +**An unquoted point is one whitespace-delimited token.** dateparser reads far more than +one token -- `June 10, 2026`, `2 days ago`, `2026-06-10 10:00 AM` all resolve, and `parse_authored_point` accepts them -- but nothing here can tell where such a date ends: dateparser also reads `June 10, 2026 The` and `2026-06-10 The`, so growing the token -until parsing fails would swallow the author's prose. A multi-word date therefore stays -content; `@occurred:2026-06-10` says the same thing in one token. - -That token boundary is also why two shapes that *do* parse are refused, so a truncated -read never becomes a plausible-looking assertion: +until parsing fails would swallow the author's prose. + +**A quoted point is exactly what the author put between the quotes**, which is how a +multi-word, relative, or month-only date is written: `@occurred:"June 10, 2026"`, +`@occurred:"2 days ago"`, `@"June 2026"`. The closing quote is the token boundary, so +whatever follows it is ordinary content, and a `\\"` inside the value does not end the +token. The scan mirrors `_split_predicate_items` in `mcp/tools/posix_tools.py`, down to +its rule that an unterminated quote is a typo to report rather than a boundary to guess +at -- scanning on to end of line would hand the author's prose to dateparser. Only the +double quote opens the form: an apostrophe is ordinary punctuation, and a scan looking +for its partner would turn `@note:'s` and its like into diagnostics. + +Inside quotes the author delimited the value, so there is nothing to truncate and +dateparser's reading is taken as written. An *unquoted* token is refused in two shapes +that do parse, so a truncated read never becomes a plausible-looking assertion: * **A short number.** dateparser reads `1` as January and `3.5` as March 5, but at the head of a line those are list markers and version numbers. A numeric point must be at @@ -34,9 +45,14 @@ silently.** Prose is full of `@` -- email addresses, handles, `@todo:` markers -- and warning about each one that is not a date would be noise, not help. -The single exception is an **unknown role**. `@asserted:2026-06-10` parses as time and -names an axis, so the author is plainly reaching for this feature and a short list of -valid roles makes the diagnostic actionable. +Three things are reported instead, because each one names its own fix: + +* an **unknown kind** (`@asserted:2026-06-10`) -- the payload parses as time and the + author is plainly reaching for this feature, so a short list of valid kinds helps; +* an **unterminated quote** -- the author opened the quoted form and mistyped; +* an unquoted point refused by the guards above **whose line continues with a digit** + (`@occurred:June 10, 2026 ...`) -- the one shape where the token rule silently costs + the author a date they clearly wrote, and the quoted form is what they wanted. A refused or unread qualifier is never peeled. Its text stays in the observation content, so the line indexes exactly as it does today and remains full-text searchable; @@ -51,41 +67,51 @@ TemporalAssertion, TemporalQualifierError, TemporalRange, - TimeRole, + TimeKind, parse_authored_point, parse_range_literal, ) -_ROLE_NAMES = frozenset(role.value for role in TimeRole) +_KIND_NAMES = frozenset(kind.value for kind in TimeKind) -# A point with no role is filed on the axis the feature is named for: the author said -# when the statement holds without narrowing *how* it holds. -DEFAULT_TIME_ROLE = TimeRole.VALID +# A point with no kind is filed as valid time, the kind this feature is named for: the +# author said when the statement holds without narrowing *how* it holds. +DEFAULT_TIME_KIND = TimeKind.VALID -_ROLE_PATTERN = r"[A-Za-z][A-Za-z0-9_]*" +_KIND_PATTERN = r"[A-Za-z][A-Za-z0-9_]*" -# `@[role]` glued to one balanced bracket group carrying a range literal's comma. The +# `@[kind]` glued to one balanced bracket group carrying a range literal's comma. The # lookahead stops `@effective[a,b)x` from half-matching, and the `^` anchor keeps # `paul@basicmemory.com` and mid-sentence `@handles` out entirely. -_RANGE_QUALIFIER = re.compile(rf"^@({_ROLE_PATTERN})?([\[(][^\[\]()]*,[^\[\]()]*[\])])(?=\s|$)") +_RANGE_QUALIFIER = re.compile(rf"^@({_KIND_PATTERN})?([\[(][^\[\]()]*,[^\[\]()]*[\])])(?=\s|$)") -# `@role:`. -_ROLE_POINT_QUALIFIER = re.compile(rf"^@({_ROLE_PATTERN}):(\S+)") +# `@[kind:]"` -- the opening of the quoted point. Only the quote is matched here; its +# partner is found by a scan, because a regex cannot honor `\"`. +_QUOTED_POINT_QUALIFIER = re.compile(rf'^@(?:({_KIND_PATTERN}):)?"') -# `@` -- the role-less point. Without a role there is nothing to +# `@kind:`. +_KIND_POINT_QUALIFIER = re.compile(rf"^@({_KIND_PATTERN}):(\S+)") + +# `@` -- the point with no kind. Without one there is nothing to # distinguish a word from a handle, so only digits open the form at all. _BARE_POINT_QUALIFIER = re.compile(r"^@(\d\S*)") +_QUOTE = '"' + # The width of a year, and the shortest numeric token worth reading as one. _MIN_NUMERIC_POINT_WIDTH = 4 +# Every diagnostic that a quote would have fixed shows the form rather than describing +# it, so the fix is one copyable edit away. +_QUOTED_EXAMPLE = "June 10, 2026" + @dataclass(frozen=True, slots=True) class ObservationTemporalParse: """What a qualifier scan found at the head of one observation's content. - Exactly three shapes exist: a peel (content shortened, one assertion, no error), an - unknown-role refusal (content untouched, no assertions, an error message), and no + Exactly three shapes exist: a peel (content shortened, one assertion, no error), a + refusal (content untouched, no assertions, an error message naming the fix), and no qualifier at all (content untouched, nothing found). """ @@ -110,10 +136,37 @@ class _ReadQualifier: token: str end: int - role_name: str | None + kind_name: str | None valid_during: TemporalRange +@dataclass(frozen=True, slots=True) +class _Refusal: + """A qualifier the author plainly meant, reported instead of silently kept.""" + + reason: str + + +@dataclass(frozen=True, slots=True) +class _PointToken: + """Where a point form ends, and the text handed to the date reader. + + `quoted` is what separates the two point forms once the boundary is found: the + author delimited a quoted value, so the truncation guards below have nothing to + guard against. + """ + + point: str + end: int + kind_name: str | None + quoted: bool + + +# What a scan of the head of one line can find: a qualifier, a reportable mistake, or +# nothing at all. +type _QualifierScan = _ReadQualifier | _Refusal | None + + def _read_range_qualifier(content: str) -> _ReadQualifier | None: """Match the bracket form and parse its literal, or report no usable qualifier.""" match = _RANGE_QUALIFIER.match(content) @@ -129,28 +182,109 @@ def _read_range_qualifier(content: str) -> _ReadQualifier | None: return _ReadQualifier(match.group(0), match.end(), match.group(1), valid_during) -def _names_a_deliberate_date(point: str, valid_during: TemporalRange) -> bool: - """Whether a one-token point is specific enough to be an assertion rather than prose. +def _scan_quoted_point(content: str, opened_at: int) -> tuple[str, int] | None: + """Read a quoted payload from `opened_at` to its closing quote. + + One pass with a backslash escape, the same scan `_split_predicate_items` uses for + find's predicate values: the delimiter rather than whitespace ends the token, and an + escaped quote belongs to the value. Returns the value and the index just past the + closing quote, or None when the quote never closed. + """ + value: list[str] = [] + escaped = False + for index in range(opened_at, len(content)): + char = content[index] + if escaped: + value.append(char) + escaped = False + elif char == "\\": + escaped = True + elif char == _QUOTE: + return "".join(value), index + 1 + else: + value.append(char) + return None + + +def _locate_point(content: str) -> _PointToken | _Refusal | None: + """Find a point form at the head of the line and delimit the date it carries.""" + quoted = _QUOTED_POINT_QUALIFIER.match(content) + if quoted is not None: + scanned = _scan_quoted_point(content, quoted.end()) + if scanned is None: + # Trigger: the author opened the quoted form and never closed it. + # Why: every other reading of the line is a guess -- taking the rest of it + # would hand prose to dateparser, and dropping the quote would put the + # truncation this form exists to prevent right back. + # Outcome: the line keeps its text and the author is told which keystroke + # is missing. + return _Refusal( + f"unterminated quote in temporal qualifier {quoted.group(0)!r}; " + f'close it, as {quoted.group(0)}{_QUOTED_EXAMPLE}"' + ) + point, end = scanned + return _PointToken(point=point, end=end, kind_name=quoted.group(1), quoted=True) + + named = _KIND_POINT_QUALIFIER.match(content) + bare = None if named is not None else _BARE_POINT_QUALIFIER.match(content) + match = named or bare + if match is None: + return None + return _PointToken( + point=match.group(2) if named is not None else match.group(1), + end=match.end(), + kind_name=match.group(1) if named is not None else None, + quoted=False, + ) + + +def _truncation_reason(point: str, valid_during: TemporalRange) -> str | None: + """Why an unquoted point is too coarse to file, or None when it names a day. - The two shapes refused here both parse, which is exactly why they need refusing -- - see the module docstring for what each one costs if it is read. + The two shapes named here both parse, which is exactly why they need refusing -- + see the module docstring for what each one costs if it is read. The wording is the + diagnostic's, so the reason a token was refused and the reason it *is* refused stay + the same sentence. A bounded span is how a coarse point announces itself: `parse_authored_point` closes a year or a month at its successor and leaves a day or a moment open, so `upper is None` *is* "this names a specific day". """ if point[0].isdigit(): - return len(point) >= _MIN_NUMERIC_POINT_WIDTH - return valid_during.upper is None + return None if len(point) >= _MIN_NUMERIC_POINT_WIDTH else "is narrower than a year" + return None if valid_during.upper is None else "names only a month or a year" + + +def _truncated_point_refusal(content: str, token: _PointToken, reason: str) -> _Refusal | None: + """Report a refused token that reads as the first word of a longer date. + + Trigger: a known (or omitted) kind, and content after the refused token starting + with a digit. + Why: `@occurred:June 10, 2026` is the one shape where the one-token rule silently + costs the author a date they clearly wrote, and the digit is the only signal that + the date kept going. Prose after the token (`@occurred:June the cat sat`) is just + prose, and an unknown kind (`@vol:2 3 pages`) is an ordinary `@word:` marker; + diagnosing either would fire all over an ordinary vault. + Outcome: one sentence naming the quoted form that files the whole date. Otherwise + the token stays ordinary content, silently, exactly as it did before quoting. + """ + if token.kind_name is not None and token.kind_name not in _KIND_NAMES: + return None + rest = content[token.end :].lstrip() + if not rest or not rest[0].isdigit(): + return None + prefix = content[: token.end - len(token.point)] + return _Refusal( + f"temporal point {content[: token.end]!r} {reason}; " + f'quote the whole date to file it, as {prefix}"{_QUOTED_EXAMPLE}"' + ) -def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _ReadQualifier | None: - """Match either point form and read its date, or report no usable qualifier.""" - roled = _ROLE_POINT_QUALIFIER.match(content) - bare = None if roled is not None else _BARE_POINT_QUALIFIER.match(content) - match = roled or bare - if match is None: - return None +def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _QualifierScan: + """Match any point form and read its date, or report why nothing was filed.""" + located = _locate_point(content) + if located is None or isinstance(located, _Refusal): + return located # Deferred, following utils.ensure_timezone_aware: the markdown parser is a # low-level module that many entrypoints import, and pulling the config stack in at @@ -160,12 +294,16 @@ def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _ReadQu from basic_memory.config import ConfigManager order = date_order if date_order is not None else ConfigManager().config.date_order - point = match.group(2) if roled is not None else match.group(1) - valid_during = parse_authored_point(point, date_order=order) - if valid_during is None or not _names_a_deliberate_date(point, valid_during): + valid_during = parse_authored_point(located.point, date_order=order) + if valid_during is None: return None - role_name = match.group(1) if roled is not None else None - return _ReadQualifier(match.group(0), match.end(), role_name, valid_during) + + # Quotes are the author's own delimiters, so a quoted value cannot be the truncated + # head of a longer date and the guards do not apply to it. + reason = None if located.quoted else _truncation_reason(located.point, valid_during) + if reason is not None: + return _truncated_point_refusal(content, located, reason) + return _ReadQualifier(content[: located.end], located.end, located.kind_name, valid_during) def parse_temporal_qualifier( @@ -178,13 +316,15 @@ def parse_temporal_qualifier( configured `date_order`; tests and callers that already hold the config pass it. """ read = _read_range_qualifier(content) or _read_point_qualifier(content, date_order) + if isinstance(read, _Refusal): + return _refuse(content, read.reason) if read is None: return _no_qualifier(content) - role_name = read.role_name - if role_name is not None and role_name not in _ROLE_NAMES: - known = ", ".join(sorted(_ROLE_NAMES)) - return _refuse(content, f"unknown temporal role {role_name!r} in {read.token!r} ({known})") + kind_name = read.kind_name + if kind_name is not None and kind_name not in _KIND_NAMES: + known = ", ".join(sorted(_KIND_NAMES)) + return _refuse(content, f"unknown temporal kind {kind_name!r} in {read.token!r} ({known})") remainder = content[read.end :].strip() if not remainder: @@ -193,7 +333,7 @@ def parse_temporal_qualifier( return _no_qualifier(content) assertion = TemporalAssertion( - time_role=TimeRole(role_name) if role_name is not None else DEFAULT_TIME_ROLE, + time_kind=TimeKind(kind_name) if kind_name is not None else DEFAULT_TIME_KIND, valid_during=read.valid_during, source_text=read.token, ) diff --git a/src/basic_memory/mcp/clients/search.py b/src/basic_memory/mcp/clients/search.py index e3bdca068..08886178f 100644 --- a/src/basic_memory/mcp/clients/search.py +++ b/src/basic_memory/mcp/clients/search.py @@ -16,7 +16,7 @@ # The valid-time fields SearchQuery carries. Named here so the skew check below stays # in step with the schema without importing the model's internals. -_TEMPORAL_QUERY_FIELDS = ("valid_at", "valid_overlaps", "time_role") +_TEMPORAL_QUERY_FIELDS = ("valid_at", "valid_overlaps", "time_kind") class SearchClient: @@ -106,7 +106,7 @@ async def search( "The search API did not apply the requested valid-time filter " "(no temporal_applied confirmation in the response). The server is " "likely older than this client; upgrade it or drop valid_at / " - "valid_overlaps / time_role from the query." + "valid_overlaps / time_kind from the query." ) return SearchResponse.model_validate(payload) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 9022d7af0..3720076df 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -395,13 +395,13 @@ def _format_search_markdown( parts.append(f"- score: {r.score:.4f}") if r.matched_chunk: parts.append(f"- match: {r.matched_chunk[:200]}") - # Name the axis and the units. A bare "2026-06-10" here would read as an edit + # Name the kind and the units. A bare "2026-06-10" here would read as an edit # date; "effective valid time ... (date)" says which time this is and that it # is a calendar date carrying no timezone. for assertion in r.temporal or []: parts.append( - f"- {assertion.role} valid time: {assertion.valid_during.literal} " - f"({assertion.valid_during.kind})" + f"- {assertion.kind} valid time: {assertion.valid_during.literal} " + f"({assertion.valid_during.axis})" ) parts.append("") @@ -585,7 +585,7 @@ async def _search_all_projects( min_similarity: float | None, valid_at: str | None, valid_overlaps: str | None, - time_role: str | None, + time_kind: str | None, context: Context | None, ) -> dict[str, Any] | str: """Search every accessible project when the caller explicitly opts in.""" @@ -595,7 +595,7 @@ async def _search_all_projects( # response that does not confirm the filter ran. So a project either honored the # valid-time filter or was dropped with a warning below; the merged answer never # silently mixes filtered and unfiltered rows. - temporal_requested = bool(valid_at or valid_overlaps or time_role) + temporal_requested = bool(valid_at or valid_overlaps or time_kind) project_refs = await _load_search_project_refs(context=context) if not project_refs: response = SearchResponse( @@ -655,7 +655,7 @@ async def _search_all_projects( min_similarity=min_similarity, valid_at=valid_at, valid_overlaps=valid_overlaps, - time_role=time_role, + time_kind=time_kind, search_all_projects=False, context=context, ) @@ -835,15 +835,15 @@ async def search_notes( "written PostgreSQL-style: '[2026-06-10,2026-07-27)', '(,2026-07-27]', " "'[2026-06-10,)'. Mutually exclusive with valid_at.", ] = None, - time_role: Annotated[ + time_kind: Annotated[ Optional[str], Field( default=None, - validation_alias=AliasChoices("time_role", "role", "time_axis"), + validation_alias=AliasChoices("time_kind", "kind"), ), - "Narrow valid-time matching to one authored axis: 'effective', 'valid', " - "'occurred', 'due', or 'mentioned'. Usable on its own to find every source " - "carrying an assertion on that axis.", + "Narrow valid-time matching to one authored kind of time: 'effective', " + "'valid', 'occurred', 'due', or 'mentioned'. Usable on its own to find every " + "source carrying an assertion of that kind.", ] = None, context: Context | None = None, ) -> dict[str, Any] | str: @@ -928,25 +928,34 @@ async def search_notes( - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. - [decision] @effective:2026-07-27 The cache layer will use Memcached. - The bracket form is an explicit range; the `@role:date` form is a point, meaning + The bracket form is an explicit range; the `@kind:date` form is a point, meaning the span its precision covers — `@2026` that year, `@2026-06` that month, and - `@2026-06-10` from that date onward. The role may be omitted (`@2026-07-27`), - which files the assertion on the `valid` axis; a role-less point has to start + `@2026-06-10` from that date onward. The kind may be omitted (`@2026-07-27`), + which files the assertion as `valid` time; a point with no kind has to start with a digit and be at least as wide as a year, so `@v2` and `@may` stay prose. - A point is **one whitespace-delimited token**. Slash dates (`@occurred:03/04/2026`, - read by the `date_order` setting) and single-word relative dates - (`@occurred:yesterday`) work; multi-word dates like `@occurred:June 10, 2026` do - not, because nothing can tell where such a date ends — write `@occurred:2026-06-10` - instead. An unreadable token is left as ordinary content, never half-read. + An unquoted point is **one whitespace-delimited token**, because nothing can tell + where a multi-word date ends. Slash dates (`@occurred:03/04/2026`, read by the + `date_order` setting) and single-word relative dates (`@occurred:yesterday`) work + as they are; anything longer goes in double quotes, which move the token boundary + to the closing quote: + + - [decision] @occurred:"June 10, 2026" The cutover ran. + - [decision] @occurred:"2 days ago" The cutover ran. + - [decision] @occurred:"June 2026" The cutover ran. + - [decision] @"June 10, 2026" The cutover ran. + + Whatever is inside the quotes is read as the date, month-only and relative forms + included, and whatever follows the closing quote is ordinary content. An unreadable + token is left as content, never half-read. These filters query that authored time, which is a different axis from `after_date` (last-indexed time) — `after_date` is never reinterpreted as valid time. - - `search_notes("cache layer", role="effective", valid_at="2026-07-28")` + - `search_notes("cache layer", kind="effective", valid_at="2026-07-28")` - Returns the Memcached decision; the Redis decision expired at the cutover. - - `search_notes("cache layer", role="effective", valid_at="2026-07-01")` + - `search_notes("cache layer", kind="effective", valid_at="2026-07-01")` - Returns the Redis decision; Memcached is not yet effective. - - `search_notes("cache layer", role="effective", valid_overlaps="[2026-06-01,2026-08-01)")` + - `search_notes("cache layer", kind="effective", valid_overlaps="[2026-06-01,2026-08-01)")` - Returns both, since each overlaps that window. - `search_notes("cache layer")` with no valid-time filter - Both compete under ordinary relevance, exactly as before. @@ -1012,8 +1021,8 @@ async def search_notes( valid_overlaps: Optional PostgreSQL-style range literal ("[2026-06-10,2026-07-27)", "(,2026-07-27]", "[2026-06-10,)"). Returns sources whose authored valid range overlaps it. Mutually exclusive with valid_at; also excludes undated sources. - time_role: Optional valid-time axis to narrow to: "effective", "valid", "occurred", - "due", or "mentioned". Valid on its own. + time_kind: Optional kind of valid time to narrow to: "effective", "valid", + "occurred", "due", or "mentioned". Valid on its own. context: Optional FastMCP context for performance caching. Returns: @@ -1183,7 +1192,7 @@ async def search_notes( min_similarity=min_similarity, valid_at=valid_at, valid_overlaps=valid_overlaps, - time_role=time_role, + time_kind=time_kind, context=context, ) return all_projects_result @@ -1213,11 +1222,11 @@ async def search_notes( or after_date or valid_at or valid_overlaps - or time_role + or time_kind ), has_tags_filter=bool(tags), has_status_filter=bool(status), - has_temporal_filter=bool(valid_at or valid_overlaps or time_role), + has_temporal_filter=bool(valid_at or valid_overlaps or time_kind), ): async with get_project_client(project, context=context, project_id=project_id) as ( client, @@ -1302,8 +1311,8 @@ async def search_notes( search_query.valid_at = valid_at if valid_overlaps: search_query.valid_overlaps = valid_overlaps - if time_role: - search_query.time_role = time_role + if time_kind: + search_query.time_kind = time_kind # Reject searches with no criteria at all if search_query.no_criteria(): @@ -1311,7 +1320,7 @@ async def search_notes( "# No Search Criteria\n\n" "Please provide at least one of: `query`, `metadata_filters`, " "`tags`, `status`, `note_types`, `entity_types`, `categories`, " - "`after_date`, `valid_at`, `valid_overlaps`, or `time_role`." + "`after_date`, `valid_at`, `valid_overlaps`, or `time_kind`." ) # Default to entity-level results to avoid returning individual diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 399127b24..49b12584d 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -434,22 +434,22 @@ class MemoryTimeIndex(Base): __tablename__ = "memory_time_index" __table_args__ = ( # The valid-time predicate selects (source_type, source_id) after filtering on - # project, role, and axis, so this index both drives the scan and covers its + # project, kind, and axis, so this index both drives the scan and covers its # projection. project_id leads it, which is why the column carries no separate # index of its own the way sibling projection tables do. Index( "ix_memory_time_index_lookup", "project_id", - "time_role", - "range_kind", + "time_kind", + "range_axis", "source_type", "source_id", ), # Fenced replace deletes by entity_id, and the cascade follows the same column. Index("ix_memory_time_index_entity_id", "entity_id"), CheckConstraint( - "range_kind IN ('date', 'instant')", - name="ck_memory_time_index_range_kind", + "range_axis IN ('date', 'instant')", + name="ck_memory_time_index_range_axis", ), # The empty range has no endpoints at all; representing it with bounds would # make two rows describe the same interval two different ways. @@ -474,8 +474,8 @@ class MemoryTimeIndex(Base): # (type, id) pair. No FK: the target table varies with source_type. source_type: Mapped[str] = mapped_column(String(32)) source_id: Mapped[int] = mapped_column(Integer) - time_role: Mapped[str] = mapped_column(String(32)) - range_kind: Mapped[str] = mapped_column(String(16)) + time_kind: Mapped[str] = mapped_column(String(32)) + range_axis: Mapped[str] = mapped_column(String(16)) # Canonical lexical bounds; NULL means unbounded on that side. lower_value: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) upper_value: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) @@ -496,7 +496,7 @@ class MemoryTimeIndex(Base): def __repr__(self) -> str: # pragma: no cover return ( f"MemoryTimeIndex(id={self.id}, entity_id={self.entity_id}, " - f"source={self.source_type}:{self.source_id}, role='{self.time_role}', " + f"source={self.source_type}:{self.source_id}, kind='{self.time_kind}', " f"range='{self.source_text}')" ) diff --git a/src/basic_memory/repository/memory_time_index_repository.py b/src/basic_memory/repository/memory_time_index_repository.py index a458bfb22..4e359a1d1 100644 --- a/src/basic_memory/repository/memory_time_index_repository.py +++ b/src/basic_memory/repository/memory_time_index_repository.py @@ -46,8 +46,8 @@ def _projection_row( entity_id=entity_id, source_type=accepted.source_type, source_id=accepted.source_id, - time_role=accepted.assertion.time_role.value, - range_kind=valid_during.kind.value, + time_kind=accepted.assertion.time_kind.value, + range_axis=valid_during.axis.value, lower_value=valid_during.lower, upper_value=valid_during.upper, lower_inclusive=valid_during.lower_inclusive, diff --git a/src/basic_memory/repository/temporal_filters.py b/src/basic_memory/repository/temporal_filters.py index 6219bb504..81c3c666c 100644 --- a/src/basic_memory/repository/temporal_filters.py +++ b/src/basic_memory/repository/temporal_filters.py @@ -8,7 +8,7 @@ typed date bind is needed on either side. * Inclusivity on the query side is known while the SQL is being built, and inclusivity on the stored side is a boolean column, so both fold into the SQL text. - The only bound parameters are the two bound values, the role, and the axis -- each + The only bound parameters are the two bound values, the kind, and the axis -- each compared directly against a column, so PostgreSQL always infers their type and asyncpg never sees a bare untyped parameter. @@ -107,17 +107,17 @@ def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) - conditions = [f"{TEMPORAL_INDEX_TABLE}.project_id = :project_id"] - if temporal.role is not None: - params["tq_role"] = temporal.role.value - conditions.append(f"{TEMPORAL_INDEX_TABLE}.time_role = :tq_role") + if temporal.kind is not None: + params["tq_kind"] = temporal.kind.value + conditions.append(f"{TEMPORAL_INDEX_TABLE}.time_kind = :tq_kind") if window is not None: # Trigger: the caller asked about a specific date or a specific instant. # Why: calendar dates and instants are different axes; converting between # them would invent a timezone or a time of day the author never wrote. # Outcome: a date query can never match an instant range, or the reverse. - params["tq_kind"] = window.kind.value - conditions.append(f"{TEMPORAL_INDEX_TABLE}.range_kind = :tq_kind") + params["tq_axis"] = window.axis.value + conditions.append(f"{TEMPORAL_INDEX_TABLE}.range_axis = :tq_axis") # The empty stored range contains no points, so it overlaps nothing. conditions.append(f"NOT {TEMPORAL_INDEX_TABLE}.is_empty") diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index e023582b7..4ef4120a1 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -77,7 +77,7 @@ class SearchQuery(BaseModel): - file_path_prefix: Limit to one directory subtree of the project - tags: Convenience frontmatter tag filter - status: Convenience frontmatter status filter - - valid_at / valid_overlaps / time_role: Authored valid-time filters (SPEC-82) + - valid_at / valid_overlaps / time_kind: Authored valid-time filters (SPEC-82) Valid time is what a note *says about the world*, written as a qualifier on an observation (``- [decision] @effective[2026-06-10,2026-07-27) ...``). It is a @@ -117,7 +117,7 @@ class SearchQuery(BaseModel): # domain values and rejects anything malformed with a visible diagnostic. valid_at: Optional[str] = None # Date or RFC 3339 instant the range must contain valid_overlaps: Optional[str] = None # Range literal, e.g. "[2026-06-10,2026-07-27)" - time_role: Optional[str] = None # effective | valid | occurred | due | mentioned + time_kind: Optional[str] = None # effective | valid | occurred | due | mentioned @model_validator(mode="after") def validate_temporal_filter(self) -> "SearchQuery": @@ -158,11 +158,11 @@ def normalize_scope(cls, value: Optional[str]) -> Optional[str]: def has_temporal_filter(self) -> bool: """Whether this query asks a valid-time question at all. - A role on its own is a legal filter: it asks for sources carrying any - assertion on that axis. Callers use this to decide whether valid time was + A kind on its own is a legal filter: it asks for sources carrying any + assertion of that kind. Callers use this to decide whether valid time was requested without parsing the values, which is why it never raises. """ - return bool(self.valid_at or self.valid_overlaps or self.time_role) + return bool(self.valid_at or self.valid_overlaps or self.time_kind) def no_criteria(self) -> bool: text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip()) @@ -215,7 +215,7 @@ class TemporalRangeValue(BaseModel): enclosing `TemporalResultMetadata` is where the author's own spelling survives. """ - kind: str # "date" (calendar dates) or "instant" (UTC timestamps) + axis: str # "date" (calendar dates) or "instant" (UTC timestamps) literal: str # e.g. "[2026-06-10,2026-07-28)", "(,2026-07-27)", "empty" lower: Optional[str] = None # None means unbounded on that side upper: Optional[str] = None @@ -228,10 +228,10 @@ class TemporalResultMetadata(BaseModel): """One authored valid-time assertion carried by a search result. Present so an agent can say *why* a source matched a valid-time query -- which - axis it was asserted on, over what interval, and in the author's own words. + kind of time it asserts, over what interval, and in the author's own words. """ - role: str # effective | valid | occurred | due | mentioned + kind: str # effective | valid | occurred | due | mentioned valid_during: TemporalRangeValue source_text: str # the qualifier exactly as authored, e.g. "@effective[2026-06-10,)" @@ -268,7 +268,7 @@ class SearchResult(BaseModel): # Authored valid-time assertions carried by this row. Collection-shaped from day # one: the MVP parser reads one qualifier per observation, but multiple assertions - # on multiple axes must not be a schema break later. + # of multiple kinds must not be a schema break later. temporal: Optional[List[TemporalResultMetadata]] = None diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index acdaf920e..5ad7d65a3 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -35,7 +35,7 @@ from basic_memory.temporal import ( TemporalFilter, TemporalQualifierError, - TimeRole, + TimeKind, parse_point, parse_range_literal, ) @@ -96,7 +96,7 @@ def build_temporal_filter(query: SearchQuery) -> TemporalFilter | None: """Parse the flat valid-time fields into one portable filter value. The boundary carries strings so HTTP and MCP callers can pass a single flat value - per axis. Every rejection here is deliberate and loud: an unknown role, a malformed + per field. Every rejection here is deliberate and loud: an unknown kind, a malformed range literal, a range mixing calendar dates with instants, or an impossible range raises rather than degrading into a filter that quietly matches something else. Callers above map the error to a 400. A timestamp written without an offset is not @@ -105,18 +105,18 @@ def build_temporal_filter(query: SearchQuery) -> TemporalFilter | None: if not query.has_temporal_filter(): return None - role: TimeRole | None = None - if query.time_role: + kind: TimeKind | None = None + if query.time_kind: try: - role = TimeRole(query.time_role) + kind = TimeKind(query.time_kind) except ValueError as exc: raise TemporalQualifierError( - f"unknown time_role {query.time_role!r}; expected one of " - f"{', '.join(item.value for item in TimeRole)}" + f"unknown time_kind {query.time_kind!r}; expected one of " + f"{', '.join(item.value for item in TimeKind)}" ) from exc return TemporalFilter( - role=role, + kind=kind, at=parse_point(query.valid_at) if query.valid_at else None, overlaps=parse_range_literal(query.valid_overlaps) if query.valid_overlaps else None, ) @@ -127,8 +127,8 @@ def _describe_temporal_criteria(temporal: TemporalFilter | None) -> str | None: if temporal is None: return None parts = [] - if temporal.role is not None: - parts.append(f"role={temporal.role.value}") + if temporal.kind is not None: + parts.append(f"kind={temporal.kind.value}") if temporal.at is not None: parts.append(f"valid_at={temporal.at.value}") elif temporal.overlaps is not None: diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index bb222e953..fa9a1ed45 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -22,7 +22,7 @@ is chronological order. That is what lets containment and overlap be plain string comparisons with identical SQL text in either dialect. -The two kinds never mix and never convert into one another. A date bound is a calendar +The two axes never mix and never convert into one another. A date bound is a calendar date: it acquires no time of day and no timezone, ever. An instant bound names a moment and is normalized to UTC, so two instants written in different offsets compare as the instants they name. A timestamp written without an offset is *read as UTC*, which is @@ -54,8 +54,8 @@ class TemporalQualifierError(ValueError): """A temporal qualifier, range literal, or bound failed to parse or validate.""" -class TimeRole(StrEnum): - """Which time axis an assertion describes. +class TimeKind(StrEnum): + """Which kind of time an assertion describes. `recorded` is deliberately absent: recorded time is never authored in markdown. """ @@ -67,7 +67,7 @@ class TimeRole(StrEnum): MENTIONED = "mentioned" -class TemporalRangeKind(StrEnum): +class TemporalRangeAxis(StrEnum): """Whether a range is measured in calendar dates or in instants.""" DATE = "date" @@ -110,11 +110,11 @@ class TemporalRangeKind(StrEnum): _RANGE_LITERAL = re.compile(r"^([\[(])([^,\[\]()]*),([^,\[\]()]*)([\])])$") -def _classify_bound(bound: str) -> TemporalRangeKind: +def _classify_bound(bound: str) -> TemporalRangeAxis: """Decide which axis an authored bound is written on.""" if _TIMESTAMP_SHAPE.match(bound): - return TemporalRangeKind.INSTANT - return TemporalRangeKind.DATE + return TemporalRangeAxis.INSTANT + return TemporalRangeAxis.DATE def _canonical_date(bound: str) -> str: @@ -154,18 +154,18 @@ def _canonical_instant(bound: str) -> str: return _instant_value(moment) -def canonical_bound(bound: str, kind: TemporalRangeKind) -> str: - """Normalize one authored bound to the canonical fixed-width form for its kind.""" - if kind is TemporalRangeKind.DATE: +def canonical_bound(bound: str, axis: TemporalRangeAxis) -> str: + """Normalize one authored bound to the canonical fixed-width form for its axis.""" + if axis is TemporalRangeAxis.DATE: return _canonical_date(bound) return _canonical_instant(bound) -def _require_canonical(value: str, kind: TemporalRangeKind) -> None: +def _require_canonical(value: str, axis: TemporalRangeAxis) -> None: """Reject a value that skipped `canonical_bound` on its way into a domain value.""" - pattern = _DATE_BOUND if kind is TemporalRangeKind.DATE else _CANONICAL_INSTANT + pattern = _DATE_BOUND if axis is TemporalRangeAxis.DATE else _CANONICAL_INSTANT if not pattern.match(value): - raise TemporalQualifierError(f"{kind.value} bound is not canonical: {value!r}") + raise TemporalQualifierError(f"{axis.value} bound is not canonical: {value!r}") def _next_calendar_day(bound: str) -> str | None: @@ -188,11 +188,11 @@ def _next_calendar_day(bound: str) -> str | None: class TemporalPoint: """One calendar date or instant that a containment question is asked about.""" - kind: TemporalRangeKind + axis: TemporalRangeAxis value: str def __post_init__(self) -> None: - _require_canonical(self.value, self.kind) + _require_canonical(self.value, self.axis) @override def __str__(self) -> str: @@ -228,7 +228,7 @@ class TemporalRange: re-parsing that rendering yields this same value. """ - kind: TemporalRangeKind + axis: TemporalRangeAxis lower: str | None = None upper: str | None = None lower_inclusive: bool = False @@ -250,7 +250,7 @@ def __post_init__(self) -> None: for bound in (self.lower, self.upper): if bound is not None: - _require_canonical(bound, self.kind) + _require_canonical(bound, self.axis) # Canonical bounds are fixed width, so string order is chronological order. # Judged on the bounds as authored: an interval written backwards is an author @@ -270,7 +270,7 @@ def __post_init__(self) -> None: # # Rewrite a date range to `[lower,upper)`. See the class docstring for why the # scalar overlap predicate needs this and why instants must not get it. - if self.kind is TemporalRangeKind.DATE: + if self.axis is TemporalRangeAxis.DATE: if self.lower is not None and not self.lower_inclusive: after_lower = _next_calendar_day(self.lower) if after_lower is None: @@ -306,9 +306,9 @@ def _become_empty(self) -> None: object.__setattr__(self, "is_empty", True) @classmethod - def empty(cls, kind: TemporalRangeKind) -> "TemporalRange": + def empty(cls, axis: TemporalRangeAxis) -> "TemporalRange": """The empty range on one axis.""" - return cls(kind=kind, is_empty=True) + return cls(axis=axis, is_empty=True) @override def __str__(self) -> str: @@ -333,12 +333,12 @@ class TemporalFilter: """A valid-time question asked of the stored assertions. Exactly one of `at` (containment) or `overlaps` may be given, or neither -- a - role-only filter asks for sources that carry *any* assertion on that axis, which + kind-only filter asks for sources that carry *any* assertion of that kind, which is a legal and useful question. A filter that asks nothing at all is refused rather than silently matching everything. """ - role: TimeRole | None = None + kind: TimeKind | None = None at: TemporalPoint | None = None overlaps: TemporalRange | None = None @@ -347,12 +347,12 @@ def __post_init__(self) -> None: raise TemporalQualifierError( "a temporal filter asks either 'at' or 'overlaps', never both" ) - if self.role is None and self.at is None and self.overlaps is None: - raise TemporalQualifierError("a temporal filter must name a role, a point, or a range") + if self.kind is None and self.at is None and self.overlaps is None: + raise TemporalQualifierError("a temporal filter must name a kind, a point, or a range") @property def window(self) -> TemporalRange | None: - """The interval this filter tests against, or None for a role-only filter. + """The interval this filter tests against, or None for a kind-only filter. Containment of a point is overlap with the closed range `[p,p]`: both ask whether the stored interval and the queried interval share at least one point. @@ -363,7 +363,7 @@ def window(self) -> TemporalRange | None: """ if self.at is not None: return TemporalRange( - kind=self.at.kind, + axis=self.at.axis, lower=self.at.value, upper=self.at.value, lower_inclusive=True, @@ -385,7 +385,7 @@ class TemporalAssertion: `valid_during` holds the normalized form. """ - time_role: TimeRole + time_kind: TimeKind valid_during: TemporalRange source_text: str extractor: str = OBSERVATION_EXTRACTOR @@ -395,20 +395,20 @@ class TemporalAssertion: # --- Literal parsing --- -def parse_range_literal(literal: str, *, kind: TemporalRangeKind | None = None) -> TemporalRange: +def parse_range_literal(literal: str, *, axis: TemporalRangeAxis | None = None) -> TemporalRange: """Parse a PostgreSQL-style range literal into a canonical `TemporalRange`. Accepts `[lower,upper)`, `(lower,upper]`, `[lower,)`, `(,upper)`, `(,)`, and the - bare token `empty`. `kind` asserts the expected axis; when omitted the axis is + bare token `empty`. `axis` asserts the axis the caller expects; when omitted it is inferred from the bounds, which is why the bound-less forms require it explicitly. """ text = literal.strip() if text == EMPTY_RANGE_LITERAL: - if kind is None: + if axis is None: raise TemporalQualifierError( - "the 'empty' range literal has no bounds, so its kind must be given" + "the 'empty' range literal has no bounds, so its axis must be given" ) - return TemporalRange.empty(kind) + return TemporalRange.empty(axis) match = _RANGE_LITERAL.match(text) if match is None: @@ -419,28 +419,28 @@ def parse_range_literal(literal: str, *, kind: TemporalRangeKind | None = None) lower_text = lower_text.strip() upper_text = upper_text.strip() - written_kinds = {_classify_bound(bound) for bound in (lower_text, upper_text) if bound} - if len(written_kinds) > 1: + written_axes = {_classify_bound(bound) for bound in (lower_text, upper_text) if bound} + if len(written_axes) > 1: raise TemporalQualifierError( f"a range must not mix date-only and timestamp bounds: {literal!r}" ) - if not written_kinds: - if kind is None: + if not written_axes: + if axis is None: raise TemporalQualifierError( f"a fully unbounded range has no bounds to classify: {literal!r}" ) - range_kind = kind + range_axis = axis else: - range_kind = written_kinds.pop() - if kind is not None and range_kind is not kind: + range_axis = written_axes.pop() + if axis is not None and range_axis is not axis: raise TemporalQualifierError( - f"expected {kind.value} bounds but found {range_kind.value} bounds: {literal!r}" + f"expected {axis.value} bounds but found {range_axis.value} bounds: {literal!r}" ) return TemporalRange( - kind=range_kind, - lower=canonical_bound(lower_text, range_kind) if lower_text else None, - upper=canonical_bound(upper_text, range_kind) if upper_text else None, + axis=range_axis, + lower=canonical_bound(lower_text, range_axis) if lower_text else None, + upper=canonical_bound(upper_text, range_axis) if upper_text else None, lower_inclusive=open_bracket == "[", upper_inclusive=close_bracket == "]", ) @@ -451,8 +451,8 @@ def parse_point(text: str) -> TemporalPoint: bound = text.strip() if not bound: raise TemporalQualifierError("a temporal point must not be empty") - kind = _classify_bound(bound) - return TemporalPoint(kind=kind, value=canonical_bound(bound, kind)) + axis = _classify_bound(bound) + return TemporalPoint(axis=axis, value=canonical_bound(bound, axis)) # --- Flexible authored points --- @@ -481,7 +481,7 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": def _calendar_span(lower: date, upper: date) -> TemporalRange: """The half-open calendar period `[lower,upper)`.""" return TemporalRange( - kind=TemporalRangeKind.DATE, + axis=TemporalRangeAxis.DATE, lower=lower.isoformat(), upper=upper.isoformat(), lower_inclusive=True, @@ -517,7 +517,7 @@ def parse_authored_point( # Outcome: ISO dates are parsed as ISO, or refused. try: return TemporalRange( - kind=TemporalRangeKind.DATE, + axis=TemporalRangeAxis.DATE, lower=date.fromisoformat(point).isoformat(), lower_inclusive=True, ) @@ -534,7 +534,7 @@ def parse_authored_point( match date_data.period: case "time": return TemporalRange( - kind=TemporalRangeKind.INSTANT, + axis=TemporalRangeAxis.INSTANT, lower=_instant_value(moment), lower_inclusive=True, ) @@ -555,7 +555,7 @@ def parse_authored_point( # Day precision, and any coarser calendar period dateparser resolves to a # specific day ("last week"): the day it named, onward. return TemporalRange( - kind=TemporalRangeKind.DATE, + axis=TemporalRangeAxis.DATE, lower=moment.date().isoformat(), lower_inclusive=True, ) diff --git a/tests/api/v2/test_search_router_temporal.py b/tests/api/v2/test_search_router_temporal.py index a85d2b38b..a49f83af0 100644 --- a/tests/api/v2/test_search_router_temporal.py +++ b/tests/api/v2/test_search_router_temporal.py @@ -67,7 +67,7 @@ async def test_temporal_filter_round_trips_through_v2_search( v2_project_url, text="cache layer", entity_types=["observation"], - time_role="effective", + time_kind="effective", valid_at="2026-07-28", ) @@ -78,10 +78,10 @@ async def test_temporal_filter_round_trips_through_v2_search( [result] = payload["results"] [assertion] = result["temporal"] - assert assertion["role"] == "effective" + assert assertion["kind"] == "effective" assert assertion["source_text"] == "@effective[2026-07-27,)" assert assertion["valid_during"] == { - "kind": "date", + "axis": "date", "literal": "[2026-07-27,)", "lower": "2026-07-27", "upper": None, @@ -249,7 +249,7 @@ async def test_temporal_only_query_is_accepted_as_criteria( await _index_note(entity_service, search_service, "Cache Layer", CACHE_LAYER_MARKDOWN) payload = await _search( - client, v2_project_url, entity_types=["observation"], time_role="effective" + client, v2_project_url, entity_types=["observation"], time_kind="effective" ) assert payload["temporal_applied"] is True diff --git a/tests/indexing/test_relation_persistence_temporal.py b/tests/indexing/test_relation_persistence_temporal.py index 15b26c4df..5590d54c2 100644 --- a/tests/indexing/test_relation_persistence_temporal.py +++ b/tests/indexing/test_relation_persistence_temporal.py @@ -37,29 +37,29 @@ from basic_memory.schemas.search import SearchItemType from basic_memory.temporal import ( TemporalAssertion, - TemporalRangeKind, - TimeRole, + TemporalRangeAxis, + TimeKind, parse_range_literal, ) -DATE = TemporalRangeKind.DATE +DATE = TemporalRangeAxis.DATE -def _assertion(literal: str, role: TimeRole = TimeRole.EFFECTIVE) -> TemporalAssertion: +def _assertion(literal: str, kind: TimeKind = TimeKind.EFFECTIVE) -> TemporalAssertion: return TemporalAssertion( - time_role=role, - valid_during=parse_range_literal(literal, kind=DATE), - source_text=f"@{role.value}{literal}", + time_kind=kind, + valid_during=parse_range_literal(literal, axis=DATE), + source_text=f"@{kind.value}{literal}", ) def _accepted( - source_id: int, literal: str, role: TimeRole = TimeRole.EFFECTIVE + source_id: int, literal: str, kind: TimeKind = TimeKind.EFFECTIVE ) -> AcceptedTemporalAssertion: return AcceptedTemporalAssertion( source_type=SearchItemType.OBSERVATION.value, source_id=source_id, - assertion=_assertion(literal, role), + assertion=_assertion(literal, kind), ) @@ -111,7 +111,7 @@ async def test_temporal_projection_replaced_under_the_generation_fence( generation=3, assertions=[ _accepted(2, "[2026-07-27,)"), - _accepted(3, "[2026-01-01,2026-06-10)", TimeRole.DUE), + _accepted(3, "[2026-01-01,2026-06-10)", TimeKind.DUE), ], ) @@ -119,7 +119,7 @@ async def test_temporal_projection_replaced_under_the_generation_fence( async with db.scoped_session(session_maker) as session: rows = await repository.find_by_entity(session, sample_entity.id) - assert [(row.source_id, row.time_role) for row in rows] == [(2, "effective"), (3, "due")] + assert [(row.source_id, row.time_kind) for row in rows] == [(2, "effective"), (3, "due")] assert rows[0].lower_value == "2026-07-27" assert rows[0].upper_value is None diff --git a/tests/markdown/test_entity_parser.py b/tests/markdown/test_entity_parser.py index e932e654f..b37632978 100644 --- a/tests/markdown/test_entity_parser.py +++ b/tests/markdown/test_entity_parser.py @@ -454,7 +454,7 @@ async def test_malformed_qualifier_logs_diagnostic_with_file_path(entity_parser) [observation] = entity.observations assert observation.temporal == [] - assert "unknown temporal role 'asserted'" in (observation.temporal_error or "") + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") # The qualifier is never dropped from the content, only from the projection. assert observation.content.startswith("@asserted[2026-06-10,)") diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index b843e37f8..bd140d854 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -2,14 +2,15 @@ Three rules shape every test here: -* **One grammar.** `@[role]` for a precise interval, - `@[role:]` for a point. The role is optional in both; a role-less point must - begin with a digit. +* **One grammar.** `@[kind]` for a precise interval, `@[kind:]` + for an unquoted point, and `@[kind:]""` for a quoted one. The kind is optional + in all three; an unquoted point that omits it must begin with a digit. * **Silent when it is not time.** If the payload does not read as a date, the token is ordinary content and nothing is reported. Prose is full of `@`, and diagnosing every one of them would be noise. -* **One diagnostic.** A payload that *does* read as time but names an unknown role is - reported, because a short list of valid roles makes that actionable. +* **Diagnostics only where the author plainly meant a qualifier.** An unknown kind, an + unterminated quote, and an unquoted date the one-token rule truncated are each + reported, because each one has a fix the message can name. And in every case a qualifier that was not accepted is **never dropped**: its text stays in the observation content, so the line indexes and round-trips exactly as it did @@ -25,7 +26,7 @@ from basic_memory.markdown.entity_parser import parse from basic_memory.markdown.schemas import Observation from basic_memory.markdown.temporal_qualifier import parse_temporal_qualifier -from basic_memory.temporal import DateOrder, TemporalRangeKind, TimeRole +from basic_memory.temporal import DateOrder, TemporalRangeAxis, TimeKind @pytest.fixture(autouse=True) @@ -48,6 +49,14 @@ def _observation(line: str) -> Observation: return observation +def _refusal(line: str) -> Observation: + """Parse a line whose qualifier must be refused, and assert the shared contract.""" + observation = _observation(line) + assert observation.temporal == [] + assert observation.temporal_error is not None + return observation + + # --- Acceptance 1: undated notes are untouched --- @@ -64,7 +73,7 @@ def test_observation_without_qualifier_parses_byte_identically(): assert str(observation) == "- [decision] The cache layer will use Redis. #infra (agreed)" -# --- Acceptance 2: round trip preserves role and bounds --- +# --- Acceptance 2: round trip preserves kind and bounds --- @pytest.mark.parametrize( @@ -82,7 +91,7 @@ def test_observation_without_qualifier_parses_byte_identically(): "@due[2026-07-27T18:42:00+02:00,)", "@mentioned[2026-07-27T18:42:00.123456Z,2026-07-28T00:00:00Z)", "@[2026-06-10,2026-07-27)", - # The point: the convenient form, with and without a role. + # The point: the convenient form, with and without a kind. "@effective:2026-07-27", "@occurred:2026-07-27T18:42:00Z", "@due:2026-07", @@ -111,15 +120,15 @@ def test_qualifier_round_trips_verbatim(qualifier: str): assert str(observation) == line -def test_qualifier_carries_its_role_and_bounds(): - """The parsed assertion is the interval the author wrote, on the axis they named.""" +def test_qualifier_carries_its_kind_and_bounds(): + """The parsed assertion is the interval the author wrote, of the kind they named.""" observation = _observation( "- [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis." ) [assertion] = observation.temporal - assert assertion.time_role is TimeRole.EFFECTIVE - assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.time_kind is TimeKind.EFFECTIVE + assert assertion.valid_during.axis is TemporalRangeAxis.DATE assert assertion.valid_during.lower == "2026-06-10" assert assertion.valid_during.upper == "2026-07-27" assert assertion.valid_during.lower_inclusive is True @@ -179,79 +188,79 @@ def test_qualifier_alone_on_the_line_still_parses(): @pytest.mark.parametrize( - ("qualifier", "literal", "kind"), + ("qualifier", "literal", "axis"), [ # A year and a month are periods the author delimited by writing them. - ("@2026", "[2026-01-01,2027-01-01)", TemporalRangeKind.DATE), - ("@2026-06", "[2026-06-01,2026-07-01)", TemporalRangeKind.DATE), + ("@2026", "[2026-01-01,2027-01-01)", TemporalRangeAxis.DATE), + ("@2026-06", "[2026-06-01,2026-07-01)", TemporalRangeAxis.DATE), # A date says when something started and leaves it open. - ("@2026-06-10", "[2026-06-10,)", TemporalRangeKind.DATE), + ("@2026-06-10", "[2026-06-10,)", TemporalRangeAxis.DATE), # So does a moment, on the instant axis. ( "@2026-06-10T14:00:00", "[2026-06-10T14:00:00.000000Z,)", - TemporalRangeKind.INSTANT, + TemporalRangeAxis.INSTANT, ), ( "@2026-06-10T14:00:00Z", "[2026-06-10T14:00:00.000000Z,)", - TemporalRangeKind.INSTANT, + TemporalRangeAxis.INSTANT, ), ( "@2026-06-10T14:00:00+02:00", "[2026-06-10T12:00:00.000000Z,)", - TemporalRangeKind.INSTANT, + TemporalRangeAxis.INSTANT, ), ], ) def test_point_qualifier_canonicalizes_to_the_span_its_precision_covers( - qualifier: str, literal: str, kind: TemporalRangeKind + qualifier: str, literal: str, axis: TemporalRangeAxis ): observation = _observation(f"- [decision] {qualifier} The cutover ran.") [assertion] = observation.temporal assert str(assertion.valid_during) == literal - assert assertion.valid_during.kind is kind + assert assertion.valid_during.axis is axis -def test_role_less_point_is_filed_on_the_valid_axis(): +def test_a_point_with_no_kind_is_filed_as_valid_time(): """`@2026-06-10` says when the statement holds, without narrowing how.""" observation = _observation("- [decision] @2026-06-10 The cache layer will use Redis.") [assertion] = observation.temporal - assert assertion.time_role is TimeRole.VALID + assert assertion.time_kind is TimeKind.VALID -def test_role_less_range_literal_is_filed_on_the_valid_axis(): - """The role is optional in both forms, and defaults the same way in both.""" +def test_a_range_literal_with_no_kind_is_filed_as_valid_time(): + """The kind is optional in both forms, and defaults the same way in both.""" observation = _observation("- [decision] @[2026-06-10,2026-07-27) Use Redis.") [assertion] = observation.temporal - assert assertion.time_role is TimeRole.VALID + assert assertion.time_kind is TimeKind.VALID assert str(assertion.valid_during) == "[2026-06-10,2026-07-27)" @pytest.mark.parametrize( - ("qualifier", "role"), + ("qualifier", "kind"), [ - ("@effective:2026-06-10", TimeRole.EFFECTIVE), - ("@occurred:2026-06-10", TimeRole.OCCURRED), - ("@due:2026-06-10", TimeRole.DUE), - ("@mentioned:2026-06-10", TimeRole.MENTIONED), - ("@valid:2026-06-10", TimeRole.VALID), + ("@effective:2026-06-10", TimeKind.EFFECTIVE), + ("@occurred:2026-06-10", TimeKind.OCCURRED), + ("@due:2026-06-10", TimeKind.DUE), + ("@mentioned:2026-06-10", TimeKind.MENTIONED), + ("@valid:2026-06-10", TimeKind.VALID), ], ) -def test_point_qualifier_names_its_axis_with_a_colon(qualifier: str, role: TimeRole): - """`:` separates role from date; a date can start with a letter, so it is needed.""" +def test_point_qualifier_names_its_kind_with_a_colon(qualifier: str, kind: TimeKind): + """`:` separates kind from date; a date can start with a letter, so it is needed.""" observation = _observation(f"- [decision] {qualifier} The cutover ran.") [assertion] = observation.temporal - assert assertion.time_role is role + assert assertion.time_kind is kind assert str(assertion.valid_during) == "[2026-06-10,)" -def test_a_roled_point_accepts_a_relative_date(): - """With a role the author has said what they mean, so any readable date is taken. +def test_a_point_with_a_kind_accepts_a_relative_date(): + """With a kind the author has said what they mean, so any readable date is taken. Relative wording resolves at parse time and is re-resolved on every index pass. That is documented behavior, not a mistake to warn about. @@ -280,11 +289,11 @@ def test_a_roled_point_accepts_a_relative_date(): "@5-3", ], ) -def test_a_role_less_point_must_be_digit_led_and_year_wide(qualifier: str): +def test_a_point_with_no_kind_must_be_digit_led_and_year_wide(qualifier: str): """A bare `@token` that short is a mention, a version, or a list marker. Accepting what dateparser makes of these would silently file wrong valid time on - ordinary prose. An author who really means one writes the role: `@occurred:may`. + ordinary prose. An author who really means one writes the kind: `@occurred:may`. """ observation = _observation(f"- [decision] {qualifier} shipped the cutover.") @@ -294,7 +303,7 @@ def test_a_role_less_point_must_be_digit_led_and_year_wide(qualifier: str): def test_a_word_point_is_read_only_when_it_names_a_specific_day(): - """A role opens the form to words, but not to words that name only a period. + """A kind opens the form to words, but not to words that name only a period. `yesterday` resolves to one day and is taken. `may` resolves to a whole month, and a bare month name at the head of a line is either prose or -- worse -- the first @@ -303,7 +312,7 @@ def test_a_word_point_is_read_only_when_it_names_a_specific_day(): """ day = _observation("- [decision] @occurred:yesterday The cutover ran.") [assertion] = day.temporal - assert assertion.time_role is TimeRole.OCCURRED + assert assertion.time_kind is TimeKind.OCCURRED assert day.content == "The cutover ran." period = _observation("- [decision] @occurred:may The cutover ran.") @@ -322,58 +331,60 @@ def test_a_word_point_is_read_only_when_it_names_a_specific_day(): @pytest.mark.parametrize( - ("qualifier", "literal", "kind"), + ("qualifier", "literal", "axis"), [ - # Single-token absolute dates, with a role and without. - ("@occurred:2026-06-10", "[2026-06-10,)", TemporalRangeKind.DATE), - ("@occurred:03/04/2026", "[2026-04-03,)", TemporalRangeKind.DATE), + # Single-token absolute dates, with a kind and without. + ("@occurred:2026-06-10", "[2026-06-10,)", TemporalRangeAxis.DATE), + ("@occurred:03/04/2026", "[2026-04-03,)", TemporalRangeAxis.DATE), ( "@occurred:2026-06-10T10:00:00", "[2026-06-10T10:00:00.000000Z,)", - TemporalRangeKind.INSTANT, + TemporalRangeAxis.INSTANT, ), - # A role admits a word, as long as it names one day. - ("@occurred:today", None, TemporalRangeKind.DATE), - ("@occurred:yesterday", None, TemporalRangeKind.DATE), + # A kind admits a word, as long as it names one day. + ("@occurred:today", None, TemporalRangeAxis.DATE), + ("@occurred:yesterday", None, TemporalRangeAxis.DATE), ], ) -def test_single_token_points_are_accepted(qualifier: str, literal: str | None, kind): +def test_single_token_points_are_accepted(qualifier: str, literal: str | None, axis): observation = _observation(f"- [decision] {qualifier} The cutover ran.") [assertion] = observation.temporal assert observation.content == "The cutover ran." - assert assertion.valid_during.kind is kind + assert assertion.valid_during.axis is axis if literal is not None: assert str(assertion.valid_during) == literal @pytest.mark.parametrize( - "qualifier", + ("qualifier", "reported"), [ # Multi-word dates: only the first token reaches the reader, and each of these # first tokens is refused, so the whole line stays content rather than being - # half-read. `@occurred:2026-06-10` says the same thing in one token. - "@occurred:June 10, 2026", - "@occurred:10 June 2026", - "@occurred:Jan 15, 2024", - "@occurred:2 days ago", - "@occurred:last week", + # half-read. `@occurred:"June 10, 2026"` says it in one delimited token. + ("@occurred:June 10, 2026", True), + ("@occurred:Jan 15, 2024", True), + ("@occurred:10 June 2026", False), + ("@occurred:2 days ago", False), + ("@occurred:last week", False), ], ) -def test_multi_word_dates_stay_content_whole(qualifier: str): - """The reader understands these; the grammar cannot delimit them. +def test_multi_word_dates_stay_content_whole(qualifier: str, reported: bool): + """The reader understands these; the unquoted grammar cannot delimit them. What matters is that an undelimitable date is left *entirely* alone: no coarse - assertion filed from its first token, and no words eaten out of the content. + assertion filed from its first token, and no words eaten out of the content. Whether + the author additionally *hears* about it is the digit-follows signal's business, + pinned below -- the line itself is untouched either way. """ line = f"- [decision] {qualifier} The cutover ran." observation = _observation(line) assert observation.temporal == [] - assert observation.temporal_error is None assert observation.content == f"{qualifier} The cutover ran." assert str(observation) == line + assert (observation.temporal_error is not None) is reported def test_a_multi_word_date_is_read_up_to_its_first_token_when_that_token_stands_alone(): @@ -387,15 +398,214 @@ def test_a_multi_word_date_is_read_up_to_its_first_token_when_that_token_stands_ [assertion] = observation.temporal assert str(assertion.valid_during) == "[2026-06-10,)" - assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.valid_during.axis is TemporalRangeAxis.DATE assert observation.content == "10:00 AM The cutover ran." +# --- The quoted point: a date the author delimited --- +# +# Quotes are how a multi-word date is written. They move the token boundary from the +# next space to the closing quote, which is the whole reason the one-token guards do not +# apply inside them: the author said where the date ends, so nothing can be truncated. + + +@pytest.mark.parametrize( + ("qualifier", "literal", "kind"), + [ + ('@occurred:"June 10, 2026"', "[2026-06-10,)", TimeKind.OCCURRED), + ('@effective:"10 June 2026"', "[2026-06-10,)", TimeKind.EFFECTIVE), + # Month-only and year-only: coarse on purpose, and delimited, so they are read. + ('@occurred:"June 2026"', "[2026-06-01,2026-07-01)", TimeKind.OCCURRED), + # With no kind, exactly like the bare point form -- filed as valid time. + ('@"June 10, 2026"', "[2026-06-10,)", TimeKind.VALID), + ], +) +def test_quoted_point_reads_a_multi_word_date(qualifier: str, literal: str, kind: TimeKind): + """The quoted form's payload goes to the date reader whole, spaces and all.""" + line = f"- [decision] {qualifier} The cutover ran." + + observation = _observation(line) + + [assertion] = observation.temporal + assert observation.temporal_error is None + assert assertion.time_kind is kind + assert str(assertion.valid_during) == literal + assert assertion.valid_during.axis is TemporalRangeAxis.DATE + assert observation.content == "The cutover ran." + # Quotes are part of the qualifier, so they round-trip with it. + assert assertion.source_text == qualifier + assert str(observation) == line + + +def test_a_quoted_relative_date_is_read_where_its_unquoted_form_is_not(): + """`2 days ago` always read fine; only the token rule kept it out.""" + quoted = _observation('- [decision] @occurred:"2 days ago" The cutover ran.') + + [assertion] = quoted.temporal + two_days_ago = datetime.now().date() - timedelta(days=2) + assert assertion.valid_during.lower == two_days_ago.isoformat() + assert quoted.content == "The cutover ran." + + unquoted = _observation("- [decision] @occurred:2 days ago The cutover ran.") + assert unquoted.temporal == [] + + +def test_a_quoted_month_is_filed_where_the_specific_day_guard_refuses_it(): + """The guard exists to catch truncation, and a delimited value cannot be truncated. + + Unquoted, `June` is refused because it may be the head of `June 2026`. Quoted, the + author has already said the date is exactly that month. + """ + quoted = _observation('- [decision] @occurred:"June 2026" The cutover ran.') + + [assertion] = quoted.temporal + assert str(assertion.valid_during) == "[2026-06-01,2026-07-01)" + assert quoted.content == "The cutover ran." + + unquoted = _observation("- [decision] @occurred:June 2026 The cutover ran.") + assert unquoted.temporal == [] + + +def test_a_quoted_clock_reading_is_read_whole_where_the_token_rule_truncates_it(): + """The one partial read the token rule allows, undone by delimiting the value. + + Unquoted, `@occurred:2026-06-10 10:00 AM` files a calendar date and leaves the clock + reading in the content (pinned above). Quoted, the same text files the instant the + author meant, and nothing is left behind. + """ + observation = _observation('- [decision] @occurred:"2026-06-10 10:00 AM" The cutover ran.') + + [assertion] = observation.temporal + assert str(assertion.valid_during) == "[2026-06-10T10:00:00.000000Z,)" + assert assertion.valid_during.axis is TemporalRangeAxis.INSTANT + assert observation.content == "The cutover ran." + + +def test_the_closing_quote_ends_the_token_and_the_rest_stays_content(): + """Content after the closing quote is ordinary content, quotes and digits included. + + Whitespace no longer delimits the token, so the peel has to stop at the quote and + hand back everything after it exactly as written -- including text that would have + been read as more date had the scan kept going. + """ + line = ( + '- [decision] @occurred:"June 10, 2026" She said "go", then 10, 2026 ' + "shipped #infra (agreed)" + ) + + observation = _observation(line) + + [assertion] = observation.temporal + assert assertion.source_text == '@occurred:"June 10, 2026"' + assert observation.content == 'She said "go", then 10, 2026 shipped #infra' + assert observation.tags == ["infra"] + assert observation.context == "agreed" + assert str(observation) == line + + +def test_content_may_follow_the_closing_quote_with_no_space(): + """The quote is the boundary, so nothing else has to mark it.""" + observation = _observation('- [decision] @occurred:"June 10, 2026"The cutover ran.') + + [assertion] = observation.temporal + assert assertion.source_text == '@occurred:"June 10, 2026"' + assert observation.content == "The cutover ran." + + +@pytest.mark.parametrize( + "qualifier", + ['@occurred:""', '@occurred:"not a date"', '@"the cutover week"'], +) +def test_a_quoted_payload_that_is_not_a_date_stays_content_silently(qualifier: str): + """Quoting says where the value ends, not that the value is a date.""" + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == f"{qualifier} The cutover ran." + + +def test_a_quoted_point_still_reports_an_unknown_kind(): + """The quoted form is a spelling of the point, so it keeps the point's diagnostic.""" + observation = _refusal('- [decision] @asserted:"June 10, 2026" The cutover ran.') + + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") + assert observation.content.startswith('@asserted:"June 10, 2026"') + + +def test_an_unterminated_quote_is_reported_instead_of_swallowing_the_line(): + """Reading on would hand the author's prose to the date reader; refusing keeps it.""" + line = '- [decision] @occurred:"June 10, 2026 The cutover ran.' + + observation = _refusal(line) + + assert "unterminated quote" in (observation.temporal_error or "") + # The fix is shown, not described. + assert '@occurred:"June 10, 2026"' in (observation.temporal_error or "") + assert observation.content == '@occurred:"June 10, 2026 The cutover ran.' + assert str(observation) == line + + +def test_an_escaped_quote_belongs_to_the_value_and_cannot_close_it(): + r"""`\"` is part of the date text, which is why this line has no closing quote left.""" + observation = _refusal('- [decision] @occurred:"June 10, 2026\\" The cutover ran.') + + assert "unterminated quote" in (observation.temporal_error or "") + assert observation.content == '@occurred:"June 10, 2026\\" The cutover ran.' + + +# --- The truncation diagnostic: when the one-token rule costs a date --- + + +def test_a_truncated_date_names_the_quoted_form_as_the_fix(): + """`@occurred:June 10, 2026` is the shape quoting exists for, so say so once.""" + observation = _refusal("- [decision] @occurred:June 10, 2026 The cutover ran.") + + error = observation.temporal_error or "" + assert "'@occurred:June'" in error + assert "names only a month or a year" in error + assert '@occurred:"June 10, 2026"' in error + # Reported, never half-read: the line is still exactly what the author wrote. + assert observation.content == "@occurred:June 10, 2026 The cutover ran." + + +def test_a_too_short_number_followed_by_a_digit_names_the_quoted_form_too(): + """The other guard gets the same treatment, with its own reason and the same fix.""" + observation = _refusal("- [note] @12 2026 was the year of the cutover.") + + error = observation.temporal_error or "" + assert "'@12'" in error + assert "is narrower than a year" in error + # A point with no kind is fixed by the quoted form with no kind. + assert '@"June 10, 2026"' in error + + +@pytest.mark.parametrize( + "line", + [ + # Prose follows, so nothing suggests a date was cut short. + "- [decision] @occurred:June the cat sat on the mat", + "- [decision] @occurred:may The cutover ran.", + "- [decision] @1 shipped the cutover.", + # Nothing follows at all. + "- [decision] @occurred:June", + # A digit follows, but `@vol:` is an ordinary `@word:` marker, not a kind. + "- [note] @vol:2 3 pages of notes", + ], +) +def test_a_refused_point_stays_silent_when_the_line_did_not_continue_the_date(line: str): + """Today's behavior, kept: the diagnostic fires on one signal, not on every refusal.""" + observation = _observation(line) + + assert observation.temporal == [] + assert observation.temporal_error is None + + @pytest.mark.parametrize( ("date_order", "expected_lower"), [("YMD", "2026-04-03"), ("DMY", "2026-04-03"), ("MDY", "2026-03-04")], ) -def test_a_roled_slash_date_follows_the_configured_order( +def test_a_slash_date_with_a_kind_follows_the_configured_order( date_order: DateOrder, expected_lower: str ): """`@occurred:03/04/2026` resolves by preference, through the real parse path.""" @@ -437,37 +647,29 @@ def test_configured_date_order_never_reinterprets_an_iso_date(monkeypatch): assert assertion.valid_during.lower == "2026-07-10" -# --- The one diagnostic: an unknown role --- - - -def _refusal(line: str) -> Observation: - """Parse a line whose qualifier must be refused, and assert the shared contract.""" - observation = _observation(line) - assert observation.temporal == [] - assert observation.temporal_error is not None - return observation +# --- The unknown-kind diagnostic --- -def test_unknown_role_in_a_range_literal_reports_diagnostic_and_keeps_text(): - """`@asserted` is well-formed but names no axis this system understands.""" +def test_unknown_kind_in_a_range_literal_reports_diagnostic_and_keeps_text(): + """`@asserted` is well-formed but names no kind this system understands.""" observation = _refusal("- [decision] @asserted[2026-06-10,) The cache layer will use Redis.") - assert "unknown temporal role 'asserted'" in (observation.temporal_error or "") - # The diagnostic names the roles that would have worked. + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") + # The diagnostic names the kinds that would have worked. assert "effective" in (observation.temporal_error or "") # Never silently dropped: the text is still searchable content. assert observation.content.startswith("@asserted[2026-06-10,)") -def test_unknown_role_in_a_point_reports_diagnostic_and_keeps_text(): - """The payload reads as a date, so the author is plainly naming an axis.""" +def test_unknown_kind_in_a_point_reports_diagnostic_and_keeps_text(): + """The payload reads as a date, so the author is plainly naming a kind.""" observation = _refusal("- [decision] @asserted:2026-06-10 The cache layer will use Redis.") - assert "unknown temporal role 'asserted'" in (observation.temporal_error or "") + assert "unknown temporal kind 'asserted'" in (observation.temporal_error or "") assert observation.content.startswith("@asserted:2026-06-10") -def test_an_unknown_role_with_an_unreadable_payload_is_left_alone(): +def test_an_unknown_kind_with_an_unreadable_payload_is_left_alone(): """`@todo:fix the thing` is prose, not a broken qualifier. The diagnostic is reserved for a payload that actually reads as time; without that, @@ -486,7 +688,7 @@ def test_an_unknown_role_with_an_unreadable_payload_is_left_alone(): @pytest.mark.parametrize( ("line", "kept"), [ - # A known role glued to something that is not a range literal. + # A known kind glued to something that is not a range literal. ("- [decision] @effective[2026-06-10 Use Redis.", "@effective[2026-06-10"), # A range mixing the two axes. ("- [decision] @effective[2026-06-10,2026-07-27T00:00:00Z) Use Redis.", "@effective["), @@ -545,7 +747,7 @@ def test_date_only_bounds_never_acquire_time_or_zone(): observation = _observation("- [decision] @effective[2026-06-10,2026-07-27) Use Redis.") [assertion] = observation.temporal - assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.valid_during.axis is TemporalRangeAxis.DATE assert assertion.valid_during.lower == "2026-06-10" assert assertion.valid_during.upper == "2026-07-27" assert "T" not in (assertion.valid_during.lower or "") @@ -561,7 +763,7 @@ def test_a_date_point_never_becomes_midnight_utc(): observation = _observation("- [decision] @effective:2026-06-10 Use Redis.") [assertion] = observation.temporal - assert assertion.valid_during.kind is TemporalRangeKind.DATE + assert assertion.valid_during.axis is TemporalRangeAxis.DATE assert assertion.valid_during.lower == "2026-06-10" assert "T00:00" not in str(assertion.valid_during) diff --git a/tests/mcp/clients/test_search_client_temporal.py b/tests/mcp/clients/test_search_client_temporal.py index e91b69028..200c61eeb 100644 --- a/tests/mcp/clients/test_search_client_temporal.py +++ b/tests/mcp/clients/test_search_client_temporal.py @@ -37,7 +37,7 @@ async def mock_call_query(client, url, **kwargs): monkeypatch.setattr("basic_memory.mcp.tools.utils.call_query", mock_call_query) -@pytest.mark.parametrize("field", ["valid_at", "valid_overlaps", "time_role"]) +@pytest.mark.parametrize("field", ["valid_at", "valid_overlaps", "time_kind"]) @pytest.mark.asyncio async def test_unconfirmed_valid_time_filter_is_refused(monkeypatch, field: str): """Every valid-time field triggers the check; none of them may pass unconfirmed.""" @@ -89,7 +89,7 @@ async def test_empty_valid_time_values_do_not_trigger_the_guard(monkeypatch): client = SearchClient(MagicMock(), "proj-123") response = await client.search( - {"text": "cache", "valid_at": None, "valid_overlaps": None, "time_role": None}, + {"text": "cache", "valid_at": None, "valid_overlaps": None, "time_kind": None}, page=1, page_size=10, ) diff --git a/tests/mcp/test_tool_contracts.py b/tests/mcp/test_tool_contracts.py index 7ade678c9..6af562a37 100644 --- a/tests/mcp/test_tool_contracts.py +++ b/tests/mcp/test_tool_contracts.py @@ -138,7 +138,7 @@ "min_similarity", "valid_at", "valid_overlaps", - "time_role", + "time_kind", ], "tail": ["timeframe", "lines", "project", "project_id"], "view_note": ["identifier", "project", "project_id"], diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py index d19e99dc2..3d90b852e 100644 --- a/tests/mcp/test_tool_search_temporal.py +++ b/tests/mcp/test_tool_search_temporal.py @@ -72,7 +72,7 @@ async def test_valid_at_after_cutover_returns_memcached_excludes_redis(client, t response = await search_notes( project=test_project.name, query="cache layer", - time_role="effective", + time_kind="effective", valid_at="2026-07-28", output_format="json", ) @@ -91,7 +91,7 @@ async def test_valid_at_before_cutover_returns_redis_excludes_memcached(client, response = await search_notes( project=test_project.name, query="cache layer", - time_role="effective", + time_kind="effective", valid_at="2026-07-01", output_format="json", ) @@ -120,14 +120,14 @@ async def test_point_qualifier_answers_the_cutover_like_a_range(client, test_pro after = await search_notes( project=test_project.name, query="cache layer", - time_role="effective", + time_kind="effective", valid_at="2026-07-28", output_format="json", ) before = await search_notes( project=test_project.name, query="cache layer", - time_role="effective", + time_kind="effective", valid_at="2026-07-01", output_format="json", ) @@ -170,7 +170,7 @@ async def test_valid_overlaps_returns_both_decisions(client, test_project): response = await search_notes( project=test_project.name, query="cache layer", - time_role="effective", + time_kind="effective", valid_overlaps="[2026-06-01,2026-08-01)", output_format="json", ) @@ -235,7 +235,7 @@ async def test_valid_at_excludes_undated_observations(client, test_project): @pytest.mark.asyncio async def test_results_carry_the_assertion_that_matched(client, test_project): - """A valid-time hit explains itself: role, canonical range, and authored text.""" + """A valid-time hit explains itself: kind, canonical range, and authored text.""" await _write_cache_layer_note(test_project.name) response = await search_notes( @@ -248,10 +248,10 @@ async def test_results_carry_the_assertion_that_matched(client, test_project): assert isinstance(response, dict), response [result] = [r for r in response["results"] if "Memcached" in (r["content"] or "")] [assertion] = result["temporal"] - assert assertion["role"] == "effective" + assert assertion["kind"] == "effective" assert assertion["source_text"] == "@effective[2026-07-27,)" assert assertion["valid_during"]["literal"] == "[2026-07-27,)" - assert assertion["valid_during"]["kind"] == "date" + assert assertion["valid_during"]["axis"] == "date" assert assertion["valid_during"]["lower"] == "2026-07-27" assert assertion["valid_during"]["lower_inclusive"] is True # JSON output drops null fields, so an unbounded end shows up as an absent key. @@ -259,8 +259,8 @@ async def test_results_carry_the_assertion_that_matched(client, test_project): @pytest.mark.asyncio -async def test_markdown_output_labels_the_time_axis(client, test_project): - """Human-readable output names the axis instead of printing a bare date.""" +async def test_markdown_output_labels_the_time_kind(client, test_project): + """Human-readable output names the kind instead of printing a bare date.""" await _write_cache_layer_note(test_project.name) rendered = await search_notes( @@ -274,8 +274,8 @@ async def test_markdown_output_labels_the_time_axis(client, test_project): @pytest.mark.asyncio -async def test_role_only_filter_finds_every_source_on_that_axis(client, test_project): - """A role with no point or range is a legal question: who asserts on this axis?""" +async def test_kind_only_filter_finds_every_source_of_that_kind(client, test_project): + """A kind with no point or range is a legal question: who asserts this kind?""" await _write_cache_layer_note(test_project.name) await write_note( project=test_project.name, @@ -287,7 +287,7 @@ async def test_role_only_filter_finds_every_source_on_that_axis(client, test_pro response = await search_notes( project=test_project.name, query="layer", - time_role="effective", + time_kind="effective", output_format="json", ) @@ -311,13 +311,13 @@ async def test_valid_at_and_valid_overlaps_together_are_refused(client, test_pro @pytest.mark.asyncio -async def test_time_role_alone_is_enough_search_criteria(client, test_project): +async def test_time_kind_alone_is_enough_search_criteria(client, test_project): """A valid-time filter is real criteria, so it must not trip the empty-query guard.""" await _write_cache_layer_note(test_project.name) response = await search_notes( project=test_project.name, - time_role="effective", + time_kind="effective", output_format="json", ) @@ -329,4 +329,4 @@ def test_tool_help_documents_undated_exclusion(): """Acceptance 8: the exclusion is documented where a caller will read it.""" doc = inspect.getdoc(search_notes) or "" assert "Sources with no temporal qualifier are excluded" in doc - assert "valid_at" in doc and "valid_overlaps" in doc and "time_role" in doc + assert "valid_at" in doc and "valid_overlaps" in doc and "time_kind" in doc diff --git a/tests/mcp/tools/test_search_notes_multi_project_temporal.py b/tests/mcp/tools/test_search_notes_multi_project_temporal.py index 8c69fe3c9..1dcfc91a2 100644 --- a/tests/mcp/tools/test_search_notes_multi_project_temporal.py +++ b/tests/mcp/tools/test_search_notes_multi_project_temporal.py @@ -90,7 +90,7 @@ async def test_all_projects_search_forwards_the_valid_time_filter(monkeypatch, c result = await search_mod.search_notes( query="cache layer", search_all_projects=True, - time_role="effective", + time_kind="effective", valid_at="2026-07-28", output_format="json", ) @@ -99,7 +99,7 @@ async def test_all_projects_search_forwards_the_valid_time_filter(monkeypatch, c assert len(payloads) == len(PROJECT_REFS) for payload in payloads: assert payload["valid_at"] == "2026-07-28" - assert payload["time_role"] == "effective" + assert payload["time_kind"] == "effective" assert payload["valid_overlaps"] is None # Every leg confirmed it ran the filter, so the merged answer confirms it too. assert result["temporal_applied"] is True diff --git a/tests/repository/test_memory_time_index_contract.py b/tests/repository/test_memory_time_index_contract.py index aaeeb4d72..7bcb9bff4 100644 --- a/tests/repository/test_memory_time_index_contract.py +++ b/tests/repository/test_memory_time_index_contract.py @@ -36,14 +36,14 @@ TemporalFilter, TemporalPoint, TemporalRange, - TemporalRangeKind, - TimeRole, + TemporalRangeAxis, + TimeKind, parse_point, parse_range_literal, ) -DATE = TemporalRangeKind.DATE -INSTANT = TemporalRangeKind.INSTANT +DATE = TemporalRangeAxis.DATE +INSTANT = TemporalRangeAxis.INSTANT # Every observation shares this word so one FTS query returns the whole population and # the temporal predicate is the only thing that narrows it. That also proves the @@ -57,15 +57,15 @@ class StoredAssertion: label: str valid_during: TemporalRange - role: TimeRole = TimeRole.EFFECTIVE + kind: TimeKind = TimeKind.EFFECTIVE def _date_range(literal: str) -> TemporalRange: - return parse_range_literal(literal, kind=DATE) + return parse_range_literal(literal, axis=DATE) def _instant_range(literal: str) -> TemporalRange: - return parse_range_literal(literal, kind=INSTANT) + return parse_range_literal(literal, axis=INSTANT) # The population under test. Labels are the vocabulary of every expectation below. @@ -76,7 +76,7 @@ def _instant_range(literal: str) -> TemporalRange: StoredAssertion("open_open", _date_range("(2026-06-10,2026-07-27)")), StoredAssertion("from_cutover", _date_range("[2026-07-27,)")), StoredAssertion("before_june", _date_range("(,2026-06-10)")), - StoredAssertion("always", TemporalRange(kind=DATE)), + StoredAssertion("always", TemporalRange(axis=DATE)), StoredAssertion("empty", TemporalRange.empty(DATE)), StoredAssertion( "instant_window", @@ -87,46 +87,46 @@ def _instant_range(literal: str) -> TemporalRange: # Authored in +02:00; normalization must make it the UTC window [14:00,15:00). _instant_range("[2026-07-27T16:00:00+02:00,2026-07-27T17:00:00+02:00)"), ), - StoredAssertion("due_window", _date_range("[2026-06-10,2026-07-27)"), role=TimeRole.DUE), + StoredAssertion("due_window", _date_range("[2026-06-10,2026-07-27)"), kind=TimeKind.DUE), # --- The discrete population --- # - # Filed on the `occurred` axis and dated in March so it never widens an expectation + # Filed as `occurred` time and dated in March so it never widens an expectation # above, and so no bound collides with the January bookkeeping timestamps that # `test_projection_rows_carry_only_authored_bounds` watches for. # # Only March 2, and only March 3: adjacent as authored bounds, disjoint as days. # This is the pair the half-open canonical form exists to tell apart. - StoredAssertion("only_mar_02", _date_range("(2026-03-01,2026-03-03)"), role=TimeRole.OCCURRED), - StoredAssertion("only_mar_03", _date_range("(2026-03-02,2026-03-04)"), role=TimeRole.OCCURRED), + StoredAssertion("only_mar_02", _date_range("(2026-03-01,2026-03-03)"), kind=TimeKind.OCCURRED), + StoredAssertion("only_mar_03", _date_range("(2026-03-02,2026-03-04)"), kind=TimeKind.OCCURRED), # Back-to-back half-open periods, the shape a sequence of effective windows takes. StoredAssertion( - "half_open_first", _date_range("[2026-03-10,2026-03-12)"), role=TimeRole.OCCURRED + "half_open_first", _date_range("[2026-03-10,2026-03-12)"), kind=TimeKind.OCCURRED ), StoredAssertion( - "half_open_second", _date_range("[2026-03-12,2026-03-14)"), role=TimeRole.OCCURRED + "half_open_second", _date_range("[2026-03-12,2026-03-14)"), kind=TimeKind.OCCURRED ), # Closed periods written by an author who means "through the 22nd": they share it. - StoredAssertion("closed_first", _date_range("[2026-03-20,2026-03-22]"), role=TimeRole.OCCURRED), + StoredAssertion("closed_first", _date_range("[2026-03-20,2026-03-22]"), kind=TimeKind.OCCURRED), StoredAssertion( - "closed_second", _date_range("[2026-03-22,2026-03-24]"), role=TimeRole.OCCURRED + "closed_second", _date_range("[2026-03-22,2026-03-24]"), kind=TimeKind.OCCURRED ), - StoredAssertion("one_day", _date_range("[2026-03-30,2026-03-30]"), role=TimeRole.OCCURRED), + StoredAssertion("one_day", _date_range("[2026-03-30,2026-03-30]"), kind=TimeKind.OCCURRED), # After the 5th and before the 6th there is no day, so this authored range is the # empty range -- something only the discrete reading can see. - StoredAssertion("no_such_day", _date_range("(2026-03-05,2026-03-06)"), role=TimeRole.OCCURRED), + StoredAssertion("no_such_day", _date_range("(2026-03-05,2026-03-06)"), kind=TimeKind.OCCURRED), # An instant range with a closed upper end, so the date rewrite is proven to stop # at the date axis rather than pushing this endpoint forward by a day. StoredAssertion( "instant_closed", _instant_range("[2026-07-27T20:00:00Z,2026-07-27T21:00:00Z]"), - role=TimeRole.OCCURRED, + kind=TimeKind.OCCURRED, ), ) DATE_LABELS = frozenset( stored.label for stored in STORED_ASSERTIONS - if stored.valid_during.kind is DATE and stored.role is TimeRole.EFFECTIVE + if stored.valid_during.axis is DATE and stored.kind is TimeKind.EFFECTIVE ) NON_EMPTY_DATE_LABELS = DATE_LABELS - {"empty"} @@ -177,8 +177,8 @@ async def temporal_population( entity_id=entity_id, source_type=SearchItemType.OBSERVATION.value, source_id=observation.id, - time_role=stored.role.value, - range_kind=stored.valid_during.kind.value, + time_kind=stored.kind.value, + range_axis=stored.valid_during.axis.value, lower_value=stored.valid_during.lower, upper_value=stored.valid_during.upper, lower_inclusive=stored.valid_during.lower_inclusive, @@ -253,7 +253,7 @@ async def test_containment_contract(search_repository, temporal_population, at, matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point(at)), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point(at)), ) assert matched == expected @@ -296,7 +296,7 @@ async def test_overlap_contract(search_repository, temporal_population, literal, matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, overlaps=_date_range(literal)), + TemporalFilter(kind=TimeKind.EFFECTIVE, overlaps=_date_range(literal)), ) assert matched == expected @@ -310,7 +310,7 @@ async def test_overlap_with_fully_unbounded_window_matches_every_non_empty_range matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, overlaps=TemporalRange(kind=DATE)), + TemporalFilter(kind=TimeKind.EFFECTIVE, overlaps=TemporalRange(axis=DATE)), ) assert matched == NON_EMPTY_DATE_LABELS @@ -322,7 +322,7 @@ async def test_overlap_with_empty_window_matches_nothing(search_repository, temp matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, overlaps=TemporalRange.empty(DATE)), + TemporalFilter(kind=TimeKind.EFFECTIVE, overlaps=TemporalRange.empty(DATE)), ) assert matched == set() @@ -335,7 +335,7 @@ async def test_stored_empty_range_matches_no_query(search_repository, temporal_p matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point(at)), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point(at)), ) assert "empty" not in matched, at @@ -352,7 +352,7 @@ async def _occurred_overlaps(search_repository, labels_by_id, literal: str) -> s return await _matching_labels( search_repository, labels_by_id, - TemporalFilter(role=TimeRole.OCCURRED, overlaps=_date_range(literal)), + TemporalFilter(kind=TimeKind.OCCURRED, overlaps=_date_range(literal)), ) @@ -360,7 +360,7 @@ async def _occurred_at(search_repository, labels_by_id, at: str) -> set[str]: return await _matching_labels( search_repository, labels_by_id, - TemporalFilter(role=TimeRole.OCCURRED, at=parse_point(at)), + TemporalFilter(kind=TimeKind.OCCURRED, at=parse_point(at)), ) @@ -493,7 +493,7 @@ async def test_instant_ranges_are_untouched_by_the_date_canonicalization( at_upper = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.OCCURRED, at=parse_point("2026-07-27T21:00:00Z")), + TemporalFilter(kind=TimeKind.OCCURRED, at=parse_point("2026-07-27T21:00:00Z")), ) assert at_upper == {"instant_closed"} @@ -501,7 +501,7 @@ async def test_instant_ranges_are_untouched_by_the_date_canonicalization( missed = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.OCCURRED, at=parse_point(outside)), + TemporalFilter(kind=TimeKind.OCCURRED, at=parse_point(outside)), ) assert missed == set(), outside @@ -563,7 +563,7 @@ async def test_date_query_does_not_match_instant_range(search_repository, tempor matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27")), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27")), ) assert "instant_window" not in matched @@ -577,7 +577,7 @@ async def test_instant_query_does_not_match_date_range(search_repository, tempor search_repository, temporal_population, TemporalFilter( - role=TimeRole.EFFECTIVE, + kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T17:00:00Z"), ), ) @@ -599,7 +599,7 @@ async def test_instant_ranges_compare_as_instants_across_offsets( inside = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T14:30:00Z")), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T14:30:00Z")), ) assert inside == {"instant_offset"} @@ -607,7 +607,7 @@ async def test_instant_ranges_compare_as_instants_across_offsets( outside = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T16:30:00Z")), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T16:30:00Z")), ) assert outside == {"instant_window"} @@ -618,36 +618,36 @@ async def test_instant_endpoints_respect_inclusivity(search_repository, temporal at_lower = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T16:00:00Z")), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T16:00:00Z")), ) assert at_lower == {"instant_window"} at_upper = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-27T18:00:00Z")), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-27T18:00:00Z")), ) assert at_upper == set() -# --- Role narrowing --- +# --- Kind narrowing --- @pytest.mark.asyncio -async def test_role_filter_narrows_to_one_axis(search_repository, temporal_population): - """Two roles can assert the same interval; a role filter separates them.""" +async def test_kind_filter_narrows_to_one_kind(search_repository, temporal_population): + """Two kinds can assert the same interval; a kind filter separates them.""" due = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.DUE, at=parse_point("2026-07-01")), + TemporalFilter(kind=TimeKind.DUE, at=parse_point("2026-07-01")), ) assert due == {"due_window"} @pytest.mark.asyncio -async def test_filter_without_role_spans_every_axis(search_repository, temporal_population): - """Omitting the role asks the question of every axis at once.""" +async def test_filter_without_a_kind_spans_every_kind(search_repository, temporal_population): + """Omitting the kind asks the question of every kind at once.""" matched = await _matching_labels( search_repository, temporal_population, @@ -665,19 +665,19 @@ async def test_filter_without_role_spans_every_axis(search_repository, temporal_ @pytest.mark.asyncio -async def test_role_only_filter_selects_every_source_on_that_axis( +async def test_kind_only_filter_selects_every_source_of_that_kind( search_repository, temporal_population ): - """A role with no window is a legal question, and the empty range still answers it. + """A kind with no window is a legal question, and the empty range still answers it. Without a window there is no axis to compare on and no interval to intersect, so - the filter asks only "does this source assert anything on this role" -- which the + the filter asks only "does this source assert anything on this kind" -- which the empty range does. """ matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE), + TemporalFilter(kind=TimeKind.EFFECTIVE), ) assert matched == DATE_LABELS | {"instant_window", "instant_offset"} @@ -795,7 +795,7 @@ async def test_temporal_filter_count_matches_search(search_repository, temporal_ The router gathers the two concurrently and derives `has_more` from the count, so a count that ignored the filter would report pages that do not exist. """ - temporal = TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-01")) + temporal = TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-01")) results = await search_repository.search( search_text=SHARED_TERM, search_item_types=[SearchItemType.OBSERVATION], @@ -817,7 +817,7 @@ async def test_temporal_filter_applies_without_search_text(search_repository, te matched = await _matching_labels( search_repository, temporal_population, - TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-08-01")), + TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-08-01")), search_text=None, ) @@ -850,7 +850,7 @@ async def test_temporal_filter_is_scoped_to_its_project( results = await other_repository.search( search_text=SHARED_TERM, search_item_types=[SearchItemType.OBSERVATION], - temporal=TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-01")), + temporal=TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-01")), limit=50, ) @@ -862,11 +862,11 @@ async def test_temporal_point_and_range_agree_on_containment( search_repository, temporal_population ): """A point question is the degenerate closed range, so the two cannot disagree.""" - point = TemporalFilter(role=TimeRole.EFFECTIVE, at=TemporalPoint(kind=DATE, value="2026-07-27")) + point = TemporalFilter(kind=TimeKind.EFFECTIVE, at=TemporalPoint(axis=DATE, value="2026-07-27")) window = TemporalFilter( - role=TimeRole.EFFECTIVE, + kind=TimeKind.EFFECTIVE, overlaps=TemporalRange( - kind=DATE, + axis=DATE, lower="2026-07-27", upper="2026-07-27", lower_inclusive=True, diff --git a/tests/repository/test_vector_temporal_filter.py b/tests/repository/test_vector_temporal_filter.py index 904768c0c..0270d7815 100644 --- a/tests/repository/test_vector_temporal_filter.py +++ b/tests/repository/test_vector_temporal_filter.py @@ -15,7 +15,7 @@ import pytest -from basic_memory.temporal import TemporalFilter, TimeRole, parse_point +from basic_memory.temporal import TemporalFilter, TimeKind, parse_point from tests.repository.test_hybrid_fusion import ( HYBRID_KWARGS, ConcreteSearchRepo as HybridSearchRepo, @@ -30,7 +30,7 @@ fake_scoped_session, ) -TEMPORAL = TemporalFilter(role=TimeRole.EFFECTIVE, at=parse_point("2026-07-28")) +TEMPORAL = TemporalFilter(kind=TimeKind.EFFECTIVE, at=parse_point("2026-07-28")) def _vector_kwargs(**overrides: Any) -> dict[str, Any]: diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py index 6b1e49dcc..862671b51 100644 --- a/tests/services/test_search_service_temporal.py +++ b/tests/services/test_search_service_temporal.py @@ -17,7 +17,7 @@ build_temporal_filter, describe_search_criteria, ) -from basic_memory.temporal import TemporalQualifierError, TimeRole +from basic_memory.temporal import TemporalQualifierError, TimeKind # The entity is created "now"; the qualifier claims June-July 2026. Keeping the two # ranges disjoint is what makes acceptance case 11 testable at all. @@ -106,7 +106,7 @@ async def test_valid_time_filter_narrows_to_the_asserting_observation( await _index_cache_layer_note(entity_service, search_service) results = await search_service.search( - SearchQuery(text="cache layer", time_role="effective", valid_at=EFFECTIVE_WINDOW_START) + SearchQuery(text="cache layer", time_kind="effective", valid_at=EFFECTIVE_WINDOW_START) ) assert [result.type for result in results] == ["observation"] @@ -137,9 +137,9 @@ async def test_undated_note_is_excluded_by_a_valid_time_filter(entity_service, s # --- Diagnostics: the boundary refuses every malformed filter --- -def test_unknown_time_role_is_refused_with_the_known_roles(): - with pytest.raises(TemporalQualifierError, match="unknown time_role 'asserted'") as exc_info: - build_temporal_filter(SearchQuery(text="cache", time_role="asserted")) +def test_unknown_time_kind_is_refused_with_the_known_kinds(): + with pytest.raises(TemporalQualifierError, match="unknown time_kind 'asserted'") as exc_info: + build_temporal_filter(SearchQuery(text="cache", time_kind="asserted")) assert "effective" in str(exc_info.value) @@ -175,11 +175,11 @@ def test_query_without_valid_time_fields_builds_no_filter(): assert build_temporal_filter(SearchQuery(text="cache")) is None -def test_role_only_query_builds_a_role_filter(): - temporal = build_temporal_filter(SearchQuery(text="cache", time_role="effective")) +def test_kind_only_query_builds_a_kind_filter(): + temporal = build_temporal_filter(SearchQuery(text="cache", time_kind="effective")) assert temporal is not None - assert temporal.role is TimeRole.EFFECTIVE + assert temporal.kind is TimeKind.EFFECTIVE assert temporal.at is None and temporal.overlaps is None @@ -195,7 +195,7 @@ def test_valid_at_and_valid_overlaps_are_mutually_exclusive_at_the_schema(): def test_a_valid_time_filter_alone_is_enough_criteria(): """A temporal filter is real criteria; the empty-query guard must not swallow it.""" assert SearchQuery(valid_at="2026-07-28").no_criteria() is False - assert SearchQuery(time_role="effective").no_criteria() is False + assert SearchQuery(time_kind="effective").no_criteria() is False assert SearchQuery(valid_overlaps="[2026-06-10,)").no_criteria() is False assert SearchQuery().no_criteria() is True @@ -203,12 +203,12 @@ def test_a_valid_time_filter_alone_is_enough_criteria(): @pytest.mark.asyncio async def test_prepared_query_carries_the_parsed_filter(search_service): prepared = search_service.prepare_query( - SearchQuery(text="cache", time_role="effective", valid_at="2026-07-28") + SearchQuery(text="cache", time_kind="effective", valid_at="2026-07-28") ) assert prepared is not None assert prepared.temporal is not None - assert prepared.temporal.role is TimeRole.EFFECTIVE + assert prepared.temporal.kind is TimeKind.EFFECTIVE assert prepared.temporal.at is not None assert prepared.temporal.at.value == "2026-07-28" @@ -217,7 +217,7 @@ async def test_prepared_query_carries_the_parsed_filter(search_service): async def test_search_trace_describes_the_valid_time_question(search_service): """A trace must show the question that actually ran, valid time included.""" containment = search_service.prepare_query( - SearchQuery(text="cache", time_role="effective", valid_at="2026-07-28") + SearchQuery(text="cache", time_kind="effective", valid_at="2026-07-28") ) overlap = search_service.prepare_query( SearchQuery(text="cache", valid_overlaps="[2026-06-10,2026-07-27)") @@ -225,6 +225,6 @@ async def test_search_trace_describes_the_valid_time_question(search_service): plain = search_service.prepare_query(SearchQuery(text="cache")) assert containment is not None and overlap is not None and plain is not None - assert "temporal=role=effective,valid_at=2026-07-28" in describe_search_criteria(containment) + assert "temporal=kind=effective,valid_at=2026-07-28" in describe_search_criteria(containment) assert "temporal=valid_overlaps=[2026-06-10,2026-07-27)" in describe_search_criteria(overlap) assert "temporal=" not in describe_search_criteria(plain) diff --git a/tests/test_memory_time_index_migration.py b/tests/test_memory_time_index_migration.py index 004ed4698..36dcc99bd 100644 --- a/tests/test_memory_time_index_migration.py +++ b/tests/test_memory_time_index_migration.py @@ -35,8 +35,8 @@ "entity_id", "source_type", "source_id", - "time_role", - "range_kind", + "time_kind", + "range_axis", "lower_value", "upper_value", "lower_inclusive", @@ -65,7 +65,7 @@ ) INSERT_SQL = """ INSERT INTO memory_time_index ( - project_id, entity_id, source_type, source_id, time_role, range_kind, + project_id, entity_id, source_type, source_id, time_kind, range_axis, lower_value, upper_value, lower_inclusive, upper_inclusive, is_empty, extractor, source_text ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -89,8 +89,8 @@ def _row_with(**overrides: Any) -> tuple[Any, ...]: "entity_id", "source_type", "source_id", - "time_role", - "range_kind", + "time_kind", + "range_axis", "lower_value", "upper_value", "lower_inclusive", @@ -152,12 +152,12 @@ def test_alembic_upgrade_creates_memory_time_index_table(tmp_path, monkeypatch): "PRAGMA index_info(ix_memory_time_index_lookup)" ).fetchall() ] - # The predicate filters on project/role/axis and projects (source_type, source_id), + # The predicate filters on project/kind/axis and projects (source_type, source_id), # so this one index both drives the scan and covers its output. assert lookup_columns == [ "project_id", - "time_role", - "range_kind", + "time_kind", + "range_axis", "source_type", "source_id", ] @@ -203,7 +203,7 @@ def test_upgraded_table_accepts_a_well_formed_assertion(tmp_path, monkeypatch): @pytest.mark.parametrize( ("overrides", "constraint"), [ - ({"range_kind": "week"}, "ck_memory_time_index_range_kind"), + ({"range_axis": "week"}, "ck_memory_time_index_range_axis"), # An empty range with endpoints would describe the same interval two ways. ({"is_empty": 1}, "ck_memory_time_index_empty_has_no_bounds"), # PostgreSQL's rule: there is no endpoint to include on an unbounded side. @@ -269,12 +269,12 @@ def test_postgres_render_carries_the_same_definition(monkeypatch): sql = buffer.getvalue() assert "CREATE TABLE memory_time_index" in sql assert "FOREIGN KEY(entity_id) REFERENCES entity (id) ON DELETE CASCADE" in sql - assert "ck_memory_time_index_range_kind" in sql + assert "ck_memory_time_index_range_axis" in sql assert "ck_memory_time_index_empty_has_no_bounds" in sql assert "ck_memory_time_index_unbounded_is_exclusive" in sql assert ( "CREATE INDEX ix_memory_time_index_lookup ON memory_time_index " - "(project_id, time_role, range_kind, source_type, source_id)" in sql + "(project_id, time_kind, range_axis, source_type, source_id)" in sql ) # Bounds stay portable text on both backends; a native range column would be a # later, generated addition rather than a change to this definition. diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 1044621a0..aab66ea3e 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -21,16 +21,16 @@ TemporalPoint, TemporalQualifierError, TemporalRange, - TemporalRangeKind, - TimeRole, + TemporalRangeAxis, + TimeKind, canonical_bound, parse_authored_point, parse_point, parse_range_literal, ) -DATE = TemporalRangeKind.DATE -INSTANT = TemporalRangeKind.INSTANT +DATE = TemporalRangeAxis.DATE +INSTANT = TemporalRangeAxis.INSTANT # --- Canonical bounds --- @@ -144,17 +144,17 @@ def test_timestamp_shaped_bound_on_a_date_that_does_not_exist_is_refused(): def test_point_rejects_a_non_canonical_value(): """A value that skipped canonicalization must not enter the domain.""" with pytest.raises(TemporalQualifierError, match="not canonical"): - TemporalPoint(kind=INSTANT, value="2026-07-27T18:42:00Z") + TemporalPoint(axis=INSTANT, value="2026-07-27T18:42:00Z") def test_point_renders_its_canonical_value(): - assert str(TemporalPoint(kind=DATE, value="2026-07-27")) == "2026-07-27" + assert str(TemporalPoint(axis=DATE, value="2026-07-27")) == "2026-07-27" def test_parse_point_infers_the_axis_from_what_was_written(): - assert parse_point("2026-07-27") == TemporalPoint(kind=DATE, value="2026-07-27") + assert parse_point("2026-07-27") == TemporalPoint(axis=DATE, value="2026-07-27") assert parse_point(" 2026-07-27T18:42:00+02:00 ") == TemporalPoint( - kind=INSTANT, value="2026-07-27T16:42:00.000000Z" + axis=INSTANT, value="2026-07-27T16:42:00.000000Z" ) @@ -166,7 +166,7 @@ def test_parse_point_refuses_an_empty_string(): def test_parse_point_reads_a_naive_timestamp_as_utc(): """The search boundary follows the same naive-is-UTC rule as authored bounds.""" assert parse_point("2026-07-27T18:42:00") == TemporalPoint( - kind=INSTANT, value="2026-07-27T18:42:00.000000Z" + axis=INSTANT, value="2026-07-27T18:42:00.000000Z" ) @@ -178,7 +178,7 @@ def test_parse_point_reads_a_naive_timestamp_as_utc(): @pytest.mark.parametrize( - ("written", "literal", "kind"), + ("written", "literal", "axis"), [ # A year and a month are periods the author delimited by writing them. ("2026", "[2026-01-01,2027-01-01)", DATE), @@ -194,12 +194,12 @@ def test_parse_point_reads_a_naive_timestamp_as_utc(): (" 2026-06-10 ", "[2026-06-10,)", DATE), ], ) -def test_authored_point_denotes_the_span_its_precision_covers(written, literal, kind): +def test_authored_point_denotes_the_span_its_precision_covers(written, literal, axis): span = parse_authored_point(written) assert span is not None assert str(span) == literal - assert span.kind is kind + assert span.axis is axis assert span.lower_inclusive is True @@ -212,7 +212,7 @@ def test_authored_date_never_acquires_a_time_of_day(): span = parse_authored_point("2026-06-10") assert span is not None - assert span.kind is DATE + assert span.axis is DATE assert span.lower == "2026-06-10" assert "T" not in span.lower and "Z" not in span.lower @@ -224,7 +224,7 @@ def test_authored_naive_timestamp_is_read_as_utc_not_local_time(): assert naive is not None and explicit is not None assert naive == explicit - assert naive.kind is INSTANT + assert naive.axis is INSTANT assert naive.lower == "2026-06-10T14:00:00.000000Z" @@ -237,7 +237,7 @@ def test_authored_relative_dates_resolve_at_parse_time(): span = parse_authored_point("yesterday") assert span is not None - assert span.kind is DATE + assert span.axis is DATE yesterday = datetime.now().date() - timedelta(days=1) assert span.lower == yesterday.isoformat() @@ -248,7 +248,7 @@ def test_authored_relative_dates_resolve_at_parse_time(): @pytest.mark.parametrize( - ("written", "literal", "kind"), + ("written", "literal", "axis"), [ # Month names, in the orders English writes them. ("June 10, 2026", "[2026-06-10,)", DATE), @@ -261,7 +261,7 @@ def test_authored_relative_dates_resolve_at_parse_time(): ("2026-06-10 10:00 AM", "[2026-06-10T10:00:00.000000Z,)", INSTANT), ], ) -def test_written_dates_read_on_the_axis_their_precision_names(written, literal, kind): +def test_written_dates_read_on_the_axis_their_precision_names(written, literal, axis): """A written date stays a date; adding a clock reading is what makes it an instant. `June 10, 2026` must never acquire a time of day -- midnight in which zone is a @@ -272,7 +272,7 @@ def test_written_dates_read_on_the_axis_their_precision_names(written, literal, assert span is not None assert str(span) == literal - assert span.kind is kind + assert span.axis is axis def test_written_relative_dates_resolve_against_now(): @@ -280,7 +280,7 @@ def test_written_relative_dates_resolve_against_now(): span = parse_authored_point("2 days ago") assert span is not None - assert span.kind is DATE + assert span.axis is DATE assert span.lower == (datetime.now().date() - timedelta(days=2)).isoformat() @@ -304,7 +304,7 @@ def test_slash_dates_resolve_by_the_configured_order(written, date_order, expect assert span is not None assert span.lower == expected_lower - assert span.kind is DATE + assert span.axis is DATE @pytest.mark.parametrize( @@ -364,7 +364,7 @@ def test_a_year_with_no_successor_is_unread(): assert parse_authored_point("9999") is None # The year before it still resolves, so the guard is the calendar edge, not 4 digits. assert parse_authored_point("9998") == TemporalRange( - kind=DATE, + axis=DATE, lower=date(9998, 1, 1).isoformat(), upper=date(9999, 1, 1).isoformat(), lower_inclusive=True, @@ -382,7 +382,7 @@ def test_unbounded_sides_are_forced_exclusive(): tests below. """ span = TemporalRange( - kind=INSTANT, + axis=INSTANT, lower=None, upper="2026-07-27T00:00:00.000000Z", lower_inclusive=True, @@ -395,7 +395,7 @@ def test_unbounded_sides_are_forced_exclusive(): def test_fully_unbounded_range_is_exclusive_on_both_sides(): - span = TemporalRange(kind=DATE, lower_inclusive=True, upper_inclusive=True) + span = TemporalRange(axis=DATE, lower_inclusive=True, upper_inclusive=True) assert (span.lower_inclusive, span.upper_inclusive) == (False, False) assert str(span) == "(,)" @@ -408,7 +408,7 @@ def test_fully_unbounded_range_is_exclusive_on_both_sides(): def test_degenerate_range_collapses_to_empty(lower_inclusive: bool, upper_inclusive: bool): """`[a,a)`, `(a,a]`, and `(a,a)` contain no points, so they *are* the empty range.""" span = TemporalRange( - kind=DATE, + axis=DATE, lower="2026-07-27", upper="2026-07-27", lower_inclusive=lower_inclusive, @@ -427,7 +427,7 @@ def test_closed_single_point_range_is_not_empty(): closing at the following day rather than by owning both endpoints. """ span = TemporalRange( - kind=DATE, + axis=DATE, lower="2026-07-27", upper="2026-07-27", lower_inclusive=True, @@ -440,26 +440,26 @@ def test_closed_single_point_range_is_not_empty(): def test_inverted_range_is_refused(): with pytest.raises(TemporalQualifierError, match="after upper bound"): - TemporalRange(kind=DATE, lower="2026-08-01", upper="2026-06-10") + TemporalRange(axis=DATE, lower="2026-08-01", upper="2026-06-10") def test_empty_range_cannot_carry_bounds(): """Two representations of the same interval would make equality lie.""" with pytest.raises(TemporalQualifierError, match="carries no bounds"): - TemporalRange(kind=DATE, lower="2026-07-27", is_empty=True) + TemporalRange(axis=DATE, lower="2026-07-27", is_empty=True) with pytest.raises(TemporalQualifierError, match="carries no bounds"): - TemporalRange(kind=DATE, is_empty=True, upper_inclusive=True) + TemporalRange(axis=DATE, is_empty=True, upper_inclusive=True) def test_range_rejects_non_canonical_bounds(): with pytest.raises(TemporalQualifierError, match="not canonical"): - TemporalRange(kind=INSTANT, lower="2026-07-27T18:42:00Z") + TemporalRange(axis=INSTANT, lower="2026-07-27T18:42:00Z") def test_empty_constructor_builds_the_empty_range_on_one_axis(): span = TemporalRange.empty(INSTANT) - assert (span.kind, span.is_empty, span.lower, span.upper) == (INSTANT, True, None, None) + assert (span.axis, span.is_empty, span.lower, span.upper) == (INSTANT, True, None, None) # --- The discrete canonical form --- @@ -492,7 +492,7 @@ def test_empty_constructor_builds_the_empty_range_on_one_axis(): ) def test_date_ranges_are_stored_half_open(authored: str, canonical: str): """Whatever the author wrote, the stored date range is `[lower,upper)`.""" - span = parse_range_literal(authored, kind=DATE) + span = parse_range_literal(authored, axis=DATE) assert str(span) == canonical # A bounded lower end is always owned, a bounded upper end never is. @@ -503,9 +503,9 @@ def test_date_ranges_are_stored_half_open(authored: str, canonical: str): def test_the_canonical_date_rendering_is_a_fixed_point(): """Re-parsing what `__str__` produced yields this same value, not a third form.""" for authored in ("(2026-06-10,2026-07-27]", "[2026-07-27,2026-07-27]", "(,2026-07-27]"): - span = parse_range_literal(authored, kind=DATE) + span = parse_range_literal(authored, axis=DATE) - assert parse_range_literal(str(span), kind=DATE) == span, authored + assert parse_range_literal(str(span), axis=DATE) == span, authored @pytest.mark.parametrize( @@ -521,7 +521,7 @@ def test_the_canonical_date_rendering_is_a_fixed_point(): ], ) def test_date_ranges_that_admit_no_day_are_the_empty_range(literal: str): - span = parse_range_literal(literal, kind=DATE) + span = parse_range_literal(literal, axis=DATE) assert span.is_empty is True assert str(span) == "empty" @@ -530,7 +530,7 @@ def test_date_ranges_that_admit_no_day_are_the_empty_range(literal: str): def test_an_inclusive_upper_end_on_the_last_date_becomes_unbounded(): """`9999-12-31` has no successor to close against, and no later day to exclude.""" span = TemporalRange( - kind=DATE, + axis=DATE, lower="2026-06-10", upper="9999-12-31", lower_inclusive=True, @@ -544,7 +544,7 @@ def test_an_inclusive_upper_end_on_the_last_date_becomes_unbounded(): def test_the_last_date_alone_is_still_one_day_not_the_empty_range(): """`[9999-12-31,9999-12-31]` survives the rewrite that drops its upper end.""" span = TemporalRange( - kind=DATE, + axis=DATE, lower="9999-12-31", upper="9999-12-31", lower_inclusive=True, @@ -557,7 +557,7 @@ def test_the_last_date_alone_is_still_one_day_not_the_empty_range(): def test_an_exclusive_lower_end_on_the_last_date_is_empty(): """A range beginning strictly after the last date admits no date at all.""" - span = TemporalRange(kind=DATE, lower="9999-12-31") + span = TemporalRange(axis=DATE, lower="9999-12-31") assert span.is_empty is True assert str(span) == "empty" @@ -583,14 +583,14 @@ def test_instant_ranges_keep_the_inclusivity_they_were_written_with(literal, exp Adding a microsecond would be an invented precision, and rewriting an instant the way a date is rewritten would move the endpoint to a moment nobody wrote. """ - span = parse_range_literal(literal, kind=INSTANT) + span = parse_range_literal(literal, axis=INSTANT) assert (span.lower_inclusive, span.upper_inclusive, span.lower, span.upper) == expected def test_an_instant_range_over_one_day_is_not_widened_by_a_day(): """The date rewrite must not reach the instant axis: `+1 day` there is a bug.""" - span = parse_range_literal("[2026-07-27T00:00:00Z,2026-07-27T23:59:59Z]", kind=INSTANT) + span = parse_range_literal("[2026-07-27T00:00:00Z,2026-07-27T23:59:59Z]", axis=INSTANT) assert span.upper == "2026-07-27T23:59:59.000000Z" assert span.upper_inclusive is True @@ -598,7 +598,7 @@ def test_an_instant_range_over_one_day_is_not_widened_by_a_day(): def test_a_degenerate_instant_range_still_holds_exactly_one_moment(): """`[t,t]` on a continuous axis stays `[t,t]`; there is no successor to close at.""" - span = parse_range_literal("[2026-07-27T18:42:00Z,2026-07-27T18:42:00Z]", kind=INSTANT) + span = parse_range_literal("[2026-07-27T18:42:00Z,2026-07-27T18:42:00Z]", axis=INSTANT) assert span.is_empty is False assert str(span) == "[2026-07-27T18:42:00.000000Z,2026-07-27T18:42:00.000000Z]" @@ -645,13 +645,13 @@ def test_range_literal_tolerates_surrounding_whitespace(): def test_empty_literal_requires_an_explicit_axis(): """`empty` carries no bounds to classify, so the caller must name the axis.""" - assert parse_range_literal("empty", kind=DATE).is_empty is True - with pytest.raises(TemporalQualifierError, match="kind must be given"): + assert parse_range_literal("empty", axis=DATE).is_empty is True + with pytest.raises(TemporalQualifierError, match="axis must be given"): parse_range_literal("empty") def test_fully_unbounded_literal_requires_an_explicit_axis(): - assert parse_range_literal("(,)", kind=INSTANT).kind is INSTANT + assert parse_range_literal("(,)", axis=INSTANT).axis is INSTANT with pytest.raises(TemporalQualifierError, match="no bounds to classify"): parse_range_literal("(,)") @@ -663,7 +663,7 @@ def test_range_literal_refuses_mixed_axes(): def test_range_literal_refuses_an_axis_it_was_not_asked_for(): with pytest.raises(TemporalQualifierError, match="expected instant bounds"): - parse_range_literal("[2026-06-10,2026-07-27)", kind=INSTANT) + parse_range_literal("[2026-06-10,2026-07-27)", axis=INSTANT) @pytest.mark.parametrize( @@ -693,8 +693,8 @@ def test_filter_refuses_asking_two_questions_at_once(): def test_filter_refuses_asking_nothing_at_all(): - """A filter that names no role, point, or range would match everything silently.""" - with pytest.raises(TemporalQualifierError, match="must name a role"): + """A filter that names no kind, point, or range would match everything silently.""" + with pytest.raises(TemporalQualifierError, match="must name a kind"): TemporalFilter() @@ -703,7 +703,7 @@ def test_point_filter_window_is_the_degenerate_closed_range(): window = TemporalFilter(at=parse_point("2026-07-27")).window assert window == TemporalRange( - kind=DATE, + axis=DATE, lower="2026-07-27", upper="2026-07-27", lower_inclusive=True, @@ -727,9 +727,9 @@ def test_overlap_filter_window_is_the_range_itself(): assert TemporalFilter(overlaps=span).window == span -def test_role_only_filter_has_no_window(): +def test_kind_only_filter_has_no_window(): """Nothing to intersect: the question is only "does this axis carry an assertion".""" - assert TemporalFilter(role=TimeRole.EFFECTIVE).window is None + assert TemporalFilter(kind=TimeKind.EFFECTIVE).window is None # --- TemporalAssertion --- @@ -737,7 +737,7 @@ def test_role_only_filter_has_no_window(): def test_assertion_defaults_to_the_observation_extractor(): assertion = TemporalAssertion( - time_role=TimeRole.EFFECTIVE, + time_kind=TimeKind.EFFECTIVE, valid_during=parse_range_literal("[2026-06-10,2026-07-27)"), source_text="@effective[2026-06-10,2026-07-27)", ) @@ -746,10 +746,10 @@ def test_assertion_defaults_to_the_observation_extractor(): assert assertion.metadata is None -def test_recorded_time_is_not_an_authorable_role(): - """Recorded time is never written in markdown, so no role names it.""" - assert "recorded" not in {role.value for role in TimeRole} - assert {role.value for role in TimeRole} == { +def test_recorded_time_is_not_an_authorable_kind(): + """Recorded time is never written in markdown, so no kind names it.""" + assert "recorded" not in {kind.value for kind in TimeKind} + assert {kind.value for kind in TimeKind} == { "effective", "valid", "occurred", From 1d23d6f654efd8195d4bff20960ba1e1afcb51e9 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 12:09:31 -0500 Subject: [PATCH 04/25] fix(core): keep calendar-edge dates from failing a note's index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @effective:9999-12 constructed date(10000, 1, 1) and raised, and nothing in the chain caught it: not the qualifier reader, not the observation parser, not entity_parser. A note whose second observation carried that qualifier failed the whole document parse — its other observations and all its relations went with it. Auditing the rest of the successor arithmetic found a worse instance the report did not name: _instant_value calls astimezone(UTC), which raises OverflowError when the offset shift crosses the calendar edge. OverflowError is not a ValueError, so it escaped even the existing except clause, and the same bounds reach the search router — where ValueError maps to 400 and this was a 500. Three spellings were reachable, including an underflow at 0001-01-01. Terminal periods now render as the unbounded range they represent (@effective:9999 -> [9999-01-01,), @effective:9999-12 -> [9999-12-01,)); a year beyond the calendar stays content, and an instant that leaves the calendar in UTC is refused at the bound rather than thrown. The other six arithmetic sites were audited and are safe, each for a stated reason. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- .../markdown/temporal_qualifier.py | 4 +- src/basic_memory/temporal.py | 72 ++++++++++++---- tests/markdown/test_temporal_qualifier.py | 47 +++++++++++ tests/test_temporal.py | 83 ++++++++++++++++--- 4 files changed, 177 insertions(+), 29 deletions(-) diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py index 6334c3b46..9254288f3 100644 --- a/src/basic_memory/markdown/temporal_qualifier.py +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -248,7 +248,9 @@ def _truncation_reason(point: str, valid_during: TemporalRange) -> str | None: A bounded span is how a coarse point announces itself: `parse_authored_point` closes a year or a month at its successor and leaves a day or a moment open, so - `upper is None` *is* "this names a specific day". + `upper is None` *is* "this names a specific day". The one period with no successor + to close at -- December 9999 -- is left open too, and so reads here as a day; no word + resolves to it, so the guard never sees that shape. """ if point[0].isdigit(): return None if len(point) >= _MIN_NUMERIC_POINT_WIDTH else "is narrower than a year" diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index fa9a1ed45..60a808aa7 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -126,16 +126,26 @@ def _canonical_date(bound: str) -> str: raise TemporalQualifierError(f"not a calendar date: {bound!r}") from exc -def _instant_value(moment: datetime) -> str: +def _instant_value(moment: datetime) -> str | None: """Render one moment as the canonical fixed-width UTC instant. A naive moment is read as UTC rather than refused. That is the house convention for every other naive datetime in the codebase, and it is what lets an author write `2026-07-27T18:42:00` without learning RFC 3339's offset syntax first. + + None means the moment has no UTC rendering: shifting it by its offset carries it off + the calendar, as `9999-12-31T23:59:59-05:00` does into year 10000. Reported the way + `_next_calendar_day` reports its own edge -- each caller decides what running off the + calendar means for it -- rather than raised, so the overflow can never escape as a + bare `OverflowError` and fail a whole note's parse. """ if moment.tzinfo is None: moment = moment.replace(tzinfo=UTC) - return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" + try: + utc = moment.astimezone(UTC) + except OverflowError: + return None + return utc.strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" def _canonical_instant(bound: str) -> str: @@ -151,7 +161,12 @@ def _canonical_instant(bound: str) -> str: moment = datetime.fromisoformat(bound.upper()) except ValueError as exc: raise TemporalQualifierError(f"not a valid timestamp: {bound!r}") from exc - return _instant_value(moment) + value = _instant_value(moment) + if value is None: + raise TemporalQualifierError( + f"timestamp bound leaves the calendar when converted to UTC: {bound!r}" + ) + return value def canonical_bound(bound: str, axis: TemporalRangeAxis) -> str: @@ -478,12 +493,33 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": ) -def _calendar_span(lower: date, upper: date) -> TemporalRange: - """The half-open calendar period `[lower,upper)`.""" +def _next_month_start(year: int, month: int) -> date | None: + """The first day of the month after `year`-`month`, or None past the calendar's end. + + Only December 9999 has no successor month; year 10000 is not a date `datetime` can + hold. Reported as None for the same reason `_next_calendar_day` reports its own + edge: the caller decides what running off the end of the calendar means for it. + """ + if month < 12: + return date(year, month + 1, 1) + if year == date.max.year: + return None + return date(year + 1, 1, 1) + + +def _calendar_span(lower: date, upper: date | None) -> TemporalRange: + """The half-open calendar period `[lower,upper)`, unbounded when it runs to the end. + + A period whose successor is off the calendar needs no upper end: nothing follows + 9999-12-31, so `[lower,)` holds exactly the days `[lower,successor)` would have. It + is the same equivalence `TemporalRange` applies to an inclusive upper bound on the + last date, and it is why December 9999 is a period this reader can express rather + than one it fails on. + """ return TemporalRange( axis=TemporalRangeAxis.DATE, lower=lower.isoformat(), - upper=upper.isoformat(), + upper=None if upper is None else upper.isoformat(), lower_inclusive=True, ) @@ -533,24 +569,26 @@ def parse_authored_point( # the components `period` vouches for may be read off `moment`. match date_data.period: case "time": + instant = _instant_value(moment) + if instant is None: + # A moment that leaves the calendar in UTC names no storable instant, + # so it reads as no date at all -- the token stays content. + return None return TemporalRange( axis=TemporalRangeAxis.INSTANT, - lower=_instant_value(moment), + lower=instant, lower_inclusive=True, ) case "year": - if moment.year >= date.max.year: - # There is no January 1 after year 9999 to close the span with. - return None - return _calendar_span(date(moment.year, 1, 1), date(moment.year + 1, 1, 1)) + # The month after December is the following January 1 -- except at year + # 9999, where there is none and `_calendar_span` leaves the span open at + # `[9999-01-01,)`, which is still exactly that year. + return _calendar_span(date(moment.year, 1, 1), _next_month_start(moment.year, 12)) case "month": - first = date(moment.year, moment.month, 1) - next_month = ( - date(first.year + 1, 1, 1) - if first.month == 12 - else date(first.year, first.month + 1, 1) + return _calendar_span( + date(moment.year, moment.month, 1), + _next_month_start(moment.year, moment.month), ) - return _calendar_span(first, next_month) case _: # Day precision, and any coarser calendar period dateparser resolves to a # specific day ("last week"): the day it named, onward. diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index bd140d854..7289f3e2f 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -697,6 +697,9 @@ def test_an_unknown_kind_with_an_unreadable_payload_is_left_alone(): # A date that the calendar does not have. ("- [decision] @effective[2026-02-30,) Use Redis.", "@effective[2026-02-30,)"), ("- [decision] @2026-02-30 Use Redis.", "@2026-02-30"), + # A moment that leaves the calendar once it is shifted to UTC. + ("- [decision] @effective[9999-12-31T23:59:59-05:00,) Use Redis.", "@effective["), + ("- [decision] @effective:9999-12-31T23:59:59-05:00 Use Redis.", "@effective:"), # Trailing junk: one broken token, not a qualifier plus content. ("- [decision] @effective[2026-06-10,2026-07-27)x Use Redis.", "@effective["), ], @@ -710,6 +713,50 @@ def test_a_payload_that_does_not_read_as_time_stays_content(line: str, kept: str assert observation.content.startswith(kept) +# --- One qualifier never costs the note its index --- + + +def test_a_qualifier_at_the_end_of_the_calendar_does_not_fail_the_note(): + """Whatever a qualifier says, the rest of the note still parses. + + `@effective:9999-12` used to build `date(10000, 1, 1)`; the `ValueError` escaped + `parse_authored_point` and `parse_temporal_qualifier` -- neither of which guards that + call -- into the markdown parser, so *the whole document* failed over one qualifier: + every other observation and relation on the page went with it. December 9999 is + representable as `[9999-12-01,)`, so it files like any other period, and the + instant beside it, which is not representable at all, is simply left as content. + """ + content = "\n".join( + [ + "## Observations", + "- [decision] @effective:9999-12 The cache layer will use Redis.", + "- [decision] @effective:9999 The contract holds all year.", + "- [decision] @effective[9999-12-31T23:59:59-05:00,) An unstorable moment.", + "- [note] An ordinary observation that must still index.", + "", + "## Relations", + "- relates_to [[Cache Layer]]", + ] + ) + + parsed = parse(content) + + month, year, unstorable, ordinary = parsed.observations + [month_assertion] = month.temporal + [year_assertion] = year.temporal + assert str(month_assertion.valid_during) == "[9999-12-01,)" + assert str(year_assertion.valid_during) == "[9999-01-01,)" + assert month.content == "The cache layer will use Redis." + assert year.content == "The contract holds all year." + # Unreadable, so never peeled: the line keeps its exact text and reports nothing. + assert unstorable.temporal == [] + assert unstorable.temporal_error is None + assert unstorable.content == "@effective[9999-12-31T23:59:59-05:00,) An unstorable moment." + # The rest of the note is what the crash used to take with it. + assert ordinary.content == "An ordinary observation that must still index." + assert [relation.target for relation in parsed.relations] == ["Cache Layer"] + + def test_qualifier_with_nothing_to_qualify_stays_content(): """Peeling it would leave an empty observation, which the plugin drops outright.""" observation = _observation("- [decision] @effective[2026-06-10,2026-07-27)") diff --git a/tests/test_temporal.py b/tests/test_temporal.py index aab66ea3e..6ebf82b82 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -10,7 +10,7 @@ of the codebase applies to naive datetimes. """ -from datetime import date, datetime, timedelta +from datetime import datetime, timedelta import pytest @@ -138,6 +138,25 @@ def test_timestamp_shaped_bound_on_a_date_that_does_not_exist_is_refused(): canonical_bound("2026-02-30T10:00:00Z", INSTANT) +@pytest.mark.parametrize( + "bound", + [ + "9999-12-31T23:59:59-05:00", # 10000-01-01 in UTC + "0001-01-01T00:00:00+05:00", # year 0 in UTC + ], +) +def test_instant_bounds_that_leave_the_calendar_in_utc_are_refused(bound: str): + """Normalizing to UTC *moves* a moment, and the move can run off the calendar. + + Refused as a `TemporalQualifierError` like every other unreadable bound, which is + what makes it survivable: `datetime.astimezone` signals this with `OverflowError`, + and an `OverflowError` is not a `ValueError`, so it slipped past every handler above + -- failing a whole note's parse, or a whole search request, over one bound. + """ + with pytest.raises(TemporalQualifierError, match="leaves the calendar"): + canonical_bound(bound, INSTANT) + + # --- TemporalPoint --- @@ -359,16 +378,58 @@ def test_text_that_names_no_date_reads_as_nothing(written: str): assert parse_authored_point(written) is None -def test_a_year_with_no_successor_is_unread(): - """Year 9999 has no January 1 after it to close the span with.""" - assert parse_authored_point("9999") is None - # The year before it still resolves, so the guard is the calendar edge, not 4 digits. - assert parse_authored_point("9998") == TemporalRange( - axis=DATE, - lower=date(9998, 1, 1).isoformat(), - upper=date(9999, 1, 1).isoformat(), - lower_inclusive=True, - ) +@pytest.mark.parametrize( + ("written", "literal"), + [ + # The last year and the last month have no successor to close at, so the + # canonical form for them is unbounded -- exactly as it is for an inclusive + # upper bound on the last date. + ("9999", "[9999-01-01,)"), + ("9999-12", "[9999-12-01,)"), + # The last day was always open-ended, like every other day. + ("9999-12-31", "[9999-12-31,)"), + ], +) +def test_periods_at_the_end_of_the_calendar_run_to_the_end_of_it(written: str, literal: str): + """Unbounded above loses no days: nothing follows 9999-12-31. + + `[9999-12-01,)` holds exactly the days a closed `[9999-12-01,10000-01-01)` would -- + and year 10000 is not a date Python can build. Constructing it raised `ValueError` + straight through `parse_authored_point` and `parse_temporal_qualifier` into the + markdown parser, failing the *whole note* over one qualifier, which is the one thing + the qualifier contract promises can never happen. + """ + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.is_empty is False + + +@pytest.mark.parametrize( + ("written", "literal"), + [("9998", "[9998-01-01,9999-01-01)"), ("9999-11", "[9999-11-01,9999-12-01)")], +) +def test_the_period_before_the_calendar_edge_still_closes(written: str, literal: str): + """The open upper end is the calendar's edge, not "four digits" or "December".""" + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + + +def test_a_year_beyond_the_calendar_is_unread(): + """Year 10000 is not a date at all, so the token names nothing and stays content.""" + assert parse_authored_point("10000") is None + + +def test_an_authored_instant_that_leaves_the_calendar_in_utc_is_unread(): + """The flexible reader has no bound to refuse, so it reads no date at all. + + Its contract is None-for-unreadable, not an exception: `parse_temporal_qualifier` + does not guard this call, so anything raised here fails the note. + """ + assert parse_authored_point("9999-12-31T23:59:59-05:00") is None # --- TemporalRange normalization --- From bef6d253e458db4da74b6c492aee1aca75ae588f Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 15:28:08 -0500 Subject: [PATCH 05/25] fix(mcp): refuse malformed valid-time filters instead of answering them Two SPEC-82 paths turned a malformed temporal value into a plausible-looking answer rather than an error. **All-projects search reported "no matches" for an invalid filter.** With `search_all_projects=True`, each per-project leg converted the API's 400 on a bad `valid_at` / `valid_overlaps` / `time_kind` into a `# Search Failed` string, which the fan-out cannot tell from a project being unavailable: it logged it and skipped on. Every project skipped left an empty response that still reported `temporal_applied=True` -- a typo wearing the shape of a successful search. `search_notes` now parses the three filter strings once, before the fan-out begins, and raises naming the bad value. Per-project availability failures are still logged and skipped exactly as before; only client-side validation failures abort. The parser itself moves to `temporal.parse_temporal_filter`, so the tool's pre-check and `search_service.build_temporal_filter` can never disagree about what is well formed. **An impossible ISO timestamp was indexed as a different instant.** `@occurred:2026-13-01T10:00:00` fell through to dateparser, which reads it as 10:00 on the 13th of January and projected `[2026-01-13T10:00:00.000000Z,)`. Every reindex reproduced the same wrong instant. The date-only branch above it already took a strict path for exactly this reason; the canonical timestamp shape now takes the same one and is refused rather than reinterpreted, leaving the qualifier as ordinary observation content. Flexible spellings the canonical form does not cover (`2026-06-10 10:00 AM`, a timestamp with no seconds) still reach dateparser unchanged. Signed-off-by: phernandez --- src/basic_memory/mcp/tools/search.py | 19 +++++- src/basic_memory/services/search_service.py | 40 ++++-------- src/basic_memory/temporal.py | 61 +++++++++++++++++++ tests/markdown/test_temporal_qualifier.py | 3 + tests/mcp/test_tool_search_temporal.py | 36 +++++++++++ ...est_search_notes_multi_project_temporal.py | 43 +++++++++++++ tests/test_temporal.py | 54 +++++++++++++++- 7 files changed, 223 insertions(+), 33 deletions(-) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 3720076df..a36a2ebc7 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -39,6 +39,7 @@ SearchResult, SearchRetrievalMode, ) +from basic_memory.temporal import TemporalQualifierError, parse_temporal_filter _SERVICE_UNAVAILABLE_HEADING = "# Search Failed - Service Temporarily Unavailable" @@ -594,7 +595,9 @@ async def _search_all_projects( # Each per-project call runs through search_notes -> SearchClient, which refuses a # response that does not confirm the filter ran. So a project either honored the # valid-time filter or was dropped with a warning below; the merged answer never - # silently mixes filtered and unfiltered rows. + # silently mixes filtered and unfiltered rows. The filter itself is already known to + # be well formed -- search_notes parses it before reaching here -- which is what + # makes "dropped with a warning" mean an unavailable project and nothing else. temporal_requested = bool(valid_at or valid_overlaps or time_kind) project_refs = await _load_search_project_refs(context=context) if not project_refs: @@ -1115,6 +1118,20 @@ async def search_notes( if valid_at and valid_overlaps: raise ValueError("Use either valid_at (containment) or valid_overlaps (overlap), not both.") + # Trigger: any valid-time filter string is supplied. + # Why: these strings are parsed server-side, so a typo comes back as a 400 that the + # fan-out below cannot tell from a project being unavailable -- it logs the + # project, skips it, and after every project is skipped reports an empty result + # that still claims the filter ran. A malformed filter would read as "no matches" + # instead of as an error. This is the only layer that can tell a client mistake + # from a per-project availability failure, and it shares the parser the search + # service uses so the two can never disagree about what is well formed. + # Outcome: one error naming the bad value, before any project is searched. + try: + parse_temporal_filter(valid_at=valid_at, valid_overlaps=valid_overlaps, time_kind=time_kind) + except TemporalQualifierError as exc: + raise ValueError(f"Invalid valid-time filter: {exc}") from exc + # Trigger: list params arrived via a direct function call instead of the MCP layer. # Why: the BeforeValidator annotations only run through MCP/Pydantic validation; direct # callers (e.g. `bm tool search-notes --type note,task` in cli/commands/tool.py, diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index 5ad7d65a3..18ee46f37 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -34,10 +34,7 @@ from basic_memory.services import FileService from basic_memory.temporal import ( TemporalFilter, - TemporalQualifierError, - TimeKind, - parse_point, - parse_range_literal, + parse_temporal_filter, ) # Maximum size for content_stems field to stay under Postgres's 8KB index row limit. @@ -93,32 +90,17 @@ def entity_embeddings_enabled(entity: Entity) -> bool: def build_temporal_filter(query: SearchQuery) -> TemporalFilter | None: - """Parse the flat valid-time fields into one portable filter value. - - The boundary carries strings so HTTP and MCP callers can pass a single flat value - per field. Every rejection here is deliberate and loud: an unknown kind, a malformed - range literal, a range mixing calendar dates with instants, or an impossible range - raises rather than degrading into a filter that quietly matches something else. - Callers above map the error to a 400. A timestamp written without an offset is not - a rejection -- like every other naive datetime in the codebase, it is read as UTC. - """ - if not query.has_temporal_filter(): - return None + """Read the query's flat valid-time fields as one portable filter value. - kind: TimeKind | None = None - if query.time_kind: - try: - kind = TimeKind(query.time_kind) - except ValueError as exc: - raise TemporalQualifierError( - f"unknown time_kind {query.time_kind!r}; expected one of " - f"{', '.join(item.value for item in TimeKind)}" - ) from exc - - return TemporalFilter( - kind=kind, - at=parse_point(query.valid_at) if query.valid_at else None, - overlaps=parse_range_literal(query.valid_overlaps) if query.valid_overlaps else None, + The parsing itself lives in `temporal.parse_temporal_filter`, which every request + surface shares, so a caller that pre-validates the same three strings can never + disagree with what runs here. `TemporalQualifierError` is a `ValueError`, so callers + above map it to a 400. + """ + return parse_temporal_filter( + valid_at=query.valid_at, + valid_overlaps=query.valid_overlaps, + time_kind=query.time_kind, ) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 60a808aa7..5615b872a 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -470,6 +470,48 @@ def parse_point(text: str) -> TemporalPoint: return TemporalPoint(axis=axis, value=canonical_bound(bound, axis)) +def parse_temporal_filter( + *, + valid_at: str | None = None, + valid_overlaps: str | None = None, + time_kind: str | None = None, +) -> TemporalFilter | None: + """Parse the three flat boundary fields into one portable filter value. + + Every request surface -- HTTP, MCP, CLI -- carries a valid-time question as these + three independent strings, so this is the one place that turns them into the domain + value. Sharing it is what lets a caller validate the question *before* asking it and + be certain the answer to "is this filter well formed?" is the same one the search + service will reach. + + Every rejection is deliberate and loud: an unknown kind, a malformed range literal, a + range mixing calendar dates with instants, or an impossible range raises rather than + degrading into a filter that quietly matches something else. A timestamp written + without an offset is not a rejection -- like every other naive datetime in the + codebase, it is read as UTC. + + Returns None when no valid-time question was asked at all. + """ + if not (valid_at or valid_overlaps or time_kind): + return None + + kind: TimeKind | None = None + if time_kind: + try: + kind = TimeKind(time_kind) + except ValueError as exc: + raise TemporalQualifierError( + f"unknown time_kind {time_kind!r}; expected one of " + f"{', '.join(item.value for item in TimeKind)}" + ) from exc + + return TemporalFilter( + kind=kind, + at=parse_point(valid_at) if valid_at else None, + overlaps=parse_range_literal(valid_overlaps) if valid_overlaps else None, + ) + + # --- Flexible authored points --- @@ -560,6 +602,25 @@ def parse_authored_point( except ValueError: return None + if _INSTANT_BOUND.match(point): + # Trigger: the text is already in the canonical RFC 3339 timestamp shape. + # Why: the leniency the branch above guards against reaches timestamps too -- + # dateparser reads "2026-13-01T10:00:00" as 10:00 on the 13th of January -- and + # every reindex would project that same wrong instant, so it is worse than an + # unread token. Only the *shape* is matched here, so the flexible spellings + # dateparser alone reads ("2026-06-10 10:00 AM", a timestamp with no seconds) + # still reach it below. + # Outcome: RFC 3339 timestamps are parsed as RFC 3339, or refused. + try: + instant = _canonical_instant(point) + except TemporalQualifierError: + return None + return TemporalRange( + axis=TemporalRangeAxis.INSTANT, + lower=instant, + lower_inclusive=True, + ) + date_data = _date_data_parser(date_order).get_date_data(point) moment = date_data.date_obj if moment is None: diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index 7289f3e2f..53fd7c943 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -697,6 +697,9 @@ def test_an_unknown_kind_with_an_unreadable_payload_is_left_alone(): # A date that the calendar does not have. ("- [decision] @effective[2026-02-30,) Use Redis.", "@effective[2026-02-30,)"), ("- [decision] @2026-02-30 Use Redis.", "@2026-02-30"), + # A timestamp the calendar does not have. Read leniently it would file 10:00 on + # the 13th of January, and every reindex would project that same wrong instant. + ("- [decision] @occurred:2026-13-01T10:00:00 Use Redis.", "@occurred:2026-13-01"), # A moment that leaves the calendar once it is shifted to UTC. ("- [decision] @effective[9999-12-31T23:59:59-05:00,) Use Redis.", "@effective["), ("- [decision] @effective:9999-12-31T23:59:59-05:00 Use Redis.", "@effective:"), diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py index 3d90b852e..4aa3e2c72 100644 --- a/tests/mcp/test_tool_search_temporal.py +++ b/tests/mcp/test_tool_search_temporal.py @@ -310,6 +310,42 @@ async def test_valid_at_and_valid_overlaps_together_are_refused(client, test_pro ) +@pytest.mark.asyncio +async def test_a_malformed_valid_time_filter_is_refused_rather_than_searched(client, test_project): + """A typo in a valid-time filter is an error, not a search that finds nothing.""" + await _write_cache_layer_note(test_project.name) + + with pytest.raises(ValueError, match="2026-13-01"): + await search_notes( + project=test_project.name, + query="cache layer", + valid_at="2026-13-01", + output_format="json", + ) + + +@pytest.mark.asyncio +async def test_all_projects_search_refuses_a_malformed_filter_instead_of_reporting_nothing( + client, test_project +): + """The same typo across every project must not come back as "no matches found". + + Through the real API each per-project leg 400s on the bad bound and returns a + `# Search Failed` string, which the fan-out logs and skips as an unavailable project. + Skipping every project leaves an empty response that still claims the filter ran -- + an invalid query wearing the shape of a successful one. + """ + await _write_cache_layer_note(test_project.name) + + with pytest.raises(ValueError, match="2026-13-01"): + await search_notes( + query="cache layer", + search_all_projects=True, + valid_at="2026-13-01", + output_format="json", + ) + + @pytest.mark.asyncio async def test_time_kind_alone_is_enough_search_criteria(client, test_project): """A valid-time filter is real criteria, so it must not trip the empty-query guard.""" diff --git a/tests/mcp/tools/test_search_notes_multi_project_temporal.py b/tests/mcp/tools/test_search_notes_multi_project_temporal.py index 1dcfc91a2..85fe04963 100644 --- a/tests/mcp/tools/test_search_notes_multi_project_temporal.py +++ b/tests/mcp/tools/test_search_notes_multi_project_temporal.py @@ -141,6 +141,49 @@ async def test_all_projects_search_without_a_filter_claims_nothing(monkeypatch, assert "temporal_applied" not in result +@pytest.mark.parametrize( + ("valid_at", "valid_overlaps", "time_kind", "bad_value"), + [ + ("2026-13-01", None, None, "2026-13-01"), + (None, "2026-06-10..2026-07-27", None, "2026-06-10..2026-07-27"), + (None, None, "asserted", "asserted"), + ], +) +@pytest.mark.asyncio +async def test_a_malformed_filter_is_refused_before_any_project_is_searched( + monkeypatch, + cloud_routing, + valid_at: str | None, + valid_overlaps: str | None, + time_kind: str | None, + bad_value: str, +): + """A typo must read as an error, never as an all-projects search with no matches. + + Each per-project leg turns the API's 400 into a `# Search Failed` string, which the + fan-out cannot tell from a project being unavailable: it logs it and skips on. With + every project skipped the merged answer is an empty success that still reports + `temporal_applied`, so a mistyped filter would come back as the plausible-looking + "no matches found" for a question that never ran anywhere. Client-side validation is + the only layer that can tell the two apart, so it runs once, before the fan-out. + """ + payloads: list[dict[str, Any]] = [] + _install_stub_client(monkeypatch, payloads, PROJECT_REFS) + search_mod = importlib.import_module("basic_memory.mcp.tools.search") + + with pytest.raises(ValueError, match=bad_value): + await search_mod.search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + valid_at=valid_at, + valid_overlaps=valid_overlaps, + time_kind=time_kind, + ) + + assert payloads == [] + + @pytest.mark.asyncio async def test_all_projects_search_with_no_projects_still_confirms_the_filter( monkeypatch, cloud_routing diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 6ebf82b82..3f02e87e1 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -369,6 +369,44 @@ def test_impossible_iso_dates_are_unread_rather_than_re_interpreted(written: str assert parse_authored_point(written) is None +@pytest.mark.parametrize( + "written", + [ + "2026-13-01T10:00:00", # RFC 3339-shaped, but there is no 13th month + "2026-02-30T10:00:00Z", # RFC 3339-shaped, but February has no 30th + "2026-06-10T25:00:00+02:00", # RFC 3339-shaped, but there is no 25th hour + ], +) +def test_impossible_iso_timestamps_are_unread_rather_than_re_interpreted(written: str): + """dateparser reads `2026-13-01T10:00:00` as 10:00 on the 13th of January. + + The canonical timestamp shape takes the same strict path the canonical date shape + does, and for the same reason: an instant nobody wrote would be re-projected by every + reindex, while an unread token merely stays observation content. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + ("2026-06-10T14:00", "2026-06-10T14:00:00.000000Z"), # no seconds + ("2026-06-10 10:00 AM", "2026-06-10T10:00:00.000000Z"), # written the human way + ], +) +def test_flexible_timestamp_spellings_still_reach_the_lenient_reader(written: str, lower: str): + """Only the *canonical* timestamp shape is held to the strict parser. + + The strict branch above is a shape test, not a ban on clock readings: a spelling the + canonical form does not cover is still the convenient form, and dateparser reads it. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + @pytest.mark.parametrize( "written", ["paul", "basicmemory.com", "ops@example.com", "someone(2026)", "Redis.", "Q3"], @@ -423,13 +461,23 @@ def test_a_year_beyond_the_calendar_is_unread(): assert parse_authored_point("10000") is None -def test_an_authored_instant_that_leaves_the_calendar_in_utc_is_unread(): +@pytest.mark.parametrize( + "written", + [ + # The canonical shape, refused by the strict timestamp branch... + "9999-12-31T23:59:59-05:00", + # ...and the same moment spelled loosely, refused after dateparser reads it. + "9999-12-31 23:59:59 -05:00", + ], +) +def test_an_authored_instant_that_leaves_the_calendar_in_utc_is_unread(written: str): """The flexible reader has no bound to refuse, so it reads no date at all. Its contract is None-for-unreadable, not an exception: `parse_temporal_qualifier` - does not guard this call, so anything raised here fails the note. + does not guard this call, so anything raised here fails the note. Both spellings are + pinned because they take different routes to the same refusal. """ - assert parse_authored_point("9999-12-31T23:59:59-05:00") is None + assert parse_authored_point(written) is None # --- TemporalRange normalization --- From 4058ee236fb647bd83489f662ad6313979a2bf34 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 16:09:43 -0500 Subject: [PATCH 06/25] fix(core): refuse ISO-shaped points with impossible calendar components The strict guards added for the canonical `YYYY-MM-DD` and `T`-separated timestamp shapes matched a token only when it was *exactly* one of them, so every other ISO-shaped spelling still reached dateparser: @occurred:2026-13 -> [2026-09-13,) (on 2026-09-01) @occurred:"2026-13-01 10:00:00" -> [2026-01-13T10:00:00.000000Z,) @occurred:2026-13-01T10:00 -> [2026-01-13T10:00:00.000000Z,) dateparser reads the impossible month as a *day* and then fills the month it never got from the current date. `2026-13` is the worst of these: the same note projects a different date on every reindex day, so a query that matched it last week can stop matching it today with nothing having been edited. Both were also peeled off the observation content, so the wrong date replaced the author's text rather than sitting beside it. Extend the same strict-branch pattern one step further out: validate the ISO calendar components a point *opens* with, and refuse the point when they name no real date. Only the leading date is judged, so the flexible spellings keep working -- `2026-06-10 10:00 AM`, `2026-06-10T14:00`, `2026-06`, `2026-1-5`, `10/07/2026` and every natural-language phrase go on reaching dateparser untouched, and an impossible *time* already read as no date at all. Refusal is `None`, as in the neighbouring branches: the token stays ordinary observation content, unindexed but still full-text searchable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 38 +++++++++++ tests/markdown/test_temporal_qualifier.py | 8 +++ tests/test_temporal.py | 81 +++++++++++++++++++++++ 3 files changed, 127 insertions(+) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 5615b872a..30f2768d3 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -535,6 +535,28 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": ) +# The ISO calendar components a point opens with, when it opens with any: `YYYY-MM` +# optionally followed by `-DD`. Not a bound grammar -- it deliberately matches a *prefix* +# and leaves whatever follows (a time, an offset, nothing) to the reader below. The +# lookahead is what keeps it to ISO shapes: `2026-06-100` and `2026-06-1` are not ones, +# so they are none of this guard's business. +_ISO_CALENDAR_PREFIX = re.compile(r"^(\d{4})-(\d{2})(?:-(\d{2}))?(?![\d-])") + + +def _names_a_real_calendar_date(year: str, month: str, day: str | None) -> bool: + """Whether ISO-shaped calendar components name a date that exists. + + A month-only prefix is judged on the first of that month: the day is a component the + author did not write, not one to guess at. `date` is the authority rather than a range + check because it already owns leap years and month lengths. + """ + try: + date(int(year), int(month), 1 if day is None else int(day)) + except ValueError: + return False + return True + + def _next_month_start(year: int, month: int) -> date | None: """The first day of the month after `year`-`month`, or None past the calendar's end. @@ -621,6 +643,22 @@ def parse_authored_point( lower_inclusive=True, ) + iso_calendar = _ISO_CALENDAR_PREFIX.match(point) + if iso_calendar is not None and not _names_a_real_calendar_date(*iso_calendar.groups()): + # Trigger: the text opens with ISO calendar components that name no real date. + # Why: the two branches above only match a token that is *exactly* a canonical + # date or timestamp, so every other ISO-shaped spelling still reached + # dateparser -- a bare `2026-13`, a space-separated `2026-13-01 10:00:00`, a + # minute-precision `2026-13-01T10:00`. It reads the impossible month as a day + # and then fills the month it never got from *today*, so `2026-13` projects a + # different date on every reindex day: the same note yields different data + # depending on when it was indexed. Only the leading calendar components are + # judged here -- an impossible *time* already reads as no date at all -- so + # every non-ISO spelling and every ISO date that does exist still reach the + # flexible reader below. + # Outcome: refused, and the token stays ordinary observation content. + return None + date_data = _date_data_parser(date_order).get_date_data(point) moment = date_data.date_obj if moment is None: diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index 53fd7c943..addb72381 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -700,6 +700,14 @@ def test_an_unknown_kind_with_an_unreadable_payload_is_left_alone(): # A timestamp the calendar does not have. Read leniently it would file 10:00 on # the 13th of January, and every reindex would project that same wrong instant. ("- [decision] @occurred:2026-13-01T10:00:00 Use Redis.", "@occurred:2026-13-01"), + # The same impossible month in the spellings the canonical shapes do not cover: + # a bare year-month, and a quoted space-separated timestamp. Both used to be + # peeled off the line *and* filed as a date in some other month. + ("- [decision] @occurred:2026-13 Use Redis.", "@occurred:2026-13"), + ( + '- [decision] @occurred:"2026-13-01 10:00:00" Use Redis.', + '@occurred:"2026-13-01 10:00:00"', + ), # A moment that leaves the calendar once it is shifted to UTC. ("- [decision] @effective[9999-12-31T23:59:59-05:00,) Use Redis.", "@effective["), ("- [decision] @effective:9999-12-31T23:59:59-05:00 Use Redis.", "@effective:"), diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 3f02e87e1..f34d9605b 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -13,6 +13,7 @@ from datetime import datetime, timedelta import pytest +from freezegun import freeze_time from basic_memory.temporal import ( DEFAULT_DATE_ORDER, @@ -407,6 +408,86 @@ def test_flexible_timestamp_spellings_still_reach_the_lenient_reader(written: st assert span.lower == lower +@pytest.mark.parametrize( + "written", + [ + # A month that does not exist, with nothing after it. The two strict branches + # above match only a token that is *exactly* a canonical date or timestamp, so + # this shape used to reach dateparser untouched. + "2026-13", + "2026-00", + # ...the same, carrying a time the canonical shape does not cover: separated by + # a space rather than `T`, or written to minute precision. + "2026-13-01 10:00:00", + "2026-13-01 10:00", + "2026-13-01T10:00", + "2026-02-30 10:00:00", + "2026-13-01 10:00:00Z", + "2026-06-31T09:30", + ], +) +def test_iso_shaped_points_with_impossible_components_are_unread(written: str): + """An ISO-shaped point must mean its components literally, whatever trails it. + + dateparser reads month 13 as *day* 13 and then supplies the month from today, so + these all used to file a date nobody wrote. Guarding only the two canonical shapes + left every other ISO spelling -- a bare year-month, a space-separated timestamp, a + minute-precision one -- on the lenient path. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) +def test_an_impossible_iso_month_is_not_completed_from_the_indexing_date(today: str): + """The worst shape of all: a date whose meaning depended on when the reindex ran. + + `2026-13` gave dateparser a year and a day but no month, and it filled the gap from + the current date -- `[2026-03-13,)` in March, `[2026-09-13,)` in September. The same + note projected different valid time on different days, so a query that matched it + last week could stop matching it today with nothing having been edited. + """ + with freeze_time(today): + assert parse_authored_point("2026-13") is None + + +@pytest.mark.parametrize( + ("written", "literal", "axis"), + [ + # A real year-month, which is still read as the month it delimits. + ("2026-06", "[2026-06-01,2026-07-01)", DATE), + ("9999-12", "[9999-12-01,)", DATE), + # A real date carrying a time the canonical `T` shape does not cover. These are + # the spellings the guard above is closest to, so they are pinned explicitly. + ("2026-06-10 14:00:00", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10 14:00:00Z", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10 14:00:00+02:00", "[2026-06-10T12:00:00.000000Z,)", INSTANT), + ("2026-06-10 10:00", "[2026-06-10T10:00:00.000000Z,)", INSTANT), + ("2026-06-10T14:00", "[2026-06-10T14:00:00.000000Z,)", INSTANT), + ("2026-06-10 10:00 AM", "[2026-06-10T10:00:00.000000Z,)", INSTANT), + # Not ISO-shaped at all: single-digit components, slashes, words, relative + # phrases. The guard must not so much as look at these. + ("2026-1-5", "[2026-01-05,)", DATE), + ("2026/03/04", "[2026-03-04,)", DATE), + ("June 10, 2026", "[2026-06-10,)", DATE), + ], +) +def test_the_iso_guard_leaves_every_readable_spelling_to_the_lenient_reader( + written: str, literal: str, axis +): + """The guard is a validity test on ISO components, not a ban on flexible spellings. + + Refusing an impossible ISO date must cost nothing that already reads. Anything whose + leading components name a real date -- and anything not ISO-shaped at all -- goes on + reaching dateparser exactly as before, so a future tightening cannot quietly take + these spellings without failing here. + """ + span = parse_authored_point(written) + + assert span is not None + assert str(span) == literal + assert span.axis is axis + + @pytest.mark.parametrize( "written", ["paul", "basicmemory.com", "ops@example.com", "someone(2026)", "Redis.", "Q3"], From 46c95fb377f0dbef74360853363482aa5bc5a33a Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 16:30:08 -0500 Subject: [PATCH 07/25] fix(core): refuse ISO points whose calendar runs are the wrong width The ISO guard matched calendar components at a fixed two-digit width, so a run of the wrong width matched nothing at all and fell through to dateparser -- the one outcome the guard exists to prevent. `@occurred:2026-01-0100` came back as `[2026-01-01,2026-02-01)` and `@occurred:2026-0100` as the whole of 2026, so a slipped keystroke silently widened one day into a range nobody wrote, and every reindex projected it again. Match each component as an unbounded run and judge it, rather than bounding the run and letting a wider one escape. Width is what separates an author's shorthand from an author's typo: a month or a day is written with one or two digits, so `2026-1-5` stays a legitimate unpadded spelling while the `0100` in `2026-01-0100` is no day at all. Width is checked before conversion because `date` takes a C long and raises OverflowError -- not the ValueError the guard catches -- once a run of digits grows past it. Every flexible spelling still reads exactly as before: `2026-06-10 10:00 AM`, `2026-06-10T14:00`, `2026-06`, `9999-12`, `2026-1-5`, `2026/03/04`, `June 10, 2026`. The lookahead still ends the run at the first character that cannot continue a calendar date, so a date carrying a time is matched on its date part alone. Signed-off-by: phernandez --- src/basic_memory/temporal.py | 31 +++++++++++++++++++++++-------- tests/test_temporal.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 30f2768d3..3139343fc 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -537,10 +537,16 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": # The ISO calendar components a point opens with, when it opens with any: `YYYY-MM` # optionally followed by `-DD`. Not a bound grammar -- it deliberately matches a *prefix* -# and leaves whatever follows (a time, an offset, nothing) to the reader below. The -# lookahead is what keeps it to ISO shapes: `2026-06-100` and `2026-06-1` are not ones, -# so they are none of this guard's business. -_ISO_CALENDAR_PREFIX = re.compile(r"^(\d{4})-(\d{2})(?:-(\d{2}))?(?![\d-])") +# and leaves whatever follows (a time, an offset, nothing) to the reader below. +# +# Each component is `\d+` rather than `\d{2}` so that an over-long run is *captured and +# judged* rather than failing to match and slipping past the guard entirely. Bounding the +# run instead would reopen the hole it closes: against `\d{2}`, `2026-01-0100` matched +# nothing -- the day `01` left a trailing `00` that the lookahead refused -- so the token +# reached dateparser and came back as the whole month of January. The lookahead still ends +# the run at the first character that cannot continue a calendar date, so a date carrying a +# time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part alone. +_ISO_CALENDAR_PREFIX = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?(?![\d-])") def _names_a_real_calendar_date(year: str, month: str, day: str | None) -> bool: @@ -550,6 +556,13 @@ def _names_a_real_calendar_date(year: str, month: str, day: str | None) -> bool: author did not write, not one to guess at. `date` is the authority rather than a range check because it already owns leap years and month lengths. """ + # A month or a day is written with one or two digits, and that width is what separates + # an author's shorthand from an author's typo: `2026-1-5` is a legitimate unpadded + # spelling of a real date, while the `0100` in `2026-01-0100` is no day at all. Judged + # before `date`, which takes a C long and raises OverflowError -- not the ValueError + # below -- once a run of digits grows past it. + if len(month) > 2 or (day is not None and len(day) > 2): + return False try: date(int(year), int(month), 1 if day is None else int(day)) except ValueError: @@ -652,10 +665,12 @@ def parse_authored_point( # minute-precision `2026-13-01T10:00`. It reads the impossible month as a day # and then fills the month it never got from *today*, so `2026-13` projects a # different date on every reindex day: the same note yields different data - # depending on when it was indexed. Only the leading calendar components are - # judged here -- an impossible *time* already reads as no date at all -- so - # every non-ISO spelling and every ISO date that does exist still reach the - # flexible reader below. + # depending on when it was indexed. A mistyped *width* is read just as freely -- + # `2026-01-0100` comes back as the whole month of January -- so a slipped + # keystroke silently widens one day into a range nobody wrote. Only the leading + # calendar components are judged here -- an impossible *time* already reads as no + # date at all -- so every non-ISO spelling and every ISO date that does exist + # still reach the flexible reader below. # Outcome: refused, and the token stays ordinary observation content. return None diff --git a/tests/test_temporal.py b/tests/test_temporal.py index f34d9605b..897d13627 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -437,6 +437,42 @@ def test_iso_shaped_points_with_impossible_components_are_unread(written: str): assert parse_authored_point(written) is None +@pytest.mark.parametrize( + "written", + [ + # The reported shape: a mistyped ISO date whose day ran on into a fourth digit. + # dateparser chopped it back to a bare year-month and filed the whole of January, + # so one slipped keystroke widened a single day into a month-long range. + "2026-01-0100", + # The same slip one component earlier, which filed the whole *year* 2026. + "2026-0100", + # Shorter over-long runs. dateparser already declined to read these, but they are + # the same malformed shape and the guard now owns them rather than trusting it to. + "2026-013", + "2026-06-100", + "2026-01-011", + # An unpadded component is a legitimate spelling, so width alone cannot decide: + # these are refused for their values, exactly as their zero-padded twins are. + "2026-1-99", + "2026-0-5", + # A run long enough to overflow the C long `date` converts to. Refused on width + # before conversion, so the guard reads it as no date rather than raising. + "2026-" + "9" * 40, + ], +) +def test_iso_shaped_points_with_malformed_calendar_runs_are_unread(written: str): + """A mistyped ISO point must stay content, not round off into a plausible range. + + The guard's first cut matched calendar components at a fixed width, so a run of the + wrong width matched *nothing* and fell through to dateparser untouched -- the one + outcome the guard exists to prevent. `2026-01-0100` came back as + `[2026-01-01,2026-02-01)`: a whole month, indistinguishable in the index from a range + the author meant to write. A silently wrong date is worse than an unread token, so a + component too wide to be a month or a day now fails the guard on that basis. + """ + assert parse_authored_point(written) is None + + @pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) def test_an_impossible_iso_month_is_not_completed_from_the_indexing_date(today: str): """The worst shape of all: a date whose meaning depended on when the reindex ran. From 6b92d37fab981bfd05f3707c39863b04558d12c6 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 16:53:21 -0500 Subject: [PATCH 08/25] fix(core): judge an ISO-shaped point on its whole text The ISO guard validated only the calendar prefix, so a token that was well formed in front and malformed behind fell through to dateparser, which drops a suffix it cannot use and answers with the date alone: @occurred:2026-01-01T -> [2026-01-01,) @occurred:2026-01-01Z -> [2026-01-01,) @occurred:2026-01-01+14:00 -> [2026-01-01,) The author reached for an instant and the index recorded a whole open-ended day, peeled off the observation and re-derived identically by every reindex. This is the fourth cut at the same guard, and the three before it were each too narrow in the same way: a bounded or prefix-only test lets a wider malformed shape slip past, because "no match" means "not this guard's business". So the fix is not another branch for the reported suffixes. Naming reserved markers would have caught exactly those three and missed `2026-01-01UTC`, `2026-01-01,` and `2026-01-01-`, and would have missed the worse defect entirely: a stray character makes dateparser abandon the ISO reading and re-guess the components under the configured order, so `2026-06-10x` came back as *October 6* -- the date itself moved. Judge the whole token instead. An ISO-shaped point has exactly two halves, and both are now checked: * The calendar head must name a real date. Its trailing `(?![\d-])` lookahead is dropped, because that lookahead was the same hole on the suffix side: `2026-01-01-` matched nothing at all and so skipped the guard. A head that always matches leaves a remainder always judged. * Whatever trails the head must be a time of day on that very date -- the only thing that can legally follow a complete calendar date. The second test is stated on what the reader *returned*, not on what the suffix looks like, and that is what closes the class rather than three examples: any trailing text dateparser silently drops or reinterprets fails it, anticipated or not. It also catches a shape nobody reported -- `2026-06 10:00` gave dateparser a clock but no day, and it filled the day from *today*, so the note projected `2026-06-07` in March and `2026-06-01` in September. Because the rule asks the reader rather than parsing the suffix, no readable spelling is lost, including ones no grammar would have admitted: `2026-06-10 10:00 AM`, `2026-06-10T14:00`, `2026-06-10 14:00:00+02:00`, `2026-06-10 noon`, `2026-06-10 2pm`, `2026-06`, `9999-12`, `2026-1-5`, `2026/03/04`, `June 10, 2026` all read exactly as before. The strict range-literal path already validated its bounds whole -- `[2026-01-01T,)` was always a hard error -- so this brings the lenient path to the same standard: the reader must account for everything the author typed. Refusal is `None`, as in the neighbouring branches: the token stays ordinary observation content, unindexed but still full-text searchable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 104 ++++++++++++++++------ tests/markdown/test_temporal_qualifier.py | 13 +++ tests/test_temporal.py | 104 ++++++++++++++++++++++ 3 files changed, 192 insertions(+), 29 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 3139343fc..30a9dc1aa 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -535,24 +535,31 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": ) -# The ISO calendar components a point opens with, when it opens with any: `YYYY-MM` -# optionally followed by `-DD`. Not a bound grammar -- it deliberately matches a *prefix* -# and leaves whatever follows (a time, an offset, nothing) to the reader below. +# The ISO calendar components a point *opens* with, when it opens with any: `YYYY-MM` +# optionally followed by `-DD`. The head of a point, not the whole of one -- a date +# carrying a time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part +# alone, because `\d+` cannot cross the separator. # -# Each component is `\d+` rather than `\d{2}` so that an over-long run is *captured and -# judged* rather than failing to match and slipping past the guard entirely. Bounding the -# run instead would reopen the hole it closes: against `\d{2}`, `2026-01-0100` matched -# nothing -- the day `01` left a trailing `00` that the lookahead refused -- so the token -# reached dateparser and came back as the whole month of January. The lookahead still ends -# the run at the first character that cannot continue a calendar date, so a date carrying a -# time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part alone. -_ISO_CALENDAR_PREFIX = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?(?![\d-])") +# Two rules keep the head from letting a malformed token escape by simply failing to +# match, which is the shape every earlier cut of this guard was wrong in: +# +# * Each component is `\d+` rather than `\d{2}`, so an over-long run is *captured and +# judged* rather than matching nothing. Against `\d{2}`, `2026-01-0100` matched nothing +# -- the day `01` left a trailing `00` no lookahead would accept -- so the token reached +# dateparser and came back as the whole month of January. +# * Nothing terminates the pattern. An earlier cut ended it with `(?![\d-])`, which put +# the same hole on the trailing side: `2026-01-01-` matched nothing at all, so the +# dangling separator reached dateparser, which dropped it and filed a bare date. +# +# A head that always matches when a point opens with ISO components leaves a remainder +# that `parse_authored_point` always judges. Between them the two cover the whole token. +_ISO_CALENDAR_HEAD = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?") def _names_a_real_calendar_date(year: str, month: str, day: str | None) -> bool: """Whether ISO-shaped calendar components name a date that exists. - A month-only prefix is judged on the first of that month: the day is a component the + A month-only head is judged on the first of that month: the day is a component the author did not write, not one to guess at. `date` is the authority rather than a range check because it already owns leap years and month lengths. """ @@ -618,6 +625,12 @@ def parse_authored_point( still holds, so closing the range at midnight would expire it overnight. Callers that need a closed interval write the range literal instead. + Non-ISO spellings are read leniently, because guessing at `June 10, 2026` is the + whole point of this reader. A token that *is* ISO-shaped is held to its own text + instead: its calendar components must name a real date, and anything trailing them + must be a time of day on that date. `2026-06-10 10:00 AM` reads; `2026-01-01T` does + not, because the author reached for an instant and no instant is there. + Returns None when the text names no date. That is not an error -- the caller leaves such a token as ordinary observation content. """ @@ -656,29 +669,62 @@ def parse_authored_point( lower_inclusive=True, ) - iso_calendar = _ISO_CALENDAR_PREFIX.match(point) - if iso_calendar is not None and not _names_a_real_calendar_date(*iso_calendar.groups()): - # Trigger: the text opens with ISO calendar components that name no real date. - # Why: the two branches above only match a token that is *exactly* a canonical - # date or timestamp, so every other ISO-shaped spelling still reached - # dateparser -- a bare `2026-13`, a space-separated `2026-13-01 10:00:00`, a - # minute-precision `2026-13-01T10:00`. It reads the impossible month as a day - # and then fills the month it never got from *today*, so `2026-13` projects a - # different date on every reindex day: the same note yields different data - # depending on when it was indexed. A mistyped *width* is read just as freely -- - # `2026-01-0100` comes back as the whole month of January -- so a slipped - # keystroke silently widens one day into a range nobody wrote. Only the leading - # calendar components are judged here -- an impossible *time* already reads as no - # date at all -- so every non-ISO spelling and every ISO date that does exist - # still reach the flexible reader below. - # Outcome: refused, and the token stays ordinary observation content. - return None + # --- An ISO-shaped point is judged on its whole text --- + # + # Everything past this section is read by dateparser, which answers "what date can I + # find in here?" rather than "does this text name a date?". It reads past what it does + # not understand, so an ISO-shaped token is checked in both halves: the calendar head + # must name a real date, and whatever trails that head must be accounted for. + iso_head = _ISO_CALENDAR_HEAD.match(point) + iso_day: date | None = None + trailing = "" + if iso_head is not None: + year, month, day = iso_head.groups() + if not _names_a_real_calendar_date(year, month, day): + # Trigger: the text opens with ISO calendar components that name no real date. + # Why: the two branches above only match a token that is *exactly* a canonical + # date or timestamp, so every other ISO-shaped spelling still reached + # dateparser -- a bare `2026-13`, a space-separated `2026-13-01 10:00:00`, a + # minute-precision `2026-13-01T10:00`. It reads the impossible month as a day + # and then fills the month it never got from *today*, so `2026-13` projects a + # different date on every reindex day: the same note yields different data + # depending on when it was indexed. A mistyped *width* is read just as freely + # -- `2026-01-0100` comes back as the whole month of January -- so a slipped + # keystroke silently widens one day into a range nobody wrote. + # Outcome: refused, and the token stays ordinary observation content. + return None + # Vouched for just above, so building the date cannot raise. None means the author + # wrote a month, which owns no day for a trailing time to fall on. + iso_day = None if day is None else date(int(year), int(month), int(day)) + trailing = point[iso_head.end() :] date_data = _date_data_parser(date_order).get_date_data(point) moment = date_data.date_obj if moment is None: return None + if trailing and not (date_data.period == "time" and moment.date() == iso_day): + # Trigger: an ISO calendar head is followed by text the reader did not turn into + # a time of day on that very date. + # Why: a calendar date is a complete point, so the only thing that can legally + # follow one is a clock reading. dateparser does not enforce that -- it drops a + # suffix it cannot use and answers with the date alone, so `2026-01-01T`, + # `2026-01-01Z` and `2026-01-01+14:00` all came back as `[2026-01-01,)`: the + # author reached for an instant and the index recorded a whole open-ended day, + # re-derived identically by every reindex. Worse, a suffix can make the reader + # abandon the ISO reading altogether and re-guess the *date* under the + # configured order -- `2026-06-10x` came back as October 6 -- or fill a + # component from *today*, so `2026-06 10:00` (a clock reading on a head that + # names no day) landed on a different date depending on when it was indexed. + # Checking what the reader *returned* rather than what the suffix looks like is + # what makes this close the class: any trailing text the reader silently drops + # or reinterprets fails here, whether or not it is a shape anyone anticipated. + # The flexible spellings are untouched, because in every one of them the trailing + # text really is the time it looks like: `T14:00`, ` 10:00 AM`, ` 14:00:00+02:00` + # and even ` noon` all come back as an instant on the date the author wrote. + # Outcome: refused, and the token stays ordinary observation content. + return None + # dateparser fills components the author did not write from today's date, so only # the components `period` vouches for may be read off `moment`. match date_data.period: diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index addb72381..47f71dd98 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -713,6 +713,19 @@ def test_an_unknown_kind_with_an_unreadable_payload_is_left_alone(): ("- [decision] @effective:9999-12-31T23:59:59-05:00 Use Redis.", "@effective:"), # Trailing junk: one broken token, not a qualifier plus content. ("- [decision] @effective[2026-06-10,2026-07-27)x Use Redis.", "@effective["), + # The same rule for the point form. A calendar date carrying an instant marker + # with no instant behind it used to be peeled off the line and filed as a bare + # date, so the author reached for a moment and the index recorded an open-ended + # day -- and reproduced it on every reindex. + ("- [decision] @occurred:2026-01-01T Use Redis.", "@occurred:2026-01-01T"), + ("- [decision] @occurred:2026-01-01Z Use Redis.", "@occurred:2026-01-01Z"), + ( + "- [decision] @occurred:2026-01-01+14:00 Use Redis.", + "@occurred:2026-01-01+14:00", + ), + # A stray keystroke that moved the date itself: June 10 was peeled off the line + # and filed as October 6. + ("- [decision] @occurred:2026-06-10x Use Redis.", "@occurred:2026-06-10x"), ], ) def test_a_payload_that_does_not_read_as_time_stays_content(line: str, kept: str): diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 897d13627..84678894a 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -473,6 +473,110 @@ def test_iso_shaped_points_with_malformed_calendar_runs_are_unread(written: str) assert parse_authored_point(written) is None +@pytest.mark.parametrize( + "written", + [ + # The reported shapes: a calendar date carrying an instant marker with no instant + # behind it. dateparser drops the marker and answers with the bare date, so the + # author reached for a moment and the index recorded a whole open-ended day. + "2026-01-01T", + "2026-01-01Z", + "2026-01-01+14:00", # a real UTC offset -- with no time for it to offset + "2026-01-01-05:00", + # The same defect wearing shapes nobody listed. Naming the marker would have + # caught the three above and missed each of these, which is why the guard asks + # what the reader *returned* rather than what the suffix looks like. + "2026-01-01UTC", + "2026-01-01TZ", + "2026-01-01T ", + "2026-01-01,", + "2026-01-01.", + # A dangling separator, which the guard's previous cut could not even see: its + # trailing `(?![\\d-])` lookahead made the head fail to match, so the token + # skipped the guard entirely and reached the lenient reader. + "2026-01-01-", + "2026-01-01-5", + ], +) +def test_iso_dates_with_a_dangling_instant_suffix_are_unread(written: str): + """A calendar date is a complete point, so only a clock reading may follow one. + + Each of these was peeled off its observation and filed as `[2026-01-01,)` -- a + plausible-looking assertion the author never wrote, re-derived identically by every + reindex. The guard is stated on the whole token rather than on a list of suffixes: + what the reader hands back must account for everything the author typed. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + "written", + [ + # A stray character next to an ISO date makes dateparser abandon the ISO reading + # and re-guess the components under the configured order: June 10 became + # *October 6*. Worse than the dangling markers above, which at least kept the day. + "2026-06-10x", + "2026-06x", + # The same re-guess with a clock reading present, so the reader does come back + # with an instant -- on the wrong date. Only comparing that date against the one + # the author wrote catches it. + "2026-06-10 14:00 x", + "2026-06-10 x 14:00", + # A relative phrase after an absolute date: the reader answers with *today* + # shifted, and the ISO date the author wrote is nowhere in the result. + "2026-06-10 tomorrow 14:00", + "2026-06-10T14:00 yesterday", + ], +) +def test_an_iso_date_whose_suffix_re_guesses_it_is_unread(written: str): + """The reader must come back with the date the author wrote, not a nearby one.""" + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) +def test_a_clock_reading_on_a_month_is_not_completed_from_the_indexing_date(today: str): + """`2026-13`'s disease in the suffix: a time of day needs a day to fall on. + + `2026-06 10:00` gave dateparser a year, a month and a clock but no day, and it filled + the day from the current date -- `[2026-06-07T10:00...,)` in March, + `[2026-06-01T10:00...,)` in September. The same note projected different valid time on + different days. A head that names only a month owns no day, so nothing may trail it. + """ + with freeze_time(today): + assert parse_authored_point("2026-06 10:00") is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # The boundary the suffix rule draws is "did the reader turn this into a time on + # that date?", not "does this look like a clock?". These carry no colon and no + # digit at all, yet each really is the time it claims to be, so each still reads. + ("2026-06-10 noon", "2026-06-10T12:00:00.000000Z"), + ("2026-06-10 midnight", "2026-06-10T00:00:00.000000Z"), + ("2026-06-10 2pm", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 at 14:00", "2026-06-10T14:00:00.000000Z"), + # An offset and a zone are only dangling when there is no time in front of them. + ("2026-06-10T14:00Z", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 14:00:00.5", "2026-06-10T14:00:00.500000Z"), + ("2026-06-10 14:00:00 UTC", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10T14:00:00+0200", "2026-06-10T12:00:00.000000Z"), + ], +) +def test_a_real_time_of_day_still_follows_an_iso_date(written: str, lower: str): + """Refusing a dangling suffix must not cost a genuine one. + + A rule written as a grammar for what may follow a date would have taken these with + it: none of them is RFC 3339, and half of them do not start with a digit. Deciding on + the reader's answer instead leaves every spelling it can genuinely read. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + @pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) def test_an_impossible_iso_month_is_not_completed_from_the_indexing_date(today: str): """The worst shape of all: a date whose meaning depended on when the reindex ran. From e447fd9a666de2ea4521d59c846435b4432bc150 Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 1 Sep 2026 17:16:42 -0500 Subject: [PATCH 09/25] refactor(core): classify an authored point instead of guarding it The previous commit closed the dangling-suffix hole by adding a fourth rejection clause to `parse_authored_point`. That is the shape the file had been drifting into, and `.agents/skills/pythonic-code/SKILL.md` names it in its opening paragraph: describe what the program supports, rather than start from a broad representation and grow a list of invalid combinations. Four rounds of review had appended four such clauses -- exact `_DATE_BOUND`, exact `_INSTANT_BOUND`, an ISO-component check, then the trailing-text check -- and the reason each round found a new gap is structural, not an oversight. Every clause tested a *shape*, and failing a shape test meant "not my business", so the token fell through to dateparser, which guesses. The next round then found another shape that failed the test. Model the positive space instead. An author writes a point in one of two languages, and they come with opposite promises: ISO calendar syntax fixes its own meaning and must be read literally; anything else (`June 10, 2026`, `2026/03/04`, `yesterday`) has no literal reading for a guess to contradict, so the flexible reader is trusted with it. A token is classified into that closed union once: type _AuthoredPoint = _IsoDay | _IsoMonth | _MalformedIso | _FlexiblePoint consumed with an exhaustive `match` and `assert_never`, following `EntityVectorPreparePlan` and the other closed unions in this codebase. What closes the class is that `_classify_authored_point` is *total*: opening with ISO syntax settles the question, and the three ISO variants are all a token can then be. There is no fourth answer and no fall-through, so no ISO-shaped token can reach an unvalidated guess -- `2026-13-01`, `2026-01-0100`, `2026-0100`, `2026-01-01T`, `2026-01-01Z`, `2026-01-01+14:00`, `2026-01-01-` and `2026-06-10x` are all just `_MalformedIso`, with no clause of their own. A fifth shape has nowhere to arrive. Two variants carry their invariant in their type rather than in a check: * `_IsoDay` holds the day the author wrote, so the flexible reader can only ever supply the *clock* -- its answer is checked against that day. * `_IsoMonth` has nowhere to put trailing text, which makes `2026-06 10:00` unrepresentable rather than merely rejected. That one mattered: a clock reading needs a day to fall on, and dateparser filled the missing day from *today*, so it read as June 7 in March and June 1 in September. Observable behavior is unchanged apart from the refusals the previous commit introduced; the full unit suite (6552 tests) and the ten temporal suites pass, and `test_the_iso_guard_leaves_every_readable_spelling_to_the_lenient_reader` is untouched and green. The split also surfaced a real gap the old shared code path hid: the flexible reader's instant branch had only ever been exercised by ISO-shaped tokens, so nothing pinned `@occurred:"June 10, 2026 2pm"` or `@occurred:"10/07/2026 14:00"`. Both are supported forms, and both are now tested. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 348 ++++++++++++++++++++--------------- tests/test_temporal.py | 38 +++- 2 files changed, 238 insertions(+), 148 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 30a9dc1aa..63e5d8abd 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -34,9 +34,11 @@ * A **range literal** (`[2026-06-10,2026-07-27)`) is the precise form. Its bounds must be written in the canonical lexical shapes above, to at most microsecond precision. -* A **point** (`2026-06-10`, `2026-06`, `2026`, `yesterday`) is the convenient form. - It is read with `dateparser` and denotes the span its precision covers, so an author - never has to spell out a range to say when something started. +* A **point** (`2026-06-10`, `2026-06`, `2026`, `yesterday`) is the convenient form. It + denotes the span its precision covers, so an author never has to spell out a range to + say when something started. A point written in ISO calendar syntax is read literally, + because its text fixes its meaning; any other spelling is read with `dateparser`, + because there is no literal reading for a guess to contradict. """ import re @@ -44,7 +46,7 @@ from datetime import UTC, date, datetime, timedelta from enum import StrEnum from functools import lru_cache -from typing import TYPE_CHECKING, Any, Literal, override +from typing import TYPE_CHECKING, Any, Literal, assert_never, override if TYPE_CHECKING: # pragma: no cover - import exists only for the annotation below from dateparser.date import DateDataParser @@ -535,48 +537,6 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": ) -# The ISO calendar components a point *opens* with, when it opens with any: `YYYY-MM` -# optionally followed by `-DD`. The head of a point, not the whole of one -- a date -# carrying a time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part -# alone, because `\d+` cannot cross the separator. -# -# Two rules keep the head from letting a malformed token escape by simply failing to -# match, which is the shape every earlier cut of this guard was wrong in: -# -# * Each component is `\d+` rather than `\d{2}`, so an over-long run is *captured and -# judged* rather than matching nothing. Against `\d{2}`, `2026-01-0100` matched nothing -# -- the day `01` left a trailing `00` no lookahead would accept -- so the token reached -# dateparser and came back as the whole month of January. -# * Nothing terminates the pattern. An earlier cut ended it with `(?![\d-])`, which put -# the same hole on the trailing side: `2026-01-01-` matched nothing at all, so the -# dangling separator reached dateparser, which dropped it and filed a bare date. -# -# A head that always matches when a point opens with ISO components leaves a remainder -# that `parse_authored_point` always judges. Between them the two cover the whole token. -_ISO_CALENDAR_HEAD = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?") - - -def _names_a_real_calendar_date(year: str, month: str, day: str | None) -> bool: - """Whether ISO-shaped calendar components name a date that exists. - - A month-only head is judged on the first of that month: the day is a component the - author did not write, not one to guess at. `date` is the authority rather than a range - check because it already owns leap years and month lengths. - """ - # A month or a day is written with one or two digits, and that width is what separates - # an author's shorthand from an author's typo: `2026-1-5` is a legitimate unpadded - # spelling of a real date, while the `0100` in `2026-01-0100` is no day at all. Judged - # before `date`, which takes a C long and raises OverflowError -- not the ValueError - # below -- once a run of digits grows past it. - if len(month) > 2 or (day is not None and len(day) > 2): - return False - try: - date(int(year), int(month), 1 if day is None else int(day)) - except ValueError: - return False - return True - - def _next_month_start(year: int, month: int) -> date | None: """The first day of the month after `year`-`month`, or None past the calendar's end. @@ -608,123 +568,178 @@ def _calendar_span(lower: date, upper: date | None) -> TemporalRange: ) -def parse_authored_point( - text: str, *, date_order: DateOrder = DEFAULT_DATE_ORDER -) -> TemporalRange | None: - """Read one authored point into the interval its precision denotes. +# --- Which language an authored point is written in --- +# +# An author writes a point in one of two languages, and they come with opposite promises. +# **ISO calendar syntax** is machine syntax: the text fixes the meaning, so it must be read +# literally or refused. **Everything else** -- `June 10, 2026`, `2026/03/04`, `10/07/2026`, +# `yesterday` -- is human syntax with no literal reading to contradict, so the flexible +# reader is trusted with it. +# +# The variants below are what a point can be once that question is settled, and settling it +# *once* is the whole design. Four review rounds went the other way: each added a shape test +# whose failure meant "not my business", so a token that failed the test fell through to the +# flexible reader and the next round found another shape that failed it. Here the classifier +# is total -- a token that opens with ISO syntax is an `_IsoDay`, an `_IsoMonth` or a +# `_MalformedIso`, and none of the three can reach the flexible reader. + +# The ISO calendar components a point *opens* with: `YYYY-MM` and an optional `-DD`. A date +# carrying a time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part +# alone, because `\d+` cannot cross the separator -- the rest is `trailing`, judged below. +# +# Each component is `\d+` rather than `\d{2}`, and nothing terminates the pattern, so the +# head matches whenever a point opens with ISO syntax at all. Both rules exist because the +# earlier cuts of this guard failed to match a malformed token and so let it escape: against +# `\d{2}` the day of `2026-01-0100` left a trailing `00` and matched nothing, and against a +# trailing `(?![\d-])` lookahead `2026-01-01-` matched nothing. Both reached the flexible +# reader, which is the one outcome ISO syntax must never have. +_ISO_CALENDAR_HEAD = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?") - The precision the author wrote is the meaning: - 2026 -> [2026-01-01,2027-01-01) the year - 2026-06 -> [2026-06-01,2026-07-01) the month - 2026-06-10 -> [2026-06-10,) from that date onward - 2026-06-10T14:00:00 -> [that instant,) from that moment onward +def _named_calendar_date(year: str, month: str, day: str | None) -> date | None: + """The date ISO-shaped calendar components name, or None when they name none. - A year or a month is a period the author delimited by writing it. A date or a - moment is not: `@effective 2026-06-10` means the decision took effect that day and - still holds, so closing the range at midnight would expire it overnight. Callers - that need a closed interval write the range literal instead. + A month-only head is placed on the first of that month: the day is a component the + author did not write, not one to guess at. `date` is the authority rather than a range + check because it already owns leap years and month lengths. + """ + # A month or a day is written with one or two digits, and that width is what separates + # an author's shorthand from an author's typo: `2026-1-5` is a legitimate unpadded + # spelling of a real date, while the `0100` in `2026-01-0100` is no day at all. Judged + # before `date`, which takes a C long and raises OverflowError -- not the ValueError + # below -- once a run of digits grows past it. + if len(month) > 2 or (day is not None and len(day) > 2): + return None + try: + return date(int(year), int(month), 1 if day is None else int(day)) + except ValueError: + return None - Non-ISO spellings are read leniently, because guessing at `June 10, 2026` is the - whole point of this reader. A token that *is* ISO-shaped is held to its own text - instead: its calendar components must name a real date, and anything trailing them - must be a time of day on that date. `2026-06-10 10:00 AM` reads; `2026-01-01T` does - not, because the author reached for an instant and no instant is there. - Returns None when the text names no date. That is not an error -- the caller leaves - such a token as ordinary observation content. +@dataclass(frozen=True, slots=True) +class _IsoDay: + """A point whose ISO head names a calendar day, and whatever was written after it. + + The day is authoritative: it is what the author typed, so no reading of `trailing` may + contradict it. `trailing` is empty for a bare date; when it is not, the point is an + instant, because a time of day is the only thing that can follow a complete date. """ - point = text.strip() - if _DATE_BOUND.match(point): - # Trigger: the text is already in the canonical ISO date shape. - # Why: dateparser is lenient with impossible components -- it reads - # "2026-13-01" as the 13th of January -- and a silently wrong date is worse - # than an unread token. - # Outcome: ISO dates are parsed as ISO, or refused. - try: - return TemporalRange( - axis=TemporalRangeAxis.DATE, - lower=date.fromisoformat(point).isoformat(), - lower_inclusive=True, - ) - except ValueError: - return None + + day: date + trailing: str + + +@dataclass(frozen=True, slots=True) +class _IsoMonth: + """A point whose ISO head names a calendar month (`2026-06`), and so denotes it. + + There is deliberately nowhere to put trailing text: nothing may follow a month. A clock + reading needs a day to fall on, and the flexible reader supplies the day it was not + given from *today*, so `2026-06 10:00` read as June 7 in March and June 1 in September + -- the same note projecting different valid time on different indexing days. + """ + + year: int + month: int + + +@dataclass(frozen=True, slots=True) +class _MalformedIso: + """A point written in ISO syntax that names nothing on the calendar. + + `2026-13-01`, `2026-01-0100`, `2026-06 10:00`. The author reached for a machine date + and missed, so there is no reading to fall back on -- only a guess, which is what this + variant exists to make unreachable. + """ + + +@dataclass(frozen=True, slots=True) +class _FlexiblePoint: + """A point in no machine syntax at all, for the flexible reader to interpret.""" + + +type _AuthoredPoint = _IsoDay | _IsoMonth | _MalformedIso | _FlexiblePoint + +_MALFORMED_ISO = _MalformedIso() +_FLEXIBLE_POINT = _FlexiblePoint() + + +def _classify_authored_point(point: str) -> _AuthoredPoint: + """Decide which language one authored point is written in, and what it names. + + Total by construction, which is the property the whole design rests on: opening with + ISO syntax settles the question, and the three ISO variants are all a token can then + be. There is no "looks ISO but is not this function's business" answer to fall through + on, which is what every earlier cut of this guard offered and what each review round + found another way to reach. + """ + head = _ISO_CALENDAR_HEAD.match(point) + if head is None: + return _FLEXIBLE_POINT + + year, month, day = head.groups() + named = _named_calendar_date(year, month, day) + if named is None: + return _MALFORMED_ISO + + trailing = point[head.end() :] + if day is None: + # Trigger: the head names a month, with or without text after it. + # Why: a month is a complete point on its own, so anything following it is part of + # a date this head cannot carry -- see `_IsoMonth` for what reading it costs. + # Outcome: a bare month denotes its own period; a month with anything after it is + # malformed. + return _MALFORMED_ISO if trailing else _IsoMonth(int(year), int(month)) + return _IsoDay(named, trailing) + + +def _read_iso_day(iso: _IsoDay, point: str, date_order: DateOrder) -> TemporalRange | None: + """Read a point whose head names a calendar day, holding it to its own text.""" + if not iso.trailing: + return TemporalRange( + axis=TemporalRangeAxis.DATE, lower=iso.day.isoformat(), lower_inclusive=True + ) if _INSTANT_BOUND.match(point): - # Trigger: the text is already in the canonical RFC 3339 timestamp shape. - # Why: the leniency the branch above guards against reaches timestamps too -- - # dateparser reads "2026-13-01T10:00:00" as 10:00 on the 13th of January -- and - # every reindex would project that same wrong instant, so it is worse than an - # unread token. Only the *shape* is matched here, so the flexible spellings - # dateparser alone reads ("2026-06-10 10:00 AM", a timestamp with no seconds) - # still reach it below. - # Outcome: RFC 3339 timestamps are parsed as RFC 3339, or refused. + # Trigger: the whole token is canonical RFC 3339. + # Why: the author wrote the one form this module defines exactly, so it is read + # exactly -- to the microsecond, and refused rather than rounded when it names no + # moment (`2026-06-10T25:00:00+02:00`) or leaves the calendar in UTC. The flexible + # reader is neither that precise nor that strict. + # Outcome: an instant, or a refusal; never a guess. try: instant = _canonical_instant(point) except TemporalQualifierError: return None - return TemporalRange( - axis=TemporalRangeAxis.INSTANT, - lower=instant, - lower_inclusive=True, - ) + return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) + + # The author wrote a clock reading in some spelling of their own, so the flexible reader + # is asked for it -- but only for it. What it hands back must be a time of day on the + # very day the head names, which is the check that keeps its guessing out of the answer: + # dateparser silently drops a suffix it cannot use (`2026-01-01T`, `2026-01-01Z`, + # `2026-01-01+14:00` all came back as the bare date), and a suffix it half-understands + # makes it abandon the ISO reading and re-guess the components under the configured + # order (`2026-06-10x` came back as October 6). Asking what it *returned* rather than + # what the suffix looks like is what covers every such shape, named or not. + date_data = _date_data_parser(date_order).get_date_data(point) + moment = date_data.date_obj + if moment is None or date_data.period != "time" or moment.date() != iso.day: + return None + instant = _instant_value(moment) + if instant is None: + # A moment that leaves the calendar in UTC names no storable instant, so it reads + # as no date at all -- the token stays content. + return None + return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) - # --- An ISO-shaped point is judged on its whole text --- - # - # Everything past this section is read by dateparser, which answers "what date can I - # find in here?" rather than "does this text name a date?". It reads past what it does - # not understand, so an ISO-shaped token is checked in both halves: the calendar head - # must name a real date, and whatever trails that head must be accounted for. - iso_head = _ISO_CALENDAR_HEAD.match(point) - iso_day: date | None = None - trailing = "" - if iso_head is not None: - year, month, day = iso_head.groups() - if not _names_a_real_calendar_date(year, month, day): - # Trigger: the text opens with ISO calendar components that name no real date. - # Why: the two branches above only match a token that is *exactly* a canonical - # date or timestamp, so every other ISO-shaped spelling still reached - # dateparser -- a bare `2026-13`, a space-separated `2026-13-01 10:00:00`, a - # minute-precision `2026-13-01T10:00`. It reads the impossible month as a day - # and then fills the month it never got from *today*, so `2026-13` projects a - # different date on every reindex day: the same note yields different data - # depending on when it was indexed. A mistyped *width* is read just as freely - # -- `2026-01-0100` comes back as the whole month of January -- so a slipped - # keystroke silently widens one day into a range nobody wrote. - # Outcome: refused, and the token stays ordinary observation content. - return None - # Vouched for just above, so building the date cannot raise. None means the author - # wrote a month, which owns no day for a trailing time to fall on. - iso_day = None if day is None else date(int(year), int(month), int(day)) - trailing = point[iso_head.end() :] +def _read_flexible_point(point: str, date_order: DateOrder) -> TemporalRange | None: + """Read a point written in no machine syntax, taking the flexible reader at its word.""" date_data = _date_data_parser(date_order).get_date_data(point) moment = date_data.date_obj if moment is None: return None - if trailing and not (date_data.period == "time" and moment.date() == iso_day): - # Trigger: an ISO calendar head is followed by text the reader did not turn into - # a time of day on that very date. - # Why: a calendar date is a complete point, so the only thing that can legally - # follow one is a clock reading. dateparser does not enforce that -- it drops a - # suffix it cannot use and answers with the date alone, so `2026-01-01T`, - # `2026-01-01Z` and `2026-01-01+14:00` all came back as `[2026-01-01,)`: the - # author reached for an instant and the index recorded a whole open-ended day, - # re-derived identically by every reindex. Worse, a suffix can make the reader - # abandon the ISO reading altogether and re-guess the *date* under the - # configured order -- `2026-06-10x` came back as October 6 -- or fill a - # component from *today*, so `2026-06 10:00` (a clock reading on a head that - # names no day) landed on a different date depending on when it was indexed. - # Checking what the reader *returned* rather than what the suffix looks like is - # what makes this close the class: any trailing text the reader silently drops - # or reinterprets fails here, whether or not it is a shape anyone anticipated. - # The flexible spellings are untouched, because in every one of them the trailing - # text really is the time it looks like: `T14:00`, ` 10:00 AM`, ` 14:00:00+02:00` - # and even ` noon` all come back as an instant on the date the author wrote. - # Outcome: refused, and the token stays ordinary observation content. - return None - # dateparser fills components the author did not write from today's date, so only # the components `period` vouches for may be read off `moment`. match date_data.period: @@ -757,3 +772,48 @@ def parse_authored_point( lower=moment.date().isoformat(), lower_inclusive=True, ) + + +def parse_authored_point( + text: str, *, date_order: DateOrder = DEFAULT_DATE_ORDER +) -> TemporalRange | None: + """Read one authored point into the interval its precision denotes. + + The precision the author wrote is the meaning: + + 2026 -> [2026-01-01,2027-01-01) the year + 2026-06 -> [2026-06-01,2026-07-01) the month + 2026-06-10 -> [2026-06-10,) from that date onward + 2026-06-10T14:00:00 -> [that instant,) from that moment onward + + A year or a month is a period the author delimited by writing it. A date or a + moment is not: `@effective 2026-06-10` means the decision took effect that day and + still holds, so closing the range at midnight would expire it overnight. Callers + that need a closed interval write the range literal instead. + + Non-ISO spellings are read leniently, because guessing at `June 10, 2026` is the + whole point of this reader. A token that *is* ISO-shaped is held to its own text + instead: its calendar components must name a real date, and anything trailing them + must be a time of day on that date. `2026-06-10 10:00 AM` reads; `2026-01-01T` does + not, because the author reached for an instant and no instant is there. + + Returns None when the text names no date. That is not an error -- the caller leaves + such a token as ordinary observation content. + """ + point = text.strip() + match _classify_authored_point(point): + case _IsoDay() as iso: + return _read_iso_day(iso, point, date_order) + case _IsoMonth() as iso: + return _calendar_span( + date(iso.year, iso.month, 1), _next_month_start(iso.year, iso.month) + ) + case _MalformedIso(): + # The author wrote a machine date that names nothing. Refusal is `None`, as + # everywhere else here: the token stays ordinary observation content, + # unindexed but still full-text searchable. + return None + case _FlexiblePoint(): + return _read_flexible_point(point, date_order) + case unreachable: # pragma: no cover - `_AuthoredPoint` is closed + assert_never(unreachable) diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 84678894a..c3f85ec04 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -577,6 +577,32 @@ def test_a_real_time_of_day_still_follows_an_iso_date(written: str, lower: str): assert span.lower == lower +@pytest.mark.parametrize( + ("written", "lower"), + [ + # A clock reading on a date written in no machine syntax at all. The ISO rules + # never see these -- there is no literal reading to hold them to -- so the + # flexible reader's answer is taken as given, clock and all. + ("10/07/2026 14:00", "2026-07-10T14:00:00.000000Z"), + ("10/07/2026 14:00:00+02:00", "2026-07-10T12:00:00.000000Z"), + ("June 10, 2026 2pm", "2026-06-10T14:00:00.000000Z"), + ("June 10, 2026 at 14:00", "2026-06-10T14:00:00.000000Z"), + ], +) +def test_a_non_iso_date_may_carry_a_clock_reading(written: str, lower: str): + """Both readers file instants, and only one of them checks the date it was given. + + An ISO head is authoritative, so a clock reading beside one is verified against it. + These spellings have no such head -- `@occurred:"June 10, 2026 2pm"` says everything + it means through the flexible reader -- so nothing here is second-guessed. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + @pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) def test_an_impossible_iso_month_is_not_completed_from_the_indexing_date(today: str): """The worst shape of all: a date whose meaning depended on when the reindex ran. @@ -685,18 +711,22 @@ def test_a_year_beyond_the_calendar_is_unread(): @pytest.mark.parametrize( "written", [ - # The canonical shape, refused by the strict timestamp branch... + # The canonical shape, read exactly and refused by the ISO reader... "9999-12-31T23:59:59-05:00", - # ...and the same moment spelled loosely, refused after dateparser reads it. + # ...the same moment spelled loosely, still ISO-headed, so the ISO reader asks + # the flexible one for the clock and then finds the moment unstorable... "9999-12-31 23:59:59 -05:00", + # ...and the same moment in no machine syntax at all, which the flexible reader + # owns outright. + "December 31, 9999 23:59:59 -05:00", ], ) def test_an_authored_instant_that_leaves_the_calendar_in_utc_is_unread(written: str): """The flexible reader has no bound to refuse, so it reads no date at all. Its contract is None-for-unreadable, not an exception: `parse_temporal_qualifier` - does not guard this call, so anything raised here fails the note. Both spellings are - pinned because they take different routes to the same refusal. + does not guard this call, so anything raised here fails the note. All three spellings + are pinned because they take different routes to the same refusal. """ assert parse_authored_point(written) is None From 57828455495b245e770cbbb7f2e7f1ce0a40665e Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 00:45:26 -0500 Subject: [PATCH 10/25] fix(core): refuse ISO points finer than a microsecond `_INSTANT_BOUND` caps a fractional second at six digits and refuses a longer one rather than truncating it, because dropping digits would store a different instant than the author wrote. That refusal only ever governed the strict path. An over-precise point never matches `_INSTANT_BOUND`, so `_read_iso_day` fell through to the flexible reader, which truncated the fraction and answered with a time of day on the very day the ISO head names. Every check that path makes then passed: `@occurred:2026-01-01T10:00:00.1234567` and its quoted form both indexed as `2026-01-01T10:00:00.123456Z`, re-derived identically on every reindex, with nothing said to the author. The day check is what guards the flexible reading, and a truncated fraction sails straight through it -- the digits it drops were never in the answer to be checked. Judged on the author's text in `_classify_authored_point` instead, so both readers refuse the same token for the same reason, and for the same reason the calendar width rule already exists: a digit run wider than the syntax allows is a typo, not a shorthand. Six digits and fewer are untouched, so `14:00:00.5` and `.123456` still read; the range-literal path already refused these correctly. This also corrects an overstated claim in the classifier's own comment. It said none of the three ISO variants can reach the flexible reader, which is false: an `_IsoDay`'s trailing text is deliberately read by it, and that is what reads `2026-06-10 10:00 AM`. What the classifier actually settles for good is the calendar -- a head naming no date dies there and is never re-guessed. The trailing is fenced by two rules, one on what the reader returned and one on what the author wrote, and the comment now says so. A sweep of 216 ISO-time spellings (T/t/space separators, minute and second precision, Z/z and offset zones with and without a colon, fractions of 0/1/6/7/9 and 30 digits) reported 54 tokens whose stored instant disagreed with a literal reading of the text. All 54 were this one defect; after the fix the sweep reports none. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 53 +++++++++++++++--- tests/markdown/test_temporal_qualifier.py | 48 ++++++++++++++++ tests/test_temporal.py | 67 +++++++++++++++++++++++ 3 files changed, 160 insertions(+), 8 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 63e5d8abd..3b0c7e1a6 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -580,8 +580,20 @@ def _calendar_span(lower: date, upper: date | None) -> TemporalRange: # *once* is the whole design. Four review rounds went the other way: each added a shape test # whose failure meant "not my business", so a token that failed the test fell through to the # flexible reader and the next round found another shape that failed it. Here the classifier -# is total -- a token that opens with ISO syntax is an `_IsoDay`, an `_IsoMonth` or a -# `_MalformedIso`, and none of the three can reach the flexible reader. +# is total: a token that opens with ISO syntax is an `_IsoDay`, an `_IsoMonth` or a +# `_MalformedIso`, and there is no fourth answer to fall through on. +# +# What that buys is narrower than "ISO-shaped text never reaches the flexible reader", and +# stating it precisely matters, because the loose version is false. An `_IsoDay`'s *trailing* +# text is still read by the flexible reader -- that is what reads `2026-06-10 10:00 AM`, and +# no grammar of clock spellings could. What the classifier settles for good is the *calendar*: +# a head that names no date dies here, and a real one is carried on the variant so the reading +# below can be held to it. The trailing is fenced by two rules instead, and dateparser's answer +# is believed only when both hold. It must come back as a time of day on the day the head names +# -- checked in `_read_iso_day`, against what it *returned*, since a suffix's looks do not say +# what it will do with it. And the text must not spell precision a canonical instant cannot +# carry -- checked here, on the text, because that is the one defect the returned-value check +# cannot see: a truncated fraction still lands on the right day. # The ISO calendar components a point *opens* with: `YYYY-MM` and an optional `-DD`. A date # carrying a time (`2026-06-10T14:00`, `2026-06-10 10:00 AM`) is matched on its date part @@ -595,6 +607,18 @@ def _calendar_span(lower: date, upper: date | None) -> TemporalRange: # reader, which is the one outcome ISO syntax must never have. _ISO_CALENDAR_HEAD = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?") +# A fractional-second run too wide for a canonical instant to carry. `_INSTANT_BOUND` caps the +# fraction at six digits and *refuses* a longer one rather than truncating it, because dropping +# digits would store a different instant than the author wrote -- but that refusal only ever +# governed the strict path. The flexible reader has no such scruple: it truncates +# `2026-01-01T10:00:00.1234567` to `...123456Z` and reports a time on the right day, so every +# check `_read_iso_day` makes passes and the authored instant is quietly rewritten on each +# reindex. Judged on the text so both paths refuse the same token for the same reason, and it +# is the same reason the calendar width rule exists: a digit run wider than the syntax allows +# is a typo, not a shorthand. Six digits and fewer are untouched -- `14:00:00.5` is precision +# a canonical instant holds exactly, so it still reads. +_OVER_PRECISE_FRACTION = re.compile(r"\.\d{7,}") + def _named_calendar_date(year: str, month: str, day: str | None) -> date | None: """The date ISO-shaped calendar components name, or None when they name none. @@ -645,11 +669,12 @@ class _IsoMonth: @dataclass(frozen=True, slots=True) class _MalformedIso: - """A point written in ISO syntax that names nothing on the calendar. + """A point written in ISO syntax that cannot be read as written. - `2026-13-01`, `2026-01-0100`, `2026-06 10:00`. The author reached for a machine date - and missed, so there is no reading to fall back on -- only a guess, which is what this - variant exists to make unreachable. + `2026-13-01`, `2026-01-0100`, `2026-06 10:00`, `2026-01-01T10:00:00.1234567`. Either the + components name nothing on the calendar, or they name a moment finer than a canonical + instant records. The author reached for a machine date and missed, so there is no reading + to fall back on -- only a guess, which is what this variant exists to make unreachable. """ @@ -690,6 +715,16 @@ def _classify_authored_point(point: str) -> _AuthoredPoint: # Outcome: a bare month denotes its own period; a month with anything after it is # malformed. return _MALFORMED_ISO if trailing else _IsoMonth(int(year), int(month)) + + # Trigger: the text after the date spells a fraction of a second wider than six digits. + # Why: no reader here can store it, and the two that try disagree -- `_canonical_instant` + # refuses it, while the flexible reader truncates it and still answers with a time on + # the head's day, which is precisely what `_read_iso_day`'s returned-value check cannot + # catch. A guard that asks what came back cannot see digits that never made it in. + # Outcome: refused as malformed, so the strict and flexible paths give the same answer to + # the same text and the token stays observation content rather than a rounded instant. + if _OVER_PRECISE_FRACTION.search(trailing): + return _MALFORMED_ISO return _IsoDay(named, trailing) @@ -794,8 +829,10 @@ def parse_authored_point( Non-ISO spellings are read leniently, because guessing at `June 10, 2026` is the whole point of this reader. A token that *is* ISO-shaped is held to its own text instead: its calendar components must name a real date, and anything trailing them - must be a time of day on that date. `2026-06-10 10:00 AM` reads; `2026-01-01T` does - not, because the author reached for an instant and no instant is there. + must be a time of day on that date, written to a precision this module can store. + `2026-06-10 10:00 AM` reads; `2026-01-01T` does not, because the author reached for an + instant and no instant is there; `2026-01-01T10:00:00.1234567` does not either, + because storing it would mean dropping the digits that made it worth writing. Returns None when the text names no date. That is not an error -- the caller leaves such a token as ordinary observation content. diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index 47f71dd98..724cb11d6 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -437,6 +437,54 @@ def test_quoted_point_reads_a_multi_word_date(qualifier: str, literal: str, kind assert str(observation) == line +@pytest.mark.parametrize( + "point", + [ + "2026-01-01T10:00:00.1234567", + "2026-01-01T10:00:00.1234567Z", + "2026-01-01T10:00:00.1234567+02:00", + "2026-01-01T10:00:00." + "1" * 30, + ], +) +def test_a_point_finer_than_a_microsecond_stays_content_in_both_forms(point: str): + """An over-precise instant is refused whichever form carries it to the reader. + + Both forms filed `[2026-01-01T10:00:00.123456Z,)` -- the authored instant with its + last digits dropped, and no sign to the author that anything was lost. The quoted form + reached it by a different route than the bare one: quoting suppresses the truncation + guards, on the reasoning that a delimited value cannot be a truncated *token*. That is + still true, and beside the point here -- the loss is inside the value, so only refusing + the point itself covers both. Pinned together so a fix to one form cannot miss the + other. + """ + for qualifier in (f"@occurred:{point}", f'@occurred:"{point}"'): + line = f"- [decision] {qualifier} The cutover ran." + + observation = _observation(line) + + assert observation.temporal == [] + # Refused, not reported: how someone spelled a date is not a diagnostic this + # feature issues. The line keeps every character and stays full-text searchable. + assert observation.temporal_error is None + assert observation.content == f"{qualifier} The cutover ran." + assert str(observation) == line + + +def test_a_point_at_exactly_microsecond_precision_is_still_filed(): + """The boundary the refusal above stops at, end to end through the parser.""" + for qualifier in ( + "@occurred:2026-01-01T10:00:00.123456", + '@occurred:"2026-01-01T10:00:00.123456"', + ): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert assertion.time_kind is TimeKind.OCCURRED + assert str(assertion.valid_during) == "[2026-01-01T10:00:00.123456Z,)" + assert assertion.valid_during.axis is TemporalRangeAxis.INSTANT + assert observation.content == "The cutover ran." + + def test_a_quoted_relative_date_is_read_where_its_unquoted_form_is_not(): """`2 days ago` always read fine; only the token rule kept it out.""" quoted = _observation('- [decision] @occurred:"2 days ago" The cutover ran.') diff --git a/tests/test_temporal.py b/tests/test_temporal.py index c3f85ec04..8605d2343 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -533,6 +533,73 @@ def test_an_iso_date_whose_suffix_re_guesses_it_is_unread(written: str): assert parse_authored_point(written) is None +@pytest.mark.parametrize( + "written", + [ + # The reported shape: one digit more than a canonical instant carries. The lenient + # reader truncated it to `...123456Z`, on the very day the head names, so every + # check the reading makes passed and the index recorded an instant 100ns off the + # one the author wrote -- re-derived identically by every reindex. + "2026-01-01T10:00:00.1234567", + # The same defect wearing every spelling of the syntax around it. None of these is + # distinguishable by what the reader *returned* -- each truncates and each lands on + # the right day -- which is why this one is judged on the text instead. + "2026-01-01t10:00:00.1234567", + "2026-01-01 10:00:00.1234567", + "2026-01-01T10:00:00.1234567Z", + "2026-01-01T10:00:00.1234567z", + "2026-01-01T10:00:00.1234567+02:00", + "2026-01-01T10:00:00.1234567-05:00", + "2026-01-01T10:00:00.1234567+0200", + # Precision far past anything a clock emits, truncated just as quietly: a 20- and a + # 30-digit fraction both stored six digits and discarded the rest without a word. + "2026-01-01T10:00:00.12345678901234567890", + "2026-01-01T10:00:00." + "1" * 30, + ], +) +def test_an_iso_point_finer_than_a_microsecond_is_unread(written: str): + """Over-precision is refused on the lenient path too, not silently rounded. + + `canonical_bound` has always refused these -- dropping digits would store a different + instant than the author wrote -- but that refusal only governed the strict path. A + point one digit too precise never matched `_INSTANT_BOUND`, so it fell to the lenient + reader, which truncated it and answered with a time on the correct day. The day check + is what guards that path, and a truncated fraction sails straight through it: the + digits it drops were never in the answer to be checked. Judged on the author's text + instead, so both readers refuse the same token for the same reason. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # Exactly six digits: the widest fraction a canonical instant carries, so it is + # stored whole and nothing is dropped. The refusal above must stop precisely here. + ("2026-01-01T10:00:00.123456", "2026-01-01T10:00:00.123456Z"), + ("2026-01-01 10:00:00.123456", "2026-01-01T10:00:00.123456Z"), + ("2026-01-01T10:00:00.123456Z", "2026-01-01T10:00:00.123456Z"), + ("2026-01-01T10:00:00.123456+02:00", "2026-01-01T08:00:00.123456Z"), + # Narrower fractions were never in question, and are pinned so a future widening + # of the rule cannot quietly take them. + ("2026-01-01T10:00:00.1", "2026-01-01T10:00:00.100000Z"), + ("2026-06-10 14:00:00.5", "2026-06-10T14:00:00.500000Z"), + ], +) +def test_a_fraction_a_canonical_instant_can_hold_still_reads(written: str, lower: str): + """Refusing over-precision must cost nothing that stores losslessly. + + Six digits is the boundary, not "any fraction is suspicious": these name a moment the + canonical form records exactly, so there is no truncation to prevent and no reason to + withhold the assertion. + """ + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + @pytest.mark.parametrize("today", ["2026-03-07", "2026-09-01"]) def test_a_clock_reading_on_a_month_is_not_completed_from_the_indexing_date(today: str): """`2026-13`'s disease in the suffix: a time of day needs a day to fall on. From 3281318271f6e76777f6ee8a0e70277603f13a9a Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 02:14:17 -0500 Subject: [PATCH 11/25] fix(core): key observation identity and note type to the owning note Two ways a valid-time query could silently return nothing, both from derived state answering a question about the note it came from. **An authored assertion could become permanently unqueryable.** A temporal qualifier is peeled off an observation before the content is stored, so two lines that differ only in their qualifier persist identical content and derive identical synthetic permalinks. The search index is unique on (permalink, project_id), so `index_entity_markdown` skipped the second as a duplicate -- while its temporal row went on addressing an observation with no search projection. Querying the second window returned nothing, and every reindex reproduced the omission from the same markdown: - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. - [decision] @effective[2027-06-10,2027-07-27) The cache layer will use Redis. Reproduced before the fix: two observation rows, one search row, two memory_time_index rows, and the 2027 query returning zero results. The duplicate check is not what is wrong -- it guards a real unique index. Its input is. Identity is derived after the peel, so it is derived from a string the note does not consider distinguishing. Nothing on the observation row separates these two, so no row-local rule can: category, content, context and tags are all equal, and the qualifier lives in its own projection by an explicit design decision. A relationship to that projection is not usable either -- all three readers of `Observation.permalink` read it on *detached* instances after their session has closed, so a lazy load would raise rather than resolve. So the ordinal is stored, exactly as `note_section.duplicate_index` already does for duplicate headings. `replace_observations_for_generation` is the one place that sees a note's whole observation set in document order, and it counts the ordinal over `observation_permalink_tail` -- shared with `Observation.permalink` so the count is taken over exactly the identity the address is built from, rather than rebuilt inline and drifting (#929). Keying on the generated tail rather than raw values also closes slug aliasing, where `Foo Bar` and `foo-bar` are different content that generate one permalink. The ordinal is 0 for the first observation of any identity, so every permalink that resolves today is byte-identical afterwards; only later twins gain a suffix. It fixes the same collision for two observations differing only in `(context)`, which had the same defect for the same reason. **A valid-time query combined with `note_types` could not match anything.** A note's type lives in its frontmatter, so only its entity row carries `metadata.note_type`; observation rows carry tags and relation rows carry nothing. Both backends read the type off each row, which asks "is this row an entity of type X?" when the question was "does this row belong to a note of type X?". Valid time selects observation rows, so the two predicates were never true of the same row and the conjunction was unsatisfiable. Resolved through the owning note instead, in one shared builder both backends call -- only the JSON accessor differs, and that is all each supplies. Every search row already carries `entity_id` and an entity row's own `id` equals it, so one non-correlated membership test covers all three row kinds. Non-correlated for the reason `temporal_filters` documents: SQLite's `search_index` is an FTS5 virtual table and a correlated EXISTS beside a MATCH is refused outright. This makes `note_types` return observation and relation rows of matching notes, where it previously collapsed to entity rows. That is the fix, not a side effect: restricting which *kind* of row may match is `entity_types`' job, the two axes are independent, and a query setting neither returns all three kinds. `test_search_type` asserted the old entity-only shape and is updated -- it recorded what the defect allowed, not an intention. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- ...7s8d9e0_add_observation_duplicate_index.py | 49 +++++++++++ src/basic_memory/models/knowledge.py | 69 +++++++++++---- .../repository/note_type_filters.py | 75 +++++++++++++++++ .../repository/observation_repository.py | 37 +++++--- .../repository/postgres_search_repository.py | 25 +++--- .../repository/sqlite_search_repository.py | 26 +++--- .../test_postgres_search_repository.py | 4 + tests/services/test_search_service.py | 29 +++++-- .../services/test_search_service_temporal.py | 84 +++++++++++++++++++ 9 files changed, 344 insertions(+), 54 deletions(-) create mode 100644 src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py create mode 100644 src/basic_memory/repository/note_type_filters.py diff --git a/src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py b/src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py new file mode 100644 index 000000000..d0ce6c648 --- /dev/null +++ b/src/basic_memory/alembic/versions/v5o6b7s8d9e0_add_observation_duplicate_index.py @@ -0,0 +1,49 @@ +"""Add the duplicate ordinal that keeps same-identity observations addressable. + +An observation's synthetic permalink is built from its category and content, both of +which survive the peel that strips a temporal qualifier and a (context) off the authored +line. Two observations differing only in one of those therefore shared one address, and +the permalink-keyed search index kept only the first -- so the second note's authored +valid time addressed an observation with no search row, and queries for its interval +found nothing (SPEC-82). + +This ordinal separates such twins. It defaults to 0, which is the value every existing +row takes and the value the first observation of any identity keeps, so no permalink that +resolves today changes. Search rows are derived state and are rebuilt from markdown, so +the second twin becomes addressable on the next index pass for that note. + +Revision ID: v5o6b7s8d9e0 +Revises: u4t5e6m7p8o9 +Create Date: 2026-09-02 10:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "v5o6b7s8d9e0" +down_revision: Union[str, None] = "u4t5e6m7p8o9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add observation.duplicate_index, defaulting every existing row to 0.""" + with op.batch_alter_table("observation", schema=None) as batch_op: + batch_op.add_column( + sa.Column( + "duplicate_index", + sa.Integer(), + server_default=sa.text("0"), + nullable=False, + ) + ) + + +def downgrade() -> None: + """Remove the duplicate ordinal.""" + with op.batch_alter_table("observation", schema=None) as batch_op: + batch_op.drop_column("duplicate_index") diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 49b12584d..2f79c6295 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -339,6 +339,45 @@ def __repr__(self) -> str: # pragma: no cover ) +def observation_permalink_tail(category: str | None, content: str) -> str: + """The part of an observation's permalink that distinguishes it within its note. + + This is the single definition of what makes two observations of one note share an + address, and it is deliberately shared with the writer that assigns + `Observation.duplicate_index`: an ordinal only disambiguates if it is counted over + exactly the identity the permalink is built from. Computing the two separately is the + defect this function exists to prevent -- rebuilding the permalink format inline is + what diverged from the search index for long observations (#929). + + Note what is *not* here. A qualifier (`@effective[...]`) and a `(context)` are peeled + off the line before the observation is stored, so neither reaches this string, and two + observations that differ only in one of them arrive identical. That is not an oversight + to correct by stuffing them back in: the peel is the feature, and valid time is its own + projection rather than an observation column. The note still distinguishes such lines, + so the *address* must too, which is what the ordinal counted over this tail supplies. + + Slug aliasing is why the count keys on this generated text rather than on the raw + values: `Foo Bar` and `foo-bar` are different content that generate one permalink, so + an ordinal counted over raw content would leave them colliding. + + Content is truncated to 200 chars to stay under PostgreSQL's btree index limit of + 2704 bytes. + """ + if len(content) > 200: + # Trigger: content exceeds the 200-char budget imposed by PostgreSQL's + # 2704-byte btree index row limit, so the permalink can only carry a prefix. + # Why: two distinct observations with the same category and an identical + # 200-char prefix would collide on the same synthetic permalink, and the + # search index (permalink-keyed upsert) silently drops the second one. + # Outcome: a short stable digest of the FULL content disambiguates + # truncated permalinks while staying well under the index limit. + digest = hashlib.sha256(content.encode("utf-8")).hexdigest()[:12] + content_for_permalink = f"{content[:200]}-{digest}" + else: + content_for_permalink = content + return generate_permalink(f"observations/{category}/{content_for_permalink}") + + class Observation(Base): """An observation about an entity. @@ -360,6 +399,11 @@ class Observation(Base): tags: Mapped[Optional[list[str]]] = mapped_column( JSON, nullable=True, default=list, server_default="[]" ) + # Which of the note's same-identity observations this one is, in document order. + # See `observation_permalink_tail` for why an ordinal is needed at all and why it is + # stored rather than derived: `permalink` is read on *detached* instances, long after + # the session that could have looked at this row's siblings has closed. + duplicate_index: Mapped[int] = mapped_column(Integer, default=0, server_default=text("0")) # Relationships entity = relationship("Entity", back_populates="observations") @@ -371,24 +415,17 @@ def permalink(self) -> str: We can construct these because observations are always defined in and owned by a single entity. - Content is truncated to 200 chars to stay under PostgreSQL's - btree index limit of 2704 bytes. + `duplicate_index` is what keeps the address faithful when one note says the same + thing twice. It is 0 for the first observation carrying a given identity, so the + overwhelming majority of permalinks are byte-identical to what they have always + been; only the second and later twins gain a trailing ordinal. """ - if len(self.content) > 200: - # Trigger: content exceeds the 200-char budget imposed by PostgreSQL's - # 2704-byte btree index row limit, so the permalink can only carry a prefix. - # Why: two distinct observations with the same category and an identical - # 200-char prefix would collide on the same synthetic permalink, and the - # search index (permalink-keyed upsert) silently drops the second one. - # Outcome: a short stable digest of the FULL content disambiguates - # truncated permalinks while staying well under the index limit. - digest = hashlib.sha256(self.content.encode("utf-8")).hexdigest()[:12] - content_for_permalink = f"{self.content[:200]}-{digest}" - else: - content_for_permalink = self.content - return generate_permalink( - f"{self.entity.permalink}/observations/{self.category}/{content_for_permalink}" + base = generate_permalink( + f"{self.entity.permalink}/{observation_permalink_tail(self.category, self.content)}" ) + if not self.duplicate_index: + return base + return f"{base}/{self.duplicate_index}" @override def __repr__(self) -> str: # pragma: no cover diff --git a/src/basic_memory/repository/note_type_filters.py b/src/basic_memory/repository/note_type_filters.py new file mode 100644 index 000000000..9da86df09 --- /dev/null +++ b/src/basic_memory/repository/note_type_filters.py @@ -0,0 +1,75 @@ +"""SQL for the note-type search predicate. + +A note type is a property of the *note*, not of the individual rows projected from it. +One markdown file becomes several search rows -- the entity itself, one per observation, +one per outgoing relation -- and only the entity row carries `metadata.note_type`, because +that is where the frontmatter lives. An observation row carries `metadata.tags`; a relation +row carries no metadata at all. + +Reading the type off each row therefore answers "is this row an entity of type X?" when the +question asked was "does this row belong to a note of type X?". Those coincide for entity +rows and for nothing else, which is invisible until a filter selects non-entity rows -- as a +valid-time filter does, since authored time lives on observations (SPEC-82). The conjunction +of the two was unsatisfiable: every row admitted by the temporal predicate was excluded by +the note-type one. + +Resolving through the owning note fixes that at the source. Every search row already carries +`entity_id`, and an entity row's own `id` equals it, so one membership test covers all three +row kinds without special-casing any of them and without copying the type onto rows that +would then have to be kept in step with the note's frontmatter. + +The predicate is a *non-correlated* subquery for the reason `temporal_filters` documents at +length: SQLite's `search_index` is an FTS5 virtual table, and a correlated `EXISTS` beside a +`MATCH` makes SQLite refuse the statement outright. A non-correlated `IN` is evaluated once, +independently, and composes with every FTS shape in this repository while leaving bm25 +ranking intact. + +One builder serves both dialects. Only the JSON accessor differs, so that is the single +thing a backend supplies -- the rule itself lives here rather than being written out once +per backend and drifting. +""" + +from __future__ import annotations + +from typing import Any, Sequence + +from basic_memory.schemas.search import SearchItemType + +SEARCH_TABLE = "search_index" + +# The alias the owning note's row carries inside the subquery. +_OWNER = "note_type_owner" + +# Each dialect's expression for the owning note's frontmatter type. +SQLITE_NOTE_TYPE_VALUE = f"json_extract({_OWNER}.metadata, '$.note_type')" +POSTGRES_NOTE_TYPE_VALUE = f"{_OWNER}.metadata->>'note_type'" + + +def build_note_type_predicate( + note_types: Sequence[str], + params: dict[str, Any], + *, + note_type_value: str, +) -> str: + """Build the WHERE-clause fragment restricting rows to notes of the given types. + + The stored type keeps the frontmatter's own casing (`Chapter`), while the filter is + documented case-insensitive, so both sides are folded to lowercase. + + Binds are added to `params` in place, following the convention the surrounding FTS + query builders already use. `project_id` is bound by the caller for the whole query. + """ + placeholders = [] + for index, note_type in enumerate(note_types): + name = f"note_type_{index}" + params[name] = note_type.lower() + placeholders.append(f":{name}") + + return ( + f"{SEARCH_TABLE}.entity_id IN (\n" + f" SELECT {_OWNER}.id\n" + f" FROM {SEARCH_TABLE} AS {_OWNER}\n" + f" WHERE {_OWNER}.type = '{SearchItemType.ENTITY.value}'\n" + f" AND {_OWNER}.project_id = :project_id\n" + f" AND LOWER({note_type_value}) IN ({', '.join(placeholders)}))" + ) diff --git a/src/basic_memory/repository/observation_repository.py b/src/basic_memory/repository/observation_repository.py index 3753184ec..7424ee353 100644 --- a/src/basic_memory/repository/observation_repository.py +++ b/src/basic_memory/repository/observation_repository.py @@ -9,6 +9,7 @@ from sqlalchemy.orm.interfaces import LoaderOption from basic_memory.models import Observation +from basic_memory.models.knowledge import observation_permalink_tail from basic_memory.repository.relation_repository import current_relation_generation_statement from basic_memory.repository.repository import Repository from basic_memory.temporal import TemporalAssertion @@ -142,17 +143,33 @@ async def replace_observations_for_generation( return ObservationGenerationWriteResult(generation_is_current=False) await self.delete_by_fields(session, entity_id=entity_id) - rows = [ - Observation( - project_id=self.project_id, - entity_id=entity_id, - content=obs.content, - category=obs.category, - context=obs.context, - tags=obs.tags, + # A note may say the same thing twice and mean two different things -- most + # sharply when a temporal qualifier or a (context) is what separates them, since + # both are peeled off before the content reaches this row. The permalink is built + # from what survives that peel, so those twins would address one row, and the + # permalink-keyed search index would keep only the first (SPEC-82). + # + # This is the one place that sees a note's whole observation set in document + # order, so it is where the ordinal that separates them can be counted at all. + # `observation_permalink_tail` is shared with `Observation.permalink` so the + # count is taken over exactly the identity the address is built from. + duplicates_seen: dict[str, int] = {} + rows = [] + for obs in observations: + identity = observation_permalink_tail(obs.category, obs.content) + duplicate_index = duplicates_seen.get(identity, 0) + duplicates_seen[identity] = duplicate_index + 1 + rows.append( + Observation( + project_id=self.project_id, + entity_id=entity_id, + content=obs.content, + category=obs.category, + context=obs.context, + tags=obs.tags, + duplicate_index=duplicate_index, + ) ) - for obs in observations - ] await self.add_all_no_return(session, rows) # add_all_no_return flushes, so every row now carries its database id. # Reading them here, inside the same transaction, is what lets the temporal diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 02cc5d7d2..b187b4053 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -35,6 +35,10 @@ build_fts_page_stage, ) from basic_memory.repository.metadata_filters import parse_metadata_filters +from basic_memory.repository.note_type_filters import ( + POSTGRES_NOTE_TYPE_VALUE, + build_note_type_predicate, +) from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex @@ -1169,19 +1173,18 @@ async def _build_fts_query_parts( # Handle note type filter (frontmatter type field, parameterized). # Trigger: caller passed `note_types` to scope by the frontmatter `type` field. - # Why: the stored note_type preserves the frontmatter casing (e.g. `Chapter`), - # but the filter is documented case-insensitive. JSONB `@>` containment is - # exact-match, so capitalized types were unfindable. - # Outcome: compare LOWER(metadata->>'note_type') against lowercased filter - # values so `note_types=["Chapter"]` matches a stored `Chapter`. + # Why: the type belongs to the note, but only its entity row carries the + # frontmatter; observation and relation rows do not. Reading it off each row + # silently excluded every non-entity row, which made `note_types` combined + # with a valid-time filter unsatisfiable. + # Outcome: resolved through the owning note in one shared builder, so both + # backends ask the same question and observation rows of a matching note + # are admitted. if note_types: - type_placeholders = [] - for idx, note_type in enumerate(note_types): - param_name = f"note_type_{idx}" - params[param_name] = note_type.lower() - type_placeholders.append(f":{param_name}") conditions.append( - f"LOWER(search_index.metadata->>'note_type') IN ({', '.join(type_placeholders)})" + build_note_type_predicate( + note_types, params, note_type_value=POSTGRES_NOTE_TYPE_VALUE + ) ) # Handle date filter diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 1e349e534..d48a737d7 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -40,6 +40,10 @@ build_fts_page_stage, ) from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path +from basic_memory.repository.note_type_filters import ( + SQLITE_NOTE_TYPE_VALUE, + build_note_type_predicate, +) from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex @@ -909,20 +913,18 @@ async def _build_fts_query_parts( # Handle note type filter (frontmatter type field, parameterized). # Trigger: caller passed `note_types` to scope by the frontmatter `type` field. - # Why: the stored note_type preserves the frontmatter casing (e.g. `Chapter`), - # but the filter is documented case-insensitive; comparing raw values - # would miss capitalized types. - # Outcome: fold both sides to lowercase so `note_types=["Chapter"]` matches a - # stored `Chapter`, `chapter`, etc. + # Why: the type belongs to the note, but only its entity row carries the + # frontmatter; observation and relation rows do not. Reading it off each row + # silently excluded every non-entity row, which made `note_types` combined + # with a valid-time filter unsatisfiable. + # Outcome: resolved through the owning note in one shared builder, so both + # backends ask the same question and observation rows of a matching note + # are admitted. if note_types: - type_placeholders = [] - for idx, t in enumerate(note_types): - param_name = f"note_type_{idx}" - params[param_name] = t.lower() - type_placeholders.append(f":{param_name}") conditions.append( - "LOWER(json_extract(search_index.metadata, '$.note_type')) " - f"IN ({', '.join(type_placeholders)})" + build_note_type_predicate( + note_types, params, note_type_value=SQLITE_NOTE_TYPE_VALUE + ) ) # Handle date filter using datetime() for proper comparison diff --git a/tests/repository/test_postgres_search_repository.py b/tests/repository/test_postgres_search_repository.py index 94c2e4c0a..e25b785e8 100644 --- a/tests/repository/test_postgres_search_repository.py +++ b/tests/repository/test_postgres_search_repository.py @@ -142,6 +142,10 @@ async def test_postgres_search_repository_index_and_search(session_maker, test_p permalink="docs/coffee-brewing", file_path="docs/coffee-brewing.md", type="entity", + # An entity row addresses itself: every indexing path sets entity_id on all three + # row kinds, and note_type is resolved through it, so a row built by hand here + # must carry it too or it belongs to no note at all. + entity_id=1, metadata={"note_type": "note"}, created_at=now, updated_at=now, diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 2fbb04610..99ed148b9 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -274,13 +274,32 @@ async def test_after_date_uses_updated_at(search_service): @pytest.mark.asyncio async def test_search_type(search_service, test_graph): - """Test search filters.""" - - # Should find only type + """`note_types` scopes results to notes of a type, not to entity rows. + + This test used to assert every result was an ENTITY row, which recorded a defect + rather than an intention: a note's type lives in its frontmatter, so only its entity + row carried it, and reading the type off each row dropped every observation and + relation belonging to the very same note. That is what made `note_types` unsatisfiable + together with a valid-time filter, since authored time lives on observation rows + (SPEC-82) -- the two predicates could not both be true of any row. + + Restricting *which kind* of row may match is `entity_types`' job, pinned by the test + directly below. The two axes are independent, and a query that sets neither returns + all three row kinds, so scoping by note type must not silently change the kinds. + """ results = await search_service.search(SearchQuery(note_types=["test"])) assert len(results) > 0 - for r in results: - assert r.type == SearchItemType.ENTITY + + # The three notes the fixture gives `note_type="test"`; "deep" and "deeper" differ. + typed_entity_ids = { + test_graph["root"].id, + test_graph["connected1"].id, + test_graph["connected2"].id, + } + # Every row belongs to a note of the requested type, whatever kind of row it is. + assert {r.entity_id for r in results} <= typed_entity_ids + # And rows other than the notes' own now survive the filter, which is the fix. + assert {r.type for r in results} - {SearchItemType.ENTITY} @pytest.mark.asyncio diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py index 862671b51..b1df63316 100644 --- a/tests/services/test_search_service_temporal.py +++ b/tests/services/test_search_service_temporal.py @@ -228,3 +228,87 @@ async def test_search_trace_describes_the_valid_time_question(search_service): assert "temporal=kind=effective,valid_at=2026-07-28" in describe_search_criteria(containment) assert "temporal=valid_overlaps=[2026-06-10,2026-07-27)" in describe_search_criteria(overlap) assert "temporal=" not in describe_search_criteria(plain) + + +# --- Every authored assertion stays queryable by its own time --- + +TWICE_DATED_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [decision] @effective[2026-06-10,2026-07-27) The cache layer will use Redis. + - [decision] @effective[2027-06-10,2027-07-27) The cache layer will use Redis. + """) + +SECOND_WINDOW_INSIDE = "2027-07-01" + + +@pytest.mark.asyncio +async def test_same_statement_at_two_times_is_queryable_at_each(entity_service, search_service): + """One note, one sentence, two authored windows -- both must remain findable. + + The qualifier is peeled off before the observation is stored, so these two lines + persist identical content and derived identical synthetic permalinks. The search + index is keyed on permalink, so the second observation was skipped as a duplicate + while its temporal assertion went on addressing a row with no search projection: + querying 2027 returned nothing, and every reindex reproduced the omission from the + same markdown. The note says two things happened at two times; both must answer. + """ + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Cache Layer Twice", + note_type="note", + directory="decisions", + content=TWICE_DATED_MARKDOWN, + ) + ) + await search_service.index_entity(entity) + + # The two rows are distinct statements and must carry distinct addresses. + first, second = entity.observations + assert first.permalink != second.permalink + + in_first = await search_service.search( + SearchQuery(text="cache layer", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + in_second = await search_service.search( + SearchQuery(text="cache layer", valid_at=SECOND_WINDOW_INSIDE) + ) + + assert [result.id for result in in_first] == [first.id] + assert [result.id for result in in_second] == [second.id] + # Each window answers with exactly one of them, never the same row twice. + assert first.id != second.id + + +@pytest.mark.asyncio +async def test_a_valid_time_query_can_also_scope_by_note_type(entity_service, search_service): + """Valid time selects observation rows; note type must not then exclude them. + + A note's type lives in its frontmatter, so only its entity row carries it. Reading the + type off each row made these two filters contradict each other -- every row the + temporal predicate admitted, the note-type predicate rejected -- so the conjunction + returned nothing however well the note matched. Resolving the type through the owning + note is what lets both questions be asked at once. + """ + entity = await _index_cache_layer_note(entity_service, search_service) + + scoped = await search_service.search( + SearchQuery( + text="cache layer", + valid_at=EFFECTIVE_WINDOW_INSIDE, + note_types=["note"], + ) + ) + + assert [result.type for result in scoped] == ["observation"] + assert scoped[0].entity_id == entity.id + # A type the note does not have still excludes it, so the filter is doing real work. + unscoped = await search_service.search( + SearchQuery( + text="cache layer", + valid_at=EFFECTIVE_WINDOW_INSIDE, + note_types=["conversation"], + ) + ) + assert unscoped == [] From cbb2ca7537294a8bc355ceeaa43a5fe13445f274 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 10:12:55 -0500 Subject: [PATCH 12/25] fix(core): refuse authored points whose meaning moves with the clock `@occurred:yesterday` was resolved against the wall clock of each parse, so reindexing a note nobody had edited replaced its stored range with a different one -- `[2026-08-31,)` on September 1, `[2026-09-08,)` on the 10th. A search that matched the note last week stopped matching it today with nothing written to disk, and the drift was silent and permanent because the projection is rebuilt from markdown on every pass. A qualifier's meaning has to be recoverable from the note's own bytes, since those are the only thing that travels to a clone or survives a re-import. The two obvious anchors both reintroduce the drift by another route: `created_at` is derived metadata that can shift on re-import or clone, and a stored resolution cannot be reproduced by a fresh clone reindexing from markdown alone, which is exactly what this table is defined to do. Refusing leaves the file as the only source of the answer, and matches what the classifier already does with an ISO-shaped token it cannot pin down. The rule is **determinism, not vocabulary**. A list of relative words is the shape this guard was refactored away from once already, and no list could cover dateparser's relative vocabulary across every language it reads. Instead the reading is taken twice against two stated reference instants, via dateparser's `RELATIVE_BASE`, and kept only if it did not move. Whatever "now" was reaching -- a word, a phrase, a component the author omitted -- lands somewhere different under each and is caught without ever being named. The comparison is on the resulting range, never on dateparser's datetime. A month or a year is delimited by what the author wrote and `_read_flexible_point` discards the components its `period` does not vouch for, so `June 2026` and `2026` answer with one range from two different datetimes. Comparing datetimes would have refused them, which is precisely the over-refusal to avoid: precision the author simply did not write is not the same thing as meaning that moves. A point that never consults the flexible reader cannot move, so the second pass costs pure-ISO tokens a regex and no date parsing at all. The range-literal path needed no change and is verified so: bounds go through `_DATE_BOUND`/`_INSTANT_BOUND` and `date.fromisoformat`, never dateparser, so `[yesterday,today)` was already refused as a malformed bound. Two consequences, both deliberate. No bare word is filed any more, because the only words that named a specific day did it by asking the clock; a month name is still *recognized*, though, so `@occurred:June 10, 2026` keeps being told that quoting is the fix -- that diagnostic asked the reading a question the reading can no longer answer, so it now asks for the shape of the reading, which is stable even when the year the word would take is not. And quoting no longer rescues a relative date: quotes fix a token-boundary problem, not a meaning that moves, and the two guards were always independent. Proven by sweeping both directions under two clocks eight months apart: 19 absolute spellings each name one identical interval under both, and 22 moving ones are refused under both -- including Spanish, French and German phrasings that no word list would have caught. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- .../markdown/temporal_qualifier.py | 58 ++++--- src/basic_memory/temporal.py | 159 ++++++++++++++---- tests/markdown/test_temporal_qualifier.py | 72 ++++---- tests/test_temporal.py | 121 +++++++++++-- 4 files changed, 302 insertions(+), 108 deletions(-) diff --git a/src/basic_memory/markdown/temporal_qualifier.py b/src/basic_memory/markdown/temporal_qualifier.py index 9254288f3..336f866f3 100644 --- a/src/basic_memory/markdown/temporal_qualifier.py +++ b/src/basic_memory/markdown/temporal_qualifier.py @@ -10,19 +10,20 @@ The bracket form carries a range literal and needs no separator, because no kind name can begin with `[` or `(`. The point forms need the `:` because a date can begin with a -letter (`yesterday`), so nothing else would tell `@occurred:yesterday` from a handle. +letter (`June 10, 2026`), so nothing else would tell a kind-and-date from a handle. **An unquoted point is one whitespace-delimited token.** dateparser reads far more than -one token -- `June 10, 2026`, `2 days ago`, `2026-06-10 10:00 AM` all resolve, and +one token -- `June 10, 2026` and `2026-06-10 10:00 AM` both resolve, and `parse_authored_point` accepts them -- but nothing here can tell where such a date ends: dateparser also reads `June 10, 2026 The` and `2026-06-10 The`, so growing the token until parsing fails would swallow the author's prose. **A quoted point is exactly what the author put between the quotes**, which is how a -multi-word, relative, or month-only date is written: `@occurred:"June 10, 2026"`, -`@occurred:"2 days ago"`, `@"June 2026"`. The closing quote is the token boundary, so -whatever follows it is ordinary content, and a `\\"` inside the value does not end the -token. The scan mirrors `_split_predicate_items` in `mcp/tools/posix_tools.py`, down to +multi-word or month-only date is written: `@occurred:"June 10, 2026"`, `@"June 2026"`. +Quoting settles where the token ends; it does not make a date mean something fixed, so +`@occurred:"2 days ago"` is delimited and still refused. The closing quote is the token +boundary, so whatever follows it is ordinary content, and a `\\"` inside the value does +not end the token. The scan mirrors `_split_predicate_items` in `mcp/tools/posix_tools.py`, down to its rule that an unterminated quote is a typo to report rather than a boundary to guess at -- scanning on to end of line would hand the author's prose to dateparser. Only the double quote opens the form: an apostrophe is ordinary punctuation, and a scan looking @@ -37,8 +38,11 @@ least as wide as a year. * **A word naming only a month or a year** (`June`, `may`, `v2`). Alone it is usually prose; as the first token of `June 10, 2026` reading it would file June 2026 and leave - `10, 2026` in the content. A word is taken only when it names a specific day - (`yesterday`, `today`), in whatever language dateparser resolves it. + `10, 2026` in the content. No bare word is filed at all now: the only ones that named a + specific day were relative (`yesterday`, `today`), and a qualifier whose meaning moves + with the calendar is refused outright -- see `basic_memory.temporal` for why. A month + name is still *recognized* here, not to file it but to tell an author who wrote + `@occurred:June 10, 2026` that quoting is the fix. Beyond those, one rule decides everything: **if the payload reads as time, the token becomes a qualifier; if it does not, the token stays ordinary observation content, @@ -68,6 +72,7 @@ TemporalQualifierError, TemporalRange, TimeKind, + names_only_a_calendar_period, parse_authored_point, parse_range_literal, ) @@ -238,23 +243,25 @@ def _locate_point(content: str) -> _PointToken | _Refusal | None: ) -def _truncation_reason(point: str, valid_during: TemporalRange) -> str | None: - """Why an unquoted point is too coarse to file, or None when it names a day. +def _truncation_reason(point: str, date_order: DateOrder) -> str | None: + """Why an unquoted point is too coarse to be the whole date, or None if it is not. - The two shapes named here both parse, which is exactly why they need refusing -- - see the module docstring for what each one costs if it is read. The wording is the - diagnostic's, so the reason a token was refused and the reason it *is* refused stay - the same sentence. + The wording is the diagnostic's, so the reason a token was refused and the reason it + *is* refused stay the same sentence. - A bounded span is how a coarse point announces itself: `parse_authored_point` closes - a year or a month at its successor and leaves a day or a moment open, so - `upper is None` *is* "this names a specific day". The one period with no successor - to close at -- December 9999 -- is left open too, and so reads here as a day; no word - resolves to it, so the guard never sees that shape. + Neither branch consults the reading, and that is what lets the question still be asked + of a point the reader threw away. A **number** is judged on its width alone, which was + always the real rule -- `1` and `3.5` are list markers and version numbers whatever + dateparser makes of them. A **word** is judged on the shape of its reading rather than + its value: no bare word is filed any more, since the only ones naming a specific day + were relative, but `June` is still recognizably a month, and that fact does not move + with the calendar even though the year it would take does. """ if point[0].isdigit(): return None if len(point) >= _MIN_NUMERIC_POINT_WIDTH else "is narrower than a year" - return None if valid_during.upper is None else "names only a month or a year" + if names_only_a_calendar_period(point, date_order=date_order): + return "names only a month or a year" + return None def _truncated_point_refusal(content: str, token: _PointToken, reason: str) -> _Refusal | None: @@ -297,14 +304,19 @@ def _read_point_qualifier(content: str, date_order: DateOrder | None) -> _Qualif order = date_order if date_order is not None else ConfigManager().config.date_order valid_during = parse_authored_point(located.point, date_order=order) - if valid_during is None: - return None # Quotes are the author's own delimiters, so a quoted value cannot be the truncated # head of a longer date and the guards do not apply to it. - reason = None if located.quoted else _truncation_reason(located.point, valid_during) + # + # Asked before the unread check below, not after: a token the reader refuses can still + # be the truncated head of a date the author wrote out in full, and a bare month name + # is now exactly that case. Bailing on `valid_during is None` first would lose the one + # diagnostic quoting exists for. + reason = None if located.quoted else _truncation_reason(located.point, order) if reason is not None: return _truncated_point_refusal(content, located, reason) + if valid_during is None: + return None return _ReadQualifier(content[: located.end], located.end, located.kind_name, valid_during) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 3b0c7e1a6..d516c07d3 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -34,11 +34,13 @@ * A **range literal** (`[2026-06-10,2026-07-27)`) is the precise form. Its bounds must be written in the canonical lexical shapes above, to at most microsecond precision. -* A **point** (`2026-06-10`, `2026-06`, `2026`, `yesterday`) is the convenient form. It - denotes the span its precision covers, so an author never has to spell out a range to +* A **point** (`2026-06-10`, `2026-06`, `2026`, `June 10, 2026`) is the convenient form. + It denotes the span its precision covers, so an author never has to spell out a range to say when something started. A point written in ISO calendar syntax is read literally, because its text fixes its meaning; any other spelling is read with `dateparser`, - because there is no literal reading for a guess to contradict. + because there is no literal reading for a guess to contradict. Either way the reading + must be the same on every pass -- `yesterday` names no fixed span and is refused, for + the reasons set out above `parse_authored_point`. """ import re @@ -517,13 +519,17 @@ def parse_temporal_filter( # --- Flexible authored points --- -@lru_cache(maxsize=8) -def _date_data_parser(date_order: DateOrder) -> "DateDataParser": - """The flexible reader for authored points, built once per configured date order. +@lru_cache(maxsize=16) +def _date_data_parser(date_order: DateOrder, relative_base: datetime) -> "DateDataParser": + """The flexible reader for authored points, built once per order and reference instant. Deferred import: dateparser costs ~0.13s and loads locale data, and the modules that carry these values are imported on every CLI start (#886). Only an observation that already looks like a qualifier ever reaches this function. + + `relative_base` is the instant the reader treats as "now". It is always supplied + explicitly, never left to the wall clock, because the wall clock is what made a + reading depend on the day it ran -- see `_STABILITY_PROBE_BASES`. """ from dateparser.date import DateDataParser @@ -533,6 +539,9 @@ def _date_data_parser(date_order: DateOrder) -> "DateDataParser": # Makes `period` report "time" when the author wrote a clock reading, # which is exactly the date-vs-instant distinction this module keeps. "RETURN_TIME_AS_PERIOD": True, + # Fixes what "now" means for this reading, so relative wording resolves + # against a stated instant rather than the moment the indexer happened to run. + "RELATIVE_BASE": relative_base, } ) @@ -572,9 +581,10 @@ def _calendar_span(lower: date, upper: date | None) -> TemporalRange: # # An author writes a point in one of two languages, and they come with opposite promises. # **ISO calendar syntax** is machine syntax: the text fixes the meaning, so it must be read -# literally or refused. **Everything else** -- `June 10, 2026`, `2026/03/04`, `10/07/2026`, -# `yesterday` -- is human syntax with no literal reading to contradict, so the flexible -# reader is trusted with it. +# literally or refused. **Everything else** -- `June 10, 2026`, `2026/03/04`, `10/07/2026` +# -- is human syntax with no literal reading to contradict, so the flexible reader is +# trusted with it. Trusted to *read* it, that is: what it hands back must still name the +# same span whenever it is asked, which is what refuses `yesterday` further down. # # The variants below are what a point can be once that question is settled, and settling it # *once* is the whole design. Four review rounds went the other way: each added a shape test @@ -728,7 +738,9 @@ def _classify_authored_point(point: str) -> _AuthoredPoint: return _IsoDay(named, trailing) -def _read_iso_day(iso: _IsoDay, point: str, date_order: DateOrder) -> TemporalRange | None: +def _read_iso_day( + iso: _IsoDay, point: str, date_order: DateOrder, relative_base: datetime +) -> TemporalRange | None: """Read a point whose head names a calendar day, holding it to its own text.""" if not iso.trailing: return TemporalRange( @@ -756,7 +768,7 @@ def _read_iso_day(iso: _IsoDay, point: str, date_order: DateOrder) -> TemporalRa # makes it abandon the ISO reading and re-guess the components under the configured # order (`2026-06-10x` came back as October 6). Asking what it *returned* rather than # what the suffix looks like is what covers every such shape, named or not. - date_data = _date_data_parser(date_order).get_date_data(point) + date_data = _date_data_parser(date_order, relative_base).get_date_data(point) moment = date_data.date_obj if moment is None or date_data.period != "time" or moment.date() != iso.day: return None @@ -768,15 +780,19 @@ def _read_iso_day(iso: _IsoDay, point: str, date_order: DateOrder) -> TemporalRa return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) -def _read_flexible_point(point: str, date_order: DateOrder) -> TemporalRange | None: +def _read_flexible_point( + point: str, date_order: DateOrder, relative_base: datetime +) -> TemporalRange | None: """Read a point written in no machine syntax, taking the flexible reader at its word.""" - date_data = _date_data_parser(date_order).get_date_data(point) + date_data = _date_data_parser(date_order, relative_base).get_date_data(point) moment = date_data.date_obj if moment is None: return None - # dateparser fills components the author did not write from today's date, so only - # the components `period` vouches for may be read off `moment`. + # dateparser fills components the author did not write from the reference instant, so + # only the components `period` vouches for may be read off `moment`. Discarding the + # rest is also what lets `June 2026` survive the stability check: the filled-in day + # differs between probes, and the month this builds from it does not. match date_data.period: case "time": instant = _instant_value(moment) @@ -809,6 +825,64 @@ def _read_flexible_point(point: str, date_order: DateOrder) -> TemporalRange | N ) +# --- Readings must not depend on when they are taken --- +# +# A qualifier's meaning has to be recoverable from the note's own bytes, because those are +# the only thing that travels. `@occurred:yesterday` failed that: dateparser resolved it +# against the wall clock of each parse, so reindexing an unedited note replaced its stored +# range with a different one -- `[2026-08-31,)` on September 1, `[2026-09-09,)` on the +# 10th -- and a search that matched last week stopped matching today with nothing having +# changed on disk. +# +# The two obvious repairs both reintroduce the drift by another route. Anchoring to the +# entity's `created_at` anchors to derived metadata that can shift on re-import or clone. +# Storing the resolved range makes the projection unreproducible: a fresh clone reindexing +# from markdown alone cannot arrive at it, and this table is rebuilt from markdown by +# definition. Refusing is what keeps the file the sole source of truth, and it is what the +# classifier already does with an ISO-shaped token it cannot pin down. +# +# The rule is *determinism*, not a vocabulary. A list of relative words is the shape this +# guard was refactored away from once already, and it could never have covered the whole +# of dateparser's relative vocabulary in every language it reads. Instead the reading is +# taken twice against two stated reference instants and kept only if it did not move. +# Whatever `now` was reaching -- a word, a phrase, an omitted component -- lands somewhere +# different under each, so it is caught without ever being named. +# +# The comparison is on the resulting *range*, never on dateparser's datetime. A month or a +# year is delimited by what the author wrote, and `_read_flexible_point` discards the +# components its `period` does not vouch for, so `June 2026` and `2026` answer with one +# range from two different datetimes. Comparing datetimes would refuse them. + +# Two reference instants that disagree in every component -- year, month, day, weekday, +# hour, minute, second -- so nothing filled in from "now" can coincide across them. +_STABILITY_PROBE_BASES = ( + datetime(2001, 3, 4, 5, 6, 7), + datetime(2097, 11, 21, 22, 33, 44), +) + + +def _read_authored_point( + point: str, date_order: DateOrder, relative_base: datetime +) -> TemporalRange | None: + """Read one already-stripped point, treating `relative_base` as the present.""" + match _classify_authored_point(point): + case _IsoDay() as iso: + return _read_iso_day(iso, point, date_order, relative_base) + case _IsoMonth() as iso: + return _calendar_span( + date(iso.year, iso.month, 1), _next_month_start(iso.year, iso.month) + ) + case _MalformedIso(): + # The author wrote a machine date that names nothing. Refusal is `None`, as + # everywhere else here: the token stays ordinary observation content, + # unindexed but still full-text searchable. + return None + case _FlexiblePoint(): + return _read_flexible_point(point, date_order, relative_base) + case unreachable: # pragma: no cover - `_AuthoredPoint` is closed + assert_never(unreachable) + + def parse_authored_point( text: str, *, date_order: DateOrder = DEFAULT_DATE_ORDER ) -> TemporalRange | None: @@ -834,23 +908,46 @@ def parse_authored_point( instant and no instant is there; `2026-01-01T10:00:00.1234567` does not either, because storing it would mean dropping the digits that made it worth writing. + A reading that would depend on when it was taken is refused, however it is spelled: + `yesterday`, `2 days ago`, `next month`, a bare `March` whose year would come from + the current one. What the author wrote must name the same interval on every pass, or + the note does not say when it holds -- see the section comment above for why anchoring + it elsewhere would not fix that. Precision the author simply did not write is a + different thing and still reads: `2026` and `June 2026` delimit their own periods. + Returns None when the text names no date. That is not an error -- the caller leaves such a token as ordinary observation content. """ point = text.strip() - match _classify_authored_point(point): - case _IsoDay() as iso: - return _read_iso_day(iso, point, date_order) - case _IsoMonth() as iso: - return _calendar_span( - date(iso.year, iso.month, 1), _next_month_start(iso.year, iso.month) - ) - case _MalformedIso(): - # The author wrote a machine date that names nothing. Refusal is `None`, as - # everywhere else here: the token stays ordinary observation content, - # unindexed but still full-text searchable. - return None - case _FlexiblePoint(): - return _read_flexible_point(point, date_order) - case unreachable: # pragma: no cover - `_AuthoredPoint` is closed - assert_never(unreachable) + early, late = _STABILITY_PROBE_BASES + reading = _read_authored_point(point, date_order, early) + if reading is None: + return None + # Trigger: the same text named a different interval when "now" was somewhere else. + # Why: then the note's bytes do not fix its meaning, and every reindex is free to + # file a different valid time for a file nobody edited. + # Outcome: refused like any other unreadable point -- the token stays content. + # A point that never consults the flexible reader cannot move, so this second pass + # costs those tokens a regex and no date parsing at all. + if _read_authored_point(point, date_order, late) != reading: + return None + return reading + + +def names_only_a_calendar_period(text: str, *, date_order: DateOrder = DEFAULT_DATE_ORDER) -> bool: + """Whether the text names a month or a year rather than a specific day. + + This explains a refusal; it never files one. `parse_authored_point` answers "does this + name one interval, whatever the date?", and a bare month name fails that because its + year comes from the present. But *that it is a month at all* does not: `June` reads as + a bounded calendar period under any present, and only which one moves. Asking for the + shape rather than the value recovers a fact the stability rule would otherwise take + with it -- the fact that lets `@occurred:June 10, 2026` be told to quote itself instead + of going silently unread. + + False for text that names nothing and for text that names a specific day, which are + the two cases with no truncated multi-word date to warn about. + """ + point = text.strip() + readings = (_read_authored_point(point, date_order, base) for base in _STABILITY_PROBE_BASES) + return all(reading is not None and reading.upper is not None for reading in readings) diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index 724cb11d6..541def245 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -17,8 +17,6 @@ before valid time existed. """ -from datetime import datetime, timedelta - import pytest from basic_memory import config as config_module @@ -259,18 +257,19 @@ def test_point_qualifier_names_its_kind_with_a_colon(qualifier: str, kind: TimeK assert str(assertion.valid_during) == "[2026-06-10,)" -def test_a_point_with_a_kind_accepts_a_relative_date(): - """With a kind the author has said what they mean, so any readable date is taken. +def test_a_point_with_a_kind_still_will_not_take_a_relative_date(): + """Naming the kind says what the author meant, not when -- the date must still say that. - Relative wording resolves at parse time and is re-resolved on every index pass. - That is documented behavior, not a mistake to warn about. + A qualifier whose meaning is re-derived from the wall clock on each index pass makes + an unedited note assert a different valid time every day, so it is refused however + plainly it was meant. Silently, like every other unread point: the line keeps its + text and stays full-text searchable. """ observation = _observation("- [decision] @occurred:yesterday The cutover ran.") - [assertion] = observation.temporal - yesterday = datetime.now().date() - timedelta(days=1) - assert assertion.valid_during.lower == yesterday.isoformat() - assert observation.content == "The cutover ran." + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == "@occurred:yesterday The cutover ran." @pytest.mark.parametrize( @@ -302,23 +301,21 @@ def test_a_point_with_no_kind_must_be_digit_led_and_year_wide(qualifier: str): assert observation.content.startswith(qualifier) -def test_a_word_point_is_read_only_when_it_names_a_specific_day(): - """A kind opens the form to words, but not to words that name only a period. +def test_no_bare_word_point_is_read_any_more(): + """The word branch is empty now, and both halves of it stay silent. - `yesterday` resolves to one day and is taken. `may` resolves to a whole month, and - a bare month name at the head of a line is either prose or -- worse -- the first - token of `May 10, 2026`, where reading it would file May 2026 and leave `10, 2026` - behind as content. + `yesterday` used to be the case that justified admitting words at all; it is refused + now because its meaning moves. `may` was always refused -- a bare month name at the + head of a line is either prose or, worse, the first token of `May 10, 2026`, where + reading it would file May 2026 and leave `10, 2026` behind as content. Neither is + reported here, because neither line continues with a digit. """ - day = _observation("- [decision] @occurred:yesterday The cutover ran.") - [assertion] = day.temporal - assert assertion.time_kind is TimeKind.OCCURRED - assert day.content == "The cutover ran." + for qualifier in ("@occurred:yesterday", "@occurred:may"): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") - period = _observation("- [decision] @occurred:may The cutover ran.") - assert period.temporal == [] - assert period.temporal_error is None - assert period.content.startswith("@occurred:may") + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content.startswith(qualifier) # --- The flexible vocabulary, as the qualifier grammar sees it --- @@ -341,9 +338,8 @@ def test_a_word_point_is_read_only_when_it_names_a_specific_day(): "[2026-06-10T10:00:00.000000Z,)", TemporalRangeAxis.INSTANT, ), - # A kind admits a word, as long as it names one day. - ("@occurred:today", None, TemporalRangeAxis.DATE), - ("@occurred:yesterday", None, TemporalRangeAxis.DATE), + # No word is admitted any more: the only ones that named a single day did it by + # asking the clock. See test_a_point_with_a_kind_still_will_not_take_a_relative_date. ], ) def test_single_token_points_are_accepted(qualifier: str, literal: str | None, axis): @@ -485,17 +481,21 @@ def test_a_point_at_exactly_microsecond_precision_is_still_filed(): assert observation.content == "The cutover ran." -def test_a_quoted_relative_date_is_read_where_its_unquoted_form_is_not(): - """`2 days ago` always read fine; only the token rule kept it out.""" - quoted = _observation('- [decision] @occurred:"2 days ago" The cutover ran.') +def test_quoting_a_relative_date_does_not_rescue_it(): + """Quotes fix a token-boundary problem; they cannot fix a meaning that moves. - [assertion] = quoted.temporal - two_days_ago = datetime.now().date() - timedelta(days=2) - assert assertion.valid_during.lower == two_days_ago.isoformat() - assert quoted.content == "The cutover ran." + `"2 days ago"` is delimited, so the one-token rule has nothing to truncate -- and it + is still refused, because what it names depends on the day it is read. The two guards + are independent, and the quoted form only ever answered the first. + """ + for line in ( + '- [decision] @occurred:"2 days ago" The cutover ran.', + "- [decision] @occurred:2 days ago The cutover ran.", + ): + observation = _observation(line) - unquoted = _observation("- [decision] @occurred:2 days ago The cutover ran.") - assert unquoted.temporal == [] + assert observation.temporal == [] + assert observation.temporal_error is None def test_a_quoted_month_is_filed_where_the_specific_day_guard_refuses_it(): diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 8605d2343..5cc0505e5 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -10,8 +10,6 @@ of the codebase applies to naive datetimes. """ -from datetime import datetime, timedelta - import pytest from freezegun import freeze_time @@ -248,18 +246,20 @@ def test_authored_naive_timestamp_is_read_as_utc_not_local_time(): assert naive.lower == "2026-06-10T14:00:00.000000Z" -def test_authored_relative_dates_resolve_at_parse_time(): - """`yesterday` is read against the clock now, and re-read on every index pass. +def test_a_relative_point_names_nothing_the_note_can_keep(): + """`yesterday` names a different day every day, so it names nothing storable. - That is documented behavior rather than a diagnostic: a file edited by hand keeps - its relative wording, and each pass resolves it fresh. + A qualifier's meaning has to be recoverable from the note's own bytes, because those + are the only thing that travels to a clone or survives a re-import. Read against the + wall clock, an unedited file's stored range changed on every index pass -- and a + search that matched it last week stopped matching today with nothing written. The two + obvious anchors are no better: `created_at` is derived metadata that can shift, and a + stored resolution cannot be reproduced by a fresh clone reindexing from markdown + alone. Refusing is what leaves the file as the only source of the answer. """ - span = parse_authored_point("yesterday") - - assert span is not None - assert span.axis is DATE - yesterday = datetime.now().date() - timedelta(days=1) - assert span.lower == yesterday.isoformat() + with freeze_time("2026-09-02"): + assert parse_authored_point("yesterday") is None + assert parse_authored_point("2 days ago") is None # The written vocabulary. These are what the *reader* accepts; the qualifier grammar @@ -295,13 +295,98 @@ def test_written_dates_read_on_the_axis_their_precision_names(written, literal, assert span.axis is axis -def test_written_relative_dates_resolve_against_now(): - """dateparser's relative vocabulary is read whole when it is handed a whole phrase.""" - span = parse_authored_point("2 days ago") +# Two clocks far enough apart that anything taken from "now" lands somewhere different +# under each. Every accepted spelling must name one interval under both; every refused one +# must be refused under both. This pair is the actual guarantee -- it catches relative +# wording nobody thought to enumerate, in languages nobody thought to test. +_FAR_APART_CLOCKS = ("2026-09-02", "2027-04-19") - assert span is not None - assert span.axis is DATE - assert span.lower == (datetime.now().date() - timedelta(days=2)).isoformat() + +@pytest.mark.parametrize( + "written", + [ + # ISO, in every precision the reader distinguishes. + "2026-06-10", + "2026-06", + "9999-12", + "2026", + "2026-1-5", + "2026-06-10T14:00", + "2026-06-10 14:00:00+02:00", + "2026-06-10 14:00:00.5", + "2026-01-01T10:00:00.123456", + # Spelled out, and slash-formatted: absolute, just not machine syntax. + "June 10, 2026", + "10 June 2026", + "Jan 15, 2024", + "June 2026", + "2026/03/04", + "10/07/2026", + # A clock reading beside an absolute date, in both syntaxes. + "2026-06-10 10:00 AM", + "2026-06-10 noon", + "10/07/2026 14:00", + "June 10, 2026 2pm", + ], +) +def test_an_accepted_point_names_the_same_interval_whenever_it_is_read(written: str): + """Precision the author did not write is fine; meaning that moves is not. + + `2026` and `June 2026` name periods the author delimited, and they must go on reading + -- refusing everything under-specified would have taken them with the relative forms. + What separates the two is not syntax or precision but whether the answer depends on + the day the question is asked. + """ + readings = [] + for clock in _FAR_APART_CLOCKS: + with freeze_time(clock): + readings.append(parse_authored_point(written)) + + assert readings[0] is not None + assert readings[0] == readings[1] + + +@pytest.mark.parametrize( + "written", + [ + # The reported shapes. + "yesterday", + "2 days ago", + # The rest of the relative vocabulary, in every direction and grain. + "today", + "tomorrow", + "now", + "last week", + "next week", + "last month", + "next month", + "last year", + "next year", + "in 3 days", + "3 hours ago", + "an hour ago", + "the day before yesterday", + # Under-specified rather than relative, and just as unstable: the year, or the + # year and month, would be taken from whenever the index happened to run. + "March", + "may", + "December", + # dateparser reads far more than English, which is exactly why the rule is + # determinism and not a list of words somebody wrote down. + "hace 2 dias", + "il y a 2 jours", + "vor 2 tagen", + ], +) +def test_a_point_whose_meaning_moves_with_the_clock_is_unread(written: str): + """Refused under every clock, not merely different under two. + + A form that read on one day and not another would be worse than either -- the note + would gain and lose an assertion as the calendar turned. + """ + for clock in _FAR_APART_CLOCKS: + with freeze_time(clock): + assert parse_authored_point(written) is None @pytest.mark.parametrize( From 3c93a92c90c799470fa1de817a1d759256bffceb Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 10:45:18 -0500 Subject: [PATCH 13/25] fix(core): count observation ordinals on the stored category A follow-up hole in the duplicate ordinal added by 32813182, and it put the original bug straight back for one shape of note. A line promoted to an observation by its hashtags alone carries no `[category]` prefix, so it reaches the writer as `None`, while an explicit `[note]` line beside it reaches it as `"note"`. The ordinal was counted on the value as given, so the two looked like different identities and both took ordinal 0. One flush later the column default landed and made them the same category, so they derived the same permalink, the permalink-keyed search index kept only the first, and the second observation's temporal assertion again addressed a row with no search projection: - @effective[2026-06-10,2026-07-27) The cache layer will use Redis. #infra - [note] @effective[2027-06-10,2027-07-27) The cache layer will use Redis. #infra Reproduced before the fix: both rows stored `category='note'` with `duplicate_index=0`, one search row, and the 2027 query returning nothing. Identity has to be computed on the value the row will *hold*, not the value it was handed. `observation_permalink_tail` now normalizes the category the same way the column does, and the default is named once as `OBSERVATION_DEFAULT_CATEGORY` and used by both, so the two cannot drift apart again. No permalink changes: the property already read a flushed row, where the category was always `"note"` -- only the ordinal was being counted on the pre-flush value. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/models/knowledge.py | 21 +++++++- .../services/test_search_service_temporal.py | 49 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 2f79c6295..58876ccaa 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -339,6 +339,15 @@ def __repr__(self) -> str: # pragma: no cover ) +# The category a row takes when the author wrote no `[category]` prefix -- a line promoted +# to an observation by its hashtags alone arrives with none. Named here so the column +# default and the identity below are the same value by construction: SQLAlchemy applies +# this at flush, so identity computed from an un-normalized `None` would be identity for a +# category the stored row never has, and two lines that end up in the same category would +# each believe they were the first of their kind. +OBSERVATION_DEFAULT_CATEGORY = "note" + + def observation_permalink_tail(category: str | None, content: str) -> str: """The part of an observation's permalink that distinguishes it within its note. @@ -360,9 +369,17 @@ def observation_permalink_tail(category: str | None, content: str) -> str: values: `Foo Bar` and `foo-bar` are different content that generate one permalink, so an ordinal counted over raw content would leave them colliding. + The category is normalized the way the column is, because identity has to be computed + on the value the row will *hold*, not the value it was handed. A hashtag-promoted line + arrives with `None` and an explicit `[note]` line with `"note"`; they look distinct + here and are the same row category after flush, so leaving them distinct would hand + both the ordinal 0 and put the collision straight back. + Content is truncated to 200 chars to stay under PostgreSQL's btree index limit of 2704 bytes. """ + if category is None: + category = OBSERVATION_DEFAULT_CATEGORY if len(content) > 200: # Trigger: content exceeds the 200-char budget imposed by PostgreSQL's # 2704-byte btree index row limit, so the permalink can only carry a prefix. @@ -394,7 +411,9 @@ class Observation(Base): project_id: Mapped[int] = mapped_column(Integer, ForeignKey("project.id"), index=True) entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE")) content: Mapped[str] = mapped_column(Text) - category: Mapped[str] = mapped_column(String, nullable=False, default="note") + category: Mapped[str] = mapped_column( + String, nullable=False, default=OBSERVATION_DEFAULT_CATEGORY + ) context: Mapped[Optional[str]] = mapped_column(Text, nullable=True) tags: Mapped[Optional[list[str]]] = mapped_column( JSON, nullable=True, default=list, server_default="[]" diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py index b1df63316..737225ed2 100644 --- a/tests/services/test_search_service_temporal.py +++ b/tests/services/test_search_service_temporal.py @@ -281,6 +281,55 @@ async def test_same_statement_at_two_times_is_queryable_at_each(entity_service, assert first.id != second.id +MIXED_CATEGORY_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - @effective[2026-06-10,2026-07-27) The cache layer will use Redis. #infra + - [note] @effective[2027-06-10,2027-07-27) The cache layer will use Redis. #infra + """) + + +@pytest.mark.asyncio +async def test_an_omitted_category_is_the_same_category_when_ordinals_are_counted( + entity_service, search_service +): + """The ordinal has to be counted on the category the row will hold, not the one given. + + A line promoted to an observation by its hashtags alone carries no `[category]`, so it + arrives as `None`, while the explicit `[note]` line beside it arrives as `"note"`. + They look like different identities at the moment the ordinal is assigned and are the + same category one flush later, when the column default lands -- so both were numbered + 0, derived one permalink, and lost a search row between them exactly as if the ordinal + had never been added. Normalizing before counting is what keeps the two halves of the + identity agreeing about what a row is. + """ + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Cache Layer Mixed", + note_type="note", + directory="decisions", + content=MIXED_CATEGORY_MARKDOWN, + ) + ) + await search_service.index_entity(entity) + + first, second = entity.observations + # The stored rows agree on category; the addresses must still tell them apart. + assert first.category == second.category + assert first.permalink != second.permalink + + in_first = await search_service.search( + SearchQuery(text="cache layer", valid_at=EFFECTIVE_WINDOW_INSIDE) + ) + in_second = await search_service.search( + SearchQuery(text="cache layer", valid_at=SECOND_WINDOW_INSIDE) + ) + + assert [result.id for result in in_first] == [first.id] + assert [result.id for result in in_second] == [second.id] + + @pytest.mark.asyncio async def test_a_valid_time_query_can_also_scope_by_note_type(entity_service, search_service): """Valid time selects observation rows; note type must not then exclude them. From 030b3c2f0e138270daff98bb53a7b67456e84d90 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 11:34:12 -0500 Subject: [PATCH 14/25] fix(core): reject empty valid-time filter values instead of ignoring them `{"text": "cache", "valid_at": ""}` ran as an ordinary unfiltered search and answered with the undated rows the filter existed to exclude, reporting itself as a plain search the whole way. The three valid-time fields were tested for truthiness wherever presence was meant, so a present-but-empty value read as absence and the requested constraint was dropped in silence. `valid_at`, `valid_overlaps` and `time_kind` are declared `Optional[str] = None`, so "no valid-time filter" already has a spelling and it is not `""`. Every caller in the tree is an MCP tool or typed client passing JSON, not a form encoder -- confirmed by grep across src, tests and test-int, which finds no caller or test passing an empty or whitespace value for any of the three. So rejecting `""` turns away no real caller; it turns away a caller who believes they applied a filter and did not, which is the same silent-unfiltered failure the rest of this PR has been closing. `None` still means absent everywhere. The only change is that a present but empty value is an error rather than a no-op, and presence is now tested against `None` rather than truthiness so the two can never be confused again. The rule lives once, in `reject_blank_temporal_value`, and is applied at both places a value can enter: * `parse_temporal_filter`, the parser every surface shares -- which is what makes `search_notes` reject a blank before any project is searched, so the all-projects fan-out cannot resurrect the silent-empty-result bug this PR already fixed once. * the `SearchQuery` field validator, because `no_criteria()` runs first: a query carrying only an empty `valid_at` would otherwise be turned away as having no criteria at all, which is true of the value and false of the request. Whitespace-only rejects too, and the message names the offending field and says that omitting it is how you ask for no valid-time filter. Five presence checks used the truthiness shape; all now test `None`: `parse_temporal_filter`, `SearchQuery.has_temporal_filter`, the fan-out's `temporal_requested`, the tool's `has_temporal_filter` span field, and the post-construction field assignment in `search_notes`. Two truthiness tests are deliberately left: the mixed `has_filters` aggregate, whose other operands are lists, and the `valid_at and valid_overlaps` pair check, where letting the blank diagnostic win over "not both" is the better message. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/mcp/tools/search.py | 17 +++++-- src/basic_memory/schemas/search.py | 30 ++++++++++- src/basic_memory/temporal.py | 35 +++++++++++-- tests/mcp/test_tool_search_temporal.py | 50 +++++++++++++++++++ .../services/test_search_service_temporal.py | 43 ++++++++++++++++ tests/test_temporal.py | 40 +++++++++++++++ 6 files changed, 203 insertions(+), 12 deletions(-) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index a36a2ebc7..a4891f543 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -598,7 +598,9 @@ async def _search_all_projects( # silently mixes filtered and unfiltered rows. The filter itself is already known to # be well formed -- search_notes parses it before reaching here -- which is what # makes "dropped with a warning" mean an unavailable project and nothing else. - temporal_requested = bool(valid_at or valid_overlaps or time_kind) + # Presence, not truthiness: a blank value is refused by `parse_temporal_filter` + # before any project is searched, so anything not None is a real question here. + temporal_requested = valid_at is not None or valid_overlaps is not None or time_kind is not None project_refs = await _load_search_project_refs(context=context) if not project_refs: response = SearchResponse( @@ -1243,7 +1245,9 @@ async def search_notes( ), has_tags_filter=bool(tags), has_status_filter=bool(status), - has_temporal_filter=bool(valid_at or valid_overlaps or time_kind), + has_temporal_filter=( + valid_at is not None or valid_overlaps is not None or time_kind is not None + ), ): async with get_project_client(project, context=context, project_id=project_id) as ( client, @@ -1324,11 +1328,14 @@ async def search_notes( search_query.status = status if min_similarity is not None: search_query.min_similarity = min_similarity - if valid_at: + # Presence, not truthiness, for the same reason as everywhere else on + # this path: these are assigned after construction, so the model's own + # blank guard never runs here, and a blank has already been refused above. + if valid_at is not None: search_query.valid_at = valid_at - if valid_overlaps: + if valid_overlaps is not None: search_query.valid_overlaps = valid_overlaps - if time_kind: + if time_kind is not None: search_query.time_kind = time_kind # Reject searches with no criteria at all diff --git a/src/basic_memory/schemas/search.py b/src/basic_memory/schemas/search.py index 4ef4120a1..89d35b982 100644 --- a/src/basic_memory/schemas/search.py +++ b/src/basic_memory/schemas/search.py @@ -9,9 +9,10 @@ from typing import Optional, List, Union, Any from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator from basic_memory.schemas.base import Permalink, normalize_note_type +from basic_memory.temporal import TemporalQualifierError, reject_blank_temporal_value class SearchItemType(str, Enum): @@ -128,6 +129,24 @@ def validate_temporal_filter(self) -> "SearchQuery": ) return self + @field_validator("valid_at", "valid_overlaps", "time_kind") + @classmethod + def check_temporal_value_not_blank( + cls, value: Optional[str], info: ValidationInfo + ) -> Optional[str]: + """Refuse a valid-time field that is present but empty. + + Enforced at the boundary as well as in the parser, because `no_criteria()` runs + first: a query carrying only an empty `valid_at` would otherwise be turned away as + having no criteria at all, which is true of the value and false of the request. + The rule itself lives in `basic_memory.temporal` so the two cannot disagree. + """ + try: + reject_blank_temporal_value(info.field_name or "", value) + except TemporalQualifierError as exc: + raise ValueError(str(exc)) from exc + return value + @field_validator("after_date") @classmethod def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]: @@ -162,7 +181,14 @@ def has_temporal_filter(self) -> bool: assertion of that kind. Callers use this to decide whether valid time was requested without parsing the values, which is why it never raises. """ - return bool(self.valid_at or self.valid_overlaps or self.time_kind) + # Presence, not truthiness: a blank value is refused above, so anything that is + # not None is a real question. Testing truthiness here is what let an empty + # `valid_at` read as "no filter requested" and run the query unfiltered. + return ( + self.valid_at is not None + or self.valid_overlaps is not None + or self.time_kind is not None + ) def no_criteria(self) -> bool: text_is_empty = self.text is None or (isinstance(self.text, str) and not self.text.strip()) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index d516c07d3..6ad8cdd00 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -474,6 +474,27 @@ def parse_point(text: str) -> TemporalPoint: return TemporalPoint(axis=axis, value=canonical_bound(bound, axis)) +TEMPORAL_FILTER_FIELDS = ("valid_at", "valid_overlaps", "time_kind") + + +def reject_blank_temporal_value(field: str, value: str | None) -> None: + """Refuse a valid-time field that is present but carries nothing. + + `None` is how a caller says "no valid-time filter"; these fields are declared optional + precisely so that spelling exists. An empty or whitespace-only string is a different + statement -- a caller who believes they applied a filter -- and reading it as absence + is the failure this whole feature keeps having to close: a query that reports itself as + filtered, runs unfiltered, and answers with the undated rows the filter was meant to + exclude. Truthiness cannot tell the two apart, so presence is tested against `None` + everywhere on this path and blankness is refused here. + """ + if value is not None and not value.strip(): + raise TemporalQualifierError( + f"{field} was given as an empty value; omit {field} to search without a " + f"valid-time filter" + ) + + def parse_temporal_filter( *, valid_at: str | None = None, @@ -494,13 +515,17 @@ def parse_temporal_filter( without an offset is not a rejection -- like every other naive datetime in the codebase, it is read as UTC. - Returns None when no valid-time question was asked at all. + Returns None when no valid-time question was asked at all -- which means all three + fields are absent, not merely falsy. See `reject_blank_temporal_value`. """ - if not (valid_at or valid_overlaps or time_kind): + for field, value in zip(TEMPORAL_FILTER_FIELDS, (valid_at, valid_overlaps, time_kind)): + reject_blank_temporal_value(field, value) + + if valid_at is None and valid_overlaps is None and time_kind is None: return None kind: TimeKind | None = None - if time_kind: + if time_kind is not None: try: kind = TimeKind(time_kind) except ValueError as exc: @@ -511,8 +536,8 @@ def parse_temporal_filter( return TemporalFilter( kind=kind, - at=parse_point(valid_at) if valid_at else None, - overlaps=parse_range_literal(valid_overlaps) if valid_overlaps else None, + at=parse_point(valid_at) if valid_at is not None else None, + overlaps=parse_range_literal(valid_overlaps) if valid_overlaps is not None else None, ) diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py index 4aa3e2c72..ed4dad09c 100644 --- a/tests/mcp/test_tool_search_temporal.py +++ b/tests/mcp/test_tool_search_temporal.py @@ -346,6 +346,56 @@ async def test_all_projects_search_refuses_a_malformed_filter_instead_of_reporti ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field", "search"), + [ + ( + "valid_at", + lambda: search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + valid_at="", + ), + ), + ( + "valid_overlaps", + lambda: search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + valid_overlaps=" ", + ), + ), + ( + "time_kind", + lambda: search_notes( + query="cache layer", + search_all_projects=True, + output_format="json", + time_kind="", + ), + ), + ], +) +async def test_all_projects_search_refuses_an_empty_filter_instead_of_running_unfiltered( + client, test_project, field: str, search +): + """An empty value must not resurrect the silent-unfiltered fan-out. + + This is the shape the fan-out is least able to survive. A blank field read as absence + meant every per-project leg ran an ordinary unfiltered search, so the merged answer + came back full of undated rows with nothing marking it as unfiltered -- the caller + asked a valid-time question and got a plain search wearing the same shape. Refused + once, before any project is searched, by the parser the per-project legs share. + """ + await _write_cache_layer_note(test_project.name) + + with pytest.raises(ValueError, match=f"{field} was given as an empty value"): + await search() + + @pytest.mark.asyncio async def test_time_kind_alone_is_enough_search_criteria(client, test_project): """A valid-time filter is real criteria, so it must not trip the empty-query guard.""" diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py index 737225ed2..aa87a596f 100644 --- a/tests/services/test_search_service_temporal.py +++ b/tests/services/test_search_service_temporal.py @@ -10,6 +10,7 @@ from textwrap import dedent import pytest +from pydantic import ValidationError from basic_memory.schemas import Entity as EntitySchema from basic_memory.schemas.search import SearchQuery @@ -183,6 +184,48 @@ def test_kind_only_query_builds_a_kind_filter(): assert temporal.at is None and temporal.overlaps is None +@pytest.mark.parametrize( + ("field", "build"), + [ + ("valid_at", lambda blank: SearchQuery(text="cache", valid_at=blank)), + ("valid_overlaps", lambda blank: SearchQuery(text="cache", valid_overlaps=blank)), + ("time_kind", lambda blank: SearchQuery(text="cache", time_kind=blank)), + ], +) +@pytest.mark.parametrize("blank", ["", " ", "\t"]) +def test_a_present_but_empty_valid_time_field_is_refused(field: str, build, blank: str): + """An empty value is a caller who thinks they filtered, not one who did not ask. + + These fields are `Optional[str] = None`, so "no valid-time filter" already has a + spelling, and it is not `""`. Read as absence, an empty value ran the query unfiltered + and answered with exactly the undated rows the filter existed to exclude -- reporting + itself as a plain search all the while. That is the same silent-unfiltered failure the + rest of this feature keeps closing, so it is refused loudly and by name. + """ + with pytest.raises(ValidationError, match=f"{field} was given as an empty value"): + build(blank) + + +def test_omitting_every_valid_time_field_still_searches_unfiltered(): + """`None` keeps meaning absent, which is the whole reason `""` can mean a mistake.""" + query = SearchQuery(text="cache") + + assert query.has_temporal_filter() is False + assert build_temporal_filter(query) is None + assert query.no_criteria() is False + + +def test_a_real_valid_time_value_still_filters(): + """The refusal must cost nothing that was actually asking a question.""" + query = SearchQuery(text="cache", valid_at=EFFECTIVE_WINDOW_INSIDE) + + assert query.has_temporal_filter() is True + temporal = build_temporal_filter(query) + assert temporal is not None + assert temporal.at is not None + assert temporal.at.value == EFFECTIVE_WINDOW_INSIDE + + def test_valid_at_and_valid_overlaps_are_mutually_exclusive_at_the_schema(): """The schema refuses the pair before any parsing or SQL can happen.""" with pytest.raises(ValueError, match="not both"): diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 5cc0505e5..49360e5c7 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -26,6 +26,7 @@ parse_authored_point, parse_point, parse_range_literal, + parse_temporal_filter, ) DATE = TemporalRangeAxis.DATE @@ -1268,3 +1269,42 @@ def test_recorded_time_is_not_an_authorable_kind(): "due", "mentioned", } + + +# --- A present but empty valid-time field --- + + +@pytest.mark.parametrize("field", ["valid_at", "valid_overlaps", "time_kind"]) +@pytest.mark.parametrize("blank", ["", " ", "\t"]) +def test_a_blank_valid_time_field_is_refused_by_the_shared_parser(field: str, blank: str): + """The parser every surface shares refuses a field that is present but says nothing. + + `parse_temporal_filter` is the one place HTTP, MCP and CLI all reach, and its contract + is that a rejection is loud rather than a filter that quietly matches something else. + Testing the three fields for truthiness broke exactly that: an empty value read as + absence and the query ran unfiltered. + """ + with pytest.raises(TemporalQualifierError, match=f"{field} was given as an empty value"): + parse_temporal_filter(**{field: blank}) + + +def test_the_message_names_the_field_and_the_way_to_say_no_filter(): + """A diagnostic is only useful if the fix is in it.""" + with pytest.raises(TemporalQualifierError) as caught: + parse_temporal_filter(valid_at="") + + assert "omit valid_at" in str(caught.value) + + +def test_absent_valid_time_fields_still_build_no_filter(): + """`None` means absent, and must go on meaning that -- only blankness changed.""" + assert parse_temporal_filter() is None + assert parse_temporal_filter(valid_at=None, valid_overlaps=None, time_kind=None) is None + + +def test_a_real_value_beside_absent_ones_still_builds_a_filter(): + temporal = parse_temporal_filter(valid_at="2026-07-28") + + assert temporal is not None + assert temporal.at is not None + assert temporal.at.value == "2026-07-28" From afea122934f967f3c7ea7dead3a1d86837729817 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 12:02:31 -0500 Subject: [PATCH 15/25] docs(mcp): stop advertising relative valid-time qualifiers cbb2ca75 made a qualifier whose meaning moves with the clock unreadable, but left the instructions promising the opposite. `search_notes`' docstring still said "single-word relative dates (`@occurred:yesterday`) work as they are" and offered `@occurred:"2 days ago"` as a quoted example that files; the man page said the same in its prose and in two gotchas. That is worse than a stale comment. The docstring is the interface an agent programs against, so the promise produced the exact failure the fix closed from the other end: the agent writes `@occurred:yesterday`, the reader declines it silently, the line stays ordinary content, and every valid-time search omits it. Both now say relative dates are not accepted, name the spellings that are refused (`yesterday`, `"2 days ago"`, `"last week"`, a bare `March`), give the reason -- they name a different span on every index pass -- and say what to write instead. They also record that quoting does not rescue one, since quotes settle where a token ends rather than what a date means. Scoped to the authored valid-time axis only. `timeframe` on recent_activity, build_context and the posix tools still takes relative wording and is untouched: those ask about edit time, which really is relative to now. Both documents now say so explicitly, because the two axes reading differently is exactly the kind of thing a reader would otherwise assume was an oversight. Two tests pin the instructions against the behaviour, so the docs cannot drift back while the reader goes on refusing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/man/man3/search-notes(3).md | 15 +++++--- src/basic_memory/mcp/tools/search.py | 21 ++++++---- tests/mcp/test_tool_search_temporal.py | 40 ++++++++++++++++++++ 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/basic_memory/man/man3/search-notes(3).md b/src/basic_memory/man/man3/search-notes(3).md index 41d81a8b5..337af10e4 100644 --- a/src/basic_memory/man/man3/search-notes(3).md +++ b/src/basic_memory/man/man3/search-notes(3).md @@ -60,10 +60,11 @@ a point, `- [decision] @effective:2026-07-27 ...` / `- [decision] @2026-07-27 ...` — and these filters match against that authored interval. A point means the span its precision covers: `@2026` is that year, `@2026-06` that month, and `@2026-06-10` from that date onward. An unquoted point is one -whitespace-delimited token; a multi-word, relative, or month-only date goes in -double quotes, which end the token at the closing quote: -`@occurred:"June 10, 2026"`, `@occurred:"2 days ago"`, `@"June 2026"`. It is a -separate axis from `after_date`, which keeps filtering last-indexed time. +whitespace-delimited token; a multi-word or month-only date goes in double +quotes, which end the token at the closing quote: `@occurred:"June 10, 2026"`, +`@"June 2026"`. A relative date is not accepted in a qualifier at all — it would +name a different span on every index pass — so write the date it should mean. It +is a separate axis from `after_date`, which keeps filtering last-indexed time. Bounds follow PostgreSQL range conventions, calendar dates and instants never convert into one another, and a source with no qualifier is excluded from any valid-time @@ -134,8 +135,10 @@ bm tool search-notes "conflict error" --project manual --page-size 2 - [gotcha] valid_at and valid_overlaps never mix calendar dates with instants: a date query matches only date ranges and an instant query only instant ranges, so `2026-07-27` and `2026-07-27T00:00:00Z` are different questions #valid-time - [gotcha] A timestamp written without an offset is read as UTC, in an authored qualifier and in a filter alike — same convention as every other naive datetime in Basic Memory #valid-time - [gotcha] An authored token that does not read as a date is left as ordinary observation content with no warning; only a qualifier the author plainly meant is reported — an unknown kind (`@asserted:2026-06-10`), an unterminated quote, or a date the one-token rule truncated #valid-time -- [gotcha] An unquoted authored point is one whitespace-delimited token: `@occurred:2026-06-10`, `@occurred:03/04/2026` and `@occurred:yesterday` work, but a multi-word date like `@occurred:June 10, 2026` is left as content because nothing can tell where it ends #valid-time -- [gotcha] Double quotes lift the one-token rule and end the point at the closing quote, so `@occurred:"June 10, 2026"`, `@occurred:"2 days ago"` and `@"June 2026"` all file — inside quotes even a month-only or year-only date is taken, since the author delimited it #valid-time +- [gotcha] An unquoted authored point is one whitespace-delimited token: `@occurred:2026-06-10` and `@occurred:03/04/2026` work, but a multi-word date like `@occurred:June 10, 2026` is left as content because nothing can tell where it ends #valid-time +- [gotcha] Double quotes lift the one-token rule and end the point at the closing quote, so `@occurred:"June 10, 2026"` and `@"June 2026"` both file — inside quotes even a month-only or year-only date is taken, since the author delimited it #valid-time +- [gotcha] A relative date is never filed as a qualifier, quoted or not: `@occurred:yesterday`, `@occurred:"2 days ago"` and a bare month name like `@occurred:March` each name a different span depending on when the note is indexed, so they stay ordinary content — quoting settles where a token ends, not what a date means #valid-time +- [gotcha] Relative wording still works for `timeframe` in recent-activity and build-context, which ask about edit time rather than authored time — only the valid-time qualifier requires a date that means the same thing on every pass #valid-time - [gotcha] Only `"` opens a quoted point, never `'`, and an unterminated quote is reported rather than swallowing the rest of the line #valid-time - [gotcha] `@occurred:03/04/2026` resolves by the `date_order` setting (YMD/DMY read it as 3 April, MDY as 4 March); ISO dates are never re-guessed #valid-time diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index a4891f543..065155e05 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -941,18 +941,25 @@ async def search_notes( An unquoted point is **one whitespace-delimited token**, because nothing can tell where a multi-word date ends. Slash dates (`@occurred:03/04/2026`, read by the - `date_order` setting) and single-word relative dates (`@occurred:yesterday`) work - as they are; anything longer goes in double quotes, which move the token boundary - to the closing quote: + `date_order` setting) work as they are; anything longer goes in double quotes, + which move the token boundary to the closing quote: - [decision] @occurred:"June 10, 2026" The cutover ran. - - [decision] @occurred:"2 days ago" The cutover ran. - [decision] @occurred:"June 2026" The cutover ran. - [decision] @"June 10, 2026" The cutover ran. - Whatever is inside the quotes is read as the date, month-only and relative forms - included, and whatever follows the closing quote is ordinary content. An unreadable - token is left as content, never half-read. + Whatever is inside the quotes is read as the date, month-only forms included, and + whatever follows the closing quote is ordinary content. An unreadable token is left + as content, never half-read. + + **Relative dates are not accepted here.** `@occurred:yesterday`, `@occurred:"2 days + ago"`, `@occurred:"last week"` and a bare month name like `@occurred:March` all name + a different span depending on the day the note is indexed, so an unedited file would + assert a different valid time on every pass. They stay ordinary content, silently, + and quoting does not change that — quotes settle where a token ends, not what a date + means. Write the date the qualifier should mean: `@occurred:2026-06-10`. This is the + authored-time axis only; `recent_activity` and `build_context` still take relative + `timeframe` values, because those ask about edit time, which really is relative to now. These filters query that authored time, which is a different axis from `after_date` (last-indexed time) — `after_date` is never reinterpreted as valid time. diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py index ed4dad09c..9df74eed4 100644 --- a/tests/mcp/test_tool_search_temporal.py +++ b/tests/mcp/test_tool_search_temporal.py @@ -416,3 +416,43 @@ def test_tool_help_documents_undated_exclusion(): doc = inspect.getdoc(search_notes) or "" assert "Sources with no temporal qualifier are excluded" in doc assert "valid_at" in doc and "valid_overlaps" in doc and "time_kind" in doc + + +# --- The instructions must not promise what the reader refuses --- + +_RELATIVE_QUALIFIER_SPELLINGS = ("yesterday", "2 days ago", "last week") + + +def test_the_tool_instructions_do_not_advertise_relative_qualifiers(): + """Docs are the interface an agent actually programs against. + + `search_notes`' docstring is what an LLM reads before writing a qualifier, so a stale + promise here is worse than a stale comment: the agent writes `@occurred:yesterday`, + the reader silently declines it, the line stays ordinary content, and every valid-time + search quietly omits it. The behaviour is pinned in tests/test_temporal.py; this pins + that the instructions agree with it. + """ + from basic_memory.mcp.tools.search import search_notes + + docs = ( + (search_notes.fn.__doc__ or "") + if hasattr(search_notes, "fn") + else (search_notes.__doc__ or "") + ) + + assert "Relative dates are not accepted" in docs + for spelling in _RELATIVE_QUALIFIER_SPELLINGS: + # Named only to refuse them; never shown as a working example. + if spelling in docs: + assert "not accepted" in docs + + +def test_the_man_page_does_not_advertise_relative_qualifiers(): + """The same promise, in the page a human reads.""" + from basic_memory.man import MAN_DIR + + page = (MAN_DIR / "man3" / "search-notes(3).md").read_text(encoding="utf-8") + + assert "A relative date is never filed as a qualifier" in page + # The quoted-form example list must not offer one as a spelling that files. + assert '`@occurred:"2 days ago"` and `@"June 2026"` all file' not in page From 6c6fa26129f791a3bdfca6db9ae359802b6cf478 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 12:32:47 -0500 Subject: [PATCH 16/25] fix(mcp): reserve the duplicate ordinal and propagate valid-time skew Two ways a valid-time answer could still come back confidently wrong. **The duplicate ordinal shared a namespace with real content.** 32813182 spelled the second twin of `foo` as `.../foo/1`, which is a tail an author's own text can produce: an observation reading `foo/1` slugs to exactly that. Whichever of the two was indexed second lost its search row to the permalink-keyed index, and if that was the dated one its temporal assertion pointed at no searchable document and the window became unanswerable -- the failure the ordinal was added to fix, reintroduced by the ordinal. I had noted this collision as theoretical when the ordinal landed and judged it far-fetched; it is cheap to close properly, and "far-fetched" is not a property worth relying on. The ordinal now lives in a segment content cannot reach. `generate_permalink` replaces every character outside `[a-z0-9/-.]` (plus CJK) with a hyphen, in both its ASCII and CJK branches, so no generated tail can contain `~`; 60k fuzzed inputs produce none. `.../foo/~1` is therefore a namespace the ordinal owns rather than one it shares. No released permalink changes -- the only permalinks carrying an ordinal were introduced by 32813182 on this branch. **An all-projects valid-time search could confirm a filter that ran nowhere.** A server predating SPEC-82 accepts a valid-time query and answers unfiltered, so `SearchClient` refuses any response omitting `temporal_applied`. The fan-out catches that refusal, logs the project and skips it -- right for one unavailable project, wrong for a whole fleet of old servers, because the merged response then reported `temporal_applied: true` over zero results. The caller could not tell "no note asserts that window" from "the filter ran nowhere", which is precisely what the confirmation field exists to prevent. The fan-out now counts the projects that answered and refuses to confirm a filter none of them applied, raising one error that names the skew instead of returning an empty result wearing the shape of a successful filtered search. A partial failure is unchanged: legs that did answer honored the filter, so the merged rows are genuinely filtered and `total_is_exact` already reports the incompleteness. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/mcp/tools/search.py | 22 ++++++++ src/basic_memory/models/knowledge.py | 15 +++++- tests/mcp/test_tool_search_temporal.py | 50 +++++++++++++++++++ .../services/test_search_service_temporal.py | 50 +++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) diff --git a/src/basic_memory/mcp/tools/search.py b/src/basic_memory/mcp/tools/search.py index 065155e05..373f6dbc4 100644 --- a/src/basic_memory/mcp/tools/search.py +++ b/src/basic_memory/mcp/tools/search.py @@ -621,6 +621,9 @@ async def _search_all_projects( total = 0 total_is_exact = True any_project_has_more = False + # How many projects actually answered. A leg that fails is skipped with a warning, + # so without this the caller cannot tell "no note matched" from "nothing ran". + projects_answered = 0 # Trigger: caller asked for an account-wide search. # Why: project_id (external UUID) routes through the cloud v2 API path, @@ -683,12 +686,29 @@ async def _search_all_projects( total_is_exact = False continue + projects_answered += 1 raw_results = _raw_results_from_search_payload(results) total += _result_total(results, raw_results) total_is_exact = total_is_exact and _result_total_is_exact(results) any_project_has_more = any_project_has_more or results.get("has_more") is True merged_results.extend(_qualify_results_for_project(raw_results, project_ref)) + # Trigger: a valid-time filter was requested and not one project answered. + # Why: each leg confirms the filter through SearchClient or is refused by it, and a + # refusal is caught above, logged, and skipped -- so a fleet of servers predating + # SPEC-82 drops every leg and arrives here indistinguishable from "no note matched". + # Claiming `temporal_applied` on that would confirm a filter that ran nowhere, which + # is the version skew the client's own check exists to make loud. + # Outcome: the skew is propagated as one error naming it, rather than returning an + # empty result wearing the shape of a successful filtered search. + if temporal_requested and projects_answered == 0: + raise ValueError( + "No project applied the requested valid-time filter: every project was " + "skipped, so the filter ran nowhere and an empty result would not mean " + "'no matches'. The servers are likely older than this client; upgrade them " + "or drop valid_at / valid_overlaps / time_kind from the query." + ) + # Each project owns retrieval and optional reranking behind its typed API client. # The MCP process only merges returned scores; it must not instantiate repository # providers with local credentials for content fetched through another route. @@ -704,6 +724,8 @@ async def _search_all_projects( "total": total, "total_is_exact": total_is_exact, "has_more": any_project_has_more or total > end or len(sorted_results) > end, + # Confirmed only because a project answered: `projects_answered` is + # non-zero here for any temporal query, guarded immediately above. "temporal_applied": True if temporal_requested else None, } ) diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 58876ccaa..9e8b63c57 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -347,6 +347,12 @@ def __repr__(self) -> str: # pragma: no cover # each believe they were the first of their kind. OBSERVATION_DEFAULT_CATEGORY = "note" +# Opens the permalink segment carrying a duplicate ordinal. `generate_permalink` replaces +# every character outside `[a-z0-9/-.]` (plus CJK) with a hyphen, so a tail generated from +# an author's own text can never contain `~` -- which is exactly what makes this a +# namespace the ordinal can own rather than one it shares with real content. +OBSERVATION_DUPLICATE_MARK = "~" + def observation_permalink_tail(category: str | None, content: str) -> str: """The part of an observation's permalink that distinguishes it within its note. @@ -438,13 +444,20 @@ def permalink(self) -> str: thing twice. It is 0 for the first observation carrying a given identity, so the overwhelming majority of permalinks are byte-identical to what they have always been; only the second and later twins gain a trailing ordinal. + + That ordinal lives in a segment natural content cannot reach. `generate_permalink` + emits only `[a-z0-9/-.]` plus CJK, so no observation's own text can ever slug to a + segment beginning with `~` -- whereas a plain `/1` is a segment content *can* + produce, and an observation whose text slugged to `foo/1` would then collide with + the second twin of `foo` and cost one of them its search row. Reserving the mark + is what makes the ordinal a namespace rather than a guess about what authors write. """ base = generate_permalink( f"{self.entity.permalink}/{observation_permalink_tail(self.category, self.content)}" ) if not self.duplicate_index: return base - return f"{base}/{self.duplicate_index}" + return f"{base}/{OBSERVATION_DUPLICATE_MARK}{self.duplicate_index}" @override def __repr__(self) -> str: # pragma: no cover diff --git a/tests/mcp/test_tool_search_temporal.py b/tests/mcp/test_tool_search_temporal.py index 9df74eed4..c5a5b388d 100644 --- a/tests/mcp/test_tool_search_temporal.py +++ b/tests/mcp/test_tool_search_temporal.py @@ -456,3 +456,53 @@ def test_the_man_page_does_not_advertise_relative_qualifiers(): assert "A relative date is never filed as a qualifier" in page # The quoted-form example list must not offer one as a spelling that files. assert '`@occurred:"2 days ago"` and `@"June 2026"` all file' not in page + + +@pytest.mark.asyncio +async def test_all_projects_search_propagates_a_filter_no_project_could_apply( + client, test_project, monkeypatch +): + """Every leg failing must not read as a successful search that found nothing. + + A server predating SPEC-82 accepts a valid-time query and answers unfiltered, so + `SearchClient` refuses any response that does not confirm `temporal_applied`. The + fan-out catches that refusal, logs the project and skips it -- correct for one + unavailable project, and wrong for a whole fleet of old servers, because the merged + answer then reported `temporal_applied: true` over zero results. The caller could not + tell "no note asserts that window" from "the filter ran nowhere", which is the exact + confusion the confirmation field exists to prevent. + """ + await _write_cache_layer_note(test_project.name) + + import sys + + # `basic_memory.mcp.tools.search` is shadowed by a `search` function exported + # from the package, so reach the module itself rather than that name. + search_module = sys.modules["basic_memory.mcp.tools.search"] + + async def refuse_every_leg(*args: Any, **kwargs: Any) -> str: + # What a per-project leg looks like once SearchClient rejects the response. + return "# Search Failed\n\nThe search API did not apply the requested valid-time filter" + + monkeypatch.setattr(search_module, "search_notes", refuse_every_leg) + + with pytest.raises(ValueError, match="No project applied the requested valid-time filter"): + await search_module._search_all_projects( + query="cache layer", + page=1, + page_size=10, + search_type="text", + output_format="json", + note_types=None, + entity_types=None, + categories=None, + after_date=None, + metadata_filters=None, + tags=None, + status=None, + min_similarity=None, + valid_at="2026-07-01", + valid_overlaps=None, + time_kind=None, + context=None, + ) diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py index aa87a596f..5fe2b9d15 100644 --- a/tests/services/test_search_service_temporal.py +++ b/tests/services/test_search_service_temporal.py @@ -324,6 +324,56 @@ async def test_same_statement_at_two_times_is_queryable_at_each(entity_service, assert first.id != second.id +# Two identical observations, plus a third whose own content slugs to the tail the +# second twin would take if the ordinal shared a namespace with real content. +ORDINAL_COLLISION_MARKDOWN = dedent(""" + # Cache Layer + + ## Observations + - [note] @effective[2026-06-10,2026-07-27) redis + - [note] @effective[2027-06-10,2027-07-27) redis + - [note] @effective[2028-06-10,2028-07-27) redis/1 + """) + +THIRD_WINDOW_INSIDE = "2028-07-01" + + +@pytest.mark.asyncio +async def test_a_duplicate_ordinal_cannot_be_taken_by_an_authors_own_text( + entity_service, search_service +): + """The ordinal needs a namespace content cannot occupy, not just a spare-looking one. + + Spelling the second twin of `redis` as `.../redis/1` put it in the same space as an + observation whose text genuinely slugs to `redis/1`. Whichever came second lost its + search row to the permalink-keyed index, and if that was the dated one, its temporal + assertion pointed at nothing and the window went unanswerable -- the very failure the + ordinal was added to fix, reintroduced by the ordinal itself. `generate_permalink` + emits no `~`, so the mark cannot be produced by any author's text. + """ + entity, _ = await entity_service.create_or_update_entity( + EntitySchema( + title="Cache Layer Ordinal", + note_type="note", + directory="decisions", + content=ORDINAL_COLLISION_MARKDOWN, + ) + ) + await search_service.index_entity(entity) + + first, second, natural = entity.observations + assert len({first.permalink, second.permalink, natural.permalink}) == 3 + + # Each of the three windows answers with its own observation, none lost. + for window, expected in ( + (EFFECTIVE_WINDOW_INSIDE, first), + (SECOND_WINDOW_INSIDE, second), + (THIRD_WINDOW_INSIDE, natural), + ): + hits = await search_service.search(SearchQuery(text="redis", valid_at=window)) + assert [result.id for result in hits] == [expected.id] + + MIXED_CATEGORY_MARKDOWN = dedent(""" # Cache Layer From 193aea02219cb3bc362b388695a6be3341c1188a Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 12:57:40 -0500 Subject: [PATCH 17/25] fix(core): claim every ISO opening, not only the ones that parse `_ISO_CALENDAR_HEAD` required digits immediately after the year's hyphen, so a token that opened `YYYY-` and then went wrong matched it not at all and fell through to the flexible reader -- the one outcome ISO syntax must never have. That reader does not report failure; it re-guesses: 2026--01 -> [2026-01-01,2026-02-01) January, invented 2026- -> [2026-01-01,2027-01-01) the whole year, invented 2026--01-02 -> [2026-01-02,) a specific day, invented 2026-x01 -> [2026-10-01,) October 1st, found nowhere in the text Every reindex reproduced these from the same markdown, so a slipped keystroke became a confident valid-time assertion nobody wrote. This is the recurring shape the classifier's own comment describes: a test whose failure means "not my business", leaving the next reviewer to find another token that fails it. The head still reads components, but the *opening* is now claimed separately -- a four-digit year followed by a hyphen is a commitment to machine syntax, and nothing else is spelled that way -- so such a token is judged as ISO or refused, never handed on because the rest of it was too broken to parse. That makes the classifier total in the way it already claimed to be. A bare `2026` carries no hyphen and is untouched, as are `2026/03/04`, `June 10, 2026` and every other spelling the guard test pins. `2026-W03` and `2026-2027` are now refused rather than guessed at, which is the honest answer for syntax this module does not implement. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 18 ++++++++++++++++++ tests/test_temporal.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 6ad8cdd00..db33130eb 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -642,6 +642,16 @@ def _calendar_span(lower: date, upper: date | None) -> TemporalRange: # reader, which is the one outcome ISO syntax must never have. _ISO_CALENDAR_HEAD = re.compile(r"^(\d{4})-(\d+)(?:-(\d+))?") +# What it takes to be *reaching* for an ISO date, as opposed to naming one. A year and a +# hyphen is a commitment to machine syntax; nothing else is spelled that way. The head +# above still needs digits after that hyphen, so `2026--01`, `2026-` and `2026-x01` matched +# it not at all and fell to the flexible reader -- which invented January 2026, the whole of +# 2026, and *October 1st* respectively, none of which appears in the text. Claiming the +# opening separately is what makes the classifier total in the way it always claimed to be: +# a token that opens in ISO syntax is judged as ISO or refused, never handed on because the +# rest of it was too broken to parse. A bare `2026` carries no hyphen and is untouched. +_ISO_CALENDAR_OPENING = re.compile(r"^\d{4}-") + # A fractional-second run too wide for a canonical instant to carry. `_INSTANT_BOUND` caps the # fraction at six digits and *refuses* a longer one rather than truncating it, because dropping # digits would store a different instant than the author wrote -- but that refusal only ever @@ -735,6 +745,14 @@ def _classify_authored_point(point: str) -> _AuthoredPoint: """ head = _ISO_CALENDAR_HEAD.match(point) if head is None: + # Trigger: the token opens `YYYY-` but no calendar components could be read from it. + # Why: the author reached for a machine date and mistyped it. Handing that to the + # flexible reader is the one outcome ISO syntax must never have -- it does not + # report failure, it re-guesses, and a slipped keystroke becomes a confident date + # nobody wrote, reproduced identically by every reindex. + # Outcome: malformed, so the token stays observation content. + if _ISO_CALENDAR_OPENING.match(point): + return _MALFORMED_ISO return _FLEXIBLE_POINT year, month, day = head.groups() diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 49360e5c7..2e235cd81 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -559,6 +559,43 @@ def test_iso_shaped_points_with_malformed_calendar_runs_are_unread(written: str) assert parse_authored_point(written) is None +@pytest.mark.parametrize( + "written", + [ + # The reported shapes: a doubled or dangling separator right after the year, which + # left no digits for the head to read. Each reached the flexible reader and came + # back with a date the author never wrote -- January 2026, and the whole of 2026. + "2026--01", + "2026---01", + "2026-", + "2026--", + # The same slip with a day still attached, which invented a specific day. + "2026--01-02", + # A space where the month should be; dateparser filled the month itself. + "2026- 01", + # The worst of them: a stray letter made dateparser abandon the ISO reading and + # re-guess, answering with October 1st -- a month and a day found nowhere in the + # text, in a token whose first four characters are the year the author wrote. + "2026-x01", + # Spellings this module does not implement. Refusing them is the honest answer; + # guessing was not. + "2026-W03", + "2026-2027", + ], +) +def test_a_year_and_a_hyphen_that_names_no_date_is_unread(written: str): + """Opening in ISO syntax settles the question even when the rest is unreadable. + + The head needed digits after the first hyphen, so these matched it not at all and fell + through to the flexible reader -- the one outcome ISO syntax must never have, and the + same "not my business" gap every earlier cut of this guard had. A reader that re-guesses + does not report failure: it answers, confidently, with a date nobody wrote, and every + reindex reproduces it. A four-digit year followed by a hyphen is a commitment to machine + syntax, so it is judged as machine syntax or refused. + """ + assert parse_authored_point(written) is None + + @pytest.mark.parametrize( "written", [ From 83e6fe6fd73e64c238841dd0611671a36dcb1fad Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 13:23:59 -0500 Subject: [PATCH 18/25] fix(core): hold a numeric date to the order it was written in `@occurred:2026/13/01` with `date_order="YMD"` was indexed as `[2026-01-13,)`. Month 13 is impossible where YMD puts the month, so dateparser silently moved the 13 into the day slot and answered January 13 -- a date the configured order does not name and the author did not write, refiled identically by every reindex. A year-first numeric date is the one non-ISO shape whose meaning is fixed rather than guessed: the text plus `date_order` determine it exactly, leaving nothing to interpret. So a run the order cannot use where the author put it is a typo, not an invitation to try the other slot. This is `2026-13-01`'s disease in the one syntax the ISO classifier deliberately does not claim, and it is now held to the same standard: refused when the order names no date, and refused when the reader answers with a different date than the order names. Which token is malformed depends on the setting, which is why the check consults it rather than banning a shape. `2026/13/01` is refused under YMD and MDY and *read* under DMY, where it legitimately means day 13 of month 1; `2026/12/31` is the reverse. Both directions are pinned. Only the year-first shape is judged. With the year written last (`01/02/2026`, `10/07/2026`) dateparser abandons the configured order for its own fallback, so this module cannot say what the text should mean without reimplementing those heuristics -- and reimplementing them is how two readings drift apart. Those forms are left exactly as they were, across every order. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 47 ++++++++++++++++++++++++++++ tests/test_temporal.py | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index db33130eb..cf82c0698 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -823,6 +823,42 @@ def _read_iso_day( return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) +# A year-first, fully numeric date. It is the one non-ISO shape whose meaning is *fixed* +# rather than guessed: `date_order` says which of the two trailing runs is the month, so +# the text plus one setting determine the date exactly, and there is nothing left to +# interpret. dateparser does not treat it that way. Handed `2026/13/01` under YMD it finds +# month 13 impossible, silently moves the 13 into the day slot, and answers January 13 -- +# a date the configured order does not name and the author did not write, refiled +# identically by every reindex. That is the `2026-13-01` disease in the one syntax the ISO +# classifier deliberately does not claim. +# +# Only the year-first shape is judged here. With the year written last (`01/02/2026`) +# dateparser abandons the configured order for its own fallback, so this module cannot say +# what the text "should" mean without reimplementing those heuristics -- and reimplementing +# them is how the two readings drift apart. +_ORDERED_NUMERIC_DATE = re.compile(r"^(\d{4})/(\d{1,2})/(\d{1,2})$") + +# Whether the run after a leading year names the month. Only DMY puts the day there. +_MONTH_LEADS_AFTER_YEAR: dict[DateOrder, bool] = {"YMD": True, "MDY": True, "DMY": False} + + +def _ordered_numeric_date(point: str, date_order: DateOrder) -> date | None | Literal[False]: + """The date a year-first numeric token names under `date_order`. + + `False` means the token is not that shape and this rule has nothing to say about it; + `None` means it is, and names no date on the calendar. + """ + ordered = _ORDERED_NUMERIC_DATE.match(point) + if ordered is None: + return False + year, first, second = (int(run) for run in ordered.groups()) + month, day = (first, second) if _MONTH_LEADS_AFTER_YEAR[date_order] else (second, first) + try: + return date(year, month, day) + except ValueError: + return None + + def _read_flexible_point( point: str, date_order: DateOrder, relative_base: datetime ) -> TemporalRange | None: @@ -832,6 +868,17 @@ def _read_flexible_point( if moment is None: return None + # Trigger: a year-first numeric date, whose meaning `date_order` fixes exactly. + # Why: the reader is free to move a run it cannot use where the author put it, and + # `2026/13/01` under YMD comes back as January 13 rather than as the impossible month + # the author actually typed. Holding it to the order is the same rule the ISO + # classifier applies to `2026-13-01`, in the syntax that classifier does not claim. + # Outcome: refused when the order names no date, and when the reader answered with a + # different one than the order names -- never quietly re-ordered. + ordered = _ordered_numeric_date(point, date_order) + if ordered is not False and moment.date() != ordered: + return None + # dateparser fills components the author did not write from the reference instant, so # only the components `period` vouches for may be read off `moment`. Discarding the # rest is also what lets `June 2026` survive the stability check: the filled-in day diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 2e235cd81..cec8fc1db 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1345,3 +1345,63 @@ def test_a_real_value_beside_absent_ones_still_builds_a_filter(): assert temporal is not None assert temporal.at is not None assert temporal.at.value == "2026-07-28" + + +# --- A numeric date must mean what the configured order says --- + + +@pytest.mark.parametrize( + ("written", "date_order"), + [ + # The reported shape: month 13 is impossible where YMD puts the month, so + # dateparser moved the 13 into the day slot and answered January 13 -- a date the + # order does not name and the author did not write. + ("2026/13/01", "YMD"), + ("2026/13/01", "MDY"), + # The same slip in the other slot, and a day the month has no room for. + ("2026/00/05", "YMD"), + ("2026/02/30", "YMD"), + # Read under DMY the runs swap roles, so it is the *other* spellings that name + # nothing: `12/31` is day 12 of month 31. Which token is malformed depends on the + # setting, which is exactly why the check has to consult it. + ("2026/12/31", "DMY"), + ("2026/03/31", "DMY"), + ], +) +def test_a_numeric_date_the_configured_order_cannot_name_is_unread(written: str, date_order): + """A fully numeric date is machine syntax whose reading `date_order` fixes. + + There is nothing left to guess once the setting is known, so a run the order cannot + use where the author put it is a typo, not an invitation to try the other slot. The + lenient reader disagrees: it silently reassigns the components and answers with a real + date, which every reindex then reproduces. This is `2026-13-01`'s disease in the one + syntax the ISO classifier deliberately does not claim. + """ + assert parse_authored_point(written, date_order=date_order) is None + + +@pytest.mark.parametrize( + ("written", "date_order", "literal"), + [ + # The same tokens under an order that *can* name them still read, so the guard is + # reading the setting rather than banning a shape. + ("2026/13/01", "DMY", "[2026-01-13,)"), + ("2026/12/31", "YMD", "[2026-12-31,)"), + # The spellings the guard test pins, across every order. + ("2026/03/04", "YMD", "[2026-03-04,)"), + ("2026/03/04", "DMY", "[2026-04-03,)"), + ("2026/03/04", "MDY", "[2026-03-04,)"), + ("2026/1/5", "YMD", "[2026-01-05,)"), + # Year-last forms are left to the reader's own fallback, untouched: predicting it + # here would mean reimplementing heuristics that could then drift out of step. + ("10/07/2026", "YMD", "[2026-07-10,)"), + ("10/07/2026", "MDY", "[2026-10-07,)"), + ("01/02/2026", "YMD", "[2026-02-01,)"), + ], +) +def test_a_numeric_date_the_order_can_name_still_reads(written: str, date_order, literal: str): + """Refusing an impossible ordering must cost nothing that the ordering allows.""" + span = parse_authored_point(written, date_order=date_order) + + assert span is not None + assert str(span) == literal From 7c1d03b3aa852f9e1fbcdfeade9420ad4bf73421 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 13:50:24 -0500 Subject: [PATCH 19/25] fix(core): refuse a digit run no reader can be handed An observation whose qualifier carried more than ~4300 digits aborted the parse of the entire note. Python declines to convert a decimal string longer than `sys.get_int_max_str_digits()` into an int, dateparser converts the runs it finds without catching that, and the `ValueError` came straight back out of the reader and past the qualifier scan. That is the one failure mode worse than a wrong date. Everywhere else an unreadable qualifier costs its own token and nothing else -- the text stays as observation content and the line indexes as it always did. Here one bad token on one line silently cost every other observation on the page its indexing. Refused on the text before either reader sees it. Both of them reach dateparser -- `_read_iso_day` hands a trailing clock reading to the same parser the flexible path uses -- so `2026-06-10 <5000 digits>` crashed by the second route and a guard on the bare numeric form alone would have left it there. The cap is a run of 32 digits: far below the smallest limit the runtime allows (`str_digits_check_threshold` is 640, so no legal setting can bring the crash within reach) and far above the widest run a readable point can carry -- six digits of a fractional second, four of a year. So it separates "no date" from "real date" without ever being the rule that decides a readable token's fate. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 24 ++++++++++++++ tests/markdown/test_temporal_qualifier.py | 19 +++++++++++ tests/test_temporal.py | 40 +++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index cf82c0698..ca37e2200 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -943,6 +943,20 @@ def _read_flexible_point( # components its `period` does not vouch for, so `June 2026` and `2026` answer with one # range from two different datetimes. Comparing datetimes would refuse them. +# A digit run no calendar component could be, and the reason this module refuses to hand +# one to a reader at all. Python declines to convert a decimal string longer than +# `sys.get_int_max_str_digits()` -- 4300 by default, and never settable below 640 -- into an +# int, and dateparser converts the runs it finds without catching that. So a token carrying +# a long enough run raised `ValueError` straight out of the reader and aborted the parse of +# the *entire note*, taking every other observation on the page with it. That is worse than +# any wrong date: elsewhere an unreadable qualifier costs its own token and nothing else. +# +# The cap is far below the smallest limit the runtime allows and far above the widest run a +# point can legitimately carry -- six digits of a fractional second, four of a year -- so it +# separates "no date" from "real date" without ever being the rule that decides a readable +# token's fate. +_UNREADABLE_DIGIT_RUN = re.compile(r"\d{32,}") + # Two reference instants that disagree in every component -- year, month, day, weekday, # hour, minute, second -- so nothing filled in from "now" can coincide across them. _STABILITY_PROBE_BASES = ( @@ -1009,6 +1023,16 @@ def parse_authored_point( such a token as ordinary observation content. """ point = text.strip() + # Trigger: a run of digits too long for any calendar component, or for Python to + # convert at all. + # Why: the readers below call int() on the runs they find, so this is refused before + # either sees it -- both of them reach dateparser, and neither can be trusted with a + # token that makes it raise. + # Outcome: no date, the same answer as any other unreadable point, and the rest of the + # note goes on indexing. + if _UNREADABLE_DIGIT_RUN.search(point): + return None + early, late = _STABILITY_PROBE_BASES reading = _read_authored_point(point, date_order, early) if reading is None: diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index 541def245..bf138082a 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -934,3 +934,22 @@ def test_scanner_takes_an_explicit_date_order(): [assertion] = result.assertions assert assertion.valid_during.lower == "2026-10-07" assert result.content == "The cutover ran." + + +def test_an_oversized_numeric_qualifier_costs_only_its_own_token(): + """The rest of the note must survive a qualifier no reader can be handed. + + A digit run past Python's integer-conversion limit made dateparser raise, and the + exception escaped the qualifier scan to abort the entire note -- so one unreadable + token on one line silently cost every other observation on the page its indexing. + """ + oversized = "9" * 5000 + observation, second = parse( + f"- [note] @{oversized} the statement\n- [note] a second observation\n" + ).observations + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == f"@{oversized} the statement" + # The line that had nothing to do with it is indexed exactly as it always was. + assert second.content == "a second observation" diff --git a/tests/test_temporal.py b/tests/test_temporal.py index cec8fc1db..4cb1b5777 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1405,3 +1405,43 @@ def test_a_numeric_date_the_order_can_name_still_reads(written: str, date_order, assert span is not None assert str(span) == literal + + +# --- A token no reader can be handed --- + +# Comfortably past Python's default integer-conversion limit of 4300 digits. +_OVERSIZED_RUN = "9" * 5000 + + +@pytest.mark.parametrize( + "written", + [ + # The reported shape: a bare numeric token long enough that converting it to an + # int is itself an error. + _OVERSIZED_RUN, + # The same run behind an ISO date, which reaches the flexible reader by the other + # route -- `_read_iso_day` hands a trailing clock reading to exactly the same + # parser, so guarding only the bare form would have left this one crashing. + f"2026-06-10 {_OVERSIZED_RUN}", + f"2026-06-10T{_OVERSIZED_RUN}", + # Runs far shorter than Python's limit but still no calendar component. + "9" * 32, + "1" * 64, + ], +) +def test_a_digit_run_no_component_could_be_is_unread(written: str): + """An unreadable qualifier must cost its own token and nothing else. + + dateparser converts the digit runs it finds without catching Python's refusal to + convert one longer than `sys.get_int_max_str_digits()`, so `ValueError` came straight + back out of the reader. Every other unreadable point returns None and leaves the token + as content; this one aborted the parse of the whole note, which is the one outcome + worse than a wrong date -- it costs every other observation on the page. + """ + assert parse_authored_point(written) is None + + +def test_a_digit_run_just_inside_the_limit_still_reads_as_no_date(): + """The guard reports "not a date", never an error, on either side of the boundary.""" + assert parse_authored_point("9" * 4299) is None + assert parse_authored_point("9" * 4301) is None From f5880ac46985a7b02b9dc65db10dea8b83f4a424 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 14:16:37 -0500 Subject: [PATCH 20/25] fix(core): put the duplicate ordinal where lookup preserves it 6c6fa261 moved the duplicate ordinal into a `~`-marked trailing segment so an author's own text could not reach it. It cannot -- and that was only half the requirement. `ContextService.build_context` re-normalizes every memory:// URL through `generate_permalink` before resolving it, and normalization replaces `~` and strips the hyphen it leaves. So `.../redis/~1` was advertised and looked up as `.../redis/1`: the second twin could not be opened through the address a temporal result had just handed back, and on a note that also carries an observation whose content slugs to `redis/1`, the lookup resolved to that different observation instead. The two requirements are not independent, and nothing appended after the content can satisfy both. Content may itself contain `/`, so any trailing segment is a position content can also occupy; the only characters that escape that are exactly the ones normalization strips. A marker that survives content is erased by lookup, and one that survives lookup is reachable by content. The leading segment has neither problem. It is generated here rather than authored, and every tail opens with it literally, so no observation can produce `observations-2` however its category or content is spelled -- including the two shapes that make later segments unsafe, a numeric category (`- [2] note/redis` yields `observations/2/note/redis`) and a category carrying a slash. And it is ordinary lowercase-and-hyphen text, so normalization returns it unchanged. First observations are untouched, as before: only the second and later twins carry the ordinal, now as `.../observations-1/note/redis`. The test asserts the addresses are distinct *and* that each survives `generate_permalink` unchanged, which is the half that was missing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/models/knowledge.py | 50 +++++++++++-------- .../services/test_search_service_temporal.py | 8 +++ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 9e8b63c57..65d380269 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -347,14 +347,24 @@ def __repr__(self) -> str: # pragma: no cover # each believe they were the first of their kind. OBSERVATION_DEFAULT_CATEGORY = "note" -# Opens the permalink segment carrying a duplicate ordinal. `generate_permalink` replaces -# every character outside `[a-z0-9/-.]` (plus CJK) with a hyphen, so a tail generated from -# an author's own text can never contain `~` -- which is exactly what makes this a -# namespace the ordinal can own rather than one it shares with real content. -OBSERVATION_DUPLICATE_MARK = "~" - - -def observation_permalink_tail(category: str | None, content: str) -> str: +# The segment every observation tail opens with, and the one place a duplicate ordinal can +# safely live. Two requirements pull against each other: the ordinal must sit where an +# author's own text can never land, and it must survive `generate_permalink`, because +# lookup re-normalizes a memory:// URL before resolving it. +# +# Nothing appended *after* the content can do both. Content may itself contain `/`, so any +# trailing segment is a position content can also occupy, and the only characters that +# would escape that are exactly the ones normalization strips -- a marker that survives +# content is erased by lookup, and one that survives lookup is reachable by content. +# +# The leading segment has neither problem. It is generated here rather than authored, and +# every tail opens with it literally, so no observation can produce `observations-2` however +# its category or content is spelled; and it is ordinary lowercase-and-hyphen text, so +# normalization returns it unchanged. +OBSERVATION_SEGMENT = "observations" + + +def observation_permalink_tail(category: str | None, content: str, duplicate_index: int = 0) -> str: """The part of an observation's permalink that distinguishes it within its note. This is the single definition of what makes two observations of one note share an @@ -398,7 +408,10 @@ def observation_permalink_tail(category: str | None, content: str) -> str: content_for_permalink = f"{content[:200]}-{digest}" else: content_for_permalink = content - return generate_permalink(f"observations/{category}/{content_for_permalink}") + segment = ( + OBSERVATION_SEGMENT if not duplicate_index else f"{OBSERVATION_SEGMENT}-{duplicate_index}" + ) + return generate_permalink(f"{segment}/{category}/{content_for_permalink}") class Observation(Base): @@ -445,19 +458,16 @@ def permalink(self) -> str: overwhelming majority of permalinks are byte-identical to what they have always been; only the second and later twins gain a trailing ordinal. - That ordinal lives in a segment natural content cannot reach. `generate_permalink` - emits only `[a-z0-9/-.]` plus CJK, so no observation's own text can ever slug to a - segment beginning with `~` -- whereas a plain `/1` is a segment content *can* - produce, and an observation whose text slugged to `foo/1` would then collide with - the second twin of `foo` and cost one of them its search row. Reserving the mark - is what makes the ordinal a namespace rather than a guess about what authors write. + That ordinal rides on the leading `observations` segment rather than trailing the + content, because only that segment satisfies both things it has to. See + `OBSERVATION_SEGMENT`: a trailing marker is either reachable by content, which puts + the collision back, or erased by the normalization every memory:// lookup performs, + which makes the advertised address resolve to a different observation. """ - base = generate_permalink( - f"{self.entity.permalink}/{observation_permalink_tail(self.category, self.content)}" + return generate_permalink( + f"{self.entity.permalink}/" + f"{observation_permalink_tail(self.category, self.content, self.duplicate_index)}" ) - if not self.duplicate_index: - return base - return f"{base}/{OBSERVATION_DUPLICATE_MARK}{self.duplicate_index}" @override def __repr__(self) -> str: # pragma: no cover diff --git a/tests/services/test_search_service_temporal.py b/tests/services/test_search_service_temporal.py index 5fe2b9d15..ef78b9fb1 100644 --- a/tests/services/test_search_service_temporal.py +++ b/tests/services/test_search_service_temporal.py @@ -19,6 +19,7 @@ describe_search_criteria, ) from basic_memory.temporal import TemporalQualifierError, TimeKind +from basic_memory.utils import generate_permalink # The entity is created "now"; the qualifier claims June-July 2026. Keeping the two # ranges disjoint is what makes acceptance case 11 testable at all. @@ -364,6 +365,13 @@ async def test_a_duplicate_ordinal_cannot_be_taken_by_an_authors_own_text( first, second, natural = entity.observations assert len({first.permalink, second.permalink, natural.permalink}) == 3 + # The address a result advertises has to survive being looked up. `build_context` + # re-normalizes every memory:// URL through `generate_permalink`, so a marker that + # normalization rewrites would send the reader to a different observation -- and with + # a natural `redis/1` on the page, to that one specifically. + for observation in (first, second, natural): + assert generate_permalink(observation.permalink) == observation.permalink + # Each of the three windows answers with its own observation, none lost. for window, expected in ( (EFFECTIVE_WINDOW_INSIDE, first), From 37ef47f2676a505335c0c3d8c43caf7e341d1a13 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 14:42:39 -0500 Subject: [PATCH 21/25] fix(core): hold year-last numeric dates to the order too 83e6fe6f validated only the year-first shape, on the reasoning that a year-last token has dateparser abandoning the configured order for its own fallback and that predicting it would mean reimplementing heuristics. That reasoning was too broad, and the scope-out hid a real bug: `MDY` and `DMY` both describe arrangements that *end* in the year, so for those the order applies as literally as it does with the year first. Only `YMD` cannot describe one. So `@occurred:13/01/2026` under MDY put 13 in the month slot, dateparser moved it to the day, and the qualifier was indexed as January 13 -- the same silent swap the year-first case already refused, in the shape that had been declared out of scope. All six combinations are now named explicitly and the reading is compared against the one the order states. Five are the order read literally. The sixth is not: `YMD` ends in no year, so the reader falls back to day-first, and that fallback is this module's behaviour too -- it was already pinned by the `10/07/2026` cases, so writing it down makes those the same rule rather than an exception to it. If dateparser ever changes that fallback the comparison fails and those tests go red, which is the failure mode to want. Newly refused, each an impossible slot rather than a shape: `13/01/2026` and `31/12/2026` under MDY, `01/13/2026` and `12/31/2026` under YMD, `01/13/2026` under DMY. Everything the guard test pins still reads, in every order. A two-digit year is still left alone. `03/04/26` states no arrangement this rule can name -- which run is even the year is the reader's call -- so there is no stated reading to hold it to. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 58 ++++++++++++++++++++++++------------ tests/test_temporal.py | 23 ++++++++++++-- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index ca37e2200..a98866629 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -823,36 +823,56 @@ def _read_iso_day( return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) -# A year-first, fully numeric date. It is the one non-ISO shape whose meaning is *fixed* -# rather than guessed: `date_order` says which of the two trailing runs is the month, so -# the text plus one setting determine the date exactly, and there is nothing left to -# interpret. dateparser does not treat it that way. Handed `2026/13/01` under YMD it finds -# month 13 impossible, silently moves the 13 into the day slot, and answers January 13 -- -# a date the configured order does not name and the author did not write, refiled +# A fully numeric date, with the year written first or last. It is the one non-ISO shape +# whose meaning is *fixed* rather than guessed: `date_order` says which of the other two +# runs is the month, so the text plus one setting determine the date exactly and there is +# nothing left to interpret. dateparser does not treat it that way. Handed a run it cannot +# use where the author put it, it silently moves that run to the other slot and answers with +# a real date -- `2026/13/01` under YMD, or `13/01/2026` under MDY, both come back as +# January 13, a date the configured order does not name and the author did not write, refiled # identically by every reindex. That is the `2026-13-01` disease in the one syntax the ISO # classifier deliberately does not claim. -# -# Only the year-first shape is judged here. With the year written last (`01/02/2026`) -# dateparser abandons the configured order for its own fallback, so this module cannot say -# what the text "should" mean without reimplementing those heuristics -- and reimplementing -# them is how the two readings drift apart. -_ORDERED_NUMERIC_DATE = re.compile(r"^(\d{4})/(\d{1,2})/(\d{1,2})$") - -# Whether the run after a leading year names the month. Only DMY puts the day there. -_MONTH_LEADS_AFTER_YEAR: dict[DateOrder, bool] = {"YMD": True, "MDY": True, "DMY": False} +_YEAR_FIRST_NUMERIC_DATE = re.compile(r"^(\d{4})/(\d{1,2})/(\d{1,2})$") +_YEAR_LAST_NUMERIC_DATE = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{4})$") + +# Whether the first of the two non-year runs names the month, given the configured order and +# where the year was written. Five of the six are the order read literally. The sixth is not: +# `YMD` describes no arrangement that ends in the year, so with the year last the reader +# falls back to day-first, and that fallback is this module's behaviour too -- pinned by the +# `10/07/2026` cases rather than left implicit. Naming all six is what lets the check compare +# against a reading it can state, instead of trusting whatever came back. +_MONTH_LEADS_THE_REMAINDER: dict[tuple[DateOrder, bool], bool] = { + ("YMD", True): True, + ("MDY", True): True, + ("DMY", True): False, + ("MDY", False): True, + ("DMY", False): False, + ("YMD", False): False, +} def _ordered_numeric_date(point: str, date_order: DateOrder) -> date | None | Literal[False]: - """The date a year-first numeric token names under `date_order`. + """The date a fully numeric token names under `date_order`. `False` means the token is not that shape and this rule has nothing to say about it; `None` means it is, and names no date on the calendar. + + A two-digit year is not this shape: `03/04/26` leaves which run is even the year to the + reader, so there is no stated reading to hold it to. """ - ordered = _ORDERED_NUMERIC_DATE.match(point) + year_first = _YEAR_FIRST_NUMERIC_DATE.match(point) + ordered = year_first or _YEAR_LAST_NUMERIC_DATE.match(point) if ordered is None: return False - year, first, second = (int(run) for run in ordered.groups()) - month, day = (first, second) if _MONTH_LEADS_AFTER_YEAR[date_order] else (second, first) + + runs = [int(run) for run in ordered.groups()] + year = runs.pop(0) if year_first else runs.pop() + first, second = runs + month, day = ( + (first, second) + if _MONTH_LEADS_THE_REMAINDER[(date_order, bool(year_first))] + else (second, first) + ) try: return date(year, month, day) except ValueError: diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 4cb1b5777..52b7ade99 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1366,6 +1366,17 @@ def test_a_real_value_beside_absent_ones_still_builds_a_filter(): # setting, which is exactly why the check has to consult it. ("2026/12/31", "DMY"), ("2026/03/31", "DMY"), + # The year written last, where the order applies just as literally. Under MDY the + # 13 sits in the month slot, and dateparser answered January 13 by moving it to the + # day -- the reported case, and the one this rule originally scoped itself out of. + ("13/01/2026", "MDY"), + ("31/12/2026", "MDY"), + # `YMD` describes no arrangement ending in the year, so the reader falls back to + # day-first and the same check applies to that reading: `01/13` is day 1, month 13. + ("01/13/2026", "YMD"), + ("12/31/2026", "YMD"), + # And under DMY, which is day-first by name rather than by fallback. + ("01/13/2026", "DMY"), ], ) def test_a_numeric_date_the_configured_order_cannot_name_is_unread(written: str, date_order): @@ -1392,11 +1403,19 @@ def test_a_numeric_date_the_configured_order_cannot_name_is_unread(written: str, ("2026/03/04", "DMY", "[2026-04-03,)"), ("2026/03/04", "MDY", "[2026-03-04,)"), ("2026/1/5", "YMD", "[2026-01-05,)"), - # Year-last forms are left to the reader's own fallback, untouched: predicting it - # here would mean reimplementing heuristics that could then drift out of step. + # Year-last forms under each order, including the two the guard test pins. `YMD` + # cannot describe them, so its day-first fallback is what they are held to -- named + # explicitly rather than trusted, which is what makes these the same rule and not + # an exception to it. ("10/07/2026", "YMD", "[2026-07-10,)"), ("10/07/2026", "MDY", "[2026-10-07,)"), + ("10/07/2026", "DMY", "[2026-07-10,)"), ("01/02/2026", "YMD", "[2026-02-01,)"), + ("13/01/2026", "DMY", "[2026-01-13,)"), + ("01/13/2026", "MDY", "[2026-01-13,)"), + # A two-digit year names no shape this rule can state -- which run is even the year + # is the reader's call -- so it is left alone. + ("03/04/26", "MDY", "[2026-03-04,)"), ], ) def test_a_numeric_date_the_order_can_name_still_reads(written: str, date_order, literal: str): From d2f71cedc93c28b1cdc94bc802309212c35d7f3c Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 15:07:35 -0500 Subject: [PATCH 22/25] fix(core): guard every reading, not only the public entry 7c1d03b3 refused an oversized digit run in `parse_authored_point`, which is one of two ways into the reader. `names_only_a_calendar_period` -- the helper that decides whether a refused word-led token was the truncated head of a date worth quoting -- calls `_read_authored_point` by its own route, so it went on handing dateparser a run Python will not convert. A word-led token is what finds it, and the shape is worth naming: the point itself is refused perfectly safely, and then the truncation diagnostic asks the same question again and raises. `@occurred:x<5000 digits> 2026 statement` aborted the note from the *diagnostic*, so the token stayed content as intended and the page was lost anyway. Moved into `_read_authored_point`, which is the one function every reading passes through. Guarding the public entry was guarding a caller; guarding here is guarding the thing that can actually raise, and there is no third route to miss next time. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 22 ++++++++++++---------- tests/markdown/test_temporal_qualifier.py | 18 ++++++++++++++++++ tests/test_temporal.py | 16 ++++++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index a98866629..5df029302 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -989,6 +989,18 @@ def _read_authored_point( point: str, date_order: DateOrder, relative_base: datetime ) -> TemporalRange | None: """Read one already-stripped point, treating `relative_base` as the present.""" + # Trigger: a run of digits too long for any calendar component, or for Python to + # convert at all. + # Why: every path below reaches dateparser, which calls int() on the runs it finds and + # does not catch the runtime's refusal to convert one. Guarding here rather than in + # `parse_authored_point` is the point: this is the one function every reading passes + # through, and `names_only_a_calendar_period` reaches it by its own route, so a guard + # on the public entry alone left the diagnostic path still raising. + # Outcome: no date, the same answer as any other unreadable point, and the rest of the + # note goes on indexing. + if _UNREADABLE_DIGIT_RUN.search(point): + return None + match _classify_authored_point(point): case _IsoDay() as iso: return _read_iso_day(iso, point, date_order, relative_base) @@ -1043,16 +1055,6 @@ def parse_authored_point( such a token as ordinary observation content. """ point = text.strip() - # Trigger: a run of digits too long for any calendar component, or for Python to - # convert at all. - # Why: the readers below call int() on the runs they find, so this is refused before - # either sees it -- both of them reach dateparser, and neither can be trusted with a - # token that makes it raise. - # Outcome: no date, the same answer as any other unreadable point, and the rest of the - # note goes on indexing. - if _UNREADABLE_DIGIT_RUN.search(point): - return None - early, late = _STABILITY_PROBE_BASES reading = _read_authored_point(point, date_order, early) if reading is None: diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index bf138082a..be04c53b0 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -953,3 +953,21 @@ def test_an_oversized_numeric_qualifier_costs_only_its_own_token(): assert observation.content == f"@{oversized} the statement" # The line that had nothing to do with it is indexed exactly as it always was. assert second.content == "a second observation" + + +def test_an_oversized_run_behind_a_word_costs_only_its_own_token(): + """The truncation diagnostic must not crash the note the qualifier could not. + + A word-led token is refused by the reader safely, and then the "did you mean to quote + this?" check asks the same question by another route. While that route was unguarded, + the note aborted from the diagnostic rather than from the parse -- so the token stayed + content and the page was lost anyway. + """ + oversized = "x" + "9" * 5000 + observation, second = parse( + f"- [note] @occurred:{oversized} 2026 statement\n- [note] a second observation\n" + ).observations + + assert observation.temporal == [] + assert observation.content == f"@occurred:{oversized} 2026 statement" + assert second.content == "a second observation" diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 52b7ade99..477d973a0 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -25,6 +25,7 @@ canonical_bound, parse_authored_point, parse_point, + names_only_a_calendar_period, parse_range_literal, parse_temporal_filter, ) @@ -1464,3 +1465,18 @@ def test_a_digit_run_just_inside_the_limit_still_reads_as_no_date(): """The guard reports "not a date", never an error, on either side of the boundary.""" assert parse_authored_point("9" * 4299) is None assert parse_authored_point("9" * 4301) is None + + +def test_the_diagnostic_reader_is_guarded_too(): + """The oversized-run guard has to sit where *every* reading passes through. + + `names_only_a_calendar_period` exists to explain a refusal, and it reaches the reader by + its own route rather than through `parse_authored_point`. A guard on the public entry + alone therefore left this path raising, and a word-led token is what finds it: the point + itself is refused safely, and then the truncation diagnostic asks the same question again + and crashes the note. + """ + word_led = "x" + "9" * 5000 + + assert parse_authored_point(word_led) is None + assert names_only_a_calendar_period(word_led) is False From 582d7afe8a61e346a4db7941c099a9d11f0f97c9 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 15:38:31 -0500 Subject: [PATCH 23/25] fix(core): describe the numeric separator instead of listing it 83e6fe6f and 37ef47f2 validated fully numeric dates against `date_order`, but wrote the shape as a literal `/`. Every other separator dateparser accepts went straight past the guard: with `date_order="YMD"`, `2026.13.01`, `2026 13 01`, `2026_13_01` and `2026\13\01` all reached the reader, which moved the impossible 13 out of the month slot and stored January 13 -- the exact projection the guard had just been written to refuse, in the spellings it did not enumerate. Year-last forms had the same hole: `13.01.2026` and `13-01-2026` under MDY. Enumerating punctuation is what produced a half-applied rule, so the pattern now describes the shape instead: `(\D)` matched twice by backreference. What makes a token fully numeric is that its runs are digits and whatever stands between them is not -- and a separator that has to be the same on both sides is the whole of it. Dots, spaces, underscores and backslashes are covered because they were never the point; the ISO `-` forms still never arrive here, since a `YYYY-` opening is claimed by the classifier first. Time-shaped tokens are unaffected: the patterns anchor on the whole token with a four-digit run at one end, so `10:00:00`, `2026-06-10T14:00:00` and `2026-06-10 14:00:00.5` are outside them, and `12:30:1990` reads as no date now exactly as it did before. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 14 +++++++++++--- tests/test_temporal.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 5df029302..9b5469b12 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -832,8 +832,13 @@ def _read_iso_day( # January 13, a date the configured order does not name and the author did not write, refiled # identically by every reindex. That is the `2026-13-01` disease in the one syntax the ISO # classifier deliberately does not claim. -_YEAR_FIRST_NUMERIC_DATE = re.compile(r"^(\d{4})/(\d{1,2})/(\d{1,2})$") -_YEAR_LAST_NUMERIC_DATE = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{4})$") +# The separator is `(\D)` matched twice rather than a list of the punctuation people +# use, because enumerating it is how this rule got half-applied the first time: written +# for `/` alone it left `2026.13.01`, `2026 13 01` and `2026_13_01` reaching the reader +# and coming back as January 13. What makes a token fully numeric is that its runs are +# digits and whatever stands between them does not, so that is what the pattern says. +_YEAR_FIRST_NUMERIC_DATE = re.compile(r"^(\d{4})(\D)(\d{1,2})\2(\d{1,2})$") +_YEAR_LAST_NUMERIC_DATE = re.compile(r"^(\d{1,2})(\D)(\d{1,2})\2(\d{4})$") # Whether the first of the two non-year runs names the month, given the configured order and # where the year was written. Five of the six are the order read literally. The sixth is not: @@ -865,7 +870,10 @@ def _ordered_numeric_date(point: str, date_order: DateOrder) -> date | None | Li if ordered is None: return False - runs = [int(run) for run in ordered.groups()] + # groups() is (run, separator, run, run); the separator is captured only so the + # backreference can require the same one twice. + year_run, _separator, second_run, third_run = ordered.groups() + runs = [int(year_run), int(second_run), int(third_run)] year = runs.pop(0) if year_first else runs.pop() first, second = runs month, day = ( diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 477d973a0..ef6c23902 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1378,6 +1378,16 @@ def test_a_real_value_beside_absent_ones_still_builds_a_filter(): ("12/31/2026", "YMD"), # And under DMY, which is day-first by name rather than by fallback. ("01/13/2026", "DMY"), + # Every separator a fully numeric date is written with, not just the slash this + # rule was first written for. Each of these reached the reader and came back as + # January 13 while the check enumerated punctuation instead of describing it. + ("2026.13.01", "YMD"), + ("2026 13 01", "YMD"), + ("2026_13_01", "YMD"), + ("2026\\13\\01", "YMD"), + ("13.01.2026", "MDY"), + ("13-01-2026", "MDY"), + ("13 01 2026", "MDY"), ], ) def test_a_numeric_date_the_configured_order_cannot_name_is_unread(written: str, date_order): @@ -1417,6 +1427,12 @@ def test_a_numeric_date_the_configured_order_cannot_name_is_unread(written: str, # A two-digit year names no shape this rule can state -- which run is even the year # is the reader's call -- so it is left alone. ("03/04/26", "MDY", "[2026-03-04,)"), + # The same separators carrying a date the order *can* name still read, so widening + # the rule cost none of them. + ("2026.03.04", "YMD", "[2026-03-04,)"), + ("2026 03 04", "YMD", "[2026-03-04,)"), + ("2026_03_04", "YMD", "[2026-03-04,)"), + ("03.04.2026", "MDY", "[2026-03-04,)"), ], ) def test_a_numeric_date_the_order_can_name_still_reads(written: str, date_order, literal: str): From 5a959902c70d50250d7a26481f10d232a5af0087 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 16:58:59 -0500 Subject: [PATCH 24/25] fix(core): judge the ISO time portion by its language, like the calendar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `@occurred:2026-01-01T10:00:00.` was stored as `[2026-01-01T10:00:00.000000Z,)`. dateparser discarded the dangling separator and answered with an instant the token never named, refiled identically by every reindex. The defect matters less than its shape. The calendar half of an ISO point has always been judged strictly -- an ISO head either names a real date or the whole point is `_MalformedIso`, with no path to the lenient reader. The time half never got that treatment. It was fenced by one returned-value check plus a text rule per defect as each was found: one for over-long fractions, one for wrong-width calendar runs. A dangling separator was simply the next defect no rule happened to name, and the third rule would not have been the last. So the time portion is now asked the same question the calendar is: which language is it written in? **Words mean a human spelling** -- `10:00 AM`, `2pm`, `noon`, `at 14:00`, `14:00:00 UTC` -- which has no literal reading for a guess to contradict, so the lenient reader keeps it, as the two-language contract requires. **Digits and punctuation alone mean machine syntax**, which must match one ISO-time grammar and then parse as a real time, or the whole point is `_MalformedIso`. `T` and `Z` are the two exceptions: ISO's own markers, not words. Testing the shape alone would not have worked, and that is worth recording: a rule keyed on a `T`-or-space followed by a digit takes `10:00 AM`, `2pm` and `14:00:00 UTC` with it, all three pinned by the guard test. What actually separates the languages is that every malformed spelling which reached the reader carried no letters at all. Two checks became redundant and are deleted rather than kept alongside: * `_OVER_PRECISE_FRACTION`, the `\.\d{7,}` text rule -- the grammar's `\.\d{1,6}` refuses a seven-digit fraction for the same reason it refuses none at all. * the `_INSTANT_BOUND` branch in `_read_iso_day`, which read only the one canonical RFC 3339 shape -- strict parsing now covers every machine spelling, including the second-less and `±HHMM` forms that branch never claimed. `_INSTANT_BOUND` itself stays: it is the grammar for range-literal bounds, a different surface with a stricter contract. The calendar width rule and the `OverflowError` handling stay untouched. Swept 336 ISO-time spellings under two clocks eight months apart, against an oracle written out in the probe rather than imported: 78 disagreed with a literal reading before, 0 after. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 94 +++++++++++++------ tests/markdown/test_temporal_qualifier.py | 29 ++++++ tests/test_temporal.py | 105 ++++++++++++++++++++++ 3 files changed, 201 insertions(+), 27 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 9b5469b12..62fdb6221 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -652,17 +652,40 @@ def _calendar_span(lower: date, upper: date | None) -> TemporalRange: # rest of it was too broken to parse. A bare `2026` carries no hyphen and is untouched. _ISO_CALENDAR_OPENING = re.compile(r"^\d{4}-") -# A fractional-second run too wide for a canonical instant to carry. `_INSTANT_BOUND` caps the -# fraction at six digits and *refuses* a longer one rather than truncating it, because dropping -# digits would store a different instant than the author wrote -- but that refusal only ever -# governed the strict path. The flexible reader has no such scruple: it truncates -# `2026-01-01T10:00:00.1234567` to `...123456Z` and reports a time on the right day, so every -# check `_read_iso_day` makes passes and the authored instant is quietly rewritten on each -# reindex. Judged on the text so both paths refuse the same token for the same reason, and it -# is the same reason the calendar width rule exists: a digit run wider than the syntax allows -# is a typo, not a shorthand. Six digits and fewer are untouched -- `14:00:00.5` is precision -# a canonical instant holds exactly, so it still reads. -_OVER_PRECISE_FRACTION = re.compile(r"\.\d{7,}") +# --- Which language the *time* portion is written in --- +# +# The calendar portion has always been judged strictly: an ISO head either names a real date +# or the whole point is `_MalformedIso`, with no path to the lenient reader. The time portion +# never got that treatment. It was fenced instead by one returned-value check plus a text +# check per defect discovered -- one for over-long fractions, one for wrong-width calendar +# runs -- and a dangling `.` was simply the next defect no existing check named. Growing that +# list is the shape this module has been refactored away from twice. +# +# So the same question is asked of the trailing text that is asked of the head: which +# language is it in? **Letters mean a human spelling** -- `10:00 AM`, `2pm`, `14:00:00 UTC`, +# `at 14:00`, `noon` -- which has no literal reading for a guess to contradict, so the +# lenient reader is trusted with it exactly as the two-language contract requires. **Digits +# and punctuation alone mean machine syntax**, and machine syntax is read literally or +# refused. `T` and `Z` are the two exceptions: they are ISO's own markers rather than words. +# +# That split is what a shape test on `[T ]digit` alone cannot do. `10:00 AM`, `2pm` and +# `14:00:00 UTC` all open that way and all must stay lenient; every malformed spelling that +# reached the reader carried no letters at all. +_HUMAN_TIME_LETTER = re.compile(r"[^\W\dTtZz_]") + +# The machine spellings of a time of day, as the trailing text after a complete ISO date. +# Seconds and the fraction are optional and the zone may be `Z`, `±HH:MM` or `±HHMM`, which +# is every ISO-time shape the pinned spellings use. A fraction must carry one to six digits, +# so this one grammar refuses both the over-long fraction and the dangling separator that +# needed a rule apiece before. +_ISO_TIME_TRAILING = re.compile( + r"^[Tt ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?(?:[Zz]|[+-]\d{2}:?\d{2})?$" +) + + +def _is_machine_time(trailing: str) -> bool: + """Whether trailing text after a date is written in machine syntax rather than words.""" + return _HUMAN_TIME_LETTER.search(trailing) is None def _named_calendar_date(year: str, month: str, day: str | None) -> date | None: @@ -776,7 +799,14 @@ def _classify_authored_point(point: str) -> _AuthoredPoint: # catch. A guard that asks what came back cannot see digits that never made it in. # Outcome: refused as malformed, so the strict and flexible paths give the same answer to # the same text and the token stays observation content rather than a rounded instant. - if _OVER_PRECISE_FRACTION.search(trailing): + # Trigger: text follows a complete date, written in machine syntax rather than words. + # Why: the head is already held to naming a real date; this holds the *time* to the same + # standard instead of leaving it to a returned-value check and a text rule per defect. + # A bare date has no trailing at all and is untouched; a worded clock is the other + # language and goes to the lenient reader, which is what the contract requires. + # Outcome: one grammar refuses every machine-syntax malformation -- the over-long + # fraction, the dangling separator, and the shapes nobody has written down yet. + if trailing and _is_machine_time(trailing) and not _ISO_TIME_TRAILING.match(trailing): return _MALFORMED_ISO return _IsoDay(named, trailing) @@ -790,20 +820,27 @@ def _read_iso_day( axis=TemporalRangeAxis.DATE, lower=iso.day.isoformat(), lower_inclusive=True ) - if _INSTANT_BOUND.match(point): - # Trigger: the whole token is canonical RFC 3339. - # Why: the author wrote the one form this module defines exactly, so it is read - # exactly -- to the microsecond, and refused rather than rounded when it names no - # moment (`2026-06-10T25:00:00+02:00`) or leaves the calendar in UTC. The flexible - # reader is neither that precise nor that strict. - # Outcome: an instant, or a refusal; never a guess. + if _is_machine_time(iso.trailing): + # Trigger: the time is written in machine syntax -- digits and punctuation, no words. + # Why: the classifier has already held its *shape* to `_ISO_TIME_TRAILING`, so what + # is left to establish is that those components name a real time. `fromisoformat` + # is the authority for that, exactly as `date` is for the calendar head: it knows + # hour 25 and minute 60 are not times, and it reads the shapes the grammar admits + # -- second-less, fractional, `Z`, `±HH:MM` and `±HHMM` alike. Upper-casing is safe + # because machine syntax carries no letters but ISO's own `t` and `z` markers. + # Outcome: an instant, or a refusal; never a guess, and never a rounded reading. try: - instant = _canonical_instant(point) - except TemporalQualifierError: + moment = datetime.fromisoformat(point.upper()) + except ValueError: + return None + instant = _instant_value(moment) + if instant is None: + # Shifting it to UTC carries it off the calendar, so it names no storable + # instant -- reported like any other unreadable point. return None return TemporalRange(axis=TemporalRangeAxis.INSTANT, lower=instant, lower_inclusive=True) - # The author wrote a clock reading in some spelling of their own, so the flexible reader + # The author wrote the clock in words, so the flexible reader # is asked for it -- but only for it. What it hands back must be a time of day on the # very day the head names, which is the check that keeps its guessing out of the answer: # dateparser silently drops a suffix it cannot use (`2026-01-01T`, `2026-01-01Z`, @@ -1046,11 +1083,14 @@ def parse_authored_point( Non-ISO spellings are read leniently, because guessing at `June 10, 2026` is the whole point of this reader. A token that *is* ISO-shaped is held to its own text - instead: its calendar components must name a real date, and anything trailing them - must be a time of day on that date, written to a precision this module can store. - `2026-06-10 10:00 AM` reads; `2026-01-01T` does not, because the author reached for an - instant and no instant is there; `2026-01-01T10:00:00.1234567` does not either, - because storing it would mean dropping the digits that made it worth writing. + instead, in both halves. Its calendar components must name a real date. And a time of + day after them is judged by the language it is written in: words are a human spelling + the lenient reader is trusted with (`10:00 AM`, `2pm`, `noon`, `14:00:00 UTC`), while + digits and punctuation alone are machine syntax, which must parse as a real ISO time or + the whole point goes unread. So `2026-06-10 10:00 AM` reads and `2026-01-01T` does not; + `2026-01-01T10:00:00.1234567` does not, because storing it would mean dropping the + digits that made it worth writing; and `2026-01-01T10:00:00.` does not either, because + a fractional separator with no fraction behind it names no moment at all. A reading that would depend on when it was taken is refused, however it is spelled: `yesterday`, `2 days ago`, `next month`, a bare `March` whose year would come from diff --git a/tests/markdown/test_temporal_qualifier.py b/tests/markdown/test_temporal_qualifier.py index be04c53b0..8425c976f 100644 --- a/tests/markdown/test_temporal_qualifier.py +++ b/tests/markdown/test_temporal_qualifier.py @@ -971,3 +971,32 @@ def test_an_oversized_run_behind_a_word_costs_only_its_own_token(): assert observation.temporal == [] assert observation.content == f"@occurred:{oversized} 2026 statement" assert second.content == "a second observation" + + +@pytest.mark.parametrize( + "point", + ["2026-01-01T10:00:00.", "2026-01-01T10:00:00.Z", "2026-01-01T14:00.."], +) +def test_a_dangling_fraction_stays_content_in_both_forms(point: str): + """A separator with no fraction behind it names no instant, quoted or not. + + dateparser discarded the dot and answered `...10:00:00.000000Z`, so the qualifier was + peeled and an instant the author never wrote was filed on every reindex. Quoting does + not help: it settles where a token ends, not whether the time inside it is one. + """ + for qualifier in (f"@occurred:{point}", f'@occurred:"{point}"'): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + assert observation.temporal == [] + assert observation.temporal_error is None + assert observation.content == f"{qualifier} The cutover ran." + + +def test_a_fraction_the_canonical_form_holds_is_still_filed(): + """The boundary the refusal stops at, end to end through the parser.""" + for qualifier in ("@occurred:2026-01-01T10:00:00.5", '@occurred:"2026-01-01T10:00:00.5"'): + observation = _observation(f"- [decision] {qualifier} The cutover ran.") + + [assertion] = observation.temporal + assert str(assertion.valid_during) == "[2026-01-01T10:00:00.500000Z,)" + assert observation.content == "The cutover ran." diff --git a/tests/test_temporal.py b/tests/test_temporal.py index ef6c23902..3019a2277 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1496,3 +1496,108 @@ def test_the_diagnostic_reader_is_guarded_too(): assert parse_authored_point(word_led) is None assert names_only_a_calendar_period(word_led) is False + + +# --- The time portion is a language, judged like the calendar --- + + +@pytest.mark.parametrize( + "written", + [ + # The reported shape: a fractional separator with no fraction behind it. dateparser + # discarded the dot and answered `...10:00:00.000000Z`, an instant the token never + # named -- the same defect class as the over-long fraction, in a spelling no text + # check named. + "2026-01-01T10:00:00.", + "2026-01-01T10:00:00.Z", + "2026-01-01T10:00:00.+02:00", + "2026-01-01T14:00.", + "2026-01-01T14:00..", + # Over-precision, which the general rule now covers on its own: the fraction must + # be one to six digits, so seven is refused for the same reason none is. + "2026-01-01T10:00:00.1234567", + "2026-01-01T10:00:00." + "1" * 30, + # Times that are shaped right and are not times. + "2026-01-01T25:00:00", + "2026-01-01T14:60", + "2026-01-01T14:00:60", + # Zone malformations, machine syntax throughout. + "2026-01-01T14:00:00+2:00", + "2026-01-01T14:00:00Z+01:00", + ], +) +def test_a_malformed_machine_time_is_unread(written: str): + """A time written in machine syntax is read literally or refused, like the calendar. + + The calendar head has always been held to naming a real date, with no path from a + malformed one to the lenient reader. The time portion was fenced instead by a + returned-value check plus a text rule per defect discovered -- one for the over-long + fraction, one for wrong-width calendar runs -- so a dangling separator was simply the + next defect no rule happened to name. Judged as a language instead: digits and + punctuation mean machine syntax, and machine syntax must parse. + """ + assert parse_authored_point(written) is None + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # One fractional digit is the boundary the refusal stops at, and six is the widest + # the canonical form holds. + ("2026-01-01T10:00:00.5", "2026-01-01T10:00:00.500000Z"), + ("2026-01-01T10:00:00.123456", "2026-01-01T10:00:00.123456Z"), + # Every machine spelling the guard test pins, read strictly rather than leniently. + ("2026-06-10T14:00", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 10:00", "2026-06-10T10:00:00.000000Z"), + ("2026-06-10 14:00:00+02:00", "2026-06-10T12:00:00.000000Z"), + ("2026-06-10T14:00Z", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10T14:00:00+0200", "2026-06-10T12:00:00.000000Z"), + ("2026-06-10t14:00:00z", "2026-06-10T14:00:00.000000Z"), + ], +) +def test_a_well_formed_machine_time_still_reads(written: str, lower: str): + """Strictness must cost nothing that names a real moment in machine syntax.""" + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + +@pytest.mark.parametrize( + ("written", "lower"), + [ + # Words are the other language, and they are what a shape test on `[T ]digit` alone + # cannot separate: each of these opens exactly like a machine time and none of them + # is one. They stay with the lenient reader, which is the two-language contract. + ("2026-06-10 10:00 AM", "2026-06-10T10:00:00.000000Z"), + ("2026-06-10 2pm", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 14:00:00 UTC", "2026-06-10T14:00:00.000000Z"), + ("2026-06-10 noon", "2026-06-10T12:00:00.000000Z"), + ("2026-06-10 at 14:00", "2026-06-10T14:00:00.000000Z"), + ], +) +def test_a_worded_clock_is_still_the_lenient_readers(written: str, lower: str): + """The split is machine-versus-words, not a grammar of what may follow a date.""" + span = parse_authored_point(written) + + assert span is not None + assert span.axis is INSTANT + assert span.lower == lower + + +@pytest.mark.parametrize( + "written", + ["9999-12-31 8pm EST", "9999-12-31 11pm PST", "9999-12-31 23:00 EST"], +) +def test_a_worded_clock_that_leaves_the_calendar_in_utc_is_unread(written: str): + """The lenient path has its own edge at the end of the calendar, and keeps it. + + Each of these is a real time on the last date there is, named in words with a zone + behind UTC -- so converting it forward carries it into year 10000, which `date` cannot + hold. The moment is genuine and the storable instant does not exist, so it reads as no + date rather than raising out of a note's parse. Pinned on a *worded* spelling because + the machine spellings that used to cover this edge are now read by the strict path, + which has its own guard for it. + """ + assert parse_authored_point(written) is None From 79deca739c59ca43ca5b211e6d8e0ab3237839d7 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 2 Sep 2026 18:18:18 -0500 Subject: [PATCH 25/25] fix(core): bound the offset minutes the parser silently normalizes `2026-01-01T10:00:00+14:60` was stored as `2025-12-31T19:00:00Z` -- the instant `+15:00` names, not the one the token does. `datetime.fromisoformat` bounds an offset's *total* magnitude below 24 hours but does not bound its minutes field, so `+14:60` is carried into the hour and read as `+15:00`, and `+14:99` as `+15:39`. Both spellings have it (`+1460` too) and both signs. This one sits inside the strict path rather than falling past it. The new grammar admits the offset shape and then delegates validity to the parser, which is the right division of labour for every other component -- and wrong for this one, because the parser normalizes it instead of refusing. The delegation is now a checked claim rather than an assumption, and the check narrows the fix rather than widening it. `fromisoformat` *does* reject `+25:00`, `+9999`, hour 25, minute 60 and second 60; minutes of the offset are the only field it normalizes. And once minutes are held below 60, its total-magnitude check is exactly RFC 3339's `00-23` on the offset hour -- so bounding one field restores the whole rule, and the fields that were already refused are pinned by tests so the claim stays checked. The finding named three surfaces -- `valid_at`, explicit range bounds, and authored qualifiers -- and they reach this through two grammars: a bound is held to the canonical RFC 3339 shape, an authored point to the wider set of machine spellings people write. The grammars stay distinct because their contracts differ, but the validity question underneath is one question, so both now go through one `_iso_instant`, and the offset rule is stated once. A test asserts all three surfaces refuse it, which is what would catch the rule being restated in only one. RFC 3339 bounds each field rather than the real-world offset range, so `+23:59` still reads along with `+14:00`, `-12:00`, `+00:00`, `Z` and `+0530`. Swept 1995 ISO-time spellings under two clocks, now including out-of-range offsets and out-of-range time components: 48 disagreed with a literal reading before, 0 after. The probe's oracle needed correcting too -- it computed offsets without bounding minutes, which would have made it agree with the bug -- and now carries a note that `fromisoformat` must never be used as the oracle, since it is blind to both this defect and the fraction truncation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014pmKq6bqCi6Zp6BTHuZjrp Signed-off-by: phernandez --- src/basic_memory/temporal.py | 48 ++++++++++++++++++----- tests/test_temporal.py | 76 ++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/src/basic_memory/temporal.py b/src/basic_memory/temporal.py index 62fdb6221..ec7dadee5 100644 --- a/src/basic_memory/temporal.py +++ b/src/basic_memory/temporal.py @@ -152,19 +152,48 @@ def _instant_value(moment: datetime) -> str | None: return utc.strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" +# The one component `datetime.fromisoformat` normalizes instead of refusing. It bounds an +# offset's *total* magnitude below 24 hours, so `+25:00` and `+9999` are rejected, but it +# does not bound the minutes field on its own: `+14:60` is carried into the hour and read as +# `+15:00`, and `+14:99` as `+15:39`. Both spellings have the hole -- `+1460` too -- and both +# signs. So the token names one instant and the stored bound names another, on every reindex. +# +# Minutes are the whole of it. Every other field is refused rather than normalized -- hour +# 25, minute 60 and second 60 of the time proper all raise -- and once minutes are held below +# 60 the total-magnitude check *is* RFC 3339's `00-23` on the hour, so the delegation that is +# right for the rest of the grammar stays right. This is the exception, not a lost trust. +_OVERFLOWING_OFFSET_MINUTES = re.compile(r"[+-]\d{2}:?[6-9]\d$") + + +def _iso_instant(text: str) -> datetime | None: + """Parse one machine-syntax ISO timestamp, or None when its text names no moment. + + The single place this module turns ISO text into a moment, shared by the range-literal + bounds and by authored points. They reach it through different grammars -- bounds are + held to the canonical RFC 3339 shape, an authored point to the wider set of machine + spellings people write -- but the validity question underneath is one question, and the + offset rule above only has to be stated once because of that. + + RFC 3339 allows lowercase `t`/`z`, which `fromisoformat` rejects. Machine syntax carries + no other letters, so upper-casing only touches those two markers. + """ + if _OVERFLOWING_OFFSET_MINUTES.search(text): + return None + try: + return datetime.fromisoformat(text.upper()) + except ValueError: + return None + + def _canonical_instant(bound: str) -> str: if not _INSTANT_BOUND.match(bound): raise TemporalQualifierError( f"timestamp bound must be RFC 3339 to microsecond precision, " f"with an optional offset or Z: {bound!r}" ) - # RFC 3339 allows lowercase `t`/`z`, which `datetime.fromisoformat` rejects. Every - # other character in a matched bound is a digit or punctuation, so upper-casing the - # whole bound only touches those two markers. - try: - moment = datetime.fromisoformat(bound.upper()) - except ValueError as exc: - raise TemporalQualifierError(f"not a valid timestamp: {bound!r}") from exc + moment = _iso_instant(bound) + if moment is None: + raise TemporalQualifierError(f"not a valid timestamp: {bound!r}") value = _instant_value(moment) if value is None: raise TemporalQualifierError( @@ -829,9 +858,8 @@ def _read_iso_day( # -- second-less, fractional, `Z`, `±HH:MM` and `±HHMM` alike. Upper-casing is safe # because machine syntax carries no letters but ISO's own `t` and `z` markers. # Outcome: an instant, or a refusal; never a guess, and never a rounded reading. - try: - moment = datetime.fromisoformat(point.upper()) - except ValueError: + moment = _iso_instant(point) + if moment is None: return None instant = _instant_value(moment) if instant is None: diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 3019a2277..dd5ed66e2 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1601,3 +1601,79 @@ def test_a_worded_clock_that_leaves_the_calendar_in_utc_is_unread(written: str): which has its own guard for it. """ assert parse_authored_point(written) is None + + +# --- The one component the parser normalizes instead of refusing --- + +_OVERFLOWING_OFFSETS = ["+14:60", "+14:99", "+1460", "-14:60", "-14:99"] + + +@pytest.mark.parametrize("offset", _OVERFLOWING_OFFSETS) +def test_an_offset_whose_minutes_overflow_is_unread(offset: str): + """`fromisoformat` carries an overflowing offset minute into the hour rather than refusing. + + `+14:60` comes back as `+15:00` and `+14:99` as `+15:39`, so the token names one instant + and the stored value names another -- reproduced on every reindex. Delegating validity to + the parser is right for every other field, which is exactly why this one needs saying: it + is the single component `fromisoformat` normalizes rather than rejects. + """ + assert parse_authored_point(f"2026-01-01T10:00:00{offset}") is None + + +@pytest.mark.parametrize("offset", _OVERFLOWING_OFFSETS) +def test_an_overflowing_offset_is_refused_on_every_surface(offset: str): + """One rule, three surfaces: an authored point, a `valid_at`, and a range bound. + + They reach it through two different grammars -- a bound is held to the canonical RFC 3339 + shape, an authored point to the wider set of machine spellings -- but the validity + question underneath is one question and is answered in one place. Asserting all three + here is what would catch the rule being restated in only one of them. + """ + bound = f"2026-01-01T10:00:00{offset}" + + assert parse_authored_point(bound) is None + with pytest.raises(TemporalQualifierError): + parse_point(bound) + with pytest.raises(TemporalQualifierError): + parse_range_literal(f"[{bound},2027-01-01T00:00:00Z)") + + +@pytest.mark.parametrize( + ("offset", "lower"), + [ + # The real maximum and minimum, the zero spellings, and a half-hour zone. + ("+14:00", "2025-12-31T20:00:00.000000Z"), + ("-12:00", "2026-01-01T22:00:00.000000Z"), + ("+00:00", "2026-01-01T10:00:00.000000Z"), + ("Z", "2026-01-01T10:00:00.000000Z"), + ("+0530", "2026-01-01T04:30:00.000000Z"), + # RFC 3339 bounds each field rather than the real-world range, so the widest legal + # offset still reads. Bounding minutes is what makes the parser's own total-magnitude + # check equal to the RFC's `00-23` on the hour. + ("+23:59", "2025-12-31T10:01:00.000000Z"), + ], +) +def test_a_legal_offset_still_reads(offset: str, lower: str): + """Bounding the minutes must cost no offset anyone can actually write.""" + span = parse_authored_point(f"2026-01-01T10:00:00{offset}") + + assert span is not None + assert span.lower == lower + + +@pytest.mark.parametrize( + "written", + [ + # Fields the parser *does* refuse, pinned so the delegation stays a checked claim + # rather than an assumption: an offset past 24 hours in either spelling, and every + # component of the time proper. + "2026-01-01T10:00:00+25:00", + "2026-01-01T10:00:00+9999", + "2026-01-01T25:00:00", + "2026-01-01T10:60:00", + "2026-01-01T10:00:60", + ], +) +def test_the_components_the_parser_refuses_stay_refused(written: str): + """What is delegated is delegated because it was checked, not because it was assumed.""" + assert parse_authored_point(written) is None