From aff07a0fac1fadc7e909ae5138ce9ce81c6c0cd6 Mon Sep 17 00:00:00 2001 From: phernandez Date: Wed, 12 Aug 2026 00:23:29 -0500 Subject: [PATCH] fix(core): default reranker document cap to 2000 chars Signed-off-by: phernandez --- docs/semantic-search.md | 11 +++++++---- src/basic_memory/config_models.py | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/semantic-search.md b/docs/semantic-search.md index e41b39057..414ea3876 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -393,7 +393,7 @@ All settings use the `BASIC_MEMORY_` environment prefix: | `reranker_provider` | `BASIC_MEMORY_RERANKER_PROVIDER` | `fastembed` | `fastembed` for a local ONNX cross-encoder or `litellm` for an API provider. | | `reranker_model` | `BASIC_MEMORY_RERANKER_MODEL` | `jinaai/jina-reranker-v1-tiny-en` | Model identifier. LiteLLM requires explicit `provider/model` routing. | | `reranker_candidates` | `BASIC_MEMORY_RERANKER_CANDIDATES` | `20` | Number of leading retrieval results rescored on every page. Larger values can improve recall but increase latency and provider usage. | -| `reranker_max_document_chars` | `BASIC_MEMORY_RERANKER_MAX_DOCUMENT_CHARS` | `0` | Maximum characters sent per candidate. `0` sends the full matched text; a positive cap bounds latency and request size. | +| `reranker_max_document_chars` | `BASIC_MEMORY_RERANKER_MAX_DOCUMENT_CHARS` | `2000` | Maximum characters sent per candidate. The default bounds worst-case latency on very long documents with no measured quality loss; `0` sends the full matched text. | | `reranker_timeout` | `BASIC_MEMORY_RERANKER_TIMEOUT` | `30.0` | Maximum seconds for each LiteLLM rerank request. FastEmbed runs locally and ignores this setting. | | `reranker_api_base` | `BASIC_MEMORY_RERANKER_API_BASE` | Unset | Optional custom endpoint for the LiteLLM provider. | | `reranker_api_key` | `BASIC_MEMORY_RERANKER_API_KEY` | Unset | Optional credential passed directly to LiteLLM. When unset, LiteLLM resolves provider credentials from its normal environment variables. | @@ -441,9 +441,12 @@ Start with the defaults, then tune only if measurements justify it: - Increase `reranker_candidates` when relevant results enter the retrieval set but remain outside the desired cutoff. This increases local inference time or hosted provider usage. -- Set `reranker_max_document_chars` to a positive value such as `1000` to - bound latency and hosted request size for long notes. The matched chunk comes - first, so a modest cap retains the strongest retrieval signal. +- To reduce rerank latency, lower `reranker_candidates` — per-query cost is + candidate-count-driven. `reranker_max_document_chars` (default `2000`) only + matters for very long documents: caps of 2000+ measured identical quality to + unbounded on a full LoCoMo sweep, while `1000` cost about 1.4 points of + recall@5. The matched chunk comes first, so the retained prefix carries the + strongest retrieval signal. Set `0` to disable the cap entirely. - Keep reranking disabled when retrieval latency matters more than the additional ranking pass. diff --git a/src/basic_memory/config_models.py b/src/basic_memory/config_models.py index 1c2d8b2ca..0ba872290 100644 --- a/src/basic_memory/config_models.py +++ b/src/basic_memory/config_models.py @@ -443,11 +443,14 @@ def __init__(self, **data: Any) -> None: ... "e.g. 'cohere/rerank-v3.5'.", ) reranker_max_document_chars: int = Field( - default=0, + default=2000, description="Max characters of each candidate's text passed to the cross-encoder. " - "0 (default) sends the full matched text — the model still truncates to its own token " - "limit. Set a positive cap (e.g. ~1000) to bound rerank latency on long notes; the " - "most-relevant matched chunk leads the text, so a modest cap keeps most of the signal.", + "The default of 2000 bounds worst-case rerank latency on very long documents with no " + "measured quality loss (issue #1234 LoCoMo sweep: caps >= 2000 score identically to " + "unbounded); the most-relevant matched chunk leads the text, so the retained prefix " + "carries the signal. 0 sends the full matched text — the model still truncates to its " + "own token limit. To reduce rerank latency generally, lower reranker_candidates; the " + "cap only matters for long documents.", ge=0, ) reranker_timeout: float = Field(