diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64ec4650..e0ffd763 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 4a848f092064e7e0320c716ff472c3f53446409e # v0.144.10 + ref: dc592024c3049a677b90f3035fcda883b6efd9f3 # v0.144.12 path: .cohort/agent-eval persist-credentials: false - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 62dd504f0509d9795011efc8addcfdb6971745b8 # v7.2.0 + ref: 5c5a9f58e35dc44992e7b8cd99911ef2c99cdd0b # v7.2.1 path: .cohort/agent-knowledge persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a231381c..9d68cdb6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -143,7 +143,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 4a848f092064e7e0320c716ff472c3f53446409e # v0.144.10 + ref: dc592024c3049a677b90f3035fcda883b6efd9f3 # v0.144.12 path: .cohort/agent-eval persist-credentials: false @@ -151,7 +151,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 62dd504f0509d9795011efc8addcfdb6971745b8 # v7.2.0 + ref: 5c5a9f58e35dc44992e7b8cd99911ef2c99cdd0b # v7.2.1 path: .cohort/agent-knowledge persist-credentials: false diff --git a/CHANGELOG.md b/CHANGELOG.md index d7549f69..e77420b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.132.1 + +- Add `candidatePopulation` to `improve(...)` results so consumers can inspect every verified optimizer candidate, including its exact profile, Interface diffs, parent lineage, and selection score. +- Consume Agent Eval 0.144.12 and Agent Knowledge 7.2.1 as one compatible dependency set. + ## 0.131.7 - Add `superviseDispatch(...)` so `agent-eval` profile matrices admit and record a recursive Runtime tree before it spends. diff --git a/README.md b/README.md index 4112c7dc..9225437a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ pnpm add @tangle-network/agent-runtime @tangle-network/agent-eval @tangle-networ - [Quickstart](#quickstart-offline-no-api-keys) - [What you do with it](#what-you-do-with-it) - [Run a chat turn](#run-a-chat-turn) +- [Retain and reconnect a run](#retain-and-reconnect-a-run) - [Supervise a team of agents](#supervise-a-team-of-agents) - [Improve an agent](#improve-an-agent) - [Improve a knowledge base](#improve-a-knowledge-base) @@ -115,6 +116,42 @@ return new Response(result.body, { headers: { 'content-type': result.contentType For a stream reconnect, call `streamPrompt` with the same `executionId` and the last event id the client received. For a repeated initial dispatch, reuse both `sessionId` and `turnId`; `executionId` alone is not an idempotency key. +### Retain and reconnect a run + +Use the retained-run API when the provider owns a job that must outlive one HTTP reader or application process. +The provider must advertise exact run identity, replay, result identity, and idempotent cancellation. + +```ts +import { reconnectRetainedRun, startRetainedRun } from '@tangle-network/agent-runtime' + +const run = await startRetainedRun({ + provider, + environment: { idempotencyKey: 'workspace-42', profile }, + turn: { turnId: 'turn-7', prompt: 'Finish the migration and run its tests.' }, + identity: { sessionId: 'thread-42', executionId: 'execution-7' }, +}) + +await journal.write(run.controlRef) + +for await (const event of run.events()) { + await journal.write(event) +} + +const recovered = await reconnectRetainedRun({ + provider: freshProvider, + controlRef: await journal.readControlRef(), +}) +if (!recovered) throw new Error('the provider no longer retains this environment') + +const snapshot = await recovered.status({ waitMs: 30_000 }) +const result = await recovered.result() +``` + +Persist `controlRef` before acknowledging dispatch to the caller. +Persist each event cursor and sequence before advancing the visible transcript. +`reconnectRetainedRun` reconstructs a client from those values and rejects any provider, environment, session, execution, run, or digest mismatch. +An unknown provider result remains unknown; the runtime never converts it into success or confirmed cancellation. + ### Supervise a team of agents One supervisor spawns and steers workers toward a goal. Where the workers run (an in-process loop, or a sandboxed coding harness) is one data value; the budget, journaling, and stopping are handled for you. @@ -238,11 +275,15 @@ With `resume: 'if-compatible'`, agent-eval resumes only when the saved run ident Set `trustResumeState: true` only when that run directory is private to the current operator. Use `resume: 'required'` to fail when no matching run exists. `result.provenance` reports the upstream package, run ID, resume status, evaluation count, and artifact directory. +`result.candidatePopulation` verifies and joins callback observations with an optimizer's official candidate graph. +It returns every unique candidate as a complete profile with ordered Interface diffs, or as an explicit materialization refusal. +GEPA candidates retain exact parent indices and selection scores; callback-only proposals report lineage as unavailable. +Methods without either artifact return `status: 'unavailable'` instead of treating the winner as the full population. There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install "agent-eval-rpc==0.144.8" +python -m pip install "agent-eval-rpc==0.144.12" python -m pip install "gepa[full]==0.1.4" ``` @@ -256,7 +297,7 @@ python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919 Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install "agent-eval-rpc==0.144.8" +python -m pip install "agent-eval-rpc==0.144.12" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` @@ -490,6 +531,7 @@ The general-purpose pieces, by import path. Every export with its one-line summa | Primitive | What it does | Import | |---|---|---| | Chat-turn runtime | Stream and persist one production chat turn (`handleChatTurn`); derive its stable execution and turn identity (`deriveExecutionId`); normalize any backend's stream into one event shape (`streamAgentTurn`) | `/durable` · `/kernel` | +| Retained provider runs | Start one detached provider job, replay exact events, reconnect after restart, continue its native context, and cancel idempotently (`startRetainedRun`, `reconnectRetainedRun`) | root | | Tool-call loop | Run one model turn, execute requested tools, feed results back, and stop on completion, repetition, time, or cost limits (`runToolLoop`, `streamToolLoop`) | `/tool-loop` | | Supervision | One agent spawns, budgets, and steers workers toward a goal (`supervise`, `delegate`), on an in-process loop or a sandboxed coding harness | `/kernel` · `/mcp` | | Loop kernel + combinators | Write a driver (`plan`/`decide`) and run it (`runAgentRounds`), or compose fixed shapes: refine (`loopUntil`), best-of-N (`fanout`), chain (`pipeline`), multi-judge (`panel`) | `/kernel` | diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index fb42280a..e6b1fe52 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.8.3 + +- Consume Runtime 0.132.1, Eval 0.144.12, and Knowledge 7.2.1 as one compatible dependency set. + ## 0.8.2 - Consume Runtime 0.131.7, Eval 0.144.10, and Knowledge 7.2.0. diff --git a/bench/package.json b/bench/package.json index af34a215..061b1070 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.8.2", + "version": "0.8.3", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index ed52e111..d67c4149 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -234,7 +234,7 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install `agent-eval-rpc==0.144.10`, then install ' + + 'install `agent-eval-rpc==0.144.12`, then install ' + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( diff --git a/docs/api/index.md b/docs/api/index.md index d7087ec9..38eb10e5 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -4884,6 +4884,262 @@ Exact complete profile instance measured on the final cases. *** +### ImprovementProfilePopulationArtifactSource + +Digest-addressed Eval artifact. + +#### Properties + +##### path + +> **path**: `string` + +##### sha256 + +> **sha256**: `` `sha256:${string}` `` + +*** + +### ImprovementProfilePopulationObservationSource + +Exact callback observation that introduced one optimizer candidate. + +#### Properties + +##### proposalSequence + +> **proposalSequence**: `number` + +One-based JSONL line sequence in the verified observation artifact. + +##### artifact + +> **artifact**: [`ImprovementProfilePopulationArtifactSource`](#improvementprofilepopulationartifactsource) + +*** + +### ImprovementProfilePopulationLineageNode + +One exact node from GEPA's accepted candidate graph. + +#### Properties + +##### index + +> **index**: `number` + +##### parentIndices + +> **parentIndices**: readonly (`number` \| `null`)[] + +##### aggregateScore + +> **aggregateScore**: `number` \| `null` + +##### selectionScores + +> **selectionScores**: readonly `object`[] + +##### discoveryEvaluationCount + +> **discoveryEvaluationCount**: `number` + +*** + +### ImprovementProfilePopulationCandidateSource + +Every verified source associated with one unique optimizer candidate. + +#### Properties + +##### candidateDigest + +> **candidateDigest**: `` `sha256:${string}` `` + +Eval identity of the external text or component candidate. + +##### observation? + +> `optional` **observation?**: [`ImprovementProfilePopulationObservationSource`](#improvementprofilepopulationobservationsource) + +Present when the candidate crossed the evaluation callback. + +##### lineage + +> **lineage**: [`ImprovementProfilePopulationLineage`](#improvementprofilepopulationlineage) + +Exact GEPA parents and scores, or an explicit statement that none were reported. + +*** + +### ImprovementMaterializedProfilePopulationCandidate + +A verified optimizer candidate that Runtime can express as an exact profile. + +#### Properties + +##### status + +> **status**: `"materialized"` + +##### source + +> **source**: [`ImprovementProfilePopulationCandidateSource`](#improvementprofilepopulationcandidatesource) + +##### value + +> **value**: `MutableSurface` + +Exact optimizer surface decoded by Eval. + +##### surfaceDigest + +> **surfaceDigest**: `` `sha256:${string}` `` + +Interface identity of `value`. + +##### profile + +> **profile**: `object` + +Exact complete profile produced by Runtime's configured materializer. + +##### profileDigest + +> **profileDigest**: `` `sha256:${string}` `` + +Interface identity of `profile`. + +##### diffs + +> **diffs**: readonly `AgentProfileDiff`[] + +Ordered Interface diffs that reproduce `profile` from the baseline. + +##### diffDigests + +> **diffDigests**: readonly `` `sha256:${string}` ``[] + +Interface identity of each entry in `diffs`. + +*** + +### ImprovementRefusedProfilePopulationCandidate + +A verified optimizer candidate that Runtime refused to materialize. + +#### Properties + +##### status + +> **status**: `"refused"` + +##### source + +> **source**: [`ImprovementProfilePopulationCandidateSource`](#improvementprofilepopulationcandidatesource) + +##### value + +> **value**: `MutableSurface` + +Exact optimizer surface decoded by Eval. + +##### surfaceDigest + +> **surfaceDigest**: `` `sha256:${string}` `` + +Interface identity of `value`. + +##### error + +> **error**: `object` + +###### name + +> **name**: `string` + +###### message + +> **message**: `string` + +*** + +### ImprovementProfileCandidatePopulationAvailable + +Complete verified population reported by one optimizer run. + +#### Properties + +##### status + +> **status**: `"available"` + +##### source + +> **source**: `object` + +###### observations? + +> `optional` **observations?**: [`ImprovementProfilePopulationArtifactSource`](#improvementprofilepopulationartifactsource) + +###### gepaCandidateGraph? + +> `optional` **gepaCandidateGraph?**: [`ImprovementProfilePopulationArtifactSource`](#improvementprofilepopulationartifactsource) & `object` + +###### Type Declaration + +###### bestIndex + +> **bestIndex**: `number` + +##### uniqueCandidates + +> **uniqueCandidates**: `number` + +Distinct candidate surfaces across all verified source artifacts. + +##### observedCandidates + +> **observedCandidates**: `number` + +Distinct candidate surfaces submitted through the evaluation callback. + +##### gepaCandidateNodes + +> **gepaCandidateNodes**: `number` + +Exact GEPA graph nodes. Multiple nodes can have the same candidate surface. + +##### materializedCandidates + +> **materializedCandidates**: `number` + +##### refusedCandidates + +> **refusedCandidates**: `number` + +##### candidates + +> **candidates**: readonly [`ImprovementProfilePopulationCandidate`](#improvementprofilepopulationcandidate)[] + +*** + +### ImprovementProfileCandidatePopulationUnavailable + +Explicit absence for methods that do not report candidate population evidence. + +#### Properties + +##### status + +> **status**: `"unavailable"` + +##### reason + +> **reason**: `"method-did-not-report-candidate-population"` + +*** + ### ImprovementCodeCandidate #### Properties @@ -5196,6 +5452,12 @@ Paired final-test confidence interval for method-based profile runs. `ImproveResultBase.liftInterval` +##### candidatePopulation + +> **candidatePopulation**: [`ImprovementProfileCandidatePopulation`](#improvementprofilecandidatepopulation) + +Every distinct verified candidate, including explicit materialization refusals. + ##### raw > **raw**: `OptimizationMethodComparison` @@ -10626,6 +10888,24 @@ The canonical improvement API: complete methods for profiles, worktrees for code *** +### ImprovementProfilePopulationLineage + +> **ImprovementProfilePopulationLineage** = \{ `status`: `"available"`; `artifact`: [`ImprovementProfilePopulationArtifactSource`](#improvementprofilepopulationartifactsource); `nodes`: readonly [`ImprovementProfilePopulationLineageNode`](#improvementprofilepopulationlineagenode)[]; \} \| \{ `status`: `"unavailable"`; `reason`: `"optimizer-did-not-report-candidate-lineage"`; \} + +*** + +### ImprovementProfilePopulationCandidate + +> **ImprovementProfilePopulationCandidate** = [`ImprovementMaterializedProfilePopulationCandidate`](#improvementmaterializedprofilepopulationcandidate) \| [`ImprovementRefusedProfilePopulationCandidate`](#improvementrefusedprofilepopulationcandidate) + +*** + +### ImprovementProfileCandidatePopulation + +> **ImprovementProfileCandidatePopulation** = [`ImprovementProfileCandidatePopulationAvailable`](#improvementprofilecandidatepopulationavailable) \| [`ImprovementProfileCandidatePopulationUnavailable`](#improvementprofilecandidatepopulationunavailable) + +*** + ### ImprovementCandidate > **ImprovementCandidate** = [`ImprovementProfileCandidate`](#improvementprofilecandidate) \| [`ImprovementCodeCandidate`](#improvementcodecandidate) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 82e87900..841ff1dd 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.131.7` and `@tangle-network/agent-eval@0.144.10` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.132.1` and `@tangle-network/agent-eval@0.144.12` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 407 exports. +Import from `@tangle-network/agent-runtime` — 418 exports. | Symbol | Kind | Summary | |---|---|---| @@ -194,6 +194,14 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `ImproveCandidateValidationInput` | interface | Exact materialized profile presented for validation before any candidate run. | | `ImproveCost` | interface | Normalized spend reported for one Runtime improvement run. | | `ImproveLineage` | interface | Optimizer ancestry sealed into downstream candidate experiments. | +| `ImprovementMaterializedProfilePopulationCandidate` | interface | A verified optimizer candidate that Runtime can express as an exact profile. | +| `ImprovementProfileCandidatePopulationAvailable` | interface | Complete verified population reported by one optimizer run. | +| `ImprovementProfileCandidatePopulationUnavailable` | interface | Explicit absence for methods that do not report candidate population evidence. | +| `ImprovementProfilePopulationArtifactSource` | interface | Digest-addressed Eval artifact. | +| `ImprovementProfilePopulationCandidateSource` | interface | Every verified source associated with one unique optimizer candidate. | +| `ImprovementProfilePopulationLineageNode` | interface | One exact node from GEPA's accepted candidate graph. | +| `ImprovementProfilePopulationObservationSource` | interface | Exact callback observation that introduced one optimizer candidate. | +| `ImprovementRefusedProfilePopulationCandidate` | interface | A verified optimizer candidate that Runtime refused to materialize. | | `ImproveMethodLineage` | interface | Method optimization always retains every identity needed to reject task reuse. | | `ImproveProfileComponents` | interface | Caller-owned mapping for optimizing several profile fields as one candidate. | | `ImproveScenarioPartitions` | interface | Redacted task evidence retained for every optimizer-visible partition. | @@ -254,7 +262,7 @@ Import from `@tangle-network/agent-runtime` — 407 exports. | `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImprovementProfileCandidatePopulation`, `ImprovementProfilePopulationCandidate`, `ImprovementProfilePopulationLineage`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -504,7 +512,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 718 exports. +Import from `@tangle-network/agent-runtime/kernel` — 731 exports. | Symbol | Kind | Summary | |---|---|---| @@ -647,6 +655,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `readWorkerProgress` | function | Fold the scope-derived facts and the executor's optional enrichment into one read. Pure: the | | `readWorkerSteerRequests` | function | Read every valid steer request in a worker's inbox. Corrupt or partial lines are skipped. | | `readWorkerTraceContext` | function | Read the inherited trace context off an `ExecutorContext`, or `undefined` when the run records no | +| `reconnectRetainedRun` | function | Rebuild a retained-run client without retaining any object from the starter. | | `registerShape` | function | Register a composed shape on the default `builtinShapes` registry — the one-call extension | | `registryScopeAnalyst` | function | A `ScopeAnalyst` backed by an `AnalystRegistry` — the panel-of-analysts seam. The registry merges | | `renderAnytimeTable` | function | One row per (strategy, satisficing target): the shareable time-to-satisfactory table. | @@ -688,6 +697,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `settledToIteration` | function | The step-8 merge-boundary adapter (M4): rehydrate a `Settled.done` into the kernel's | | `settledWorkerOut` | function | What a settled worker exposes as its output artifact (the blob the brain's | | `spendFromUsageEvents` | function | Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate | +| `startRetainedRun` | function | Dispatch one detached, replayable run and return only after exact durable | | `stopSentinel` | function | A unique, attributable stop sentinel for a node (ralph-loop style). Deterministic from the | | `streamAgentTurn` | function | Run ONE agent turn on any backend kind and stream its events. Yields the | | `structuralRollout` | function | Build the structuralRollout `Strategy`: k shots → score each by the frozen visible | @@ -899,6 +909,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `ProviderAsSandboxClientOptions` | interface | Options for exposing an `AgentEnvironmentProvider` through the legacy sandbox client port. | | `ProviderExecutorOptions` | interface | Options for running a provider as a supervise-mode executor. | | `ProviderSeam` | interface | Generic environment provider executor config. External packages implement | +| `ReconnectRetainedRunOptions` | interface | Inputs sufficient to rebuild a control client in a new process. | | `RegisteredPrompt` | interface | One registry entry: the handle plus the text it pins. | | `RegistryAnalyzeProjection` | interface | Project a `ScopeAnalyzeInput` into the `AnalystRegistry.run` arguments. The registry runs over a | | `RenderCorpusToInstructionsOptions` | interface | Project accreted corpus facts into an `AgentProfile`'s instruction seams — the learning-flywheel | @@ -908,6 +919,12 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `ResultBlobStore` | interface | Content-addressed result blobs (the `outRef` → artifact map) backing the replay | | `ResumedKeyState` | interface | What the journal proves about one keyed assignment at resume time. | | `ResumedWork` | interface | The committed work a resumed run inherits from its journal. `settled` is the replayed | +| `RetainedRunCancellation` | interface | Durable acknowledgement state for one retained control operation. | +| `RetainedRunCancelOptions` | interface | Options for an idempotent retained cancellation. | +| `RetainedRunEventOptions` | interface | Options for replaying canonical events strictly after a saved point. | +| `RetainedRunHandle` | interface | Reconstructable control of one provider-retained run. | +| `RetainedRunReplayPoint` | interface | Cursor plus runtime sequence needed to continue one ordered replay. | +| `RetainedRunSnapshot` | interface | Stable status snapshot for a retained run. | | `RootHandle` | interface | Live root handle — a chat/pi-viz client uses it to inspect and control one root run. | | `RouterSeam` | interface | Router/inline transport seam. The profile owns model, prompt, and generation behavior. | | `RouterToolsSeam` | interface | Router seam WITH tool use — the tool-using router backend. Same direct | @@ -944,6 +961,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `SpawnForestTree` | interface | One journal tree in a recursively loaded supervision forest. | | `SpawnJournal` | interface | The spawn-tree event source (mirrors `ConversationJournal`'s begin/append/load shape). | | `Spend` | interface | Conserved spend, reconciled from the normalized `UsageEvent` stream. Tokens and usd | +| `StartRetainedRunOptions` | interface | A retained start is retry-safe only when environment and turn keys are explicit. | | `SteerableRootHandle` | interface | A Runtime-minted root handle that can deliver raw steering or answers to a live manager inbox. | | `SteerableSandboxSession` | interface | What the steerable session exposes to its executor: the usage stream plus the live reads. | | `SteerContext` | interface | How a combinator's `act` consumes findings to steer — the SINGLE firewalled steer surface a | @@ -1025,6 +1043,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `LoopUntil` | type | `loopUntil(spec)` — build the iterative-deepening combinator. `seed` is the initial state. | | `MaterializedModelIdentity` | type | A named model carried into an execution, or an explicit reason the exact model is unknowable. | | `MountRecorder` | type | Records a mounted resource into the run's provenance manifest. Passed to | +| `NativeContextContinuationExecution` | type | Result of one verified same-session continuation. | +| `NativeContextContinuationInput` | type | Runtime controls plus the exact user turn bound into a continuation request. | | `NodeId` | type | Deterministic node id — `${parent}:s${seq}` from the cursor order, never wall-clock. | | `NodeStatus` | type | `'acquiring'` is first-class (M1): a node spends real time + reaps an orphan box | | `ObserveSupervisorNodeEvent` | type | Context-aware observer used internally to bind product transactions to the actual live node. | @@ -1039,6 +1059,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 718 exports. | `ResolveDriveHarness` | type | Resolve an external harness for one exact Runtime-owned manager identity. | | `ResolveSupervisorTools` | type | Product policy for the tools one exact supervisor node may call. Resolved once per node. | | `Restart` | type | OTP child-spec restart class. | +| `RetainedRunEffect` | type | Effect recorded for one retained control operation. | | `RootMaterialization` | type | Trusted root composition evidence. Generic `Agent.act` roots omit this and remain unknown. | | `RootSignal` | type | Out-of-band message to a running root. Open by intent — a client extends it. | | `RunContext` | type | The stores a supervised run needs, in-memory or file-backed. `InMemoryRunContext` is the | @@ -1590,7 +1611,7 @@ Import from `@tangle-network/agent-eval` — 61 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 358 exports. +Import from `@tangle-network/agent-eval/campaign` — 363 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1658,6 +1679,7 @@ Import from `@tangle-network/agent-eval/campaign` — 358 exports. | `provenanceRecordPath` | function | Canonical durable paths under the run dir. | | `provenanceSpansPath` | function | Canonical path for the durable OTLP spans JSONL file under a loop run directory. | | `readExternalOptimizerObservationArtifact` | function | Read and verify the exact callback observation artifact addressed by method provenance. | +| `readGepaCandidatePopulationArtifact` | function | Read GEPA's exact candidate graph from the artifact addressed by method provenance. | | `renderScoreboardMarkdown` | function | Render the scoreboard as a launch-readiness Markdown document — the literal | | `renderSurfaceDiff` | function | Canonical customer-visible description of the exact before/after surfaces. | | `resolveExternalOptimizerCallbackLimits` | function | _(no summary — add a TSDoc line at the declaration)_ | @@ -1777,7 +1799,7 @@ Import from `@tangle-network/agent-eval/campaign` — 358 exports. | `ProposalFindingOrigin` | type | Data sources that candidate generation may intentionally learn from. | | `SearchLedgerTrustedHeadMode` | type | How this ledger uses its trusted head — the `(sequence, entryHash)` pin kept | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalyzeCrossSurfaceInteractionsInput`, `AutoevalsScoreLike`, `AxisEvidence`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `BuildTraceAnalystSurfaceDispatchOptions`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareOptimizationMethodsOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DefaultProductionGateOptions`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `ExternalOptimizationExample`, `ExternalOptimizerCallbackLimits`, `ExternalOptimizerExecutionSummary`, `ExternalOptimizerModelBudget`, `ExternalOptimizerObservationArtifact`, `ExternalOptimizerObservationSummary`, `ExternalOptimizerProcessLimits`, `ExternalOptimizerRunnerCommand`, `ExternalOptimizerSubmittedCandidate`, `ExternalTextEvaluationResponse`, `ExternalTextOptimizerContext`, `ExternalTextOptimizerResult`, `FsLabeledScenarioStoreOptions`, `GateContext`, `GateContribution`, `GateResult`, `GenerationRecord`, `GepaOptimizationMethodConfig`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `LoopProvenanceEvidence`, `LoopProvenanceOptimizationMethod`, `NeutralizationGateOptions`, `OpenAutoPrOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizationMethodComparison`, `OptimizationMethodPairwise`, `OptimizationMethodProvenance`, `OptimizationMethodResult`, `OptimizationMethodScore`, `OptimizationPackageSource`, `OptimizationTokenUsage`, `OptimizerConfig`, `PairedHoldout`, `ParetoSignificanceGateOptions`, `PendingCostCallView`, `PhoenixEvaluationResultLike`, `PhoenixEvaluatorLike`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `PremeasuredOptimizationBaseline`, `ProfileSummary`, `PromotionObjective`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunCampaignOptions`, `RunEvalOptions`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SingleRunLock`, `SkillOptOptimizationMethodConfig`, `SkillOptTrainerConfig`, `TraceAnalystArtifact`, `TraceAnalystScenario`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `AutoevalsScorerLike`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `DefaultProductionGateCheck`, `DefaultProductionRewardHackingOptions`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `ExternalOptimizerEndpointFormat`, `ExternalOptimizerEvaluationRefusalReason`, `ExternalTextCandidate`, `OptimizerModelBudget`, `RedactionStatus`, `RunImprovementLoopOptions`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`, `SequentialDecision`, `SkillOptRunnerCommand`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalyzeCrossSurfaceInteractionsInput`, `AutoevalsScoreLike`, `AxisEvidence`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `BuildTraceAnalystSurfaceDispatchOptions`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareOptimizationMethodsOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DefaultProductionGateOptions`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `ExternalOptimizationExample`, `ExternalOptimizerCallbackLimits`, `ExternalOptimizerExecutionSummary`, `ExternalOptimizerModelBudget`, `ExternalOptimizerObservationArtifact`, `ExternalOptimizerObservationSummary`, `ExternalOptimizerProcessLimits`, `ExternalOptimizerRunnerCommand`, `ExternalOptimizerSubmittedCandidate`, `ExternalTextEvaluationResponse`, `ExternalTextOptimizerContext`, `ExternalTextOptimizerResult`, `FsLabeledScenarioStoreOptions`, `GateContext`, `GateContribution`, `GateResult`, `GenerationRecord`, `GepaCandidatePopulationArtifact`, `GepaCandidatePopulationCandidate`, `GepaCandidatePopulationSummary`, `GepaCandidateSelectionScore`, `GepaOptimizationMethodConfig`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `LoopProvenanceEvidence`, `LoopProvenanceOptimizationMethod`, `NeutralizationGateOptions`, `OpenAutoPrOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizationMethodComparison`, `OptimizationMethodPairwise`, `OptimizationMethodProvenance`, `OptimizationMethodResult`, `OptimizationMethodScore`, `OptimizationPackageSource`, `OptimizationTokenUsage`, `OptimizerConfig`, `PairedHoldout`, `ParetoSignificanceGateOptions`, `PendingCostCallView`, `PhoenixEvaluationResultLike`, `PhoenixEvaluatorLike`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `PremeasuredOptimizationBaseline`, `ProfileSummary`, `PromotionObjective`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunCampaignOptions`, `RunEvalOptions`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SingleRunLock`, `SkillOptOptimizationMethodConfig`, `SkillOptTrainerConfig`, `TraceAnalystArtifact`, `TraceAnalystScenario`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `AutoevalsScorerLike`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `DefaultProductionGateCheck`, `DefaultProductionRewardHackingOptions`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `ExternalOptimizerEndpointFormat`, `ExternalOptimizerEvaluationRefusalReason`, `ExternalTextCandidate`, `OptimizerModelBudget`, `RedactionStatus`, `RunImprovementLoopOptions`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`, `SequentialDecision`, `SkillOptRunnerCommand`. ### TOKEN / USAGE — usage extraction + run-record usage types diff --git a/docs/api/runtime.md b/docs/api/runtime.md index adab7e2b..fbb79a21 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -876,7 +876,7 @@ One flattened node with the journal tree that owns its records. ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`status`](#status-9) +[`NodeSnapshot`](#nodesnapshot).[`status`](#status-12) ##### runtime @@ -920,7 +920,7 @@ Manager-scoped assignment identity, including deterministic ids for unkeyed sibl ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`identity`](#identity-6) +[`NodeSnapshot`](#nodesnapshot).[`identity`](#identity-7) ##### materialization? @@ -5886,6 +5886,346 @@ Per-turn deadline (ms). *** +### RetainedRunReplayPoint + +**`Stable`** + +Cursor plus runtime sequence needed to continue one ordered replay. + +#### Properties + +##### cursor + +> `readonly` **cursor**: `string` + +##### sequence + +> `readonly` **sequence**: `number` + +*** + +### RetainedRunEventOptions + +**`Stable`** + +Options for replaying canonical events strictly after a saved point. + +#### Properties + +##### after? + +> `readonly` `optional` **after?**: [`RetainedRunReplayPoint`](#retainedrunreplaypoint) + +##### signal? + +> `readonly` `optional` **signal?**: `AbortSignal` + +*** + +### RetainedRunSnapshot + +**`Stable`** + +Stable status snapshot for a retained run. + +#### Properties + +##### runId + +> `readonly` **runId**: `string` + +##### controlRef + +> `readonly` **controlRef**: `AgentExactRunControlRef` + +##### status + +> `readonly` **status**: `AgentSessionStatus` \| `null` + +##### effect + +> `readonly` **effect**: [`RetainedRunEffect`](#retainedruneffect) + +##### observedAt + +> `readonly` **observedAt**: `string` + +##### reason? + +> `readonly` `optional` **reason?**: `string` + +##### signal? + +> `readonly` `optional` **signal?**: `string` + +*** + +### RetainedRunCancellation + +**`Stable`** + +Durable acknowledgement state for one retained control operation. + +#### Properties + +##### operationId + +> `readonly` **operationId**: `string` + +##### requestDigest + +> `readonly` **requestDigest**: `` `sha256:${string}` `` + +##### status + +> `readonly` **status**: `"unknown"` \| `"replayed"` \| `"accepted"` \| `"conflict"` + +##### effect + +> `readonly` **effect**: [`RetainedRunEffect`](#retainedruneffect) + +##### snapshot + +> `readonly` **snapshot**: [`RetainedRunSnapshot`](#retainedrunsnapshot) + +##### reason? + +> `readonly` `optional` **reason?**: `string` + +##### signal? + +> `readonly` `optional` **signal?**: `string` + +*** + +### RetainedRunCancelOptions + +**`Stable`** + +Options for an idempotent retained cancellation. + +#### Properties + +##### operationId + +> `readonly` **operationId**: `string` + +##### reason? + +> `readonly` `optional` **reason?**: `string` + +##### signal? + +> `readonly` `optional` **signal?**: `AbortSignal` + +*** + +### RetainedRunHandle + +**`Stable`** + +Reconstructable control of one provider-retained run. + +#### Properties + +##### controlRef + +> `readonly` **controlRef**: `AgentExactRunControlRef` + +#### Methods + +##### status() + +> **status**(`options?`): `Promise`\<[`RetainedRunSnapshot`](#retainedrunsnapshot)\> + +###### Parameters + +###### options? + +###### waitMs? + +`number` + +###### signal? + +`AbortSignal` + +###### Returns + +`Promise`\<[`RetainedRunSnapshot`](#retainedrunsnapshot)\> + +##### events() + +> **events**(`options?`): `AsyncIterable`\<`RuntimeEventEnvelope`\> + +###### Parameters + +###### options? + +[`RetainedRunEventOptions`](#retainedruneventoptions) + +###### Returns + +`AsyncIterable`\<`RuntimeEventEnvelope`\> + +##### result() + +> **result**(): `Promise`\<`AgentTurnResult`\> + +###### Returns + +`Promise`\<`AgentTurnResult`\> + +##### respondToInteraction() + +> **respondToInteraction**(`command`, `options?`): `Promise`\<\{ \}\> + +###### Parameters + +###### command + +###### options? + +###### signal? + +`AbortSignal` + +###### Returns + +`Promise`\<\{ \}\> + +##### contextBoundary() + +> **contextBoundary**(`options?`): `Promise`\<\{ \} \| `null`\> + +###### Parameters + +###### options? + +###### signal? + +`AbortSignal` + +###### Returns + +`Promise`\<\{ \} \| `null`\> + +##### continueNative() + +> **continueNative**(`request`, `turn`): `Promise`\<\{ \} \| \{ \}\> + +###### Parameters + +###### request + +`NativeContextContinuationRequest` + +###### turn + +[`NativeContextContinuationInput`](#nativecontextcontinuationinput) + +###### Returns + +`Promise`\<\{ \} \| \{ \}\> + +##### cancel() + +> **cancel**(`options`): `Promise`\<[`RetainedRunCancellation`](#retainedruncancellation)\> + +###### Parameters + +###### options + +[`RetainedRunCancelOptions`](#retainedruncanceloptions) + +###### Returns + +`Promise`\<[`RetainedRunCancellation`](#retainedruncancellation)\> + +*** + +### StartRetainedRunOptions + +**`Stable`** + +A retained start is retry-safe only when environment and turn keys are explicit. + +#### Properties + +##### provider + +> `readonly` **provider**: `AgentEnvironmentProvider` + +##### environment + +> `readonly` **environment**: `CreateAgentEnvironmentInput` & `object` + +###### Type Declaration + +###### idempotencyKey + +> **idempotencyKey**: `string` + +##### turn + +> `readonly` **turn**: `AgentTurnInput` & `object` + +###### Type Declaration + +###### turnId + +> **turnId**: `string` + +##### identity? + +> `readonly` `optional` **identity?**: `object` + +Runtime-owned coordinates for providers that support deterministic retained dispatch. + +###### sessionId + +> `readonly` **sessionId**: `string` + +###### executionId + +> `readonly` **executionId**: `string` + +##### now? + +> `readonly` `optional` **now?**: () => `number` + +###### Returns + +`number` + +*** + +### ReconnectRetainedRunOptions + +**`Stable`** + +Inputs sufficient to rebuild a control client in a new process. + +#### Properties + +##### provider + +> `readonly` **provider**: `AgentEnvironmentProvider` + +##### controlRef + +> `readonly` **controlRef**: `AgentExactRunControlRef` + +##### now? + +> `readonly` `optional` **now?**: () => `number` + +###### Returns + +`number` + +*** + ### RouterTransportConfig Connection details for Runtime's Router-backed executors. @@ -7486,7 +7826,7 @@ The gen0 field. Default [sample, refine, sampleThenRefine]. ##### objective? -> `optional` **objective?**: `"score"` \| `"cost"` +> `optional` **objective?**: `"cost"` \| `"score"` What "better" means for PROMOTION. 'score' (default): the candidate must beat the incumbent's score (superiority gate). 'cost': the candidate must prove score @@ -13897,7 +14237,7 @@ breaker, or a recursive parent. ###### Inherited from -[`SupervisorNodeContext`](#supervisornodecontext).[`runId`](#runid-15) +[`SupervisorNodeContext`](#supervisornodecontext).[`runId`](#runid-16) ##### runNamespace @@ -13943,7 +14283,7 @@ Stable identity of this manager's coordination stream. ###### Inherited from -[`SupervisorNodeContext`](#supervisornodecontext).[`identity`](#identity-1) +[`SupervisorNodeContext`](#supervisornodecontext).[`identity`](#identity-2) ##### assignmentId? @@ -15643,7 +15983,7 @@ Phantom: binds the handle to the supervised run's output type. Type-only — nev ###### Inherited from -[`RootHandle`](#roothandle-1).[`signal`](#signal-17) +[`RootHandle`](#roothandle-1).[`signal`](#signal-21) ##### abort() @@ -18588,6 +18928,36 @@ judge/verdict/score scheme is rejected. Fail loud — a tainted finding aborts. *** +### RetainedRunEffect + +> **RetainedRunEffect** = `"cancel_requested"` \| `"cancelled"` \| `"not_live"` \| `"unknown"` + +**`Stable`** + +Effect recorded for one retained control operation. + +*** + +### NativeContextContinuationInput + +> **NativeContextContinuationInput** = `NativeContextContinuationTurn` & `Omit`\<`AgentNativeContextContinuationOptions`, `"turn"`\> + +**`Stable`** + +Runtime controls plus the exact user turn bound into a continuation request. + +*** + +### NativeContextContinuationExecution + +> **NativeContextContinuationExecution** = `AgentNativeContextContinuationResult` + +**`Stable`** + +Result of one verified same-session continuation. + +*** + ### Environment > **Environment** = [`AgenticSurface`](#agenticsurface) @@ -21945,6 +22315,47 @@ that `resolveBenchClient` builds on — reuse this instead of hand-rolling the *** +### startRetainedRun() + +> **startRetainedRun**(`options`): `Promise`\<[`RetainedRunHandle`](#retainedrunhandle)\> + +**`Stable`** + +Dispatch one detached, replayable run and return only after exact durable +coordinates are confirmed by the provider. + +#### Parameters + +##### options + +[`StartRetainedRunOptions`](#startretainedrunoptions) + +#### Returns + +`Promise`\<[`RetainedRunHandle`](#retainedrunhandle)\> + +*** + +### reconnectRetainedRun() + +> **reconnectRetainedRun**(`options`): `Promise`\<[`RetainedRunHandle`](#retainedrunhandle) \| `null`\> + +**`Stable`** + +Rebuild a retained-run client without retaining any object from the starter. + +#### Parameters + +##### options + +[`ReconnectRetainedRunOptions`](#reconnectretainedrunoptions) + +#### Returns + +`Promise`\<[`RetainedRunHandle`](#retainedrunhandle) \| `null`\> + +*** + ### runBenchmark() > **runBenchmark**(`cfg`): `Promise`\<[`BenchmarkReport`](#benchmarkreport)\> diff --git a/docs/api/testing.md b/docs/api/testing.md index def58d12..a50a8887 100644 --- a/docs/api/testing.md +++ b/docs/api/testing.md @@ -383,7 +383,7 @@ The run journal the edge ledger and every spawn/settle ride. Default: in-memory. ###### Inherited from -[`RunGraphOptions`](runtime.md#rungraphoptions).[`runId`](runtime.md#runid-10) +[`RunGraphOptions`](runtime.md#rungraphoptions).[`runId`](runtime.md#runid-11) ##### perWorker? @@ -438,7 +438,7 @@ Product authority over every steer/answer instruction (the filter seam). `runGra ###### Inherited from -[`RunGraphOptions`](runtime.md#rungraphoptions).[`signal`](runtime.md#signal-11) +[`RunGraphOptions`](runtime.md#rungraphoptions).[`signal`](runtime.md#signal-15) ##### now? @@ -450,7 +450,7 @@ Product authority over every steer/answer instruction (the filter seam). `runGra ###### Inherited from -[`RunGraphOptions`](runtime.md#rungraphoptions).[`now`](runtime.md#now-4) +[`RunGraphOptions`](runtime.md#rungraphoptions).[`now`](runtime.md#now-6) ##### otel? @@ -522,7 +522,7 @@ root scope and every live child, including acquisition and backend execution. ###### Inherited from -[`SuperviseOptions`](runtime.md#superviseoptions).[`signal`](runtime.md#signal-13) +[`SuperviseOptions`](runtime.md#superviseoptions).[`signal`](runtime.md#signal-17) ##### execution? @@ -1159,7 +1159,7 @@ Give the supervisor brain a chapter-lifecycle on its OWN context window (router ###### Inherited from -[`SuperviseOptions`](runtime.md#superviseoptions).[`runId`](runtime.md#runid-14) +[`SuperviseOptions`](runtime.md#superviseoptions).[`runId`](runtime.md#runid-15) ##### now? @@ -1171,7 +1171,7 @@ Give the supervisor brain a chapter-lifecycle on its OWN context window (router ###### Inherited from -[`SuperviseOptions`](runtime.md#superviseoptions).[`now`](runtime.md#now-10) +[`SuperviseOptions`](runtime.md#superviseoptions).[`now`](runtime.md#now-12) ##### allowedModels? diff --git a/docs/canonical-api.md b/docs/canonical-api.md index eeacac27..193cc0ea 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,9 +4,9 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.131.7.** +> **Version 0.132.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.144.10 <0.145.0`. +> `agent-eval` must satisfy `>=0.144.12 <0.145.0`. > `sandbox` must satisfy `>=0.19.4 <0.20.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.46.1 <0.47.0`. > @@ -28,7 +28,16 @@ The system is four steps, each with a named entry point: 1. **Describe the agent as data.** A **profile** is the whole agent: `systemPrompt + skills + tools + mcp + knowledge + memory + rag`, one combined surface. 2. **Run it.** A driver steers workers over rounds: `runPersonified` composes a combinator (`loopUntil`, `fanout`, …) over the `Supervisor`, spending K rounds against one persistent, journaled artifact from a *conserved budget pool*, so any two topologies you compare cost the same by construction. 3. **Score it on a benchmark.** Either the `ADAPTERS` registry driven by `runGate` over the Supervisor, or an `AgenticSurface` driven by `runBenchmark`/`runAgentic`. -4. **Improve it on three partitions.** `improve(profile, { executionRef, method, trainScenarios, selectionScenarios, testScenarios, agent })` runs a complete agent-eval `OptimizationMethod`. The method receives train and selection cases only. Runtime materializes each surface as a complete detached profile and passes that exact profile to `agent`. `executionRef` identifies the callback, component mapping, model, tools, and closure settings. Agent Eval scores the selected profile on the untouched final test. Runtime returns `ship` only when the paired interval clears the required lift and all spend is accounted for. +4. **Improve it on three partitions.** + `improve(profile, { executionRef, method, trainScenarios, selectionScenarios, testScenarios, agent })` runs a complete agent-eval `OptimizationMethod`. + The method receives train and selection cases only. + Runtime materializes each surface as a complete detached profile and passes that profile to `agent`. + `executionRef` identifies the callback, component mapping, model, tools, and closure settings. + Agent Eval scores the selected profile on the untouched final test. + Runtime returns `ship` only when the paired interval clears the required lift and all spend is accounted for. + `candidatePopulation` joins verified callback observations with the optimizer's official graph. + It returns every unique candidate as an exact profile plus Interface diffs, or as an explicit refusal. + Official GEPA graph nodes retain parent indices and selection scores. Two standing rules: the model that picks the best attempt is never the model that grades it, and observation attaches to the *loop* via `RuntimeHooks`, never to the portable profile. One known limit: the current `Supervisor` records completed settlements but does not resume a live tree after coordinator restart. diff --git a/package.json b/package.json index 20078f0c..a4dab846 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.131.7", + "version": "0.132.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -170,7 +170,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.144.10 <0.145.0", + "@tangle-network/agent-eval": ">=0.144.12 <0.145.0", "@tangle-network/agent-interface": ">=0.46.1 <0.47.0", "@tangle-network/sandbox": ">=0.19.4 <0.20.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f573aa6..820d8a15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,14 +16,14 @@ catalogs: specifier: 0.5.4 version: 0.5.4 '@tangle-network/agent-eval': - specifier: 0.144.10 - version: 0.144.10 + specifier: 0.144.12 + version: 0.144.12 '@tangle-network/agent-interface': specifier: 0.46.1 version: 0.46.1 '@tangle-network/agent-knowledge': - specifier: 7.2.0 - version: 7.2.0 + specifier: 7.2.1 + version: 7.2.1 '@tangle-network/agent-profile-materialize': specifier: 0.13.1 version: 0.13.1 @@ -58,7 +58,7 @@ importers: version: 0.5.4 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.2.0(@tangle-network/agent-eval@0.144.10)(@tangle-network/agent-interface@0.46.1) + version: 7.2.1(@tangle-network/agent-eval@0.144.12)(@tangle-network/agent-interface@0.46.1) '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.13.1(@tangle-network/agent-interface@0.46.1) @@ -80,7 +80,7 @@ importers: version: 1.30.0(supports-color@10.2.2)(zod@4.4.3) '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.144.10 + version: 0.144.12 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.46.1 @@ -128,13 +128,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.144.10 + version: 0.144.12 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.46.1 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 7.2.0(@tangle-network/agent-eval@0.144.10)(@tangle-network/agent-interface@0.46.1) + version: 7.2.1(@tangle-network/agent-eval@0.144.12)(@tangle-network/agent-interface@0.46.1) '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -1127,20 +1127,20 @@ packages: '@tangle-network/agent-core@0.5.4': resolution: {integrity: sha512-k6gYv3BlagkfuWrGyTJH6mKUBgsLY6TXxizACqt0QF8a1/5uqy0UYc6R2Wo9nqQVJuaRxDnoiRf8YtsRVqA75g==} - '@tangle-network/agent-eval@0.144.10': - resolution: {integrity: sha512-mDUf902qRCDnEzQ7cyN9ytohyPtho/+Aumw8kMTjMcJo7XoUKeH4bEqpsTS0q4aLMQYgg5R+xH8FRCnN1wt8Jg==} + '@tangle-network/agent-eval@0.144.12': + resolution: {integrity: sha512-HTyCiA05Z26fUUpBOQriD49U+2UnAxzTIncR8jJl8Jvp7XDTU0Ng3fIMgfo85ZvnRuaHuay6gDD8DrvG6KO92A==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@0.46.1': resolution: {integrity: sha512-6a3GRkDxS+r6Bmlu8y6LQpiVE20oCYPzE2opb5o+AZeDzRW5KemyxreyYIprjgKfDrvBTMo0tgvC2JB51Sl84w==} - '@tangle-network/agent-knowledge@7.2.0': - resolution: {integrity: sha512-DPpicRkEcRSd+5RBMmFg7uaoSQx4RpoTvrLo2Zm7m1NtDbQRKXgws/iaSPTWaOewUD5lI7OuJiXaBoLctGKn9g==} + '@tangle-network/agent-knowledge@7.2.1': + resolution: {integrity: sha512-0MSgFBBqYEtSoppne/tAkcCE2Akke5oFZhjWCDPSCWOvEV+wUALgRExO83OUUJJ65xWGZ0UamTI645PlagaodA==} engines: {node: '>=20.19.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.144.10 <0.145.0' + '@tangle-network/agent-eval': '>=0.144.11 <0.145.0' '@tangle-network/agent-interface': '>=0.46.1 <0.47.0' '@tangle-network/agent-profile-materialize@0.13.1': @@ -3170,7 +3170,7 @@ snapshots: '@tangle-network/agent-interface': 0.46.1 zod: 4.4.3 - '@tangle-network/agent-eval@0.144.10': + '@tangle-network/agent-eval@0.144.12': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.0.12(hono@4.12.32) @@ -3188,9 +3188,9 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@7.2.0(@tangle-network/agent-eval@0.144.10)(@tangle-network/agent-interface@0.46.1)': + '@tangle-network/agent-knowledge@7.2.1(@tangle-network/agent-eval@0.144.12)(@tangle-network/agent-interface@0.46.1)': dependencies: - '@tangle-network/agent-eval': 0.144.10 + '@tangle-network/agent-eval': 0.144.12 '@tangle-network/agent-interface': 0.46.1 proper-lockfile: 4.1.2 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ca6d7a11..cca34395 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -20,9 +20,9 @@ catalog: '@modelcontextprotocol/sdk': 1.30.0 '@tangle-network/agent-core': 0.5.4 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.144.10 + '@tangle-network/agent-eval': 0.144.12 '@tangle-network/agent-interface': 0.46.1 - '@tangle-network/agent-knowledge': 7.2.0 + '@tangle-network/agent-knowledge': 7.2.1 '@tangle-network/agent-profile-materialize': 0.13.1 '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': 0.19.4 diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs index 2d74723c..eac87039 100644 --- a/scripts/verify-official-optimizers.mjs +++ b/scripts/verify-official-optimizers.mjs @@ -170,11 +170,10 @@ try { const installedKnowledge = readJson( join(appDir, 'node_modules', '@tangle-network', 'agent-knowledge', 'package.json'), ) - assertInstalledKnowledgeSharedPeer(installedKnowledge, '@tangle-network/agent-eval', agentEvalVersion) + assertInstalledKnowledgeSharedPeer(installedKnowledge, '@tangle-network/agent-eval') assertInstalledKnowledgeSharedPeer( installedKnowledge, '@tangle-network/agent-interface', - workspaceAgentInterfaceVersion, ) run( process.execPath, @@ -315,15 +314,13 @@ function requiredPackedDependency(packageJson, packageName) { ) } -function assertInstalledKnowledgeSharedPeer(packageJson, packageName, expectedVersion) { +function assertInstalledKnowledgeSharedPeer(packageJson, packageName) { if (packageJson.dependencies?.[packageName] !== undefined) { throw new Error(`installed Agent Knowledge must not nest ${packageName} as a runtime dependency`) } - assertVersion( - requiredPackedDevelopmentDependency(packageJson, packageName), - expectedVersion, - `installed Agent Knowledge development ${packageName}`, - ) + // npm installs with strict peer checks above. This confirms Knowledge's tested lower bound + // without requiring its development patch to equal the compatible patch selected by Runtime. + requiredPackedDevelopmentDependency(packageJson, packageName) assertPeerMatchesDevelopmentDependency(packageJson, packageName) } diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 91de1bbc..d22f703e 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -186,6 +186,11 @@ try { SandboxSizePreset, Sha256Digest, } from '@tangle-network/agent-interface' + import type { + ImproveMethodResult, + ImprovementProfileCandidatePopulation, + ImprovementProfilePopulationCandidateSource, + } from '@tangle-network/agent-runtime' import { driverAgent, type AgentProfileImprovementFixture, @@ -239,6 +244,10 @@ try { declare const profileStateResolver: AgentImprovementProfileStateResolver declare const profileEvaluation: AgentImprovementEvaluation declare const activeProfile: AgentProfile + declare const improvementResult: ImproveMethodResult + const candidatePopulation: ImprovementProfileCandidatePopulation = + improvementResult.candidatePopulation + declare const populationCandidateSource: ImprovementProfilePopulationCandidateSource const proposalFixture: AgentImprovementProposal = loadAgentImprovementProposalFixture() const profileFixture: AgentProfileImprovementFixture = loadAgentProfileImprovementFixture() @@ -333,6 +342,8 @@ try { void currentDigest void profileDiffs void profilePrepared + void candidatePopulation + void populationCandidateSource void proposalFixture void profileFixture void fixtureProfileExperimentDigest diff --git a/scripts/verify-packed-cohort.mjs b/scripts/verify-packed-cohort.mjs index a34e6c52..5bcd5547 100644 --- a/scripts/verify-packed-cohort.mjs +++ b/scripts/verify-packed-cohort.mjs @@ -619,21 +619,15 @@ function assertSharedContractPeer(owner, dependency) { if (owner.packageJson.dependencies?.[dependency.name] !== undefined) { throw new Error(`${owner.name} must not nest ${dependency.name} as a runtime dependency`) } - assertVersion( - requiredPackedDevelopmentDependency(owner.packageJson, dependency.name), - dependency.version, - `${owner.name} development ${dependency.name}`, - ) + // A required peer deliberately admits later compatible patches. Requiring every owner's + // development pin to equal the consumer-selected patch recreates the release lockstep that + // peer dependencies removed. The strict packed install below proves the selected version is + // admitted and resolves to one physical package; this check proves the owner's lower bound. + requiredPackedDevelopmentDependency(owner.packageJson, dependency.name) assertPeerMatchesDevelopmentDependency(owner.packageJson, dependency.name) assertRequiredPeer(owner, dependency) } -function assertVersion(actual, expected, label) { - if (actual !== expected) { - throw new Error(`${label} must be ${expected}, found ${String(actual)}`) - } -} - function assertCleanGitCheckout(sourceRepo, packageName) { if (!existsSync(join(sourceRepo, '.git'))) { throw new Error(`${packageName} source is not a Git checkout: ${sourceRepo}`) diff --git a/src/improvement/improve-types.ts b/src/improvement/improve-types.ts index 7379b576..1432557c 100644 --- a/src/improvement/improve-types.ts +++ b/src/improvement/improve-types.ts @@ -13,7 +13,7 @@ import type { SelfImproveOptions, SelfImproveResult, } from '@tangle-network/agent-eval/contract' -import type { AgentProfile, Sha256Digest } from '@tangle-network/agent-interface' +import type { AgentProfile, AgentProfileDiff, Sha256Digest } from '@tangle-network/agent-interface' import type { AgenticGeneratorExecutorForWorktree, Verifier } from './agentic-generator' import type { CandidateGenerator } from './improvement-driver' import type { ReadonlyAgentProfile } from './profile-types' @@ -233,6 +233,118 @@ export interface ImprovementProfileCandidate { profile: ReadonlyAgentProfile } +/** Digest-addressed Eval artifact. */ +export interface ImprovementProfilePopulationArtifactSource { + path: string + sha256: Sha256Digest +} + +/** Exact callback observation that introduced one optimizer candidate. */ +export interface ImprovementProfilePopulationObservationSource { + /** One-based JSONL line sequence in the verified observation artifact. */ + proposalSequence: number + artifact: ImprovementProfilePopulationArtifactSource +} + +/** One exact node from GEPA's accepted candidate graph. */ +export interface ImprovementProfilePopulationLineageNode { + index: number + parentIndices: readonly (number | null)[] + aggregateScore: number | null + selectionScores: readonly { + scenarioId: string + score: number + }[] + discoveryEvaluationCount: number +} + +export type ImprovementProfilePopulationLineage = + | { + status: 'available' + artifact: ImprovementProfilePopulationArtifactSource + nodes: readonly ImprovementProfilePopulationLineageNode[] + } + | { + status: 'unavailable' + reason: 'optimizer-did-not-report-candidate-lineage' + } + +/** Every verified source associated with one unique optimizer candidate. */ +export interface ImprovementProfilePopulationCandidateSource { + /** Eval identity of the external text or component candidate. */ + candidateDigest: Sha256Digest + /** Present when the candidate crossed the evaluation callback. */ + observation?: ImprovementProfilePopulationObservationSource + /** Exact GEPA parents and scores, or an explicit statement that none were reported. */ + lineage: ImprovementProfilePopulationLineage +} + +/** A verified optimizer candidate that Runtime can express as an exact profile. */ +export interface ImprovementMaterializedProfilePopulationCandidate { + status: 'materialized' + source: ImprovementProfilePopulationCandidateSource + /** Exact optimizer surface decoded by Eval. */ + value: MutableSurface + /** Interface identity of `value`. */ + surfaceDigest: Sha256Digest + /** Exact complete profile produced by Runtime's configured materializer. */ + profile: ReadonlyAgentProfile + /** Interface identity of `profile`. */ + profileDigest: Sha256Digest + /** Ordered Interface diffs that reproduce `profile` from the baseline. */ + diffs: readonly AgentProfileDiff[] + /** Interface identity of each entry in `diffs`. */ + diffDigests: readonly Sha256Digest[] +} + +/** A verified optimizer candidate that Runtime refused to materialize. */ +export interface ImprovementRefusedProfilePopulationCandidate { + status: 'refused' + source: ImprovementProfilePopulationCandidateSource + /** Exact optimizer surface decoded by Eval. */ + value: MutableSurface + /** Interface identity of `value`. */ + surfaceDigest: Sha256Digest + error: { + name: string + message: string + } +} + +export type ImprovementProfilePopulationCandidate = + | ImprovementMaterializedProfilePopulationCandidate + | ImprovementRefusedProfilePopulationCandidate + +/** Complete verified population reported by one optimizer run. */ +export interface ImprovementProfileCandidatePopulationAvailable { + status: 'available' + source: { + observations?: ImprovementProfilePopulationArtifactSource + gepaCandidateGraph?: ImprovementProfilePopulationArtifactSource & { + bestIndex: number + } + } + /** Distinct candidate surfaces across all verified source artifacts. */ + uniqueCandidates: number + /** Distinct candidate surfaces submitted through the evaluation callback. */ + observedCandidates: number + /** Exact GEPA graph nodes. Multiple nodes can have the same candidate surface. */ + gepaCandidateNodes: number + materializedCandidates: number + refusedCandidates: number + candidates: readonly ImprovementProfilePopulationCandidate[] +} + +/** Explicit absence for methods that do not report candidate population evidence. */ +export interface ImprovementProfileCandidatePopulationUnavailable { + status: 'unavailable' + reason: 'method-did-not-report-candidate-population' +} + +export type ImprovementProfileCandidatePopulation = + | ImprovementProfileCandidatePopulationAvailable + | ImprovementProfileCandidatePopulationUnavailable + export interface ImprovementCodeCandidate { surface: 'code' value: MutableSurface @@ -314,6 +426,8 @@ export interface ImproveMethodResult extends ImproveResultBase { + for (const root of populationFixtureRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) const improvementJudge: JudgeConfig = { name: 'improvement', @@ -101,6 +114,117 @@ function fixedMethod( } } +function populationMethod( + storage: ReturnType, + winnerSurface = 'improved prompt', +): OptimizationMethod { + const runId = 'in-memory-population' + const graphRoot = mkdtempSync(join(tmpdir(), 'agent-runtime-candidate-population-')) + populationFixtureRoots.push(graphRoot) + const graphPath = join(graphRoot, 'gepa-candidate-population.json') + const observationPath = 'mem://population/external-optimizer-observations.jsonl' + const selectedCandidate = 'improved prompt' + const callbackOnlyCandidate = 'callback only prompt' + const selectedHash = contentHash({ + kind: 'external-text-candidate', + candidate: selectedCandidate, + }) + const callbackOnlyHash = contentHash({ + kind: 'external-text-candidate', + candidate: callbackOnlyCandidate, + }) + const observations = `${[ + { + kind: 'proposal', + sequence: 1, + candidate: selectedCandidate, + candidateHash: selectedHash, + }, + { + kind: 'proposal', + sequence: 2, + candidate: callbackOnlyCandidate, + candidateHash: callbackOnlyHash, + }, + ] + .map(canonicalJson) + .join('\n')}\n` + const graph = JSON.stringify({ + schemaVersion: 1, + scope: 'gepa-candidate-population', + runId, + bestIndex: 0, + candidates: [ + { + index: 0, + candidate: selectedCandidate, + parentIndices: [null], + aggregateScore: 1, + selectionScores: [{ scenarioId: 'selection', score: 1 }], + discoveryEvaluationCount: 0, + }, + { + index: 1, + candidate: selectedCandidate, + parentIndices: [0], + aggregateScore: 0.75, + selectionScores: [{ scenarioId: 'selection', score: 0.75 }], + discoveryEvaluationCount: 1, + }, + ], + }) + writeFileSync(graphPath, graph) + storage.write(observationPath, observations) + return { + name: 'in-memory-population', + async optimize() { + return { + winnerSurface, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, + durationMs: 1, + provenance: { + source: { + kind: 'package', + evidence: 'declared', + package: 'in-memory-population-fixture', + version: '1.0.0', + }, + runId, + resumed: false, + evaluationCount: 0, + artifactDir: 'mem://population', + observations: { + scope: 'callback-submitted-candidates', + path: observationPath, + sha256: `sha256:${createHash('sha256').update(observations).digest('hex')}`, + submittedCandidates: 2, + evaluations: 0, + refusals: 0, + }, + gepaCandidatePopulation: { + scope: 'gepa-candidate-population', + path: graphPath, + sha256: `sha256:${createHash('sha256').update(graph).digest('hex')}`, + bytes: Buffer.byteLength(graph), + runId, + candidates: 2, + bestIndex: 0, + maxCandidates: 2, + maxCandidateChars: 100, + scenarioIds: ['selection'], + surfaceKind: 'text', + }, + }, + } + }, + } +} + function methodOptions(method: OptimizationMethod): { method: OptimizationMethod executionRef: typeof executionRef @@ -167,10 +291,99 @@ describe('improve method execution', () => { expect(result.lift).toBe(1) expect(result.liftInterval.low).toBeGreaterThan(0) expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') + expect(result.candidatePopulation).toEqual({ + status: 'unavailable', + reason: 'method-did-not-report-candidate-population', + }) expect(profile.prompt?.systemPrompt).toBe('baseline') expect(Object.isFrozen(result.candidate)).toBe(true) }) + it('joins custom-storage observations with the file-backed GEPA graph', async () => { + const storage = inMemoryCampaignStorage() + const result = await improve(promptProfile(), { + ...methodOptions(populationMethod(storage)), + storage, + optimizationRunOptions: { storage }, + runDir: 'mem://in-memory-population-run', + }) + + expect(result.candidatePopulation).toMatchObject({ + status: 'available', + uniqueCandidates: 2, + observedCandidates: 2, + gepaCandidateNodes: 2, + materializedCandidates: 2, + refusedCandidates: 0, + candidates: expect.arrayContaining([ + expect.objectContaining({ + status: 'materialized', + value: 'improved prompt', + source: expect.objectContaining({ + lineage: expect.objectContaining({ + status: 'available', + nodes: expect.arrayContaining([ + expect.objectContaining({ index: 0, parentIndices: [null], aggregateScore: 1 }), + expect.objectContaining({ index: 1, parentIndices: [0], aggregateScore: 0.75 }), + ]), + }), + }), + }), + expect.objectContaining({ + status: 'materialized', + value: 'callback only prompt', + source: expect.objectContaining({ + lineage: { + status: 'unavailable', + reason: 'optimizer-did-not-report-candidate-lineage', + }, + }), + }), + ]), + }) + if (result.candidatePopulation.status !== 'available') { + throw new Error('expected an available candidate population') + } + const joined = result.candidatePopulation.candidates.find( + (candidate) => candidate.value === 'improved prompt', + ) + expect(joined?.source.lineage).toEqual({ + status: 'available', + artifact: { + path: result.candidatePopulation.source.gepaCandidateGraph?.path, + sha256: result.candidatePopulation.source.gepaCandidateGraph?.sha256, + }, + nodes: [ + { + index: 0, + parentIndices: [null], + aggregateScore: 1, + selectionScores: [{ scenarioId: 'selection', score: 1 }], + discoveryEvaluationCount: 0, + }, + { + index: 1, + parentIndices: [0], + aggregateScore: 0.75, + selectionScores: [{ scenarioId: 'selection', score: 0.75 }], + discoveryEvaluationCount: 1, + }, + ], + }) + }) + + it('refuses a method winner that differs from the verified GEPA best candidate', async () => { + const storage = inMemoryCampaignStorage() + await expect( + improve(promptProfile(), { + ...methodOptions(populationMethod(storage, 'improved method winner')), + storage, + optimizationRunOptions: { storage }, + runDir: 'mem://mismatched-population-winner', + }), + ).rejects.toThrow(/method winner does not equal the verified GEPA bestIndex candidate/u) + }) + it('resumes an identical profile run without dispatching another agent call', async () => { const storage = inMemoryCampaignStorage() let agentCalls = 0 diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 6d887932..0b1518ed 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -41,7 +41,18 @@ export type { ImproveMethodSource, ImprovementCandidate, ImprovementCodeCandidate, + ImprovementMaterializedProfilePopulationCandidate, ImprovementProfileCandidate, + ImprovementProfileCandidatePopulation, + ImprovementProfileCandidatePopulationAvailable, + ImprovementProfileCandidatePopulationUnavailable, + ImprovementProfilePopulationArtifactSource, + ImprovementProfilePopulationCandidate, + ImprovementProfilePopulationCandidateSource, + ImprovementProfilePopulationLineage, + ImprovementProfilePopulationLineageNode, + ImprovementProfilePopulationObservationSource, + ImprovementRefusedProfilePopulationCandidate, ImproveOptimizationRunOptions, ImproveOptions, ImproveProfileAgent, diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 11ed901e..9c33115a 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -42,7 +42,18 @@ export { type ImproveMethodSource, type ImprovementCandidate, type ImprovementCodeCandidate, + type ImprovementMaterializedProfilePopulationCandidate, type ImprovementProfileCandidate, + type ImprovementProfileCandidatePopulation, + type ImprovementProfileCandidatePopulationAvailable, + type ImprovementProfileCandidatePopulationUnavailable, + type ImprovementProfilePopulationArtifactSource, + type ImprovementProfilePopulationCandidate, + type ImprovementProfilePopulationCandidateSource, + type ImprovementProfilePopulationLineage, + type ImprovementProfilePopulationLineageNode, + type ImprovementProfilePopulationObservationSource, + type ImprovementRefusedProfilePopulationCandidate, type ImproveOptimizationRunOptions, type ImproveOptions, type ImproveProfileAgent, diff --git a/src/improvement/method-execution.ts b/src/improvement/method-execution.ts index 8dda1b99..1e79a2ce 100644 --- a/src/improvement/method-execution.ts +++ b/src/improvement/method-execution.ts @@ -1,13 +1,21 @@ import { randomUUID } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' import { assertProposalFindings } from '@tangle-network/agent-eval/analyst' import { + type CampaignStorage, compareOptimizationMethods, + decodeExternalTextCandidate, type OptimizationMethod, type OptimizationMethodComparison, + readExternalOptimizerObservationArtifact, + readGepaCandidatePopulationArtifact, } from '@tangle-network/agent-eval/campaign' -import type { Scenario } from '@tangle-network/agent-eval/contract' +import type { MutableSurface, Scenario } from '@tangle-network/agent-eval/contract' import { type AgentProfile, + applyAgentProfileDiff, + diffAgentProfiles, + canonicalCandidateDigest as interfaceCandidateDigest, type Sha256Digest, sha256DigestSchema, } from '@tangle-network/agent-interface' @@ -21,6 +29,9 @@ import type { ImproveMethodResult, ImproveMethodSource, ImprovementProfileCandidate, + ImprovementProfileCandidatePopulation, + ImprovementProfilePopulationCandidateSource, + ImprovementProfilePopulationLineageNode, } from './improve-types' import { methodRuntimeControlsOf } from './method-controls' import { @@ -69,6 +80,214 @@ function validateExecutionRef(value: unknown): Sha256Digest { return parsed.data } +function materializationError(error: unknown): { name: string; message: string } { + if (error instanceof Error) { + return { name: error.name, message: error.message } + } + return { + name: 'Error', + message: + typeof error === 'string' ? error : 'candidate materialization threw a non-Error value', + } +} + +function profileCandidatePopulation( + provenance: OptimizationMethodComparison['best']['provenance'], + baselineProfile: AgentProfile, + materializeProfile: (candidateSurface: MutableSurface) => AgentProfile, + winnerSurface: MutableSurface, + storage?: CampaignStorage, +): ImprovementProfileCandidatePopulation { + const observationSummary = provenance?.observations + const graphSummary = provenance?.gepaCandidatePopulation + if (!observationSummary && !graphSummary) { + return Object.freeze({ + status: 'unavailable', + reason: 'method-did-not-report-candidate-population', + }) + } + + const observations = observationSummary + ? readExternalOptimizerObservationArtifact({ + summary: observationSummary, + ...(storage ? { storage } : {}), + }) + : undefined + const graph = graphSummary + ? readGepaCandidatePopulationArtifact({ + summary: graphSummary, + }) + : undefined + if (!graph && (observations?.candidates.length ?? 0) === 0) { + return Object.freeze({ + status: 'unavailable', + reason: 'method-did-not-report-candidate-population', + }) + } + interface PopulationEntry { + candidateDigest: Sha256Digest + value: MutableSurface + observation?: { + proposalSequence: number + artifact: { path: string; sha256: Sha256Digest } + } + lineageNodes: ImprovementProfilePopulationLineageNode[] + } + const entries = new Map() + const entryFor = (candidateDigest: Sha256Digest, value: MutableSurface): PopulationEntry => { + const existing = entries.get(candidateDigest) + if (existing) { + if (!isDeepStrictEqual(existing.value, value)) { + throw new ConfigError( + `improve(): optimizer artifacts disagree on candidate ${candidateDigest}`, + ) + } + return existing + } + const entry: PopulationEntry = { + candidateDigest, + value: immutableCandidateValue(value), + lineageNodes: [], + } + entries.set(candidateDigest, entry) + return entry + } + + for (const submitted of observations?.candidates ?? []) { + const entry = entryFor( + submitted.candidateDigest, + decodeExternalTextCandidate(submitted.candidate), + ) + entry.observation = { + proposalSequence: submitted.proposalSequence, + artifact: { + path: submitted.provenance.path, + sha256: submitted.provenance.sha256, + }, + } + } + for (const graphCandidate of graph?.candidates ?? []) { + const entry = entryFor( + graphCandidate.candidateDigest, + decodeExternalTextCandidate(graphCandidate.candidate), + ) + entry.lineageNodes.push({ + index: graphCandidate.index, + parentIndices: [...graphCandidate.parentIndices], + aggregateScore: graphCandidate.aggregateScore, + selectionScores: graphCandidate.selectionScores.map((score) => ({ ...score })), + discoveryEvaluationCount: graphCandidate.discoveryEvaluationCount, + }) + } + + if (graph) { + const best = graph.candidates.find((candidate) => candidate.index === graph.bestIndex) + if (!best) { + throw new ConfigError( + `improve(): GEPA candidate population has no bestIndex node ${graph.bestIndex}`, + ) + } + const verifiedBest = decodeExternalTextCandidate(best.candidate) + if (!isDeepStrictEqual(verifiedBest, winnerSurface)) { + throw new ConfigError( + 'improve(): method winner does not equal the verified GEPA bestIndex candidate', + ) + } + } else if ( + ![...entries.values()].some((entry) => isDeepStrictEqual(entry.value, winnerSurface)) + ) { + throw new ConfigError( + 'improve(): method winner does not appear in the verified optimizer observations', + ) + } + + let materializedCandidates = 0 + let refusedCandidates = 0 + const candidates = [...entries.values()].map((entry) => { + const source: ImprovementProfilePopulationCandidateSource = { + candidateDigest: entry.candidateDigest, + ...(entry.observation ? { observation: entry.observation } : {}), + lineage: + entry.lineageNodes.length > 0 && graph + ? { + status: 'available', + artifact: { + path: graph.summary.path, + sha256: graph.summary.sha256, + }, + nodes: entry.lineageNodes, + } + : { + status: 'unavailable', + reason: 'optimizer-did-not-report-candidate-lineage', + }, + } + const surfaceDigest = interfaceCandidateDigest(entry.value) + let candidateProfile: AgentProfile + try { + candidateProfile = materializeProfile(entry.value) + } catch (error) { + refusedCandidates += 1 + return { + status: 'refused' as const, + source, + value: entry.value, + surfaceDigest, + error: materializationError(error), + } + } + + const profileDigest = interfaceCandidateDigest(candidateProfile) + const diffs = diffAgentProfiles(baselineProfile, candidateProfile) + const reproduced = diffs.reduce(applyAgentProfileDiff, baselineProfile) + if (interfaceCandidateDigest(reproduced) !== profileDigest) { + throw new ConfigError( + `improve(): Interface profile diffs do not reproduce optimizer candidate ${entry.candidateDigest}`, + ) + } + materializedCandidates += 1 + return { + status: 'materialized' as const, + source, + value: entry.value, + surfaceDigest, + profile: candidateProfile, + profileDigest, + diffs, + diffDigests: diffs.map(interfaceCandidateDigest), + } + }) + + return immutableCandidateValue({ + status: 'available', + source: { + ...(observations + ? { + observations: { + path: observations.summary.path, + sha256: observations.summary.sha256, + }, + } + : {}), + ...(graph + ? { + gepaCandidateGraph: { + path: graph.summary.path, + sha256: graph.summary.sha256, + bestIndex: graph.bestIndex, + }, + } + : {}), + }, + uniqueCandidates: entries.size, + observedCandidates: observations?.candidates.length ?? 0, + gepaCandidateNodes: graph?.candidates.length ?? 0, + materializedCandidates, + refusedCandidates, + candidates, + }) +} + export async function runMethodImprovement( profile: AgentProfile, opts: ImproveMethodOptions, @@ -228,6 +447,13 @@ export async function runMethodImprovement minimumLift ? 'ship' : 'hold', lift: score.lift, liftInterval: { ...score.liftCi }, + candidatePopulation, cost, durationMs: Date.now() - startedAt, lineage: Object.freeze({ diff --git a/src/improvement/official-optimizers.test.ts b/src/improvement/official-optimizers.test.ts index 5b618094..03e16e8b 100644 --- a/src/improvement/official-optimizers.test.ts +++ b/src/improvement/official-optimizers.test.ts @@ -6,6 +6,7 @@ import type { OpenAICompatibleOptimizerModel } from '@tangle-network/agent-eval/ import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import { type AgentProfile, + applyAgentProfileDiff, canonicalCandidateDigest, defineAgentProfilePublicConfig, } from '@tangle-network/agent-interface' @@ -115,8 +116,11 @@ function fakeRunner( exampleId: string responsePath: string candidate?: string + bestCandidate?: string + allowFailure?: boolean }, ) { + const bestCandidate = callback?.bestCandidate ?? callback?.candidate ?? 'improved prompt' const optimizerSource = { package: optimizer, version: optimizer === 'gepa' ? 'test' : '0.2.0', @@ -126,7 +130,7 @@ function fakeRunner( const output = optimizer === 'gepa' ? [ - ` bestCandidate: ${JSON.stringify(callback?.candidate ?? 'improved prompt')},`, + ' bestCandidate,', ' bestScore: 1,', ` totalEvaluations: ${callback ? 1 : 0},`, ' recipeKind: input.recipe.kind,', @@ -135,7 +139,7 @@ function fakeRunner( ' upstream: optimizerSource,', ] : [ - ` bestCandidate: ${JSON.stringify(callback?.candidate ?? 'improved prompt')},`, + ' bestCandidate,', ' bestScore: 1,', ` totalEvaluations: ${callback ? 1 : 0},`, ' totalSteps: 0,', @@ -144,11 +148,13 @@ function fakeRunner( ] const source = [ "const fs = require('node:fs')", + "const crypto = require('node:crypto')", ';(async () => {', "const inputPath = process.argv[process.argv.indexOf('--input') + 1]", "const outputPath = process.argv[process.argv.indexOf('--output') + 1]", 'const input = JSON.parse(fs.readFileSync(inputPath, "utf8"))', `const optimizerSource = ${JSON.stringify(optimizerSource)}`, + `const bestCandidate = ${JSON.stringify(bestCandidate)}`, 'if (input.operation === "inspect") {', ' fs.writeFileSync(outputPath, JSON.stringify({ runtime: {', ' python: { implementation: "CPython", version: "3.12.0" },', @@ -169,15 +175,25 @@ function fakeRunner( ' },', ` body: JSON.stringify({ candidate: ${JSON.stringify(callback.candidate ?? 'improved prompt')}, exampleId: ${JSON.stringify(callback.exampleId)} }),`, '})', - 'if (!callbackResponse.ok) throw new Error("callback failed: " + callbackResponse.status)', + `if (!callbackResponse.ok && !${JSON.stringify(callback.allowFailure ?? false)}) throw new Error("callback failed: " + callbackResponse.status)`, 'const callbackBody = await callbackResponse.json()', `fs.writeFileSync(${JSON.stringify(callback.responsePath)}, JSON.stringify(callbackBody))`, ] : []), + 'let candidatePopulation', + 'if (optimizerSource.package === "gepa" && input.recipe.kind === "engine" && input.recipe.run.engine === "gepa") {', + ' fs.mkdirSync(input.outputDir, { recursive: true })', + ' const populationPath = input.outputDir + "/candidate-population-test.json"', + ' const scenarioIds = (input.selectionSet.length ? input.selectionSet : input.trainSet).map((scenario) => scenario.id)', + ` const populationContents = JSON.stringify({ schemaVersion: 1, scope: "gepa-candidate-population", runId: input.runId, bestIndex: 0, candidates: [{ index: 0, candidate: bestCandidate, parentIndices: [null], aggregateScore: 1, selectionScores: [{ scenarioId: scenarioIds[0], score: 1 }], discoveryEvaluationCount: ${callback ? 1 : 0} }] }, null, 2) + "\\n"`, + ' fs.writeFileSync(populationPath, populationContents)', + ' candidatePopulation = { scope: "gepa-candidate-population", path: populationPath, sha256: "sha256:" + crypto.createHash("sha256").update(populationContents).digest("hex"), bytes: Buffer.byteLength(populationContents), runId: input.runId, candidates: 1, bestIndex: 0, maxCandidates: input.maxPopulationCandidates, maxCandidateChars: input.maxCandidateChars, scenarioIds, surfaceKind: typeof input.seedCandidate === "string" ? "text" : "components" }', + '}', 'fs.writeFileSync(outputPath, JSON.stringify({', ...output, ' runId: input.runId,', ' resumed: false,', + ' ...(candidatePopulation ? { candidatePopulation } : {}),', '}))', '})().catch((error) => { process.stderr.write(String(error?.stack ?? error)); process.exit(1) })', ].join('\n') @@ -341,6 +357,26 @@ describe('official optimizer methods', () => { expect(result.raw.best.provenance?.bridge?.version).toBe('0.126.1') expect(result.decision).toBe('ship') expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') + expect(result.candidatePopulation).toMatchObject({ + status: 'available', + uniqueCandidates: 1, + observedCandidates: 0, + gepaCandidateNodes: 1, + materializedCandidates: 1, + refusedCandidates: 0, + candidates: [ + { + status: 'materialized', + value: 'improved prompt', + source: { + lineage: { status: 'available', nodes: [{ index: 0, parentIndices: [null] }] }, + }, + }, + ], + }) + if (result.candidatePopulation.status === 'available') { + expect(result.candidatePopulation.candidates[0]?.source.observation).toBeUndefined() + } }) it('changes upstream identity when feedback transformation logic changes', async () => { @@ -519,6 +555,104 @@ describe('official optimizer methods', () => { expect(JSON.stringify(observedInput)).not.toContain('SECRET') expect(JSON.stringify(observedResponse)).not.toContain('SECRET') expect(result.provenance?.evaluationCount).toBe(1) + expect(result.candidatePopulation).toMatchObject({ + status: 'available', + uniqueCandidates: 1, + observedCandidates: 1, + gepaCandidateNodes: 1, + materializedCandidates: 1, + refusedCandidates: 0, + }) + if (result.candidatePopulation.status !== 'available') { + throw new Error('expected the official optimizer candidate population') + } + const [candidate] = result.candidatePopulation.candidates + expect(candidate?.status).toBe('materialized') + if (candidate?.status !== 'materialized') { + throw new Error('expected one materialized candidate') + } + expect(candidate.profile.prompt?.systemPrompt).toBe('improved prompt') + expect(candidate.source.candidateDigest).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(candidate.source.observation?.artifact).toEqual( + result.candidatePopulation.source.observations, + ) + expect(candidate.source.lineage).toMatchObject({ + status: 'available', + nodes: [{ index: 0, parentIndices: [null], aggregateScore: 1 }], + }) + expect(candidate.diffDigests).toEqual(candidate.diffs.map(canonicalCandidateDigest)) + const reproduced = candidate.diffs.reduce(applyAgentProfileDiff, profile) + expect(canonicalCandidateDigest(reproduced)).toBe(candidate.profileDigest) + expect(Object.isFrozen(result.candidatePopulation)).toBe(true) + expect(Object.isFrozen(candidate.profile)).toBe(true) + expect(Object.isFrozen(candidate.diffs)).toBe(true) + }) + + it('reports every submitted candidate that profile materialization refuses', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-refused-input.json') + const observedResponsePath = join(root, 'observed-refused-response.json') + const mcpProfile: AgentProfile = { ...profile, mcp: {} } + const result = await improve(mcpProfile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent MCP configuration.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + authorizeSensitiveCandidate: (input) => + input.surface === 'mcp' && + input.sensitivePaths.length === 1 && + input.sensitivePaths[0] === '$' && + (input.isBaseline + ? Object.keys(input.profile.mcp ?? {}).length === 0 + : input.candidateSurface === 'not-json' || input.candidateSurface === '{}'), + runner: fakeRunner('gepa', observedInputPath, { + exampleId: 'train', + responsePath: observedResponsePath, + candidate: 'not-json', + bestCandidate: '{}', + allowFailure: true, + }), + }), + ), + surface: 'mcp', + runDir: join(root, 'run'), + }) + + expect(result.candidatePopulation).toMatchObject({ + status: 'available', + uniqueCandidates: 2, + observedCandidates: 1, + gepaCandidateNodes: 1, + materializedCandidates: 1, + refusedCandidates: 1, + }) + if (result.candidatePopulation.status !== 'available') { + throw new Error('expected the official optimizer candidate population') + } + expect(result.candidatePopulation.candidates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: 'refused', + value: 'not-json', + error: expect.objectContaining({ name: 'ConfigError' }), + }), + expect.objectContaining({ status: 'materialized', value: '{}' }), + ]), + ) + const refused = result.candidatePopulation.candidates.find( + (candidate) => candidate.status === 'refused', + ) + expect(refused?.source.lineage).toEqual({ + status: 'unavailable', + reason: 'optimizer-did-not-report-candidate-lineage', + }) + expect(JSON.parse(readFileSync(observedResponsePath, 'utf8'))).toEqual({ + error: 'evaluation failed', + }) }) it('applies a caller redactor to arbitrary PII in supplied descriptors', async () => { diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index dc3828da..77b0a7bc 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -25,7 +25,7 @@ import { withMethodRuntimeControls } from './method-controls' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.10"`' +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.144.12"`' const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 91c1fdfb..21eae90c 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -344,6 +344,21 @@ export { type ResolveSandboxClientOptions, resolveSandboxClient, } from './resolve-sandbox-client' +export { + type NativeContextContinuationExecution, + type NativeContextContinuationInput, + type ReconnectRetainedRunOptions, + type RetainedRunCancellation, + type RetainedRunCancelOptions, + type RetainedRunEffect, + type RetainedRunEventOptions, + type RetainedRunHandle, + type RetainedRunReplayPoint, + type RetainedRunSnapshot, + reconnectRetainedRun, + type StartRetainedRunOptions, + startRetainedRun, +} from './retained-run' // Router requests are an internal transport adapter. Public execution always enters through an // exact AgentProfile (`createExecutor` + `streamAgentTurn`); callers may configure only the // endpoint/auth transport used by that path. diff --git a/src/runtime/retained-run-binding.ts b/src/runtime/retained-run-binding.ts new file mode 100644 index 00000000..ea787813 --- /dev/null +++ b/src/runtime/retained-run-binding.ts @@ -0,0 +1,328 @@ +import { + type AgentExactRunControlRef, + AgentExactRunControlRefSchema, + type AgentRunCancellationAcknowledgement, + AgentRunCancellationAcknowledgementSchema, + type AgentRunCancellationRequest, + AgentRunCancellationRequestSchema, + agentRunCancellationAcknowledgementMatchesRequest, + agentRunCancellationRequestDigest, + canonicalCandidateDigest, + type InteractionResponseCommand, + type NativeContextBoundaryProof, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironment, + AgentEnvironmentEvent, + AgentSession, + AgentSessionStatus, + AgentTurnResult, +} from '@tangle-network/agent-interface/environment-provider' + +export function exactSession( + environment: AgentEnvironment, + controlRef: AgentExactRunControlRef, +): { session: AgentSession; controlRef: AgentExactRunControlRef } { + if (environment.id !== controlRef.environmentId || environment.provider !== controlRef.provider) { + throw new Error('provider reconstructed a different retained environment') + } + if (!environment.session || !controlRef.sessionId) { + throw new Error('retained run control requires a provider session id') + } + const session = environment.session(controlRef.sessionId, { controlRef }) + if (session.id !== controlRef.sessionId) { + throw new Error('provider reconstructed a different session') + } + if (session.controlRef === undefined) { + throw new Error('provider session did not return its provider-owned run control reference') + } + const currentControlRef = AgentExactRunControlRefSchema.parse(session.controlRef) + if (!sameControlCoordinates(currentControlRef, controlRef)) { + throw new Error('provider session reconstructed a different retained session') + } + if (canonicalCandidateDigest(currentControlRef) !== canonicalCandidateDigest(controlRef)) { + throw new Error('provider session did not retain the exact run control reference') + } + return { session, controlRef: currentControlRef } +} + +export function sameControlCoordinates( + left: AgentExactRunControlRef, + right: AgentExactRunControlRef, +): boolean { + return canonicalCandidateDigest(left) === canonicalCandidateDigest(right) +} + +export function freezeControlRef(controlRef: AgentExactRunControlRef): AgentExactRunControlRef { + return Object.freeze({ ...AgentExactRunControlRefSchema.parse(controlRef) }) +} + +export function copyControlRef(controlRef: AgentExactRunControlRef): AgentExactRunControlRef { + return { ...controlRef } +} + +export function exactControlRef( + candidate: unknown, + expected: { provider: string; environmentId: string; sessionId: string }, +): AgentExactRunControlRef { + if (!candidate) throw new Error('provider dispatch returned no durable run control reference') + const controlRef = AgentExactRunControlRefSchema.parse(candidate) + if ( + controlRef.provider !== expected.provider || + controlRef.environmentId !== expected.environmentId || + controlRef.sessionId !== expected.sessionId + ) { + throw new Error('provider dispatch returned control coordinates for another run') + } + return controlRef +} + +export function exactContinuedControlRef( + candidate: AgentExactRunControlRef, + initial: AgentExactRunControlRef, +): AgentExactRunControlRef { + const controlRef = AgentExactRunControlRefSchema.parse(candidate) + if ( + controlRef.provider !== initial.provider || + controlRef.environmentId !== initial.environmentId || + controlRef.sessionId !== initial.sessionId + ) { + throw new Error('provider continuation advanced to another retained session') + } + if (controlRef.runId === initial.runId && controlRef.executionId === initial.executionId) { + throw new Error('provider continuation did not return a new exact run execution') + } + return controlRef +} + +export function assertInteractionBinding( + controlRef: AgentExactRunControlRef, + command: InteractionResponseCommand, +): void { + if ( + command.binding.runId !== controlRef.runId || + command.binding.provider !== controlRef.provider || + command.binding.environmentId !== controlRef.environmentId || + command.binding.sessionId !== controlRef.sessionId || + command.binding.executionId !== controlRef.executionId || + command.binding.requestDigest !== controlRef.requestDigest + ) { + throw new Error('interaction response command does not target this retained run') + } +} + +export function assertBoundaryBinding( + controlRef: AgentExactRunControlRef, + proof: NativeContextBoundaryProof, +): void { + if ( + proof.runId !== controlRef.runId || + proof.provider !== controlRef.provider || + proof.environmentId !== controlRef.environmentId || + proof.sessionId !== controlRef.sessionId || + proof.executionId !== controlRef.executionId || + proof.requestDigest !== controlRef.requestDigest + ) { + throw new Error('native context proof does not identify this retained run') + } +} + +export function assertSequence(value: unknown, label: string): asserts value is number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`${label} must be a non-negative safe integer`) + } +} + +export function assertResultBinding( + controlRef: AgentExactRunControlRef, + result: AgentTurnResult, +): void { + if (result.sessionId !== controlRef.sessionId) { + throw new Error('provider returned a result for another retained session') + } + const metadata = result.metadata + if (controlRef.executionId !== undefined && metadata?.executionId !== controlRef.executionId) { + throw new Error('provider returned a result for another retained execution') + } + if (metadata?.runId !== controlRef.runId) { + throw new Error('provider returned a result for another retained run') + } + if (metadata?.requestDigest !== controlRef.requestDigest) { + throw new Error('provider returned a result for another retained request') + } +} + +interface ExactCancelOptions { + readonly signal?: AbortSignal +} + +export class RetainedRunProviderContractError extends Error { + constructor(message: string) { + super(message) + this.name = 'RetainedRunProviderContractError' + } +} + +export function cancellationRequest( + controlRef: AgentExactRunControlRef, + operationId: string, + reason?: string, +): AgentRunCancellationRequest { + const material = { + operationId, + run: AgentExactRunControlRefSchema.parse(controlRef), + ...(reason === undefined ? {} : { reason }), + } + return AgentRunCancellationRequestSchema.parse({ + ...material, + requestDigest: agentRunCancellationRequestDigest(material), + }) +} + +export async function exactSessionCancel( + session: AgentSession, + request: AgentRunCancellationRequest, + options: ExactCancelOptions, +): Promise { + if (!session.cancelRun) { + throw new Error('provider session does not expose durable cancellation operations') + } + let acknowledgement: unknown + try { + acknowledgement = await awaitAbortable( + Promise.resolve().then(() => session.cancelRun!(request, options)), + options.signal, + ) + } catch (error) { + if (error instanceof RetainedRunProviderContractError) throw error + throw error + } + let exactAcknowledgement: AgentRunCancellationAcknowledgement + try { + exactAcknowledgement = AgentRunCancellationAcknowledgementSchema.parse(acknowledgement) + } catch { + throw new RetainedRunProviderContractError( + 'provider returned an invalid retained cancellation acknowledgement', + ) + } + if (!agentRunCancellationAcknowledgementMatchesRequest(request, exactAcknowledgement)) { + throw new RetainedRunProviderContractError( + 'provider returned a retained cancellation acknowledgement for another request', + ) + } + return exactAcknowledgement +} + +export function hasDurableCancel(session: AgentSession): boolean { + return typeof session.cancelRun === 'function' +} + +export async function awaitAbortable( + work: PromiseLike, + signal: AbortSignal | undefined, +): Promise { + if (signal === undefined) return await work + if (signal.aborted) { + void Promise.resolve(work).catch(() => undefined) + throw abortError(signal.reason) + } + return await new Promise((resolve, reject) => { + let settled = false + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + queueMicrotask(() => { + if (settled) return + settled = true + cleanup() + reject(abortError(signal.reason)) + }) + } + signal.addEventListener('abort', onAbort, { once: true }) + Promise.resolve(work).then( + (value) => { + if (settled) return + settled = true + cleanup() + resolve(value) + }, + (error) => { + if (settled) return + settled = true + cleanup() + reject(error) + }, + ) + }) +} + +export async function exactSessionResult(session: AgentSession): Promise { + return session.result() +} + +export function assertEventBinding( + source: AgentEnvironmentEvent, + controlRef: AgentExactRunControlRef, +): void { + const transport = isRecord(source) ? source : undefined + const record = isRecord(source.data) ? source.data : undefined + const providerEvent = isRecord(source.providerEvent) ? source.providerEvent : undefined + const values = [transport, record, providerEvent].filter( + (value): value is Record => value !== undefined, + ) + for (const value of values) { + if (value.runId !== undefined && value.runId !== controlRef.runId) { + throw new Error('provider returned an event for another retained run') + } + if (value.executionId !== undefined && value.executionId !== controlRef.executionId) { + throw new Error('provider returned an event for another retained execution') + } + if (value.sessionId !== undefined && value.sessionId !== controlRef.sessionId) { + throw new Error('provider returned an event for another retained session') + } + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function assertStableText(value: string, label: string): void { + if (value.length === 0 || value.trim() !== value) { + throw new Error(`${label} must be non-empty and have no outer whitespace`) + } +} + +export function abortError(reason: unknown): Error { + const error = new Error(reason === undefined ? 'aborted' : String(reason)) + error.name = 'AbortError' + return error +} + +export function assertWaitDuration(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${label} must be a non-negative safe integer`) + } +} + +export function isTerminalSessionStatus(status: AgentSessionStatus | null): boolean { + return ( + status === 'completed' || status === 'failed' || status === 'cancelled' || status === 'expired' + ) +} + +export function delay(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(abortError(signal.reason)) + return new Promise((resolveDelay, rejectDelay) => { + const timer = setTimeout(finish, ms) + function finish() { + signal?.removeEventListener('abort', onAbort) + resolveDelay() + } + function onAbort() { + clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + rejectDelay(abortError(signal?.reason)) + } + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} diff --git a/src/runtime/retained-run-events.ts b/src/runtime/retained-run-events.ts new file mode 100644 index 00000000..408fb25e --- /dev/null +++ b/src/runtime/retained-run-events.ts @@ -0,0 +1,115 @@ +import { + type AgentExactRunControlRef, + type RuntimeEventEnvelope, + RuntimeEventEnvelopeSchema, + type StreamEvent, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironmentEvent, + AgentSession, +} from '@tangle-network/agent-interface/environment-provider' +import { + abortError, + assertEventBinding, + assertSequence, + assertStableText, + awaitAbortable, +} from './retained-run-binding' +import type { RetainedRunEventOptions } from './retained-run-types' +import { extractTransportEventIdentity, parseCanonicalTransportEvent } from './sandbox-events' + +export async function* retainedRunEvents( + session: AgentSession, + controlRef: AgentExactRunControlRef, + options: RetainedRunEventOptions | undefined, + now: () => number, +): AsyncGenerator { + const after = options?.after + if (after) { + assertStableText(after.cursor, 'replay cursor') + assertSequence(after.sequence, 'replay sequence') + } + let nextSequence = after === undefined ? 1 : after.sequence + 1 + let lastSequence = after?.sequence ?? -1 + const seen = new Set() + let firstAfterEvent = after !== undefined + const iterator = session + .events({ + ...(after === undefined ? {} : { since: after.cursor }), + ...(controlRef.executionId === undefined ? {} : { executionId: controlRef.executionId }), + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }) + [Symbol.asyncIterator]() + try { + for (;;) { + const next = await awaitAbortable( + Promise.resolve().then(() => iterator.next()), + options?.signal, + ) + if (next.done) break + const source = next.value + const identity = extractTransportEventIdentity(source) + const sourceCursor = identity.cursor ?? identity.eventId + assertEventBinding(source, controlRef) + if (sourceCursor === after?.cursor) continue + const event = canonicalEvent(source) + if (!event) continue + if (firstAfterEvent && after !== undefined) { + if (identity.sequence !== undefined && identity.sequence <= after.sequence) { + throw new Error( + 'provider replay did not prove that the first event follows the requested cursor', + ) + } + firstAfterEvent = false + } + const eventId = identity.eventId ?? identity.cursor + if (!eventId) { + throw new Error('replayable canonical event has no stable provider event id or cursor') + } + assertStableText(eventId, 'provider event id') + if (seen.has(eventId)) throw new Error(`provider replay repeated event id "${eventId}"`) + seen.add(eventId) + const sourceSequence = identity.sequence + const sequence = sourceSequence ?? nextSequence + if (sequence <= lastSequence) { + throw new Error( + `provider event sequence is not monotonic: ${sequence} follows ${lastSequence}`, + ) + } + const occurredAt = identity.occurredAt + const envelope = RuntimeEventEnvelopeSchema.parse({ + runId: controlRef.runId, + eventId, + sequence, + cursor: identity.cursor ?? eventId, + ...(occurredAt === undefined ? {} : { occurredAt }), + receivedAt: new Date(now()).toISOString(), + event, + }) + yield envelope + lastSequence = sequence + nextSequence = sequence + 1 + } + } catch (error) { + if (options?.signal?.aborted) { + try { + void Promise.resolve(iterator.return?.()).catch(() => undefined) + } catch { + // The caller's abort is the only observable result of an interrupted read. + } + throw abortError(options.signal.reason) + } + throw error + } finally { + if (!options?.signal?.aborted) await iterator.return?.() + } +} + +function canonicalEvent(source: AgentEnvironmentEvent): StreamEvent | undefined { + return parseCanonicalTransportEvent( + source.type, + source.data, + source.normalized ?? source.data.normalized, + 'provider', + ) +} diff --git a/src/runtime/retained-run-handle.ts b/src/runtime/retained-run-handle.ts new file mode 100644 index 00000000..fd0fe15f --- /dev/null +++ b/src/runtime/retained-run-handle.ts @@ -0,0 +1,262 @@ +import { + type AgentExactRunControlRef, + AgentNativeContextContinuationResultSchema, + AgentTurnResultSchema, + agentNativeContextContinuationResultMatchesRequest, + canonicalCandidateDigest, + type InteractionAcknowledgement, + InteractionAcknowledgementSchema, + InteractionResponseCommandSchema, + type NativeContextBoundaryProof, + NativeContextBoundaryProofSchema, + NativeContextContinuationRequestSchema, + nativeContextContinuationTurnDigest, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironment, + AgentEnvironmentCapabilities, + AgentSession, + AgentSessionStatus, +} from '@tangle-network/agent-interface/environment-provider' +import { + abortError, + assertBoundaryBinding, + assertInteractionBinding, + assertResultBinding, + assertStableText, + assertWaitDuration, + awaitAbortable, + cancellationRequest, + copyControlRef, + delay, + exactContinuedControlRef, + exactSessionCancel, + exactSessionResult, + freezeControlRef, + hasDurableCancel, + isTerminalSessionStatus, + RetainedRunProviderContractError, + sameControlCoordinates, +} from './retained-run-binding' +import { retainedRunEvents } from './retained-run-events' +import type { + NativeContextContinuationExecution, + RetainedRunCancellation, + RetainedRunEffect, + RetainedRunHandle, + RetainedRunSnapshot, +} from './retained-run-types' + +export function createRetainedRunHandle( + environment: AgentEnvironment, + session: AgentSession, + initialControlRef: AgentExactRunControlRef, + capabilities: AgentEnvironmentCapabilities, + now: (() => number) | undefined, +): RetainedRunHandle { + const clock = now ?? Date.now + let activeControlRef = freezeControlRef(initialControlRef) + const snapshot = async (reason?: string, signal?: AbortSignal): Promise => { + if (signal?.aborted) throw abortError(signal.reason) + let status: AgentSessionStatus | null + try { + status = await awaitAbortable( + Promise.resolve().then(() => session.status({ signal })), + signal, + ) + } catch { + if (signal?.aborted) throw abortError(signal.reason) + status = 'unknown' + } + const effect: RetainedRunEffect = + status === 'cancelled' + ? 'cancelled' + : status === 'completed' || status === 'failed' || status === 'expired' + ? 'not_live' + : 'unknown' + return { + runId: activeControlRef.runId, + controlRef: copyControlRef(activeControlRef), + status, + effect, + observedAt: new Date(clock()).toISOString(), + ...(reason === undefined ? {} : { reason }), + ...(signal?.aborted ? { signal: String(signal.reason ?? 'aborted') } : {}), + } + } + return { + get controlRef() { + return copyControlRef(activeControlRef) + }, + status: async (options) => { + const waitMs = options?.waitMs ?? 0 + assertWaitDuration(waitMs, 'retained status wait') + const initial = await snapshot(undefined, options?.signal) + if (waitMs === 0 || isTerminalSessionStatus(initial.status)) return initial + const deadline = Date.now() + waitMs + let current = initial + while (Date.now() < deadline) { + await delay(Math.min(25, Math.max(1, deadline - Date.now())), options?.signal) + current = await snapshot(undefined, options?.signal) + if (current.status !== initial.status || isTerminalSessionStatus(current.status)) { + return current + } + } + return current + }, + events: (options) => + retainedRunEvents(session, copyControlRef(activeControlRef), options, clock), + result: async () => { + const result = AgentTurnResultSchema.parse(await exactSessionResult(session)) + assertResultBinding(activeControlRef, result) + return structuredClone(result) + }, + async respondToInteraction(command, options): Promise { + const exactCommand = InteractionResponseCommandSchema.parse(command) + assertInteractionBinding(activeControlRef, exactCommand) + if (capabilities.interactions?.responseIdempotency !== true) { + throw new Error( + `provider "${activeControlRef.provider}" does not promise retry-safe interaction responses`, + ) + } + const respond = session.respondToInteraction ?? environment.respondToInteraction + if (!respond) { + throw new Error( + `provider "${activeControlRef.provider}" does not support interaction responses`, + ) + } + const acknowledgement = InteractionAcknowledgementSchema.parse( + await awaitAbortable( + Promise.resolve().then(() => + respond.call( + session.respondToInteraction ? session : environment, + exactCommand, + options, + ), + ), + options?.signal, + ), + ) + if ( + acknowledgement.operationId !== exactCommand.operationId || + acknowledgement.commandDigest !== exactCommand.commandDigest || + canonicalCandidateDigest(acknowledgement.binding) !== + canonicalCandidateDigest(exactCommand.binding) + ) { + throw new Error('provider returned an interaction acknowledgement for another command') + } + return structuredClone(acknowledgement) + }, + async contextBoundary(options): Promise { + if (!session.contextBoundary) return null + const proof = await awaitAbortable( + Promise.resolve().then(() => session.contextBoundary!(options)), + options?.signal, + ) + if (proof === null) return null + const exactProof = NativeContextBoundaryProofSchema.parse(proof) + assertBoundaryBinding(activeControlRef, exactProof) + return structuredClone(exactProof) + }, + async continueNative(request, turn): Promise { + const exactRequest = NativeContextContinuationRequestSchema.parse(request) + if (!sameControlCoordinates(exactRequest.run, activeControlRef)) { + throw new Error('native continuation request targets another retained run') + } + const { timeoutMs, signal, ...semanticTurn } = turn + if (nativeContextContinuationTurnDigest(semanticTurn) !== exactRequest.turnDigest) { + throw new Error('native continuation request targets another user turn') + } + if ( + capabilities.nativeContinuation?.atomicBoundary !== true || + capabilities.nativeContinuation.requestIdempotency !== true || + !session.continueNative + ) { + throw new Error( + `provider "${activeControlRef.provider}" does not support retry-safe native continuation`, + ) + } + const outcome = AgentNativeContextContinuationResultSchema.parse( + await awaitAbortable( + Promise.resolve().then(() => + session.continueNative!(exactRequest, { + turn: semanticTurn, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + ...(signal === undefined ? {} : { signal }), + }), + ), + signal, + ), + ) + if ( + outcome.acknowledgement.operationId !== exactRequest.operationId || + outcome.acknowledgement.requestDigest !== exactRequest.requestDigest + ) { + throw new Error('provider returned a native continuation result for another request') + } + if (outcome.acknowledgement.actualBoundary !== undefined) { + const actualBoundary = NativeContextBoundaryProofSchema.parse( + outcome.acknowledgement.actualBoundary, + ) + assertBoundaryBinding(exactRequest.run, actualBoundary) + } + if ( + outcome.acknowledgement.status !== 'accepted' && + outcome.acknowledgement.status !== 'replayed' + ) { + return structuredClone(outcome) + } + if (!('result' in outcome) || !('controlRef' in outcome)) { + throw new Error('provider omitted the successful native continuation result') + } + if (!agentNativeContextContinuationResultMatchesRequest(exactRequest, outcome)) { + throw new Error('provider returned a native continuation result for another request') + } + const nextControlRef = exactContinuedControlRef(outcome.controlRef, activeControlRef) + assertResultBinding(nextControlRef, outcome.result) + activeControlRef = freezeControlRef(nextControlRef) + return structuredClone({ ...outcome, controlRef: copyControlRef(activeControlRef) }) + }, + async cancel(options): Promise { + assertStableText(options.operationId, 'retained cancellation operation id') + if (options.reason !== undefined) + assertStableText(options.reason, 'retained cancellation reason') + if (options.signal?.aborted) throw abortError(options.signal.reason) + if (!hasDurableCancel(session)) { + throw new Error( + `provider "${activeControlRef.provider}" does not expose durable cancellation operations`, + ) + } + const request = cancellationRequest(activeControlRef, options.operationId, options.reason) + let effect: RetainedRunEffect = 'unknown' + let providerStatus: RetainedRunCancellation['status'] | undefined + try { + const acknowledgement = await exactSessionCancel(session, request, { + signal: options.signal, + }) + effect = acknowledgement.effect + providerStatus = acknowledgement.status + } catch (error) { + if (error instanceof RetainedRunProviderContractError) throw error + if (error instanceof Error && error.name === 'AbortError') throw error + effect = 'unknown' + } + const current = await snapshot(options.reason, options.signal) + const observedEffect = + effect === 'unknown' && providerStatus === undefined && current.status === 'cancelled' + ? 'cancelled' + : effect + const acknowledgement: RetainedRunCancellation = { + operationId: options.operationId, + requestDigest: request.requestDigest, + status: + providerStatus ?? (observedEffect === 'unknown' ? 'unknown' : ('accepted' as const)), + effect: observedEffect, + snapshot: { ...current, effect: observedEffect }, + ...(options.reason === undefined ? {} : { reason: options.reason }), + ...(options.signal?.aborted ? { signal: String(options.signal.reason ?? 'aborted') } : {}), + } + return structuredClone(acknowledgement) + }, + } +} diff --git a/src/runtime/retained-run-start.ts b/src/runtime/retained-run-start.ts new file mode 100644 index 00000000..76ff4257 --- /dev/null +++ b/src/runtime/retained-run-start.ts @@ -0,0 +1,125 @@ +import { + AgentEnvironmentCapabilitiesSchema, + AgentExactRunControlRefSchema, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironmentCapabilities, + AgentEnvironmentProvider, +} from '@tangle-network/agent-interface/environment-provider' +import { assertStableText, exactControlRef, exactSession } from './retained-run-binding' +import { createRetainedRunHandle } from './retained-run-handle' +import type { + ReconnectRetainedRunOptions, + RetainedRunHandle, + StartRetainedRunOptions, +} from './retained-run-types' +import { freshTurnInput } from './turn-input' + +/** + * Dispatch one detached, replayable run and return only after exact durable + * coordinates are confirmed by the provider. + * + * @stable + */ +export async function startRetainedRun( + options: StartRetainedRunOptions, +): Promise { + assertStableText(options.environment.idempotencyKey, 'environment idempotency key') + assertStableText(options.turn.turnId, 'turn idempotency key') + if (options.identity !== undefined) { + assertStableText(options.identity.sessionId, 'retained session id') + assertStableText(options.identity.executionId, 'retained execution id') + } + const capabilities = await assertRetainedCapabilities(options.provider) + if (!options.provider.get) { + throw new Error(`provider "${options.provider.name}" cannot reconstruct an environment by id`) + } + + const environment = await options.provider.create(options.environment) + if (!environment.dispatch || !environment.session) { + try { + await environment.destroy?.() + } catch (cleanupError) { + throw new AggregateError( + [ + new Error(`provider "${options.provider.name}" does not expose detached session control`), + cleanupError, + ], + 'retained run could not start and its unused environment could not be destroyed', + ) + } + throw new Error(`provider "${options.provider.name}" does not expose detached session control`) + } + + // Once dispatch begins, its outcome may be unknown to this process. Keep the + // idempotently-created environment so a retry or reconnect can recover the + // retained operation instead of destroying work that may already be live. + const reference = await environment.dispatch( + freshTurnInput(options.turn, { + turnId: options.turn.turnId, + detach: true, + ...(options.identity === undefined ? {} : options.identity), + }), + ) + if (reference.provider !== undefined && reference.provider !== options.provider.name) { + throw new Error('provider dispatch returned a session reference for another provider') + } + const controlRef = exactControlRef(reference.controlRef, { + provider: options.provider.name, + environmentId: environment.id, + sessionId: reference.id, + }) + const exact = exactSession(environment, controlRef) + return createRetainedRunHandle( + environment, + exact.session, + exact.controlRef, + capabilities, + options.now, + ) +} + +/** Rebuild a retained-run client without retaining any object from the starter. @stable */ +export async function reconnectRetainedRun( + options: ReconnectRetainedRunOptions, +): Promise { + const controlRef = AgentExactRunControlRefSchema.parse(options.controlRef) + if (controlRef.provider !== options.provider.name) { + throw new Error( + `run provider "${controlRef.provider}" does not match "${options.provider.name}"`, + ) + } + const capabilities = await assertRetainedCapabilities(options.provider) + if (!options.provider.get) { + throw new Error(`provider "${options.provider.name}" cannot reconstruct an environment by id`) + } + const environment = await options.provider.get(controlRef.environmentId) + if (!environment) return null + const exact = exactSession(environment, controlRef) + return createRetainedRunHandle( + environment, + exact.session, + exact.controlRef, + capabilities, + options.now, + ) +} + +export async function assertRetainedCapabilities( + provider: AgentEnvironmentProvider, +): Promise { + const capabilities = AgentEnvironmentCapabilitiesSchema.parse(await provider.capabilities()) + const retained = capabilities.retainedControl + if ( + retained?.exactRunIdentity !== true || + retained.resultIdentity !== true || + retained.eventIdentity !== true || + retained.cancellationIdempotency !== true || + !capabilities.streaming.detach || + !capabilities.streaming.replay || + !capabilities.streaming.turnIdempotency + ) { + throw new Error(`provider "${provider.name}" cannot control a retry-safe retained run`) + } + return capabilities +} diff --git a/src/runtime/retained-run-types.ts b/src/runtime/retained-run-types.ts new file mode 100644 index 00000000..f8850055 --- /dev/null +++ b/src/runtime/retained-run-types.ts @@ -0,0 +1,108 @@ +import type { + AgentExactRunControlRef, + AgentNativeContextContinuationOptions, + AgentNativeContextContinuationResult, + AgentSessionStatus, + InteractionAcknowledgement, + InteractionResponseCommand, + NativeContextBoundaryProof, + NativeContextContinuationRequest, + NativeContextContinuationTurn, + RuntimeEventEnvelope, + Sha256Digest, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironmentProvider, + AgentTurnInput, + AgentTurnResult, + CreateAgentEnvironmentInput, +} from '@tangle-network/agent-interface/environment-provider' + +/** Cursor plus runtime sequence needed to continue one ordered replay. @stable */ +export interface RetainedRunReplayPoint { + readonly cursor: string + readonly sequence: number +} + +/** Options for replaying canonical events strictly after a saved point. @stable */ +export interface RetainedRunEventOptions { + readonly after?: RetainedRunReplayPoint + readonly signal?: AbortSignal +} + +/** Effect recorded for one retained control operation. @stable */ +export type RetainedRunEffect = 'cancel_requested' | 'cancelled' | 'not_live' | 'unknown' + +/** Stable status snapshot for a retained run. @stable */ +export interface RetainedRunSnapshot { + readonly runId: string + readonly controlRef: AgentExactRunControlRef + readonly status: AgentSessionStatus | null + readonly effect: RetainedRunEffect + readonly observedAt: string + readonly reason?: string + readonly signal?: string +} + +/** Durable acknowledgement state for one retained control operation. @stable */ +export interface RetainedRunCancellation { + readonly operationId: string + readonly requestDigest: Sha256Digest + readonly status: 'accepted' | 'replayed' | 'conflict' | 'unknown' + readonly effect: RetainedRunEffect + readonly snapshot: RetainedRunSnapshot + readonly reason?: string + readonly signal?: string +} + +/** Options for an idempotent retained cancellation. @stable */ +export interface RetainedRunCancelOptions { + readonly operationId: string + readonly reason?: string + readonly signal?: AbortSignal +} + +/** Runtime controls plus the exact user turn bound into a continuation request. @stable */ +export type NativeContextContinuationInput = NativeContextContinuationTurn & + Omit + +/** Result of one verified same-session continuation. @stable */ +export type NativeContextContinuationExecution = AgentNativeContextContinuationResult + +/** Reconstructable control of one provider-retained run. @stable */ +export interface RetainedRunHandle { + readonly controlRef: AgentExactRunControlRef + status(options?: { waitMs?: number; signal?: AbortSignal }): Promise + events(options?: RetainedRunEventOptions): AsyncIterable + result(): Promise + respondToInteraction( + command: InteractionResponseCommand, + options?: { signal?: AbortSignal }, + ): Promise + contextBoundary(options?: { signal?: AbortSignal }): Promise + continueNative( + request: NativeContextContinuationRequest, + turn: NativeContextContinuationInput, + ): Promise + cancel(options: RetainedRunCancelOptions): Promise +} + +/** A retained start is retry-safe only when environment and turn keys are explicit. @stable */ +export interface StartRetainedRunOptions { + readonly provider: AgentEnvironmentProvider + readonly environment: CreateAgentEnvironmentInput & { idempotencyKey: string } + readonly turn: AgentTurnInput & { turnId: string } + /** Runtime-owned coordinates for providers that support deterministic retained dispatch. */ + readonly identity?: { + readonly sessionId: string + readonly executionId: string + } + readonly now?: () => number +} + +/** Inputs sufficient to rebuild a control client in a new process. @stable */ +export interface ReconnectRetainedRunOptions { + readonly provider: AgentEnvironmentProvider + readonly controlRef: AgentExactRunControlRef + readonly now?: () => number +} diff --git a/src/runtime/retained-run.test.ts b/src/runtime/retained-run.test.ts new file mode 100644 index 00000000..360c2de9 --- /dev/null +++ b/src/runtime/retained-run.test.ts @@ -0,0 +1,1274 @@ +import { spawn } from 'node:child_process' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + type AgentRunCancellationAcknowledgement, + type AgentRunCancellationRequest, + AgentRunCancellationRequestSchema, + interactionResponseCommandDigest, + type RuntimeEventEnvelope, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironment, + AgentEnvironmentEvent, + AgentEnvironmentProvider, + AgentSession, + AgentSessionStatus, + AgentTurnInput, +} from '@tangle-network/agent-interface/environment-provider' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { reconnectRetainedRun, startRetainedRun } from './retained-run' + +const childScript = new URL('../../tests/helpers/retained-run-child.ts', import.meta.url).pathname +const retainedRequestDigest = `sha256:${'a'.repeat(64)}` as const + +interface ChildExit { + readonly code: number | null + readonly signal: NodeJS.Signals | null + readonly stdout: string + readonly stderr: string +} + +async function runChild( + stateFile: string, + referenceFile: string, + phase: 'start' | 'reconnect', +): Promise { + return await new Promise((resolveChild, rejectChild) => { + const child = spawn( + process.execPath, + ['--import', 'tsx', childScript, stateFile, referenceFile, phase], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8').on('data', (chunk: string) => { + stdout += chunk + }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk + }) + child.once('error', rejectChild) + child.once('close', (code, signal) => resolveChild({ code, signal, stdout, stderr })) + }) +} + +describe('retained runtime run control', () => { + let directory: string + + beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'agent-runtime-retained-')) + }) + + afterEach(async () => { + await rm(directory, { recursive: true, force: true }) + }) + + it('reconstructs in a new process, resumes after an exclusive cursor, answers, and cancels', async () => { + const stateFile = join(directory, 'provider.json') + const referenceFile = join(directory, 'reference.json') + + const first = await runChild(stateFile, referenceFile, 'start') + expect(first.code, first.stderr).toBe(0) + expect(first.signal).toBeNull() + const started = JSON.parse(await readFile(`${referenceFile}.output`, 'utf8')) as { + first: { eventId: string; cursor: string; sequence: number; runId: string } + controlRef: { runId: string } + } + expect(started.first).toMatchObject({ + eventId: 'event-0', + cursor: 'event-0', + sequence: 0, + runId: started.controlRef.runId, + }) + + const second = await runChild(stateFile, referenceFile, 'reconnect') + expect(second.code, second.stderr).toBe(0) + expect(second.signal).toBeNull() + const reconnected = JSON.parse(await readFile(`${referenceFile}.output`, 'utf8')) as { + events: Array<{ eventId: string; cursor: string; sequence: number; runId: string }> + statusBefore: { status: string; runId: string; effect: string } + interaction: { status: string; operationId: string } + cancellation: { + operationId: string + status: string + effect: string + snapshot: { status: string; reason?: string } + } + continuation: { + acknowledgement: { status: string; operationId: string } + controlRef: { runId: string; executionId: string } + } + result: { text: string; success: boolean } + } + expect( + reconnected.events.map(({ eventId, cursor, sequence }) => ({ + eventId, + cursor, + sequence, + })), + ).toEqual([ + { eventId: 'event-1', cursor: 'event-1', sequence: 1 }, + { eventId: 'event-2', cursor: 'event-2', sequence: 2 }, + ]) + expect(new Set(reconnected.events.map((event) => event.runId))).toEqual( + new Set(['native-run-restart-native-operation']), + ) + expect(reconnected.statusBefore).toMatchObject({ status: 'running', effect: 'unknown' }) + expect(reconnected.interaction).toMatchObject({ + operationId: 'answer-after-restart', + status: 'accepted', + }) + expect(reconnected.cancellation).toMatchObject({ + operationId: 'restart-cancel-operation', + status: 'accepted', + effect: 'cancelled', + reason: 'test cleanup', + snapshot: { status: 'cancelled', reason: 'test cleanup' }, + }) + expect(reconnected.continuation).toMatchObject({ + acknowledgement: { + operationId: 'restart-native-operation', + status: 'replayed', + }, + controlRef: { + runId: 'native-run-restart-native-operation', + executionId: 'native-execution-restart-native-operation', + }, + }) + expect(reconnected.result).toMatchObject({ text: 'durable result', success: true }) + + const durableState = JSON.parse(await readFile(stateFile, 'utf8')) as { + environments: Record< + string, + { + sessions: Record }> + } + > + } + expect( + durableState.environments['environment-restart-proof']?.sessions['session-restart-proof'] + ?.status, + ).toBe('cancelled') + expect( + Object.keys( + durableState.environments['environment-restart-proof']?.sessions['session-restart-proof'] + ?.nativeOperations ?? {}, + ), + ).toEqual(['restart-native-operation']) + }) + + it('keeps an environment after dispatch becomes uncertain, but cleans an unused one', async () => { + let destroys = 0 + const uncertain = providerWithEnvironment({ + async dispatch() { + throw new Error('connection lost after dispatch') + }, + async destroy() { + destroys += 1 + }, + }) + await expect( + startRetainedRun({ + provider: uncertain, + environment: { profile: { name: 'worker' }, idempotencyKey: 'environment-key' }, + turn: { prompt: 'go', turnId: 'turn-key' }, + }), + ).rejects.toThrow('connection lost after dispatch') + expect(destroys).toBe(0) + + const unusable = providerWithEnvironment({ + dispatch: undefined, + session: undefined, + async destroy() { + destroys += 1 + }, + }) + await expect( + startRetainedRun({ + provider: unusable, + environment: { profile: { name: 'worker' }, idempotencyKey: 'unused-key' }, + turn: { prompt: 'go', turnId: 'unused-turn' }, + }), + ).rejects.toThrow('does not expose detached session control') + expect(destroys).toBe(1) + }) + + it('allowlists a fresh retained start when JavaScript supplies stale run fields', async () => { + const controlRef = { + runId: 'fresh-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'fresh-session', + executionId: 'fresh-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true, sessionId: controlRef.sessionId }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + let recorded: AgentTurnInput | undefined + const provider = providerWithEnvironment({ + async dispatch(input) { + recorded = input + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const staleTurn = { + prompt: 'fresh prompt', + turnId: 'fresh-turn', + runId: 'old-run', + sessionId: 'old-session', + executionId: 'old-execution', + lastEventId: 'old-event', + detach: false, + controlRef: { ...controlRef, runId: 'old-run' }, + contextTransfer: { stale: true }, + nativeContinuation: { stale: true }, + } as unknown as AgentTurnInput & { turnId: string } + + await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'fresh-environment' }, + turn: staleTurn, + }) + + expect(recorded).toEqual({ prompt: 'fresh prompt', turnId: 'fresh-turn', detach: true }) + }) + + it('injects explicit Runtime-owned identity into a retained dispatch', async () => { + const controlRef = { + runId: 'owned-execution', + provider: 'test-provider', + environmentId: 'owned-environment', + sessionId: 'owned-session', + executionId: 'owned-execution', + requestDigest: retainedRequestDigest, + } + let recorded: AgentTurnInput | undefined + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true, sessionId: controlRef.sessionId }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + id: controlRef.environmentId, + async dispatch(input) { + recorded = input + return { id: controlRef.sessionId, provider: controlRef.provider, controlRef } + }, + session: () => session, + }) + + await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: controlRef.environmentId }, + turn: { prompt: 'owned task', turnId: 'owned-turn', sessionId: 'stale-session' }, + identity: { sessionId: controlRef.sessionId, executionId: controlRef.executionId }, + }) + + expect(recorded).toEqual({ + prompt: 'owned task', + turnId: 'owned-turn', + detach: true, + sessionId: controlRef.sessionId, + executionId: controlRef.executionId, + }) + }) + + it('rejects a schema-valid result bound to another session', async () => { + const controlRef = { + runId: 'result-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'result-session', + executionId: 'result-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'foreign result', success: true, sessionId: 'foreign-session' }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + async dispatch() { + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'result-environment' }, + turn: { prompt: 'go', turnId: 'result-turn' }, + }) + + await expect(run.result()).rejects.toThrow('another retained session') + }) + + it('requires exact run and execution coordinates on reconnect and result', async () => { + const controlRef = { + runId: 'exact-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'exact-session', + executionId: 'exact-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ + text: 'foreign execution', + success: true, + sessionId: controlRef.sessionId, + metadata: { executionId: 'another-execution' }, + }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + async dispatch() { + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'exact-environment' }, + turn: { prompt: 'go', turnId: 'exact-turn' }, + }) + await expect( + reconnectRetainedRun({ + provider, + controlRef: { ...controlRef, runId: 'another-run', executionId: 'another-execution' }, + }), + ).rejects.toThrow('different retained session') + await expect(run.result()).rejects.toThrow('another retained execution') + }) + + it('acknowledges cancellation by operation id and never repeats its effect', async () => { + const controlRef = { + runId: 'cancel-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'cancel-session', + executionId: 'cancel-execution', + requestDigest: retainedRequestDigest, + } + let cancelCalls = 0 + let cancellationReason: string | undefined + let cancellationDigest: AgentRunCancellationRequest['requestDigest'] | undefined + let cancellationSeen = false + let status: AgentSessionStatus = 'running' + const session = { + id: controlRef.sessionId, + controlRef, + status: async () => status, + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true, sessionId: controlRef.sessionId }), + prompt: async () => ({ text: 'continued', success: true }), + async cancel(options?: { executionId?: string }) { + cancelCalls += 1 + expect(options).toMatchObject({ executionId: controlRef.executionId }) + status = 'cancelled' + }, + async cancelRun( + request: AgentRunCancellationRequest, + ): Promise { + const exactRequest = AgentRunCancellationRequestSchema.parse(request) + if (cancellationSeen) { + return { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + run: exactRequest.run, + status: exactRequest.reason === cancellationReason ? 'replayed' : 'conflict', + effect: exactRequest.reason === cancellationReason ? 'cancelled' : 'unknown', + ...(exactRequest.reason === cancellationReason + ? {} + : { existingRequestDigest: cancellationDigest }), + } + } + cancellationSeen = true + cancellationReason = exactRequest.reason + cancellationDigest = exactRequest.requestDigest + cancelCalls += 1 + status = 'cancelled' + return { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + run: exactRequest.run, + status: 'accepted', + effect: 'cancelled', + } + }, + } as AgentSession & { + cancelRun(request: AgentRunCancellationRequest): Promise + } + const provider = providerWithEnvironment({ + async dispatch() { + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'cancel-environment' }, + turn: { prompt: 'go', turnId: 'cancel-turn' }, + }) + const first = await run.cancel({ operationId: 'cancel-operation', reason: 'stop now' }) + const replay = await run.cancel({ operationId: 'cancel-operation', reason: 'stop now' }) + const conflict = await run.cancel({ operationId: 'cancel-operation', reason: 'different' }) + const reconnected = await reconnectRetainedRun({ provider, controlRef }) + if (!reconnected) throw new Error('expected retained run reconnection') + const crossProcessReplay = await reconnected.cancel({ + operationId: 'cancel-operation', + reason: 'stop now', + }) + expect(first).toMatchObject({ + operationId: 'cancel-operation', + status: 'accepted', + effect: 'cancelled', + reason: 'stop now', + snapshot: { status: 'cancelled', reason: 'stop now' }, + }) + expect(replay).toMatchObject({ status: 'replayed', effect: 'cancelled' }) + expect(conflict).toMatchObject({ status: 'conflict', effect: 'unknown' }) + expect(crossProcessReplay).toMatchObject({ status: 'replayed', effect: 'cancelled' }) + expect(cancelCalls).toBe(1) + }) + + it('omits an undefined cancellation reason from the durable operation digest', async () => { + const controlRef = { + runId: 'cancel-no-reason-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'cancel-no-reason-session', + executionId: 'cancel-no-reason-execution', + requestDigest: retainedRequestDigest, + } + let seenReason: string | undefined = 'sentinel' + const session = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running' as const, + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + async cancelRun(request: AgentRunCancellationRequest) { + const exactRequest = AgentRunCancellationRequestSchema.parse(request) + seenReason = exactRequest.reason + return { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + run: exactRequest.run, + status: 'accepted' as const, + effect: 'not_live' as const, + } + }, + } as AgentSession & { + cancelRun(request: AgentRunCancellationRequest): Promise + } + const provider = providerWithEnvironment({ + async dispatch() { + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'cancel-no-reason' }, + turn: { prompt: 'go', turnId: 'cancel-no-reason-turn' }, + }) + + await expect(run.cancel({ operationId: 'cancel-without-reason' })).resolves.toMatchObject({ + status: 'accepted', + effect: 'not_live', + }) + expect(seenReason).toBeUndefined() + }) + + it('rejects cancellation when the provider has no durable operation contract', async () => { + const controlRef = { + runId: 'cancel-unsupported-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'cancel-unsupported-session', + executionId: 'cancel-unsupported-execution', + requestDigest: retainedRequestDigest, + } + let legacyCancelCalls = 0 + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => { + legacyCancelCalls += 1 + }, + } + const provider = providerWithEnvironment({ + async dispatch() { + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'cancel-unsupported' }, + turn: { prompt: 'go', turnId: 'cancel-unsupported-turn' }, + }) + + await expect(run.cancel({ operationId: 'cancel-unsupported-operation' })).rejects.toThrow( + 'does not expose durable cancellation operations', + ) + expect(legacyCancelCalls).toBe(0) + }) + + it('rejects a caller-supplied control reference when reconnecting a session without provider identity', async () => { + const controlRef = { + runId: 'forged-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'forged-session', + executionId: 'forged-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ session: () => session }) + + await expect(reconnectRetainedRun({ provider, controlRef })).rejects.toThrow( + 'provider session did not return its provider-owned run control reference', + ) + }) + + it('rejects contradictory provider and environment identities', async () => { + const controlRef = { + runId: 'identity-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'identity-session', + executionId: 'identity-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const contradictoryReference = providerWithEnvironment({ + dispatch: async () => ({ + id: session.id, + provider: 'another-provider', + controlRef, + }), + session: () => session, + }) + + await expect( + startRetainedRun({ + provider: contradictoryReference, + environment: { profile: { name: 'worker' }, idempotencyKey: 'identity-environment' }, + turn: { prompt: 'go', turnId: 'identity-turn' }, + }), + ).rejects.toThrow('session reference for another provider') + + const wrongEnvironment = providerWithEnvironment({ + id: 'another-environment', + session: () => session, + }) + await expect(reconnectRetainedRun({ provider: wrongEnvironment, controlRef })).rejects.toThrow( + 'different retained environment', + ) + }) + + it('rejects a provider that cannot promise detach, replay, and turn idempotency', async () => { + let creates = 0 + const provider = providerWithEnvironment({}, false) + const originalCreate = provider.create + provider.create = async (input) => { + creates += 1 + return originalCreate(input) + } + await expect( + startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'unsafe-key' }, + turn: { prompt: 'go', turnId: 'unsafe-turn' }, + }), + ).rejects.toThrow('cannot control a retry-safe retained run') + expect(creates).toBe(0) + }) + + it('rejects legacy replay flags without exact retained identity promises', async () => { + let creates = 0 + const controlRef = { + runId: 'no-session-rebuild-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'no-session-rebuild-session', + executionId: 'no-session-rebuild-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + async dispatch() { + return { id: session.id, provider: 'test-provider', controlRef } + }, + session: () => session, + }) + const capabilities = provider.capabilities + provider.capabilities = async () => { + const { retainedControl: _retainedControl, ...legacy } = await capabilities() + return legacy + } + const originalCreate = provider.create + provider.create = async (input) => { + creates += 1 + return originalCreate(input) + } + + await expect( + startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'no-session-rebuild' }, + turn: { prompt: 'go', turnId: 'no-session-rebuild-turn' }, + }), + ).rejects.toThrow('cannot control a retry-safe retained run') + expect(creates).toBe(0) + }) + + it('waits for a status change up to the caller deadline and honors cancellation', async () => { + const controlRef = { + runId: 'status-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'status-session', + executionId: 'status-execution', + requestDigest: retainedRequestDigest, + } + let statusCalls = 0 + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => (++statusCalls < 3 ? 'running' : 'completed'), + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'status-environment' }, + turn: { prompt: 'go', turnId: 'status-turn' }, + }) + + await expect(run.status({ waitMs: 200 })).resolves.toMatchObject({ + status: 'completed', + effect: 'not_live', + }) + expect(statusCalls).toBe(3) + await expect(run.status({ waitMs: -1 })).rejects.toThrow('non-negative safe integer') + + statusCalls = 0 + const controller = new AbortController() + const pending = run.status({ waitMs: 200, signal: controller.signal }) + controller.abort('caller stopped waiting') + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + }) + + it('interrupts a provider status call that ignores the abort signal', async () => { + const controlRef = { + runId: 'status-abort-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'status-abort-session', + executionId: 'status-abort-execution', + requestDigest: retainedRequestDigest, + } + let resolveStarted!: () => void + const started = new Promise((resolve) => { + resolveStarted = resolve + }) + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => { + resolveStarted() + return await new Promise(() => {}) + }, + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'status-abort-environment' }, + turn: { prompt: 'go', turnId: 'status-abort-turn' }, + }) + const controller = new AbortController() + const pending = run.status({ signal: controller.signal }) + await started + controller.abort('caller stopped status') + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'caller stopped status', + }) + }) + + it('interrupts a durable cancellation call that ignores the abort signal', async () => { + const controlRef = { + runId: 'cancel-abort-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'cancel-abort-session', + executionId: 'cancel-abort-execution', + requestDigest: retainedRequestDigest, + } + let resolveStarted!: () => void + const started = new Promise((resolve) => { + resolveStarted = resolve + }) + let cancelCalls = 0 + const session = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running' as const, + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + async cancelRun() { + cancelCalls += 1 + resolveStarted() + return await new Promise(() => {}) + }, + } as AgentSession & { + cancelRun( + request: AgentRunCancellationRequest, + options?: { signal?: AbortSignal }, + ): Promise + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'cancel-abort-environment' }, + turn: { prompt: 'go', turnId: 'cancel-abort-turn' }, + }) + const controller = new AbortController() + const pending = run.cancel({ operationId: 'cancel-abort-operation', signal: controller.signal }) + await started + controller.abort('caller stopped cancellation') + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'caller stopped cancellation', + }) + expect(cancelCalls).toBe(1) + }) + + it('interrupts a retained event read and closes its provider iterator', async () => { + const controlRef = { + runId: 'events-abort-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'events-abort-session', + executionId: 'events-abort-execution', + requestDigest: retainedRequestDigest, + } + let resolveStarted!: () => void + const started = new Promise((resolve) => { + resolveStarted = resolve + }) + let returnCalls = 0 + const eventStream: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next: async () => { + resolveStarted() + return await new Promise>(() => {}) + }, + return: async () => { + returnCalls += 1 + return { done: true, value: undefined } + }, + } + }, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + events: () => eventStream, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'events-abort-environment' }, + turn: { prompt: 'go', turnId: 'events-abort-turn' }, + }) + const controller = new AbortController() + const iterator = run.events({ signal: controller.signal })[Symbol.asyncIterator]() + const pending = iterator.next() + await started + controller.abort('caller stopped events') + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'caller stopped events', + }) + expect(returnCalls).toBe(1) + }) + + it('does not call an interaction method unless retry safety was declared', async () => { + let responses = 0 + const controlRef = { + runId: 'undeclared-interaction-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'undeclared-interaction-session', + executionId: 'undeclared-interaction-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield* [] + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + async respondToInteraction(command) { + responses += 1 + return { + operationId: command.operationId, + binding: command.binding, + commandDigest: command.commandDigest, + status: 'accepted', + } + }, + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'interaction-environment' }, + turn: { prompt: 'go', turnId: 'interaction-turn' }, + }) + + const binding = { + runId: controlRef.runId, + provider: controlRef.provider, + environmentId: controlRef.environmentId, + sessionId: controlRef.sessionId, + executionId: controlRef.executionId, + interactionId: 'interaction-1', + requestDigest: controlRef.requestDigest, + } + const response = { id: 'interaction-1', outcome: 'accepted' as const } + await expect( + run.respondToInteraction({ + operationId: 'undeclared-interaction-operation', + binding, + commandDigest: interactionResponseCommandDigest({ binding, response }), + response, + }), + ).rejects.toThrow('does not promise retry-safe interaction responses') + expect(responses).toBe(0) + }) + + it('replays identity stored in data when the transport has no top-level id', async () => { + const controlRef = { + runId: 'fallback-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'fallback-session', + executionId: 'fallback-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: 'fallback-session', + controlRef, + status: async () => 'running', + async *events() { + yield { + type: 'status', + data: { + normalized: { type: 'status', status: 'completed' }, + eventId: 'fallback-event', + cursor: 'fallback-cursor', + sequence: 4, + occurredAt: '2026-08-02T03:00:00.000Z', + }, + } + yield { + type: 'status', + data: { + status: 'completed', + eventId: 'fallback-next-event', + cursor: 'fallback-next-cursor', + sequence: 5, + occurredAt: '2026-08-02T03:00:01.000Z', + }, + } + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'fallback-environment' }, + turn: { prompt: 'go', turnId: 'fallback-turn' }, + }) + const events: RuntimeEventEnvelope[] = [] + for await (const event of run.events()) events.push(event) + + expect(events.map(({ eventId, cursor, sequence }) => ({ eventId, cursor, sequence }))).toEqual([ + { eventId: 'fallback-event', cursor: 'fallback-cursor', sequence: 4 }, + { eventId: 'fallback-next-event', cursor: 'fallback-next-cursor', sequence: 5 }, + ]) + const replayed: RuntimeEventEnvelope[] = [] + for await (const event of run.events({ + after: { cursor: 'fallback-cursor', sequence: 4 }, + })) { + replayed.push(event) + } + expect(replayed).toHaveLength(1) + expect(replayed[0]).toMatchObject({ + eventId: 'fallback-next-event', + cursor: 'fallback-next-cursor', + sequence: 5, + }) + }) + + it('assigns contiguous positive sequences when a replayable provider omits them', async () => { + const controlRef = { + runId: 'generated-sequence-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'generated-sequence-session', + executionId: 'generated-sequence-execution', + requestDigest: retainedRequestDigest, + } + const source = [ + { + id: 'generated-event-1', + type: 'status', + data: { cursor: 'generated-cursor-1', runId: controlRef.runId }, + normalized: { type: 'status', status: 'started' }, + }, + { + id: 'generated-event-2', + type: 'status', + data: { cursor: 'generated-cursor-2', runId: controlRef.runId }, + normalized: { type: 'status', status: 'completed' }, + }, + ] satisfies AgentEnvironmentEvent[] + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'completed', + async *events(options) { + const start = options?.since === 'generated-cursor-1' ? 1 : 0 + yield* source.slice(start) + }, + result: async () => ({ + text: 'done', + success: true, + sessionId: controlRef.sessionId, + metadata: { + runId: controlRef.runId, + executionId: controlRef.executionId, + requestDigest: controlRef.requestDigest, + }, + }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'generated-sequence' }, + turn: { prompt: 'go', turnId: 'generated-sequence-turn' }, + }) + + const fresh: RuntimeEventEnvelope[] = [] + for await (const event of run.events()) fresh.push(event) + expect(fresh.map(({ eventId, sequence }) => ({ eventId, sequence }))).toEqual([ + { eventId: 'generated-event-1', sequence: 1 }, + { eventId: 'generated-event-2', sequence: 2 }, + ]) + + const replayed: RuntimeEventEnvelope[] = [] + for await (const event of run.events({ + after: { cursor: 'generated-cursor-1', sequence: 1 }, + })) { + replayed.push(event) + } + expect(replayed.map(({ eventId, sequence }) => ({ eventId, sequence }))).toEqual([ + { eventId: 'generated-event-2', sequence: 2 }, + ]) + }) + + it('validates a replay anchor before skipping it', async () => { + const controlRef = { + runId: 'anchor-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'anchor-session', + executionId: 'anchor-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield { + type: 'status', + data: { + status: 'completed', + runId: 'foreign-run', + eventId: 'anchor-event', + cursor: 'anchor-cursor', + sequence: 4, + }, + } + yield { + type: 'status', + data: { + status: 'completed', + runId: controlRef.runId, + eventId: 'next-event', + cursor: 'next-cursor', + sequence: 5, + }, + } + }, + result: async () => ({ + text: 'done', + success: true, + sessionId: controlRef.sessionId, + metadata: { runId: controlRef.runId }, + }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'anchor-environment' }, + turn: { prompt: 'go', turnId: 'anchor-turn' }, + }) + + await expect( + collectRetainedEvents(run.events({ after: { cursor: 'anchor-cursor', sequence: 4 } })), + ).rejects.toThrow('another retained run') + }) + + it('rejects retained events bound to another run on the transport object', async () => { + const controlRef = { + runId: 'transport-binding-run', + provider: 'test-provider', + environmentId: 'environment-1', + sessionId: 'transport-binding-session', + executionId: 'transport-binding-execution', + requestDigest: retainedRequestDigest, + } + const session: AgentSession = { + id: controlRef.sessionId, + controlRef, + status: async () => 'running', + async *events() { + yield { + type: 'status', + data: { status: 'completed', eventId: 'transport-binding-event' }, + runId: 'foreign-run', + } as unknown as AgentEnvironmentEvent + }, + result: async () => ({ text: 'done', success: true }), + prompt: async () => ({ text: 'continued', success: true }), + cancel: async () => {}, + } + const provider = providerWithEnvironment({ + dispatch: async () => ({ id: session.id, provider: 'test-provider', controlRef }), + session: () => session, + }) + const run = await startRetainedRun({ + provider, + environment: { profile: { name: 'worker' }, idempotencyKey: 'transport-binding' }, + turn: { prompt: 'go', turnId: 'transport-binding-turn' }, + }) + + await expect(collectRetainedEvents(run.events())).rejects.toThrow('another retained run') + }) +}) + +async function collectRetainedEvents(events: AsyncIterable): Promise { + for await (const _event of events) { + // The first event is expected to fail its retained-run binding check. + } +} + +function providerWithEnvironment( + overrides: Partial, + replay = true, +): AgentEnvironmentProvider { + const environment: AgentEnvironment = { + id: 'environment-1', + provider: 'test-provider', + status: async () => 'running', + async *stream() { + yield* [] + }, + async dispatch() { + throw new Error('dispatch should have been overridden') + }, + session() { + throw new Error('session should not be reached') + }, + ...overrides, + } + return { + name: 'test-provider', + capabilities: () => ({ + profile: { + namedProfiles: true, + systemPrompt: { replace: true, append: true }, + instructions: true, + tools: true, + permissions: true, + mcp: true, + subagents: true, + resources: { files: true, instructions: true }, + runtimeUpdate: true, + validation: true, + }, + streaming: { live: true, replay, detach: replay, turnIdempotency: replay }, + sessions: { continue: true, list: true, messages: true }, + ...(replay + ? { + retainedControl: { + exactRunIdentity: true, + resultIdentity: true, + eventIdentity: true, + cancellationIdempotency: true, + }, + } + : {}), + workspace: { + read: false, + write: false, + exec: false, + git: false, + upload: false, + download: false, + }, + branching: { checkpoint: false, fork: false }, + placement: false, + usage: false, + confidential: false, + }), + async create() { + return environment + }, + async get() { + return environment + }, + } +} diff --git a/src/runtime/retained-run.ts b/src/runtime/retained-run.ts new file mode 100644 index 00000000..c6b8323e --- /dev/null +++ b/src/runtime/retained-run.ts @@ -0,0 +1,21 @@ +/** + * Compatibility facade for retained provider runs. + * + * Public contracts stay at this import path while implementation modules own + * startup, replay, binding checks, and handle operations. + */ + +export { reconnectRetainedRun, startRetainedRun } from './retained-run-start' +export type { + NativeContextContinuationExecution, + NativeContextContinuationInput, + ReconnectRetainedRunOptions, + RetainedRunCancellation, + RetainedRunCancelOptions, + RetainedRunEffect, + RetainedRunEventOptions, + RetainedRunHandle, + RetainedRunReplayPoint, + RetainedRunSnapshot, + StartRetainedRunOptions, +} from './retained-run-types' diff --git a/src/runtime/sandbox-events.ts b/src/runtime/sandbox-events.ts index 150d503d..033c557a 100644 --- a/src/runtime/sandbox-events.ts +++ b/src/runtime/sandbox-events.ts @@ -467,3 +467,9 @@ export function mapSandboxEvent( return extractLlmCallEvent(event, opts.agentRunName ?? 'agent') } + +export { + extractTransportEventIdentity, + parseCanonicalTransportEvent, + type TransportEventIdentity, +} from './sandbox-transport-events' diff --git a/src/runtime/sandbox-transport-events.ts b/src/runtime/sandbox-transport-events.ts new file mode 100644 index 00000000..7e2f0288 --- /dev/null +++ b/src/runtime/sandbox-transport-events.ts @@ -0,0 +1,116 @@ +import { CanonicalStreamEventSchema, type StreamEvent } from '@tangle-network/agent-interface' +import { assertRuntimeTimestamp } from './timestamps' + +/** Parse canonical payload fields without treating transport identity as event data. */ +export function parseCanonicalTransportEvent( + type: unknown, + data: Record, + normalized: unknown, + source: string, +): StreamEvent | undefined { + if (!isRecord(data)) { + throw new Error(`${source} emitted a canonical event without an object payload`) + } + const outerType = String(type ?? '') + if (normalized !== undefined) { + if ( + typeof normalized === 'object' && + normalized !== null && + 'type' in normalized && + typeof normalized.type === 'string' && + normalized.type !== outerType + ) { + throw new Error( + `${source} canonical event type "${normalized.type}" does not match transport type "${outerType}"`, + ) + } + const candidate = CanonicalStreamEventSchema.safeParse(normalized) + if (!candidate.success) { + throw new Error(`${source} emitted an invalid normalized canonical event`, { + cause: candidate.error, + }) + } + return candidate.data + } + const { + type: embeddedType, + eventId: _eventId, + cursor: _cursor, + sequence: _sequence, + occurredAt: _occurredAt, + normalized: _normalized, + ...payload + } = data + if (embeddedType !== undefined && embeddedType !== outerType) { + throw new Error( + `${source} canonical event type "${String(embeddedType)}" does not match transport type "${outerType}"`, + ) + } + const candidate = CanonicalStreamEventSchema.safeParse({ ...payload, type: outerType }) + return candidate.success ? candidate.data : undefined +} + +/** Identity fields carried by either the transport envelope or its data payload. */ +export interface TransportEventIdentity { + readonly eventId?: string + readonly cursor?: string + readonly sequence?: number + readonly occurredAt?: string +} + +/** Extract shared event identity without privileging one provider wire shape. */ +export function extractTransportEventIdentity(event: unknown): TransportEventIdentity { + const record = isRecord(event) ? event : {} + const data = isRecord(record.data) ? record.data : {} + const providerEvent = isRecord(record.providerEvent) ? record.providerEvent : {} + const providerData = isRecord(providerEvent.data) ? providerEvent.data : {} + const eventId = stableString( + record.id ?? + record.eventId ?? + data.eventId ?? + providerEvent.id ?? + providerEvent.eventId ?? + providerData.eventId, + ) + const cursor = stableString( + record.cursor ?? data.cursor ?? providerEvent.cursor ?? providerData.cursor, + ) + const sequence = optionalSequence( + record.sequence ?? data.sequence ?? providerEvent.sequence ?? providerData.sequence, + ) + const occurredAt = optionalTimestamp( + record.occurredAt ?? data.occurredAt ?? providerEvent.occurredAt ?? providerData.occurredAt, + ) + return { + ...(eventId === undefined ? {} : { eventId }), + ...(cursor === undefined ? {} : { cursor }), + ...(sequence === undefined ? {} : { sequence }), + ...(occurredAt === undefined ? {} : { occurredAt }), + } +} + +function stableString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 && value.trim() === value ? value : undefined +} + +function finiteNonNegativeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined +} + +function optionalSequence(value: unknown): number | undefined { + if (value === undefined) return undefined + if (finiteNonNegativeInteger(value) === undefined) { + throw new Error('transport event sequence must be a non-negative safe integer') + } + return value as number +} + +function optionalTimestamp(value: unknown): string | undefined { + if (value === undefined) return undefined + assertRuntimeTimestamp(value, 'occurredAt') + return value +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/runtime/timestamps.ts b/src/runtime/timestamps.ts new file mode 100644 index 00000000..51263f1d --- /dev/null +++ b/src/runtime/timestamps.ts @@ -0,0 +1,29 @@ +import { RuntimeEventEnvelopeSchema } from '@tangle-network/agent-interface' + +const validTimestamp = '2026-01-01T00:00:00.000Z' + +/** Validate a timestamp with the public runtime envelope contract. */ +export function assertRuntimeTimestamp(value: unknown, label: string): asserts value is string { + try { + RuntimeEventEnvelopeSchema.parse({ + runId: 'runtime', + eventId: 'event', + sequence: 0, + occurredAt: value, + receivedAt: validTimestamp, + event: { type: 'status', status: 'processing' }, + }) + } catch (error) { + throw new Error(`${label} must be a valid ISO timestamp`, { cause: error }) + } +} + +/** Return whether a value satisfies the public runtime timestamp contract. */ +export function isRuntimeTimestamp(value: unknown): value is string { + try { + assertRuntimeTimestamp(value, 'occurredAt') + return true + } catch { + return false + } +} diff --git a/src/runtime/turn-input.ts b/src/runtime/turn-input.ts new file mode 100644 index 00000000..1f54448b --- /dev/null +++ b/src/runtime/turn-input.ts @@ -0,0 +1,36 @@ +import type { ContextTransferRequest } from '@tangle-network/agent-interface' +import type { AgentTurnInput } from '@tangle-network/agent-interface/environment-provider' + +/** + * Copy only fields that describe a new provider turn. + * + * Session, replay, continuation, and transfer coordinates are runtime-owned + * and are injected by the caller that owns that lifecycle. Keeping this copy + * explicit also makes JavaScript callers subject to the same boundary. + */ +export function freshTurnInput( + input: AgentTurnInput, + runtime: { + readonly turnId: string + readonly detach: true + readonly sessionId?: string + readonly executionId?: string + readonly contextTransfer?: ContextTransferRequest + }, +): AgentTurnInput { + const fresh: AgentTurnInput = { + ...(input.prompt === undefined ? {} : { prompt: input.prompt }), + ...(input.parts === undefined ? {} : { parts: input.parts }), + ...(input.model === undefined ? {} : { model: input.model }), + ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), + ...(input.context === undefined ? {} : { context: input.context }), + ...(input.providerOptions === undefined ? {} : { providerOptions: input.providerOptions }), + ...(input.signal === undefined ? {} : { signal: input.signal }), + turnId: runtime.turnId, + detach: runtime.detach, + ...(runtime.sessionId === undefined ? {} : { sessionId: runtime.sessionId }), + ...(runtime.executionId === undefined ? {} : { executionId: runtime.executionId }), + ...(runtime.contextTransfer === undefined ? {} : { contextTransfer: runtime.contextTransfer }), + } + return fresh +} diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index f60c3213..d21ff3dd 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:125dadc9b6ce877cbcf4821930fa68e3997f5b5a29b7a7fd43d13e230aee696d", + "digest": "sha256:d7e382f866e8b8f947210b2f8c8c52bd56ff616cb10f8e872ba94912e13165a7", "evaluation": { "decision": { "contributingChecks": [ @@ -4870,7 +4870,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.131.7" + "runtimeVersion": "0.132.1" }, "objectives": [ { @@ -4981,8 +4981,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:4c5b48bb0e2e628bb48d5ef0f1c73581a8368e319990c1d30dd169c841af5dd3", - "runId": "agent-runtime-0.131.7-proposal-fixture", + "recordDigest": "sha256:81894ad1952e77f0b1e66765854897f16538eee7a57e3e370a2d53b102fa263e", + "runId": "agent-runtime-0.132.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5009,5 +5009,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.131.7-proposal-fixture" + "runId": "agent-runtime-0.132.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index ace56b8d..b9adc00a 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:70a5e01569bf5f72d742134c45236c014dd43d2e3674d6b579112f1202143fe4", + "digest": "sha256:1637f21204f32f323a46abf1093a57fe8db5ad2fa2f918f1678937262209015b", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.131.7" + "runtimeVersion": "0.132.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:ac4dbeecf9b4080d091c2b0284685d052bd47189601fe271be48a645c93f8a7b", + "recordDigest": "sha256:d63fed2a06ba61e109301816ddc66a6915314098a67b1b881d64730093e6a867", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/helpers/durable-retained-provider.ts b/tests/helpers/durable-retained-provider.ts new file mode 100644 index 00000000..d6c35b55 --- /dev/null +++ b/tests/helpers/durable-retained-provider.ts @@ -0,0 +1,553 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { + type AgentExactRunControlRef, + type AgentNativeContextContinuationResult, + type AgentRunCancellationAcknowledgement, + type AgentRunCancellationRequest, + AgentRunCancellationRequestSchema, + canonicalCandidateDigest, + type InteractionAcknowledgement, + type InteractionResponseCommand, + interactionRequestDigest, + type NativeContextBoundaryProof, + type NativeContextContinuationRequest, + NativeContextContinuationRequestSchema, +} from '@tangle-network/agent-interface' +import type { + AgentEnvironment, + AgentEnvironmentCapabilities, + AgentEnvironmentEvent, + AgentEnvironmentProvider, + AgentNativeContextContinuationOptions, + AgentSession, + AgentSessionStatus, + AgentTurnInput, +} from '@tangle-network/agent-interface/environment-provider' + +interface StoredSession { + readonly id: string + controlRef: AgentExactRunControlRef + status: AgentSessionStatus + readonly events: AgentEnvironmentEvent[] + readonly prompts: Array> + readonly interactionOperations: Record< + string, + { digest: string; acknowledgement: InteractionAcknowledgement } + > + readonly cancellationOperations: Record< + string, + { digest: string; status: 'accepted' | 'replayed'; effect: 'cancelled' | 'not_live' } + > + readonly nativeOperations: Record< + string, + { + requestDigest: NativeContextContinuationRequest['requestDigest'] + result: AgentNativeContextContinuationResult + } + > + readonly nativeResponseLosses: Record +} + +interface StoredEnvironment { + readonly id: string + readonly sessions: Record +} + +interface DurableProviderState { + readonly environments: Record +} + +/** Test-only provider whose complete control state is reconstructed from one JSON file. */ +export function durableRetainedProvider(stateFile: string): AgentEnvironmentProvider { + const providerName = 'durable-test' + return { + name: providerName, + capabilities: retainedCapabilities, + async create(input) { + if (!input.idempotencyKey) throw new Error('durable test provider requires a create key') + const state = readState(stateFile) + const id = `environment-${input.idempotencyKey}` + state.environments[id] ??= { id, sessions: {} } + writeState(stateFile, state) + return environmentFor(stateFile, providerName, id) + }, + async get(id) { + return readState(stateFile).environments[id] + ? environmentFor(stateFile, providerName, id) + : null + }, + } +} + +function environmentFor(stateFile: string, provider: string, id: string): AgentEnvironment { + return { + id, + provider, + status: async () => 'running', + async *stream(): AsyncIterable { + yield* [] + }, + async dispatch(input) { + if (!input.turnId) throw new Error('durable test provider requires a turn key') + const state = readState(stateFile) + const environment = state.environments[id] + if (!environment) throw new Error(`missing environment ${id}`) + const sessionId = `session-${input.turnId}` + let session = environment.sessions[sessionId] + if (!session) { + const controlRef: AgentExactRunControlRef = { + runId: `run-${input.turnId}`, + provider, + environmentId: id, + sessionId, + executionId: `execution-${input.turnId}`, + requestDigest: canonicalCandidateDigest({ + environmentId: id, + sessionId, + turnId: input.turnId, + prompt: input.prompt ?? null, + parts: input.parts ?? null, + }), + } + session = { + id: sessionId, + controlRef, + status: 'running', + events: retainedEvents(controlRef), + prompts: [], + interactionOperations: {}, + cancellationOperations: {}, + nativeOperations: {}, + nativeResponseLosses: {}, + } + environment.sessions[sessionId] = session + writeState(stateFile, state) + } + return { id: session.id, provider, controlRef: session.controlRef } + }, + session(sessionId, options) { + const stored = sessionState(stateFile, id, sessionId) + if ( + options?.controlRef && + canonicalCandidateDigest(options.controlRef) !== canonicalCandidateDigest(stored.controlRef) + ) { + throw new Error('durable test provider received the wrong control reference') + } + return sessionFor(stateFile, id, stored) + }, + } +} + +function sessionFor( + stateFile: string, + environmentId: string, + initial: StoredSession, +): AgentSession { + return { + id: initial.id, + controlRef: initial.controlRef, + async status() { + return sessionState(stateFile, environmentId, initial.id).status + }, + async *events(options): AsyncIterable { + const events = sessionState(stateFile, environmentId, initial.id).events + const start = + options?.since === undefined + ? 0 + : events.findIndex((event) => event.id === options.since) + 1 + if (options?.since !== undefined && start === 0) { + throw new Error(`unknown event cursor ${options.since}`) + } + for (const event of events.slice(start)) yield structuredClone(event) + }, + async result() { + const current = sessionState(stateFile, environmentId, initial.id) + const latest = Object.values(current.nativeOperations).at(-1)?.result.controlRef + return { + text: 'durable result', + success: true, + sessionId: initial.id, + ...(latest?.executionId + ? { + metadata: { + executionId: latest.executionId, + runId: latest.runId, + requestDigest: latest.requestDigest, + }, + } + : { + metadata: { + executionId: initial.controlRef.executionId, + runId: initial.controlRef.runId, + requestDigest: initial.controlRef.requestDigest, + }, + }), + } + }, + async prompt(input) { + const state = readState(stateFile) + const session = requireSession(state, environmentId, initial.id) + session.prompts.push(serializableTurn(input)) + writeState(stateFile, state) + return { text: 'continued', success: true, sessionId: initial.id } + }, + async respondToInteraction(command) { + const state = readState(stateFile) + const session = requireSession(state, environmentId, initial.id) + const digest = canonicalCandidateDigest(command) + const existing = session.interactionOperations[command.operationId] + if (existing) { + return existing.digest === digest + ? existing.acknowledgement + : acknowledgement(command, 'already_resolved_different') + } + const exact = + (command.binding.runId === session.controlRef.runId || + command.binding.runId === + Object.values(session.nativeOperations).at(-1)?.result.controlRef.runId) && + command.binding.environmentId === environmentId && + command.binding.sessionId === initial.id && + command.binding.interactionId === 'interaction-1' + const result = acknowledgement(command, exact ? 'accepted' : 'binding_mismatch') + session.interactionOperations[command.operationId] = { digest, acknowledgement: result } + writeState(stateFile, state) + return result + }, + async contextBoundary(): Promise { + const current = sessionState(stateFile, environmentId, initial.id) + return { + runId: current.controlRef.runId, + provider: current.controlRef.provider, + environmentId, + sessionId: initial.id, + executionId: current.controlRef.executionId, + requestDigest: current.controlRef.requestDigest, + boundary: { + kind: 'messages', + messageIds: ['message-1'], + digest: `sha256:${'1'.repeat(64)}`, + }, + observedAt: '2026-08-02T00:00:03.000Z', + } + }, + async continueNative( + request, + options: AgentNativeContextContinuationOptions, + ): Promise { + const exactRequest = NativeContextContinuationRequestSchema.parse(request) + const state = readState(stateFile) + const session = requireSession(state, environmentId, initial.id) + const prior = session.nativeOperations[exactRequest.operationId] + if (prior) { + if (prior.requestDigest !== exactRequest.requestDigest) { + return { + acknowledgement: { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + status: 'conflict', + historyMessagesSent: 0, + existingRequestDigest: prior.requestDigest, + }, + } + } + return { + ...structuredClone(prior.result), + acknowledgement: { ...prior.result.acknowledgement, status: 'replayed' }, + } + } + if ( + exactRequest.run.provider !== session.controlRef.provider || + exactRequest.run.environmentId !== environmentId || + exactRequest.run.sessionId !== session.id || + exactRequest.expectedBoundary.runId !== exactRequest.run.runId || + exactRequest.expectedBoundary.provider !== session.controlRef.provider || + exactRequest.expectedBoundary.environmentId !== environmentId || + exactRequest.expectedBoundary.sessionId !== session.id + ) { + return { + acknowledgement: { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + status: 'unknown_session', + historyMessagesSent: 0, + }, + } + } + const expectedBoundaryDigest = canonicalCandidateDigest( + exactRequest.expectedBoundary.boundary, + ) + const currentBoundaryDigest = canonicalCandidateDigest({ + kind: 'messages', + messageIds: ['message-1'], + digest: `sha256:${'1'.repeat(64)}`, + }) + if (expectedBoundaryDigest !== currentBoundaryDigest) { + return { + acknowledgement: { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + status: 'boundary_mismatch', + historyMessagesSent: 0, + actualBoundary: nativeBoundaryFor(exactRequest.run), + }, + } + } + if (options.turn.prompt === undefined && options.turn.parts === undefined) { + return { + acknowledgement: { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + status: 'transport_failure', + historyMessagesSent: 0, + message: 'continuation turn is empty', + }, + } + } + const nextControlRef: AgentExactRunControlRef = { + ...session.controlRef, + runId: `native-run-${exactRequest.operationId}`, + executionId: `native-execution-${exactRequest.operationId}`, + requestDigest: exactRequest.requestDigest, + } + const result: AgentNativeContextContinuationResult = { + acknowledgement: { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + status: 'accepted', + historyMessagesSent: 0, + actualBoundary: nativeBoundaryFor(exactRequest.run), + }, + result: { + text: 'continued', + success: true, + sessionId: session.id, + metadata: { + runId: nextControlRef.runId, + executionId: nextControlRef.executionId, + requestDigest: nextControlRef.requestDigest, + }, + }, + controlRef: nextControlRef, + } + session.nativeOperations[exactRequest.operationId] = { + requestDigest: exactRequest.requestDigest, + result, + } + writeState(stateFile, state) + if ( + exactRequest.operationId === 'restart-native-operation' && + !session.nativeResponseLosses[exactRequest.operationId] + ) { + session.nativeResponseLosses[exactRequest.operationId] = true + writeState(stateFile, state) + throw new Error('connection lost after continuation commit') + } + return structuredClone(result) + }, + async cancelRun( + request: AgentRunCancellationRequest, + ): Promise { + const exactRequest = AgentRunCancellationRequestSchema.parse(request) + const state = readState(stateFile) + const session = requireSession(state, environmentId, initial.id) + const latestControlRef = + Object.values(session.nativeOperations).at(-1)?.result.controlRef ?? session.controlRef + if ( + canonicalCandidateDigest(exactRequest.run) !== canonicalCandidateDigest(latestControlRef) + ) { + return { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + run: exactRequest.run, + status: 'unknown', + effect: 'unknown', + message: 'durable test cancellation received the wrong execution', + retryable: false, + } + } + const digest = exactRequest.requestDigest + const prior = session.cancellationOperations[exactRequest.operationId] + if (prior) { + return { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + run: exactRequest.run, + status: prior.digest === digest ? 'replayed' : 'conflict', + effect: prior.digest === digest ? prior.effect : 'unknown', + } + } + const effect = session.status === 'running' ? 'cancelled' : 'not_live' + session.cancellationOperations[exactRequest.operationId] = { + digest, + status: 'accepted', + effect, + } + if (effect === 'cancelled') session.status = 'cancelled' + writeState(stateFile, state) + return { + operationId: exactRequest.operationId, + requestDigest: exactRequest.requestDigest, + run: exactRequest.run, + status: 'accepted', + effect, + } + }, + async cancel() { + const state = readState(stateFile) + requireSession(state, environmentId, initial.id).status = 'cancelled' + writeState(stateFile, state) + }, + } +} + +function acknowledgement( + command: InteractionResponseCommand, + status: InteractionAcknowledgement['status'], +): InteractionAcknowledgement { + return { + operationId: command.operationId, + binding: command.binding, + commandDigest: command.commandDigest, + status, + } +} + +function nativeBoundaryFor(controlRef: AgentExactRunControlRef): NativeContextBoundaryProof { + return { + runId: controlRef.runId, + provider: controlRef.provider, + environmentId: controlRef.environmentId, + sessionId: controlRef.sessionId, + executionId: controlRef.executionId, + requestDigest: controlRef.requestDigest, + boundary: { + kind: 'messages', + messageIds: ['message-1'], + digest: `sha256:${'1'.repeat(64)}`, + }, + observedAt: '2026-08-02T00:00:03.000Z', + } +} + +function retainedEvents(controlRef: AgentExactRunControlRef): AgentEnvironmentEvent[] { + const interaction = { + id: 'interaction-1', + kind: 'question', + title: 'Continue?', + answerSpec: { fields: [] }, + binding: { + runId: controlRef.runId, + provider: controlRef.provider, + environmentId: controlRef.environmentId, + sessionId: controlRef.sessionId, + executionId: controlRef.executionId, + interactionId: 'interaction-1', + }, + } + return [ + { + id: 'event-0', + type: 'status', + data: { sequence: 0, occurredAt: '2026-08-02T00:00:00.000Z' }, + normalized: { type: 'status', status: 'started' }, + }, + { + id: 'event-1', + type: 'interaction', + data: { sequence: 1, occurredAt: '2026-08-02T00:00:01.000Z' }, + normalized: { + type: 'interaction', + request: { + ...interaction, + requestDigest: interactionRequestDigest(interaction), + }, + }, + }, + { + id: 'event-2', + type: 'session.updated', + data: { sequence: 2, occurredAt: '2026-08-02T00:00:02.000Z' }, + normalized: { type: 'session.updated', sessionId: controlRef.sessionId }, + }, + ] +} + +function serializableTurn(input: AgentTurnInput): Record { + const { signal: _signal, controlRef, nativeContinuation, contextTransfer, ...rest } = input + return { + ...rest, + ...(controlRef === undefined ? {} : { controlRef }), + ...(nativeContinuation === undefined ? {} : { nativeContinuation }), + ...(contextTransfer === undefined ? {} : { contextTransfer }), + } +} + +function sessionState(stateFile: string, environmentId: string, sessionId: string): StoredSession { + return requireSession(readState(stateFile), environmentId, sessionId) +} + +function requireSession( + state: DurableProviderState, + environmentId: string, + sessionId: string, +): StoredSession { + const session = state.environments[environmentId]?.sessions[sessionId] + if (!session) throw new Error(`missing session ${environmentId}/${sessionId}`) + return session +} + +function readState(file: string): DurableProviderState { + if (!existsSync(file)) return { environments: {} } + return JSON.parse(readFileSync(file, 'utf8')) as DurableProviderState +} + +function writeState(file: string, state: DurableProviderState): void { + writeFileSync(file, `${JSON.stringify(state)}\n`, { encoding: 'utf8', mode: 0o600 }) +} + +function retainedCapabilities(): AgentEnvironmentCapabilities { + return { + profile: { + namedProfiles: true, + systemPrompt: { replace: true, append: true }, + instructions: true, + tools: true, + permissions: true, + mcp: true, + subagents: true, + resources: { files: true, instructions: true }, + runtimeUpdate: true, + validation: true, + }, + streaming: { live: true, replay: true, detach: true, turnIdempotency: true }, + sessions: { continue: true, list: true, messages: true }, + retainedControl: { + exactRunIdentity: true, + resultIdentity: true, + eventIdentity: true, + cancellationIdempotency: true, + }, + nativeContinuation: { atomicBoundary: true, requestIdempotency: true }, + interactions: { + kinds: ['question'], + answerFieldTypes: ['text'], + responseScopes: ['interaction'], + secretAnswers: false, + concurrentRequests: false, + replay: true, + responseIdempotency: true, + }, + workspace: { + read: false, + write: false, + exec: false, + git: false, + upload: false, + download: false, + }, + branching: { checkpoint: false, fork: false }, + placement: false, + usage: false, + confidential: false, + } +} diff --git a/tests/helpers/retained-run-child.ts b/tests/helpers/retained-run-child.ts new file mode 100644 index 00000000..daac2509 --- /dev/null +++ b/tests/helpers/retained-run-child.ts @@ -0,0 +1,124 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { + type AgentExactRunControlRef, + interactionResponseCommandDigest, + NativeContextContinuationRequestSchema, + type NativeContextContinuationTurn, + nativeContextContinuationRequestDigest, + nativeContextContinuationTurnDigest, +} from '@tangle-network/agent-interface' +import { reconnectRetainedRun, startRetainedRun } from '../../src/runtime/retained-run' +import { durableRetainedProvider } from './durable-retained-provider' + +const [stateFile, referenceFile, phase] = process.argv.slice(2) +if (!stateFile || !referenceFile || (phase !== 'start' && phase !== 'reconnect')) { + throw new Error('usage: retained-run-child ') +} + +if (phase === 'start') { + const run = await startRetainedRun({ + provider: durableRetainedProvider(stateFile), + environment: { profile: { name: 'worker' }, idempotencyKey: 'restart-proof' }, + turn: { prompt: 'start', turnId: 'restart-proof' }, + now: () => Date.parse('2026-08-02T00:00:10.000Z'), + }) + const iterator = run.events()[Symbol.asyncIterator]() + const first = await iterator.next() + if (first.done) throw new Error('retained test run emitted no event') + const continuationTurn: NativeContextContinuationTurn = { + prompt: 'continue after the client restarted', + } + const expectedBoundary = await run.contextBoundary() + if (!expectedBoundary) throw new Error('durable provider did not return a continuation boundary') + const continuationMaterial = { + operationId: 'restart-native-operation', + run: run.controlRef, + expectedBoundary, + turnDigest: nativeContextContinuationTurnDigest(continuationTurn), + } + const continuation = NativeContextContinuationRequestSchema.parse({ + requestDigest: nativeContextContinuationRequestDigest(continuationMaterial), + ...continuationMaterial, + }) + const reference = { + controlRef: run.controlRef, + after: { cursor: first.value.cursor ?? first.value.eventId, sequence: first.value.sequence }, + continuation, + continuationTurn, + } + await writeFile(referenceFile, `${JSON.stringify(reference)}\n`, 'utf8') + await expectLostContinuation(run, continuation, continuationTurn) + await writeFile( + `${referenceFile}.output`, + `${JSON.stringify({ first: first.value, controlRef: run.controlRef })}\n`, + 'utf8', + ) +} else { + const reference = JSON.parse(await readFile(referenceFile, 'utf8')) as { + controlRef: AgentExactRunControlRef + after: { cursor: string; sequence: number } + continuation: ReturnType + continuationTurn: NativeContextContinuationTurn + } + const run = await reconnectRetainedRun({ + provider: durableRetainedProvider(stateFile), + controlRef: reference.controlRef, + now: () => Date.parse('2026-08-02T00:00:11.000Z'), + }) + if (!run) throw new Error('retained run was not reconstructable') + const continuation = await run.continueNative(reference.continuation, reference.continuationTurn) + const events = [] + for await (const event of run.events({ after: reference.after })) events.push(event) + const statusBefore = await run.status() + const interactionBinding = { + runId: run.controlRef.runId, + provider: run.controlRef.provider, + environmentId: run.controlRef.environmentId, + sessionId: run.controlRef.sessionId, + executionId: run.controlRef.executionId, + interactionId: 'interaction-1', + requestDigest: run.controlRef.requestDigest, + } + const interactionResponse = { id: 'interaction-1', outcome: 'accepted' as const } + const interaction = await run.respondToInteraction({ + operationId: 'answer-after-restart', + binding: interactionBinding, + commandDigest: interactionResponseCommandDigest({ + binding: interactionBinding, + response: interactionResponse, + }), + response: interactionResponse, + }) + const cancellation = await run.cancel({ + operationId: 'restart-cancel-operation', + reason: 'test cleanup', + }) + await writeFile( + `${referenceFile}.output`, + `${JSON.stringify({ + events, + statusBefore, + interaction, + cancellation, + continuation, + result: await run.result(), + })}\n`, + 'utf8', + ) +} + +async function expectLostContinuation( + run: Awaited>, + request: ReturnType, + turn: NativeContextContinuationTurn, +): Promise { + try { + await run.continueNative(request, turn) + } catch (error) { + if (error instanceof Error && error.message === 'connection lost after continuation commit') { + return + } + throw error + } + throw new Error('continuation response was not lost') +}