From 997f590817b6bac214e1a33d165e92a1d157c6c2 Mon Sep 17 00:00:00 2001 From: Gal Be Date: Tue, 8 Sep 2026 11:53:21 +0300 Subject: [PATCH 1/4] docs: drop the Go port framing, stand on its own The README explained this project as a port of its Go sibling, which made the Go repository a prerequisite for reading it. Nothing here depends on that repository, so the framing only sent readers somewhere else. The intro now names pytest-benchmark as the metric source directly. The status note, the decision-procedure note and the Windows limits section make the same points without the comparison; two code comments that justified a choice by pointing at the Go implementation now carry the reasoning themselves. A test guards the README against the framing coming back. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VaGHC3qsuNHw3gX8EvGgxY --- README.md | 20 ++++++++------------ src/autor3search_python/stats.py | 8 ++++---- tests/test_e2e.py | 7 +++---- tests/test_templates.py | 31 +++++++++++++++++++++---------- 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 271e73f..f0ee060 100644 --- a/README.md +++ b/README.md @@ -12,26 +12,23 @@ wake up to a branch of accepted commits and a log of every experiment, including the failures. Inspired by [karpathy/autoresearch](https://github.com/karpathy/autoresearch). -This is a Python port of [g4lb/autor3search-go](https://github.com/g4lb/autor3search-go), -its Go sibling — same design, same guarantees, a different metric source: where -the Go harness reads `ns/op` out of `go test -bench`, this one reads per-round -timings out of [pytest-benchmark](https://pytest-benchmark.readthedocs.io/). +The metric source is [pytest-benchmark](https://pytest-benchmark.readthedocs.io/): +the harness reads per-round timings out of its JSON output. > **Status: early but working.** Validated against one real library — > [`humanize`](https://github.com/python-humanize/humanize) — where it found and > kept a genuine 5.81% win across its 15 benchmarks (hoisting a per-call > `import math` out of nine function bodies; `clamp` −23.2%, `apnumber` −14.9%, -> no regressions). That is one library, not the three the Go original was -> exercised against, so treat this as a tool that inherited a validated design -> and has begun earning its own record rather than one that already has it. +> no regressions). That is one library, so treat this as a tool that has begun +> earning its record rather than one that already has it. > > Every number in this README is a real measurement taken on the machine that > wrote it, never an illustration. Where a number would have been guessed, there > is no number. > > The decision procedure — the scoring rules, the Bonferroni correction, the -> asymmetric regression guard, the four exit codes — is carried over unchanged -> from the Go original. If you run this against your own project, the harness's +> asymmetric regression guard, the four exit codes — is fixed, not tuned per +> project. If you run this against your own project, the harness's > `results.tsv` and `report` output are the honest record of what it actually did > there; that is rather the point of the whole design. @@ -391,10 +388,9 @@ abandoned. Plain `stop` is unaffected and behaves identically everywhere. you about less. One residual race is real: a grandchild started in the microseconds between spawning a benchmark subprocess and putting it in its job object is outside that job and would survive a timeout kill. Windows offers no -way to create a process directly into a job through `subprocess`, and the Go -original has the same window. +way to create a process directly into a job through `subprocess`. -Ported from the Go original: +Accepted limits of the design: - No attempt to make the harness tamper-proof against a same-user attacker. The worktree-integrity check catches accidental clobbering and careless diff --git a/src/autor3search_python/stats.py b/src/autor3search_python/stats.py index 5c677fe..c147612 100644 --- a/src/autor3search_python/stats.py +++ b/src/autor3search_python/stats.py @@ -1,9 +1,9 @@ """The statistics behind a verdict. -The Go original gets these from golang.org/x/perf/benchmath. Implementing them -here keeps the harness free of a scipy dependency, and — more usefully — keeps -the exact small-sample behavior under this project's own tests, since small -samples are the entire operating regime (`count` defaults to 10 per side). +These are implemented here rather than taken from scipy: it keeps the harness +dependency-free, and — more usefully — keeps the exact small-sample behavior +under this project's own tests, since small samples are the entire operating +regime (`count` defaults to 10 per side). """ from __future__ import annotations diff --git a/tests/test_e2e.py b/tests/test_e2e.py index f495b92..5daf544 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -72,10 +72,9 @@ def test_full_run(demo_repo): # times the largest no-op delta observed, and well clear of the KEEP step # below, which turns a quadratic per-character concatenation into a single # join and measures -25.8% (score 0.7418) on the machine this was written - # on. The Go original raises the same floor for the same reason; it uses - # 15%, against a fixture whose true effect is an order of magnitude rather - # than this one's quarter, so the ratio here is the thing being matched, - # not the number. + # on. What matters is that ratio — the floor sitting well inside the gap + # between the two effects — not the absolute number, so a fixture with a + # different true effect would want a different floor. raised = cfg_path.read_text().replace("min_effect_pct = 1.0", "min_effect_pct = 10.0") assert "min_effect_pct = 10.0" in raised, "init's config no longer has the key this rewrites" cfg_path.write_text(raised) diff --git a/tests/test_templates.py b/tests/test_templates.py index 93e1ba1..ab65500 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,8 +1,21 @@ +import pathlib import re from autor3search_python import pipeline, templates, verdict +GO_ARTIFACTS = ( + "autor3search-go", + "go.mod", + "go.sum", + "_test.go", + "config.yaml", + "ns/op", + "allocs_delta", + "go test", +) + + def test_program_md_exit_code_table_agrees_with_verdict(): """A stale table would send the agent branching on the wrong exit code.""" text = templates.program_md() @@ -35,16 +48,14 @@ def test_program_md_documents_every_exit_code(): def test_program_md_names_no_go_artifacts(): """A stale Go reference would send the agent looking for a file that is not there.""" text = templates.program_md() - for stale in ( - "autor3search-go", - "go.mod", - "go.sum", - "_test.go", - "config.yaml", - "ns/op", - "allocs_delta", - "go test", - ): + for stale in GO_ARTIFACTS: + assert stale not in text + + +def test_readme_names_no_go_artifacts(): + """The README stands on its own; it does not explain this project as a port.""" + text = (pathlib.Path(__file__).parent.parent / "README.md").read_text(encoding="utf-8") + for stale in GO_ARTIFACTS + ("Go original", "Go sibling", "Go harness", "Python port"): assert stale not in text From 213e494d148752809db3ac8c842b8e5083972957 Mon Sep 17 00:00:00 2001 From: Gal Be Date: Tue, 8 Sep 2026 11:56:21 +0300 Subject: [PATCH 2/4] style: satisfy ruff's import block spacing Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VaGHC3qsuNHw3gX8EvGgxY --- tests/test_templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index ab65500..7e9911c 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -3,7 +3,6 @@ from autor3search_python import pipeline, templates, verdict - GO_ARTIFACTS = ( "autor3search-go", "go.mod", From 53389e1f62ed4a6d20b4b8f7e5427a2779787d06 Mon Sep 17 00:00:00 2001 From: Gal Be Date: Tue, 8 Sep 2026 11:57:32 +0300 Subject: [PATCH 3/4] test: name the guard for what it checks, not for Go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hoisted constant put "GO" in the vocabulary of a project that has none. Two of its entries are not Go anyway — config.yaml and allocs_delta are leftovers of an earlier config format and metric set. What the guard actually checks is that the agent-facing template and the README name nothing absent from this project, so ABSENT_NAMES says that; the README's prose check moves to PORT_FRAMING beside it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VaGHC3qsuNHw3gX8EvGgxY --- tests/test_templates.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index 7e9911c..c2e054d 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -3,7 +3,11 @@ from autor3search_python import pipeline, templates, verdict -GO_ARTIFACTS = ( +# Names that do not exist in this project: files it never writes, a config +# format it does not read, metrics it does not report. Naming one in the +# agent-facing template or the README sends the reader after something that +# is not there. +ABSENT_NAMES = ( "autor3search-go", "go.mod", "go.sum", @@ -14,6 +18,9 @@ "go test", ) +# Prose that would explain this project as a port rather than on its own terms. +PORT_FRAMING = ("Go original", "Go sibling", "Go harness", "Python port") + def test_program_md_exit_code_table_agrees_with_verdict(): """A stale table would send the agent branching on the wrong exit code.""" @@ -44,18 +51,18 @@ def test_program_md_documents_every_exit_code(): assert token in text -def test_program_md_names_no_go_artifacts(): - """A stale Go reference would send the agent looking for a file that is not there.""" +def test_program_md_names_nothing_absent(): + """A name for a file that is not there sends the agent looking for it.""" text = templates.program_md() - for stale in GO_ARTIFACTS: - assert stale not in text + for absent in ABSENT_NAMES: + assert absent not in text -def test_readme_names_no_go_artifacts(): +def test_readme_names_nothing_absent(): """The README stands on its own; it does not explain this project as a port.""" text = (pathlib.Path(__file__).parent.parent / "README.md").read_text(encoding="utf-8") - for stale in GO_ARTIFACTS + ("Go original", "Go sibling", "Go harness", "Python port"): - assert stale not in text + for absent in ABSENT_NAMES + PORT_FRAMING: + assert absent not in text def test_program_md_forbids_editing_conftest(): From 8bcb10738ea94d665c9fe128a16bea9f4c59ca95 Mon Sep 17 00:00:00 2001 From: Gal Be Date: Tue, 8 Sep 2026 12:00:21 +0300 Subject: [PATCH 4/4] refactor: drop Go from an error message and three docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchtime error told the user their value 'uses go test's fixed-iteration-count form' — a Go reference in output this project prints at runtime. The form is refused for its own reasons, which the rest of the message already gives, so naming another tool's flag adds nothing. Three docstrings explained a gate or a fixture by analogy to go build and go test; they now say what the thing does. The guard extends over src/ as well, since error messages reach users the same way the README does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VaGHC3qsuNHw3gX8EvGgxY --- src/autor3search_python/config.py | 2 +- src/autor3search_python/runner.py | 2 +- tests/test_config.py | 2 +- tests/test_runner.py | 4 ++-- tests/test_templates.py | 9 +++++++++ 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/autor3search_python/config.py b/src/autor3search_python/config.py index 80b4b62..d7206c4 100644 --- a/src/autor3search_python/config.py +++ b/src/autor3search_python/config.py @@ -174,7 +174,7 @@ def validate(cfg: Config) -> None: raise ConfigError(f"gc must be one of {list(ALLOWED_GC)}, got {cfg.gc!r}") if _ITERATION_COUNT_FORM.match(str(cfg.benchtime)): raise ConfigError( - f"benchtime {cfg.benchtime!r} uses go test's fixed-iteration-count form (Nx), " + f"benchtime {cfg.benchtime!r} uses the fixed-iteration-count form (Nx), " f"which is deliberately unsupported: a fixed count makes rounds incomparable, " f"because a candidate that is twice as fast finishes in half the wall time and " f"is therefore measured under different thermal conditions — exactly what the " diff --git a/src/autor3search_python/runner.py b/src/autor3search_python/runner.py index 45a684c..2c274b6 100644 --- a/src/autor3search_python/runner.py +++ b/src/autor3search_python/runner.py @@ -309,7 +309,7 @@ def python_run(self, *args: str) -> Result: return self.run(self.python, *args) def compile_gate(self, paths: Sequence[str]) -> Result: - """The syntax gate. `go build`'s role: is this even valid code?""" + """The syntax gate: is this even valid code, before anything runs it?""" return self.python_run("-m", "compileall", "-q", "-x", _compile_exclude(), *paths) def import_gate(self, modules: Sequence[str]) -> Result: diff --git a/tests/test_config.py b/tests/test_config.py index 91bae20..a60c7e0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -130,7 +130,7 @@ def test_parse_duration_rejects(text): def test_benchtime_rejects_the_iteration_count_form(tmp_path): - """go test's `100x` form is a real flag value someone may copy across; say why not.""" + """`100x` is a plausible thing to type for a benchtime; say why it is refused.""" with pytest.raises(config.ConfigError, match="fixed-iteration"): config.load(write(tmp_path, 'benchtime = "100x"\n')) diff --git a/tests/test_runner.py b/tests/test_runner.py index 7367371..0d9e025 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -43,8 +43,8 @@ def test_timeout_is_reported_not_raised(tmp_path): def test_timeout_kills_grandchildren(tmp_path): - """go test runs its benchmark as a grandchild; pytest does too. A survivor - burns CPU and corrupts every later measurement on the machine.""" + """pytest runs its benchmark as a grandchild. A survivor burns CPU and + corrupts every later measurement on the machine.""" marker = tmp_path / "alive.txt" # marker.as_posix(): a Windows path interpolated raw would put "\\U" of # C:\\Users into the inner script's own non-raw string literal, and the diff --git a/tests/test_templates.py b/tests/test_templates.py index c2e054d..e1ae36f 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -65,6 +65,15 @@ def test_readme_names_nothing_absent(): assert absent not in text +def test_shipped_source_names_nothing_absent(): + """Error messages and docstrings reach users too, so they get the same guard.""" + src = pathlib.Path(__file__).parent.parent / "src" + for path in sorted(src.rglob("*.py")): + text = path.read_text(encoding="utf-8") + for absent in ABSENT_NAMES + PORT_FRAMING: + assert absent not in text, f"{path.relative_to(src)} names {absent!r}" + + def test_program_md_forbids_editing_conftest(): assert "conftest.py" in templates.program_md()