e2e test fixture cases

.gitignore

@@ -24,3 +24,4 @@ wheels/
 !.env.example
 !.markdownlint-cli2.yaml
 !maps-example.yaml
+!tests/e2e/fixtures/**/maps.yaml

dev/TODO.md

@@ -29,14 +29,6 @@
 - How do we avoid stampedes (e.g., bulk repo sync triggering thousands
   of re-runs)?
 
-## High priority: End to End test cases
-
-- Create test cases. Each test case should contain:
-  - Before state
-  - maps.yaml file
-  - Expected after state
-- Script to run the script, and verify the after state matches the expected after state
-
 ## High priority: Verify perms are updated when a user's SAML groups change
 
 - If a user gets added to a new SAML group, which hits a mapping, ensure they

dev/test-fixture-cases.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from tests.e2e.test_permission_fixture_cases import FixtureRunResult
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+
+def _format_delta(before: int, after: int) -> str:
+    return f"{after - before:+d}"
+
+
+def _format_expected(value: int | None) -> str:
+    if value is None:
+        return "n/a"
+    return str(value)
+
+
+def _print_result(result: FixtureRunResult) -> None:
+    status = "PASS" if result.passed else "FAIL"
+    permission_pair_delta = _format_delta(
+        result.before_counts.permission_pairs,
+        result.actual_counts.permission_pairs,
+    )
+    print(f"{status} {result.name} — {result.description}")
+    print(f"  scope: users={result.before_counts.users} repos={result.before_counts.repos}")
+    print(
+        "  permission pairs: "
+        f"before={result.before_counts.permission_pairs} "
+        f"expected={result.expected_counts.permission_pairs} "
+        f"actual={result.actual_counts.permission_pairs} "
+        f"delta={permission_pair_delta}"
+    )
+    print(
+        "  changed repos: "
+        f"expected={result.expected_changed_repos} "
+        f"actual={result.actual_changed_repos}"
+    )
+    print(
+        "  mutations: "
+        f"expected={_format_expected(result.expected_mutations)} "
+        f"actual={result.actual_mutations}"
+    )
+    if result.failure is not None:
+        print(f"  failure: {result.failure}")
+    print()
+
+
+def main() -> int:
+    from tests.e2e.test_permission_fixture_cases import fixture_case_dirs, run_fixture_case
+
+    results = [run_fixture_case(case_dir) for case_dir in fixture_case_dirs()]
+    for result in results:
+        _print_result(result)
+
+    passed = sum(1 for result in results if result.passed)
+    failed = len(results) - passed
+    print(f"Summary: {passed} passed, {failed} failed, {len(results)} total.")
+    return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/e2e/__init__.py

@@ -0,0 +1 @@
+"""Fixture-backed end-to-end tests."""

tests/e2e/fixtures/add-users-preserves-existing/after.json

@@ -0,0 +1,63 @@
+{
+  "endpoint": "https://fixture.sourcegraph.test",
+  "authProviders": [
+    {
+      "serviceType": "builtin",
+      "serviceID": "",
+      "clientID": "",
+      "displayName": "Builtin username/password",
+      "isBuiltin": true,
+      "configID": ""
+    }
+  ],
+  "externalServices": [
+    {
+      "id": 1,
+      "kind": "GITHUB",
+      "displayName": "GitHub",
+      "url": "https://github.com/",
+      "config": "{}"
+    }
+  ],
+  "users": [
+    {
+      "id": 1,
+      "username": "alice",
+      "builtinAuth": true,
+      "createdAt": "2026-01-01T00:00:00Z",
+      "emails": [{ "email": "alice@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 2,
+      "username": "bob",
+      "builtinAuth": true,
+      "createdAt": "2026-01-02T00:00:00Z",
+      "emails": [{ "email": "bob@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 3,
+      "username": "carol",
+      "builtinAuth": true,
+      "createdAt": "2026-01-03T00:00:00Z",
+      "emails": [{ "email": "carol@example.com", "verified": true }],
+      "externalAccounts": []
+    }
+  ],
+  "repos": [
+    {
+      "id": 101,
+      "name": "github.com/acme/app",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["alice", "bob"]
+    },
+    {
+      "id": 102,
+      "name": "github.com/acme/lib",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["bob", "carol"]
+    }
+  ],
+  "pendingBindIDs": []
+}

tests/e2e/fixtures/add-users-preserves-existing/before.json

@@ -0,0 +1,63 @@
+{
+  "endpoint": "https://fixture.sourcegraph.test",
+  "authProviders": [
+    {
+      "serviceType": "builtin",
+      "serviceID": "",
+      "clientID": "",
+      "displayName": "Builtin username/password",
+      "isBuiltin": true,
+      "configID": ""
+    }
+  ],
+  "externalServices": [
+    {
+      "id": 1,
+      "kind": "GITHUB",
+      "displayName": "GitHub",
+      "url": "https://github.com/",
+      "config": "{}"
+    }
+  ],
+  "users": [
+    {
+      "id": 1,
+      "username": "alice",
+      "builtinAuth": true,
+      "createdAt": "2026-01-01T00:00:00Z",
+      "emails": [{ "email": "alice@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 2,
+      "username": "bob",
+      "builtinAuth": true,
+      "createdAt": "2026-01-02T00:00:00Z",
+      "emails": [{ "email": "bob@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 3,
+      "username": "carol",
+      "builtinAuth": true,
+      "createdAt": "2026-01-03T00:00:00Z",
+      "emails": [{ "email": "carol@example.com", "verified": true }],
+      "externalAccounts": []
+    }
+  ],
+  "repos": [
+    {
+      "id": 101,
+      "name": "github.com/acme/app",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["alice"]
+    },
+    {
+      "id": 102,
+      "name": "github.com/acme/lib",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["carol"]
+    }
+  ],
+  "pendingBindIDs": []
+}

tests/e2e/fixtures/add-users-preserves-existing/case.json

@@ -0,0 +1,7 @@
+{
+  "description": "Additive --users mode grants mapped repos to one user without dropping existing repo users.",
+  "set": {
+    "users": ["bob"]
+  },
+  "expectedMutations": 2
+}

tests/e2e/fixtures/add-users-preserves-existing/maps.yaml

@@ -0,0 +1,10 @@
+maps:
+  - name: Grant app and lib to selected users
+    users:
+      usernames:
+        - alice
+        - bob
+    repos:
+      names:
+        - github.com/acme/app
+        - github.com/acme/lib

tests/e2e/fixtures/full-overwrite-unions/after.json

@@ -0,0 +1,69 @@
+{
+  "endpoint": "https://fixture.sourcegraph.test",
+  "authProviders": [
+    {
+      "serviceType": "builtin",
+      "serviceID": "",
+      "clientID": "",
+      "displayName": "Builtin username/password",
+      "isBuiltin": true,
+      "configID": ""
+    }
+  ],
+  "externalServices": [
+    {
+      "id": 1,
+      "kind": "GITHUB",
+      "displayName": "GitHub",
+      "url": "https://github.com/",
+      "config": "{}"
+    }
+  ],
+  "users": [
+    {
+      "id": 1,
+      "username": "alice",
+      "builtinAuth": true,
+      "createdAt": "2026-01-01T00:00:00Z",
+      "emails": [{ "email": "alice@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 2,
+      "username": "bob",
+      "builtinAuth": true,
+      "createdAt": "2026-01-02T00:00:00Z",
+      "emails": [{ "email": "bob@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 3,
+      "username": "mallory",
+      "builtinAuth": true,
+      "createdAt": "2026-01-03T00:00:00Z",
+      "emails": [{ "email": "mallory@example.com", "verified": true }],
+      "externalAccounts": []
+    }
+  ],
+  "repos": [
+    {
+      "id": 101,
+      "name": "github.com/acme/app",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["alice"]
+    },
+    {
+      "id": 102,
+      "name": "github.com/acme/lib",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["alice", "bob"]
+    },
+    {
+      "id": 103,
+      "name": "github.com/acme/archive",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["mallory"]
+    }
+  ],
+  "pendingBindIDs": []
+}

tests/e2e/fixtures/full-overwrite-unions/before.json

@@ -0,0 +1,69 @@
+{
+  "endpoint": "https://fixture.sourcegraph.test",
+  "authProviders": [
+    {
+      "serviceType": "builtin",
+      "serviceID": "",
+      "clientID": "",
+      "displayName": "Builtin username/password",
+      "isBuiltin": true,
+      "configID": ""
+    }
+  ],
+  "externalServices": [
+    {
+      "id": 1,
+      "kind": "GITHUB",
+      "displayName": "GitHub",
+      "url": "https://github.com/",
+      "config": "{}"
+    }
+  ],
+  "users": [
+    {
+      "id": 1,
+      "username": "alice",
+      "builtinAuth": true,
+      "createdAt": "2026-01-01T00:00:00Z",
+      "emails": [{ "email": "alice@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 2,
+      "username": "bob",
+      "builtinAuth": true,
+      "createdAt": "2026-01-02T00:00:00Z",
+      "emails": [{ "email": "bob@example.com", "verified": true }],
+      "externalAccounts": []
+    },
+    {
+      "id": 3,
+      "username": "mallory",
+      "builtinAuth": true,
+      "createdAt": "2026-01-03T00:00:00Z",
+      "emails": [{ "email": "mallory@example.com", "verified": true }],
+      "externalAccounts": []
+    }
+  ],
+  "repos": [
+    {
+      "id": 101,
+      "name": "github.com/acme/app",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["mallory"]
+    },
+    {
+      "id": 102,
+      "name": "github.com/acme/lib",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["mallory"]
+    },
+    {
+      "id": 103,
+      "name": "github.com/acme/archive",
+      "externalServiceID": 1,
+      "explicitPermissionsUsers": ["mallory"]
+    }
+  ],
+  "pendingBindIDs": []
+}

tests/e2e/fixtures/full-overwrite-unions/case.json

@@ -0,0 +1,7 @@
+{
+  "description": "Full set mode unions users across rules, overwrites mapped repos, and leaves unmapped repos alone.",
+  "set": {
+    "full": true
+  },
+  "expectedMutations": 2
+}