flex scripts

scripts/create_more_training_sets.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+"""Create a child version + an unversioned "mix" training set (DE-8692 manual test).
+
+Usage:
+    poetry run python scripts/create_more_training_sets.py \
+        --model <model_id> --api-key <key> --parent <trs_id>
+
+- Child version: downstream of --parent, prunes one inherited item and adds one new one.
+- Mix: unversioned root, unioning the parent set + the child set (+ one extra item).
+"""
+import argparse
+
+from nucleus import NucleusClient
+
+# Item chosen to prune from the parent, and items to add to the child version.
+REMOVE_FROM_PARENT = "di_d6ng124z6a1006gm86c0"
+ADD_TO_CHILD = ["di_d6ng124z6a1006gm86eg"]
+# Extra item folded into the unversioned "mix" set alongside the two source sets.
+EXTRA_FOR_MIX = ["di_d6ng124z6a1006gm86e0"]
+
+
+def summarize(client: NucleusClient, ts) -> None:
+    page = client.list_training_set_items(ts.id, limit=1000)
+    print(
+        f"  {ts.id}  v{ts.version_major}.{ts.version_minor}  ({page.total} items)"
+    )
+    for item_id in page.item_ids:
+        print(f"    - {item_id}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--api-key", required=True)
+    parser.add_argument(
+        "--parent", required=True, help="Parent training set id (trs_*)"
+    )
+    parser.add_argument(
+        "--endpoint", default="http://localhost:3000/v1/nucleus"
+    )
+    args = parser.parse_args()
+
+    client = NucleusClient(api_key=args.api_key, endpoint=args.endpoint)
+
+    print(
+        f"Creating child version of {args.parent} "
+        f"(prune {REMOVE_FROM_PARENT}, add {ADD_TO_CHILD})..."
+    )
+    # Downstream version: inherits the parent's members, adds item_ids, prunes removed_item_ids —
+    # final set = parent ∪ added ∖ removed, in one call.
+    child = client.create_training_set(
+        "training-set-de-8692-child",
+        model=args.model,
+        parent_training_set_id=args.parent,
+        item_ids=ADD_TO_CHILD,
+        removed_item_ids=[REMOVE_FROM_PARENT],
+        wait_for_completion=True,
+    )
+    print("Child version:")
+    summarize(client, child)
+
+    print(
+        f"\nCreating unversioned mix of {args.parent} + {child.id} (+ {EXTRA_FOR_MIX})..."
+    )
+    mix = client.create_training_set(
+        "training-set-de-8692-mix",
+        model=args.model,
+        training_set_ids=[args.parent, child.id],
+        item_ids=EXTRA_FOR_MIX,
+        wait_for_completion=True,
+    )
+    print("Mix set:")
+    summarize(client, mix)
+
+
+if __name__ == "__main__":
+    main()

scripts/create_training_set.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""Create a training set from a fixed set of dataset items against a local backend.
+
+Usage:
+    python scripts/create_training_set.py --model <model_id> --api-key <key>
+
+The model id and API key are required (training sets are model-scoped). The
+backend defaults to the local prod instance at http://localhost:3000.
+"""
+import argparse
+
+from nucleus import NucleusClient
+
+# The four unique item ids from the request. Note the second token in the
+# original prompt was URL-encoded (`di_..b0%2Cdi_..bg`), i.e. two ids joined by
+# a comma, so the list below has 4 members, not 3.
+ITEM_IDS = [
+    "di_d6ng124z6a1006gm86ag",
+    "di_d6ng124z6a1006gm86b0",
+    "di_d6ng124z6a1006gm86bg",
+    "di_d6ng124z6a1006gm86c0",
+]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="Model id to scope/attach the training set to",
+    )
+    parser.add_argument(
+        "--api-key", required=True, help="Scale API key for the target backend"
+    )
+    parser.add_argument(
+        "--name", default="training-set-de-8692", help="Training set name"
+    )
+    parser.add_argument(
+        "--endpoint",
+        default="http://localhost:3000/v1/nucleus",
+        help="Nucleus API base URL (default: local prod backend)",
+    )
+    args = parser.parse_args()
+
+    client = NucleusClient(api_key=args.api_key, endpoint=args.endpoint)
+
+    print(
+        f"Creating training set {args.name!r} on model {args.model} from {len(ITEM_IDS)} items..."
+    )
+    training_set = client.create_training_set(
+        args.name,
+        model=args.model,
+        item_ids=ITEM_IDS,
+        wait_for_completion=True,
+    )
+
+    print(f"\nCreated training set: {training_set.id}")
+    print(f"  status:  {getattr(training_set, 'status', '?')}")
+    print(
+        f"  version: v{training_set.version_major}.{training_set.version_minor}"
+    )
+    page = training_set.items()
+    print(f"  members: {page.total}")
+    for item_id in page.item_ids:
+        print(f"    - {item_id}")
+
+
+if __name__ == "__main__":
+    main()

scripts/flex_benchmark_exclusion_effect.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+"""Prove benchmark exclusion rules actually drop content from the *benchmark's own views*.
+
+Companion to flex_benchmark_exclusions.py (which flexes the create/update/validation API surface).
+This one is a live, eyeball-it verification of the piece-2/3 backend change that makes a benchmark's
+Insights / ground-truth coverage / item grid honor its own exclusion rules — not just evaluations.
+
+It finalizes TWO benchmarks over the SAME member items:
+
+  * control  — no exclusion rules
+  * excluded — one annotation-scope groundTruth label rule dropping "Bomber Aircraft"
+
+Both are created as `ready` (draft=False), so the server runs the build job and recomputes the frozen
+content metrics through the new exclusion filter. Then it prints each benchmark id and its Insights
+URL so you can compare the label distribution:
+
+  control  : 5 classes / 27 annotations, Bomber Aircraft 8 (30%)
+  excluded : 4 classes / 19 annotations, no Bomber Aircraft bucket
+
+(Exact counts depend on the seed GT; the point is control minus excluded == the Bomber Aircraft count.)
+
+IMPORTANT: the backend must be running the piece-2/3 code. Because exclusions are applied at metric
+recompute (finalize/build), a benchmark finalized by an OLD backend will still show the full 27 — so
+restart the backend before running this.
+
+Usage:
+    # uses $NUCLEUS_PRODCLONE_API_KEY + local prodclone backend by default
+    python scripts/flex_benchmark_exclusion_effect.py
+
+    # clean them up afterwards instead of leaving them for inspection
+    python scripts/flex_benchmark_exclusion_effect.py --delete
+"""
+import argparse
+import os
+import time
+
+from nucleus import LabelExclusionRule, NucleusClient
+
+# Same scattered prodclone items the sibling flex scripts use (ds_d6ng124awzzg0gtc8x50, all with GT).
+ITEM_IDS = [
+    "di_d6ng124z6a1006gm86ag",
+    "di_d6ng124z6a1006gm86b0",
+    "di_d6ng124z6a1006gm86bg",
+    "di_d6ng124z6a1006gm86c0",
+    "di_d6ng124z6a1006gm86e0",
+    "di_d6ng124z6a1006gm86eg",
+]
+
+# The class we drop. Annotation-scope groundTruth => individual "Bomber Aircraft" GT boxes disappear
+# from the benchmark's content views (the item stays a member via its other-class GT).
+EXCLUDED_LABEL = "Bomber Aircraft"
+BOMBER_RULE = LabelExclusionRule(
+    scope="annotation", target="groundTruth", labels=[EXCLUDED_LABEL]
+)
+
+
+def make_ready_benchmark(client, name, description, exclusion_rules):
+    """Create a finalized (ready) benchmark and block until its build/recompute completes."""
+    print(f"\nCreating {name!r} (ready)...")
+    bm = client.create_benchmark(
+        name,
+        description=description,
+        metadata={"purpose": "flex_benchmark_exclusion_effect.py"},
+        item_ids=ITEM_IDS,
+        exclusion_rules=exclusion_rules,  # None => omit; a list => set
+        # draft defaults to False -> server builds + recomputes content metrics now.
+    )
+    print(
+        f"  {bm.id} status={bm.status!r} items={bm.item_count} "
+        f"exclusion_rules={len(bm.exclusion_rules or [])}"
+    )
+    return bm
+
+
+def insights_url(base_ui: str, spoof: str, bm_id: str) -> str:
+    return f"{base_ui}/pubsec-data-engine/{bm_id}?spoof={spoof}&bm_tab=insights"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--api-key",
+        default=os.environ.get("NUCLEUS_PRODCLONE_API_KEY"),
+        help="Scale API key (default: $NUCLEUS_PRODCLONE_API_KEY).",
+    )
+    parser.add_argument(
+        "--endpoint",
+        default="http://localhost:3000/v1/nucleus",
+        help="Nucleus API base URL (default: local prodclone backend).",
+    )
+    parser.add_argument(
+        "--ui-base",
+        default="http://localhost:3003",
+        help="Base URL of the pubsec UI, for the printed Insights links.",
+    )
+    parser.add_argument(
+        "--spoof",
+        default="66ab902a5d92f6f39ca2ef9a",
+        help="Spoof user id for the printed Insights links.",
+    )
+    parser.add_argument(
+        "--delete",
+        action="store_true",
+        help="Delete both benchmarks at the end instead of leaving them for inspection.",
+    )
+    args = parser.parse_args()
+    if not args.api_key:
+        parser.error(
+            "no API key: pass --api-key or set NUCLEUS_PRODCLONE_API_KEY"
+        )
+
+    client = NucleusClient(api_key=args.api_key, endpoint=args.endpoint)
+    stamp = int(time.time())
+
+    control = make_ready_benchmark(
+        client,
+        f"flex-excl-effect-{stamp}-control",
+        "control: no exclusions (expect full GT)",
+        exclusion_rules=None,
+    )
+    excluded = make_ready_benchmark(
+        client,
+        f"flex-excl-effect-{stamp}-excluded",
+        f"excluded: drop {EXCLUDED_LABEL} (expect fewer GT)",
+        exclusion_rules=[BOMBER_RULE],
+    )
+
+    print("\n" + "=" * 68)
+    print("Open both Insights tabs and compare the label distribution:")
+    print(f"\n  control  ({control.id}) — expect all classes incl. {EXCLUDED_LABEL}:")
+    print(f"    {insights_url(args.ui_base, args.spoof, control.id)}")
+    print(f"\n  excluded ({excluded.id}) — expect NO {EXCLUDED_LABEL}, fewer annotations:")
+    print(f"    {insights_url(args.ui_base, args.spoof, excluded.id)}")
+    print(
+        "\nThe difference in total annotations between the two == the number of "
+        f"{EXCLUDED_LABEL!r}\nGT boxes across these items. If the excluded benchmark still shows "
+        f"{EXCLUDED_LABEL},\nthe backend that finalized it predates the piece-2/3 change — restart it "
+        "and rerun."
+    )
+
+    if args.delete:
+        print("\n--delete: cleaning up...")
+        for bm in (control, excluded):
+            client.delete_benchmark(bm.id)
+            print(f"  deleted {bm.id}")
+    else:
+        print(f"\nLeaving both benchmarks in place (pass --delete to remove).")
+
+
+if __name__ == "__main__":
+    main()