cuda.core: require a cuda-bindings floor and call the driver through its resolved pointers (#2783)

.github/RELEASE-core.md

@@ -62,9 +62,14 @@ platforms as appropriate for each release.
 ## Check (or update if needed) the dependency requirements
 
 Review `cuda_core/pyproject.toml` and verify that all dependency
-requirements are current.
-
-Update the cuda_core dependency in `cuda_python/setup.py`.
+requirements are current. The `cu12`/`cu13` extras declare the
+`cuda-bindings` floors; see "Bumping the cuda-bindings floor" in
+`cuda_core/AGENTS.md`.
+
+Update the `cuda-core` pin in `cuda_python/setup.py` to the new minor
+series (`cuda-core~=X.Y.0`), and plan a `cuda-python` release right after
+this one. Until that release, `pip install cuda-python` resolves to the
+previous `cuda-core`.
 
 ---
 

.github/workflows/coverage.yml

@@ -80,8 +80,8 @@ jobs:
         uses: ./.github/actions/install_unix_deps
         continue-on-error: false
         with:
-          dependencies: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1"
-          dependent_exes: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1"
+          dependencies: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1 yq"
+          dependent_exes: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1 yq"
 
       - name: Setup proxy cache
         uses: nv-gha-runners/setup-proxy-cache@main

.github/workflows/test-wheel-linux.yml

@@ -144,8 +144,9 @@ jobs:
           # for artifact fetching, graphics libs, g++ required for cffi in
           # example; util-linux for `nsenter` (custom-DRIVER rows re-exec
           # install_gpu_driver.sh onto the host through nsenter)
-          dependencies: "jq wget libgl1 libegl1 g++ util-linux"
-          dependent_exes: "jq wget"
+          # yq: ci/tools/env-vars reads ci/versions.yml with it
+          dependencies: "jq wget libgl1 libegl1 g++ util-linux yq"
+          dependent_exes: "jq wget yq"
 
       - name: Install GPU driver
         if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
@@ -252,14 +253,14 @@ jobs:
           fi
 
       - name: Display structure of downloaded cuda-python artifacts
-        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }}
+        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }}
         run: |
           pwd
           ls -lah cuda_python*.whl cuda_pathfinder/
 
       - name: Display structure of downloaded cuda.bindings artifacts
         if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
-                env.BINDINGS_SOURCE != 'published' }}
+                env.BINDINGS_SOURCE != 'floor' }}
         run: |
           pwd
           ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR

.github/workflows/test-wheel-windows.yml

@@ -232,14 +232,14 @@ jobs:
           fi
 
       - name: Display structure of downloaded cuda-python artifacts
-        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }}
+        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }}
         run: |
           Get-Location
           Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName
 
       - name: Display structure of downloaded cuda.bindings artifacts
         if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
-                env.BINDINGS_SOURCE != 'published' }}
+                env.BINDINGS_SOURCE != 'floor' }}
         run: |
           Get-Location
           Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName

.pre-commit-config.yaml

@@ -42,6 +42,15 @@ repos:
         pass_filenames: false
         verbose: true
 
+      - id: check-cuda-core-bindings-floor
+        name: Check the cuda-bindings floor against ci/versions.yml and the docs
+        entry: python ./toolshed/check_cuda_core_bindings_floor.py
+        language: python
+        files: ^(cuda_core/pyproject\.toml|ci/versions\.yml|cuda_core/cuda/core/_bindings_floor\.py|cuda_core/docs/source/.*\.rst|toolshed/check_cuda_core_bindings_floor\.py)$
+        pass_filenames: false
+        additional_dependencies:
+          - "tomli>=1.1.0; python_version < '3.11'"
+
       - id: check-spdx
         name: Check SPDX Headers
         entry: python ./toolshed/check_spdx.py

ci/tools/cuda_core_bindings_floor.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Print the cuda-bindings floor of a cuda-core wheel for one CUDA major.
+
+    cuda_core_bindings_floor.py --wheel dist/cuda_core-*.whl --major 13
+    -> 13.4.1
+
+CI installs `cuda-bindings==<floor>` next to a freshly built cuda-core wheel to
+test the oldest cuda-bindings that wheel supports (BINDINGS_SOURCE=floor in
+ci/tools/env-vars). This script reads the floor from the wheel under test, not
+from the checkout, so a nightly job that tests a wheel built from another
+commit reads that wheel's floor.
+
+The floors are declared once, by the `cu12`/`cu13` extras in
+cuda_core/pyproject.toml, and the wheel's METADATA carries them as
+`Requires-Dist: cuda-bindings[all]<14,>=13.4.1; extra == "cu13"`. This script
+reads that line.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+import zipfile
+from pathlib import Path
+
+# The cuda-bindings requirement of one `cu<major>` extra in METADATA. The name
+# is followed by `[`, an operator or whitespace so that other names do not match.
+_REQUIRES_DIST_RE = re.compile(
+    r"^Requires-Dist:\s*cuda-bindings(?=[\[<>=!~\s])(?P<spec>[^;]*);\s*extra\s*==\s*['\"]cu(?P<major>\d+)['\"]"
+)
+_FLOOR_RE = re.compile(r">=\s*(\d+)\.(\d+)\.(\d+)")
+
+
+def floor_from_metadata(metadata: str, major: int) -> str:
+    """The floor that the `cu<major>` extra of a wheel's METADATA pins cuda-bindings to."""
+    for line in metadata.splitlines():
+        m = _REQUIRES_DIST_RE.match(line)
+        if m is None or int(m.group("major")) != major:
+            continue
+        floor = _FLOOR_RE.search(m.group("spec"))
+        if floor is None:
+            raise SystemExit(f"METADATA pins cuda-bindings without a floor for the cu{major} extra: {line.strip()!r}")
+        if int(floor.group(1)) != major:
+            raise SystemExit(
+                f"METADATA pins a cuda-bindings floor of another major for the cu{major} extra: {line.strip()!r}"
+            )
+        return ".".join(floor.groups())
+    raise SystemExit(f"METADATA declares no cuda-bindings requirement for the cu{major} extra")
+
+
+def floor_from_wheel(wheel: Path, major: int) -> str:
+    with zipfile.ZipFile(wheel) as zf:
+        metadata = [name for name in zf.namelist() if name.endswith(".dist-info/METADATA")]
+        if len(metadata) != 1:
+            raise SystemExit(f"{wheel.name} contains {len(metadata)} METADATA files, expected 1. Is it a wheel?")
+        return floor_from_metadata(zf.read(metadata[0]).decode("utf-8"), major)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--wheel", type=Path, required=True, help="the cuda-core wheel under test")
+    parser.add_argument("--major", type=int, required=True, help="CUDA major series (12 or 13)")
+    args = parser.parse_args(argv)
+    print(floor_from_wheel(args.wheel, args.major))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/tools/env-vars

@@ -64,23 +64,45 @@ elif [[ "${1}" == "test" ]]; then
   # BINDINGS_SOURCE controls which cuda-bindings to install at test time:
   #   main      — use the just-built bindings wheel from this CI run
   #   backport  — fetch bindings from the prior (N-1) branch
-  #   published — install from PyPI (cuda-bindings==${TEST_CUDA_MAJOR}.${TEST_CUDA_MINOR}.*)
+  #   floor     — install from PyPI the oldest cuda-bindings that the cuda-core
+  #               wheel under test supports, its per-major floor (see
+  #               cuda_core/cuda/core/_bindings_floor.py and ci/tools/run-tests).
+  #               Selected when the test CTK minor differs from the minor the
+  #               wheel was built against. Those rows exercise a new cuda-core
+  #               with the floor cuda-bindings and older CTK libraries, the skew
+  #               that cuda-core supports. cuda-bindings older than the floor is
+  #               unsupported and fails at import
+  #               (https://github.com/NVIDIA/cuda-python/issues/2783).
   #
   # SKIP_CUDA_BINDINGS_TEST / SKIP_CYTHON_TEST control which *tests* to run
   # (they do NOT affect installation — that's BINDINGS_SOURCE's job).
 
   BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${BUILD_CUDA_VER})"
   TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${CUDA_VER})"
+  # CI builds the prior-major half of the cuda-core wheel against the prev_build
+  # toolkit in ci/versions.yml and the backport branch's bindings.
+  if ! command -v yq >/dev/null 2>&1; then
+    echo "Error: yq is required to read ci/versions.yml" >&2
+    exit 1
+  fi
+  BUILD_PREV_CUDA_VER="$(yq -r '.cuda.prev_build.version' ci/versions.yml)"
+  BUILD_PREV_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${BUILD_PREV_CUDA_VER})"
 
   if [[ ${BUILD_CUDA_MAJOR} != ${TEST_CUDA_MAJOR} ]]; then
-    # Major mismatch (e.g. build=13.x, test=12.x): use the backport branch.
-    BINDINGS_SOURCE=backport
     SKIP_CUDA_BINDINGS_TEST=1
     SKIP_CYTHON_TEST=1
+    if [[ ${BUILD_PREV_CUDA_MINOR} != ${TEST_CUDA_MINOR} ]]; then
+      # Prior major, minor mismatch (e.g. built against 12.9, test=12.6): floor
+      # bindings from PyPI with the older CTK libraries.
+      BINDINGS_SOURCE=floor
+    else
+      # Prior major, same minor (e.g. build=13.x, test=12.9): the backport branch.
+      BINDINGS_SOURCE=backport
+    fi
   elif [[ ${BUILD_CUDA_MINOR} != ${TEST_CUDA_MINOR} ]]; then
-    # Same major, minor mismatch (e.g. build=13.2, test=13.0): use published
-    # bindings from PyPI to test the real-world backward-compat scenario.
-    BINDINGS_SOURCE=published
+    # Same major, minor mismatch (e.g. build=13.4, test=13.0): floor bindings
+    # from PyPI with the older CTK libraries.
+    BINDINGS_SOURCE=floor
     SKIP_CUDA_BINDINGS_TEST=1
     SKIP_CYTHON_TEST=1
   else

ci/tools/merge_cuda_core_wheels.py

@@ -148,10 +148,11 @@ def merge_wheels(wheels: list[Path], output_dir: Path, show_wheel_contents: bool
 
         print("\n=== Removing files from cuda/core/ directory ===", file=sys.stderr)
         # Only what cuda/core/__init__.py uses before it rewrites __path__ to the
-        # versioned subpackage stays at top level: it imports _version, then
-        # redirects every later import into the versioned tree. Anything else
-        # left at top level is a dead copy that nothing imports.
-        items_to_keep = {"__init__.py", "_version.py", *versioned_dirs}
+        # versioned subpackage stays at top level: it imports _version and the
+        # build-independent floor logic in _bindings_floor, then redirects every
+        # later import into the versioned tree. Anything else left at top level
+        # is a dead copy that nothing imports.
+        items_to_keep = {"__init__.py", "_version.py", "_bindings_floor.py", *versioned_dirs}
         all_items = os.scandir(base_wheel / base_dir)
         removed_count = 0
         for f in all_items:

ci/tools/run-tests

@@ -74,10 +74,14 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then
 
   # Resolve bindings based on BINDINGS_SOURCE (set by env-vars):
   #   main/backport → local wheel from artifacts dir
-  #   published     → install from PyPI by version
+  #   floor         → the oldest cuda-bindings that the cuda-core wheel under test
+  #                   supports, read from that wheel and installed from PyPI
   BINDINGS_ARGS=()
-  if [[ "${BINDINGS_SOURCE}" == "published" ]]; then
-    BINDINGS_ARGS+=("cuda-bindings==${TEST_CUDA_MAJOR}.${TEST_CUDA_MINOR}.*")
+  if [[ "${BINDINGS_SOURCE}" == "floor" ]]; then
+    CORE_WHL_FOR_FLOOR=("${CUDA_CORE_ARTIFACTS_DIR}"/*.whl)
+    BINDINGS_FLOOR="$(python ci/tools/cuda_core_bindings_floor.py --wheel "${CORE_WHL_FOR_FLOOR[0]}" --major "${TEST_CUDA_MAJOR}")"
+    echo "cuda-bindings floor of ${CORE_WHL_FOR_FLOOR[0]##*/} for CUDA ${TEST_CUDA_MAJOR}: ${BINDINGS_FLOOR}"
+    BINDINGS_ARGS+=("cuda-bindings==${BINDINGS_FLOOR}")
   else
     BINDINGS_ARGS=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)
     if [[ "${LOCAL_CTK}" != 1 ]]; then

ci/tools/tests/test_cuda_core_bindings_floor.py

@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import importlib.util
+import zipfile
+from pathlib import Path
+
+import pytest
+
+TOOLS = Path(__file__).resolve().parent.parent
+
+
+def _load(name, path):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+tool = _load("cuda_core_bindings_floor", TOOLS / "cuda_core_bindings_floor.py")
+
+# METADATA as setuptools writes it: pip's normalized specifier order, extras in quotes.
+METADATA = """\
+Metadata-Version: 2.4
+Name: cuda-core
+Version: 1.3.0
+Requires-Dist: cuda-pathfinder>=1.4.2
+Requires-Dist: numpy
+Provides-Extra: cu12
+Requires-Dist: cuda-bindings[all]<13,>=12.9.8; extra == "cu12"
+Requires-Dist: cuda-toolkit==12.*; extra == "cu12"
+Provides-Extra: cu13
+Requires-Dist: cuda-bindings[all]<14,>=13.4.1; extra == "cu13"
+Requires-Dist: cuda-toolkit==13.*; extra == "cu13"
+"""
+
+
+def _wheel(tmp_path, metadata=METADATA):
+    path = tmp_path / "cuda_core-1.3.0-cp312-cp312-linux_x86_64.whl"
+    with zipfile.ZipFile(path, "w") as zf:
+        zf.writestr("cuda/core/__init__.py", "")
+        if metadata is not None:
+            zf.writestr("cuda_core-1.3.0.dist-info/METADATA", metadata)
+    return path
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+@pytest.mark.parametrize(("major", "floor"), [(12, "12.9.8"), (13, "13.4.1")])
+def test_reads_the_floor_of_each_extra(tmp_path, major, floor):
+    assert tool.floor_from_wheel(_wheel(tmp_path), major) == floor
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_accepts_other_spellings_of_the_requirement():
+    metadata = 'Requires-Dist: cuda-bindings>=13.4.1,==13.*; extra == "cu13"\n'
+    assert tool.floor_from_metadata(metadata, 13) == "13.4.1"
+    metadata = "Requires-Dist: cuda-bindings [all] >= 13.4.1, < 14 ; extra == 'cu13'\n"
+    assert tool.floor_from_metadata(metadata, 13) == "13.4.1"
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_ignores_lookalike_names_and_other_extras():
+    metadata = (
+        'Requires-Dist: cuda-bindings-extra>=1.0; extra == "cu13"\n'
+        'Requires-Dist: cuda-bindings[all]<13,>=12.9.8; extra == "cu12"\n'
+        'Requires-Dist: cuda-bindings[all]<14,>=13.4.1; extra == "cu13"\n'
+    )
+    assert tool.floor_from_metadata(metadata, 13) == "13.4.1"
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_rejects_a_missing_extra(tmp_path):
+    with pytest.raises(SystemExit, match="no cuda-bindings requirement for the cu11 extra"):
+        tool.floor_from_wheel(_wheel(tmp_path), 11)
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_rejects_a_requirement_without_a_floor():
+    with pytest.raises(SystemExit, match="without a floor for the cu13 extra"):
+        tool.floor_from_metadata('Requires-Dist: cuda-bindings==13.*; extra == "cu13"\n', 13)
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_rejects_a_floor_of_another_major():
+    with pytest.raises(SystemExit, match="floor of another major for the cu13 extra"):
+        tool.floor_from_metadata('Requires-Dist: cuda-bindings>=12.9.8,<14; extra == "cu13"\n', 13)
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_rejects_a_zip_that_is_not_a_wheel(tmp_path):
+    with pytest.raises(SystemExit, match="contains 0 METADATA files"):
+        tool.floor_from_wheel(_wheel(tmp_path, metadata=None), 13)
+
+
+@pytest.mark.agent_authored(model="claude-fable-5-1")
+def test_cli_prints_the_floor(tmp_path, capsys):
+    assert tool.main(["--wheel", str(_wheel(tmp_path)), "--major", "13"]) == 0
+    assert capsys.readouterr().out.strip() == "13.4.1"

cuda_bindings/AGENTS.md

@@ -51,6 +51,10 @@ the `legacy_tests` subdirectory.
 
 - `CUDA_HOME` or `CUDA_PATH` must point to a valid CUDA Toolkit for source
   builds.
+- The toolkit's `cuda.h` must have the same major.minor as the generated
+  sources, `CUDA_VERSION` in `cuda/bindings/cydriver.pxd`. `build_hooks.py`
+  checks this before cythonize and fails with a message that names both
+  versions.
 - `CUDA_PYTHON_PARALLEL_LEVEL` controls build parallelism.
 - Runtime behavior is affected by
   `CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM` and