diff --git a/cuda_core/cuda/core/_cpp/resource_handles.cpp b/cuda_core/cuda/core/_cpp/resource_handles.cpp index ed6e630b9d9..ee116a9f353 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.cpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.cpp @@ -111,6 +111,13 @@ decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit = nullptr; void* p_cuDevSmResourceSplit = nullptr; #endif +// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13020 +decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync = nullptr; +#else +void* p_cuMemcpyWithAttributesAsync = nullptr; +#endif + // NVRTC function pointers decltype(&nvrtcDestroyProgram) p_nvrtcDestroyProgram = nullptr; @@ -2834,4 +2841,25 @@ bool has_sm_resource_split() noexcept { return p_cuDevSmResourceSplit != nullptr; } +// ============================================================================ +// cuMemcpyWithAttributesAsync wrapper +// ============================================================================ + +CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, + void* attr, CUstream hStream) { +#if CUDA_VERSION >= 13020 + if (!p_cuMemcpyWithAttributesAsync) { + return CUDA_ERROR_NOT_SUPPORTED; + } + return p_cuMemcpyWithAttributesAsync( + dst, src, size, static_cast(attr), hStream); +#else + return CUDA_ERROR_NOT_SUPPORTED; +#endif +} + +bool has_memcpy_with_attributes_async() noexcept { + return p_cuMemcpyWithAttributesAsync != nullptr; +} + } // namespace cuda_core diff --git a/cuda_core/cuda/core/_cpp/resource_handles.hpp b/cuda_core/cuda/core/_cpp/resource_handles.hpp index a55353bb0ec..ff1a12a4618 100644 --- a/cuda_core/cuda/core/_cpp/resource_handles.hpp +++ b/cuda_core/cuda/core/_cpp/resource_handles.hpp @@ -146,6 +146,15 @@ extern decltype(&cuDevSmResourceSplit) p_cuDevSmResourceSplit; extern void* p_cuDevSmResourceSplit; #endif +// cuMemcpyWithAttributesAsync (13.2+ — may be null on older drivers/bindings) +#if CUDA_VERSION >= 13020 +extern decltype(&cuMemcpyWithAttributesAsync) p_cuMemcpyWithAttributesAsync; +#else +// cuMemcpyWithAttributesAsync doesn't exist in CUDA < 13.2 headers, so use a +// void* placeholder. The pointer is always null when built against older CUDA. +extern void* p_cuMemcpyWithAttributesAsync; +#endif + // ============================================================================ // NVRTC function pointers // @@ -1110,4 +1119,21 @@ CUresult sm_resource_split(CUdevResource* result, unsigned int nbGroups, // Returns true if the cuDevSmResourceSplit function pointer is available. bool has_sm_resource_split() noexcept; +// ============================================================================ +// cuMemcpyWithAttributesAsync wrapper (13.2+) +// +// Calls through p_cuMemcpyWithAttributesAsync if available, otherwise returns +// CUDA_ERROR_NOT_SUPPORTED. This avoids a direct Cython cimport of the +// cydriver cdef function, which would fail at module init on cuda-bindings +// < 13.2 (see https://github.com/NVIDIA/cuda-python/issues/2063). +// ============================================================================ + +// attr is void* so the Cython declaration doesn't reference CUmemcpyAttributes +// (absent from cuda-bindings built against CUDA < 12.8). The C++ side casts it. +CUresult memcpy_with_attributes_async(CUdeviceptr dst, CUdeviceptr src, size_t size, + void* attr, CUstream hStream); + +// Returns true if the cuMemcpyWithAttributesAsync function pointer is available. +bool has_memcpy_with_attributes_async() noexcept; + } // namespace cuda_core diff --git a/cuda_core/cuda/core/_memory/_buffer.pyi b/cuda_core/cuda/core/_memory/_buffer.pyi index 5bf6511fa79..b441754503c 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pyi +++ b/cuda_core/cuda/core/_memory/_buffer.pyi @@ -3,6 +3,7 @@ from __future__ import annotations import cython +from cuda.core._memory._copy_enums import CopyOptions from cuda.core._memory._device_memory_resource import DeviceMemoryResource from cuda.core._memory._ipc import IPCBufferDescriptor from cuda.core._memory._pinned_memory_resource import PinnedMemoryResource @@ -170,7 +171,7 @@ class Buffer: def __exit__(self, exc_type, exc_val, exc_tb): ... - def copy_to(self, dst: Buffer | None=None, *, stream: Stream | GraphBuilder) -> Buffer: + def copy_to(self, dst: Buffer | None=None, *, stream: Stream | GraphBuilder, options: CopyOptions | None=None) -> Buffer: """Copy from this buffer to the dst buffer asynchronously on the given stream. Copies the data from this buffer to the provided dst buffer. @@ -185,10 +186,32 @@ class Buffer: stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder` Keyword argument specifying the stream for the asynchronous copy + options : :class:`~utils.CopyOptions`, optional + Transfer hints (source access order, location hints, overlap mode). + Honored when cuda.bindings and the driver are both CUDA 13.2 or + newer. Not accepted with ``LEGACY_DEFAULT_STREAM``; use + ``PER_THREAD_DEFAULT_STREAM`` instead. Not accepted with a + capturing stream either, since a graph cannot represent these + attributes; use :meth:`graph.GraphNode.memcpy` for a plain, + non-attributed copy node, or pass ``options=None``. On an older + cuda.bindings/driver, ``src_access_order`` values of ``STREAM`` + and ``ANY`` are silently ignored; ``DURING_API_CALL`` raises + instead of silently downgrading its guarantee. + + Raises + ------ + TypeError + If ``options`` is not a :class:`~utils.CopyOptions` instance, or + if ``options`` is given together with ``LEGACY_DEFAULT_STREAM`` + or a stream currently in graph capture mode. + RuntimeError + If ``options.src_access_order`` is ``DURING_API_CALL`` and + cuda.bindings or the driver is older than CUDA 13.2: falling + back to a plain copy cannot honor that guarantee. """ - def copy_from(self, src: Buffer, *, stream: Stream | GraphBuilder) -> None: + def copy_from(self, src: Buffer, *, stream: Stream | GraphBuilder, options: CopyOptions | None=None) -> None: """Copy from the src buffer to this buffer asynchronously on the given stream. Parameters @@ -198,7 +221,28 @@ class Buffer: stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder` Keyword argument specifying the stream for the asynchronous copy + options : :class:`~utils.CopyOptions`, optional + Transfer hints (source access order, location hints, overlap mode). + Honored when cuda.bindings and the driver are both CUDA 13.2 or + newer. Not accepted with ``LEGACY_DEFAULT_STREAM``; use + ``PER_THREAD_DEFAULT_STREAM`` instead. Not accepted with a + capturing stream either, since a graph cannot represent these + attributes; use :meth:`graph.GraphNode.memcpy` for a plain, + non-attributed copy node, or pass ``options=None``. On an older + cuda.bindings/driver, ``src_access_order`` values of ``STREAM`` + and ``ANY`` are silently ignored; ``DURING_API_CALL`` raises + instead of silently downgrading its guarantee. + Raises + ------ + TypeError + If ``options`` is not a :class:`~utils.CopyOptions` instance, or + if ``options`` is given together with ``LEGACY_DEFAULT_STREAM`` + or a stream currently in graph capture mode. + RuntimeError + If ``options.src_access_order`` is ``DURING_API_CALL`` and + cuda.bindings or the driver is older than CUDA 13.2: falling + back to a plain copy cannot honor that guarantee. """ def fill(self, value: int | BufferProtocol, *, stream: Stream | GraphBuilder) -> None: diff --git a/cuda_core/cuda/core/_memory/_buffer.pyx b/cuda_core/cuda/core/_memory/_buffer.pyx index 552bb0bcc8d..6c983c1de26 100644 --- a/cuda_core/cuda/core/_memory/_buffer.pyx +++ b/cuda_core/cuda/core/_memory/_buffer.pyx @@ -27,13 +27,20 @@ from cuda.core._resource_handles cimport ( ) from cuda.core.typing import DevicePointerType -from cuda.core._stream cimport Stream, Stream_accept, default_stream +from cuda.core._memory._copy_attributes cimport _with_attributes_available +from cuda.core._memory._copy_attributes cimport _to_cu_memcpy_attributes # no-cython-lint + +IF CUDA_CORE_BUILD_MAJOR >= 13: + from cuda.core._resource_handles cimport memcpy_with_attributes_async + +from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token, default_stream from cuda.core._utils.cuda_utils cimport HANDLE_RETURN, _parse_fill_value import sys from collections.abc import Sequence from typing import TYPE_CHECKING +from cuda.core._memory._copy_enums import CopyOptions, _reject_unsupported_during_api_call from cuda.core._utils.pycompat import BufferProtocol from cuda.core._dlpack import classify_dl_device, make_py_capsule from cuda.core._device import Device @@ -159,6 +166,75 @@ cdef inline void _init_memory_attrs(Buffer self): self._mem_attrs_inited.store(True, memory_order_release) +cdef bint _stream_is_capturing(Stream s): + cdef cydriver.CUstreamCaptureStatus cap_status + IF CUDA_CORE_BUILD_MAJOR >= 13: + HANDLE_RETURN(cydriver.cuStreamGetCaptureInfo(as_cu(s._h_stream), &cap_status, + NULL, NULL, NULL, NULL, NULL)) + ELSE: + HANDLE_RETURN(cydriver.cuStreamGetCaptureInfo(as_cu(s._h_stream), &cap_status, + NULL, NULL, NULL, NULL)) + return cap_status == cydriver.CU_STREAM_CAPTURE_STATUS_ACTIVE + + +cdef void _do_copy_with_attributes( + cydriver.CUdeviceptr dst, cydriver.CUdeviceptr src, size_t nbytes, + object options, cydriver.CUstream hstream, +): + IF CUDA_CORE_BUILD_MAJOR >= 13: + # Routed through the memcpy_with_attributes_async() C++ shim since + # cydriver.cuMemcpyWithAttributesAsync is absent from cuda-bindings < 13.2. + cdef cydriver.CUmemcpyAttributes cu_attr = _to_cu_memcpy_attributes(options) + with nogil: + HANDLE_RETURN(memcpy_with_attributes_async(dst, src, nbytes, &cu_attr, hstream)) + ELSE: + pass # unreachable: _with_attributes_available() is always False on CUDA 12 + + +cdef void _dispatch_buffer_copy( + cydriver.CUdeviceptr dst, cydriver.CUdeviceptr src, size_t nbytes, + Stream s, object options, str method_name, +): + """Submit a single copy, honoring CopyOptions when the attributes path is usable.""" + if options is None: + with nogil: + HANDLE_RETURN(cydriver.cuMemcpyAsync(dst, src, nbytes, as_cu(s._h_stream))) + return + if not isinstance(options, CopyOptions): + raise TypeError( + f"{method_name}: options must be CopyOptions, got {type(options).__name__}" + ) + if Stream_is_legacy_default_token(s): + raise TypeError( + f"{method_name} does not accept LEGACY_DEFAULT_STREAM with options " + "(matches copy_batch); cuMemcpyWithAttributesAsync rejects it outright, " + "unlike PER_THREAD_DEFAULT_STREAM, which is a real stream to the driver " + "and is accepted. Pass an explicit stream, PER_THREAD_DEFAULT_STREAM, " + "or options=None." + ) + if _stream_is_capturing(s): + raise TypeError( + f"{method_name} does not support graph capture with options " + "(matches copy_batch); the driver has no graph-node form of " + "cuMemcpyWithAttributesAsync, so options cannot be honored in a graph. " + "Use GraphNode.memcpy for a plain (non-attributed) copy node, or pass " + "options=None." + ) + if _with_attributes_available(): + _do_copy_with_attributes(dst, src, nbytes, options, as_cu(s._h_stream)) + else: + _reject_unsupported_during_api_call( + options.src_access_order, + "cuda.bindings and the driver to both report CUDA 13.2 or newer " + "(cuMemcpyWithAttributesAsync is unavailable here)", + ) + # STREAM and ANY never require access sooner than stream order, so + # cuMemcpyAsync satisfies them; options are otherwise silently + # ignored on this pre-CUDA-13.2 fallback path, matching copy_batch. + with nogil: + HANDLE_RETURN(cydriver.cuMemcpyAsync(dst, src, nbytes, as_cu(s._h_stream))) + + cdef class Buffer: """Represent a handle to allocated memory. @@ -393,7 +469,8 @@ cdef class Buffer: self.close() return False - def copy_to(self, dst: Buffer | None = None, *, stream: Stream | GraphBuilder) -> Buffer: + def copy_to(self, dst: Buffer | None = None, *, stream: Stream | GraphBuilder, + options: CopyOptions | None = None) -> Buffer: """Copy from this buffer to the dst buffer asynchronously on the given stream. Copies the data from this buffer to the provided dst buffer. @@ -408,6 +485,28 @@ cdef class Buffer: stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder` Keyword argument specifying the stream for the asynchronous copy + options : :class:`~utils.CopyOptions`, optional + Transfer hints (source access order, location hints, overlap mode). + Honored when cuda.bindings and the driver are both CUDA 13.2 or + newer. Not accepted with ``LEGACY_DEFAULT_STREAM``; use + ``PER_THREAD_DEFAULT_STREAM`` instead. Not accepted with a + capturing stream either, since a graph cannot represent these + attributes; use :meth:`graph.GraphNode.memcpy` for a plain, + non-attributed copy node, or pass ``options=None``. On an older + cuda.bindings/driver, ``src_access_order`` values of ``STREAM`` + and ``ANY`` are silently ignored; ``DURING_API_CALL`` raises + instead of silently downgrading its guarantee. + + Raises + ------ + TypeError + If ``options`` is not a :class:`~utils.CopyOptions` instance, or + if ``options`` is given together with ``LEGACY_DEFAULT_STREAM`` + or a stream currently in graph capture mode. + RuntimeError + If ``options.src_access_order`` is ``DURING_API_CALL`` and + cuda.bindings or the driver is older than CUDA 13.2: falling + back to a plain copy cannot honor that guarantee. """ cdef Stream s = Stream_accept(stream) @@ -424,12 +523,12 @@ cdef class Buffer: raise ValueError( "buffer sizes mismatch between src and dst (sizes " f"are: src={src_size}, dst={dst_size})" ) - with nogil: - HANDLE_RETURN(cydriver.cuMemcpyAsync( - as_cu(dst._h_ptr), as_cu(self._h_ptr), src_size, as_cu(s._h_stream))) + _dispatch_buffer_copy( + as_cu(dst._h_ptr), as_cu(self._h_ptr), src_size, s, options, "copy_to") return dst - def copy_from(self, src: Buffer, *, stream: Stream | GraphBuilder) -> None: + def copy_from(self, src: Buffer, *, stream: Stream | GraphBuilder, + options: CopyOptions | None = None) -> None: """Copy from the src buffer to this buffer asynchronously on the given stream. Parameters @@ -439,7 +538,28 @@ cdef class Buffer: stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder` Keyword argument specifying the stream for the asynchronous copy + options : :class:`~utils.CopyOptions`, optional + Transfer hints (source access order, location hints, overlap mode). + Honored when cuda.bindings and the driver are both CUDA 13.2 or + newer. Not accepted with ``LEGACY_DEFAULT_STREAM``; use + ``PER_THREAD_DEFAULT_STREAM`` instead. Not accepted with a + capturing stream either, since a graph cannot represent these + attributes; use :meth:`graph.GraphNode.memcpy` for a plain, + non-attributed copy node, or pass ``options=None``. On an older + cuda.bindings/driver, ``src_access_order`` values of ``STREAM`` + and ``ANY`` are silently ignored; ``DURING_API_CALL`` raises + instead of silently downgrading its guarantee. + Raises + ------ + TypeError + If ``options`` is not a :class:`~utils.CopyOptions` instance, or + if ``options`` is given together with ``LEGACY_DEFAULT_STREAM`` + or a stream currently in graph capture mode. + RuntimeError + If ``options.src_access_order`` is ``DURING_API_CALL`` and + cuda.bindings or the driver is older than CUDA 13.2: falling + back to a plain copy cannot honor that guarantee. """ cdef Stream s = Stream_accept(stream) cdef size_t dst_size = self._size @@ -449,9 +569,8 @@ cdef class Buffer: raise ValueError( "buffer sizes mismatch between src and dst (sizes " f"are: src={src_size}, dst={dst_size})" ) - with nogil: - HANDLE_RETURN(cydriver.cuMemcpyAsync( - as_cu(self._h_ptr), as_cu(src._h_ptr), dst_size, as_cu(s._h_stream))) + _dispatch_buffer_copy( + as_cu(self._h_ptr), as_cu(src._h_ptr), dst_size, s, options, "copy_from") def fill(self, value: int | BufferProtocol, *, stream: Stream | GraphBuilder) -> None: """Fill this buffer with a repeating byte pattern. diff --git a/cuda_core/cuda/core/_memory/_copy_attributes.pxd b/cuda_core/cuda/core/_memory/_copy_attributes.pxd index 726e2553922..96c213dfec5 100644 --- a/cuda_core/cuda/core/_memory/_copy_attributes.pxd +++ b/cuda_core/cuda/core/_memory/_copy_attributes.pxd @@ -11,8 +11,19 @@ from cuda.core._utils.version cimport cy_binding_version, cy_driver_version # n IF CUDA_CORE_BUILD_MAJOR >= 13: + from cuda.core._resource_handles cimport has_memcpy_with_attributes_async + cdef inline bint _with_attributes_available(): - return cy_driver_version() >= (13, 2, 0) and cy_binding_version() >= (13, 2, 0) + # has_memcpy_with_attributes_async() says whether the installed + # cuda-bindings actually exports cuMemcpyWithAttributesAsync (13.2+); + # the version checks alone are not sufficient, since cuda.core's build + # can be paired with a cuda-bindings install older than what it built + # against (see https://github.com/NVIDIA/cuda-python/issues/2063). + return ( + has_memcpy_with_attributes_async() + and cy_driver_version() >= (13, 2, 0) + and cy_binding_version() >= (13, 2, 0) + ) ELSE: cdef inline bint _with_attributes_available(): return False diff --git a/cuda_core/cuda/core/_memory/_copy_enums.py b/cuda_core/cuda/core/_memory/_copy_enums.py index 568aca02ddf..84c72e71110 100644 --- a/cuda_core/cuda/core/_memory/_copy_enums.py +++ b/cuda_core/cuda/core/_memory/_copy_enums.py @@ -160,6 +160,40 @@ def _to_driver_flags(self) -> int: _OVERLAP_MODE_TO_DRIVER = {} +def _reject_unsupported_during_api_call( + src_access_order: MemcpySrcAccessOrder, requirement: str, *, index: int | None = None +) -> None: + """Raise if ``src_access_order`` is DURING_API_CALL but the native attributes + path (``cuMemcpyWithAttributesAsync`` / ``cuMemcpyBatchAsync``) is unavailable. + + STREAM and ANY never promise access sooner than stream order, so a plain + ``cuMemcpyAsync`` fallback satisfies them; DURING_API_CALL specifically + promises all source reads complete before the call returns, which + ``cuMemcpyAsync`` cannot provide (it reads the source in stream order + only). Silently downgrading that guarantee would let a caller reuse or + overwrite the source buffer before the real, stream-ordered read + happens: a silent data race, not a missed optimization. ``requirement`` + names what the native path needs and why it is unavailable here; + ``index`` identifies the offending copy within a batch. + + Internal, but deliberately importable: shared between the per-buffer and + batched fallback paths so both raise identically, and directly testable + without needing an actual old driver/bindings install. + """ + if src_access_order != MemcpySrcAccessOrder.DURING_API_CALL: + return + where = f" at index {index}" if index is not None else "" + raise RuntimeError( + f"src_access_order=DURING_API_CALL{where} requires {requirement}. A " + "plain cuMemcpyAsync fallback reads the source in stream order only, " + "which would silently violate the guarantee that all source reads " + "complete before the call returns, letting the caller reuse the " + "source buffer before the real (stream-ordered) read happens. Use " + "src_access_order=STREAM or ANY, or omit options, if that works for " + "your use case." + ) + + def _attr_run_starts(attrs: Sequence[CopyOptions]) -> list[int]: """Return the start index of each maximal run of equal attributes. diff --git a/cuda_core/cuda/core/_memory/_copy_ops.pyi b/cuda_core/cuda/core/_memory/_copy_ops.pyi index f281f0913c7..9923f2a0042 100644 --- a/cuda_core/cuda/core/_memory/_copy_ops.pyi +++ b/cuda_core/cuda/core/_memory/_copy_ops.pyi @@ -42,7 +42,10 @@ def copy_batch(stream: Stream, srcs: Sequence[Buffer], dsts: Sequence[Buffer], * (mirrors :func:`launch`). Does not accept a capturing stream (including a :class:`~graph.GraphBuilder`'s underlying stream); use :meth:`graph.GraphNode.memcpy` or per-buffer - :meth:`Buffer.copy_to` to build copies into a graph. + :meth:`Buffer.copy_to` to build copies into a graph. Does not accept + ``LEGACY_DEFAULT_STREAM``, which ``cuMemcpyBatchAsync`` rejects + outright; ``PER_THREAD_DEFAULT_STREAM`` is a real stream to the + driver and is accepted. srcs : Sequence[:class:`Buffer`] Source buffers. Must be a sequence, not a single Buffer. dsts : Sequence[:class:`Buffer`] @@ -57,10 +60,14 @@ def copy_batch(stream: Stream, srcs: Sequence[Buffer], dsts: Sequence[Buffer], * ValueError If lengths or sizes mismatch. TypeError - If a single Buffer is passed instead of a sequence, if a - default-stream token (``LEGACY_DEFAULT_STREAM`` / - ``PER_THREAD_DEFAULT_STREAM``) is passed, or if the stream is - currently in graph capture mode. + If a single Buffer is passed instead of a sequence, if + ``LEGACY_DEFAULT_STREAM`` is passed, or if the stream is currently + in graph capture mode. + RuntimeError + If any copy requests ``src_access_order=DURING_API_CALL`` and the + native ``cuMemcpyBatchAsync`` path is unavailable (see Notes): the + per-copy ``cuMemcpyAsync`` fallback reads the source in stream + order only, which cannot honor that guarantee. Notes ----- @@ -79,7 +86,11 @@ def copy_batch(stream: Stream, srcs: Sequence[Buffer], dsts: Sequence[Buffer], * On pre-CUDA 13 installs the copies fall back to a Python-level loop over ``cuMemcpyAsync``, so the potential performance benefit of - asynchronous batched copies is not realized. :class:`CopyOptions` are - silently ignored on the fallback path. + asynchronous batched copies is not realized. ``src_access_order`` values + of ``STREAM`` and ``ANY`` are silently ignored on the fallback path + (stream-ordered access already satisfies both); ``DURING_API_CALL`` + raises ``RuntimeError`` instead, since silently downgrading it to + stream-ordered access would let a caller reuse the source buffer before + the real read happens. """ \ No newline at end of file diff --git a/cuda_core/cuda/core/_memory/_copy_ops.pyx b/cuda_core/cuda/core/_memory/_copy_ops.pyx index 646ce9000dd..e57be2e40a0 100644 --- a/cuda_core/cuda/core/_memory/_copy_ops.pyx +++ b/cuda_core/cuda/core/_memory/_copy_ops.pyx @@ -13,7 +13,7 @@ from cuda.bindings cimport cydriver from cuda.core._memory._buffer cimport Buffer, Buffer_coerce_batch from cuda.core._memory._copy_attributes cimport _to_cu_memcpy_attributes # no-cython-lint from cuda.core._resource_handles cimport as_cu -from cuda.core._stream cimport Stream, Stream_accept, Stream_is_default_token +from cuda.core._stream cimport Stream, Stream_accept, Stream_is_legacy_default_token from cuda.core._utils.cuda_utils cimport HANDLE_RETURN # cy_driver_version and _attr_run_starts are referenced only from CUDA 13 @@ -21,7 +21,11 @@ from cuda.core._utils.cuda_utils cimport HANDLE_RETURN # a pragma to be seen as used. from cuda.core._utils.version cimport cy_driver_version # no-cython-lint -from cuda.core._memory._copy_enums import CopyOptions, _attr_run_starts # no-cython-lint +from cuda.core._memory._copy_enums import ( + CopyOptions, + _attr_run_starts, # no-cython-lint + _reject_unsupported_during_api_call, +) _SINGLE_COPY_HINT = "Buffer.copy_to / Buffer.copy_from" @@ -110,7 +114,10 @@ def copy_batch( (mirrors :func:`launch`). Does not accept a capturing stream (including a :class:`~graph.GraphBuilder`\'s underlying stream); use :meth:`graph.GraphNode.memcpy` or per-buffer - :meth:`Buffer.copy_to` to build copies into a graph. + :meth:`Buffer.copy_to` to build copies into a graph. Does not accept + ``LEGACY_DEFAULT_STREAM``, which ``cuMemcpyBatchAsync`` rejects + outright; ``PER_THREAD_DEFAULT_STREAM`` is a real stream to the + driver and is accepted. srcs : Sequence[:class:`Buffer`] Source buffers. Must be a sequence, not a single Buffer. dsts : Sequence[:class:`Buffer`] @@ -125,10 +132,14 @@ def copy_batch( ValueError If lengths or sizes mismatch. TypeError - If a single Buffer is passed instead of a sequence, if a - default-stream token (``LEGACY_DEFAULT_STREAM`` / - ``PER_THREAD_DEFAULT_STREAM``) is passed, or if the stream is - currently in graph capture mode. + If a single Buffer is passed instead of a sequence, if + ``LEGACY_DEFAULT_STREAM`` is passed, or if the stream is currently + in graph capture mode. + RuntimeError + If any copy requests ``src_access_order=DURING_API_CALL`` and the + native ``cuMemcpyBatchAsync`` path is unavailable (see Notes): the + per-copy ``cuMemcpyAsync`` fallback reads the source in stream + order only, which cannot honor that guarantee. Notes ----- @@ -147,8 +158,12 @@ def copy_batch( On pre-CUDA 13 installs the copies fall back to a Python-level loop over ``cuMemcpyAsync``, so the potential performance benefit of - asynchronous batched copies is not realized. :class:`CopyOptions` are - silently ignored on the fallback path. + asynchronous batched copies is not realized. ``src_access_order`` values + of ``STREAM`` and ``ANY`` are silently ignored on the fallback path + (stream-ordered access already satisfies both); ``DURING_API_CALL`` + raises ``RuntimeError`` instead, since silently downgrading it to + stream-ordered access would let a caller reuse the source buffer before + the real read happens. """ cdef tuple src_bufs = Buffer_coerce_batch(srcs, "copy_batch", _SINGLE_COPY_HINT) @@ -162,11 +177,12 @@ def copy_batch( cdef Stream s = Stream_accept(stream) - if Stream_is_default_token(s): + if Stream_is_legacy_default_token(s): raise TypeError( - "copy_batch does not accept a default-stream token " - "(LEGACY_DEFAULT_STREAM / PER_THREAD_DEFAULT_STREAM); " - "pass an explicit stream" + "copy_batch does not accept LEGACY_DEFAULT_STREAM; cuMemcpyBatchAsync " + "rejects it outright, unlike PER_THREAD_DEFAULT_STREAM, which is a real " + "stream to the driver and is accepted. Pass an explicit stream or " + "PER_THREAD_DEFAULT_STREAM." ) cdef cydriver.CUstreamCaptureStatus _cap_status @@ -208,17 +224,36 @@ cdef void _do_copy_batch(tuple src_bufs, tuple dst_bufs, Stream s, tuple attr_tu if _batch_entry_point_available(): _do_copy_batch_native(src_bufs, dst_bufs, s, attr_tuple) else: + _reject_during_api_call_fallback(attr_tuple) _do_copy_batch_loop(src_bufs, dst_bufs, s) ELSE: + _reject_during_api_call_fallback(attr_tuple) _do_copy_batch_loop(src_bufs, dst_bufs, s) +cdef void _reject_during_api_call_fallback(tuple attr_tuple): + """Raise before the per-copy cuMemcpyAsync loop if any copy needs + DURING_API_CALL, which that fallback cannot honor (see + _reject_unsupported_during_api_call for why this must raise rather than + silently ignore the option, unlike STREAM and ANY). + """ + cdef Py_ssize_t i + for i in range(len(attr_tuple)): + _reject_unsupported_during_api_call( + (attr_tuple[i]).src_access_order, + "cuda.core built against CUDA 13 headers and cuda.bindings/driver " + "13.0 or newer (cuMemcpyBatchAsync is unavailable here)", + index=i, + ) + + cdef void _do_copy_batch_loop(tuple src_bufs, tuple dst_bufs, Stream s): """Per-copy cuMemcpyAsync fallback where the batch entry point is absent. Issues copies one at a time, so the performance benefit of batching is - not realized. Callers guarantee the options are defaults; copy_batch - rejects anything else before reaching here. + not realized. STREAM and ANY are silently ignored here (satisfied by + stream-ordered cuMemcpyAsync regardless); DURING_API_CALL is rejected by + _reject_during_api_call_fallback before this is ever called. """ cdef Py_ssize_t n = len(src_bufs) cdef Py_ssize_t i diff --git a/cuda_core/cuda/core/_resource_handles.pxd b/cuda_core/cuda/core/_resource_handles.pxd index 2717610a01a..568af27ac2e 100644 --- a/cuda_core/cuda/core/_resource_handles.pxd +++ b/cuda_core/cuda/core/_resource_handles.pxd @@ -351,3 +351,11 @@ cdef cydriver.CUresult sm_resource_split( const cydriver.CUdevResource* input, cydriver.CUdevResource* remainder, unsigned int flags, void* groupParams) nogil cdef bint has_sm_resource_split() noexcept nogil + +# cuMemcpyWithAttributesAsync (13.2+ — calls through function pointer, safe on older bindings) +# attr is void* here to avoid referencing CUmemcpyAttributes (absent from +# cuda-bindings built against CUDA < 12.8). The C++ side casts it. +cdef cydriver.CUresult memcpy_with_attributes_async( + cydriver.CUdeviceptr dst, cydriver.CUdeviceptr src, size_t size, + void* attr, cydriver.CUstream hStream) nogil +cdef bint has_memcpy_with_attributes_async() noexcept nogil diff --git a/cuda_core/cuda/core/_resource_handles.pyx b/cuda_core/cuda/core/_resource_handles.pyx index aabdb2ea51e..c7de24666f8 100644 --- a/cuda_core/cuda/core/_resource_handles.pyx +++ b/cuda_core/cuda/core/_resource_handles.pyx @@ -243,6 +243,14 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": unsigned int flags, void* groupParams) nogil bint has_sm_resource_split "cuda_core::has_sm_resource_split" () noexcept nogil + # cuMemcpyWithAttributesAsync (13.2+ wrapper — avoids direct cydriver cimport) + # attr is void* to avoid referencing CUmemcpyAttributes (absent from + # cuda-bindings built against CUDA < 12.8). The C++ side casts it. + cydriver.CUresult memcpy_with_attributes_async "cuda_core::memcpy_with_attributes_async" ( + cydriver.CUdeviceptr dst, cydriver.CUdeviceptr src, size_t size, + void* attr, cydriver.CUstream hStream) nogil + bint has_memcpy_with_attributes_async "cuda_core::has_memcpy_with_attributes_async" () noexcept nogil + # Array / mipmapped-array / texture / surface handles (PR #467) OpaqueArrayHandle create_array_handle "cuda_core::create_array_handle" ( const cydriver.CUDA_ARRAY3D_DESCRIPTOR& desc) except+ nogil @@ -372,6 +380,9 @@ cdef extern from "_cpp/resource_handles.hpp" namespace "cuda_core": # SM resource split (13.1+) void* p_cuDevSmResourceSplit "reinterpret_cast(cuda_core::p_cuDevSmResourceSplit)" + # cuMemcpyWithAttributesAsync (13.2+) + void* p_cuMemcpyWithAttributesAsync "reinterpret_cast(cuda_core::p_cuMemcpyWithAttributesAsync)" + # NVRTC void* p_nvrtcDestroyProgram "reinterpret_cast(cuda_core::p_nvrtcDestroyProgram)" @@ -418,6 +429,7 @@ cdef void _init_driver_fn_pointers() noexcept: global p_cuLinkDestroy global p_cuGraphicsUnmapResources, p_cuGraphicsUnregisterResource global p_cuDevSmResourceSplit + global p_cuMemcpyWithAttributesAsync global p_cuArray3DCreate, p_cuArrayDestroy global p_cuMipmappedArrayCreate, p_cuMipmappedArrayDestroy, p_cuMipmappedArrayGetLevel global p_cuTexObjectCreate, p_cuTexObjectDestroy @@ -506,6 +518,9 @@ cdef void _init_driver_fn_pointers() noexcept: # SM resource split (13.1+ — may not exist in older cuda-bindings) p_cuDevSmResourceSplit = _get_optional_driver_fn("cuDevSmResourceSplit") + # cuMemcpyWithAttributesAsync (13.2+ — may not exist in older cuda-bindings) + p_cuMemcpyWithAttributesAsync = _get_optional_driver_fn("cuMemcpyWithAttributesAsync") + _init_driver_fn_pointers() initialize_deferred_cleanup() diff --git a/cuda_core/cuda/core/_stream.pxd b/cuda_core/cuda/core/_stream.pxd index dc9a2da826c..b9c8677e130 100644 --- a/cuda_core/cuda/core/_stream.pxd +++ b/cuda_core/cuda/core/_stream.pxd @@ -24,3 +24,4 @@ cdef class Stream: cpdef Stream default_stream() cpdef Stream Stream_accept(arg, bint allow_stream_protocol=*) cdef bint Stream_is_default_token(Stream self) noexcept nogil +cdef bint Stream_is_legacy_default_token(Stream self) noexcept nogil diff --git a/cuda_core/cuda/core/_stream.pyx b/cuda_core/cuda/core/_stream.pyx index c8c5faf74bc..a376db96e11 100644 --- a/cuda_core/cuda/core/_stream.pyx +++ b/cuda_core/cuda/core/_stream.pyx @@ -493,6 +493,18 @@ cdef inline bint Stream_is_default_token(Stream self) noexcept nogil: return h == cydriver.CU_STREAM_LEGACY or h == cydriver.CU_STREAM_PER_THREAD +cdef inline bint Stream_is_legacy_default_token(Stream self) noexcept nogil: + """Return True only for CU_STREAM_LEGACY. + + Unlike CU_STREAM_PER_THREAD, the legacy default stream token is rejected + outright (CUDA_ERROR_INVALID_VALUE) by cuMemcpyWithAttributesAsync and + cuMemcpyBatchAsync; CU_STREAM_PER_THREAD is a real stream to those entry + points and is accepted normally. Use this narrower check, not + Stream_is_default_token, wherever that distinction matters. + """ + return as_cu(self._h_stream) == cydriver.CU_STREAM_LEGACY + + cdef inline int Stream_get_ctx(Stream self, ContextHandle* h_context) except?-1 nogil: """Resolve the stream's context handle into ``h_context``.""" cdef cydriver.CUcontext ctx diff --git a/cuda_core/docs/source/release/1.2.0-notes.rst b/cuda_core/docs/source/release/1.2.0-notes.rst index 34c39bc690a..6344047026a 100644 --- a/cuda_core/docs/source/release/1.2.0-notes.rst +++ b/cuda_core/docs/source/release/1.2.0-notes.rst @@ -9,14 +9,25 @@ New features ------------ -- Added :func:`utils.copy_batch`, which submits many buffer-to-buffer copies - in a single ``cuMemcpyBatchAsync`` call. Per-copy behavior is controlled - by the new :class:`utils.CopyOptions` dataclass. Requires ``cuda.core`` - built against CUDA 13, ``cuda.bindings`` 13.0+, and a driver reporting - CUDA 13.0 or newer; otherwise falls back to a per-copy ``cuMemcpyAsync`` - loop with options silently ignored. Graph capture and default-stream tokens - are rejected. Copies within a batch must not alias. - (`#1333 `__) +- Added :class:`utils.CopyOptions` (source access order, location hints, + overlap mode) for buffer-to-buffer copies. The new + :func:`utils.copy_batch` accepts it and submits many copies in a single + ``cuMemcpyBatchAsync`` call, requiring ``cuda.core`` built against CUDA 13 + plus ``cuda.bindings``/driver 13.0 or newer. :meth:`Buffer.copy_to` and + :meth:`Buffer.copy_from` also accept it now, as a new ``options`` keyword, + submitting a single copy via ``cuMemcpyWithAttributesAsync`` and requiring + ``cuda.bindings``/driver 13.2 or newer. Both reject + ``LEGACY_DEFAULT_STREAM`` with ``TypeError`` (``PER_THREAD_DEFAULT_STREAM`` + is accepted); ``copy_batch`` always rejects graph capture, while + ``Buffer.copy_to``/``copy_from`` reject it only when ``options`` is given. + On an older ``cuda.bindings``/driver install, ``src_access_order`` values + of ``STREAM`` and ``ANY`` silently fall back to plain ``cuMemcpyAsync``; + ``DURING_API_CALL`` raises ``RuntimeError`` instead, since that fallback + cannot honor its guarantee that all source reads complete before the call + returns. Copies within a ``copy_batch`` call must not alias. + (`#1333 `__, + `#2365 `__) + - Added the ``programmatic_stream_serialization`` option to :class:`LaunchConfig`, which sets ``cudaLaunchAttributeProgrammaticStreamSerialization`` so a kernel can diff --git a/cuda_core/tests/memory/test_copy_batch.py b/cuda_core/tests/memory/test_copy_batch.py index 81abe9f9636..73fb438fc2d 100644 --- a/cuda_core/tests/memory/test_copy_batch.py +++ b/cuda_core/tests/memory/test_copy_batch.py @@ -275,13 +275,33 @@ def test_capturing_stream_is_rejected(self, copy_batch_device, device_bufs): gb.close() @pytest.mark.agent_authored(model="Claude Sonnet 4.6") - @pytest.mark.parametrize( - "default_stream", - [LEGACY_DEFAULT_STREAM, PER_THREAD_DEFAULT_STREAM], - ids=["legacy", "per_thread"], - ) - def test_default_stream_token_is_rejected(self, init_cuda, h2d_bufs, default_stream): - """Default-stream tokens must be rejected with a clear TypeError.""" + def test_legacy_default_stream_token_is_rejected(self, init_cuda, h2d_bufs): + """LEGACY_DEFAULT_STREAM must be rejected with a clear TypeError. + + cuMemcpyBatchAsync rejects the legacy token outright + (CUDA_ERROR_INVALID_VALUE); copy_batch surfaces this before ever + calling the driver. + """ srcs, dsts = h2d_bufs - with pytest.raises(TypeError, match="default-stream token"): - copy_batch(default_stream, srcs, dsts) + with pytest.raises(TypeError, match="LEGACY_DEFAULT_STREAM"): + copy_batch(LEGACY_DEFAULT_STREAM, srcs, dsts) + + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + def test_per_thread_default_stream_token_is_accepted(self, copy_batch_device): + """PER_THREAD_DEFAULT_STREAM is a real stream to the driver and works + like any explicit stream for copy_batch, unlike LEGACY_DEFAULT_STREAM. + """ + pinned_mr = LegacyPinnedMemoryResource() + device_mr = copy_batch_device.memory_resource + src = pinned_mr.allocate(COPY_BATCH_SIZE) + dst = device_mr.allocate(COPY_BATCH_SIZE, stream=PER_THREAD_DEFAULT_STREAM) + set_buffer(src, 99) + + copy_batch(PER_THREAD_DEFAULT_STREAM, [src], [dst]) + copy_batch_device.sync() + + assert compare_buffer_to_constant(dst, 99) + + src.close(PER_THREAD_DEFAULT_STREAM) + dst.close(PER_THREAD_DEFAULT_STREAM) + copy_batch_device.sync() diff --git a/cuda_core/tests/memory/test_copy_batch_options.py b/cuda_core/tests/memory/test_copy_batch_options.py index 31d3a28f4cc..275c5c3f327 100644 --- a/cuda_core/tests/memory/test_copy_batch_options.py +++ b/cuda_core/tests/memory/test_copy_batch_options.py @@ -20,10 +20,12 @@ ) from cuda.core import Host, LegacyPinnedMemoryResource -from cuda.core._memory._copy_enums import _attr_run_starts +from cuda.core._memory._copy_enums import _attr_run_starts, _reject_unsupported_during_api_call from cuda.core._memory._copy_ops import ( _normalize_copy_options, ) +from cuda.core._stream import PER_THREAD_DEFAULT_STREAM +from cuda.core._utils.version import binding_version, driver_version from cuda.core.utils import ( CopyOptions, MemcpyOverlapMode, @@ -32,6 +34,11 @@ ) +def _batch_native_available(): + """True when copy_batch will actually use cuMemcpyBatchAsync.""" + return binding_version() >= (13, 0, 0) and driver_version() >= (13, 0, 0) + + class TestOptionsEncoding: """How ``options`` becomes the driver's ``attrs`` / ``attrsIdxs`` pair. @@ -95,6 +102,43 @@ def test_single_element(self): assert _attr_run_starts([CopyOptions()]) == [0] +class TestRejectUnsupportedDuringApiCall: + """``_reject_unsupported_during_api_call`` guards the one hazardous fallback. + + Pure logic, no CUDA: this is what both ``Buffer.copy_to``/``copy_from`` + and ``copy_batch`` call before falling back to a plain ``cuMemcpyAsync`` + when the native attributes path is unavailable. STREAM and ANY never + promise access sooner than stream order, so cuMemcpyAsync satisfies them + silently; DURING_API_CALL promises all source reads complete before the + call returns, which cuMemcpyAsync cannot provide, so it must raise + instead of silently downgrading that guarantee. + """ + + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + def test_during_api_call_raises(self): + with pytest.raises(RuntimeError, match="src_access_order=DURING_API_CALL"): + _reject_unsupported_during_api_call(MemcpySrcAccessOrder.DURING_API_CALL, "some requirement") + + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + def test_during_api_call_message_names_requirement_and_index(self): + with pytest.raises(RuntimeError, match="requires some requirement") as exc_info: + _reject_unsupported_during_api_call(MemcpySrcAccessOrder.DURING_API_CALL, "some requirement", index=5) + assert "at index 5" in str(exc_info.value) + + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + def test_during_api_call_message_omits_index_when_not_given(self): + with pytest.raises(RuntimeError) as exc_info: + _reject_unsupported_during_api_call(MemcpySrcAccessOrder.DURING_API_CALL, "some requirement") + assert "at index" not in str(exc_info.value) + + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + @pytest.mark.parametrize("order", [MemcpySrcAccessOrder.STREAM, MemcpySrcAccessOrder.ANY]) + def test_stream_and_any_do_not_raise(self, order): + """Stream-ordered access satisfies both, so no fallback hazard exists.""" + _reject_unsupported_during_api_call(order, "some requirement") + _reject_unsupported_during_api_call(order, "some requirement", index=0) + + class TestCopyBatchOptions: """Each ``CopyOptions`` field is accepted and does not corrupt the copy.""" @@ -103,11 +147,18 @@ class TestCopyBatchOptions: ("order", "marker"), [ (MemcpySrcAccessOrder.STREAM, 31), - (MemcpySrcAccessOrder.DURING_API_CALL, 32), (MemcpySrcAccessOrder.ANY, 33), ], ) def test_src_access_order(self, h2d_bufs, copy_stream, order, marker): + """STREAM and ANY are accepted and never corrupt the copy. + + Both are satisfied by stream-ordered access at worst, so this holds + whether the native cuMemcpyBatchAsync path is used or the copy falls + back to a per-copy cuMemcpyAsync loop. DURING_API_CALL is different + (see test_during_api_call): its stronger guarantee cannot be + silently downgraded, so it is tested separately. + """ srcs, dsts = h2d_bufs for i, src in enumerate(srcs): set_buffer(src, i + marker) @@ -118,20 +169,50 @@ def test_src_access_order(self, h2d_bufs, copy_stream, order, marker): for i, dst in enumerate(dsts): assert compare_buffer_to_constant(dst, i + marker) + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + def test_during_api_call(self, h2d_bufs, copy_stream): + """DURING_API_CALL is honored on the native cuMemcpyBatchAsync path. + + On the per-copy cuMemcpyAsync fallback (pre-CUDA-13 build, or + driver/bindings older than 13.0) it must raise RuntimeError instead + of silently downgrading to stream-ordered access, which cannot honor + the guarantee that all source reads complete before the call + returns (see TestRejectUnsupportedDuringApiCall). CI runs both + generations (see ci/test-matrix.yml), so this test must handle both + outcomes rather than assuming the native path is available. + """ + srcs, dsts = h2d_bufs + marker = 32 + for i, src in enumerate(srcs): + set_buffer(src, i + marker) + + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.DURING_API_CALL) + if _batch_native_available(): + copy_batch(copy_stream, srcs, dsts, options=opts) + copy_stream.sync() + for i, dst in enumerate(dsts): + assert compare_buffer_to_constant(dst, i + marker) + else: + with pytest.raises(RuntimeError, match="DURING_API_CALL"): + copy_batch(copy_stream, srcs, dsts, options=opts) + @pytest.mark.agent_authored(model="Claude Opus 5") def test_per_copy_options(self, h2d_bufs, copy_stream): srcs, dsts = h2d_bufs for i, src in enumerate(srcs): set_buffer(src, i + 40) + # DURING_API_CALL is deliberately excluded here: it raises RuntimeError + # rather than silently falling back on pre-CUDA-13 driver/bindings (see + # test_during_api_call), which CI also exercises (ci/test-matrix.yml). + # STREAM and ANY are enough to prove distinct per-copy options don't + # corrupt the data; the encoding itself is covered by TestOptionsEncoding. per_copy_options = [ CopyOptions(src_access_order=MemcpySrcAccessOrder.STREAM), CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY), - CopyOptions(src_access_order=MemcpySrcAccessOrder.DURING_API_CALL), + CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY), CopyOptions(src_access_order=MemcpySrcAccessOrder.STREAM), ] - # The encoding itself is covered by TestOptionsEncoding; here the - # point is that distinct per-copy options do not corrupt the data. copy_batch(copy_stream, srcs, dsts, options=per_copy_options) copy_stream.sync() @@ -241,6 +322,34 @@ def test_default_overlap_mode_does_not_warn(self, h2d_bufs, copy_stream): copy_batch(copy_stream, srcs, dsts, options=CopyOptions()) copy_stream.sync() + @pytest.mark.agent_authored(model="Claude Sonnet 4.6") + def test_options_on_per_thread_default_stream(self, copy_batch_device): + """CopyOptions work on PER_THREAD_DEFAULT_STREAM like any explicit stream. + + Unlike LEGACY_DEFAULT_STREAM (rejected outright, see + TestCopyBatchStreamSemantics in test_copy_batch.py), + PER_THREAD_DEFAULT_STREAM is a real stream to cuMemcpyBatchAsync. + """ + pinned_mr = LegacyPinnedMemoryResource() + device_mr = copy_batch_device.memory_resource + src = pinned_mr.allocate(COPY_BATCH_SIZE) + dst = device_mr.allocate(COPY_BATCH_SIZE, stream=PER_THREAD_DEFAULT_STREAM) + set_buffer(src, 44) + + copy_batch( + PER_THREAD_DEFAULT_STREAM, + [src], + [dst], + options=CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY), + ) + copy_batch_device.sync() + + assert compare_buffer_to_constant(dst, 44) + + src.close(PER_THREAD_DEFAULT_STREAM) + dst.close(PER_THREAD_DEFAULT_STREAM) + copy_batch_device.sync() + class TestCopyOptionsValidation: """``CopyOptions`` rejects invalid enum values at construction.""" diff --git a/cuda_core/tests/memory/test_copy_single_options.py b/cuda_core/tests/memory/test_copy_single_options.py new file mode 100644 index 00000000000..bfaad76dcd7 --- /dev/null +++ b/cuda_core/tests/memory/test_copy_single_options.py @@ -0,0 +1,476 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CopyOptions support for Buffer.copy_to / Buffer.copy_from (issue #2365).""" + +import pytest +from conftest import create_managed_memory_resource_or_skip +from helpers.buffers import compare_equal_buffers, make_scratch_buffer, set_buffer +from helpers.copy_batch import assert_managed_holds + +from cuda.core import Device, Host, LegacyPinnedMemoryResource +from cuda.core._stream import LEGACY_DEFAULT_STREAM, PER_THREAD_DEFAULT_STREAM +from cuda.core._utils.version import binding_version, driver_version +from cuda.core.utils import CopyOptions, MemcpyOverlapMode, MemcpySrcAccessOrder + +SIZE = 4096 + + +def _options_honored(): + """True when cuMemcpyWithAttributesAsync will actually be used for options. + + Mirrors _with_attributes_available() in _buffer.pyx. CI runs a matrix + that includes pre-CUDA-13.2 driver/bindings combinations (see + ci/test-matrix.yml), where this is False and the DURING_API_CALL tests + below must expect a RuntimeError instead of a successful copy. + """ + return driver_version() >= (13, 2, 0) and binding_version() >= (13, 2, 0) + + +@pytest.fixture +def single_copy_device(init_cuda): + device = Device() + device.set_current() + return device + + +@pytest.fixture +def single_copy_stream(single_copy_device): + s = single_copy_device.create_stream() + yield s + s.close() + + +@pytest.fixture +def pinned_mr(): + return LegacyPinnedMemoryResource() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_none_copy_to_data_correct(single_copy_device, single_copy_stream, pinned_mr): + """options=None (default) continues to copy the right bytes.""" + src = make_scratch_buffer(single_copy_device, 0x55, SIZE) + dst = pinned_mr.allocate(SIZE) + + src.copy_to(dst, stream=single_copy_stream) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_none_copy_from_data_correct(single_copy_device, single_copy_stream, pinned_mr): + """copy_from with options=None copies the right bytes.""" + src = make_scratch_buffer(single_copy_device, 0xAA, SIZE) + dst = pinned_mr.allocate(SIZE) + + dst.copy_from(src, stream=single_copy_stream) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +@pytest.mark.parametrize( + ("order", "marker"), + [ + (MemcpySrcAccessOrder.STREAM, 0x31), + (MemcpySrcAccessOrder.ANY, 0x33), + ], +) +def test_src_access_order_copy_to(single_copy_device, single_copy_stream, pinned_mr, order, marker): + """STREAM and ANY are accepted and never corrupt copy_to. + + Both are satisfied by stream-ordered access at worst, so whether + cuMemcpyWithAttributesAsync actually honors the hint (CUDA 13.2+ driver + and cuda.bindings) or the call silently falls back to cuMemcpyAsync, the + copied bytes must be identical either way. DURING_API_CALL is different + (see test_during_api_call_copy_to): its stronger guarantee cannot be + silently downgraded, so it is tested separately. + """ + src = make_scratch_buffer(single_copy_device, marker, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=order) + + src.copy_to(dst, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +@pytest.mark.parametrize( + ("order", "marker"), + [ + (MemcpySrcAccessOrder.STREAM, 0x41), + (MemcpySrcAccessOrder.ANY, 0x43), + ], +) +def test_src_access_order_copy_from(single_copy_device, single_copy_stream, pinned_mr, order, marker): + """STREAM and ANY are accepted and never corrupt copy_from. See + test_src_access_order_copy_to for why DURING_API_CALL is tested + separately. + """ + src = make_scratch_buffer(single_copy_device, marker, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=order) + + dst.copy_from(src, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_during_api_call_copy_to(single_copy_device, single_copy_stream, pinned_mr): + """DURING_API_CALL is honored on the native (CUDA 13.2+) path. + + On the pre-13.2 fallback it must raise RuntimeError instead of silently + downgrading to stream-ordered cuMemcpyAsync, which cannot honor the + guarantee that all source reads complete before the call returns (see + TestRejectUnsupportedDuringApiCall in test_copy_batch_options.py). CI + runs both driver generations (see ci/test-matrix.yml), so this test must + handle both outcomes rather than assuming the native path is available. + """ + src = make_scratch_buffer(single_copy_device, 0x32, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.DURING_API_CALL) + + if _options_honored(): + src.copy_to(dst, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + assert compare_equal_buffers(src, dst) + else: + with pytest.raises(RuntimeError, match="DURING_API_CALL"): + src.copy_to(dst, stream=single_copy_stream, options=opts) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_during_api_call_copy_from(single_copy_device, single_copy_stream, pinned_mr): + """Same as test_during_api_call_copy_to, exercising copy_from instead.""" + src = make_scratch_buffer(single_copy_device, 0x42, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.DURING_API_CALL) + + if _options_honored(): + dst.copy_from(src, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + assert compare_equal_buffers(src, dst) + else: + with pytest.raises(RuntimeError, match="DURING_API_CALL"): + dst.copy_from(src, stream=single_copy_stream, options=opts) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_overlap_mode_copies_correctly(single_copy_device, single_copy_stream, pinned_mr): + """The overlap hint is advisory and must not change the bytes copied.""" + src = make_scratch_buffer(single_copy_device, 0x77, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(overlap_mode=MemcpyOverlapMode.PREFER_OVERLAP_WITH_COMPUTE) + + src.copy_to(dst, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_legacy_default_stream_token_rejected_with_options(single_copy_device): + """LEGACY_DEFAULT_STREAM with options raises TypeError, matching copy_batch. + + cuMemcpyWithAttributesAsync rejects the legacy default-stream token + outright with CUDA_ERROR_INVALID_VALUE on every driver version, so + copy_to / copy_from surface this before ever calling the driver, just + like copy_batch does. options=None is unaffected: it never touches the + attributes path, so LEGACY_DEFAULT_STREAM keeps working as it always has. + """ + pinned_mr = LegacyPinnedMemoryResource() + src = pinned_mr.allocate(SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY) + + with pytest.raises(TypeError, match="LEGACY_DEFAULT_STREAM"): + src.copy_to(dst, stream=LEGACY_DEFAULT_STREAM, options=opts) + + with pytest.raises(TypeError, match="LEGACY_DEFAULT_STREAM"): + dst.copy_from(src, stream=LEGACY_DEFAULT_STREAM, options=opts) + + # options=None never reaches the attributes path, so this keeps working. + src.copy_to(dst, stream=LEGACY_DEFAULT_STREAM) + single_copy_device.sync() + + src.close() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_per_thread_default_stream_token_accepted_with_options(single_copy_device): + """PER_THREAD_DEFAULT_STREAM is a real stream to the driver, so options are + honored on it just like an explicit stream (subject to the usual CUDA + 13.2+ attributes gate), unlike LEGACY_DEFAULT_STREAM. + """ + pinned_mr = LegacyPinnedMemoryResource() + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY) + + src = pinned_mr.allocate(SIZE) + set_buffer(src, 0x22) + dst = pinned_mr.allocate(SIZE) + src.copy_to(dst, stream=PER_THREAD_DEFAULT_STREAM, options=opts) + single_copy_device.sync() + assert compare_equal_buffers(src, dst) + + set_buffer(src, 0x23) + dst.copy_from(src, stream=PER_THREAD_DEFAULT_STREAM, options=opts) + single_copy_device.sync() + assert compare_equal_buffers(src, dst) + + src.close() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_location_hints_do_not_corrupt_copy(single_copy_device, single_copy_stream): + """Device and host location hints are accepted and leave the bytes intact. + + Hints are only honored by the driver for managed memory; for other + allocation types they are silently ignored. This exercises the + src_location_hint / dst_location_hint → to_cumemlocation path through + cuMemcpyWithAttributesAsync rather than cuMemcpyBatchAsync. + """ + dev = single_copy_device + mr = create_managed_memory_resource_or_skip() + src = mr.allocate(SIZE, stream=single_copy_stream) + dst = mr.allocate(SIZE, stream=single_copy_stream) + + src.fill(0x88, stream=single_copy_stream) + + opts = CopyOptions( + src_access_order=MemcpySrcAccessOrder.STREAM, + src_location_hint=dev, + dst_location_hint=Host(), + ) + src.copy_to(dst, stream=single_copy_stream, options=opts) + + assert_managed_holds(dev, dst, 0x88, stream=single_copy_stream) + + src.close(single_copy_stream) + dst.close(single_copy_stream) + single_copy_stream.sync() + mr.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_host_numa_location_hint(single_copy_device, single_copy_stream): + """A NUMA-specific host hint is accepted and does not corrupt the copy.""" + dev = single_copy_device + numa_id = dev.properties.host_numa_id + if numa_id < 0: + pytest.skip("System does not report a host NUMA node for this device") + mr = create_managed_memory_resource_or_skip() + src = mr.allocate(SIZE, stream=single_copy_stream) + dst = mr.allocate(SIZE, stream=single_copy_stream) + + src.fill(0x99, stream=single_copy_stream) + + opts = CopyOptions(dst_location_hint=Host(numa_id=numa_id)) + src.copy_to(dst, stream=single_copy_stream, options=opts) + + assert_managed_holds(dev, dst, 0x99, stream=single_copy_stream) + + src.close(single_copy_stream) + dst.close(single_copy_stream) + single_copy_stream.sync() + mr.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_host_numa_current_location_hint(single_copy_device, single_copy_stream): + """Host.numa_current() as a location hint is accepted and does not corrupt the copy.""" + dev = single_copy_device + if dev.properties.host_numa_id < 0: + pytest.skip("System does not report a host NUMA node for this device") + mr = create_managed_memory_resource_or_skip() + src = mr.allocate(SIZE, stream=single_copy_stream) + dst = mr.allocate(SIZE, stream=single_copy_stream) + + src.fill(0xAB, stream=single_copy_stream) + + opts = CopyOptions(dst_location_hint=Host.numa_current()) + src.copy_to(dst, stream=single_copy_stream, options=opts) + + assert_managed_holds(dev, dst, 0xAB, stream=single_copy_stream) + + src.close(single_copy_stream) + dst.close(single_copy_stream) + single_copy_stream.sync() + mr.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_copy_to_data_correct(single_copy_device, single_copy_stream, pinned_mr): + """copy_to with non-None options copies the right bytes on all driver versions.""" + src = make_scratch_buffer(single_copy_device, 0x77, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY) + + src.copy_to(dst, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_copy_from_data_correct(single_copy_device, single_copy_stream, pinned_mr): + """copy_from with non-None options copies the right bytes on all driver versions.""" + src = make_scratch_buffer(single_copy_device, 0x33, SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.STREAM) + + dst.copy_from(src, stream=single_copy_stream, options=opts) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close(single_copy_stream) + single_copy_stream.sync() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_copy_to_rejected_under_graph_capture(single_copy_stream, pinned_mr): + """copy_to with options raises TypeError when the stream is capturing, + matching copy_batch. Use GraphNode.memcpy to build attributed copies + into a graph instead; options=None keeps working under capture as it + always has (captured as a plain cuMemcpyAsync node). + """ + src = pinned_mr.allocate(SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY) + + gb = single_copy_stream.create_graph_builder().begin_building() + try: + with pytest.raises(TypeError, match="graph capture"): + src.copy_to(dst, stream=gb, options=opts) + finally: + gb.end_building() + gb.close() + + src.close() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_copy_from_rejected_under_graph_capture(single_copy_stream, pinned_mr): + """Same as the copy_to variant, exercising copy_from instead.""" + src = pinned_mr.allocate(SIZE) + dst = pinned_mr.allocate(SIZE) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.STREAM) + + gb = single_copy_stream.create_graph_builder().begin_building() + try: + with pytest.raises(TypeError, match="graph capture"): + dst.copy_from(src, stream=gb, options=opts) + finally: + gb.end_building() + gb.close() + + src.close() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_options_none_copy_to_still_works_under_graph_capture(single_copy_stream, pinned_mr): + """options=None never touches the attributes path, so copy_to keeps + working under graph capture exactly as it did before options existed. + """ + src = pinned_mr.allocate(SIZE) + set_buffer(src, 0xBB) + dst = pinned_mr.allocate(SIZE) + + gb = single_copy_stream.create_graph_builder().begin_building() + src.copy_to(dst, stream=gb) + graph = gb.end_building().complete() + graph.launch(single_copy_stream) + single_copy_stream.sync() + + assert compare_equal_buffers(src, dst) + + src.close() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 5") +@pytest.mark.parametrize("bad_options", [42, "not-copyoptions", object()]) +def test_copy_to_rejects_invalid_options_type(single_copy_stream, pinned_mr, bad_options): + src = pinned_mr.allocate(SIZE) + dst = pinned_mr.allocate(SIZE) + + with pytest.raises(TypeError, match="options must be CopyOptions"): + src.copy_to(dst, stream=single_copy_stream, options=bad_options) + + with pytest.raises(TypeError, match="options must be CopyOptions"): + dst.copy_from(src, stream=single_copy_stream, options=bad_options) + + with pytest.raises(TypeError, match="options must be CopyOptions"): + src.copy_to(dst, stream=LEGACY_DEFAULT_STREAM, options="not-copyoptions") + + src.close() + dst.close() + + +@pytest.mark.agent_authored(model="Claude Sonnet 4.6") +def test_dst_none_with_options(single_copy_device, single_copy_stream, pinned_mr): + """dst=None auto-allocation works correctly with options on all driver versions.""" + mr = single_copy_device.memory_resource + src = mr.allocate(SIZE, stream=single_copy_stream) + src.fill(0xF0, stream=single_copy_stream) + opts = CopyOptions(src_access_order=MemcpySrcAccessOrder.ANY) + + dst = src.copy_to(stream=single_copy_stream, options=opts) + + # Read back via pinned buffer to verify bytes. + host = pinned_mr.allocate(SIZE) + dst.copy_to(host, stream=single_copy_stream) + single_copy_stream.sync() + + ref = make_scratch_buffer(single_copy_device, 0xF0, SIZE) + assert compare_equal_buffers(ref, host) + + src.close(single_copy_stream) + dst.close(single_copy_stream) + single_copy_stream.sync() + host.close() + ref.close(single_copy_stream) + single_copy_stream.sync() diff --git a/cuda_core/tests/test_memory.py b/cuda_core/tests/test_memory.py index 3fdbe98885f..60883544e9e 100644 --- a/cuda_core/tests/test_memory.py +++ b/cuda_core/tests/test_memory.py @@ -243,6 +243,36 @@ def test_buffer_copy_from(): buffer_copy_from(DummyPinnedMemoryResource(device), device, check=True) +def test_buffer_copy_to_size_mismatch_raises(): + device = Device() + device.set_current() + mr = DummyDeviceMemoryResource(device) + stream = device.create_stream() + src_buffer = mr.allocate(size=1024) + dst_buffer = mr.allocate(size=2048) + + with pytest.raises(ValueError, match="buffer sizes mismatch"): + src_buffer.copy_to(dst_buffer, stream=stream) + + dst_buffer.close() + src_buffer.close() + + +def test_buffer_copy_from_size_mismatch_raises(): + device = Device() + device.set_current() + mr = DummyDeviceMemoryResource(device) + stream = device.create_stream() + src_buffer = mr.allocate(size=1024) + dst_buffer = mr.allocate(size=2048) + + with pytest.raises(ValueError, match="buffer sizes mismatch"): + dst_buffer.copy_from(src_buffer, stream=stream) + + dst_buffer.close() + src_buffer.close() + + def _bytes_repeat(pattern: bytes, size: int) -> bytes: assert len(pattern) > 0 assert size % len(pattern) == 0