From 37287be38e9b73202e92363bbc789a4aeddaa643 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Fri, 2 Oct 2026 12:55:46 -0400 Subject: [PATCH 1/4] store bitpacked block offsets as an optional child Signed-off-by: Matt Katz --- .../fastlanes/benches/bitpack_compare.rs | 6 +- .../benches/bitpack_compare_sweep.rs | 6 +- .../bitpacking/array/bitpack_decompress.rs | 22 +- .../fastlanes/src/bitpacking/array/mod.rs | 157 ++++++-- .../fastlanes/src/bitpacking/array/tests.rs | 338 ++++++++++++++++++ .../src/bitpacking/array/unpack_iter.rs | 6 +- .../src/bitpacking/compute/between.rs | 6 + .../fastlanes/src/bitpacking/compute/cast.rs | 15 +- .../src/bitpacking/compute/compare.rs | 6 + .../src/bitpacking/compute/compare_fused.rs | 7 +- .../src/bitpacking/compute/filter.rs | 14 +- .../src/bitpacking/compute/is_constant.rs | 5 + .../fastlanes/src/bitpacking/compute/slice.rs | 19 +- .../fastlanes/src/bitpacking/compute/take.rs | 11 +- encodings/fastlanes/src/bitpacking/mod.rs | 1 + encodings/fastlanes/src/bitpacking/plugin.rs | 14 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 55 ++- .../src/bitpacking/vtable/operations.rs | 2 +- .../src/for_/array/for_decompress.rs | 7 +- encodings/fastlanes/src/for_/vtable/mod.rs | 8 +- .../src/schemes/integer/bitpacking.rs | 4 +- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 5 +- .../src/dynamic_dispatch/plan_builder.rs | 68 +++- vortex-cuda/src/kernel/encodings/bitpacked.rs | 19 +- vortex-python/python/vortex/_lib/arrays.pyi | 2 +- vortex-python/src/arrays/fastlanes.rs | 12 +- 26 files changed, 738 insertions(+), 77 deletions(-) create mode 100644 encodings/fastlanes/src/bitpacking/array/tests.rs diff --git a/encodings/fastlanes/benches/bitpack_compare.rs b/encodings/fastlanes/benches/bitpack_compare.rs index ecac3446d8e..0d89e770cc1 100644 --- a/encodings/fastlanes/benches/bitpack_compare.rs +++ b/encodings/fastlanes/benches/bitpack_compare.rs @@ -30,6 +30,7 @@ use vortex_buffer::BufferMut; use vortex_fastlanes::BitPacked; use vortex_fastlanes::BitPackedArray; use vortex_fastlanes::BitPackedData; +use vortex_fastlanes::BitWidths; use vortex_session::VortexSession; #[global_allocator] @@ -58,12 +59,15 @@ const BIT_WIDTHS: &[u8] = &[4, 16]; fn page_aligned(array: BitPackedArray) -> BitPackedArray { let ptype = array.dtype().as_ptype(); let parts = BitPacked::into_parts(array); + let BitWidths::Global(bit_width) = parts.bit_widths else { + unreachable!("bitpack_encode packs every block at one bit width") + }; BitPacked::try_new( parts.packed.ensure_aligned(Alignment::new(4096)).unwrap(), ptype, parts.validity, parts.patches, - parts.bit_width, + bit_width, parts.len, parts.offset, ) diff --git a/encodings/fastlanes/benches/bitpack_compare_sweep.rs b/encodings/fastlanes/benches/bitpack_compare_sweep.rs index 1d1f91f7671..f3da868902b 100644 --- a/encodings/fastlanes/benches/bitpack_compare_sweep.rs +++ b/encodings/fastlanes/benches/bitpack_compare_sweep.rs @@ -34,6 +34,7 @@ use vortex_buffer::BufferMut; use vortex_fastlanes::BitPacked; use vortex_fastlanes::BitPackedArray; use vortex_fastlanes::BitPackedData; +use vortex_fastlanes::BitWidths; use vortex_session::VortexSession; #[global_allocator] @@ -84,12 +85,15 @@ impl_bench_int!(u8, u16, u32, u64, i8, i16, i32, i64); fn page_aligned(array: BitPackedArray) -> BitPackedArray { let ptype = array.dtype().as_ptype(); let parts = BitPacked::into_parts(array); + let BitWidths::Global(bit_width) = parts.bit_widths else { + unreachable!("bitpack_encode packs every block at one bit width") + }; BitPacked::try_new( parts.packed.ensure_aligned(Alignment::new(4096)).unwrap(), ptype, parts.validity, parts.patches, - parts.bit_width, + bit_width, parts.len, parts.offset, ) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 0684aea5e6a..a8d1bb6834c 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -17,11 +17,12 @@ use vortex_array::match_each_integer_ptype; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; use vortex_array::scalar::Scalar; -use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use crate::BitPacked; use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::FL_CHUNK_SIZE; use crate::unpack_iter::BitPacked as BitPackedUnpack; use crate::unpack_iter::BitUnpackedChunks; @@ -114,18 +115,19 @@ where } let len = array.len(); + let mut scratch = [const { MaybeUninit::::uninit() }; FL_CHUNK_SIZE]; + let mut chunks = array.unpacked_chunks::(&mut scratch)?; + let validity = array.validity()?.execute_mask(len, ctx)?; let mut uninit_range = builder.uninit_range(len); // SAFETY: We initialize all `len` values below via `decode` and the patch loop. unsafe { - uninit_range.append_mask(&array.validity()?.execute_mask(len, ctx)?); + uninit_range.append_mask(&validity); } // SAFETY: `decode` writes a value to every slot in this range. let uninit_slice = unsafe { uninit_range.slice_uninit_mut(0, len) }; - let mut scratch = [const { MaybeUninit::::uninit() }; FL_CHUNK_SIZE]; - let mut chunks = array.unpacked_chunks::(&mut scratch)?; decode(&mut chunks, uninit_slice, &map); if let Some(patches) = array.patches() { @@ -164,8 +166,11 @@ pub(crate) fn apply_patches_to_uninit_range, index: usize) -> Scalar { - let bit_width = array.bit_width() as usize; +pub fn unpack_single(array: ArrayView<'_, BitPacked>, index: usize) -> VortexResult { + let BitWidths::Global(bit_width) = array.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; + let bit_width = bit_width as usize; let ptype = array.dtype().as_ptype(); // let packed = array.packed().into_primitive()?; let index_in_encoded = index + array.offset() as usize; @@ -176,7 +181,7 @@ pub fn unpack_single(array: ArrayView<'_, BitPacked>, index: usize) -> Scalar { } }); // Cast to fix signedness and nullability - scalar.cast(array.dtype()).vortex_expect("cast failure") + scalar.cast(array.dtype()) } /// # Safety @@ -227,6 +232,7 @@ mod tests { use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_buffer::buffer; + use vortex_error::VortexExpect; use vortex_session::VortexSession; use super::*; @@ -259,7 +265,7 @@ mod tests { .iter() .enumerate() .for_each(|(i, v)| { - let scalar: u16 = (&unpack_single(compressed.as_view(), i)) + let scalar: u16 = (&unpack_single(compressed.as_view(), i).unwrap()) .try_into() .unwrap(); assert_eq!(scalar, *v); diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index c5f547788c1..d89fce2b79c 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -16,6 +16,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; +use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::PatchSlotIndices; use vortex_array::patches::Patches; use vortex_array::patches::PatchesData; @@ -25,11 +26,15 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_error::vortex_panic; pub mod bitpack_compress; pub mod bitpack_decompress; pub mod unpack_iter; +#[cfg(test)] +mod tests; + use crate::BitPackedArray; use crate::FL_CHUNK_SIZE; use crate::bitpack_compress::bitpack_encode; @@ -50,6 +55,11 @@ pub struct BitPackedSlots { /// The validity bitmap indicating which elements are non-null. #[slot(3)] pub validity_child: Option, + /// Byte boundaries of the packed blocks as non-nullable unsigned integers, including one + /// trailing boundary, when the blocks have no global bit width. + /// Block `i` is packed at `(block_offsets[i + 1] - block_offsets[i]) / 128` bits. + #[slot(4)] + pub block_offsets: Option, } pub(crate) const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { @@ -58,9 +68,81 @@ pub(crate) const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { chunk_offsets: BitPackedSlots::PATCH_CHUNK_OFFSETS, }; +/// Check that `offsets` holds `num_blocks + 1` boundaries spanning `packed_len` bytes, each block a +/// whole number of 128-byte rows with a bit width supported by `ptype`. +/// +/// Boundaries are only inspected when they are materialized on the host. +pub(crate) fn validate_block_offsets( + offsets: &ArrayRef, + ptype: PType, + num_blocks: usize, + packed_len: usize, +) -> VortexResult<()> { + vortex_ensure!( + offsets.dtype().is_unsigned_int() && !offsets.dtype().is_nullable(), + "Expected non-nullable unsigned integer block offsets, got {}", + offsets.dtype() + ); + vortex_ensure!( + offsets.len() == num_blocks + 1, + "Expected {} block boundaries, got {}", + num_blocks + 1, + offsets.len() + ); + let max_bit_width = ptype.bit_width() as u64; + if let Some(primitive) = offsets.as_opt::() + && primitive.buffer_handle().is_on_host() + { + match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { + validate_primitive_offsets(primitive.as_slice::(), max_bit_width, packed_len) + }) + } else { + Ok(()) + } +} + +/// Check that each block between `boundaries` is a whole number of 128-byte rows of at most +/// `max_bit_width` bits, and that the boundaries span `packed_len` bytes. +fn validate_primitive_offsets( + boundaries: &[T], + max_bit_width: u64, + packed_len: usize, +) -> VortexResult<()> +where + u64: From, +{ + for pair in boundaries.windows(2) { + let size = u64::from(pair[1]).checked_sub(u64::from(pair[0])); + vortex_ensure!( + size.is_some_and(|size| size % 128 == 0 && size / 128 <= max_bit_width), + "Block boundaries {} and {} do not hold a supported bit width (at most {max_bit_width} bits)", + pair[0], + pair[1] + ); + } + let span = match boundaries { + [first, .., last] => u64::from(*last) - u64::from(*first), + _ => 0, + }; + vortex_ensure!( + span == packed_len as u64, + "Block offsets span {span} bytes, but the packed buffer has {packed_len}" + ); + Ok(()) +} + +/// How the blocks of a [`BitPackedArray`] are packed. +#[derive(Clone, Debug)] +pub enum BitWidths { + /// Every block is packed at this bit width. + Global(u8), + /// Byte boundaries of the packed blocks, from which each block's bit width is derived. + Blocked(ArrayRef), +} + pub struct BitPackedDataParts { pub offset: u16, - pub bit_width: u8, + pub bit_widths: BitWidths, pub len: usize, pub packed: BufferHandle, pub patches: Option, @@ -72,7 +154,9 @@ pub struct BitPackedData { /// The offset within the first block (created with a slice). /// 0 <= offset < 1024 pub(super) offset: u16, - pub(super) bit_width: u8, + /// The bit width shared by every block, or `None` when the block offsets child holds each + /// block's boundaries. + pub(super) global_bit_width: Option, pub(super) packed: BufferHandle, /// Patch metadata for reconstructing Patches from slots. pub(super) patches_data: Option, @@ -80,7 +164,10 @@ pub struct BitPackedData { impl Display for BitPackedData { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "bit_width: {}, offset: {}", self.bit_width, self.offset) + match self.global_bit_width { + Some(bit_width) => write!(f, "bit_width: {}, offset: {}", bit_width, self.offset), + None => write!(f, "offset: {}", self.offset), + } } } @@ -140,7 +227,26 @@ impl BitPackedData { Ok(Self { offset, - bit_width, + global_bit_width: Some(bit_width), + packed, + patches_data: patches.as_ref().map(PatchesData::from_patches), + }) + } + + /// Create the packed payload for blocks whose bit widths come from a block offsets child. + pub(crate) fn try_new_blocked( + packed: BufferHandle, + patches: Option, + offset: u16, + ) -> VortexResult { + vortex_ensure!( + offset < 1024, + "Offset must be less than the full block i.e., 1024, got {offset}" + ); + + Ok(Self { + offset, + global_bit_width: None, packed, patches_data: patches.as_ref().map(PatchesData::from_patches), }) @@ -151,12 +257,11 @@ impl BitPackedData { ptype: PType, validity: &Validity, patches: Option<&Patches>, - bit_width: u8, + bit_width: Option, length: usize, offset: u16, ) -> VortexResult<()> { vortex_ensure!(ptype.is_int(), MismatchedTypes: "integer", ptype); - vortex_ensure!(bit_width <= 64, "Unsupported bit width {bit_width}"); if let Some(validity_len) = validity.maybe_len() { vortex_ensure_eq!( @@ -171,10 +276,16 @@ impl BitPackedData { Self::validate_patches(patches, ptype, length)?; } - // Validate packed buffer - let expected_packed_len = - (length + offset as usize).div_ceil(1024) * (128 * bit_width as usize); - vortex_ensure_eq!(packed.len(), expected_packed_len); + // Validate packed buffer. Block offsets are validated against it separately. + if let Some(bit_width) = bit_width { + vortex_ensure!( + usize::from(bit_width) <= ptype.bit_width(), + "Unsupported bit width {bit_width} for {ptype}" + ); + let expected_packed_len = + (length + offset as usize).div_ceil(1024) * (128 * bit_width as usize); + vortex_ensure_eq!(packed.len(), expected_packed_len); + } Ok(()) } @@ -236,12 +347,6 @@ impl BitPackedData { BitUnpackedChunks::try_new(self, len, scratch) } - /// Bit-width of the packed values - #[inline] - pub fn bit_width(&self) -> u8 { - self.bit_width - } - #[inline] pub fn offset(&self) -> u16 { self.offset @@ -269,14 +374,6 @@ impl BitPackedData { .map_err(|a| vortex_err!(InvalidArgument: "Bitpacking can only encode primitive arrays, got {}", a.encoding_id()))?; bitpack_encode(&parray, bit_width, None, ctx) } - - /// Calculate the maximum value that **can** be contained by this array, given its bit-width. - /// - /// Note that this value need not actually be present in the array. - #[inline] - pub fn max_packed_value(&self) -> usize { - (1 << self.bit_width()) - 1 - } } pub trait BitPackedArrayExt: BitPackedArraySlotsExt { @@ -285,9 +382,17 @@ pub trait BitPackedArrayExt: BitPackedArraySlotsExt { BitPackedData::packed(self) } + /// How the blocks are packed: at one global bit width, or at the widths implied by the block + /// offsets. #[inline] - fn bit_width(&self) -> u8 { - BitPackedData::bit_width(self) + fn bit_widths(&self) -> BitWidths { + match (self.global_bit_width, self.block_offsets()) { + (Some(bit_width), None) => BitWidths::Global(bit_width), + (None, Some(block_offsets)) => BitWidths::Blocked(block_offsets.clone()), + _ => vortex_panic!( + "BitPacked must have exactly one of a global bit width and block offsets" + ), + } } #[inline] diff --git a/encodings/fastlanes/src/bitpacking/array/tests.rs b/encodings/fastlanes/src/bitpacking/array/tests.rs new file mode 100644 index 00000000000..cac25b7441c --- /dev/null +++ b/encodings/fastlanes/src/bitpacking/array/tests.rs @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Tests for the global bit width and block offsets of bit-packed arrays. + +use std::sync::LazyLock; + +use rstest::rstest; +use vortex_array::Array; +use vortex_array::ArrayParts; +use vortex_array::ArrayRef; +use vortex_array::ArraySlots; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::assert_arrays_eq; +use vortex_array::buffer::BufferHandle; +use vortex_array::builders::ArrayBuilder; +use vortex_array::builders::PrimitiveBuilder; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::match_each_unsigned_integer_ptype; +use vortex_array::scalar_fn::fns::cast::CastKernel; +use vortex_array::scalar_fn::fns::cast::CastReduce; +use vortex_array::validity::Validity; +use vortex_buffer::ByteBuffer; +use vortex_buffer::buffer; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_session::VortexSession; + +use crate::BitPacked; +use crate::BitPackedArray; +use crate::BitPackedArrayExt; +use crate::BitPackedArraySlotsExt; +use crate::BitPackedData; +use crate::BitWidths; +use crate::bitpacking::bitpack_compress::bitpack_to_best_bit_width; + +static SESSION: LazyLock = LazyLock::new(|| { + let session = vortex_array::array_session(); + crate::initialize(&session); + session +}); + +fn encode(values: &[u32]) -> VortexResult { + let mut ctx = SESSION.create_execution_ctx(); + bitpack_to_best_bit_width(&PrimitiveArray::from_iter(values.iter().copied()), &mut ctx) +} + +fn uniform() -> VortexResult { + encode(&(0..3000u32).map(|i| i % 128).collect::>()) +} + +fn with_block_offsets(array: &BitPackedArray, offsets: ArrayRef) -> VortexResult { + BitPacked::try_new_with_block_offsets( + array.packed().clone(), + array.dtype().as_ptype(), + array.validity()?, + array.patches(), + offsets, + array.len(), + array.offset(), + ) +} + +#[test] +fn global_bit_width_has_no_block_offsets() -> VortexResult<()> { + let uniform = uniform()?; + assert!(matches!(uniform.bit_widths(), BitWidths::Global(7))); + assert!(uniform.block_offsets().is_none()); + Ok(()) +} + +#[rstest] +#[case::both(true, true)] +#[case::neither(false, false)] +fn global_bit_width_and_block_offsets_are_exclusive( + #[case] constant: bool, + #[case] offsets: bool, +) -> VortexResult<()> { + let packed = BufferHandle::new_host(ByteBuffer::zeroed(128)); + let data = if constant { + BitPackedData::try_new(packed, None, 1, 0)? + } else { + BitPackedData::try_new_blocked(packed, None, 0)? + }; + let block_offsets = offsets.then(|| buffer![0u64, 128].into_array()); + let slots: ArraySlots = [None, None, None, None, block_offsets] + .into_iter() + .collect(); + let dtype = DType::Primitive(PType::U32, Nullability::NonNullable); + assert!( + Array::::try_from_parts( + ArrayParts::new(BitPacked, dtype, 1024, data).with_slots(slots) + ) + .is_err() + ); + Ok(()) +} + +#[rstest] +fn unsigned_block_offsets_are_supported( + #[values(PType::U8, PType::U16, PType::U32, PType::U64)] ptype: PType, +) -> VortexResult<()> { + let offsets = match_each_unsigned_integer_ptype!(ptype, |T| { + PrimitiveArray::from_iter([127 as T, 255 as T]).into_array() + }); + let array = BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(128)), + PType::U32, + Validity::NonNullable, + None, + offsets.clone(), + 1024, + 0, + )?; + assert!( + array + .block_offsets() + .is_some_and(|block_offsets| ArrayRef::ptr_eq(&offsets, block_offsets)) + ); + Ok(()) +} + +#[rstest] +#[case::unaligned([0, 127])] +#[case::decreasing([128, 0])] +#[case::wrong_span([0, 0])] +fn invalid_unsigned_block_offsets_are_rejected( + #[case] boundaries: [u8; 2], + #[values(PType::U8, PType::U16, PType::U32, PType::U64)] ptype: PType, +) { + let offsets = match_each_unsigned_integer_ptype!(ptype, |T| { + PrimitiveArray::from_iter(boundaries.map(T::from)).into_array() + }); + assert!( + BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(128)), + PType::U32, + Validity::NonNullable, + None, + offsets, + 1024, + 0, + ) + .is_err() + ); +} + +#[rstest] +#[case::equal_steps(buffer![0u64, 896, 1792, 2688])] +#[case::different_widths(buffer![0u64, 384, 1408, 2688])] +fn block_offsets_have_no_constant_width( + #[case] offsets: vortex_buffer::Buffer, +) -> VortexResult<()> { + let array = with_block_offsets(&uniform()?, offsets.into_array())?; + assert!(matches!(array.bit_widths(), BitWidths::Blocked(_))); + // Decoding per-block widths is not supported yet. + assert!( + array + .into_array() + .execute::(&mut SESSION.create_execution_ctx()) + .is_err() + ); + Ok(()) +} + +#[rstest] +#[case::equal_steps(buffer![0u64, 896, 1792, 2688])] +#[case::different_widths(buffer![0u64, 384, 1408, 2688])] +fn casts_with_block_offsets_decline( + #[case] offsets: vortex_buffer::Buffer, + #[values( + DType::Primitive(PType::U32, Nullability::Nullable), + DType::Primitive(PType::U64, Nullability::NonNullable) + )] + dtype: DType, +) -> VortexResult<()> { + let array = with_block_offsets(&uniform()?, offsets.into_array())?; + let mut ctx = SESSION.create_execution_ctx(); + + assert!(::cast(array.as_view(), &dtype)?.is_none()); + assert!(::cast(array.as_view(), &dtype, &mut ctx)?.is_none()); + Ok(()) +} + +#[rstest] +#[case::too_few_boundaries(buffer![0u64, 896, 1792].into_array())] +#[case::unaligned_block(buffer![0u64, 896, 1791, 2688].into_array())] +#[case::decreasing(buffer![0u64, 896, 768, 2688].into_array())] +#[case::span_disagrees_with_packed_len(buffer![0u64, 768, 1536, 2304].into_array())] +#[case::signed(buffer![0i32, 896, 1792, 2688].into_array())] +#[case::float(buffer![0f32, 896.0, 1792.0, 2688.0].into_array())] +#[case::nullable( + PrimitiveArray::from_option_iter([Some(0u32), Some(896), Some(1792), Some(2688)]).into_array() +)] +fn invalid_block_offsets_are_rejected(#[case] offsets: ArrayRef) -> VortexResult<()> { + assert!(with_block_offsets(&uniform()?, offsets).is_err()); + Ok(()) +} + +#[rstest] +fn block_width_must_fit_ptype( + #[values( + PType::U8, PType::I8, PType::U16, PType::I16, PType::U32, PType::I32, PType::U64, + PType::I64 + )] + ptype: PType, + #[values(false, true)] block_offsets: bool, + #[values(false, true)] too_wide: bool, +) -> VortexResult<()> { + let bit_width = u8::try_from(ptype.bit_width())? + u8::from(too_wide); + let packed = BufferHandle::new_host(ByteBuffer::zeroed(128 * usize::from(bit_width))); + let result = if block_offsets { + let end = 128 * u64::from(bit_width); + BitPacked::try_new_with_block_offsets( + packed, + ptype, + Validity::NonNullable, + None, + buffer![0u64, end].into_array(), + 1024, + 0, + ) + } else { + BitPacked::try_new( + packed, + ptype, + Validity::NonNullable, + None, + bit_width, + 1024, + 0, + ) + }; + assert_eq!(result.is_err(), too_wide); + Ok(()) +} + +#[rstest] +#[case::equal_steps(buffer![0u64, 512, 1024])] +#[case::different_widths(buffer![0u64, 384, 1024])] +fn unsupported_offsets_leave_builder_unchanged( + #[case] offsets: vortex_buffer::Buffer, +) -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(1024)), + PType::U32, + Validity::AllValid, + None, + offsets.into_array(), + 2048, + 0, + )?; + let mut builder = PrimitiveBuilder::::with_capacity_in( + Nullability::Nullable, + array.len() + 2, + ctx.allocator(), + ); + builder.append_null(); + assert!(array.append_to_builder(&mut builder, &mut ctx).is_err()); + builder.append_value(7); + assert_arrays_eq!( + builder.finish_into_primitive(), + PrimitiveArray::from_option_iter([None, Some(7u32)]), + &mut ctx + ); + Ok(()) +} + +#[test] +fn construct_blocks_without_a_uniform_width() -> VortexResult<()> { + // Two blocks at widths 3 and 4 occupy 896 bytes, which no constant width can represent. + let array = BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(896)), + PType::U32, + Validity::NonNullable, + None, + buffer![0u64, 384, 896].into_array(), + 2048, + 0, + )?; + assert!(matches!(array.bit_widths(), BitWidths::Blocked(_))); + Ok(()) +} + +#[test] +fn empty_and_zero_width_arrays() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + assert!(matches!(encode(&[])?.bit_widths(), BitWidths::Global(0))); + let zeros = encode(&vec![0u32; 2049])?; + assert!(matches!(zeros.bit_widths(), BitWidths::Global(0))); + assert_eq!(zeros.packed().len(), 0); + assert_arrays_eq!(zeros, PrimitiveArray::from_iter(vec![0u32; 2049]), &mut ctx); + Ok(()) +} + +#[test] +fn into_parts_preserves_global_bit_width() -> VortexResult<()> { + let parts = BitPacked::into_parts(uniform()?); + assert!(matches!(parts.bit_widths, BitWidths::Global(7))); + Ok(()) +} + +#[rstest] +#[case::equal_steps(buffer![128u64, 640, 1152].into_array())] +#[case::different_widths(buffer![128u64, 512, 1152].into_array())] +fn into_parts_preserves_block_offsets(#[case] offsets: ArrayRef) -> VortexResult<()> { + let array = BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(1024)), + PType::U32, + Validity::NonNullable, + None, + offsets.clone(), + 1500, + 17, + )?; + let parts = BitPacked::into_parts(array); + let BitWidths::Blocked(block_offsets) = parts.bit_widths else { + vortex_bail!("expected block offsets"); + }; + assert!(ArrayRef::ptr_eq(&offsets, &block_offsets)); + let rebuilt = BitPacked::try_new_with_block_offsets( + parts.packed, + PType::U32, + parts.validity, + parts.patches, + block_offsets, + parts.len, + parts.offset, + )?; + assert_eq!(rebuilt.len(), 1500); + assert_eq!(rebuilt.offset(), 17); + Ok(()) +} diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index cc2036fb06f..d84ae4d9c10 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -12,6 +12,7 @@ use lending_iterator::prelude::Item; use lending_iterator::prelude::LendingIterator; use vortex_array::dtype::PhysicalPType; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; @@ -107,10 +108,13 @@ impl<'a, T: BitPacked> BitUnpackedChunks<'a, T> { len: usize, scratch: &'a mut [MaybeUninit; CHUNK_SIZE], ) -> VortexResult { + let Some(bit_width) = array.global_bit_width else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; Self::try_new_with_strategy( BitPackingStrategy, array.packed_slice::(), - array.bit_width() as usize, + bit_width as usize, array.offset() as usize, len, scratch, diff --git a/encodings/fastlanes/src/bitpacking/compute/between.rs b/encodings/fastlanes/src/bitpacking/compute/between.rs index 8010fa208d2..fb59ef863af 100644 --- a/encodings/fastlanes/src/bitpacking/compute/between.rs +++ b/encodings/fastlanes/src/bitpacking/compute/between.rs @@ -22,6 +22,8 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::bitpacking::compute::stream_predicate::stream_predicate; impl BetweenKernel for BitPacked { @@ -32,6 +34,10 @@ impl BetweenKernel for BitPacked { options: &BetweenOptions, ctx: &mut ExecutionCtx, ) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } // Only accelerate constant-bounds between; vary-by-row bounds fall through to the // default `compare + and` pipeline. let (Some(lower_const), Some(upper_const)) = (lower.as_constant(), upper.as_constant()) diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index cdbb8141cea..4287e7029d8 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -15,8 +15,10 @@ use vortex_array::scalar_fn::fns::cast::CastKernel; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_array::validity::Validity; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use crate::bitpacking::BitPacked; +use crate::bitpacking::BitWidths; use crate::bitpacking::array::BitPackedArrayExt; use crate::bitpacking::array::bitpack_decompress::unpack_map_into_builder; @@ -36,6 +38,9 @@ fn build_with_validity( dtype: &DType, new_validity: Validity, ) -> VortexResult { + let BitWidths::Global(bit_width) = array.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; Ok(BitPacked::try_new( array.packed().clone(), dtype.as_ptype(), @@ -44,7 +49,7 @@ fn build_with_validity( .patches() .map(|patches| patches.map_values(|values| values.cast(dtype.clone()))) .transpose()?, - array.bit_width(), + bit_width, array.len(), array.offset(), )? @@ -53,6 +58,10 @@ fn build_with_validity( impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } if !array.dtype().eq_ignore_nullability(dtype) { return Ok(None); } @@ -72,6 +81,10 @@ impl CastKernel for BitPacked { dtype: &DType, ctx: &mut ExecutionCtx, ) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } // Nullability-only change: keep the values bit-packed, just adjust validity. if array.dtype().eq_ignore_nullability(dtype) { let new_validity = diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index c9d6b815b0d..b8f1eb5c1a2 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -27,6 +27,8 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::bitpacking::compute::compare_fused::stream_compare_fused; use crate::unpack_iter::BitPacked as BitPackedIter; @@ -37,6 +39,10 @@ impl CompareKernel for BitPacked { operator: CompareOperator, ctx: &mut ExecutionCtx, ) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(lhs.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } // Only accelerate compare-against-constant. let Some(constant) = rhs.as_constant() else { return Ok(None); diff --git a/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs b/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs index 1259ed815fe..aa78aed09a7 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs @@ -44,10 +44,12 @@ use vortex_buffer::BitBufferMut; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use super::stream_predicate::stream_predicate; use crate::BitPacked; use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::unpack_iter::BitPacked as BitPackedIter; use crate::unpack_iter::for_each_packed_chunk; @@ -78,7 +80,10 @@ where F: Fn(T, T) -> bool + Copy, { let len = array.len(); - let bit_width = array.bit_width() as usize; + let BitWidths::Global(bit_width) = array.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; + let bit_width = bit_width as usize; let offset = array.offset() as usize; // A degenerate width has no packed payload for the fused kernel to consume; defer to the scalar diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 0b1b9422f86..297e1ae0472 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -18,6 +18,7 @@ use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_mask::Mask; use vortex_mask::MaskValuesRef; @@ -26,6 +27,7 @@ use super::take::UNPACK_CHUNK_THRESHOLD; use crate::BitPacked; use crate::BitPackedArrayExt; use crate::BitPackedData; +use crate::BitWidths; /// The threshold over which it is faster to fully unpack the entire [`BitPackedArray`](crate::BitPackedArray) and then /// filter the result than to unpack only specific bitpacked values into the output buffer. @@ -49,6 +51,10 @@ impl FilterKernel for BitPacked { mask: &Mask, ctx: &mut ExecutionCtx, ) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } let values = match mask { Mask::AllTrue(_) | Mask::AllFalse(_) => { return Ok(None); @@ -110,7 +116,10 @@ fn filter_primitive_without_patches( array: ArrayView<'_, BitPacked>, selection: &MaskValuesRef, ) -> VortexResult<(Buffer, Validity)> { - let values = filter_with_indices(array.data(), selection.indices()); + let BitWidths::Global(bit_width) = array.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; + let values = filter_with_indices(array.data(), bit_width, selection.indices()); let validity = array .validity()? .filter(&Mask::Values(MaskValuesRef::clone(selection)))?; @@ -120,10 +129,11 @@ fn filter_primitive_without_patches( fn filter_with_indices( array: &BitPackedData, + bit_width: u8, indices: &[usize], ) -> BufferMut { let offset = array.offset() as usize; - let bit_width = array.bit_width() as usize; + let bit_width = bit_width as usize; let mut values = BufferMut::with_capacity(indices.len()); // Some re-usable memory to store per-chunk indices. diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 0ab01a635ba..f0a28c5ec8a 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -23,6 +23,7 @@ use vortex_error::VortexResult; use crate::BitPacked; use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::unpack_iter::BitPacked as BitPackedUnpack; /// BitPacked-specific is_constant kernel with SIMD support. @@ -43,6 +44,10 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { let Some(array) = batch.as_opt::() else { return Ok(None); }; + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } let result = match_each_integer_ptype!(array.dtype().as_ptype(), |P| { bitpacked_is_constant::() }>(array, ctx)? diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index 996565a2672..a7b769e0de6 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -12,12 +12,18 @@ use vortex_array::arrays::slice::SliceKernel; use vortex_array::arrays::slice::SliceReduce; use vortex_array::patches::Patches; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use crate::BitPacked; +use crate::BitWidths; use crate::bitpacking::array::BitPackedArrayExt; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } // We cannot access buffers (to slice the patches). if array.patches().is_some() { return Ok(None); @@ -33,6 +39,10 @@ impl SliceKernel for BitPacked { range: Range, _ctx: &mut ExecutionCtx, ) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } let patches = array .patches() .map(|p| p.slice(range.clone())) @@ -54,15 +64,18 @@ fn slice_bitpacked( let block_start = max(0, offset_start - offset); let block_stop = offset_stop.div_ceil(1024) * 1024; - let encoded_start = (block_start / 8) * array.bit_width() as usize; - let encoded_stop = (block_stop / 8) * array.bit_width() as usize; + let BitWidths::Global(bit_width) = array.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; + let encoded_start = (block_start / 8) * bit_width as usize; + let encoded_stop = (block_stop / 8) * bit_width as usize; Ok(BitPacked::try_new( array.packed().slice(encoded_start..encoded_stop), array.dtype().as_ptype(), array.validity()?.slice(range.clone())?, patches, - array.bit_width(), + bit_width, range.len(), offset as u16, )? diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 86e97623cf6..86152dbc04e 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -21,10 +21,12 @@ use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use super::chunked_indices; use crate::BitPacked; use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::bitpack_decompress; // TODO(connor): This is duplicated in `encodings/fastlanes/src/bitpacking/kernels/mod.rs`. @@ -39,6 +41,10 @@ impl TakeExecute for BitPacked { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { + // Blocks packed at different widths fall back to decoding. + if !matches!(array.bit_widths(), BitWidths::Global(_)) { + return Ok(None); + } // If the indices are large enough, it's faster to flatten and take the primitive array. if indices.len() * UNPACK_CHUNK_THRESHOLD > array.len() { let prim = array.array().clone().execute::(ctx)?; @@ -81,7 +87,10 @@ fn take_primitive( } let offset = array.offset() as usize; - let bit_width = array.bit_width() as usize; + let BitWidths::Global(bit_width) = array.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; + let bit_width = bit_width as usize; let packed = array.packed_slice::(); diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index f6af27bf728..e6090df3277 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -7,6 +7,7 @@ pub use array::BitPackedArraySlotsExt; pub use array::BitPackedData; pub use array::BitPackedDataParts; pub use array::BitPackedSlots; +pub use array::BitWidths; pub use array::bitpack_compress; pub use array::bitpack_decompress; pub use array::unpack_iter; diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index 6383a041247..0eaab906214 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -37,6 +37,8 @@ use crate::BitPacked; use crate::BitPackedArray; use crate::BitPackedArrayExt; use crate::BitPackedData; +use crate::BitWidths; +use crate::bitpacking::array::BitPackedSlots; #[derive(Clone, prost::Message)] pub struct BitPackedMetadata { @@ -68,8 +70,11 @@ impl ArrayPlugin for BitPackedPlugin { let view = array.as_opt::().ok_or_else(|| { vortex_err!("BitPacked plugin cannot serialize {}", array.encoding_id()) })?; + let BitWidths::Global(bit_width) = view.bit_widths() else { + vortex_bail!("BitPacked plugin cannot serialize per-block bit widths"); + }; let metadata = BitPackedMetadata { - bit_width: view.bit_width() as u32, + bit_width: u32::from(bit_width), offset: view.offset() as u32, patches: view .patches() @@ -148,9 +153,10 @@ impl ArrayPlugin for BitPackedPlugin { .transpose()?; let slots = { - let mut s = ArraySlots::with_capacity(4); + let mut s = ArraySlots::with_capacity(BitPackedSlots::COUNT); PatchesData::push_slots(&mut s, patches.as_ref()); s.push(validity_to_child(&validity, len)); + s.push(None); s }; let data = BitPackedData::try_new( @@ -223,7 +229,9 @@ impl ArrayPlugin for BitPackedPatchedPlugin { let packed = bitpacked.packed().clone(); let ptype = bitpacked.dtype().as_ptype(); let validity = bitpacked.validity()?; - let bw = bitpacked.bit_width; + let BitWidths::Global(bw) = bitpacked.bit_widths() else { + vortex_bail!("BitPacked patched plugin cannot serialize per-block bit widths"); + }; let len = bitpacked.len(); let offset = bitpacked.offset(); diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 641c6b3a2fe..086b23deffa 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -41,11 +41,13 @@ use vortex_session::registry::CachedId; use crate::BitPackedArrayExt; use crate::BitPackedData; use crate::BitPackedDataParts; +use crate::FL_CHUNK_SIZE; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; use crate::bitpacking::array::BitPackedSlots; use crate::bitpacking::array::BitPackedSlotsView; use crate::bitpacking::array::PATCH_SLOTS; +use crate::bitpacking::array::validate_block_offsets; use crate::bitpacking::vtable::rules::RULES; mod kernels; mod operations; @@ -62,7 +64,7 @@ pub(crate) fn initialize(session: &VortexSession) { impl ArrayHash for BitPackedData { fn array_hash(&self, state: &mut H, accuracy: EqMode) { self.offset.hash(state); - self.bit_width.hash(state); + self.global_bit_width.hash(state); self.packed.array_hash(state, accuracy); self.patches_data.hash(state); } @@ -71,7 +73,7 @@ impl ArrayHash for BitPackedData { impl ArrayEq for BitPackedData { fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool { self.offset == other.offset - && self.bit_width == other.bit_width + && self.global_bit_width == other.global_bit_width && self.packed.array_eq(&other.packed, accuracy) && self.patches_data == other.patches_data } @@ -95,7 +97,20 @@ impl VTable for BitPacked { len: usize, slots: &[Option], ) -> VortexResult<()> { + vortex_ensure_eq!(slots.len(), BitPackedSlots::COUNT); let bp_slots = BitPackedSlotsView::from_slots(slots); + match (data.global_bit_width, bp_slots.block_offsets) { + (Some(_), None) => {} + (None, Some(block_offsets)) => validate_block_offsets( + block_offsets, + dtype.as_ptype(), + (len + data.offset as usize).div_ceil(FL_CHUNK_SIZE), + data.packed.len(), + )?, + _ => { + vortex_bail!("BitPacked needs exactly one of a global bit width and block offsets") + } + } let validity = child_to_validity(bp_slots.validity_child, dtype.nullability()); let patches = @@ -105,7 +120,7 @@ impl VTable for BitPacked { dtype.as_ptype(), &validity, patches.as_ref(), - data.bit_width, + data.global_bit_width, len, data.offset, ) @@ -210,6 +225,7 @@ impl VTable for BitPacked { pub struct BitPacked; impl BitPacked { + /// Construct a bit-packed array whose blocks all use `bit_width`. pub fn try_new( packed: BufferHandle, ptype: PType, @@ -221,23 +237,52 @@ impl BitPacked { ) -> VortexResult { let dtype = DType::Primitive(ptype, validity.nullability()); let slots = { - let mut s = ArraySlots::with_capacity(4); + let mut s = ArraySlots::with_capacity(BitPackedSlots::COUNT); PatchesData::push_slots(&mut s, patches.as_ref()); s.push(validity_to_child(&validity, len)); + s.push(None); s }; let data = BitPackedData::try_new(packed, patches, bit_width, offset)?; Array::try_from_parts(ArrayParts::new(BitPacked, dtype, len, data).with_slots(slots)) } + /// Construct a bit-packed array from packed data and explicit block byte boundaries. + /// + /// `block_offsets` must be non-nullable unsigned integers with one boundary per block and a + /// trailing end boundary. Each block's bit width is derived from the distance between its + /// boundaries. + pub fn try_new_with_block_offsets( + packed: BufferHandle, + ptype: PType, + validity: Validity, + patches: Option, + block_offsets: ArrayRef, + len: usize, + offset: u16, + ) -> VortexResult { + let dtype = DType::Primitive(ptype, validity.nullability()); + let slots = { + let mut s = ArraySlots::with_capacity(BitPackedSlots::COUNT); + PatchesData::push_slots(&mut s, patches.as_ref()); + s.push(validity_to_child(&validity, len)); + s.push(Some(block_offsets)); + s + }; + let data = BitPackedData::try_new_blocked(packed, patches, offset)?; + Array::try_from_parts(ArrayParts::new(BitPacked, dtype, len, data).with_slots(slots)) + } + + /// Split the array into its parts. pub fn into_parts(array: BitPackedArray) -> BitPackedDataParts { let len = array.len(); let patches = array.patches(); let validity = array.validity().vortex_expect("BitPacked validity"); + let bit_widths = array.bit_widths(); let data = array.into_data(); BitPackedDataParts { offset: data.offset, - bit_width: data.bit_width, + bit_widths, len, packed: data.packed, patches, diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 2816407ac03..a71650a8369 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -24,7 +24,7 @@ impl OperationsVTable for BitPacked { { patch } else { - bitpack_decompress::unpack_single(array, index) + bitpack_decompress::unpack_single(array, index)? }, ) } diff --git a/encodings/fastlanes/src/for_/array/for_decompress.rs b/encodings/fastlanes/src/for_/array/for_decompress.rs index f20bbc34475..de9a7977698 100644 --- a/encodings/fastlanes/src/for_/array/for_decompress.rs +++ b/encodings/fastlanes/src/for_/array/for_decompress.rs @@ -30,10 +30,12 @@ use vortex_compute::lane_kernels::IndexedSinkExt; use vortex_compute::lane_kernels::IndexedSourceExt; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_err; use crate::BitPacked; use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::FL_CHUNK_SIZE; use crate::FoRArray; use crate::for_::array::FoRArrayExt; @@ -310,7 +312,10 @@ fn unpack_chunks< output: &mut [MaybeUninit], ) -> VortexResult<()> { let offset = usize::from(bp.offset()); - let bit_width = bp.bit_width() as usize; + let BitWidths::Global(bit_width) = bp.bit_widths() else { + vortex_bail!("BitPacked array has per-block bit widths"); + }; + let bit_width = bit_width as usize; // SAFETY: `T::Physical` is `T` with the same size and alignment, and the unpack is the same // wrapping addition in two's complement whichever signedness `T` has. let output = diff --git a/encodings/fastlanes/src/for_/vtable/mod.rs b/encodings/fastlanes/src/for_/vtable/mod.rs index 2577b7a20a7..1096519ba6d 100644 --- a/encodings/fastlanes/src/for_/vtable/mod.rs +++ b/encodings/fastlanes/src/for_/vtable/mod.rs @@ -35,6 +35,8 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::BitPacked; +use crate::BitPackedArrayExt; +use crate::BitWidths; use crate::FoRData; use crate::for_::array::FoRArrayExt; use crate::for_::array::FoRArraySlotsExt; @@ -149,9 +151,11 @@ impl VTable for FoR { require_child!(array, array.references(), FoRSlots::REFERENCES => Primitive) }; // The fused unpack reads a bit-packed child's buffers directly. Its chunks line up with - // the FoR chunks when the references are constant or the offsets match. + // the FoR chunks when the references are constant or the offsets match. Blocks packed at + // different widths are decoded first. let fused = array.encoded().as_opt::().is_some_and(|bp| { - array.constant_reference().is_some() || bp.offset() == array.offset() + matches!(bp.bit_widths(), BitWidths::Global(_)) + && (array.constant_reference().is_some() || bp.offset() == array.offset()) }); let array = if fused { array diff --git a/vortex-btrblocks/src/schemes/integer/bitpacking.rs b/vortex-btrblocks/src/schemes/integer/bitpacking.rs index 5ac7d0e4078..8025633d835 100644 --- a/vortex-btrblocks/src/schemes/integer/bitpacking.rs +++ b/vortex-btrblocks/src/schemes/integer/bitpacking.rs @@ -97,7 +97,7 @@ impl Scheme for BitPackingScheme { ptype, parts.validity, None, - parts.bit_width, + bw, parts.len, parts.offset, )? @@ -122,7 +122,7 @@ impl Scheme for BitPackingScheme { ptype, parts.validity, parts.patches, - parts.bit_width, + bw, parts.len, parts.offset, )? diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 82afa0a843d..da456fb9d31 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -47,6 +47,7 @@ use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::BitPackedData; +use vortex::encodings::fastlanes::BitWidths; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::fastlanes::FoRArraySlotsExt; @@ -475,8 +476,8 @@ mod standalone { cuda_session: &CudaSession, cuda_ctx: &mut CudaExecutionCtx, ) -> Self { - assert_eq!(values_bp.bit_width(), 6); - assert_eq!(codes_bp.bit_width(), 6); + assert!(matches!(values_bp.bit_widths(), BitWidths::Global(6))); + assert!(matches!(codes_bp.bit_widths(), BitWidths::Global(6))); let values_packed = block_on(cuda_ctx.ensure_on_device(values_bp.packed().clone())) .vortex_expect("values packed"); diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index bd143116cdd..d5822313f31 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -30,6 +30,7 @@ use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArrayExt; +use vortex::encodings::fastlanes::BitWidths; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::fastlanes::FoRArraySlotsExt; @@ -89,7 +90,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { return matches!(arr.dtype().as_ptype(), PType::F32 | PType::F64); } if id == BitPacked.id() { - return true; + return matches!(array.as_::().bit_widths(), BitWidths::Global(_)); } if id == Dict.id() { let arr = array.as_::(); @@ -156,11 +157,16 @@ fn is_dyn_dispatch_cast_compatible(array: &ArrayRef) -> bool { /// Returns `true` if a registered standalone kernel can decode the entire /// `array` tree in a single launch without recursing into `execute_cuda` /// for child encodings. +/// +/// `FoR` requires a constant reference, and `BitPacked` requires a constant bit +/// width. pub fn has_standalone_kernel(array: &ArrayRef) -> bool { let id = array.encoding_id(); - // Leaf encodings: no children to recurse into. - if id == BitPacked.id() || id == Sequence.id() { + if id == BitPacked.id() { + return is_bitpacked_with_global_bit_width(array); + } + if id == Sequence.id() { return true; } @@ -172,10 +178,10 @@ pub fn has_standalone_kernel(array: &ArrayRef) -> bool { } let child = for_arr.encoded(); if child.encoding_id() == BitPacked.id() { - return true; + return is_bitpacked_with_global_bit_width(child); } if let Some(slice) = child.as_opt::() { - return slice.child().encoding_id() == BitPacked.id(); + return is_bitpacked_with_global_bit_width(slice.child()); } return false; } @@ -183,6 +189,12 @@ pub fn has_standalone_kernel(array: &ArrayRef) -> bool { false } +fn is_bitpacked_with_global_bit_width(array: &ArrayRef) -> bool { + array + .as_opt::() + .is_some_and(|array| matches!(array.bit_widths(), BitWidths::Global(_))) +} + /// Patch payload attached to the op that consumes it. /// /// `range` is the logical output range to apply when materializing the patch descriptor on the GPU. @@ -567,10 +579,13 @@ impl FusedPlan { let source_ptype = ptype_to_tag(PType::try_from(bp.dtype()).map_err(|_| { vortex_err!("BitPacked must have primitive dtype, got {:?}", bp.dtype()) })?); + let BitWidths::Global(bit_width) = bp.bit_widths() else { + vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); + }; let buf_index = self.source_buffers.len(); self.source_buffers.push(Some(packed)); return Ok(Stage::new( - SourceOp::bitunpack(bp.bit_width(), bitpacked_offset), + SourceOp::bitunpack(bit_width, bitpacked_offset), Some(buf_index), source_ptype, ) @@ -622,10 +637,13 @@ impl FusedPlan { let source_ptype = ptype_to_tag(PType::try_from(bp.dtype()).map_err(|_| { vortex_err!("BitPacked must have primitive dtype, got {:?}", bp.dtype()) })?); + let BitWidths::Global(bit_width) = bp.bit_widths() else { + vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); + }; let buf_index = self.source_buffers.len(); self.source_buffers.push(Some(bp.packed().clone())); Ok(Stage::new( - SourceOp::bitunpack(bp.bit_width(), bp.offset()), + SourceOp::bitunpack(bit_width, bp.offset()), Some(buf_index), source_ptype, ) @@ -903,14 +921,50 @@ impl FusedPlan { #[cfg(test)] mod tests { + use rstest::rstest; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; + use vortex::array::arrays::SliceArray; use vortex::array::builtins::ArrayBuiltins; + use vortex::buffer::Buffer; + use vortex::buffer::ByteBuffer; + use vortex::buffer::buffer; use vortex::dtype::DType; use vortex::dtype::Nullability; use super::*; + #[rstest] + #[case::equal_steps(buffer![0u64, 512, 1024])] + #[case::different_widths(buffer![0u64, 384, 1024])] + fn materialized_bitpacked_offsets_have_no_standalone_kernel( + #[case] offsets: Buffer, + ) -> VortexResult<()> { + let bitpacked = BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(1024)), + PType::U32, + Validity::NonNullable, + None, + offsets.into_array(), + 2048, + 0, + )? + .into_array(); + assert!(!has_standalone_kernel(&bitpacked)); + assert!(matches!( + DispatchPlan::new(&bitpacked, CudaDispatchMode::Auto)?, + DispatchPlan::Unfused + )); + + let for_bitpacked = FoR::try_new(bitpacked.clone(), 100u32.into())?.into_array(); + assert!(!has_standalone_kernel(&for_bitpacked)); + + let sliced = SliceArray::new(bitpacked, 100..1500).into_array(); + let for_sliced = FoR::try_new(sliced, 100u32.into())?.into_array(); + assert!(!has_standalone_kernel(&for_sliced)); + Ok(()) + } + #[test] fn cast_to_non_primitive_target_is_not_dyn_dispatch_compatible() -> VortexResult<()> { let cast = PrimitiveArray::from_iter([0u8, 1]) diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 86b7a88b276..55da3aaf08a 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -26,8 +26,10 @@ use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::BitPackedDataParts; +use vortex::encodings::fastlanes::BitWidths; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_ensure; use vortex::error::vortex_err; @@ -61,8 +63,11 @@ pub(crate) fn bitpacked_slice_view( let block_start = offset_start - bitpacked_offset; let block_stop = offset_stop.div_ceil(PATCH_CHUNK_SIZE) * PATCH_CHUNK_SIZE; - let encoded_start = (block_start / 8) * bp.bit_width() as usize; - let encoded_stop = (block_stop / 8) * bp.bit_width() as usize; + let BitWidths::Global(bit_width) = bp.bit_widths() else { + vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); + }; + let encoded_start = (block_start / 8) * bit_width as usize; + let encoded_stop = (block_stop / 8) * bit_width as usize; Ok(( bp.packed().slice(encoded_start..encoded_stop), @@ -91,12 +96,15 @@ impl BitPackedExecutor { let offset = slice.data().slice_range().start; let len = array.len(); let (packed, bitpacked_offset, patch_range) = bitpacked_slice_view(bp, offset, len)?; + let BitWidths::Global(bit_width) = bp.bit_widths() else { + vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); + }; let sliced = BitPacked::try_new( packed, bp.ptype(bp.dtype()), child.validity()?.slice(patch_range.clone())?, bp.patches(), - bp.bit_width(), + bit_width, len, bitpacked_offset, )?; @@ -162,12 +170,15 @@ where { let BitPackedDataParts { offset, - bit_width, + bit_widths, len, packed, patches, validity, } = BitPacked::into_parts(array); + let BitWidths::Global(bit_width) = bit_widths else { + vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); + }; vortex_ensure!(len > 0, "Non empty array"); let offset = offset as usize; diff --git a/vortex-python/python/vortex/_lib/arrays.pyi b/vortex-python/python/vortex/_lib/arrays.pyi index 1c1d74bd093..0ded6ebf993 100644 --- a/vortex-python/python/vortex/_lib/arrays.pyi +++ b/vortex-python/python/vortex/_lib/arrays.pyi @@ -135,7 +135,7 @@ class ZigZagArray(Array): @final class FastLanesBitPackedArray(Array): @property - def bit_width(self) -> int: ... + def bit_width(self) -> int | None: ... @final class FastLanesDeltaArray(Array): ... diff --git a/vortex-python/src/arrays/fastlanes.rs b/vortex-python/src/arrays/fastlanes.rs index 31b49e8d804..18d6665f106 100644 --- a/vortex-python/src/arrays/fastlanes.rs +++ b/vortex-python/src/arrays/fastlanes.rs @@ -3,10 +3,11 @@ use pyo3::prelude::*; use vortex::encodings::fastlanes::BitPacked; +use vortex::encodings::fastlanes::BitPackedArrayExt; +use vortex::encodings::fastlanes::BitWidths; use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::FoR; -use crate::arrays::native::AsArrayRef; use crate::arrays::native::EncodingSubclass; use crate::arrays::native::PyNativeArray; @@ -20,10 +21,13 @@ impl EncodingSubclass for PyFastLanesBitPackedArray { #[pymethods] impl PyFastLanesBitPackedArray { - /// Returns the bit width of the packed values. + /// Returns the bit width shared by every block, or `None` if blocks have different widths. #[getter] - fn bit_width(self_: PyRef<'_, Self>) -> u8 { - self_.as_array_ref().bit_width() + fn bit_width(self_: PyRef<'_, Self>) -> Option { + match self_.as_super().inner().as_::().bit_widths() { + BitWidths::Global(bit_width) => Some(bit_width), + BitWidths::Blocked(_) => None, + } } } From f3d97d93e225f925b5ea53dfcb004e95e2a3c54d Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Fri, 2 Oct 2026 14:48:05 -0400 Subject: [PATCH 2/4] check bitpacked widths once per kernel Signed-off-by: Matt Katz --- .../fastlanes/src/bitpacking/array/mod.rs | 8 ++++++++ .../src/bitpacking/compute/between.rs | 4 +--- .../fastlanes/src/bitpacking/compute/cast.rs | 19 +++++++----------- .../src/bitpacking/compute/compare.rs | 20 +++++++++---------- .../src/bitpacking/compute/compare_fused.rs | 6 +----- .../src/bitpacking/compute/filter.rs | 13 +++++------- .../src/bitpacking/compute/is_constant.rs | 4 +--- .../fastlanes/src/bitpacking/compute/slice.rs | 19 +++++++----------- .../fastlanes/src/bitpacking/compute/take.rs | 13 +++++------- encodings/fastlanes/src/bitpacking/plugin.rs | 3 ++- encodings/fastlanes/src/for_/vtable/mod.rs | 7 +++---- .../src/dynamic_dispatch/plan_builder.rs | 10 ++++------ vortex-cuda/src/kernel/encodings/bitpacked.rs | 12 +++++------ vortex-cuda/src/kernel/encodings/for_.rs | 6 +++++- vortex-python/src/arrays/fastlanes.rs | 3 ++- 15 files changed, 67 insertions(+), 80 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index d89fce2b79c..519b06edfd6 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -140,6 +140,14 @@ pub enum BitWidths { Blocked(ArrayRef), } +impl BitWidths { + /// Returns `true` if every block is packed at one bit width. + #[inline] + pub fn is_global(&self) -> bool { + matches!(self, Self::Global(_)) + } +} + pub struct BitPackedDataParts { pub offset: u16, pub bit_widths: BitWidths, diff --git a/encodings/fastlanes/src/bitpacking/compute/between.rs b/encodings/fastlanes/src/bitpacking/compute/between.rs index fb59ef863af..2a96d484303 100644 --- a/encodings/fastlanes/src/bitpacking/compute/between.rs +++ b/encodings/fastlanes/src/bitpacking/compute/between.rs @@ -23,7 +23,6 @@ use vortex_error::VortexResult; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; use crate::bitpacking::compute::stream_predicate::stream_predicate; impl BetweenKernel for BitPacked { @@ -34,8 +33,7 @@ impl BetweenKernel for BitPacked { options: &BetweenOptions, ctx: &mut ExecutionCtx, ) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + if !array.bit_widths().is_global() { return Ok(None); } // Only accelerate constant-bounds between; vary-by-row bounds fall through to the diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 4287e7029d8..217615be8d6 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -15,7 +15,6 @@ use vortex_array::scalar_fn::fns::cast::CastKernel; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_array::validity::Validity; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use crate::bitpacking::BitPacked; use crate::bitpacking::BitWidths; @@ -35,12 +34,10 @@ fn is_widening_int_cast(src: PType, tgt: PType) -> bool { fn build_with_validity( array: ArrayView<'_, BitPacked>, + bit_width: u8, dtype: &DType, new_validity: Validity, ) -> VortexResult { - let BitWidths::Global(bit_width) = array.bit_widths() else { - vortex_bail!("BitPacked array has per-block bit widths"); - }; Ok(BitPacked::try_new( array.packed().clone(), dtype.as_ptype(), @@ -58,10 +55,9 @@ fn build_with_validity( impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = array.bit_widths() else { return Ok(None); - } + }; if !array.dtype().eq_ignore_nullability(dtype) { return Ok(None); } @@ -71,7 +67,7 @@ impl CastReduce for BitPacked { else { return Ok(None); }; - build_with_validity(array, dtype, new_validity).map(Some) + build_with_validity(array, bit_width, dtype, new_validity).map(Some) } } @@ -81,17 +77,16 @@ impl CastKernel for BitPacked { dtype: &DType, ctx: &mut ExecutionCtx, ) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = array.bit_widths() else { return Ok(None); - } + }; // Nullability-only change: keep the values bit-packed, just adjust validity. if array.dtype().eq_ignore_nullability(dtype) { let new_validity = array .validity()? .cast_nullability(dtype.nullability(), array.len(), ctx)?; - return build_with_validity(array, dtype, new_validity).map(Some); + return build_with_validity(array, bit_width, dtype, new_validity).map(Some); } // Widening integer cast: unpack each FastLanes chunk into a cache-resident scratch buffer diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index b8f1eb5c1a2..3eb5b5034d7 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -39,10 +39,9 @@ impl CompareKernel for BitPacked { operator: CompareOperator, ctx: &mut ExecutionCtx, ) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(lhs.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = lhs.bit_widths() else { return Ok(None); - } + }; // Only accelerate compare-against-constant. let Some(constant) = rhs.as_constant() else { return Ok(None); @@ -63,7 +62,7 @@ impl CompareKernel for BitPacked { let rhs: T = constant_prim .typed_value::() .vortex_expect("compare adaptor strips null constants"); - compare_constant_typed::(lhs, rhs, operator, nullability, ctx)? + compare_constant_typed::(lhs, bit_width, rhs, operator, nullability, ctx)? }); Ok(Some(result)) } @@ -75,6 +74,7 @@ impl CompareKernel for BitPacked { /// kernel's dispatch shape). `NotEq` has no direct method, so use `!is_eq`. fn compare_constant_typed( lhs: ArrayView<'_, BitPacked>, + bit_width: u8, rhs: T, operator: CompareOperator, nullability: Nullability, @@ -88,22 +88,22 @@ where { match operator { CompareOperator::Eq => { - stream_compare_fused::(lhs, rhs, nullability, |a, b| a.is_eq(b), ctx) + stream_compare_fused::(lhs, bit_width, rhs, nullability, |a, b| a.is_eq(b), ctx) } CompareOperator::NotEq => { - stream_compare_fused::(lhs, rhs, nullability, |a, b| !a.is_eq(b), ctx) + stream_compare_fused::(lhs, bit_width, rhs, nullability, |a, b| !a.is_eq(b), ctx) } CompareOperator::Lt => { - stream_compare_fused::(lhs, rhs, nullability, |a, b| a.is_lt(b), ctx) + stream_compare_fused::(lhs, bit_width, rhs, nullability, |a, b| a.is_lt(b), ctx) } CompareOperator::Lte => { - stream_compare_fused::(lhs, rhs, nullability, |a, b| a.is_le(b), ctx) + stream_compare_fused::(lhs, bit_width, rhs, nullability, |a, b| a.is_le(b), ctx) } CompareOperator::Gt => { - stream_compare_fused::(lhs, rhs, nullability, |a, b| a.is_gt(b), ctx) + stream_compare_fused::(lhs, bit_width, rhs, nullability, |a, b| a.is_gt(b), ctx) } CompareOperator::Gte => { - stream_compare_fused::(lhs, rhs, nullability, |a, b| a.is_ge(b), ctx) + stream_compare_fused::(lhs, bit_width, rhs, nullability, |a, b| a.is_ge(b), ctx) } } } diff --git a/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs b/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs index aa78aed09a7..ecc1be33de8 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare_fused.rs @@ -44,12 +44,10 @@ use vortex_buffer::BitBufferMut; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use super::stream_predicate::stream_predicate; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; use crate::unpack_iter::BitPacked as BitPackedIter; use crate::unpack_iter::for_each_packed_chunk; @@ -67,6 +65,7 @@ const WORDS_PER_CHUNK: usize = CHUNK_SIZE / U64_BITS; /// [`BitPackedArray`]: crate::BitPackedArray pub(super) fn stream_compare_fused( array: ArrayView<'_, BitPacked>, + bit_width: u8, rhs: T, nullability: Nullability, cmp: F, @@ -80,9 +79,6 @@ where F: Fn(T, T) -> bool + Copy, { let len = array.len(); - let BitWidths::Global(bit_width) = array.bit_widths() else { - vortex_bail!("BitPacked array has per-block bit widths"); - }; let bit_width = bit_width as usize; let offset = array.offset() as usize; diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 297e1ae0472..330c60b9e3b 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -18,7 +18,6 @@ use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use vortex_mask::Mask; use vortex_mask::MaskValuesRef; @@ -51,10 +50,9 @@ impl FilterKernel for BitPacked { mask: &Mask, ctx: &mut ExecutionCtx, ) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = array.bit_widths() else { return Ok(None); - } + }; let values = match mask { Mask::AllTrue(_) | Mask::AllFalse(_) => { return Ok(None); @@ -71,7 +69,8 @@ impl FilterKernel for BitPacked { // Filter and patch using the correct unsigned type for FastLanes, then cast to signed if needed. let primitive = match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { - let (buffer, validity) = filter_primitive_without_patches::(array, values)?; + let (buffer, validity) = + filter_primitive_without_patches::(array, bit_width, values)?; // reinterpret_cast for signed types. let primitive = PrimitiveArray::new(buffer, validity); if array.dtype().as_ptype().is_signed_int() { @@ -114,11 +113,9 @@ impl FilterKernel for BitPacked { /// Returns a tuple of (values buffer, validity mask). fn filter_primitive_without_patches( array: ArrayView<'_, BitPacked>, + bit_width: u8, selection: &MaskValuesRef, ) -> VortexResult<(Buffer, Validity)> { - let BitWidths::Global(bit_width) = array.bit_widths() else { - vortex_bail!("BitPacked array has per-block bit widths"); - }; let values = filter_with_indices(array.data(), bit_width, selection.indices()); let validity = array .validity()? diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index f0a28c5ec8a..0ad8005ba0f 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -23,7 +23,6 @@ use vortex_error::VortexResult; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; use crate::unpack_iter::BitPacked as BitPackedUnpack; /// BitPacked-specific is_constant kernel with SIMD support. @@ -44,8 +43,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { let Some(array) = batch.as_opt::() else { return Ok(None); }; - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + if !array.bit_widths().is_global() { return Ok(None); } diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index a7b769e0de6..c693550dcde 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -12,7 +12,6 @@ use vortex_array::arrays::slice::SliceKernel; use vortex_array::arrays::slice::SliceReduce; use vortex_array::patches::Patches; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use crate::BitPacked; use crate::BitWidths; @@ -20,16 +19,15 @@ use crate::bitpacking::array::BitPackedArrayExt; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = array.bit_widths() else { return Ok(None); - } + }; // We cannot access buffers (to slice the patches). if array.patches().is_some() { return Ok(None); } - Ok(Some(slice_bitpacked(array, range, None)?)) + Ok(Some(slice_bitpacked(array, bit_width, range, None)?)) } } @@ -39,22 +37,22 @@ impl SliceKernel for BitPacked { range: Range, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = array.bit_widths() else { return Ok(None); - } + }; let patches = array .patches() .map(|p| p.slice(range.clone())) .transpose()? .flatten(); - Ok(Some(slice_bitpacked(array, range, patches)?)) + Ok(Some(slice_bitpacked(array, bit_width, range, patches)?)) } } fn slice_bitpacked( array: ArrayView<'_, BitPacked>, + bit_width: u8, range: Range, patches: Option, ) -> VortexResult { @@ -64,9 +62,6 @@ fn slice_bitpacked( let block_start = max(0, offset_start - offset); let block_stop = offset_stop.div_ceil(1024) * 1024; - let BitWidths::Global(bit_width) = array.bit_widths() else { - vortex_bail!("BitPacked array has per-block bit widths"); - }; let encoded_start = (block_start / 8) * bit_width as usize; let encoded_stop = (block_stop / 8) * bit_width as usize; diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 86152dbc04e..c3d8b54a7ab 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -21,7 +21,6 @@ use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use super::chunked_indices; use crate::BitPacked; @@ -41,10 +40,9 @@ impl TakeExecute for BitPacked { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - // Blocks packed at different widths fall back to decoding. - if !matches!(array.bit_widths(), BitWidths::Global(_)) { + let BitWidths::Global(bit_width) = array.bit_widths() else { return Ok(None); - } + }; // If the indices are large enough, it's faster to flatten and take the primitive array. if indices.len() * UNPACK_CHUNK_THRESHOLD > array.len() { let prim = array.array().clone().execute::(ctx)?; @@ -60,7 +58,7 @@ impl TakeExecute for BitPacked { let indices = indices.clone().execute::(ctx)?; let taken = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |T| { match_each_integer_ptype!(indices.ptype(), |I| { - take_primitive::(array, &indices, taken_validity, ctx)? + take_primitive::(array, bit_width, &indices, taken_validity, ctx)? }) }); let taken = if ptype.is_signed_int() { @@ -78,6 +76,7 @@ impl TakeExecute for BitPacked { fn take_primitive( array: ArrayView<'_, BitPacked>, + bit_width: u8, indices: &PrimitiveArray, taken_validity: Validity, ctx: &mut ExecutionCtx, @@ -87,9 +86,6 @@ fn take_primitive( } let offset = array.offset() as usize; - let BitWidths::Global(bit_width) = array.bit_widths() else { - vortex_bail!("BitPacked array has per-block bit widths"); - }; let bit_width = bit_width as usize; let packed = array.packed_slice::(); @@ -288,6 +284,7 @@ mod test { let taken_primitive = take_primitive::( start.as_view(), + 1, &PrimitiveArray::from_iter([0u64, 1, 2, 3]), Validity::NonNullable, &mut ctx, diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index 0eaab906214..3dde44ab7a3 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -31,6 +31,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::BitPacked; @@ -230,7 +231,7 @@ impl ArrayPlugin for BitPackedPatchedPlugin { let ptype = bitpacked.dtype().as_ptype(); let validity = bitpacked.validity()?; let BitWidths::Global(bw) = bitpacked.bit_widths() else { - vortex_bail!("BitPacked patched plugin cannot serialize per-block bit widths"); + vortex_panic!("BitPacked plugin always deserializes a global bit width"); }; let len = bitpacked.len(); let offset = bitpacked.offset(); diff --git a/encodings/fastlanes/src/for_/vtable/mod.rs b/encodings/fastlanes/src/for_/vtable/mod.rs index 1096519ba6d..7d18b9c0b58 100644 --- a/encodings/fastlanes/src/for_/vtable/mod.rs +++ b/encodings/fastlanes/src/for_/vtable/mod.rs @@ -36,7 +36,6 @@ use vortex_session::VortexSession; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; use crate::FoRData; use crate::for_::array::FoRArrayExt; use crate::for_::array::FoRArraySlotsExt; @@ -151,10 +150,10 @@ impl VTable for FoR { require_child!(array, array.references(), FoRSlots::REFERENCES => Primitive) }; // The fused unpack reads a bit-packed child's buffers directly. Its chunks line up with - // the FoR chunks when the references are constant or the offsets match. Blocks packed at - // different widths are decoded first. + // the FoR chunks when the references are constant or the offsets match. It also needs a + // global bit width. let fused = array.encoded().as_opt::().is_some_and(|bp| { - matches!(bp.bit_widths(), BitWidths::Global(_)) + bp.bit_widths().is_global() && (array.constant_reference().is_some() || bp.offset() == array.offset()) }); let array = if fused { diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index d5822313f31..0ddc08dcdfb 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -90,7 +90,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { return matches!(arr.dtype().as_ptype(), PType::F32 | PType::F64); } if id == BitPacked.id() { - return matches!(array.as_::().bit_widths(), BitWidths::Global(_)); + return is_bitpacked_with_global_bit_width(array); } if id == Dict.id() { let arr = array.as_::(); @@ -192,7 +192,7 @@ pub fn has_standalone_kernel(array: &ArrayRef) -> bool { fn is_bitpacked_with_global_bit_width(array: &ArrayRef) -> bool { array .as_opt::() - .is_some_and(|array| matches!(array.bit_widths(), BitWidths::Global(_))) + .is_some_and(|array| array.bit_widths().is_global()) } /// Patch payload attached to the op that consumes it. @@ -574,14 +574,12 @@ impl FusedPlan { let bp = child.as_::(); let offset = slice_arr.data().slice_range().start; let len = array.len(); - let (packed, bitpacked_offset, patch_range) = bitpacked_slice_view(bp, offset, len)?; + let (packed, bit_width, bitpacked_offset, patch_range) = + bitpacked_slice_view(bp, offset, len)?; let source_ptype = ptype_to_tag(PType::try_from(bp.dtype()).map_err(|_| { vortex_err!("BitPacked must have primitive dtype, got {:?}", bp.dtype()) })?); - let BitWidths::Global(bit_width) = bp.bit_widths() else { - vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); - }; let buf_index = self.source_buffers.len(); self.source_buffers.push(Some(packed)); return Ok(Stage::new( diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 55da3aaf08a..9db84b97d7f 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -50,12 +50,13 @@ pub(crate) struct BitPackedExecutor; /// Bit-unpack kernels decode full FastLanes chunks, so the packed buffer is /// widened to chunk boundaries and `offset` is converted into the in-chunk /// starting position. The returned logical range is passed to patch -/// materialization so exception metadata is sliced consistently. +/// materialization so exception metadata is sliced consistently. The global +/// bit width of `bp` is returned alongside the view. pub(crate) fn bitpacked_slice_view( bp: ArrayView<'_, BitPacked>, offset: usize, len: usize, -) -> VortexResult<(BufferHandle, u16, Range)> { +) -> VortexResult<(BufferHandle, u8, u16, Range)> { let patch_range = offset..offset + len; let offset_start = patch_range.start + bp.offset() as usize; let offset_stop = offset_start + len; @@ -71,6 +72,7 @@ pub(crate) fn bitpacked_slice_view( Ok(( bp.packed().slice(encoded_start..encoded_stop), + bit_width, u16::try_from(bitpacked_offset)?, patch_range, )) @@ -95,10 +97,8 @@ impl BitPackedExecutor { let bp = child.as_::(); let offset = slice.data().slice_range().start; let len = array.len(); - let (packed, bitpacked_offset, patch_range) = bitpacked_slice_view(bp, offset, len)?; - let BitWidths::Global(bit_width) = bp.bit_widths() else { - vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); - }; + let (packed, bit_width, bitpacked_offset, patch_range) = + bitpacked_slice_view(bp, offset, len)?; let sliced = BitPacked::try_new( packed, bp.ptype(bp.dtype()), diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 694f7fe1902..a5e420939bf 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -20,6 +20,7 @@ use vortex::array::match_each_integer_ptype; use vortex::array::match_each_native_simd_ptype; use vortex::dtype::NativePType; use vortex::encodings::fastlanes::BitPacked; +use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArray; use vortex::encodings::fastlanes::FoRArrayExt; @@ -65,7 +66,9 @@ impl CudaExecute for FoRExecutor { }; // Fuse FOR + BP => FFOR - if let Some(bitpacked) = array.encoded().as_opt::() { + if let Some(bitpacked) = array.encoded().as_opt::() + && bitpacked.bit_widths().is_global() + { match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { let reference: P = (&reference).try_into()?; return decode_bitpacked(bitpacked.into_owned(), reference, None, ctx).await; @@ -75,6 +78,7 @@ impl CudaExecute for FoRExecutor { // Fuse FOR + SLICE + BP => SLICE + FFOR if let Some(slice_array) = array.encoded().as_opt::() && let Some(bitpacked) = slice_array.child().as_opt::() + && bitpacked.bit_widths().is_global() { let slice_range = slice_array.slice_range().clone(); let unpacked = match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { diff --git a/vortex-python/src/arrays/fastlanes.rs b/vortex-python/src/arrays/fastlanes.rs index 18d6665f106..8b8b9cc0b03 100644 --- a/vortex-python/src/arrays/fastlanes.rs +++ b/vortex-python/src/arrays/fastlanes.rs @@ -21,7 +21,8 @@ impl EncodingSubclass for PyFastLanesBitPackedArray { #[pymethods] impl PyFastLanesBitPackedArray { - /// Returns the bit width shared by every block, or `None` if blocks have different widths. + /// Returns the global bit width of the packed values, or `None` if the array has per-block + /// bit widths. #[getter] fn bit_width(self_: PyRef<'_, Self>) -> Option { match self_.as_super().inner().as_::().bit_widths() { From ad8ba8cfa08194edb575b11a2aeb45afb8f96909 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Fri, 2 Oct 2026 14:59:35 -0400 Subject: [PATCH 3/4] only check bitpacked block boundaries in debug builds Signed-off-by: Matt Katz --- .../fastlanes/src/bitpacking/array/mod.rs | 19 ++-- .../fastlanes/src/bitpacking/array/tests.rs | 94 +++++++++++++------ 2 files changed, 76 insertions(+), 37 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 519b06edfd6..834c5a1f408 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -22,6 +22,7 @@ use vortex_array::patches::Patches; use vortex_array::patches::PatchesData; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; @@ -68,10 +69,11 @@ pub(crate) const PATCH_SLOTS: PatchSlotIndices = PatchSlotIndices { chunk_offsets: BitPackedSlots::PATCH_CHUNK_OFFSETS, }; -/// Check that `offsets` holds `num_blocks + 1` boundaries spanning `packed_len` bytes, each block a -/// whole number of 128-byte rows with a bit width supported by `ptype`. +/// Check that `offsets` holds `num_blocks + 1` non-nullable unsigned boundaries. /// -/// Boundaries are only inspected when they are materialized on the host. +/// Debug builds also assert that host-resident boundaries span `packed_len` bytes, each block a +/// whole number of 128-byte rows with a bit width supported by `ptype`. Release builds don't check +/// the boundary values, so decoders must bounds-check them. pub(crate) fn validate_block_offsets( offsets: &ArrayRef, ptype: PType, @@ -89,16 +91,17 @@ pub(crate) fn validate_block_offsets( num_blocks + 1, offsets.len() ); - let max_bit_width = ptype.bit_width() as u64; - if let Some(primitive) = offsets.as_opt::() + if cfg!(debug_assertions) + && let Some(primitive) = offsets.as_opt::() && primitive.buffer_handle().is_on_host() { + let max_bit_width = ptype.bit_width() as u64; match_each_unsigned_integer_ptype!(primitive.ptype(), |T| { validate_primitive_offsets(primitive.as_slice::(), max_bit_width, packed_len) + .vortex_expect("invalid BitPacked block offsets") }) - } else { - Ok(()) } + Ok(()) } /// Check that each block between `boundaries` is a whole number of 128-byte rows of at most @@ -284,7 +287,7 @@ impl BitPackedData { Self::validate_patches(patches, ptype, length)?; } - // Validate packed buffer. Block offsets are validated against it separately. + // Validate packed buffer. Block offsets are only checked against it in debug builds. if let Some(bit_width) = bit_width { vortex_ensure!( usize::from(bit_width) <= ptype.bit_width(), diff --git a/encodings/fastlanes/src/bitpacking/array/tests.rs b/encodings/fastlanes/src/bitpacking/array/tests.rs index cac25b7441c..bcaf822d02f 100644 --- a/encodings/fastlanes/src/bitpacking/array/tests.rs +++ b/encodings/fastlanes/src/bitpacking/array/tests.rs @@ -124,29 +124,29 @@ fn unsigned_block_offsets_are_supported( Ok(()) } +#[cfg(debug_assertions)] #[rstest] #[case::unaligned([0, 127])] #[case::decreasing([128, 0])] #[case::wrong_span([0, 0])] -fn invalid_unsigned_block_offsets_are_rejected( +#[should_panic(expected = "invalid BitPacked block offsets")] +fn invalid_unsigned_block_boundaries_panic_in_debug( #[case] boundaries: [u8; 2], #[values(PType::U8, PType::U16, PType::U32, PType::U64)] ptype: PType, ) { let offsets = match_each_unsigned_integer_ptype!(ptype, |T| { PrimitiveArray::from_iter(boundaries.map(T::from)).into_array() }); - assert!( - BitPacked::try_new_with_block_offsets( - BufferHandle::new_host(ByteBuffer::zeroed(128)), - PType::U32, - Validity::NonNullable, - None, - offsets, - 1024, - 0, - ) - .is_err() - ); + BitPacked::try_new_with_block_offsets( + BufferHandle::new_host(ByteBuffer::zeroed(128)), + PType::U32, + Validity::NonNullable, + None, + offsets, + 1024, + 0, + ) + .unwrap(); } #[rstest] @@ -188,9 +188,6 @@ fn casts_with_block_offsets_decline( #[rstest] #[case::too_few_boundaries(buffer![0u64, 896, 1792].into_array())] -#[case::unaligned_block(buffer![0u64, 896, 1791, 2688].into_array())] -#[case::decreasing(buffer![0u64, 896, 768, 2688].into_array())] -#[case::span_disagrees_with_packed_len(buffer![0u64, 768, 1536, 2304].into_array())] #[case::signed(buffer![0i32, 896, 1792, 2688].into_array())] #[case::float(buffer![0f32, 896.0, 1792.0, 2688.0].into_array())] #[case::nullable( @@ -201,19 +198,20 @@ fn invalid_block_offsets_are_rejected(#[case] offsets: ArrayRef) -> VortexResult Ok(()) } +#[cfg(debug_assertions)] #[rstest] -fn block_width_must_fit_ptype( - #[values( - PType::U8, PType::I8, PType::U16, PType::I16, PType::U32, PType::I32, PType::U64, - PType::I64 - )] - ptype: PType, - #[values(false, true)] block_offsets: bool, - #[values(false, true)] too_wide: bool, -) -> VortexResult<()> { - let bit_width = u8::try_from(ptype.bit_width())? + u8::from(too_wide); +#[case::unaligned_block(buffer![0u64, 896, 1791, 2688])] +#[case::decreasing(buffer![0u64, 896, 768, 2688])] +#[case::span_disagrees_with_packed_len(buffer![0u64, 768, 1536, 2304])] +#[should_panic(expected = "invalid BitPacked block offsets")] +fn invalid_block_boundaries_panic_in_debug(#[case] offsets: vortex_buffer::Buffer) { + with_block_offsets(&uniform().unwrap(), offsets.into_array()).unwrap(); +} + +/// One block of `ptype` values packed at `bit_width`, either globally or through block offsets. +fn single_block(ptype: PType, bit_width: u8, block_offsets: bool) -> VortexResult { let packed = BufferHandle::new_host(ByteBuffer::zeroed(128 * usize::from(bit_width))); - let result = if block_offsets { + if block_offsets { let end = 128 * u64::from(bit_width); BitPacked::try_new_with_block_offsets( packed, @@ -234,11 +232,49 @@ fn block_width_must_fit_ptype( 1024, 0, ) - }; - assert_eq!(result.is_err(), too_wide); + } +} + +#[rstest] +fn bit_width_must_fit_ptype( + #[values( + PType::U8, PType::I8, PType::U16, PType::I16, PType::U32, PType::I32, PType::U64, + PType::I64 + )] + ptype: PType, + #[values(false, true)] too_wide: bool, +) -> VortexResult<()> { + let bit_width = u8::try_from(ptype.bit_width())? + u8::from(too_wide); + assert_eq!(single_block(ptype, bit_width, false).is_err(), too_wide); Ok(()) } +#[rstest] +fn block_offsets_can_fill_ptype( + #[values( + PType::U8, PType::I8, PType::U16, PType::I16, PType::U32, PType::I32, PType::U64, + PType::I64 + )] + ptype: PType, +) -> VortexResult<()> { + single_block(ptype, u8::try_from(ptype.bit_width())?, true)?; + Ok(()) +} + +#[cfg(debug_assertions)] +#[rstest] +#[should_panic(expected = "invalid BitPacked block offsets")] +fn too_wide_block_panics_in_debug( + #[values( + PType::U8, PType::I8, PType::U16, PType::I16, PType::U32, PType::I32, PType::U64, + PType::I64 + )] + ptype: PType, +) { + let bit_width = u8::try_from(ptype.bit_width()).unwrap() + 1; + single_block(ptype, bit_width, true).unwrap(); +} + #[rstest] #[case::equal_steps(buffer![0u64, 512, 1024])] #[case::different_widths(buffer![0u64, 384, 1024])] From 9c01ab903d9f75a874cd24f8af0148b7d695fbee Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Mon, 5 Oct 2026 12:19:38 -0400 Subject: [PATCH 4/4] borrow block offsets in bit_widths Signed-off-by: Matt Katz --- .../bitpacking/array/bitpack_decompress.rs | 4 +-- .../fastlanes/src/bitpacking/array/mod.rs | 36 ++++++++++++++----- .../fastlanes/src/bitpacking/array/tests.rs | 14 +++++--- .../fastlanes/src/bitpacking/compute/cast.rs | 6 ++-- .../src/bitpacking/compute/compare.rs | 4 +-- .../src/bitpacking/compute/filter.rs | 4 +-- .../fastlanes/src/bitpacking/compute/slice.rs | 6 ++-- .../fastlanes/src/bitpacking/compute/take.rs | 4 +-- encodings/fastlanes/src/bitpacking/mod.rs | 1 + encodings/fastlanes/src/bitpacking/plugin.rs | 6 ++-- .../fastlanes/src/bitpacking/vtable/mod.rs | 3 +- .../src/for_/array/for_decompress.rs | 4 +-- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 6 ++-- .../src/dynamic_dispatch/plan_builder.rs | 4 +-- vortex-cuda/src/kernel/encodings/bitpacked.rs | 3 +- vortex-python/src/arrays/fastlanes.rs | 6 ++-- 16 files changed, 69 insertions(+), 42 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index a8d1bb6834c..0f7d8fa7533 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -22,7 +22,7 @@ use vortex_error::vortex_bail; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; +use crate::BitWidthsView; use crate::FL_CHUNK_SIZE; use crate::unpack_iter::BitPacked as BitPackedUnpack; use crate::unpack_iter::BitUnpackedChunks; @@ -167,7 +167,7 @@ pub(crate) fn apply_patches_to_uninit_range, index: usize) -> VortexResult { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { vortex_bail!("BitPacked array has per-block bit widths"); }; let bit_width = bit_width as usize; diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 834c5a1f408..00e35dc05c0 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -134,16 +134,16 @@ where Ok(()) } -/// How the blocks of a [`BitPackedArray`] are packed. -#[derive(Clone, Debug)] -pub enum BitWidths { +/// How the blocks of a [`BitPackedArray`] are packed, borrowing block offsets if present. +#[derive(Clone, Copy, Debug)] +pub enum BitWidthsView<'a> { /// Every block is packed at this bit width. Global(u8), /// Byte boundaries of the packed blocks, from which each block's bit width is derived. - Blocked(ArrayRef), + Blocked(&'a ArrayRef), } -impl BitWidths { +impl BitWidthsView<'_> { /// Returns `true` if every block is packed at one bit width. #[inline] pub fn is_global(&self) -> bool { @@ -151,6 +151,26 @@ impl BitWidths { } } +/// How the blocks of a [`BitPackedArray`] are packed, owning block offsets if present. +/// +/// This is the owned form of [`BitWidthsView`], held by [`BitPackedDataParts`]. +#[derive(Clone, Debug)] +pub enum BitWidths { + /// Every block is packed at this bit width. + Global(u8), + /// Byte boundaries of the packed blocks, from which each block's bit width is derived. + Blocked(ArrayRef), +} + +impl From> for BitWidths { + fn from(view: BitWidthsView<'_>) -> Self { + match view { + BitWidthsView::Global(bit_width) => Self::Global(bit_width), + BitWidthsView::Blocked(block_offsets) => Self::Blocked(block_offsets.clone()), + } + } +} + pub struct BitPackedDataParts { pub offset: u16, pub bit_widths: BitWidths, @@ -396,10 +416,10 @@ pub trait BitPackedArrayExt: BitPackedArraySlotsExt { /// How the blocks are packed: at one global bit width, or at the widths implied by the block /// offsets. #[inline] - fn bit_widths(&self) -> BitWidths { + fn bit_widths(&self) -> BitWidthsView<'_> { match (self.global_bit_width, self.block_offsets()) { - (Some(bit_width), None) => BitWidths::Global(bit_width), - (None, Some(block_offsets)) => BitWidths::Blocked(block_offsets.clone()), + (Some(bit_width), None) => BitWidthsView::Global(bit_width), + (None, Some(block_offsets)) => BitWidthsView::Blocked(block_offsets), _ => vortex_panic!( "BitPacked must have exactly one of a global bit width and block offsets" ), diff --git a/encodings/fastlanes/src/bitpacking/array/tests.rs b/encodings/fastlanes/src/bitpacking/array/tests.rs index bcaf822d02f..346f96fa713 100644 --- a/encodings/fastlanes/src/bitpacking/array/tests.rs +++ b/encodings/fastlanes/src/bitpacking/array/tests.rs @@ -36,6 +36,7 @@ use crate::BitPackedArrayExt; use crate::BitPackedArraySlotsExt; use crate::BitPackedData; use crate::BitWidths; +use crate::BitWidthsView; use crate::bitpacking::bitpack_compress::bitpack_to_best_bit_width; static SESSION: LazyLock = LazyLock::new(|| { @@ -68,7 +69,7 @@ fn with_block_offsets(array: &BitPackedArray, offsets: ArrayRef) -> VortexResult #[test] fn global_bit_width_has_no_block_offsets() -> VortexResult<()> { let uniform = uniform()?; - assert!(matches!(uniform.bit_widths(), BitWidths::Global(7))); + assert!(matches!(uniform.bit_widths(), BitWidthsView::Global(7))); assert!(uniform.block_offsets().is_none()); Ok(()) } @@ -156,7 +157,7 @@ fn block_offsets_have_no_constant_width( #[case] offsets: vortex_buffer::Buffer, ) -> VortexResult<()> { let array = with_block_offsets(&uniform()?, offsets.into_array())?; - assert!(matches!(array.bit_widths(), BitWidths::Blocked(_))); + assert!(matches!(array.bit_widths(), BitWidthsView::Blocked(_))); // Decoding per-block widths is not supported yet. assert!( array @@ -319,16 +320,19 @@ fn construct_blocks_without_a_uniform_width() -> VortexResult<()> { 2048, 0, )?; - assert!(matches!(array.bit_widths(), BitWidths::Blocked(_))); + assert!(matches!(array.bit_widths(), BitWidthsView::Blocked(_))); Ok(()) } #[test] fn empty_and_zero_width_arrays() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - assert!(matches!(encode(&[])?.bit_widths(), BitWidths::Global(0))); + assert!(matches!( + encode(&[])?.bit_widths(), + BitWidthsView::Global(0) + )); let zeros = encode(&vec![0u32; 2049])?; - assert!(matches!(zeros.bit_widths(), BitWidths::Global(0))); + assert!(matches!(zeros.bit_widths(), BitWidthsView::Global(0))); assert_eq!(zeros.packed().len(), 0); assert_arrays_eq!(zeros, PrimitiveArray::from_iter(vec![0u32; 2049]), &mut ctx); Ok(()) diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index 217615be8d6..799ecb6601a 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -17,7 +17,7 @@ use vortex_array::validity::Validity; use vortex_error::VortexResult; use crate::bitpacking::BitPacked; -use crate::bitpacking::BitWidths; +use crate::bitpacking::BitWidthsView; use crate::bitpacking::array::BitPackedArrayExt; use crate::bitpacking::array::bitpack_decompress::unpack_map_into_builder; @@ -55,7 +55,7 @@ fn build_with_validity( impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { return Ok(None); }; if !array.dtype().eq_ignore_nullability(dtype) { @@ -77,7 +77,7 @@ impl CastKernel for BitPacked { dtype: &DType, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { return Ok(None); }; // Nullability-only change: keep the values bit-packed, just adjust validity. diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index 3eb5b5034d7..836336401cb 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -28,7 +28,7 @@ use vortex_error::VortexResult; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; +use crate::BitWidthsView; use crate::bitpacking::compute::compare_fused::stream_compare_fused; use crate::unpack_iter::BitPacked as BitPackedIter; @@ -39,7 +39,7 @@ impl CompareKernel for BitPacked { operator: CompareOperator, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let BitWidths::Global(bit_width) = lhs.bit_widths() else { + let BitWidthsView::Global(bit_width) = lhs.bit_widths() else { return Ok(None); }; // Only accelerate compare-against-constant. diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 330c60b9e3b..952e7728e52 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -26,7 +26,7 @@ use super::take::UNPACK_CHUNK_THRESHOLD; use crate::BitPacked; use crate::BitPackedArrayExt; use crate::BitPackedData; -use crate::BitWidths; +use crate::BitWidthsView; /// The threshold over which it is faster to fully unpack the entire [`BitPackedArray`](crate::BitPackedArray) and then /// filter the result than to unpack only specific bitpacked values into the output buffer. @@ -50,7 +50,7 @@ impl FilterKernel for BitPacked { mask: &Mask, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { return Ok(None); }; let values = match mask { diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index c693550dcde..8437c0e85c9 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -14,12 +14,12 @@ use vortex_array::patches::Patches; use vortex_error::VortexResult; use crate::BitPacked; -use crate::BitWidths; +use crate::BitWidthsView; use crate::bitpacking::array::BitPackedArrayExt; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { return Ok(None); }; // We cannot access buffers (to slice the patches). @@ -37,7 +37,7 @@ impl SliceKernel for BitPacked { range: Range, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { return Ok(None); }; let patches = array diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index c3d8b54a7ab..1bb778212c3 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -25,7 +25,7 @@ use vortex_error::VortexResult; use super::chunked_indices; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; +use crate::BitWidthsView; use crate::bitpack_decompress; // TODO(connor): This is duplicated in `encodings/fastlanes/src/bitpacking/kernels/mod.rs`. @@ -40,7 +40,7 @@ impl TakeExecute for BitPacked { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let BitWidths::Global(bit_width) = array.bit_widths() else { + let BitWidthsView::Global(bit_width) = array.bit_widths() else { return Ok(None); }; // If the indices are large enough, it's faster to flatten and take the primitive array. diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index e6090df3277..30334f1bbee 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -8,6 +8,7 @@ pub use array::BitPackedData; pub use array::BitPackedDataParts; pub use array::BitPackedSlots; pub use array::BitWidths; +pub use array::BitWidthsView; pub use array::bitpack_compress; pub use array::bitpack_decompress; pub use array::unpack_iter; diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index 3dde44ab7a3..d159bce8e9d 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -38,7 +38,7 @@ use crate::BitPacked; use crate::BitPackedArray; use crate::BitPackedArrayExt; use crate::BitPackedData; -use crate::BitWidths; +use crate::BitWidthsView; use crate::bitpacking::array::BitPackedSlots; #[derive(Clone, prost::Message)] @@ -71,7 +71,7 @@ impl ArrayPlugin for BitPackedPlugin { let view = array.as_opt::().ok_or_else(|| { vortex_err!("BitPacked plugin cannot serialize {}", array.encoding_id()) })?; - let BitWidths::Global(bit_width) = view.bit_widths() else { + let BitWidthsView::Global(bit_width) = view.bit_widths() else { vortex_bail!("BitPacked plugin cannot serialize per-block bit widths"); }; let metadata = BitPackedMetadata { @@ -230,7 +230,7 @@ impl ArrayPlugin for BitPackedPatchedPlugin { let packed = bitpacked.packed().clone(); let ptype = bitpacked.dtype().as_ptype(); let validity = bitpacked.validity()?; - let BitWidths::Global(bw) = bitpacked.bit_widths() else { + let BitWidthsView::Global(bw) = bitpacked.bit_widths() else { vortex_panic!("BitPacked plugin always deserializes a global bit width"); }; let len = bitpacked.len(); diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 086b23deffa..e79d246bdcf 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -41,6 +41,7 @@ use vortex_session::registry::CachedId; use crate::BitPackedArrayExt; use crate::BitPackedData; use crate::BitPackedDataParts; +use crate::BitWidths; use crate::FL_CHUNK_SIZE; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; @@ -278,7 +279,7 @@ impl BitPacked { let len = array.len(); let patches = array.patches(); let validity = array.validity().vortex_expect("BitPacked validity"); - let bit_widths = array.bit_widths(); + let bit_widths: BitWidths = array.bit_widths().into(); let data = array.into_data(); BitPackedDataParts { offset: data.offset, diff --git a/encodings/fastlanes/src/for_/array/for_decompress.rs b/encodings/fastlanes/src/for_/array/for_decompress.rs index de9a7977698..2f364c022b7 100644 --- a/encodings/fastlanes/src/for_/array/for_decompress.rs +++ b/encodings/fastlanes/src/for_/array/for_decompress.rs @@ -35,7 +35,7 @@ use vortex_error::vortex_err; use crate::BitPacked; use crate::BitPackedArrayExt; -use crate::BitWidths; +use crate::BitWidthsView; use crate::FL_CHUNK_SIZE; use crate::FoRArray; use crate::for_::array::FoRArrayExt; @@ -312,7 +312,7 @@ fn unpack_chunks< output: &mut [MaybeUninit], ) -> VortexResult<()> { let offset = usize::from(bp.offset()); - let BitWidths::Global(bit_width) = bp.bit_widths() else { + let BitWidthsView::Global(bit_width) = bp.bit_widths() else { vortex_bail!("BitPacked array has per-block bit widths"); }; let bit_width = bit_width as usize; diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index da456fb9d31..a61270bf926 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -47,7 +47,7 @@ use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::BitPackedData; -use vortex::encodings::fastlanes::BitWidths; +use vortex::encodings::fastlanes::BitWidthsView; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::fastlanes::FoRArraySlotsExt; @@ -476,8 +476,8 @@ mod standalone { cuda_session: &CudaSession, cuda_ctx: &mut CudaExecutionCtx, ) -> Self { - assert!(matches!(values_bp.bit_widths(), BitWidths::Global(6))); - assert!(matches!(codes_bp.bit_widths(), BitWidths::Global(6))); + assert!(matches!(values_bp.bit_widths(), BitWidthsView::Global(6))); + assert!(matches!(codes_bp.bit_widths(), BitWidthsView::Global(6))); let values_packed = block_on(cuda_ctx.ensure_on_device(values_bp.packed().clone())) .vortex_expect("values packed"); diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index 0ddc08dcdfb..415d4bc0f82 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -30,7 +30,7 @@ use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArrayExt; -use vortex::encodings::fastlanes::BitWidths; +use vortex::encodings::fastlanes::BitWidthsView; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::fastlanes::FoRArraySlotsExt; @@ -635,7 +635,7 @@ impl FusedPlan { let source_ptype = ptype_to_tag(PType::try_from(bp.dtype()).map_err(|_| { vortex_err!("BitPacked must have primitive dtype, got {:?}", bp.dtype()) })?); - let BitWidths::Global(bit_width) = bp.bit_widths() else { + let BitWidthsView::Global(bit_width) = bp.bit_widths() else { vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); }; let buf_index = self.source_buffers.len(); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 9db84b97d7f..5e2cd411674 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -27,6 +27,7 @@ use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::BitPackedDataParts; use vortex::encodings::fastlanes::BitWidths; +use vortex::encodings::fastlanes::BitWidthsView; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; use vortex::error::vortex_bail; @@ -64,7 +65,7 @@ pub(crate) fn bitpacked_slice_view( let block_start = offset_start - bitpacked_offset; let block_stop = offset_stop.div_ceil(PATCH_CHUNK_SIZE) * PATCH_CHUNK_SIZE; - let BitWidths::Global(bit_width) = bp.bit_widths() else { + let BitWidthsView::Global(bit_width) = bp.bit_widths() else { vortex_bail!("CUDA does not support BitPacked arrays with per-block bit widths"); }; let encoded_start = (block_start / 8) * bit_width as usize; diff --git a/vortex-python/src/arrays/fastlanes.rs b/vortex-python/src/arrays/fastlanes.rs index 8b8b9cc0b03..45ddd847b83 100644 --- a/vortex-python/src/arrays/fastlanes.rs +++ b/vortex-python/src/arrays/fastlanes.rs @@ -4,7 +4,7 @@ use pyo3::prelude::*; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArrayExt; -use vortex::encodings::fastlanes::BitWidths; +use vortex::encodings::fastlanes::BitWidthsView; use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::FoR; @@ -26,8 +26,8 @@ impl PyFastLanesBitPackedArray { #[getter] fn bit_width(self_: PyRef<'_, Self>) -> Option { match self_.as_super().inner().as_::().bit_widths() { - BitWidths::Global(bit_width) => Some(bit_width), - BitWidths::Blocked(_) => None, + BitWidthsView::Global(bit_width) => Some(bit_width), + BitWidthsView::Blocked(_) => None, } } }