Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion vortex-array/src/aggregate_fn/fns/is_constant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ mod varbin;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_mask::Mask;
use vortex_session::registry::CachedId;

use self::bool::check_bool_constant;
Expand Down Expand Up @@ -76,7 +77,7 @@ fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> Vor
// Compare values element-wise. Result is null where both inputs are null,
// true/false where both are valid.
let eq_result = a.binary(b.clone(), Operator::Eq)?;
let eq_result = eq_result.null_as_false().execute(ctx)?;
let eq_result = eq_result.fill_null(false)?.execute::<Mask>(ctx)?;

Ok(eq_result.true_count() == valid_count)
}
Expand Down
3 changes: 2 additions & 1 deletion vortex-array/src/arrays/bool/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use crate::array::ArrayView;
use crate::arrays::Bool;
use crate::arrays::BoolArray;
use crate::arrays::bool::BoolArrayExt;
use crate::builtins::ArrayBuiltins;
use crate::scalar_fn::fns::zip::ZipKernel;
use crate::scalar_fn::fns::zip::zip_validity;

Expand All @@ -34,7 +35,7 @@ impl ZipKernel for Bool {
};

// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
let mask = mask.clone().null_as_false().execute(ctx)?;
let mask = mask.clone().fill_null(false)?.execute::<Mask>(ctx)?;
let mask_values = match &mask {
// Defer trivial masks to the generic zip, which just casts the surviving side.
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/listview/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ impl ZipKernel for ListView {
};

// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
let mask = mask.clone().null_as_false().execute(ctx)?;
let mask = mask.clone().fill_null(false)?.execute::<Mask>(ctx)?;
match &mask {
// Defer the trivial masks to the generic zip, which just casts one side.
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
Expand Down
3 changes: 2 additions & 1 deletion vortex-array/src/arrays/primitive/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use crate::IntoArray;
use crate::array::ArrayView;
use crate::arrays::Primitive;
use crate::arrays::PrimitiveArray;
use crate::builtins::ArrayBuiltins;
use crate::dtype::NativePType;
use crate::match_each_native_ptype;
use crate::scalar_fn::fns::zip::ZipKernel;
Expand Down Expand Up @@ -46,7 +47,7 @@ impl ZipKernel for Primitive {
}

// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
let mask = mask.clone().null_as_false().execute(ctx)?;
let mask = mask.clone().fill_null(false)?.execute::<Mask>(ctx)?;
match &mask {
// Defer trivial masks to the generic zip, which just casts the surviving side.
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
Expand Down
3 changes: 2 additions & 1 deletion vortex-array/src/arrays/struct_/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use std::ops::BitOr;
use std::ops::Not;

use vortex_error::VortexResult;
use vortex_mask::Mask;

use crate::ArrayRef;
use crate::ExecutionCtx;
Expand Down Expand Up @@ -48,7 +49,7 @@ impl ZipKernel for Struct {
(Validity::AllInvalid, Validity::AllInvalid) => Validity::AllInvalid,

(v1, v2) => {
let mask_mask = mask.clone().null_as_false().execute(ctx)?;
let mask_mask = mask.clone().fill_null(false)?.execute::<Mask>(ctx)?;
let v1m = v1.execute_mask(if_true.len(), ctx)?;
let v2m = v2.execute_mask(if_false.len(), ctx)?;

Expand Down
3 changes: 2 additions & 1 deletion vortex-array/src/arrays/varbinview/compute/zip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use crate::arrays::varbinview::BinaryView;
use crate::arrays::varbinview::VarBinViewArrayExt;
use crate::builders::DeduplicatedBuffers;
use crate::builders::LazyBitBufferBuilder;
use crate::builtins::ArrayBuiltins;
use crate::scalar_fn::fns::zip::ZipKernel;

// A dedicated VarBinView zip kernel that builds the result directly by adjusting views and validity,
Expand Down Expand Up @@ -57,7 +58,7 @@ impl ZipKernel for VarBinView {
let true_validity = if_true.varbinview_validity().execute_mask(len, ctx)?;
let false_validity = if_false.varbinview_validity().execute_mask(len, ctx)?;

let mask = mask.clone().null_as_false().execute(ctx)?;
let mask = mask.clone().fill_null(false)?.execute::<Mask>(ctx)?;
let if_false_view = if_false;
match mask.slices() {
AllOr::All => push_range(
Expand Down
82 changes: 7 additions & 75 deletions vortex-array/src/mask.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ use crate::Executable;
use crate::ExecutionCtx;
use crate::IntoArray;
use crate::arrays::BoolArray;
use crate::builtins::ArrayBuiltins;
use crate::columnar::Columnar;
use crate::dtype::DType;
use crate::dtype::Nullability;
use crate::validity::Validity;

impl Executable for Mask {
/// Executes a boolean array into a [`Mask`].
Expand Down Expand Up @@ -42,53 +42,20 @@ impl Executable for Mask {
}
}

/// An adapter that coerces null elements of a boolean array to `false` before executing it into a
/// [`Mask`]. Created by [`ArrayRef::null_as_false`].
///
/// Use for filter and pruning predicates over nullable data, where SQL semantics treat `NULL` as
/// not matching.
#[deprecated = "Use array.fill_null(false)?.execute::<Mask>(ctx)"]
pub struct NullAsFalse(ArrayRef);

impl ArrayRef {
/// Returns an adapter that treats null elements of this boolean array as `false` when executed
/// into a [`Mask`]. See [`NullAsFalse`].
#[deprecated = "Use array.fill_null(false)?.execute::<Mask>(ctx)"]
#[allow(deprecated)]
pub fn null_as_false(self) -> NullAsFalse {
NullAsFalse(self)
}
}

#[allow(deprecated)]
impl NullAsFalse {
/// Executes the boolean array into a [`Mask`], coercing null elements to `false`.
///
/// Canonicalizes the (possibly lazy) array exactly once and folds validity into the value bits
/// with a single `AND` that reuses the value buffer when it is uniquely owned.
#[deprecated = "Use array.fill_null(false)?.execute::<Mask>(ctx)"]
pub fn execute(self, ctx: &mut ExecutionCtx) -> VortexResult<Mask> {
let array = self.0;
if !matches!(array.dtype(), DType::Bool(_)) {
vortex_bail!("Mask array must have boolean dtype, not {}", array.dtype());
}
// Non-nullable input needs no coercion; defer to the strict `Mask` execution.
if !array.dtype().is_nullable() {
return array.execute::<Mask>(ctx);
}

let len = array.len();
Ok(match array.execute::<Columnar>(ctx)? {
Columnar::Constant(c) => Mask::new(len, c.scalar().as_bool().value().unwrap_or(false)),
Columnar::Canonical(c) => {
let bool = c.into_array().execute::<BoolArray>(ctx)?;
match bool.as_ref().validity()? {
Validity::NonNullable | Validity::AllValid => {
Mask::from_buffer(bool.into_bit_buffer())
}
Validity::AllInvalid => Mask::new_false(len),
Validity::Array(v) => {
let validity_bits = v.execute::<BoolArray>(ctx)?.into_bit_buffer();
Mask::from_buffer(bool.into_bit_buffer() & &validity_bits)
}
}
}
})
self.0.fill_null(false)?.execute::<Mask>(ctx)
}
}

Expand Down Expand Up @@ -142,39 +109,4 @@ mod tests {
assert_eq!(mask, Mask::new_false(4));
Ok(())
}

#[test]
fn null_as_false_non_nullable() -> VortexResult<()> {
let array = BoolArray::from_iter([true, false, true]).into_array();
let mask = array.null_as_false().execute(&mut ctx())?;
assert_eq!(mask, Mask::from_iter([true, false, true]));
Ok(())
}

#[test]
fn null_as_false_treats_null_as_false() -> VortexResult<()> {
let array = BoolArray::from_iter([Some(true), None, Some(false), None]).into_array();
let mask = array.null_as_false().execute(&mut ctx())?;
assert_eq!(mask, Mask::from_iter([true, false, false, false]));
Ok(())
}

#[test]
fn null_as_false_null_constant() -> VortexResult<()> {
let array =
ConstantArray::new(Scalar::null(DType::Bool(Nullability::Nullable)), 4).into_array();
let mask = array.null_as_false().execute(&mut ctx())?;
assert_eq!(mask, Mask::new_false(4));
Ok(())
}

#[test]
fn null_as_false_matches_fill_null_then_mask() -> VortexResult<()> {
let array =
BoolArray::from_iter([Some(true), None, Some(false), Some(true), None]).into_array();
let via_fill_null = array.fill_null(false)?.execute::<Mask>(&mut ctx())?;
let via_coerce = array.null_as_false().execute(&mut ctx())?;
assert_eq!(via_coerce, via_fill_null);
Ok(())
}
}
5 changes: 3 additions & 2 deletions vortex-array/src/scalar_fn/fns/fill_null/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ use crate::ArrayRef;
use crate::CanonicalView;
use crate::ColumnarView;
use crate::ExecutionCtx;
use crate::IntoArray;
use crate::arrays::Bool;
use crate::arrays::Decimal;
use crate::arrays::Primitive;
use crate::arrays::ScalarFnArray;
use crate::builtins::ArrayBuiltins;
use crate::dtype::DType;
use crate::expr::BoundExpression;
use crate::scalar::Scalar;
Expand Down Expand Up @@ -114,7 +114,8 @@ impl ScalarFnVTable for FillNull {
);

let Some(columnar) = input.as_opt::<AnyColumnar>() else {
return input.execute::<ArrayRef>(ctx)?.fill_null(fill_scalar);
let input = input.execute::<ArrayRef>(ctx)?;
return Ok(FillNull::try_new(input, fill_value)?.into_array());
};

match columnar {
Expand Down
5 changes: 3 additions & 2 deletions vortex-cuda/src/layout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use vortex::array::MaskFuture;
use vortex::array::ProstMetadata;
use vortex::array::VortexSessionExecute;
use vortex::array::arrays::Constant;
use vortex::array::builtins::ArrayBuiltins;
use vortex::array::expr::BoundExpression;
use vortex::array::expr::stats::Precision;
use vortex::array::expr::stats::Stat;
Expand Down Expand Up @@ -314,12 +315,12 @@ impl LayoutReader for CudaFlatReader {
let array = array.apply_bound(&expr)?;
let array = array.filter(mask.clone())?;
let mut ctx = session.create_execution_ctx();
let array_mask = array.null_as_false().execute(&mut ctx)?;
let array_mask = array.fill_null(false)?.execute::<Mask>(&mut ctx)?;
mask.intersect_by_rank(&array_mask)
} else {
let array = array.apply_bound(&expr)?;
let mut ctx = session.create_execution_ctx();
let array_mask = array.null_as_false().execute(&mut ctx)?;
let array_mask = array.fill_null(false)?.execute::<Mask>(&mut ctx)?;
mask.bitand(&array_mask)
};

Expand Down
6 changes: 5 additions & 1 deletion vortex-layout/src/layouts/dict/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use vortex_array::MaskFuture;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::DictArray;
use vortex_array::arrays::SharedArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::dtype::FieldMask;
use vortex_array::dtype::Nullability;
Expand Down Expand Up @@ -277,7 +278,10 @@ impl LayoutReader for DictReader {
let mask = mask.await?;

let mut ctx = session.create_execution_ctx();
let dict_mask = values.take(codes)?.null_as_false().execute(&mut ctx)?;
let dict_mask = values
.take(codes)?
.fill_null(false)?
.execute::<Mask>(&mut ctx)?;

Ok(mask.bitand(&dict_mask))
}))
Expand Down
5 changes: 3 additions & 2 deletions vortex-layout/src/layouts/flat/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use tracing::trace;
use vortex_array::ArrayRef;
use vortex_array::MaskFuture;
use vortex_array::VortexSessionExecute;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::dtype::FieldMask;
use vortex_array::expr::BoundExpression;
Expand Down Expand Up @@ -156,14 +157,14 @@ impl LayoutReader for FlatReader {
let array = array.apply_bound(&expr)?;
let array = array.filter(mask.clone())?;
let mut ctx = session.create_execution_ctx();
let array_mask = array.null_as_false().execute(&mut ctx)?;
let array_mask = array.fill_null(false)?.execute::<Mask>(&mut ctx)?;

mask.intersect_by_rank(&array_mask)
} else {
// Run over the full array, with a simpler bitand at the end.
let array = array.apply_bound(&expr)?;
let mut ctx = session.create_execution_ctx();
let array_mask = array.null_as_false().execute(&mut ctx)?;
let array_mask = array.fill_null(false)?.execute::<Mask>(&mut ctx)?;

mask.bitand(&array_mask)
};
Expand Down
10 changes: 3 additions & 7 deletions vortex-layout/src/layouts/list/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -437,17 +437,18 @@ impl LayoutReader for ListReader {
return Ok(mask);
}

let mut ctx = session.create_execution_ctx();
if mask.density() < EXPR_EVAL_THRESHOLD {
let predicate = reader
.projection_evaluation(&row_range, &expr, MaskFuture::ready(mask.clone()))?
.await?;
let predicate_mask = predicate_array_to_mask(predicate, &session)?;
let predicate_mask = predicate.fill_null(false)?.execute::<Mask>(&mut ctx)?;
Ok(mask.intersect_by_rank(&predicate_mask))
} else {
let predicate = reader
.projection_evaluation(&row_range, &expr, MaskFuture::new_true(len))?
.await?;
let predicate_mask = predicate_array_to_mask(predicate, &session)?;
let predicate_mask = predicate.fill_null(false)?.execute::<Mask>(&mut ctx)?;
Ok(mask & &predicate_mask)
}
}))
Expand Down Expand Up @@ -599,11 +600,6 @@ fn apply_lengths_validity(
}
}

fn predicate_array_to_mask(array: ArrayRef, session: &VortexSession) -> VortexResult<Mask> {
let mut ctx = session.create_execution_ctx();
array.null_as_false().execute(&mut ctx)
}

#[cfg(test)]
mod tests {
use std::ops::Range;
Expand Down
6 changes: 4 additions & 2 deletions vortex-layout/src/layouts/partitioned.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ use vortex_array::IntoArray;
use vortex_array::MaskFuture;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::StructArray;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::dtype::Nullability;
use vortex_array::expr::BoundExpression;
use vortex_array::expr::transform::BoundPartitionedExpr;
use vortex_array::validity::Validity;
use vortex_error::VortexError;
use vortex_error::VortexResult;
use vortex_mask::Mask;
use vortex_session::VortexSession;

use crate::ArrayFuture;
Expand Down Expand Up @@ -93,8 +95,8 @@ impl<P: Send + Sync + 'static> BoundPartitionedExprEval<P> for BoundPartitionedE
let mut ctx = session.create_execution_ctx();
let root_mask = root_scope
.apply_bound(&self.root)?
.null_as_false()
.execute(&mut ctx)?;
.fill_null(false)?
.execute::<Mask>(&mut ctx)?;

let mask = mask.bitand(&root_mask);

Expand Down
5 changes: 3 additions & 2 deletions vortex-layout/src/layouts/row_idx/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::MaskFuture;
use vortex_array::VortexSessionExecute;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::dtype::FieldMask;
use vortex_array::dtype::FieldName;
Expand Down Expand Up @@ -319,8 +320,8 @@ fn row_idx_mask_future(
let mut ctx = session.create_execution_ctx();
let result_mask = array
.apply_bound(&expr)?
.null_as_false()
.execute(&mut ctx)?;
.fill_null(false)?
.execute::<Mask>(&mut ctx)?;

Ok(result_mask.bitand(&mask.await?))
})
Expand Down
Loading
Loading