feat(array): builders no longer canonicalize their children [builders-child-stack]

vortex-array/src/builders/extension.rs

@@ -13,8 +13,8 @@ use crate::IntoArray;
 use crate::arrays::ExtensionArray;
 use crate::arrays::extension::ExtensionArrayExt;
 use crate::builders::ArrayBuilder;
+use crate::builders::ChildBuilder;
 use crate::builders::DEFAULT_BUILDER_CAPACITY;
-use crate::builders::builder_with_capacity;
 use crate::canonical::Canonical;
 use crate::dtype::DType;
 use crate::dtype::extension::ExtDTypeRef;
@@ -24,7 +24,7 @@ use crate::scalar::Scalar;
 /// The builder for building a [`ExtensionArray`].
 pub struct ExtensionBuilder {
     dtype: DType,
-    storage: Box<dyn ArrayBuilder>,
+    storage: ChildBuilder,
 }
 
 impl ExtensionBuilder {
@@ -36,7 +36,7 @@ impl ExtensionBuilder {
     /// Creates a new `ExtensionBuilder` with the given `capacity`.
     pub fn with_capacity(ext_dtype: ExtDTypeRef, capacity: usize) -> Self {
         Self {
-            storage: builder_with_capacity(ext_dtype.storage_dtype(), capacity),
+            storage: ChildBuilder::with_capacity(ext_dtype.storage_dtype(), capacity),
             dtype: DType::Extension(ext_dtype),
         }
     }
@@ -53,9 +53,7 @@ impl ExtensionBuilder {
         array: &ExtensionArray,
         ctx: &mut ExecutionCtx,
     ) -> VortexResult<()> {
-        array
-            .storage_array()
-            .append_to_builder(self.storage.as_mut(), ctx)
+        self.storage.append_array(array.storage_array(), ctx)
     }
 
     /// Finishes the builder directly into a [`ExtensionArray`].

vortex-array/src/builders/fixed_size_list.rs

@@ -17,9 +17,9 @@ use crate::IntoArray;
 use crate::arrays::FixedSizeListArray;
 use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt;
 use crate::builders::ArrayBuilder;
+use crate::builders::ChildBuilder;
 use crate::builders::DEFAULT_BUILDER_CAPACITY;
 use crate::builders::LazyBitBufferBuilder;
-use crate::builders::builder_with_capacity;
 use crate::canonical::Canonical;
 use crate::dtype::DType;
 use crate::dtype::Nullability;
@@ -34,7 +34,7 @@ pub struct FixedSizeListBuilder {
     /// The builder for the underlying elements of the [`FixedSizeListArray`].
     ///
     /// This builder will have a capacity equal to the `list_size * capacity`.
-    elements_builder: Box<dyn ArrayBuilder>,
+    elements_builder: ChildBuilder,
 
     /// The null map builder of the [`FixedSizeListArray`].
     ///
@@ -62,7 +62,7 @@ impl FixedSizeListBuilder {
     ) -> Self {
         let elements_capacity = capacity * list_size as usize;
 
-        let elements_builder = builder_with_capacity(&element_dtype, elements_capacity);
+        let elements_builder = ChildBuilder::with_capacity(&element_dtype, elements_capacity);
         let fsl_dtype = DType::FixedSizeList(element_dtype, list_size, nullability);
         let nulls = LazyBitBufferBuilder::new(capacity);
 
@@ -98,7 +98,7 @@ impl FixedSizeListBuilder {
             self.list_size()
         );
 
-        array.append_to_builder(self.elements_builder.as_mut(), ctx)?;
+        self.elements_builder.append_array(array, ctx)?;
         self.nulls.append_non_null();
 
         Ok(())
@@ -115,9 +115,7 @@ impl FixedSizeListBuilder {
             return Ok(());
         }
 
-        array
-            .elements()
-            .append_to_builder(self.elements_builder.as_mut(), ctx)?;
+        self.elements_builder.append_array(array.elements(), ctx)?;
         self.nulls
             .append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
         Ok(())

vortex-array/src/builders/list.rs

@@ -25,10 +25,10 @@ use crate::arrays::PrimitiveArray;
 use crate::arrays::list::ListArraySlotsExt;
 use crate::arrays::listview::ListViewArraySlotsExt;
 use crate::builders::ArrayBuilder;
+use crate::builders::ChildBuilder;
 use crate::builders::DEFAULT_BUILDER_CAPACITY;
 use crate::builders::LazyBitBufferBuilder;
 use crate::builders::PrimitiveBuilder;
-use crate::builders::builder_with_capacity;
 use crate::dtype::DType;
 use crate::dtype::IntegerPType;
 use crate::dtype::Nullability;
@@ -44,7 +44,7 @@ pub struct ListBuilder<O: IntegerPType> {
     dtype: DType,
 
     /// The builder for the underlying elements of the [`ListArray`].
-    elements_builder: Box<dyn ArrayBuilder>,
+    elements_builder: ChildBuilder,
 
     /// The builder for the `offsets` into the `elements` array.
     offsets_builder: PrimitiveBuilder<O>,
@@ -79,7 +79,7 @@ impl<O: IntegerPType> ListBuilder<O> {
         elements_capacity: usize,
         capacity: usize,
     ) -> Self {
-        let elements_builder = builder_with_capacity(value_dtype.as_ref(), elements_capacity);
+        let elements_builder = ChildBuilder::with_capacity(value_dtype.as_ref(), elements_capacity);
         let mut offsets_builder = PrimitiveBuilder::<O>::with_capacity(NonNullable, capacity + 1);
 
         // The first offset is always 0 and represents an empty list.
@@ -111,8 +111,7 @@ impl<O: IntegerPType> ListBuilder<O> {
             self.element_dtype()
         );
 
-        self.elements_builder.reserve_exact(array.len());
-        array.append_to_builder(self.elements_builder.as_mut(), ctx)?;
+        self.elements_builder.append_array(array, ctx)?;
         self.nulls.append_non_null();
         self.offsets_builder.append_value(
             O::from_usize(self.elements_builder.len())
@@ -210,7 +209,7 @@ where
             let list_elements = new_elements
                 .slice(offset..offset + size)
                 .vortex_expect("list builder slice");
-            list_elements.append_to_builder(builder.elements_builder.as_mut(), ctx)?;
+            builder.elements_builder.append_array(&list_elements, ctx)?;
             curr_offset += size;
         }
 
@@ -320,11 +319,8 @@ impl<O: IntegerPType> ArrayBuilder for ListBuilder<O> {
             // in bulk and the offsets rebased onto this builder's elements.
             let elements_base = self.elements_builder.len();
             if last > first {
-                self.elements_builder.reserve_exact(last - first);
-                array
-                    .elements()
-                    .slice(first..last)?
-                    .append_to_builder(self.elements_builder.as_mut(), ctx)?;
+                self.elements_builder
+                    .append_array(&array.elements().slice(first..last)?, ctx)?;
             }
 
             self.offsets_builder.reserve_exact(num_lists);

vortex-array/src/builders/listview.rs

@@ -31,10 +31,10 @@ use crate::arrays::list::ListArraySlotsExt;
 use crate::arrays::listview::ListViewArraySlotsExt;
 use crate::arrays::listview::ListViewRebuildMode;
 use crate::builders::ArrayBuilder;
+use crate::builders::ChildBuilder;
 use crate::builders::DEFAULT_BUILDER_CAPACITY;
 use crate::builders::PrimitiveBuilder;
 use crate::builders::UninitRange;
-use crate::builders::builder_with_capacity;
 use crate::builders::lazy_null_builder::LazyBitBufferBuilder;
 use crate::builtins::ArrayBuiltins;
 use crate::dtype::DType;
@@ -58,7 +58,7 @@ pub struct ListViewBuilder<O: IntegerPType, S: IntegerPType> {
     dtype: DType,
 
     /// The builder for the underlying elements of the [`ListArray`](crate::arrays::ListArray).
-    elements_builder: Box<dyn ArrayBuilder>,
+    elements_builder: ChildBuilder,
 
     /// The builder for the `offsets` into the `elements` array.
     offsets_builder: PrimitiveBuilder<O>,
@@ -96,7 +96,7 @@ impl<O: IntegerPType, S: IntegerPType> ListViewBuilder<O, S> {
         elements_capacity: usize,
         capacity: usize,
     ) -> Self {
-        let elements_builder = builder_with_capacity(&element_dtype, elements_capacity);
+        let elements_builder = ChildBuilder::with_capacity(&element_dtype, elements_capacity);
 
         let offsets_builder =
             PrimitiveBuilder::<O>::with_capacity(Nullability::NonNullable, capacity);
@@ -142,8 +142,7 @@ impl<O: IntegerPType, S: IntegerPType> ListViewBuilder<O, S> {
             "appending this list would cause an offset overflow"
         );
 
-        self.elements_builder.reserve_exact(num_elements);
-        array.append_to_builder(self.elements_builder.as_mut(), ctx)?;
+        self.elements_builder.append_array(array, ctx)?;
         self.nulls.append_non_null();
 
         self.offsets_builder.append_value(
@@ -365,10 +364,7 @@ impl<O: IntegerPType, S: IntegerPType> ArrayBuilder for ListViewBuilder<O, S> {
         // Bulk append the new elements (which should have no gaps or overlaps).
         let old_elements_len = self.elements_builder.len();
         self.elements_builder
-            .reserve_exact(listview.elements().len());
-        listview
-            .elements()
-            .append_to_builder(self.elements_builder.as_mut(), ctx)?;
+            .append_array(listview.elements(), ctx)?;
         let new_elements_len = self.elements_builder.len();
 
         // Reserve enough space for the new views.
@@ -432,10 +428,9 @@ where
     );
 
     if last > first {
-        builder.elements_builder.reserve_exact(last - first);
-        elements
-            .slice(first..last)?
-            .append_to_builder(builder.elements_builder.as_mut(), ctx)?;
+        builder
+            .elements_builder
+            .append_array(&elements.slice(first..last)?, ctx)?;
     }
 
     builder.offsets_builder.reserve_exact(num_lists);

vortex-array/src/builders/mod.rs

@@ -6,6 +6,12 @@
 //! Every logical type in Vortex has a canonical (uncompressed) in-memory encoding. This module
 //! provides pre-allocated builders to construct new canonical arrays.
 //!
+//! Canonical form is not recursive, and neither are these builders: appending an array to a nested
+//! builder keeps the child in the encoding it arrived in instead of decoding it. The fields of a
+//! [`StructArray`](crate::arrays::StructArray), the elements of a list, and the storage of an
+//! [`ExtensionArray`](crate::arrays::ExtensionArray) may therefore come back compressed, or as a
+//! [`ChunkedArray`](crate::arrays::ChunkedArray) when several arrays were appended in turn.
+//!
 //! ## Example:
 //!
 //! ```
@@ -53,6 +59,7 @@ mod lazy_null_builder;
 pub(crate) use lazy_null_builder::LazyBitBufferBuilder;
 
 mod bool;
+mod child;
 mod decimal;
 pub mod dict;
 mod extension;
@@ -65,6 +72,7 @@ mod struct_;
 mod varbinview;
 
 pub use bool::*;
+pub(crate) use child::ChildBuilder;
 pub use decimal::*;
 pub use extension::*;
 pub use fixed_size_list::*;
@@ -180,6 +188,9 @@ pub trait ArrayBuilder: Send {
 
     /// Constructs an Array from the builder components.
     ///
+    /// The returned array is canonical at the top level only; its children keep whatever encoding
+    /// they were appended with.
+    ///
     /// # Panics
     ///
     /// This function may panic if the builder's methods are called with invalid arguments. If only

vortex-array/src/builders/struct_.rs

@@ -17,9 +17,9 @@ use crate::IntoArray;
 use crate::arrays::StructArray;
 use crate::arrays::struct_::StructArrayExt;
 use crate::builders::ArrayBuilder;
+use crate::builders::ChildBuilder;
 use crate::builders::DEFAULT_BUILDER_CAPACITY;
 use crate::builders::LazyBitBufferBuilder;
-use crate::builders::builder_with_capacity;
 use crate::canonical::Canonical;
 use crate::dtype::DType;
 use crate::dtype::Nullability;
@@ -30,7 +30,7 @@ use crate::scalar::StructScalar;
 /// The builder for building a [`StructArray`].
 pub struct StructBuilder {
     dtype: DType,
-    builders: Vec<Box<dyn ArrayBuilder>>,
+    builders: Vec<ChildBuilder>,
     nulls: LazyBitBufferBuilder,
 }
 
@@ -48,7 +48,7 @@ impl StructBuilder {
     ) -> Self {
         let builders = struct_dtype
             .fields()
-            .map(|dt| builder_with_capacity(&dt, capacity))
+            .map(|dt| ChildBuilder::with_capacity(&dt, capacity))
             .collect();
 
         Self {
@@ -131,7 +131,7 @@ impl StructBuilder {
             .iter_unmasked_fields()
             .zip_eq(self.builders.iter_mut())
         {
-            field.append_to_builder(builder.as_mut(), ctx)?;
+            builder.append_array(field, ctx)?;
         }
 
         self.nulls