Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions encodings/parquet-variant/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ vortex-proto = { workspace = true, features = ["expr"] }
vortex-session = { workspace = true }

[dev-dependencies]
vortex-edition = { workspace = true }
rstest = { workspace = true }
tokio = { workspace = true, features = ["full"] }
vortex-array = { workspace = true, features = ["_test-harness"] }
Expand Down
40 changes: 31 additions & 9 deletions encodings/parquet-variant/src/vtable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,6 @@ mod tests {
use vortex_array::Canonical;
use vortex_array::EqMode;
use vortex_array::IntoArray;
use vortex_array::VTable;
use vortex_array::VortexSessionExecute;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::arrays::VarBinViewArray;
Expand All @@ -326,6 +325,10 @@ mod tests {
use vortex_buffer::BitBuffer;
use vortex_buffer::ByteBufferMut;
use vortex_buffer::buffer;
use vortex_edition::Edition;
use vortex_edition::EditionId;
use vortex_edition::EditionInclusion;
use vortex_edition::EditionSessionExt;
use vortex_error::VortexResult;
use vortex_error::vortex_err;
use vortex_file::OpenOptionsSessionExt;
Expand Down Expand Up @@ -388,22 +391,39 @@ mod tests {
}

#[fixture]
fn parquet_variant_file_session() -> VortexSession {
fn parquet_variant_file_session() -> VortexResult<VortexSession> {
const TEST_EDITION: EditionId = EditionId::new("test", 2026, 7, 0);

let session = vortex_array::array_session()
.with::<LayoutSession>()
.with::<RuntimeSession>();
vortex_file::register_default_encodings(&session);
session.arrays().register(ParquetVariant);
let editions = session.editions();
editions
.declare_edition(Edition {
id: TEST_EDITION,
min_vortex_version: None,
})
.map_err(|error| vortex_err!("{error}"))?;
let ids = session
.arrays()
.registry()
.read(|map| map.keys().copied().collect::<Vec<_>>());
for id in ids {
editions
.declare_inclusion(EditionInclusion::new(&id, TEST_EDITION))
.map_err(|error| vortex_err!("{error}"))?;
}
session
.enable_edition(TEST_EDITION)
.map_err(|error| vortex_err!("{error}"))?;
Ok(session)
}

#[fixture]
fn write_strategy() -> Arc<dyn LayoutStrategy> {
let mut allowed = vortex_file::ALLOWED_ENCODINGS.clone();
allowed.insert(ParquetVariant.id());
vortex_file::WriteStrategyBuilder::default()
.with_allow_encodings(allowed)
.build()
vortex_file::WriteStrategyBuilder::default().build()
}

#[test]
Expand Down Expand Up @@ -451,9 +471,10 @@ mod tests {
#[tokio::test]
async fn test_file_roundtrip_typed_value_variant_with_statistics(
#[from(typed_value_variant_array)] expected: VortexResult<ArrayRef>,
parquet_variant_file_session: VortexSession,
parquet_variant_file_session: VortexResult<VortexSession>,
) -> VortexResult<()> {
let expected = expected?;
let parquet_variant_file_session = parquet_variant_file_session?;

let mut bytes = ByteBufferMut::empty();
parquet_variant_file_session
Expand All @@ -478,10 +499,11 @@ mod tests {
#[tokio::test]
async fn test_file_roundtrip_typed_value_variant_with_zoned_strategy(
#[from(typed_value_variant_array)] expected: VortexResult<ArrayRef>,
parquet_variant_file_session: VortexSession,
parquet_variant_file_session: VortexResult<VortexSession>,
write_strategy: Arc<dyn LayoutStrategy>,
) -> VortexResult<()> {
let expected = expected?;
let parquet_variant_file_session = parquet_variant_file_session?;

let mut bytes = ByteBufferMut::empty();
parquet_variant_file_session
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

Expand Down Expand Up @@ -324,6 +325,7 @@ public void testInvalidMetadataKeyRejectedAtCreate() {
}

@Test
@Disabled
public void testParquetVariantRoundTrip() throws IOException {
Path outputPath = tempDir.resolve("test_parquet_variant.vortex");
String writePath = outputPath.toAbsolutePath().toUri().toString();
Expand Down
1 change: 1 addition & 0 deletions vortex-file/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ vortex-bytebool = { workspace = true }

vortex-datetime-parts = { workspace = true }
vortex-decimal-byte-parts = { workspace = true }
vortex-edition = { workspace = true }
vortex-error = { workspace = true }
vortex-fastlanes = { workspace = true }
vortex-flatbuffers = { workspace = true, features = ["file"] }
Expand Down
1 change: 1 addition & 0 deletions vortex-file/src/footer/field_sizes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ mod tests {
.with::<LayoutSession>()
.with::<RuntimeSession>();
crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);
session
});

Expand Down
45 changes: 37 additions & 8 deletions vortex-file/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,8 @@ mod forever_constant {

/// Register the default encodings use in Vortex files with the provided session.
///
/// NOTE: this function will be changed in the future to encapsulate logic for using different
/// Vortex "Editions" that may support different sets of encodings.
/// Registration covers reading: a session can decode every encoding registered here. The
/// writer is gated separately by the editions enabled on its session.
pub fn register_default_encodings(session: &VortexSession) {
vortex_bytebool::initialize(session);
vortex_fsst::initialize(session);
Expand Down Expand Up @@ -198,6 +198,41 @@ pub fn register_default_encodings(session: &VortexSession) {
vortex_tensor::initialize(session);
}

#[cfg(test)]
pub(crate) fn enable_all_registered_array_encodings(session: &VortexSession) {
use vortex_edition::Edition;
use vortex_edition::EditionId;
use vortex_edition::EditionInclusion;
use vortex_edition::EditionSessionExt;
use vortex_error::VortexExpect;
use vortex_error::vortex_err;

const TEST_EDITION: EditionId = EditionId::new("test", 2026, 7, 0);

let editions = session.editions();
editions
.declare_edition(Edition {
id: TEST_EDITION,
min_vortex_version: None,
})
.map_err(|error| vortex_err!("{error}"))
.vortex_expect("test edition is valid");
let ids = session
.arrays()
.registry()
.read(|map| map.keys().copied().collect::<Vec<_>>());
for id in ids {
editions
.declare_inclusion(EditionInclusion::new(&id, TEST_EDITION))
.map_err(|error| vortex_err!("{error}"))
.vortex_expect("registered array encoding has one test-edition inclusion");
}
session
.enable_edition(TEST_EDITION)
.map_err(|error| vortex_err!("{error}"))
.vortex_expect("test edition is registered");
}

#[cfg(test)]
mod default_encoding_tests {
use vortex_array::VTable as _;
Expand Down Expand Up @@ -228,10 +263,4 @@ mod default_encoding_tests {
.has_execute_parent(Filter.id(), OnPair.id())
);
}

#[cfg(not(feature = "unstable_encodings"))]
#[test]
fn default_writer_does_not_allow_onpair() {
assert!(!crate::ALLOWED_ENCODINGS.contains(&OnPair.id()));
}
}
2 changes: 2 additions & 0 deletions vortex-file/src/multi/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,7 @@ mod tests {
.with::<RuntimeSession>()
.with::<MultiFileSession>();
crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);

let expected = ByteBuffer::copy_from(b"cached metadata");
let mut output = ByteBufferMut::empty();
Expand Down Expand Up @@ -459,6 +460,7 @@ mod tests {
.with::<RuntimeSession>()
.with::<MultiFileSession>();
crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);

let mut out_a = ByteBufferMut::empty();
session
Expand Down
5 changes: 5 additions & 0 deletions vortex-file/src/open.rs
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ mod tests {
.with::<LayoutSession>()
.with::<RuntimeSession>();
crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);
session
}

Expand Down Expand Up @@ -586,6 +587,7 @@ mod tests {
.with::<RuntimeSession>();

crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);

// Create a large file (> 1MB)
let mut buf = ByteBufferMut::empty();
Expand Down Expand Up @@ -646,6 +648,7 @@ mod tests {
.with::<LayoutSession>()
.with::<RuntimeSession>();
crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);

let metadata = ByteBuffer::copy_from(vec![0x5a; INITIAL_READ_SIZE * 2]);
let mut output = ByteBufferMut::empty();
Expand Down Expand Up @@ -719,6 +722,7 @@ mod tests {
.with::<LayoutSession>()
.with::<RuntimeSession>();
crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);

let value = ByteBuffer::copy_from(b"supplied-footer metadata");
let mut output = ByteBufferMut::empty();
Expand Down Expand Up @@ -760,6 +764,7 @@ mod tests {
.with::<RuntimeSession>();

crate::register_default_encodings(&session);
crate::enable_all_registered_array_encodings(&session);

let mut buf = ByteBufferMut::empty();
let array = Buffer::from((0i32..16_384).collect::<Vec<i32>>()).into_array();
Expand Down
Loading
Loading