From b704f9d122eb27fd9ec17121e0f12bbe94f0d664 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 5 Mar 2026 18:19:06 +0000 Subject: [PATCH 1/5] Layout readers don't need a mutable context Signed-off-by: Robert Kruszewski --- encodings/fastlanes/src/rle/array/mod.rs | 10 ++++- encodings/pco/src/test.rs | 3 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 4 +- vortex-array/src/arrays/masked/vtable/mod.rs | 4 +- vortex-array/src/context.rs | 8 ---- vortex-array/src/lib.rs | 6 ++- vortex-array/src/serde.rs | 5 ++- vortex-cuda/src/layout.rs | 9 ++-- vortex-file/src/footer/file_layout.rs | 12 +++-- vortex-file/src/footer/mod.rs | 16 +++---- vortex-file/src/footer/serializer.rs | 6 +-- vortex-file/src/writer.rs | 3 +- vortex-ipc/src/messages/decoder.rs | 8 ++-- vortex-layout/src/children.rs | 13 +++--- vortex-layout/src/encoding.rs | 7 ++- vortex-layout/src/flatbuffers.rs | 5 +-- vortex-layout/src/layout.rs | 5 +-- vortex-layout/src/layouts/chunked/mod.rs | 4 +- vortex-layout/src/layouts/dict/mod.rs | 4 +- vortex-layout/src/layouts/flat/mod.rs | 12 ++--- vortex-layout/src/layouts/flat/writer.rs | 4 +- vortex-layout/src/layouts/struct_/mod.rs | 4 +- vortex-layout/src/layouts/zoned/mod.rs | 4 +- vortex-layout/src/vtable.rs | 4 +- vortex-python/src/serde/context.rs | 44 ++++++++++++++++++- vortex-python/src/serde/mod.rs | 3 +- vortex-python/src/serde/parts.rs | 4 +- vortex-session/src/registry.rs | 23 ++++++++++ 28 files changed, 149 insertions(+), 85 deletions(-) delete mode 100644 vortex-array/src/context.rs diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 797ba66224d..83f75ad45ae 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -229,6 +229,7 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; + use vortex_session::registry::ReadContext; use crate::RLEArray; use crate::test::SESSION; @@ -458,7 +459,7 @@ mod tests { .decode( &DType::Primitive(PType::U32, Nullability::NonNullable), 2048, - &ctx, + &ReadContext::new(ctx.to_ids()), &SESSION, ) .unwrap(); @@ -498,7 +499,12 @@ mod tests { let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts - .decode(sliced.dtype(), sliced.len(), &ctx, &SESSION) + .decode( + sliced.dtype(), + sliced.len(), + &ReadContext::new(ctx.to_ids()), + &SESSION, + ) .unwrap(); let original_data = sliced.to_primitive(); diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index 2758085846d..a7d038e59d4 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -26,6 +26,7 @@ use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_mask::Mask; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; static SESSION: LazyLock = LazyLock::new(|| { @@ -166,7 +167,7 @@ fn test_serde() -> VortexResult<()> { let decoded = parts.decode( &DType::Primitive(PType::I32, Nullability::NonNullable), 1_000_000, - &context, + &ReadContext::new(context.to_ids()), &SESSION, )?; let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 65cc99da536..2bab9f813af 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -239,7 +239,7 @@ impl DecimalVTable { mod tests { use vortex_buffer::ByteBufferMut; use vortex_buffer::buffer; - + use vortex_session::registry::ReadContext; use crate::ArrayContext; use crate::IntoArray; use crate::LEGACY_SESSION; @@ -273,7 +273,7 @@ mod tests { let concat = concat.freeze(); let parts = ArrayParts::try_from(concat).unwrap(); - let decoded = parts.decode(&dtype, 5, &ctx, &LEGACY_SESSION).unwrap(); + let decoded = parts.decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION).unwrap(); assert!(decoded.is::()); } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 3ba48543c85..f799c5dbdc7 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -217,7 +217,7 @@ mod tests { use rstest::rstest; use vortex_buffer::ByteBufferMut; use vortex_error::VortexError; - + use vortex_session::registry::ReadContext; use crate::ArrayContext; use crate::Canonical; use crate::IntoArray; @@ -269,7 +269,7 @@ mod tests { let concat = concat.freeze(); let parts = ArrayParts::try_from(concat).unwrap(); - let decoded = parts.decode(&dtype, len, &ctx, &LEGACY_SESSION).unwrap(); + let decoded = parts.decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION).unwrap(); assert!(decoded.is::()); assert_eq!( diff --git a/vortex-array/src/context.rs b/vortex-array/src/context.rs deleted file mode 100644 index 962a3dfb1ae..00000000000 --- a/vortex-array/src/context.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_session::registry::Context; - -use crate::vtable::DynVTable; - -pub type ArrayContext = Context<&'static dyn DynVTable>; diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index d85569c046f..b9602d9a21b 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -18,14 +18,15 @@ use std::sync::LazyLock; pub use array::*; pub use canonical::*; pub use columnar::*; -pub use context::*; pub use executor::*; pub use hash::*; pub use mask_future::*; pub use metadata::*; +use vortex_session::registry::Context; use vortex_session::VortexSession; use crate::session::ArraySession; +use crate::vtable::DynVTable; pub mod accessor; #[doc(hidden)] @@ -39,7 +40,6 @@ pub mod builtins; mod canonical; mod columnar; pub mod compute; -mod context; pub mod display; pub mod dtype; mod executor; @@ -80,3 +80,5 @@ pub mod flatbuffers { // here... pub static LEGACY_SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); + +pub type ArrayContext = Context<&'static dyn DynVTable>; diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index e749e2696c3..e741065b493 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -23,6 +23,7 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::array as fba; use vortex_flatbuffers::array::Compression; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; use vortex_utils::aliases::hash_map::HashMap; @@ -318,7 +319,7 @@ impl ArrayParts { &self, dtype: &DType, len: usize, - ctx: &ArrayContext, + ctx: &ReadContext, session: &VortexSession, ) -> VortexResult { let encoding_idx = self.flatbuffer().encoding(); @@ -623,7 +624,7 @@ impl ArrayParts { struct ArrayPartsChildren<'a> { parts: &'a ArrayParts, - ctx: &'a ArrayContext, + ctx: &'a ReadContext, session: &'a VortexSession, } diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 1fa4e86e885..8bea6099974 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -63,6 +63,7 @@ use vortex::scalar::Scalar; use vortex::scalar::ScalarTruncation; use vortex::scalar::lower_bound; use vortex::scalar::upper_bound; +use vortex::session::registry::ReadContext; use vortex::session::VortexSession; use vortex::utils::aliases::hash_map::HashMap; @@ -94,7 +95,7 @@ pub struct CudaFlatLayout { row_count: u64, dtype: DType, segment_id: SegmentId, - ctx: ArrayContext, + ctx: ReadContext, array_tree: ByteBuffer, /// Small buffers kept on host, keyed by global buffer index. host_buffers: Arc>, @@ -107,7 +108,7 @@ impl CudaFlatLayout { } #[inline] - pub fn array_ctx(&self) -> &ArrayContext { + pub fn array_ctx(&self) -> &ReadContext { &self.ctx } @@ -195,7 +196,7 @@ impl VTable for CudaFlatVTable { metadata: &::Output, segment_ids: Vec, _children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult { if segment_ids.len() != 1 { vortex_bail!("CudaFlatLayout must have exactly one segment ID"); @@ -535,7 +536,7 @@ impl LayoutStrategy for CudaFlatLayoutStrategy { row_count, dtype: stream.dtype().clone(), segment_id, - ctx: ctx.clone(), + ctx: ReadContext::new(ctx.to_ids()), array_tree, host_buffers: Arc::new(host_buffer_map), } diff --git a/vortex-file/src/footer/file_layout.rs b/vortex-file/src/footer/file_layout.rs index 684955d7c80..e24fed66e1b 100644 --- a/vortex-file/src/footer/file_layout.rs +++ b/vortex-file/src/footer/file_layout.rs @@ -11,13 +11,11 @@ use std::sync::Arc; use flatbuffers::FlatBufferBuilder; use flatbuffers::WIPOffset; -use vortex_array::ArrayContext; use vortex_error::VortexResult; use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::footer as fb; -use vortex_layout::LayoutContext; - +use vortex_session::registry::ReadContext; use crate::footer::segment::SegmentSpec; /// A writer for serializing a file layout to a FlatBuffer. @@ -26,9 +24,9 @@ use crate::footer::segment::SegmentSpec; /// which describes the structure of the data in the file. pub(crate) struct FooterFlatBufferWriter { /// The array context containing encodings used in the file. - pub(crate) ctx: ArrayContext, + pub(crate) ctx: ReadContext, /// The layout context containing the layouts used in the file. - pub(crate) layout_ctx: LayoutContext, + pub(crate) layout_ctx: ReadContext, /// Specifications for all segments in the file. pub(crate) segment_specs: Arc<[SegmentSpec]>, } @@ -47,7 +45,7 @@ impl WriteFlatBuffer for FooterFlatBufferWriter { let array_specs = self .ctx - .to_ids() + .ids() .iter() .map(|e| { let id = fbb.create_string(e.as_ref()); @@ -58,7 +56,7 @@ impl WriteFlatBuffer for FooterFlatBufferWriter { let layout_specs = self .layout_ctx - .to_ids() + .ids() .iter() .map(|e| { let id = fbb.create_string(e.as_ref()); diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index 4142a60ded9..249e85edc6e 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -24,7 +24,6 @@ pub use file_statistics::FileStatistics; use flatbuffers::root; use itertools::Itertools; pub use segment::*; -use vortex_array::ArrayContext; use vortex_array::dtype::DType; use vortex_array::vtable::ArrayId; use vortex_buffer::ByteBuffer; @@ -38,6 +37,7 @@ use vortex_layout::LayoutEncodingId; use vortex_layout::LayoutRef; use vortex_layout::layout_from_flatbuffer; use vortex_layout::session::LayoutSessionExt; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; /// Captures the layout information of a Vortex file. @@ -47,7 +47,7 @@ pub struct Footer { segments: Arc<[SegmentSpec]>, statistics: Option, // The specific arrays used within the file, in the order they were registered. - array_ctx: ArrayContext, + read_ctx: ReadContext, // The approximate size of the footer in bytes, used for caching and memory management. approx_byte_size: Option, } @@ -57,13 +57,13 @@ impl Footer { root_layout: LayoutRef, segments: Arc<[SegmentSpec]>, statistics: Option, - array_ctx: ArrayContext, + read_ctx: ReadContext, ) -> Self { Self { root_layout, segments, statistics, - array_ctx, + read_ctx, approx_byte_size: None, } } @@ -95,18 +95,18 @@ impl Footer { // Create an ArrayContext from the registry. let array_specs = fb_footer.array_specs(); - let array_ids = array_specs + let array_ids: Arc<[_]> = array_specs .iter() .flat_map(|e| e.iter()) .map(|encoding| ArrayId::new_arc(Arc::from(encoding.id()))) .collect(); - let array_ctx = ArrayContext::new(array_ids); + let read_ctx = ReadContext::new(array_ids); let root_layout = layout_from_flatbuffer( layout_bytes, &dtype, &layout_ctx, - &array_ctx, + &read_ctx, session.layouts().registry(), )?; @@ -126,7 +126,7 @@ impl Footer { root_layout, segments, statistics, - array_ctx, + read_ctx, approx_byte_size: Some(approx_byte_size), }) } diff --git a/vortex-file/src/footer/serializer.rs b/vortex-file/src/footer/serializer.rs index cc1d773115f..5eb1d3087ce 100644 --- a/vortex-file/src/footer/serializer.rs +++ b/vortex-file/src/footer/serializer.rs @@ -10,7 +10,7 @@ use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::WriteFlatBufferExt; use vortex_layout::LayoutContext; - +use vortex_session::registry::ReadContext; use crate::EOF_SIZE; use crate::Footer; use crate::MAGIC_BYTES; @@ -94,8 +94,8 @@ impl FooterSerializer { let (buffer, footer_segment) = write_flatbuffer( &mut self.offset, &FooterFlatBufferWriter { - ctx: self.footer.array_ctx.clone(), - layout_ctx, + ctx: self.footer.read_ctx.clone(), + layout_ctx: ReadContext::new(layout_ctx.to_ids()), segment_specs: self.footer.segments.clone(), }, )?; diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 6bc6aef2dda..4f35df3ad23 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -43,6 +43,7 @@ use vortex_layout::layouts::file_stats::accumulate_stats; use vortex_layout::sequence::SequenceId; use vortex_layout::sequence::SequentialStreamAdapter; use vortex_layout::sequence::SequentialStreamExt; +use vortex_session::registry::ReadContext; use vortex_session::SessionExt; use vortex_session::VortexSession; @@ -211,7 +212,7 @@ impl VortexWriteOptions { &dtype, )) }, - ctx, + ReadContext::new(ctx.to_ids()), ); // Emit the footer buffers and EOF. diff --git a/vortex-ipc/src/messages/decoder.rs b/vortex-ipc/src/messages/decoder.rs index d2595d25235..ca863410855 100644 --- a/vortex-ipc/src/messages/decoder.rs +++ b/vortex-ipc/src/messages/decoder.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use bytes::Buf; use flatbuffers::root; use flatbuffers::root_unchecked; -use vortex_array::ArrayContext; use vortex_array::serde::ArrayParts; use vortex_array::vtable::ArrayId; use vortex_buffer::AlignedBuf; @@ -21,11 +20,12 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::message as fb; use vortex_flatbuffers::message::MessageHeader; use vortex_flatbuffers::message::MessageVersion; +use vortex_session::registry::ReadContext; /// A message decoded from an IPC stream. #[derive(Debug)] pub enum DecoderMessage { - Array((ArrayParts, ArrayContext, usize)), + Array((ArrayParts, ReadContext, usize)), Buffer(ByteBuffer), DType(FlatBuffer), } @@ -121,14 +121,14 @@ impl MessageDecoder { .header_as_array_message() .vortex_expect("header is array"); - let encoding_ids: Vec<_> = header + let encoding_ids: Arc<_> = header .encodings() .iter() .flat_map(|e| e.iter()) .map(|id| ArrayId::new_arc(Arc::from(id.to_string()))) .collect(); - let ctx = ArrayContext::new(encoding_ids); + let ctx = ReadContext::new(encoding_ids); let row_count = header.row_count() as usize; self.state = Default::default(); diff --git a/vortex-layout/src/children.rs b/vortex-layout/src/children.rs index e87f0d05031..f5c1cdc8f45 100644 --- a/vortex-layout/src/children.rs +++ b/vortex-layout/src/children.rs @@ -7,14 +7,13 @@ use std::sync::Arc; use flatbuffers::Follow; use itertools::Itertools; -use vortex_array::ArrayContext; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::layout as fbl; - +use vortex_session::registry::ReadContext; use crate::LayoutContext; use crate::LayoutRef; use crate::segments::SegmentId; @@ -101,7 +100,7 @@ impl LayoutChildren for OwnedLayoutChildren { pub(crate) struct ViewedLayoutChildren { flatbuffer: FlatBuffer, flatbuffer_loc: usize, - array_ctx: ArrayContext, + read_ctx: ReadContext, layout_ctx: LayoutContext, layouts: LayoutRegistry, } @@ -115,14 +114,14 @@ impl ViewedLayoutChildren { pub(super) unsafe fn new_unchecked( flatbuffer: FlatBuffer, flatbuffer_loc: usize, - array_ctx: ArrayContext, + read_ctx: ReadContext, layout_ctx: LayoutContext, layouts: LayoutRegistry, ) -> Self { Self { flatbuffer, flatbuffer_loc, - array_ctx, + read_ctx, layout_ctx, layouts, } @@ -151,7 +150,7 @@ impl LayoutChildren for ViewedLayoutChildren { let viewed_children = ViewedLayoutChildren { flatbuffer: self.flatbuffer.clone(), flatbuffer_loc: fb_child._tab.loc(), - array_ctx: self.array_ctx.clone(), + read_ctx: self.read_ctx.clone(), layout_ctx: self.layout_ctx.clone(), layouts: self.layouts.clone(), }; @@ -178,7 +177,7 @@ impl LayoutChildren for ViewedLayoutChildren { .map(SegmentId::from) .collect_vec(), &viewed_children, - &self.array_ctx, + &self.read_ctx, ) } diff --git a/vortex-layout/src/encoding.rs b/vortex-layout/src/encoding.rs index d6b0892327e..f928be12e85 100644 --- a/vortex-layout/src/encoding.rs +++ b/vortex-layout/src/encoding.rs @@ -7,13 +7,12 @@ use std::fmt::Display; use std::fmt::Formatter; use arcref::ArcRef; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; - +use vortex_session::registry::ReadContext; use crate::IntoLayout; use crate::LayoutChildren; use crate::LayoutRef; @@ -35,7 +34,7 @@ pub trait LayoutEncoding: 'static + Send + Sync + Debug + private::Sealed { metadata: &[u8], segment_ids: Vec, children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult; } @@ -58,7 +57,7 @@ impl LayoutEncoding for LayoutEncodingAdapter { metadata: &[u8], segment_ids: Vec, children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult { let metadata = ::deserialize(metadata)?; let layout = V::build( diff --git a/vortex-layout/src/flatbuffers.rs b/vortex-layout/src/flatbuffers.rs index 88010bcd083..7de81bd3dd8 100644 --- a/vortex-layout/src/flatbuffers.rs +++ b/vortex-layout/src/flatbuffers.rs @@ -8,7 +8,6 @@ use flatbuffers::FlatBufferBuilder; use flatbuffers::VerifierOptions; use flatbuffers::WIPOffset; use flatbuffers::root_with_opts; -use vortex_array::ArrayContext; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -16,7 +15,7 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::layout; - +use vortex_session::registry::ReadContext; use crate::Layout; use crate::LayoutContext; use crate::LayoutRef; @@ -46,7 +45,7 @@ pub fn layout_from_flatbuffer( flatbuffer: FlatBuffer, dtype: &DType, layout_ctx: &LayoutContext, - ctx: &ArrayContext, + ctx: &ReadContext, layouts: &LayoutRegistry, ) -> VortexResult { let fb_layout = root_with_opts::(&LAYOUT_VERIFIER, &flatbuffer)?; diff --git a/vortex-layout/src/layout.rs b/vortex-layout/src/layout.rs index 91df3db8061..8c860726c7d 100644 --- a/vortex-layout/src/layout.rs +++ b/vortex-layout/src/layout.rs @@ -326,8 +326,7 @@ mod private { #[cfg(test)] mod tests { use rstest::rstest; - use vortex_array::ArrayContext; - + use vortex_session::registry::ReadContext; use super::*; #[test] @@ -482,7 +481,7 @@ mod tests { use crate::layouts::struct_::StructLayout; use crate::segments::SegmentId; - let ctx = ArrayContext::empty(); + let ctx = ReadContext::new([]); // Create a flat layout for dict values (utf8 strings) let dict_values = diff --git a/vortex-layout/src/layouts/chunked/mod.rs b/vortex-layout/src/layouts/chunked/mod.rs index e1395fa88f2..8ca30671c23 100644 --- a/vortex-layout/src/layouts/chunked/mod.rs +++ b/vortex-layout/src/layouts/chunked/mod.rs @@ -6,11 +6,11 @@ pub mod writer; use std::sync::Arc; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::EmptyMetadata; use vortex_array::dtype::DType; use vortex_error::VortexResult; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; use crate::LayoutChildType; @@ -90,7 +90,7 @@ impl VTable for ChunkedVTable { _metadata: &::Output, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { Ok(ChunkedLayout::new( row_count, diff --git a/vortex-layout/src/layouts/dict/mod.rs b/vortex-layout/src/layouts/dict/mod.rs index a8e63931fa9..192f9145d69 100644 --- a/vortex-layout/src/layouts/dict/mod.rs +++ b/vortex-layout/src/layouts/dict/mod.rs @@ -7,7 +7,6 @@ pub mod writer; use std::sync::Arc; use reader::DictReader; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::ProstMetadata; use vortex_array::dtype::DType; @@ -19,6 +18,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; use crate::LayoutChildType; @@ -108,7 +108,7 @@ impl VTable for DictVTable { metadata: &::Output, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { let values = children.child(0, dtype)?; let codes_nullable = metadata diff --git a/vortex-layout/src/layouts/flat/mod.rs b/vortex-layout/src/layouts/flat/mod.rs index b4199648186..161a5705a24 100644 --- a/vortex-layout/src/layouts/flat/mod.rs +++ b/vortex-layout/src/layouts/flat/mod.rs @@ -7,7 +7,6 @@ pub mod writer; use std::env; use std::sync::Arc; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::ProstMetadata; use vortex_array::dtype::DType; @@ -15,6 +14,7 @@ use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; use crate::LayoutChildType; @@ -101,7 +101,7 @@ impl VTable for FlatVTable { metadata: &::Output, segment_ids: Vec, _children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult { if segment_ids.len() != 1 { vortex_bail!("Flat layout must have exactly one segment ID"); @@ -136,12 +136,12 @@ pub struct FlatLayout { row_count: u64, dtype: DType, segment_id: SegmentId, - ctx: ArrayContext, + ctx: ReadContext, array_tree: Option, } impl FlatLayout { - pub fn new(row_count: u64, dtype: DType, segment_id: SegmentId, ctx: ArrayContext) -> Self { + pub fn new(row_count: u64, dtype: DType, segment_id: SegmentId, ctx: ReadContext) -> Self { Self { row_count, dtype, @@ -155,7 +155,7 @@ impl FlatLayout { row_count: u64, dtype: DType, segment_id: SegmentId, - ctx: ArrayContext, + ctx: ReadContext, metadata: Option, ) -> Self { Self { @@ -173,7 +173,7 @@ impl FlatLayout { } #[inline] - pub fn array_ctx(&self) -> &ArrayContext { + pub fn array_ctx(&self) -> &ReadContext { &self.ctx } diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index f58b4729346..2cbc518c134 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -24,7 +24,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_io::runtime::Handle; - +use vortex_session::registry::ReadContext; use crate::IntoLayout; use crate::LayoutRef; use crate::LayoutStrategy; @@ -178,7 +178,7 @@ impl LayoutStrategy for FlatLayoutStrategy { row_count, stream.dtype().clone(), segment_id, - ctx.clone(), + ReadContext::new(ctx.to_ids()), array_node, ) .into_layout()) diff --git a/vortex-layout/src/layouts/struct_/mod.rs b/vortex-layout/src/layouts/struct_/mod.rs index ec41978ac50..9dba2cdf0e9 100644 --- a/vortex-layout/src/layouts/struct_/mod.rs +++ b/vortex-layout/src/layouts/struct_/mod.rs @@ -7,7 +7,6 @@ pub mod writer; use std::sync::Arc; use reader::StructReader; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::EmptyMetadata; use vortex_array::dtype::DType; @@ -20,6 +19,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_session::registry::ReadContext; use vortex_session::SessionExt; use vortex_session::VortexSession; @@ -131,7 +131,7 @@ impl VTable for StructVTable { _metadata: &::Output, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { let struct_dt = dtype .as_struct_fields_opt() diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index 68c93a8c8a1..3ef6e38a1e7 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -10,7 +10,6 @@ use std::sync::Arc; pub use builder::MAX_IS_TRUNCATED; pub use builder::MIN_IS_TRUNCATED; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; @@ -22,6 +21,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; use crate::LayoutChildType; @@ -116,7 +116,7 @@ impl VTable for ZonedVTable { metadata: &ZonedMetadata, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { Ok(ZonedLayout { dtype: dtype.clone(), diff --git a/vortex-layout/src/vtable.rs b/vortex-layout/src/vtable.rs index 804d97dfc0d..31f988e911d 100644 --- a/vortex-layout/src/vtable.rs +++ b/vortex-layout/src/vtable.rs @@ -5,12 +5,12 @@ use std::fmt::Debug; use std::ops::Deref; use std::sync::Arc; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_session::registry::ReadContext; use vortex_session::VortexSession; use crate::IntoLayout; @@ -73,7 +73,7 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { metadata: &::Output, segment_ids: Vec, children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult; /// Replaces the children of the layout with the given layout references. diff --git a/vortex-python/src/serde/context.rs b/vortex-python/src/serde/context.rs index a31c22b83c4..48d1552882b 100644 --- a/vortex-python/src/serde/context.rs +++ b/vortex-python/src/serde/context.rs @@ -2,11 +2,12 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::ops::Deref; - +use std::sync::Arc; use itertools::Itertools; use pyo3::pyclass; use pyo3::pymethods; use vortex::array::ArrayContext; +use vortex::session::registry::{Id, ReadContext}; /// An ArrayContext captures an ordered set of encodings. /// @@ -44,3 +45,44 @@ impl PyArrayContext { self.to_ids().len() } } + +/// A ReadContext captures an ordered set of encodings. +/// +/// In a serialized array, encodings are identified by a positional index into such an +/// :class:`~vortex.ReadContext`. +#[pyclass(name = "ReadContext", module = "vortex", frozen)] +pub(crate) struct PyReadContext(ReadContext); + +impl From for PyReadContext { + fn from(context: ReadContext) -> Self { + Self(context) + } +} + +impl Deref for PyReadContext { + type Target = ReadContext; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[pymethods] +impl PyReadContext { + #[new] + fn new(ids: Vec) -> Self { + Self(ReadContext::new( + ids.into_iter() + .map(|i| Id::new_arc(Arc::from(i))) + .collect::>(), + )) + } + + fn __str__(&self) -> String { + self.0.ids().iter().join(", ") + } + + fn __len__(&self) -> usize { + self.ids().len() + } +} diff --git a/vortex-python/src/serde/mod.rs b/vortex-python/src/serde/mod.rs index 0c6587966c8..9f6786fbd3c 100644 --- a/vortex-python/src/serde/mod.rs +++ b/vortex-python/src/serde/mod.rs @@ -19,7 +19,7 @@ use crate::SESSION; use crate::arrays::PyArrayRef; use crate::error::PyVortexResult; use crate::install_module; -use crate::serde::context::PyArrayContext; +use crate::serde::context::{PyArrayContext, PyReadContext}; use crate::serde::parts::PyArrayParts; /// Register serde functions and classes. @@ -30,6 +30,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(decode_ipc_array, &m)?)?; m.add_function(wrap_pyfunction!(decode_ipc_array_buffers, &m)?)?; diff --git a/vortex-python/src/serde/parts.rs b/vortex-python/src/serde/parts.rs index 68c3c51344f..1f558a4d0a6 100644 --- a/vortex-python/src/serde/parts.rs +++ b/vortex-python/src/serde/parts.rs @@ -17,7 +17,7 @@ use crate::SESSION; use crate::arrays::PyArrayRef; use crate::dtype::PyDType; use crate::error::PyVortexResult; -use crate::serde::context::PyArrayContext; +use crate::serde::context::PyReadContext; /// ArrayParts is a parsed representation of a serialized array. /// @@ -56,7 +56,7 @@ impl PyArrayParts { /// The decoded array. fn decode( &self, - ctx: &PyArrayContext, + ctx: &PyReadContext, dtype: PyDType, len: usize, ) -> PyVortexResult { diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 392579b0056..9c9a866da46 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -66,6 +66,29 @@ impl Registry { } } +/// A [`ReadContext`] holds a set of interned IDs for use during deserialization, mapping +/// u16 indices to IDs. +#[derive(Clone, Debug)] +pub struct ReadContext { + ids: Arc<[Id]>, +} + +impl ReadContext { + /// Create a context with the given initial IDs. + pub fn new(ids: impl Into>) -> Self { + Self { ids: ids.into() } + } + + /// Resolve an interned ID by its index. + pub fn resolve(&self, idx: u16) -> Option { + self.ids.get(idx as usize).cloned() + } + + pub fn ids(&self) -> &[Id] { + &self.ids + } +} + /// A [`Context`] holds a set of interned IDs for use during serialization/deserialization, mapping /// IDs to u16 indices. /// From c0136969f55dc76807b8b21cbb06f5a82cf04c19 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 6 Mar 2026 11:26:52 +0000 Subject: [PATCH 2/5] format Signed-off-by: Robert Kruszewski --- encodings/pco/src/test.rs | 2 +- vortex-array/public-api.lock | 2 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 5 +- vortex-array/src/arrays/masked/vtable/mod.rs | 10 +- vortex-array/src/lib.rs | 2 +- vortex-array/src/serde.rs | 2 +- vortex-cuda/src/layout.rs | 2 +- vortex-file/src/footer/file_layout.rs | 1 + vortex-file/src/footer/mod.rs | 2 +- vortex-file/src/footer/serializer.rs | 1 + vortex-file/src/writer.rs | 2 +- vortex-ipc/public-api.lock | 2 +- vortex-layout/public-api.lock | 92 +++++++++---------- vortex-layout/src/children.rs | 1 + vortex-layout/src/encoding.rs | 1 + vortex-layout/src/flatbuffers.rs | 1 + vortex-layout/src/layout.rs | 1 + vortex-layout/src/layouts/chunked/mod.rs | 2 +- vortex-layout/src/layouts/dict/mod.rs | 2 +- vortex-layout/src/layouts/flat/mod.rs | 2 +- vortex-layout/src/layouts/flat/writer.rs | 1 + vortex-layout/src/layouts/struct_/mod.rs | 2 +- vortex-layout/src/layouts/zoned/mod.rs | 2 +- vortex-layout/src/vtable.rs | 2 +- vortex-python/src/serde/context.rs | 4 +- vortex-python/src/serde/mod.rs | 3 +- vortex-session/public-api.lock | 18 ++++ vortex-session/src/registry.rs | 2 +- 28 files changed, 104 insertions(+), 65 deletions(-) diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index a7d038e59d4..85d7dd64220 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -26,8 +26,8 @@ use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_mask::Mask; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; static SESSION: LazyLock = LazyLock::new(|| { let session = VortexSession::empty().with::(); diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 94d962a4868..1947a1d84a2 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -15982,7 +15982,7 @@ pub fn vortex_array::serde::ArrayParts::buffer_lengths(&self) -> alloc::vec::Vec pub fn vortex_array::serde::ArrayParts::child(&self, idx: usize) -> vortex_array::serde::ArrayParts -pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_array::ArrayContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::serde::ArrayParts::encoding_id(&self) -> u16 diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 2bab9f813af..2f49a8c545b 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -240,6 +240,7 @@ mod tests { use vortex_buffer::ByteBufferMut; use vortex_buffer::buffer; use vortex_session::registry::ReadContext; + use crate::ArrayContext; use crate::IntoArray; use crate::LEGACY_SESSION; @@ -273,7 +274,9 @@ mod tests { let concat = concat.freeze(); let parts = ArrayParts::try_from(concat).unwrap(); - let decoded = parts.decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION).unwrap(); + let decoded = parts + .decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION) + .unwrap(); assert!(decoded.is::()); } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index f799c5dbdc7..4d10c12f843 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -218,6 +218,7 @@ mod tests { use vortex_buffer::ByteBufferMut; use vortex_error::VortexError; use vortex_session::registry::ReadContext; + use crate::ArrayContext; use crate::Canonical; use crate::IntoArray; @@ -269,7 +270,14 @@ mod tests { let concat = concat.freeze(); let parts = ArrayParts::try_from(concat).unwrap(); - let decoded = parts.decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION).unwrap(); + let decoded = parts + .decode( + &dtype, + len, + &ReadContext::new(ctx.to_ids()), + &LEGACY_SESSION, + ) + .unwrap(); assert!(decoded.is::()); assert_eq!( diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index b9602d9a21b..9e56c9c4560 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -22,8 +22,8 @@ pub use executor::*; pub use hash::*; pub use mask_future::*; pub use metadata::*; -use vortex_session::registry::Context; use vortex_session::VortexSession; +use vortex_session::registry::Context; use crate::session::ArraySession; use crate::vtable::DynVTable; diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index e741065b493..c4f885f6c2a 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -23,8 +23,8 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::array as fba; use vortex_flatbuffers::array::Compression; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayContext; diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 8bea6099974..0bf4711fdae 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -63,8 +63,8 @@ use vortex::scalar::Scalar; use vortex::scalar::ScalarTruncation; use vortex::scalar::lower_bound; use vortex::scalar::upper_bound; -use vortex::session::registry::ReadContext; use vortex::session::VortexSession; +use vortex::session::registry::ReadContext; use vortex::utils::aliases::hash_map::HashMap; /// A buffer inlined into layout metadata for host-side access. diff --git a/vortex-file/src/footer/file_layout.rs b/vortex-file/src/footer/file_layout.rs index e24fed66e1b..22e5a3e593d 100644 --- a/vortex-file/src/footer/file_layout.rs +++ b/vortex-file/src/footer/file_layout.rs @@ -16,6 +16,7 @@ use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::footer as fb; use vortex_session::registry::ReadContext; + use crate::footer::segment::SegmentSpec; /// A writer for serializing a file layout to a FlatBuffer. diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index 249e85edc6e..9b1a092145f 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -37,8 +37,8 @@ use vortex_layout::LayoutEncodingId; use vortex_layout::LayoutRef; use vortex_layout::layout_from_flatbuffer; use vortex_layout::session::LayoutSessionExt; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; /// Captures the layout information of a Vortex file. #[derive(Debug, Clone)] diff --git a/vortex-file/src/footer/serializer.rs b/vortex-file/src/footer/serializer.rs index 5eb1d3087ce..ec96cc95cd6 100644 --- a/vortex-file/src/footer/serializer.rs +++ b/vortex-file/src/footer/serializer.rs @@ -11,6 +11,7 @@ use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::WriteFlatBufferExt; use vortex_layout::LayoutContext; use vortex_session::registry::ReadContext; + use crate::EOF_SIZE; use crate::Footer; use crate::MAGIC_BYTES; diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 4f35df3ad23..ef91dc701c0 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -43,9 +43,9 @@ use vortex_layout::layouts::file_stats::accumulate_stats; use vortex_layout::sequence::SequenceId; use vortex_layout::sequence::SequentialStreamAdapter; use vortex_layout::sequence::SequentialStreamExt; -use vortex_session::registry::ReadContext; use vortex_session::SessionExt; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::Footer; use crate::MAGIC_BYTES; diff --git a/vortex-ipc/public-api.lock b/vortex-ipc/public-api.lock index 94762154cae..09d90971d2f 100644 --- a/vortex-ipc/public-api.lock +++ b/vortex-ipc/public-api.lock @@ -46,7 +46,7 @@ pub mod vortex_ipc::messages pub enum vortex_ipc::messages::DecoderMessage -pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_array::context::ArrayContext, usize)) +pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_session::registry::ReadContext, usize)) pub vortex_ipc::messages::DecoderMessage::Buffer(vortex_buffer::ByteBuffer) diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index bf69112d7dd..9da043fa465 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -36,7 +36,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::buffered::Buffere pub fn vortex_layout::layouts::buffered::BufferedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::chunked @@ -58,7 +58,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::chunked::writer:: pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::chunked::ChunkedLayout @@ -124,7 +124,7 @@ pub type vortex_layout::layouts::chunked::ChunkedVTable::Layout = vortex_layout: pub type vortex_layout::layouts::chunked::ChunkedVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::chunked::ChunkedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -160,7 +160,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::collect::CollectS pub fn vortex_layout::layouts::collect::CollectStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::compressed @@ -182,7 +182,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::compressed::Compr pub fn vortex_layout::layouts::compressed::CompressingStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait vortex_layout::layouts::compressed::CompressorPlugin: core::marker::Send + core::marker::Sync + 'static @@ -274,7 +274,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::dict::writer::Dic pub fn vortex_layout::layouts::dict::writer::DictStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn vortex_layout::layouts::dict::writer::dict_layout_supported(dtype: &vortex_array::dtype::DType) -> bool @@ -372,7 +372,7 @@ pub type vortex_layout::layouts::dict::DictVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::dict::DictVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::dict::DictVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -442,19 +442,19 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::flat::writer::Fla pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::flat::FlatLayout impl vortex_layout::layouts::flat::FlatLayout -pub fn vortex_layout::layouts::flat::FlatLayout::array_ctx(&self) -> &vortex_array::context::ArrayContext +pub fn vortex_layout::layouts::flat::FlatLayout::array_ctx(&self) -> &vortex_session::registry::ReadContext pub fn vortex_layout::layouts::flat::FlatLayout::array_tree(&self) -> core::option::Option<&vortex_buffer::ByteBuffer> -pub fn vortex_layout::layouts::flat::FlatLayout::new(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_array::context::ArrayContext) -> Self +pub fn vortex_layout::layouts::flat::FlatLayout::new(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_session::registry::ReadContext) -> Self -pub fn vortex_layout::layouts::flat::FlatLayout::new_with_metadata(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_array::context::ArrayContext, metadata: core::option::Option) -> Self +pub fn vortex_layout::layouts::flat::FlatLayout::new_with_metadata(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_session::registry::ReadContext, metadata: core::option::Option) -> Self pub fn vortex_layout::layouts::flat::FlatLayout::segment_id(&self) -> vortex_layout::segments::SegmentId @@ -536,7 +536,7 @@ pub type vortex_layout::layouts::flat::FlatVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::flat::FlatVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::flat::FlatVTable::child(_layout: &Self::Layout, _idx: usize) -> vortex_error::VortexResult @@ -576,7 +576,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::repartition::Repa pub fn vortex_layout::layouts::repartition::RepartitionStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::repartition::RepartitionWriterOptions @@ -658,7 +658,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::struct_::writer:: pub fn vortex_layout::layouts::struct_::writer::StructStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::struct_::StructLayout @@ -730,7 +730,7 @@ pub type vortex_layout::layouts::struct_::StructVTable::Layout = vortex_layout:: pub type vortex_layout::layouts::struct_::StructVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::struct_::StructVTable::child(layout: &Self::Layout, index: usize) -> vortex_error::VortexResult @@ -774,7 +774,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::table::TableStrat pub fn vortex_layout::layouts::table::TableStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::zoned @@ -804,7 +804,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::zoned::writer::Zo pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::zoned::zone_map @@ -938,7 +938,7 @@ pub type vortex_layout::layouts::zoned::ZonedVTable::Layout = vortex_layout::lay pub type vortex_layout::layouts::zoned::ZonedVTable::Metadata = vortex_layout::layouts::zoned::ZonedMetadata -pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::zoned::ZonedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1266,7 +1266,7 @@ pub type vortex_layout::vtable::VTable::Layout: 'static + core::marker::Send + c pub type vortex_layout::vtable::VTable::Metadata: vortex_array::metadata::SerializeMetadata + vortex_array::metadata::DeserializeMetadata + core::fmt::Debug -pub fn vortex_layout::vtable::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::vtable::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::vtable::VTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1298,7 +1298,7 @@ pub type vortex_layout::layouts::chunked::ChunkedVTable::Layout = vortex_layout: pub type vortex_layout::layouts::chunked::ChunkedVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::chunked::ChunkedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1330,7 +1330,7 @@ pub type vortex_layout::layouts::dict::DictVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::dict::DictVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::dict::DictVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1362,7 +1362,7 @@ pub type vortex_layout::layouts::flat::FlatVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::flat::FlatVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::flat::FlatVTable::child(_layout: &Self::Layout, _idx: usize) -> vortex_error::VortexResult @@ -1394,7 +1394,7 @@ pub type vortex_layout::layouts::struct_::StructVTable::Layout = vortex_layout:: pub type vortex_layout::layouts::struct_::StructVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::struct_::StructVTable::child(layout: &Self::Layout, index: usize) -> vortex_error::VortexResult @@ -1426,7 +1426,7 @@ pub type vortex_layout::layouts::zoned::ZonedVTable::Layout = vortex_layout::lay pub type vortex_layout::layouts::zoned::ZonedVTable::Metadata = vortex_layout::layouts::zoned::ZonedMetadata -pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::zoned::ZonedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1526,7 +1526,7 @@ impl vortex_layout::LayoutEncoding for vortex_layout:: pub fn vortex_layout::LayoutEncodingAdapter::as_any(&self) -> &dyn core::any::Any -pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::LayoutEncodingAdapter::id(&self) -> vortex_layout::LayoutEncodingId @@ -1646,7 +1646,7 @@ pub trait vortex_layout::LayoutEncoding: 'static + core::marker::Send + core::ma pub fn vortex_layout::LayoutEncoding::as_any(&self) -> &dyn core::any::Any -pub fn vortex_layout::LayoutEncoding::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::LayoutEncoding::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::LayoutEncoding::id(&self) -> vortex_layout::LayoutEncodingId @@ -1654,7 +1654,7 @@ impl vortex_layout::LayoutEncoding for vortex_layout:: pub fn vortex_layout::LayoutEncodingAdapter::as_any(&self) -> &dyn core::any::Any -pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::LayoutEncodingAdapter::id(&self) -> vortex_layout::LayoutEncodingId @@ -1694,73 +1694,73 @@ pub trait vortex_layout::LayoutStrategy: 'static + core::marker::Send + core::ma pub fn vortex_layout::LayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::LayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::LayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for alloc::sync::Arc pub fn alloc::sync::Arc::buffered_bytes(&self) -> u64 -pub fn alloc::sync::Arc::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn alloc::sync::Arc::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::buffered::BufferedStrategy pub fn vortex_layout::layouts::buffered::BufferedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::collect::CollectStrategy pub fn vortex_layout::layouts::collect::CollectStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::compressed::CompressingStrategy pub fn vortex_layout::layouts::compressed::CompressingStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::dict::writer::DictStrategy pub fn vortex_layout::layouts::dict::writer::DictStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::flat::writer::FlatLayoutStrategy pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::repartition::RepartitionStrategy pub fn vortex_layout::layouts::repartition::RepartitionStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::struct_::writer::StructStrategy pub fn vortex_layout::layouts::struct_::writer::StructStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::table::TableStrategy pub fn vortex_layout::layouts::table::TableStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::zoned::writer::ZonedStrategy pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait vortex_layout::VTable: 'static + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug @@ -1770,7 +1770,7 @@ pub type vortex_layout::VTable::Layout: 'static + core::marker::Send + core::mar pub type vortex_layout::VTable::Metadata: vortex_array::metadata::SerializeMetadata + vortex_array::metadata::DeserializeMetadata + core::fmt::Debug -pub fn vortex_layout::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::VTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1802,7 +1802,7 @@ pub type vortex_layout::layouts::chunked::ChunkedVTable::Layout = vortex_layout: pub type vortex_layout::layouts::chunked::ChunkedVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::chunked::ChunkedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1834,7 +1834,7 @@ pub type vortex_layout::layouts::dict::DictVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::dict::DictVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::dict::DictVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1866,7 +1866,7 @@ pub type vortex_layout::layouts::flat::FlatVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::flat::FlatVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::flat::FlatVTable::child(_layout: &Self::Layout, _idx: usize) -> vortex_error::VortexResult @@ -1898,7 +1898,7 @@ pub type vortex_layout::layouts::struct_::StructVTable::Layout = vortex_layout:: pub type vortex_layout::layouts::struct_::StructVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::struct_::StructVTable::child(layout: &Self::Layout, index: usize) -> vortex_error::VortexResult @@ -1930,7 +1930,7 @@ pub type vortex_layout::layouts::zoned::ZonedVTable::Layout = vortex_layout::lay pub type vortex_layout::layouts::zoned::ZonedVTable::Metadata = vortex_layout::layouts::zoned::ZonedMetadata -pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::zoned::ZonedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1954,7 +1954,7 @@ pub fn vortex_layout::layouts::zoned::ZonedVTable::segment_ids(_layout: &Self::L pub fn vortex_layout::layouts::zoned::ZonedVTable::with_children(layout: &mut Self::Layout, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> -pub fn vortex_layout::layout_from_flatbuffer(flatbuffer: vortex_flatbuffers::FlatBuffer, dtype: &vortex_array::dtype::DType, layout_ctx: &vortex_layout::LayoutContext, ctx: &vortex_array::context::ArrayContext, layouts: &vortex_layout::session::LayoutRegistry) -> vortex_error::VortexResult +pub fn vortex_layout::layout_from_flatbuffer(flatbuffer: vortex_flatbuffers::FlatBuffer, dtype: &vortex_array::dtype::DType, layout_ctx: &vortex_layout::LayoutContext, ctx: &vortex_session::registry::ReadContext, layouts: &vortex_layout::session::LayoutRegistry) -> vortex_error::VortexResult pub type vortex_layout::ArrayFuture = futures_core::future::BoxFuture<'static, vortex_error::VortexResult> diff --git a/vortex-layout/src/children.rs b/vortex-layout/src/children.rs index f5c1cdc8f45..838fc5e7c8e 100644 --- a/vortex-layout/src/children.rs +++ b/vortex-layout/src/children.rs @@ -14,6 +14,7 @@ use vortex_error::vortex_err; use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::layout as fbl; use vortex_session::registry::ReadContext; + use crate::LayoutContext; use crate::LayoutRef; use crate::segments::SegmentId; diff --git a/vortex-layout/src/encoding.rs b/vortex-layout/src/encoding.rs index f928be12e85..ce0086e2ffc 100644 --- a/vortex-layout/src/encoding.rs +++ b/vortex-layout/src/encoding.rs @@ -13,6 +13,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; use vortex_session::registry::ReadContext; + use crate::IntoLayout; use crate::LayoutChildren; use crate::LayoutRef; diff --git a/vortex-layout/src/flatbuffers.rs b/vortex-layout/src/flatbuffers.rs index 7de81bd3dd8..d55fab9101a 100644 --- a/vortex-layout/src/flatbuffers.rs +++ b/vortex-layout/src/flatbuffers.rs @@ -16,6 +16,7 @@ use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::layout; use vortex_session::registry::ReadContext; + use crate::Layout; use crate::LayoutContext; use crate::LayoutRef; diff --git a/vortex-layout/src/layout.rs b/vortex-layout/src/layout.rs index 8c860726c7d..a4f9e6e0830 100644 --- a/vortex-layout/src/layout.rs +++ b/vortex-layout/src/layout.rs @@ -327,6 +327,7 @@ mod private { mod tests { use rstest::rstest; use vortex_session::registry::ReadContext; + use super::*; #[test] diff --git a/vortex-layout/src/layouts/chunked/mod.rs b/vortex-layout/src/layouts/chunked/mod.rs index 8ca30671c23..21752432ca4 100644 --- a/vortex-layout/src/layouts/chunked/mod.rs +++ b/vortex-layout/src/layouts/chunked/mod.rs @@ -10,8 +10,8 @@ use vortex_array::DeserializeMetadata; use vortex_array::EmptyMetadata; use vortex_array::dtype::DType; use vortex_error::VortexResult; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; diff --git a/vortex-layout/src/layouts/dict/mod.rs b/vortex-layout/src/layouts/dict/mod.rs index 192f9145d69..05c379f7ceb 100644 --- a/vortex-layout/src/layouts/dict/mod.rs +++ b/vortex-layout/src/layouts/dict/mod.rs @@ -18,8 +18,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; diff --git a/vortex-layout/src/layouts/flat/mod.rs b/vortex-layout/src/layouts/flat/mod.rs index 161a5705a24..8e0a880fd8a 100644 --- a/vortex-layout/src/layouts/flat/mod.rs +++ b/vortex-layout/src/layouts/flat/mod.rs @@ -14,8 +14,8 @@ use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 2cbc518c134..2b38446bccc 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -25,6 +25,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_io::runtime::Handle; use vortex_session::registry::ReadContext; + use crate::IntoLayout; use crate::LayoutRef; use crate::LayoutStrategy; diff --git a/vortex-layout/src/layouts/struct_/mod.rs b/vortex-layout/src/layouts/struct_/mod.rs index 9dba2cdf0e9..96929d519a5 100644 --- a/vortex-layout/src/layouts/struct_/mod.rs +++ b/vortex-layout/src/layouts/struct_/mod.rs @@ -19,9 +19,9 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use vortex_session::registry::ReadContext; use vortex_session::SessionExt; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index 3ef6e38a1e7..717f1ea0b9b 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -21,8 +21,8 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; diff --git a/vortex-layout/src/vtable.rs b/vortex-layout/src/vtable.rs index 31f988e911d..f8349758ba0 100644 --- a/vortex-layout/src/vtable.rs +++ b/vortex-layout/src/vtable.rs @@ -10,8 +10,8 @@ use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_session::registry::ReadContext; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::IntoLayout; use crate::Layout; diff --git a/vortex-python/src/serde/context.rs b/vortex-python/src/serde/context.rs index 48d1552882b..6a4f6d5d3a2 100644 --- a/vortex-python/src/serde/context.rs +++ b/vortex-python/src/serde/context.rs @@ -3,11 +3,13 @@ use std::ops::Deref; use std::sync::Arc; + use itertools::Itertools; use pyo3::pyclass; use pyo3::pymethods; use vortex::array::ArrayContext; -use vortex::session::registry::{Id, ReadContext}; +use vortex::session::registry::Id; +use vortex::session::registry::ReadContext; /// An ArrayContext captures an ordered set of encodings. /// diff --git a/vortex-python/src/serde/mod.rs b/vortex-python/src/serde/mod.rs index 9f6786fbd3c..e3290821194 100644 --- a/vortex-python/src/serde/mod.rs +++ b/vortex-python/src/serde/mod.rs @@ -19,7 +19,8 @@ use crate::SESSION; use crate::arrays::PyArrayRef; use crate::error::PyVortexResult; use crate::install_module; -use crate::serde::context::{PyArrayContext, PyReadContext}; +use crate::serde::context::PyArrayContext; +use crate::serde::context::PyReadContext; use crate::serde::parts::PyArrayParts; /// Register serde functions and classes. diff --git a/vortex-session/public-api.lock b/vortex-session/public-api.lock index 4e00c75a886..4163cd885b5 100644 --- a/vortex-session/public-api.lock +++ b/vortex-session/public-api.lock @@ -30,6 +30,24 @@ impl core::default::Default for vortex_session::registry::Context pub fn vortex_session::registry::Context::default() -> Self +pub struct vortex_session::registry::ReadContext + +impl vortex_session::registry::ReadContext + +pub fn vortex_session::registry::ReadContext::ids(&self) -> &[vortex_session::registry::Id] + +pub fn vortex_session::registry::ReadContext::new(ids: impl core::convert::Into>) -> Self + +pub fn vortex_session::registry::ReadContext::resolve(&self, idx: u16) -> core::option::Option + +impl core::clone::Clone for vortex_session::registry::ReadContext + +pub fn vortex_session::registry::ReadContext::clone(&self) -> vortex_session::registry::ReadContext + +impl core::fmt::Debug for vortex_session::registry::ReadContext + +pub fn vortex_session::registry::ReadContext::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub struct vortex_session::registry::Registry(_) impl vortex_session::registry::Registry diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 9c9a866da46..a929b092ada 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -83,7 +83,7 @@ impl ReadContext { pub fn resolve(&self, idx: u16) -> Option { self.ids.get(idx as usize).cloned() } - + pub fn ids(&self) -> &[Id] { &self.ids } From fab5158f5758d5658848aa10c261ee24a74fc099 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 6 Mar 2026 14:35:42 +0000 Subject: [PATCH 3/5] locks Signed-off-by: Robert Kruszewski --- vortex-file/src/footer/mod.rs | 13 ++++++------- vortex-layout/src/children.rs | 19 +++++++++---------- vortex-layout/src/flatbuffers.rs | 2 +- vortex-session/src/registry.rs | 5 ----- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index 9b1a092145f..997cbb4487f 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -32,7 +32,6 @@ use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::footer as fb; -use vortex_layout::LayoutContext; use vortex_layout::LayoutEncodingId; use vortex_layout::LayoutRef; use vortex_layout::layout_from_flatbuffer; @@ -86,12 +85,12 @@ impl Footer { // Create a LayoutContext from the registry. let layout_specs = fb_footer.layout_specs(); - let layout_ids = layout_specs + let layout_ids: Arc<[_]> = layout_specs .iter() .flat_map(|e| e.iter()) .map(|encoding| LayoutEncodingId::new_arc(Arc::from(encoding.id()))) .collect(); - let layout_ctx = LayoutContext::new(layout_ids); + let layout_read_ctx = ReadContext::new(layout_ids); // Create an ArrayContext from the registry. let array_specs = fb_footer.array_specs(); @@ -100,13 +99,13 @@ impl Footer { .flat_map(|e| e.iter()) .map(|encoding| ArrayId::new_arc(Arc::from(encoding.id()))) .collect(); - let read_ctx = ReadContext::new(array_ids); + let array_read_ctx = ReadContext::new(array_ids); let root_layout = layout_from_flatbuffer( layout_bytes, &dtype, - &layout_ctx, - &read_ctx, + &layout_read_ctx, + &array_read_ctx, session.layouts().registry(), )?; @@ -126,7 +125,7 @@ impl Footer { root_layout, segments, statistics, - read_ctx, + read_ctx: array_read_ctx, approx_byte_size: Some(approx_byte_size), }) } diff --git a/vortex-layout/src/children.rs b/vortex-layout/src/children.rs index 838fc5e7c8e..e20aa487b78 100644 --- a/vortex-layout/src/children.rs +++ b/vortex-layout/src/children.rs @@ -15,7 +15,6 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::layout as fbl; use vortex_session::registry::ReadContext; -use crate::LayoutContext; use crate::LayoutRef; use crate::segments::SegmentId; use crate::session::LayoutRegistry; @@ -101,8 +100,8 @@ impl LayoutChildren for OwnedLayoutChildren { pub(crate) struct ViewedLayoutChildren { flatbuffer: FlatBuffer, flatbuffer_loc: usize, - read_ctx: ReadContext, - layout_ctx: LayoutContext, + arra_read_ctx: ReadContext, + layout_read_ctx: ReadContext, layouts: LayoutRegistry, } @@ -116,14 +115,14 @@ impl ViewedLayoutChildren { flatbuffer: FlatBuffer, flatbuffer_loc: usize, read_ctx: ReadContext, - layout_ctx: LayoutContext, + layout_ctx: ReadContext, layouts: LayoutRegistry, ) -> Self { Self { flatbuffer, flatbuffer_loc, - read_ctx, - layout_ctx, + arra_read_ctx: read_ctx, + layout_read_ctx: layout_ctx, layouts, } } @@ -151,13 +150,13 @@ impl LayoutChildren for ViewedLayoutChildren { let viewed_children = ViewedLayoutChildren { flatbuffer: self.flatbuffer.clone(), flatbuffer_loc: fb_child._tab.loc(), - read_ctx: self.read_ctx.clone(), - layout_ctx: self.layout_ctx.clone(), + arra_read_ctx: self.arra_read_ctx.clone(), + layout_read_ctx: self.layout_read_ctx.clone(), layouts: self.layouts.clone(), }; let encoding_id = self - .layout_ctx + .layout_read_ctx .resolve(fb_child.encoding()) .ok_or_else(|| vortex_err!("Encoding not found: {}", fb_child.encoding()))?; let encoding = self.layouts.find(&encoding_id).ok_or_else(|| { @@ -178,7 +177,7 @@ impl LayoutChildren for ViewedLayoutChildren { .map(SegmentId::from) .collect_vec(), &viewed_children, - &self.read_ctx, + &self.arra_read_ctx, ) } diff --git a/vortex-layout/src/flatbuffers.rs b/vortex-layout/src/flatbuffers.rs index d55fab9101a..f180ab9753c 100644 --- a/vortex-layout/src/flatbuffers.rs +++ b/vortex-layout/src/flatbuffers.rs @@ -45,7 +45,7 @@ static LAYOUT_VERIFIER: LazyLock = LazyLock::new(|| { pub fn layout_from_flatbuffer( flatbuffer: FlatBuffer, dtype: &DType, - layout_ctx: &LayoutContext, + layout_ctx: &ReadContext, ctx: &ReadContext, layouts: &LayoutRegistry, ) -> VortexResult { diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index a929b092ada..35a65b086a3 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -160,11 +160,6 @@ impl Context { Some(u16::try_from(idx).vortex_expect("checked already")) } - /// Resolve an interned ID by its index. - pub fn resolve(&self, idx: u16) -> Option { - self.ids.read().get(idx as usize).cloned() - } - /// Get the list of interned IDs. pub fn to_ids(&self) -> Vec { self.ids.read().clone() From 2a7b5c54769d600b9b4a7d7710bbb0b9656f6e2d Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 6 Mar 2026 14:38:16 +0000 Subject: [PATCH 4/5] renames Signed-off-by: Robert Kruszewski --- vortex-file/src/footer/mod.rs | 8 ++++---- vortex-file/src/footer/serializer.rs | 2 +- vortex-layout/src/children.rs | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index 997cbb4487f..228fa860b2f 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -46,7 +46,7 @@ pub struct Footer { segments: Arc<[SegmentSpec]>, statistics: Option, // The specific arrays used within the file, in the order they were registered. - read_ctx: ReadContext, + array_read_ctx: ReadContext, // The approximate size of the footer in bytes, used for caching and memory management. approx_byte_size: Option, } @@ -56,13 +56,13 @@ impl Footer { root_layout: LayoutRef, segments: Arc<[SegmentSpec]>, statistics: Option, - read_ctx: ReadContext, + array_read_ctx: ReadContext, ) -> Self { Self { root_layout, segments, statistics, - read_ctx, + array_read_ctx, approx_byte_size: None, } } @@ -125,7 +125,7 @@ impl Footer { root_layout, segments, statistics, - read_ctx: array_read_ctx, + array_read_ctx: array_read_ctx, approx_byte_size: Some(approx_byte_size), }) } diff --git a/vortex-file/src/footer/serializer.rs b/vortex-file/src/footer/serializer.rs index ec96cc95cd6..493d10871a8 100644 --- a/vortex-file/src/footer/serializer.rs +++ b/vortex-file/src/footer/serializer.rs @@ -95,7 +95,7 @@ impl FooterSerializer { let (buffer, footer_segment) = write_flatbuffer( &mut self.offset, &FooterFlatBufferWriter { - ctx: self.footer.read_ctx.clone(), + ctx: self.footer.array_read_ctx.clone(), layout_ctx: ReadContext::new(layout_ctx.to_ids()), segment_specs: self.footer.segments.clone(), }, diff --git a/vortex-layout/src/children.rs b/vortex-layout/src/children.rs index e20aa487b78..7871e5bf151 100644 --- a/vortex-layout/src/children.rs +++ b/vortex-layout/src/children.rs @@ -100,7 +100,7 @@ impl LayoutChildren for OwnedLayoutChildren { pub(crate) struct ViewedLayoutChildren { flatbuffer: FlatBuffer, flatbuffer_loc: usize, - arra_read_ctx: ReadContext, + array_read_ctx: ReadContext, layout_read_ctx: ReadContext, layouts: LayoutRegistry, } @@ -114,15 +114,15 @@ impl ViewedLayoutChildren { pub(super) unsafe fn new_unchecked( flatbuffer: FlatBuffer, flatbuffer_loc: usize, - read_ctx: ReadContext, - layout_ctx: ReadContext, + array_read_ctx: ReadContext, + layout_read_ctx: ReadContext, layouts: LayoutRegistry, ) -> Self { Self { flatbuffer, flatbuffer_loc, - arra_read_ctx: read_ctx, - layout_read_ctx: layout_ctx, + array_read_ctx, + layout_read_ctx, layouts, } } @@ -150,7 +150,7 @@ impl LayoutChildren for ViewedLayoutChildren { let viewed_children = ViewedLayoutChildren { flatbuffer: self.flatbuffer.clone(), flatbuffer_loc: fb_child._tab.loc(), - arra_read_ctx: self.arra_read_ctx.clone(), + array_read_ctx: self.array_read_ctx.clone(), layout_read_ctx: self.layout_read_ctx.clone(), layouts: self.layouts.clone(), }; @@ -177,7 +177,7 @@ impl LayoutChildren for ViewedLayoutChildren { .map(SegmentId::from) .collect_vec(), &viewed_children, - &self.arra_read_ctx, + &self.array_read_ctx, ) } From 7a82d6f167f61fc13f0c5edbfd410ed557858942 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 6 Mar 2026 15:08:11 +0000 Subject: [PATCH 5/5] format Signed-off-by: Robert Kruszewski --- vortex-file/src/footer/mod.rs | 2 +- vortex-layout/public-api.lock | 2 +- vortex-session/public-api.lock | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index 228fa860b2f..f4e4c477259 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -125,7 +125,7 @@ impl Footer { root_layout, segments, statistics, - array_read_ctx: array_read_ctx, + array_read_ctx, approx_byte_size: Some(approx_byte_size), }) } diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 9da043fa465..f68706b04dc 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -1954,7 +1954,7 @@ pub fn vortex_layout::layouts::zoned::ZonedVTable::segment_ids(_layout: &Self::L pub fn vortex_layout::layouts::zoned::ZonedVTable::with_children(layout: &mut Self::Layout, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> -pub fn vortex_layout::layout_from_flatbuffer(flatbuffer: vortex_flatbuffers::FlatBuffer, dtype: &vortex_array::dtype::DType, layout_ctx: &vortex_layout::LayoutContext, ctx: &vortex_session::registry::ReadContext, layouts: &vortex_layout::session::LayoutRegistry) -> vortex_error::VortexResult +pub fn vortex_layout::layout_from_flatbuffer(flatbuffer: vortex_flatbuffers::FlatBuffer, dtype: &vortex_array::dtype::DType, layout_ctx: &vortex_session::registry::ReadContext, ctx: &vortex_session::registry::ReadContext, layouts: &vortex_layout::session::LayoutRegistry) -> vortex_error::VortexResult pub type vortex_layout::ArrayFuture = futures_core::future::BoxFuture<'static, vortex_error::VortexResult> diff --git a/vortex-session/public-api.lock b/vortex-session/public-api.lock index 4163cd885b5..65dc969ad79 100644 --- a/vortex-session/public-api.lock +++ b/vortex-session/public-api.lock @@ -12,8 +12,6 @@ pub fn vortex_session::registry::Context::intern(&self, id: &vortex_session:: pub fn vortex_session::registry::Context::new(ids: alloc::vec::Vec) -> Self -pub fn vortex_session::registry::Context::resolve(&self, idx: u16) -> core::option::Option - pub fn vortex_session::registry::Context::to_ids(&self) -> alloc::vec::Vec pub fn vortex_session::registry::Context::with_registry(self, registry: vortex_session::registry::Registry) -> Self