diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 797ba66224d..83f75ad45ae 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -229,6 +229,7 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; + use vortex_session::registry::ReadContext; use crate::RLEArray; use crate::test::SESSION; @@ -458,7 +459,7 @@ mod tests { .decode( &DType::Primitive(PType::U32, Nullability::NonNullable), 2048, - &ctx, + &ReadContext::new(ctx.to_ids()), &SESSION, ) .unwrap(); @@ -498,7 +499,12 @@ mod tests { let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts - .decode(sliced.dtype(), sliced.len(), &ctx, &SESSION) + .decode( + sliced.dtype(), + sliced.len(), + &ReadContext::new(ctx.to_ids()), + &SESSION, + ) .unwrap(); let original_data = sliced.to_primitive(); diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index 2758085846d..85d7dd64220 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -27,6 +27,7 @@ use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; static SESSION: LazyLock = LazyLock::new(|| { let session = VortexSession::empty().with::(); @@ -166,7 +167,7 @@ fn test_serde() -> VortexResult<()> { let decoded = parts.decode( &DType::Primitive(PType::I32, Nullability::NonNullable), 1_000_000, - &context, + &ReadContext::new(context.to_ids()), &SESSION, )?; let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 94d962a4868..1947a1d84a2 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -15982,7 +15982,7 @@ pub fn vortex_array::serde::ArrayParts::buffer_lengths(&self) -> alloc::vec::Vec pub fn vortex_array::serde::ArrayParts::child(&self, idx: usize) -> vortex_array::serde::ArrayParts -pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_array::ArrayContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub fn vortex_array::serde::ArrayParts::encoding_id(&self) -> u16 diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 65cc99da536..2f49a8c545b 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -239,6 +239,7 @@ impl DecimalVTable { mod tests { use vortex_buffer::ByteBufferMut; use vortex_buffer::buffer; + use vortex_session::registry::ReadContext; use crate::ArrayContext; use crate::IntoArray; @@ -273,7 +274,9 @@ mod tests { let concat = concat.freeze(); let parts = ArrayParts::try_from(concat).unwrap(); - let decoded = parts.decode(&dtype, 5, &ctx, &LEGACY_SESSION).unwrap(); + let decoded = parts + .decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION) + .unwrap(); assert!(decoded.is::()); } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 3ba48543c85..4d10c12f843 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -217,6 +217,7 @@ mod tests { use rstest::rstest; use vortex_buffer::ByteBufferMut; use vortex_error::VortexError; + use vortex_session::registry::ReadContext; use crate::ArrayContext; use crate::Canonical; @@ -269,7 +270,14 @@ mod tests { let concat = concat.freeze(); let parts = ArrayParts::try_from(concat).unwrap(); - let decoded = parts.decode(&dtype, len, &ctx, &LEGACY_SESSION).unwrap(); + let decoded = parts + .decode( + &dtype, + len, + &ReadContext::new(ctx.to_ids()), + &LEGACY_SESSION, + ) + .unwrap(); assert!(decoded.is::()); assert_eq!( diff --git a/vortex-array/src/context.rs b/vortex-array/src/context.rs deleted file mode 100644 index 962a3dfb1ae..00000000000 --- a/vortex-array/src/context.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_session::registry::Context; - -use crate::vtable::DynVTable; - -pub type ArrayContext = Context<&'static dyn DynVTable>; diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index d85569c046f..9e56c9c4560 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -18,14 +18,15 @@ use std::sync::LazyLock; pub use array::*; pub use canonical::*; pub use columnar::*; -pub use context::*; pub use executor::*; pub use hash::*; pub use mask_future::*; pub use metadata::*; use vortex_session::VortexSession; +use vortex_session::registry::Context; use crate::session::ArraySession; +use crate::vtable::DynVTable; pub mod accessor; #[doc(hidden)] @@ -39,7 +40,6 @@ pub mod builtins; mod canonical; mod columnar; pub mod compute; -mod context; pub mod display; pub mod dtype; mod executor; @@ -80,3 +80,5 @@ pub mod flatbuffers { // here... pub static LEGACY_SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); + +pub type ArrayContext = Context<&'static dyn DynVTable>; diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index e749e2696c3..c4f885f6c2a 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -24,6 +24,7 @@ use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::array as fba; use vortex_flatbuffers::array::Compression; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayContext; @@ -318,7 +319,7 @@ impl ArrayParts { &self, dtype: &DType, len: usize, - ctx: &ArrayContext, + ctx: &ReadContext, session: &VortexSession, ) -> VortexResult { let encoding_idx = self.flatbuffer().encoding(); @@ -623,7 +624,7 @@ impl ArrayParts { struct ArrayPartsChildren<'a> { parts: &'a ArrayParts, - ctx: &'a ArrayContext, + ctx: &'a ReadContext, session: &'a VortexSession, } diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 1fa4e86e885..0bf4711fdae 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -64,6 +64,7 @@ use vortex::scalar::ScalarTruncation; use vortex::scalar::lower_bound; use vortex::scalar::upper_bound; use vortex::session::VortexSession; +use vortex::session::registry::ReadContext; use vortex::utils::aliases::hash_map::HashMap; /// A buffer inlined into layout metadata for host-side access. @@ -94,7 +95,7 @@ pub struct CudaFlatLayout { row_count: u64, dtype: DType, segment_id: SegmentId, - ctx: ArrayContext, + ctx: ReadContext, array_tree: ByteBuffer, /// Small buffers kept on host, keyed by global buffer index. host_buffers: Arc>, @@ -107,7 +108,7 @@ impl CudaFlatLayout { } #[inline] - pub fn array_ctx(&self) -> &ArrayContext { + pub fn array_ctx(&self) -> &ReadContext { &self.ctx } @@ -195,7 +196,7 @@ impl VTable for CudaFlatVTable { metadata: &::Output, segment_ids: Vec, _children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult { if segment_ids.len() != 1 { vortex_bail!("CudaFlatLayout must have exactly one segment ID"); @@ -535,7 +536,7 @@ impl LayoutStrategy for CudaFlatLayoutStrategy { row_count, dtype: stream.dtype().clone(), segment_id, - ctx: ctx.clone(), + ctx: ReadContext::new(ctx.to_ids()), array_tree, host_buffers: Arc::new(host_buffer_map), } diff --git a/vortex-file/src/footer/file_layout.rs b/vortex-file/src/footer/file_layout.rs index 684955d7c80..22e5a3e593d 100644 --- a/vortex-file/src/footer/file_layout.rs +++ b/vortex-file/src/footer/file_layout.rs @@ -11,12 +11,11 @@ use std::sync::Arc; use flatbuffers::FlatBufferBuilder; use flatbuffers::WIPOffset; -use vortex_array::ArrayContext; use vortex_error::VortexResult; use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::footer as fb; -use vortex_layout::LayoutContext; +use vortex_session::registry::ReadContext; use crate::footer::segment::SegmentSpec; @@ -26,9 +25,9 @@ use crate::footer::segment::SegmentSpec; /// which describes the structure of the data in the file. pub(crate) struct FooterFlatBufferWriter { /// The array context containing encodings used in the file. - pub(crate) ctx: ArrayContext, + pub(crate) ctx: ReadContext, /// The layout context containing the layouts used in the file. - pub(crate) layout_ctx: LayoutContext, + pub(crate) layout_ctx: ReadContext, /// Specifications for all segments in the file. pub(crate) segment_specs: Arc<[SegmentSpec]>, } @@ -47,7 +46,7 @@ impl WriteFlatBuffer for FooterFlatBufferWriter { let array_specs = self .ctx - .to_ids() + .ids() .iter() .map(|e| { let id = fbb.create_string(e.as_ref()); @@ -58,7 +57,7 @@ impl WriteFlatBuffer for FooterFlatBufferWriter { let layout_specs = self .layout_ctx - .to_ids() + .ids() .iter() .map(|e| { let id = fbb.create_string(e.as_ref()); diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index 4142a60ded9..f4e4c477259 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -24,7 +24,6 @@ pub use file_statistics::FileStatistics; use flatbuffers::root; use itertools::Itertools; pub use segment::*; -use vortex_array::ArrayContext; use vortex_array::dtype::DType; use vortex_array::vtable::ArrayId; use vortex_buffer::ByteBuffer; @@ -33,12 +32,12 @@ use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::footer as fb; -use vortex_layout::LayoutContext; use vortex_layout::LayoutEncodingId; use vortex_layout::LayoutRef; use vortex_layout::layout_from_flatbuffer; use vortex_layout::session::LayoutSessionExt; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; /// Captures the layout information of a Vortex file. #[derive(Debug, Clone)] @@ -47,7 +46,7 @@ pub struct Footer { segments: Arc<[SegmentSpec]>, statistics: Option, // The specific arrays used within the file, in the order they were registered. - array_ctx: ArrayContext, + array_read_ctx: ReadContext, // The approximate size of the footer in bytes, used for caching and memory management. approx_byte_size: Option, } @@ -57,13 +56,13 @@ impl Footer { root_layout: LayoutRef, segments: Arc<[SegmentSpec]>, statistics: Option, - array_ctx: ArrayContext, + array_read_ctx: ReadContext, ) -> Self { Self { root_layout, segments, statistics, - array_ctx, + array_read_ctx, approx_byte_size: None, } } @@ -86,27 +85,27 @@ impl Footer { // Create a LayoutContext from the registry. let layout_specs = fb_footer.layout_specs(); - let layout_ids = layout_specs + let layout_ids: Arc<[_]> = layout_specs .iter() .flat_map(|e| e.iter()) .map(|encoding| LayoutEncodingId::new_arc(Arc::from(encoding.id()))) .collect(); - let layout_ctx = LayoutContext::new(layout_ids); + let layout_read_ctx = ReadContext::new(layout_ids); // Create an ArrayContext from the registry. let array_specs = fb_footer.array_specs(); - let array_ids = array_specs + let array_ids: Arc<[_]> = array_specs .iter() .flat_map(|e| e.iter()) .map(|encoding| ArrayId::new_arc(Arc::from(encoding.id()))) .collect(); - let array_ctx = ArrayContext::new(array_ids); + let array_read_ctx = ReadContext::new(array_ids); let root_layout = layout_from_flatbuffer( layout_bytes, &dtype, - &layout_ctx, - &array_ctx, + &layout_read_ctx, + &array_read_ctx, session.layouts().registry(), )?; @@ -126,7 +125,7 @@ impl Footer { root_layout, segments, statistics, - array_ctx, + array_read_ctx, approx_byte_size: Some(approx_byte_size), }) } diff --git a/vortex-file/src/footer/serializer.rs b/vortex-file/src/footer/serializer.rs index cc1d773115f..493d10871a8 100644 --- a/vortex-file/src/footer/serializer.rs +++ b/vortex-file/src/footer/serializer.rs @@ -10,6 +10,7 @@ use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::WriteFlatBufferExt; use vortex_layout::LayoutContext; +use vortex_session::registry::ReadContext; use crate::EOF_SIZE; use crate::Footer; @@ -94,8 +95,8 @@ impl FooterSerializer { let (buffer, footer_segment) = write_flatbuffer( &mut self.offset, &FooterFlatBufferWriter { - ctx: self.footer.array_ctx.clone(), - layout_ctx, + ctx: self.footer.array_read_ctx.clone(), + layout_ctx: ReadContext::new(layout_ctx.to_ids()), segment_specs: self.footer.segments.clone(), }, )?; diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 6bc6aef2dda..ef91dc701c0 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -45,6 +45,7 @@ use vortex_layout::sequence::SequentialStreamAdapter; use vortex_layout::sequence::SequentialStreamExt; use vortex_session::SessionExt; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::Footer; use crate::MAGIC_BYTES; @@ -211,7 +212,7 @@ impl VortexWriteOptions { &dtype, )) }, - ctx, + ReadContext::new(ctx.to_ids()), ); // Emit the footer buffers and EOF. diff --git a/vortex-ipc/public-api.lock b/vortex-ipc/public-api.lock index 94762154cae..09d90971d2f 100644 --- a/vortex-ipc/public-api.lock +++ b/vortex-ipc/public-api.lock @@ -46,7 +46,7 @@ pub mod vortex_ipc::messages pub enum vortex_ipc::messages::DecoderMessage -pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_array::context::ArrayContext, usize)) +pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_session::registry::ReadContext, usize)) pub vortex_ipc::messages::DecoderMessage::Buffer(vortex_buffer::ByteBuffer) diff --git a/vortex-ipc/src/messages/decoder.rs b/vortex-ipc/src/messages/decoder.rs index d2595d25235..ca863410855 100644 --- a/vortex-ipc/src/messages/decoder.rs +++ b/vortex-ipc/src/messages/decoder.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use bytes::Buf; use flatbuffers::root; use flatbuffers::root_unchecked; -use vortex_array::ArrayContext; use vortex_array::serde::ArrayParts; use vortex_array::vtable::ArrayId; use vortex_buffer::AlignedBuf; @@ -21,11 +20,12 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::message as fb; use vortex_flatbuffers::message::MessageHeader; use vortex_flatbuffers::message::MessageVersion; +use vortex_session::registry::ReadContext; /// A message decoded from an IPC stream. #[derive(Debug)] pub enum DecoderMessage { - Array((ArrayParts, ArrayContext, usize)), + Array((ArrayParts, ReadContext, usize)), Buffer(ByteBuffer), DType(FlatBuffer), } @@ -121,14 +121,14 @@ impl MessageDecoder { .header_as_array_message() .vortex_expect("header is array"); - let encoding_ids: Vec<_> = header + let encoding_ids: Arc<_> = header .encodings() .iter() .flat_map(|e| e.iter()) .map(|id| ArrayId::new_arc(Arc::from(id.to_string()))) .collect(); - let ctx = ArrayContext::new(encoding_ids); + let ctx = ReadContext::new(encoding_ids); let row_count = header.row_count() as usize; self.state = Default::default(); diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index bf69112d7dd..f68706b04dc 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -36,7 +36,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::buffered::Buffere pub fn vortex_layout::layouts::buffered::BufferedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::chunked @@ -58,7 +58,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::chunked::writer:: pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::chunked::ChunkedLayout @@ -124,7 +124,7 @@ pub type vortex_layout::layouts::chunked::ChunkedVTable::Layout = vortex_layout: pub type vortex_layout::layouts::chunked::ChunkedVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::chunked::ChunkedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -160,7 +160,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::collect::CollectS pub fn vortex_layout::layouts::collect::CollectStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::compressed @@ -182,7 +182,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::compressed::Compr pub fn vortex_layout::layouts::compressed::CompressingStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait vortex_layout::layouts::compressed::CompressorPlugin: core::marker::Send + core::marker::Sync + 'static @@ -274,7 +274,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::dict::writer::Dic pub fn vortex_layout::layouts::dict::writer::DictStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn vortex_layout::layouts::dict::writer::dict_layout_supported(dtype: &vortex_array::dtype::DType) -> bool @@ -372,7 +372,7 @@ pub type vortex_layout::layouts::dict::DictVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::dict::DictVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::dict::DictVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -442,19 +442,19 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::flat::writer::Fla pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::flat::FlatLayout impl vortex_layout::layouts::flat::FlatLayout -pub fn vortex_layout::layouts::flat::FlatLayout::array_ctx(&self) -> &vortex_array::context::ArrayContext +pub fn vortex_layout::layouts::flat::FlatLayout::array_ctx(&self) -> &vortex_session::registry::ReadContext pub fn vortex_layout::layouts::flat::FlatLayout::array_tree(&self) -> core::option::Option<&vortex_buffer::ByteBuffer> -pub fn vortex_layout::layouts::flat::FlatLayout::new(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_array::context::ArrayContext) -> Self +pub fn vortex_layout::layouts::flat::FlatLayout::new(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_session::registry::ReadContext) -> Self -pub fn vortex_layout::layouts::flat::FlatLayout::new_with_metadata(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_array::context::ArrayContext, metadata: core::option::Option) -> Self +pub fn vortex_layout::layouts::flat::FlatLayout::new_with_metadata(row_count: u64, dtype: vortex_array::dtype::DType, segment_id: vortex_layout::segments::SegmentId, ctx: vortex_session::registry::ReadContext, metadata: core::option::Option) -> Self pub fn vortex_layout::layouts::flat::FlatLayout::segment_id(&self) -> vortex_layout::segments::SegmentId @@ -536,7 +536,7 @@ pub type vortex_layout::layouts::flat::FlatVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::flat::FlatVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::flat::FlatVTable::child(_layout: &Self::Layout, _idx: usize) -> vortex_error::VortexResult @@ -576,7 +576,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::repartition::Repa pub fn vortex_layout::layouts::repartition::RepartitionStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::repartition::RepartitionWriterOptions @@ -658,7 +658,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::struct_::writer:: pub fn vortex_layout::layouts::struct_::writer::StructStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct vortex_layout::layouts::struct_::StructLayout @@ -730,7 +730,7 @@ pub type vortex_layout::layouts::struct_::StructVTable::Layout = vortex_layout:: pub type vortex_layout::layouts::struct_::StructVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::struct_::StructVTable::child(layout: &Self::Layout, index: usize) -> vortex_error::VortexResult @@ -774,7 +774,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::table::TableStrat pub fn vortex_layout::layouts::table::TableStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::zoned @@ -804,7 +804,7 @@ impl vortex_layout::LayoutStrategy for vortex_layout::layouts::zoned::writer::Zo pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub mod vortex_layout::layouts::zoned::zone_map @@ -938,7 +938,7 @@ pub type vortex_layout::layouts::zoned::ZonedVTable::Layout = vortex_layout::lay pub type vortex_layout::layouts::zoned::ZonedVTable::Metadata = vortex_layout::layouts::zoned::ZonedMetadata -pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::zoned::ZonedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1266,7 +1266,7 @@ pub type vortex_layout::vtable::VTable::Layout: 'static + core::marker::Send + c pub type vortex_layout::vtable::VTable::Metadata: vortex_array::metadata::SerializeMetadata + vortex_array::metadata::DeserializeMetadata + core::fmt::Debug -pub fn vortex_layout::vtable::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::vtable::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::vtable::VTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1298,7 +1298,7 @@ pub type vortex_layout::layouts::chunked::ChunkedVTable::Layout = vortex_layout: pub type vortex_layout::layouts::chunked::ChunkedVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::chunked::ChunkedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1330,7 +1330,7 @@ pub type vortex_layout::layouts::dict::DictVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::dict::DictVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::dict::DictVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1362,7 +1362,7 @@ pub type vortex_layout::layouts::flat::FlatVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::flat::FlatVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::flat::FlatVTable::child(_layout: &Self::Layout, _idx: usize) -> vortex_error::VortexResult @@ -1394,7 +1394,7 @@ pub type vortex_layout::layouts::struct_::StructVTable::Layout = vortex_layout:: pub type vortex_layout::layouts::struct_::StructVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::struct_::StructVTable::child(layout: &Self::Layout, index: usize) -> vortex_error::VortexResult @@ -1426,7 +1426,7 @@ pub type vortex_layout::layouts::zoned::ZonedVTable::Layout = vortex_layout::lay pub type vortex_layout::layouts::zoned::ZonedVTable::Metadata = vortex_layout::layouts::zoned::ZonedMetadata -pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::zoned::ZonedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1526,7 +1526,7 @@ impl vortex_layout::LayoutEncoding for vortex_layout:: pub fn vortex_layout::LayoutEncodingAdapter::as_any(&self) -> &dyn core::any::Any -pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::LayoutEncodingAdapter::id(&self) -> vortex_layout::LayoutEncodingId @@ -1646,7 +1646,7 @@ pub trait vortex_layout::LayoutEncoding: 'static + core::marker::Send + core::ma pub fn vortex_layout::LayoutEncoding::as_any(&self) -> &dyn core::any::Any -pub fn vortex_layout::LayoutEncoding::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::LayoutEncoding::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::LayoutEncoding::id(&self) -> vortex_layout::LayoutEncodingId @@ -1654,7 +1654,7 @@ impl vortex_layout::LayoutEncoding for vortex_layout:: pub fn vortex_layout::LayoutEncodingAdapter::as_any(&self) -> &dyn core::any::Any -pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::LayoutEncodingAdapter::build(&self, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &[u8], segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::LayoutEncodingAdapter::id(&self) -> vortex_layout::LayoutEncodingId @@ -1694,73 +1694,73 @@ pub trait vortex_layout::LayoutStrategy: 'static + core::marker::Send + core::ma pub fn vortex_layout::LayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::LayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::LayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for alloc::sync::Arc pub fn alloc::sync::Arc::buffered_bytes(&self) -> u64 -pub fn alloc::sync::Arc::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn alloc::sync::Arc::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::buffered::BufferedStrategy pub fn vortex_layout::layouts::buffered::BufferedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::buffered::BufferedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::collect::CollectStrategy pub fn vortex_layout::layouts::collect::CollectStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::collect::CollectStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::compressed::CompressingStrategy pub fn vortex_layout::layouts::compressed::CompressingStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::dict::writer::DictStrategy pub fn vortex_layout::layouts::dict::writer::DictStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::dict::writer::DictStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::flat::writer::FlatLayoutStrategy pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::flat::writer::FlatLayoutStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, _eof: vortex_layout::sequence::SequencePointer, _handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::repartition::RepartitionStrategy pub fn vortex_layout::layouts::repartition::RepartitionStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::repartition::RepartitionStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::struct_::writer::StructStrategy pub fn vortex_layout::layouts::struct_::writer::StructStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::struct_::writer::StructStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::table::TableStrategy pub fn vortex_layout::layouts::table::TableStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::table::TableStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl vortex_layout::LayoutStrategy for vortex_layout::layouts::zoned::writer::ZonedStrategy pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::buffered_bytes(&self) -> u64 -pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::context::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn vortex_layout::layouts::zoned::writer::ZonedStrategy::write_stream<'life0, 'async_trait>(&'life0 self, ctx: vortex_array::ArrayContext, segment_sink: vortex_layout::segments::SegmentSinkRef, stream: vortex_layout::sequence::SendableSequentialStream, eof: vortex_layout::sequence::SequencePointer, handle: vortex_io::runtime::handle::Handle) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait vortex_layout::VTable: 'static + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug @@ -1770,7 +1770,7 @@ pub type vortex_layout::VTable::Layout: 'static + core::marker::Send + core::mar pub type vortex_layout::VTable::Metadata: vortex_array::metadata::SerializeMetadata + vortex_array::metadata::DeserializeMetadata + core::fmt::Debug -pub fn vortex_layout::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::VTable::build(encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::VTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1802,7 +1802,7 @@ pub type vortex_layout::layouts::chunked::ChunkedVTable::Layout = vortex_layout: pub type vortex_layout::layouts::chunked::ChunkedVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::chunked::ChunkedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::chunked::ChunkedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1834,7 +1834,7 @@ pub type vortex_layout::layouts::dict::DictVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::dict::DictVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::dict::DictVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::dict::DictVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1866,7 +1866,7 @@ pub type vortex_layout::layouts::flat::FlatVTable::Layout = vortex_layout::layou pub type vortex_layout::layouts::flat::FlatVTable::Metadata = vortex_array::metadata::ProstMetadata -pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::flat::FlatVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, metadata: &::Output, segment_ids: alloc::vec::Vec, _children: &dyn vortex_layout::LayoutChildren, ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::flat::FlatVTable::child(_layout: &Self::Layout, _idx: usize) -> vortex_error::VortexResult @@ -1898,7 +1898,7 @@ pub type vortex_layout::layouts::struct_::StructVTable::Layout = vortex_layout:: pub type vortex_layout::layouts::struct_::StructVTable::Metadata = vortex_array::metadata::EmptyMetadata -pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::struct_::StructVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, row_count: u64, _metadata: &::Output, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::struct_::StructVTable::child(layout: &Self::Layout, index: usize) -> vortex_error::VortexResult @@ -1930,7 +1930,7 @@ pub type vortex_layout::layouts::zoned::ZonedVTable::Layout = vortex_layout::lay pub type vortex_layout::layouts::zoned::ZonedVTable::Metadata = vortex_layout::layouts::zoned::ZonedMetadata -pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_array::context::ArrayContext) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::zoned::ZonedVTable::build(_encoding: &Self::Encoding, dtype: &vortex_array::dtype::DType, _row_count: u64, metadata: &vortex_layout::layouts::zoned::ZonedMetadata, _segment_ids: alloc::vec::Vec, children: &dyn vortex_layout::LayoutChildren, _ctx: &vortex_session::registry::ReadContext) -> vortex_error::VortexResult pub fn vortex_layout::layouts::zoned::ZonedVTable::child(layout: &Self::Layout, idx: usize) -> vortex_error::VortexResult @@ -1954,7 +1954,7 @@ pub fn vortex_layout::layouts::zoned::ZonedVTable::segment_ids(_layout: &Self::L pub fn vortex_layout::layouts::zoned::ZonedVTable::with_children(layout: &mut Self::Layout, children: alloc::vec::Vec) -> vortex_error::VortexResult<()> -pub fn vortex_layout::layout_from_flatbuffer(flatbuffer: vortex_flatbuffers::FlatBuffer, dtype: &vortex_array::dtype::DType, layout_ctx: &vortex_layout::LayoutContext, ctx: &vortex_array::context::ArrayContext, layouts: &vortex_layout::session::LayoutRegistry) -> vortex_error::VortexResult +pub fn vortex_layout::layout_from_flatbuffer(flatbuffer: vortex_flatbuffers::FlatBuffer, dtype: &vortex_array::dtype::DType, layout_ctx: &vortex_session::registry::ReadContext, ctx: &vortex_session::registry::ReadContext, layouts: &vortex_layout::session::LayoutRegistry) -> vortex_error::VortexResult pub type vortex_layout::ArrayFuture = futures_core::future::BoxFuture<'static, vortex_error::VortexResult> diff --git a/vortex-layout/src/children.rs b/vortex-layout/src/children.rs index e87f0d05031..7871e5bf151 100644 --- a/vortex-layout/src/children.rs +++ b/vortex-layout/src/children.rs @@ -7,15 +7,14 @@ use std::sync::Arc; use flatbuffers::Follow; use itertools::Itertools; -use vortex_array::ArrayContext; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::layout as fbl; +use vortex_session::registry::ReadContext; -use crate::LayoutContext; use crate::LayoutRef; use crate::segments::SegmentId; use crate::session::LayoutRegistry; @@ -101,8 +100,8 @@ impl LayoutChildren for OwnedLayoutChildren { pub(crate) struct ViewedLayoutChildren { flatbuffer: FlatBuffer, flatbuffer_loc: usize, - array_ctx: ArrayContext, - layout_ctx: LayoutContext, + array_read_ctx: ReadContext, + layout_read_ctx: ReadContext, layouts: LayoutRegistry, } @@ -115,15 +114,15 @@ impl ViewedLayoutChildren { pub(super) unsafe fn new_unchecked( flatbuffer: FlatBuffer, flatbuffer_loc: usize, - array_ctx: ArrayContext, - layout_ctx: LayoutContext, + array_read_ctx: ReadContext, + layout_read_ctx: ReadContext, layouts: LayoutRegistry, ) -> Self { Self { flatbuffer, flatbuffer_loc, - array_ctx, - layout_ctx, + array_read_ctx, + layout_read_ctx, layouts, } } @@ -151,13 +150,13 @@ impl LayoutChildren for ViewedLayoutChildren { let viewed_children = ViewedLayoutChildren { flatbuffer: self.flatbuffer.clone(), flatbuffer_loc: fb_child._tab.loc(), - array_ctx: self.array_ctx.clone(), - layout_ctx: self.layout_ctx.clone(), + array_read_ctx: self.array_read_ctx.clone(), + layout_read_ctx: self.layout_read_ctx.clone(), layouts: self.layouts.clone(), }; let encoding_id = self - .layout_ctx + .layout_read_ctx .resolve(fb_child.encoding()) .ok_or_else(|| vortex_err!("Encoding not found: {}", fb_child.encoding()))?; let encoding = self.layouts.find(&encoding_id).ok_or_else(|| { @@ -178,7 +177,7 @@ impl LayoutChildren for ViewedLayoutChildren { .map(SegmentId::from) .collect_vec(), &viewed_children, - &self.array_ctx, + &self.array_read_ctx, ) } diff --git a/vortex-layout/src/encoding.rs b/vortex-layout/src/encoding.rs index d6b0892327e..ce0086e2ffc 100644 --- a/vortex-layout/src/encoding.rs +++ b/vortex-layout/src/encoding.rs @@ -7,12 +7,12 @@ use std::fmt::Display; use std::fmt::Formatter; use arcref::ArcRef; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; +use vortex_session::registry::ReadContext; use crate::IntoLayout; use crate::LayoutChildren; @@ -35,7 +35,7 @@ pub trait LayoutEncoding: 'static + Send + Sync + Debug + private::Sealed { metadata: &[u8], segment_ids: Vec, children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult; } @@ -58,7 +58,7 @@ impl LayoutEncoding for LayoutEncodingAdapter { metadata: &[u8], segment_ids: Vec, children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult { let metadata = ::deserialize(metadata)?; let layout = V::build( diff --git a/vortex-layout/src/flatbuffers.rs b/vortex-layout/src/flatbuffers.rs index 88010bcd083..f180ab9753c 100644 --- a/vortex-layout/src/flatbuffers.rs +++ b/vortex-layout/src/flatbuffers.rs @@ -8,7 +8,6 @@ use flatbuffers::FlatBufferBuilder; use flatbuffers::VerifierOptions; use flatbuffers::WIPOffset; use flatbuffers::root_with_opts; -use vortex_array::ArrayContext; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -16,6 +15,7 @@ use vortex_flatbuffers::FlatBuffer; use vortex_flatbuffers::FlatBufferRoot; use vortex_flatbuffers::WriteFlatBuffer; use vortex_flatbuffers::layout; +use vortex_session::registry::ReadContext; use crate::Layout; use crate::LayoutContext; @@ -45,8 +45,8 @@ static LAYOUT_VERIFIER: LazyLock = LazyLock::new(|| { pub fn layout_from_flatbuffer( flatbuffer: FlatBuffer, dtype: &DType, - layout_ctx: &LayoutContext, - ctx: &ArrayContext, + layout_ctx: &ReadContext, + ctx: &ReadContext, layouts: &LayoutRegistry, ) -> VortexResult { let fb_layout = root_with_opts::(&LAYOUT_VERIFIER, &flatbuffer)?; diff --git a/vortex-layout/src/layout.rs b/vortex-layout/src/layout.rs index 91df3db8061..a4f9e6e0830 100644 --- a/vortex-layout/src/layout.rs +++ b/vortex-layout/src/layout.rs @@ -326,7 +326,7 @@ mod private { #[cfg(test)] mod tests { use rstest::rstest; - use vortex_array::ArrayContext; + use vortex_session::registry::ReadContext; use super::*; @@ -482,7 +482,7 @@ mod tests { use crate::layouts::struct_::StructLayout; use crate::segments::SegmentId; - let ctx = ArrayContext::empty(); + let ctx = ReadContext::new([]); // Create a flat layout for dict values (utf8 strings) let dict_values = diff --git a/vortex-layout/src/layouts/chunked/mod.rs b/vortex-layout/src/layouts/chunked/mod.rs index e1395fa88f2..21752432ca4 100644 --- a/vortex-layout/src/layouts/chunked/mod.rs +++ b/vortex-layout/src/layouts/chunked/mod.rs @@ -6,12 +6,12 @@ pub mod writer; use std::sync::Arc; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::EmptyMetadata; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; @@ -90,7 +90,7 @@ impl VTable for ChunkedVTable { _metadata: &::Output, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { Ok(ChunkedLayout::new( row_count, diff --git a/vortex-layout/src/layouts/dict/mod.rs b/vortex-layout/src/layouts/dict/mod.rs index a8e63931fa9..05c379f7ceb 100644 --- a/vortex-layout/src/layouts/dict/mod.rs +++ b/vortex-layout/src/layouts/dict/mod.rs @@ -7,7 +7,6 @@ pub mod writer; use std::sync::Arc; use reader::DictReader; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::ProstMetadata; use vortex_array::dtype::DType; @@ -20,6 +19,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; @@ -108,7 +108,7 @@ impl VTable for DictVTable { metadata: &::Output, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { let values = children.child(0, dtype)?; let codes_nullable = metadata diff --git a/vortex-layout/src/layouts/flat/mod.rs b/vortex-layout/src/layouts/flat/mod.rs index b4199648186..8e0a880fd8a 100644 --- a/vortex-layout/src/layouts/flat/mod.rs +++ b/vortex-layout/src/layouts/flat/mod.rs @@ -7,7 +7,6 @@ pub mod writer; use std::env; use std::sync::Arc; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::ProstMetadata; use vortex_array::dtype::DType; @@ -16,6 +15,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; @@ -101,7 +101,7 @@ impl VTable for FlatVTable { metadata: &::Output, segment_ids: Vec, _children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult { if segment_ids.len() != 1 { vortex_bail!("Flat layout must have exactly one segment ID"); @@ -136,12 +136,12 @@ pub struct FlatLayout { row_count: u64, dtype: DType, segment_id: SegmentId, - ctx: ArrayContext, + ctx: ReadContext, array_tree: Option, } impl FlatLayout { - pub fn new(row_count: u64, dtype: DType, segment_id: SegmentId, ctx: ArrayContext) -> Self { + pub fn new(row_count: u64, dtype: DType, segment_id: SegmentId, ctx: ReadContext) -> Self { Self { row_count, dtype, @@ -155,7 +155,7 @@ impl FlatLayout { row_count: u64, dtype: DType, segment_id: SegmentId, - ctx: ArrayContext, + ctx: ReadContext, metadata: Option, ) -> Self { Self { @@ -173,7 +173,7 @@ impl FlatLayout { } #[inline] - pub fn array_ctx(&self) -> &ArrayContext { + pub fn array_ctx(&self) -> &ReadContext { &self.ctx } diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index f58b4729346..2b38446bccc 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -24,6 +24,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_io::runtime::Handle; +use vortex_session::registry::ReadContext; use crate::IntoLayout; use crate::LayoutRef; @@ -178,7 +179,7 @@ impl LayoutStrategy for FlatLayoutStrategy { row_count, stream.dtype().clone(), segment_id, - ctx.clone(), + ReadContext::new(ctx.to_ids()), array_node, ) .into_layout()) diff --git a/vortex-layout/src/layouts/struct_/mod.rs b/vortex-layout/src/layouts/struct_/mod.rs index ec41978ac50..96929d519a5 100644 --- a/vortex-layout/src/layouts/struct_/mod.rs +++ b/vortex-layout/src/layouts/struct_/mod.rs @@ -7,7 +7,6 @@ pub mod writer; use std::sync::Arc; use reader::StructReader; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::EmptyMetadata; use vortex_array::dtype::DType; @@ -22,6 +21,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::SessionExt; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; @@ -131,7 +131,7 @@ impl VTable for StructVTable { _metadata: &::Output, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { let struct_dt = dtype .as_struct_fields_opt() diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index 68c93a8c8a1..717f1ea0b9b 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -10,7 +10,6 @@ use std::sync::Arc; pub use builder::MAX_IS_TRUNCATED; pub use builder::MIN_IS_TRUNCATED; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; @@ -23,6 +22,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::LayoutChildType; use crate::LayoutEncodingRef; @@ -116,7 +116,7 @@ impl VTable for ZonedVTable { metadata: &ZonedMetadata, _segment_ids: Vec, children: &dyn LayoutChildren, - _ctx: &ArrayContext, + _ctx: &ReadContext, ) -> VortexResult { Ok(ZonedLayout { dtype: dtype.clone(), diff --git a/vortex-layout/src/vtable.rs b/vortex-layout/src/vtable.rs index 804d97dfc0d..f8349758ba0 100644 --- a/vortex-layout/src/vtable.rs +++ b/vortex-layout/src/vtable.rs @@ -5,13 +5,13 @@ use std::fmt::Debug; use std::ops::Deref; use std::sync::Arc; -use vortex_array::ArrayContext; use vortex_array::DeserializeMetadata; use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::VortexSession; +use vortex_session::registry::ReadContext; use crate::IntoLayout; use crate::Layout; @@ -73,7 +73,7 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { metadata: &::Output, segment_ids: Vec, children: &dyn LayoutChildren, - ctx: &ArrayContext, + ctx: &ReadContext, ) -> VortexResult; /// Replaces the children of the layout with the given layout references. diff --git a/vortex-python/src/serde/context.rs b/vortex-python/src/serde/context.rs index a31c22b83c4..6a4f6d5d3a2 100644 --- a/vortex-python/src/serde/context.rs +++ b/vortex-python/src/serde/context.rs @@ -2,11 +2,14 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::ops::Deref; +use std::sync::Arc; use itertools::Itertools; use pyo3::pyclass; use pyo3::pymethods; use vortex::array::ArrayContext; +use vortex::session::registry::Id; +use vortex::session::registry::ReadContext; /// An ArrayContext captures an ordered set of encodings. /// @@ -44,3 +47,44 @@ impl PyArrayContext { self.to_ids().len() } } + +/// A ReadContext captures an ordered set of encodings. +/// +/// In a serialized array, encodings are identified by a positional index into such an +/// :class:`~vortex.ReadContext`. +#[pyclass(name = "ReadContext", module = "vortex", frozen)] +pub(crate) struct PyReadContext(ReadContext); + +impl From for PyReadContext { + fn from(context: ReadContext) -> Self { + Self(context) + } +} + +impl Deref for PyReadContext { + type Target = ReadContext; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[pymethods] +impl PyReadContext { + #[new] + fn new(ids: Vec) -> Self { + Self(ReadContext::new( + ids.into_iter() + .map(|i| Id::new_arc(Arc::from(i))) + .collect::>(), + )) + } + + fn __str__(&self) -> String { + self.0.ids().iter().join(", ") + } + + fn __len__(&self) -> usize { + self.ids().len() + } +} diff --git a/vortex-python/src/serde/mod.rs b/vortex-python/src/serde/mod.rs index 0c6587966c8..e3290821194 100644 --- a/vortex-python/src/serde/mod.rs +++ b/vortex-python/src/serde/mod.rs @@ -20,6 +20,7 @@ use crate::arrays::PyArrayRef; use crate::error::PyVortexResult; use crate::install_module; use crate::serde::context::PyArrayContext; +use crate::serde::context::PyReadContext; use crate::serde::parts::PyArrayParts; /// Register serde functions and classes. @@ -30,6 +31,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(decode_ipc_array, &m)?)?; m.add_function(wrap_pyfunction!(decode_ipc_array_buffers, &m)?)?; diff --git a/vortex-python/src/serde/parts.rs b/vortex-python/src/serde/parts.rs index 68c3c51344f..1f558a4d0a6 100644 --- a/vortex-python/src/serde/parts.rs +++ b/vortex-python/src/serde/parts.rs @@ -17,7 +17,7 @@ use crate::SESSION; use crate::arrays::PyArrayRef; use crate::dtype::PyDType; use crate::error::PyVortexResult; -use crate::serde::context::PyArrayContext; +use crate::serde::context::PyReadContext; /// ArrayParts is a parsed representation of a serialized array. /// @@ -56,7 +56,7 @@ impl PyArrayParts { /// The decoded array. fn decode( &self, - ctx: &PyArrayContext, + ctx: &PyReadContext, dtype: PyDType, len: usize, ) -> PyVortexResult { diff --git a/vortex-session/public-api.lock b/vortex-session/public-api.lock index 4e00c75a886..65dc969ad79 100644 --- a/vortex-session/public-api.lock +++ b/vortex-session/public-api.lock @@ -12,8 +12,6 @@ pub fn vortex_session::registry::Context::intern(&self, id: &vortex_session:: pub fn vortex_session::registry::Context::new(ids: alloc::vec::Vec) -> Self -pub fn vortex_session::registry::Context::resolve(&self, idx: u16) -> core::option::Option - pub fn vortex_session::registry::Context::to_ids(&self) -> alloc::vec::Vec pub fn vortex_session::registry::Context::with_registry(self, registry: vortex_session::registry::Registry) -> Self @@ -30,6 +28,24 @@ impl core::default::Default for vortex_session::registry::Context pub fn vortex_session::registry::Context::default() -> Self +pub struct vortex_session::registry::ReadContext + +impl vortex_session::registry::ReadContext + +pub fn vortex_session::registry::ReadContext::ids(&self) -> &[vortex_session::registry::Id] + +pub fn vortex_session::registry::ReadContext::new(ids: impl core::convert::Into>) -> Self + +pub fn vortex_session::registry::ReadContext::resolve(&self, idx: u16) -> core::option::Option + +impl core::clone::Clone for vortex_session::registry::ReadContext + +pub fn vortex_session::registry::ReadContext::clone(&self) -> vortex_session::registry::ReadContext + +impl core::fmt::Debug for vortex_session::registry::ReadContext + +pub fn vortex_session::registry::ReadContext::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub struct vortex_session::registry::Registry(_) impl vortex_session::registry::Registry diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 392579b0056..35a65b086a3 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -66,6 +66,29 @@ impl Registry { } } +/// A [`ReadContext`] holds a set of interned IDs for use during deserialization, mapping +/// u16 indices to IDs. +#[derive(Clone, Debug)] +pub struct ReadContext { + ids: Arc<[Id]>, +} + +impl ReadContext { + /// Create a context with the given initial IDs. + pub fn new(ids: impl Into>) -> Self { + Self { ids: ids.into() } + } + + /// Resolve an interned ID by its index. + pub fn resolve(&self, idx: u16) -> Option { + self.ids.get(idx as usize).cloned() + } + + pub fn ids(&self) -> &[Id] { + &self.ids + } +} + /// A [`Context`] holds a set of interned IDs for use during serialization/deserialization, mapping /// IDs to u16 indices. /// @@ -137,11 +160,6 @@ impl Context { Some(u16::try_from(idx).vortex_expect("checked already")) } - /// Resolve an interned ID by its index. - pub fn resolve(&self, idx: u16) -> Option { - self.ids.read().get(idx as usize).cloned() - } - /// Get the list of interned IDs. pub fn to_ids(&self) -> Vec { self.ids.read().clone()