Skip to content

layout reader ctx #8037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
onursatici wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

layout reader ctx #8037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
67ac6d6
layout reader ctx
onursatici May 20, 2026
e8f8e0d
break
onursatici May 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions vortex-cuda/src/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ impl VTable for CudaFlat {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
_ctx: &vortex::layout::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(CudaFlatReader {
layout: layout.clone(),
Expand Down
7 changes: 6 additions & 1 deletion vortex-file/src/file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,12 @@ impl VortexFile {
self.footer
.layout()
// TODO(ngates): we may want to allow the user pass in a name here?
.new_reader("".into(), segment_source, &self.session)
.new_reader(
"".into(),
segment_source,
&self.session,
&Default::default(),
)
}

/// Create a [`DataSource`](vortex_scan::DataSource) from this file for scanning.
Expand Down
8 changes: 4 additions & 4 deletions vortex-file/src/v2/file_stats_reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

let reader =
FileStatsLayoutReader::new(child, test_file_stats(0, 100), SESSION.clone());
Expand Down Expand Up @@ -349,7 +349,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

let reader =
FileStatsLayoutReader::new(child, test_file_stats(0, 100), SESSION.clone());
Expand Down Expand Up @@ -400,7 +400,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

// File-level stats: 1 null in deleted_at.
let mut stats = StatsSet::default();
Expand Down Expand Up @@ -449,7 +449,7 @@ mod tests {
)
.await?;

let child = layout.new_reader("".into(), segments, &SESSION)?;
let child = layout.new_reader("".into(), segments, &SESSION, &Default::default())?;

let reader =
FileStatsLayoutReader::new(child, test_file_null_count_stats(5), SESSION.clone());
Expand Down
64 changes: 43 additions & 21 deletions vortex-layout/public-api.lock
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 11 additions & 1 deletion vortex-layout/src/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use vortex_session::registry::Id;

use crate::LayoutEncodingId;
use crate::LayoutEncodingRef;
use crate::LayoutReaderContext;
use crate::LayoutReaderRef;
use crate::VTable;
use crate::display::DisplayLayoutTree;
Expand Down Expand Up @@ -63,11 +64,19 @@ pub trait Layout: 'static + Send + Sync + Debug + private::Sealed {
/// Get the segment IDs for this layout.
fn segment_ids(&self) -> Vec<SegmentId>;

/// Construct a new reader for this layout, using the given dependency context.
///
/// `ctx` is the typed-data registry threaded through reader construction (see
/// [`LayoutReaderContext`]). Top-level callers (file open, tests) typically pass
/// `&LayoutReaderContext::new()`; recursive callers inside layout implementations
/// must propagate the `ctx` they were handed so ancestor-published values reach
/// descendants.
fn new_reader(
&self,
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &LayoutReaderContext,
) -> VortexResult<LayoutReaderRef>;
}

Expand Down Expand Up @@ -311,8 +320,9 @@ impl<V: VTable> Layout for LayoutAdapter<V> {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
V::new_reader(&self.0, name, segment_source, session)
V::new_reader(&self.0, name, segment_source, session, ctx)
}
}

Expand Down
2 changes: 2 additions & 0 deletions vortex-layout/src/layouts/chunked/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,14 @@ impl VTable for Chunked {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(ChunkedReader::new(
layout.clone(),
name,
segment_source,
session,
ctx.clone(),
)))
}

Expand Down
11 changes: 9 additions & 2 deletions vortex-layout/src/layouts/chunked/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ impl ChunkedReader {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: crate::LayoutReaderContext,
) -> Self {
let nchildren = layout.nchildren();

Expand All @@ -66,6 +67,7 @@ impl ChunkedReader {
names,
segment_source,
session.clone(),
ctx,
);

Self {
Expand Down Expand Up @@ -435,7 +437,12 @@ mod test {
) {
let layout = nested_chunked_layout();
let reader = layout
.new_reader("".into(), Arc::new(TestSegments::default()), &SESSION)
.new_reader(
"".into(),
Arc::new(TestSegments::default()),
&SESSION,
&Default::default(),
)
.unwrap();

let splits = SplitBy::Layout
Expand All @@ -451,7 +458,7 @@ mod test {
) {
block_on(|_h| async {
let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down
2 changes: 2 additions & 0 deletions vortex-layout/src/layouts/dict/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,14 @@ impl VTable for Dict {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(DictReader::try_new(
layout.clone(),
name,
segment_source,
session.clone(),
ctx.clone(),
)?))
}

Expand Down
18 changes: 11 additions & 7 deletions vortex-layout/src/layouts/dict/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,17 +58,21 @@ impl DictReader {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: VortexSession,
ctx: crate::LayoutReaderContext,
) -> VortexResult<Self> {
let values_len = usize::try_from(layout.values.row_count())?;
let values = layout.values.new_reader(
format!("{name}.values").into(),
Arc::clone(&segment_source),
&session,
&ctx,
)?;
let codes = layout.codes.new_reader(
format!("{name}.codes").into(),
segment_source,
&session,
&ctx,
)?;
let codes =
layout
.codes
.new_reader(format!("{name}.codes").into(), segment_source, &session)?;

Ok(Self {
layout,
Expand Down Expand Up @@ -370,7 +374,7 @@ mod tests {
);
assert!(layout.encoding_id() == LayoutId::new("vortex.dict"));
let actual = layout
.new_reader("".into(), segments, &session)
.new_reader("".into(), segments, &session, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -454,7 +458,7 @@ mod tests {
)),
);
let mask = layout
.new_reader("".into(), segments, &session)
.new_reader("".into(), segments, &session, &Default::default())
.unwrap()
.filter_evaluation(&(0..3), &filter, MaskFuture::new_true(3))
.unwrap()
Expand Down Expand Up @@ -515,7 +519,7 @@ mod tests {
let expression = is_not_null(root());
assert_eq!(layout.encoding_id(), LayoutId::new("vortex.dict"));
let actual = layout
.new_reader("".into(), segments, &session)
.new_reader("".into(), segments, &session, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down
1 change: 1 addition & 0 deletions vortex-layout/src/layouts/flat/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ impl VTable for Flat {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
_ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(FlatReader::new(
layout.clone(),
Expand Down
6 changes: 3 additions & 3 deletions vortex-layout/src/layouts/flat/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ mod test {
);

let result = layout
.new_reader("".into(), segments, &SESSION)?
.new_reader("".into(), segments, &SESSION, &Default::default())?
.projection_evaluation(
&(0..layout.row_count()),
&root(),
Expand Down Expand Up @@ -313,7 +313,7 @@ mod test {

let expr = gt(root(), lit(3i32));
let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -350,7 +350,7 @@ mod test {
.unwrap();

let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(&(2..4), &root(), MaskFuture::new_true(2))
.unwrap()
Expand Down
6 changes: 3 additions & 3 deletions vortex-layout/src/layouts/flat/writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ mod tests {
.unwrap();

let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -311,7 +311,7 @@ mod tests {
.unwrap();

let result = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down Expand Up @@ -384,7 +384,7 @@ mod tests {

// We should be able to read the array we just wrote.
let result: ArrayRef = layout
.new_reader("".into(), segments, &SESSION)
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap()
.projection_evaluation(
&(0..layout.row_count()),
Expand Down
1 change: 1 addition & 0 deletions vortex-layout/src/layouts/foreign/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ impl Layout for ForeignLayout {
_name: Arc<str>,
_segment_source: Arc<dyn SegmentSource>,
_session: &VortexSession,
_ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
vortex_bail!(
"Cannot read unknown layout encoding '{}'",
Expand Down
12 changes: 9 additions & 3 deletions vortex-layout/src/layouts/row_idx/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,9 @@ mod tests {
let expr = eq(root(), lit(3i32));
let result = RowIdxLayoutReader::new(
0,
layout.new_reader("".into(), segments, &SESSION).unwrap(),
layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap(),
SESSION.clone(),
)
.projection_evaluation(
Expand Down Expand Up @@ -411,7 +413,9 @@ mod tests {
let expr = gt(row_idx(), lit(3u64));
let result = RowIdxLayoutReader::new(
0,
layout.new_reader("".into(), segments, &SESSION).unwrap(),
layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap(),
SESSION.clone(),
)
.projection_evaluation(
Expand Down Expand Up @@ -456,7 +460,9 @@ mod tests {

let result = RowIdxLayoutReader::new(
0,
layout.new_reader("".into(), segments, &SESSION).unwrap(),
layout
.new_reader("".into(), segments, &SESSION, &Default::default())
.unwrap(),
SESSION.clone(),
)
.projection_evaluation(
Expand Down
2 changes: 2 additions & 0 deletions vortex-layout/src/layouts/struct_/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,14 @@ impl VTable for Struct {
name: Arc<str>,
segment_source: Arc<dyn SegmentSource>,
session: &VortexSession,
ctx: &crate::LayoutReaderContext,
) -> VortexResult<LayoutReaderRef> {
Ok(Arc::new(StructReader::try_new(
layout.clone(),
name,
segment_source,
session.session(),
ctx.clone(),
)?))
}

Expand Down
Loading
Loading