diff --git a/Cargo.lock b/Cargo.lock index 6f2b7bb670e..65d522aa2f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,35 +145,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest 0.10.7", - "liblzma", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.4", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum 0.27.2", - "strum_macros 0.27.2", - "thiserror 2.0.18", - "uuid", - "zstd", -] - [[package]] name = "approx" version = "0.5.1" @@ -282,6 +253,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-avro" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap 2.14.0", + "liblzma", + "rand 0.9.4", + "serde", + "serde_json", + "snap", + "strum_macros 0.28.0", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" version = "58.3.0" @@ -787,7 +782,6 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] [[package]] @@ -799,7 +793,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -901,31 +895,6 @@ dependencies = [ "piper", ] -[[package]] -name = "bon" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "borsh" version = "1.6.1" @@ -1669,6 +1638,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1979,45 +1963,91 @@ dependencies = [ "arrow-schema", "async-trait", "bytes", + "chrono", + "datafusion-catalog 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-catalog-listing 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-arrow 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-csv 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource-json 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-nested 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-table 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-optimizer 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-optimizer 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-sql 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.4", + "regex", + "sqlparser 0.61.0", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", + "datafusion-catalog 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-catalog-listing 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource-arrow 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", + "datafusion-datasource-csv 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource-json 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-nested 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-table 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-window 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-optimizer 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-adapter 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-optimizer 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-sql 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "flate2", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "liblzma", "log", "object_store 0.13.2", "parking_lot", "parquet", - "rand 0.9.4", - "regex", - "sqlparser", + "sqlparser 0.62.0", "tempfile", "tokio", "url", @@ -2032,9 +2062,9 @@ dependencies = [ "anyhow", "clap", "custom-labels", - "datafusion", - "datafusion-common", - "datafusion-physical-plan", + "datafusion 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "itertools 0.14.0", "object_store 0.13.2", @@ -2060,14 +2090,38 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "itertools 0.14.0", "log", @@ -2084,15 +2138,37 @@ checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-adapter 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "itertools 0.14.0", "log", @@ -2106,7 +2182,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", - "apache-avro", "arrow", "arrow-ipc", "chrono", @@ -2117,11 +2192,34 @@ dependencies = [ "libc", "log", "object_store 0.13.2", - "parquet", "paste", + "sqlparser 0.61.0", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "libc", + "log", + "object_store 0.13.2", + "parquet", "recursive", - "sqlparser", + "sqlparser 0.62.0", "tokio", + "uuid", "web-time", ] @@ -2136,11 +2234,49 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-common-runtime" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-datasource" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-adapter 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "rand 0.9.4", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" dependencies = [ "arrow", "async-compression", @@ -2148,15 +2284,15 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-adapter 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "flate2", "futures", "glob", @@ -2164,6 +2300,7 @@ dependencies = [ "liblzma", "log", "object_store 0.13.2", + "parking_lot", "rand 0.9.4", "tokio", "tokio-util", @@ -2181,14 +2318,37 @@ dependencies = [ "arrow-ipc", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "itertools 0.14.0", + "object_store 0.13.2", + "tokio", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "itertools 0.14.0", "object_store 0.13.2", @@ -2198,20 +2358,18 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-adapter 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", - "num-traits", "object_store 0.13.2", ] @@ -2224,14 +2382,36 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "object_store 0.13.2", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "object_store 0.13.2", "regex", @@ -2247,14 +2427,14 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-session 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "object_store 0.13.2", "serde_json", @@ -2262,27 +2442,49 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "datafusion-datasource-json" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "futures", + "object_store 0.13.2", + "tokio", + "tokio-stream", +] + [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-adapter 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-pruning 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-session 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "itertools 0.14.0", "log", @@ -2298,6 +2500,11 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +[[package]] +name = "datafusion-doc" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" + [[package]] name = "datafusion-execution" version = "53.1.0" @@ -2309,9 +2516,30 @@ dependencies = [ "async-trait", "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.4", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "log", "object_store 0.13.2", @@ -2330,18 +2558,39 @@ dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 2.14.0", "itertools 0.14.0", "paste", + "serde_json", + "sqlparser 0.61.0", +] + +[[package]] +name = "datafusion-expr" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-doc 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-window-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "indexmap 2.14.0", + "itertools 0.14.0", "recursive", "serde_json", - "sqlparser", + "sqlparser 0.62.0", ] [[package]] @@ -2351,12 +2600,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 2.14.0", "itertools 0.14.0", "paste", ] +[[package]] +name = "datafusion-expr-common" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "indexmap 2.14.0", + "itertools 0.14.0", +] + [[package]] name = "datafusion-functions" version = "53.1.0" @@ -2370,16 +2630,16 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex", "itertools 0.14.0", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-traits", "rand 0.9.4", @@ -2389,6 +2649,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-functions" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-doc 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-macros 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "hex", + "itertools 0.14.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.4", + "regex", + "sha2 0.11.0", + "uuid", +] + [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" @@ -2397,20 +2688,40 @@ checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "half", "log", "num-traits", "paste", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-doc 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-macros 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" @@ -2419,9 +2730,20 @@ checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", ] [[package]] @@ -2432,16 +2754,16 @@ checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.16.1", "itertools 0.14.0", "itoa", @@ -2449,6 +2771,30 @@ dependencies = [ "paste", ] +[[package]] +name = "datafusion-functions-nested" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-doc 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-macros 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + [[package]] name = "datafusion-functions-table" version = "53.1.0" @@ -2457,14 +2803,29 @@ checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot", "paste", ] +[[package]] +name = "datafusion-functions-table" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "parking_lot", +] + [[package]] name = "datafusion-functions-window" version = "53.1.0" @@ -2472,25 +2833,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log", "paste", ] +[[package]] +name = "datafusion-functions-window" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-doc 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-window-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-macros 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "log", +] + [[package]] name = "datafusion-functions-window-common" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", ] [[package]] @@ -2499,7 +2885,17 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ - "datafusion-doc", + "datafusion-doc 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "datafusion-macros" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "datafusion-doc 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "quote", "syn 2.0.117", ] @@ -2512,10 +2908,28 @@ checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-optimizer" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2532,11 +2946,11 @@ checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "half", "hashbrown 0.16.1", "indexmap 2.14.0", @@ -2544,6 +2958,26 @@ dependencies = [ "parking_lot", "paste", "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "petgraph", "recursive", "tokio", ] @@ -2555,11 +2989,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "itertools 0.14.0", ] @@ -2572,29 +3020,62 @@ dependencies = [ "ahash 0.8.12", "arrow", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.16.1", "indexmap 2.14.0", "itertools 0.14.0", - "parking_lot", + "parking_lot", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-pruning 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-pruning 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "itertools 0.14.0", "recursive", ] @@ -2610,15 +3091,15 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common-runtime 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "half", "hashbrown 0.16.1", @@ -2631,6 +3112,38 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-plan" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-window-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "datafusion-pruning" version = "53.1.0" @@ -2638,16 +3151,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-datasource 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.14.0", "log", ] +[[package]] +name = "datafusion-pruning" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "log", +] + [[package]] name = "datafusion-session" version = "53.1.0" @@ -2655,37 +3183,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "async-trait", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "parking_lot", ] [[package]] name = "datafusion-spark" version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-catalog 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-aggregate-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-nested 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "log", + "num-traits", "percent-encoding", "rand 0.9.4", "serde_json", - "sha1 0.10.6", - "sha2 0.10.9", + "sha1", + "sha2 0.11.0", + "twox-hash", "url", ] @@ -2698,29 +3241,44 @@ dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-nested 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 2.14.0", + "log", + "regex", + "sqlparser 0.61.0", +] + +[[package]] +name = "datafusion-sql" +version = "53.1.0" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions-nested 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "indexmap 2.14.0", "log", "recursive", "regex", - "sqlparser", + "sqlparser 0.62.0", ] [[package]] name = "datafusion-sqllogictest" version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e5a4a7a49143a68936992b6dbb0db44121c635e9992b2482817278f1e69c56" +source = "git+https://github.com/apache/datafusion?branch=branch-54#70caf372a54a088b4bcfd3b4479490f3388971d8" dependencies = [ "arrow", "async-trait", "bigdecimal", "clap", - "datafusion", + "datafusion 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "datafusion-spark", - "datafusion-substrait", "futures", "half", "indicatif", @@ -2728,32 +3286,12 @@ dependencies = [ "log", "object_store 0.13.2", "sqllogictest", - "sqlparser", + "sqlparser 0.62.0", "tempfile", "thiserror 2.0.18", "tokio", ] -[[package]] -name = "datafusion-substrait" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" -dependencies = [ - "async-recursion", - "async-trait", - "chrono", - "datafusion", - "half", - "itertools 0.14.0", - "object_store 0.13.2", - "pbjson-types", - "prost 0.14.3", - "substrait", - "tokio", - "url", -] - [[package]] name = "deepsize" version = "0.2.0" @@ -4324,11 +4862,11 @@ dependencies = [ "chrono", "crossbeam-skiplist", "dashmap", - "datafusion", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-plan 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "deepsize", "either", "futures", @@ -4433,8 +4971,8 @@ dependencies = [ "byteorder", "bytes", "chrono", - "datafusion-common", - "datafusion-sql", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-sql 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "deepsize", "futures", "itertools 0.13.0", @@ -4474,10 +5012,10 @@ dependencies = [ "arrow-select", "async-trait", "chrono", - "datafusion", - "datafusion-common", - "datafusion-functions", - "datafusion-physical-expr", + "datafusion 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures", "jsonb", "lance-arrow", @@ -4567,7 +5105,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "datafusion-common", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "deepsize", "futures", "lance-arrow", @@ -4605,11 +5143,11 @@ dependencies = [ "bytes", "chrono", "crossbeam-queue", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-sql", + "datafusion 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-common 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-sql 53.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "deepsize", "dirs", "fst", @@ -5209,6 +5747,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.0" @@ -5516,7 +6064,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -5687,7 +6234,7 @@ dependencies = [ "humantime", "hyper", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml", @@ -6009,43 +6556,6 @@ dependencies = [ "stfu8", ] -[[package]] -name = "pbjson" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" -dependencies = [ - "base64", - "serde", -] - -[[package]] -name = "pbjson-build" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095" -dependencies = [ - "heck", - "itertools 0.14.0", - "prost 0.14.3", - "prost-types", -] - -[[package]] -name = "pbjson-types" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e748e28374f10a330ee3bb9f29b828c0ac79831a32bab65015ad9b661ead526" -dependencies = [ - "bytes", - "chrono", - "pbjson", - "pbjson-build", - "prost 0.14.3", - "prost-build", - "serde", -] - [[package]] name = "pbkdf2" version = "0.13.0" @@ -6144,7 +6654,7 @@ checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ "rand 0.9.4", "socket2", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -6357,7 +6867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph", @@ -6389,7 +6899,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -6565,12 +7075,6 @@ dependencies = [ "url", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quick-xml" version = "0.39.4" @@ -7011,16 +7515,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "regress" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" -dependencies = [ - "hashbrown 0.16.1", - "memchr", -] - [[package]] name = "relative-path" version = "1.9.3" @@ -7419,18 +7913,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "schemars" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" -dependencies = [ - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", -] - [[package]] name = "schemars" version = "0.9.0" @@ -7455,18 +7937,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "schemars_derive" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.117", -] - [[package]] name = "scoped-tls" version = "1.0.1" @@ -7546,16 +8016,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -7576,17 +8036,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "serde_json" version = "1.0.149" @@ -7640,18 +8089,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_tokenstream" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c49585c52c01f13c5c2ebb333f14f6885d76daa768d8a037d28017ec538c69" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.117", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -7696,19 +8133,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.14.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "serial_test" version = "3.4.0" @@ -7735,17 +8159,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha1" version = "0.11.0" @@ -7987,7 +8400,7 @@ dependencies = [ "humantime", "itertools 0.13.0", "libtest-mimic", - "md-5", + "md-5 0.10.6", "owo-colors", "rand 0.8.6", "regex", @@ -8003,6 +8416,16 @@ name = "sqlparser" version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -8147,31 +8570,6 @@ dependencies = [ "unicode-width 0.1.14", ] -[[package]] -name = "substrait" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" -dependencies = [ - "heck", - "pbjson", - "pbjson-build", - "pbjson-types", - "prettyplease", - "prost 0.14.3", - "prost-build", - "prost-types", - "regress", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_yaml", - "syn 2.0.117", - "typify", - "walkdir", -] - [[package]] name = "subtle" version = "2.6.1" @@ -8876,53 +9274,6 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" -[[package]] -name = "typify" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5bcc6f62eb1fa8aa4098f39b29f93dcb914e17158b76c50360911257aa629" -dependencies = [ - "typify-impl", - "typify-macro", -] - -[[package]] -name = "typify-impl" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1eb359f7ffa4f9ebe947fa11a1b2da054564502968db5f317b7e37693cb2240" -dependencies = [ - "heck", - "log", - "proc-macro2", - "quote", - "regress", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "syn 2.0.117", - "thiserror 2.0.18", - "unicode-ident", -] - -[[package]] -name = "typify-macro" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" -dependencies = [ - "proc-macro2", - "quote", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_tokenstream", - "syn 2.0.117", - "typify-impl", -] - [[package]] name = "uint" version = "0.10.0" @@ -8998,12 +9349,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -9537,19 +9882,19 @@ dependencies = [ "anyhow", "arrow-schema", "async-trait", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-catalog 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-common-runtime 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-datasource 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-execution 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-functions 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-adapter 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-expr-common 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-physical-plan 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", + "datafusion-pruning 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "futures", "insta", "itertools 0.14.0", @@ -10080,7 +10425,7 @@ dependencies = [ "async-trait", "bigdecimal", "clap", - "datafusion", + "datafusion 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "datafusion-sqllogictest", "futures", "indicatif", @@ -10139,7 +10484,7 @@ dependencies = [ "clap", "console_error_panic_hook", "crossterm 0.29.0", - "datafusion", + "datafusion 53.1.0 (git+https://github.com/apache/datafusion?branch=branch-54)", "env_logger", "flatbuffers", "futures", @@ -11082,7 +11427,7 @@ dependencies = [ "memchr", "pbkdf2", "ppmd-rust", - "sha1 0.11.0", + "sha1", "time", "typed-path", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index fb87a953154..dd547e9629f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,20 +132,22 @@ cudarc = { version = "0.19.0", features = [ ] } custom-labels = "0.4.4" dashmap = "6.1.0" -datafusion = { version = "53", default-features = false, features = ["sql"] } -datafusion-catalog = { version = "53" } -datafusion-common = { version = "53" } -datafusion-common-runtime = { version = "53" } -datafusion-datasource = { version = "53", default-features = false } -datafusion-execution = { version = "53" } -datafusion-expr = { version = "53" } -datafusion-functions = { version = "53" } -datafusion-physical-expr = { version = "53" } -datafusion-physical-expr-adapter = { version = "53" } -datafusion-physical-expr-common = { version = "53" } -datafusion-physical-plan = { version = "53" } -datafusion-pruning = { version = "53" } -datafusion-sqllogictest = { version = "53" } +datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-54", default-features = false, features = [ + "sql", +] } +datafusion-catalog = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-common-runtime = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-datasource = { git = "https://github.com/apache/datafusion", branch = "branch-54", default-features = false } +datafusion-execution = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-functions = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-physical-expr = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-physical-plan = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-pruning = { git = "https://github.com/apache/datafusion", branch = "branch-54" } +datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", branch = "branch-54" } dirs = "6.0.0" divan = { package = "codspeed-divan-compat", version = "4.0.4" } enum-iterator = "2.0.0" diff --git a/benchmarks/datafusion-bench/src/lib.rs b/benchmarks/datafusion-bench/src/lib.rs index 469c9be6177..c8353eb1f85 100644 --- a/benchmarks/datafusion-bench/src/lib.rs +++ b/benchmarks/datafusion-bench/src/lib.rs @@ -14,7 +14,7 @@ use datafusion::datasource::provider::DefaultTableFactory; use datafusion::execution::SessionStateBuilder; use datafusion::execution::cache::DefaultListFilesCache; use datafusion::execution::cache::cache_manager::CacheManagerConfig; -use datafusion::execution::cache::cache_unit::DefaultFileStatisticsCache; +use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::prelude::SessionConfig; use datafusion::prelude::SessionContext; @@ -37,7 +37,7 @@ pub fn get_session_context() -> SessionContext { let file_static_cache = Arc::new(DefaultFileStatisticsCache::default()); let list_file_cache = Arc::new(DefaultListFilesCache::default()); let cache_config = CacheManagerConfig::default() - .with_files_statistics_cache(Some(file_static_cache)) + .with_file_statistics_cache(Some(file_static_cache)) .with_list_files_cache(Some(list_file_cache)); rt_builder = rt_builder.with_cache_manager(cache_config); diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index 6a9ac0fdc52..b6d741fc5ec 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -127,7 +127,6 @@ impl DefaultExpressionConvertor { let mut result = self.convert(source_expr.as_ref())?; for expr in field_names { let field_name = expr - .as_any() .downcast_ref::() .ok_or_else(|| exec_datafusion_err!("get_field field name must be a literal"))? .value() @@ -195,7 +194,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { fn convert(&self, df: &dyn PhysicalExpr) -> DFResult { // TODO(joe): Don't return an error when we have an unsupported node, bubble up "TRUE" as in keep // for that node, up to any `and` or `or` node. - if let Some(binary_expr) = df.as_any().downcast_ref::() { + if let Some(binary_expr) = df.downcast_ref::() { let left = self.convert(binary_expr.left().as_ref())?; let right = self.convert(binary_expr.right().as_ref())?; let operator = try_operator_from_df(binary_expr.op())?; @@ -203,11 +202,11 @@ impl ExpressionConvertor for DefaultExpressionConvertor { return Ok(Binary.new_expr(operator, [left, right])); } - if let Some(col_expr) = df.as_any().downcast_ref::() { + if let Some(col_expr) = df.downcast_ref::() { return Ok(get_item(col_expr.name().to_owned(), root())); } - if let Some(like) = df.as_any().downcast_ref::() { + if let Some(like) = df.downcast_ref::() { let child = self.convert(like.expr().as_ref())?; let pattern = self.convert(like.pattern().as_ref())?; return Ok(Like.new_expr( @@ -219,42 +218,34 @@ impl ExpressionConvertor for DefaultExpressionConvertor { )); } - if let Some(literal) = df.as_any().downcast_ref::() { + if let Some(literal) = df.downcast_ref::() { let value = Scalar::from_df(literal.value()); return Ok(lit(value)); } - if let Some(cast_expr) = df.as_any().downcast_ref::() { - let cast_dtype = DType::from_arrow((cast_expr.cast_type(), Nullability::Nullable)); + if let Some(cast_expr) = df.downcast_ref::() { + let cast_dtype = DType::from_arrow(cast_expr.target_field().as_ref()); let child = self.convert(cast_expr.expr().as_ref())?; return Ok(cast(child, cast_dtype)); } - if let Some(cast_col_expr) = df.as_any().downcast_ref::() { - let target = cast_col_expr.target_field(); - - let target_dtype = DType::from_arrow((target.data_type(), target.is_nullable().into())); - let child = self.convert(cast_col_expr.expr().as_ref())?; - return Ok(cast(child, target_dtype)); - } - - if let Some(is_null_expr) = df.as_any().downcast_ref::() { + if let Some(is_null_expr) = df.downcast_ref::() { let arg = self.convert(is_null_expr.arg().as_ref())?; return Ok(is_null(arg)); } - if let Some(is_not_null_expr) = df.as_any().downcast_ref::() { + if let Some(is_not_null_expr) = df.downcast_ref::() { let arg = self.convert(is_not_null_expr.arg().as_ref())?; return Ok(is_not_null(arg)); } - if let Some(in_list) = df.as_any().downcast_ref::() { + if let Some(in_list) = df.downcast_ref::() { let value = self.convert(in_list.expr().as_ref())?; let list_elements: Vec<_> = in_list .list() .iter() .map(|e| { - if let Some(lit) = e.as_any().downcast_ref::() { + if let Some(lit) = e.downcast_ref::() { Ok(Scalar::from_df(lit.value())) } else { Err(exec_datafusion_err!("Failed to cast sub-expression")) @@ -272,11 +263,11 @@ impl ExpressionConvertor for DefaultExpressionConvertor { return Ok(if in_list.negated() { not(expr) } else { expr }); } - if let Some(scalar_fn) = df.as_any().downcast_ref::() { + if let Some(scalar_fn) = df.downcast_ref::() { return self.try_convert_scalar_function(scalar_fn); } - if let Some(case_expr) = df.as_any().downcast_ref::() { + if let Some(case_expr) = df.downcast_ref::() { return self.try_convert_case_expr(case_expr); } @@ -297,7 +288,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { for projection_expr in source_projection.iter() { let r = projection_expr.expr.apply(|node| { // We only pull column children of scalar functions that we can't push into the scan. - if let Some(scalar_fn_expr) = node.as_any().downcast_ref::() + if let Some(scalar_fn_expr) = node.downcast_ref::() && !can_scalar_fn_be_pushed_down(scalar_fn_expr) { scan_projection.extend( @@ -312,7 +303,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { // DataFusion assumes different decimal types can be coerced. // Vortex expects a perfect match so we don't push it down. - if let Some(binary_expr) = node.as_any().downcast_ref::() + if let Some(binary_expr) = node.downcast_ref::() && binary_expr.op().is_numerical_operators() && (is_decimal(&binary_expr.left().data_type(input_schema)?) && is_decimal(&binary_expr.right().data_type(input_schema)?)) @@ -406,14 +397,13 @@ fn try_operator_from_df(value: &DFOperator) -> DFResult { } } -fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> bool { +fn can_be_pushed_down_impl(expr: &Arc, schema: &Schema) -> bool { // We currently do not support pushdown of dynamic expressions in DF. // See issue: https://github.com/vortex-data/vortex/issues/4034 - if is_dynamic_physical_expr(df_expr) { + if is_dynamic_physical_expr(expr) { return false; } - let expr = df_expr.as_any(); if let Some(binary) = expr.downcast_ref::() { can_binary_be_pushed_down(binary, schema) } else if let Some(col) = expr.downcast_ref::() { @@ -429,9 +419,6 @@ fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> } else if let Some(cast_expr) = expr.downcast_ref::() { // CastExpr child must be an expression type that convert() can handle is_convertible_expr(cast_expr.expr()) - } else if let Some(cast_col_expr) = expr.downcast_ref::() { - // CastColumnExpr child must be an expression type that convert() can handle - is_convertible_expr(cast_col_expr.expr()) } else if let Some(is_null) = expr.downcast_ref::() { can_be_pushed_down_impl(is_null.arg(), schema) } else if let Some(is_not_null) = expr.downcast_ref::() { @@ -447,7 +434,7 @@ fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> } else if let Some(case_expr) = expr.downcast_ref::() { can_case_be_pushed_down(case_expr, schema) } else { - tracing::debug!(%df_expr, "DataFusion expression can't be pushed down"); + tracing::debug!(%expr, "DataFusion expression can't be pushed down"); false } } @@ -455,9 +442,7 @@ fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> /// Checks if an expression type is one that convert() can handle. /// This is less restrictive than can_be_pushed_down since it only checks /// expression types, not data type support. -fn is_convertible_expr(df_expr: &Arc) -> bool { - let expr = df_expr.as_any(); - +fn is_convertible_expr(expr: &Arc) -> bool { // Expression types that convert() handles expr.downcast_ref::().is_some() || expr.downcast_ref::().is_some() @@ -466,9 +451,6 @@ fn is_convertible_expr(df_expr: &Arc) -> bool { || expr .downcast_ref::() .is_some_and(|e| is_convertible_expr(e.expr())) - || expr - .downcast_ref::() - .is_some_and(|e| is_convertible_expr(e.expr())) || expr.downcast_ref::().is_some() || expr.downcast_ref::().is_some() || expr.downcast_ref::().is_some() @@ -568,6 +550,8 @@ mod tests { use arrow_schema::Field; use arrow_schema::Schema; use arrow_schema::TimeUnit as ArrowTimeUnit; + use datafusion::arrow::array::AsArray; + use datafusion::arrow::datatypes::Int32Type; use datafusion_common::ScalarValue; use datafusion_expr::Operator as DFOperator; use datafusion_physical_expr::PhysicalExpr; @@ -988,12 +972,7 @@ mod tests { let vortex_as_arrow = vortex_result.into_primitive().as_slice::().to_vec(); // Convert DataFusion result to Vec for comparison - let df_as_arrow: Vec = df_array - .as_any() - .downcast_ref::() - .unwrap() - .values() - .to_vec(); + let df_as_arrow: Vec = df_array.as_primitive::().values().to_vec(); // Compare results // Expected: [0, 0, 50, 100, 100] for values [1, 5, 10, 15, 20] diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index b00dfadaaf0..e1b1775a1d2 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::any::Any; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -299,10 +298,6 @@ impl FileFormatFactory for VortexFormatFactory { fn default(&self) -> Arc { Arc::new(VortexFormat::new(self.session.clone())) } - - fn as_any(&self) -> &dyn Any { - self - } } impl VortexFormat { @@ -330,10 +325,6 @@ impl VortexFormat { #[async_trait] impl FileFormat for VortexFormat { - fn as_any(&self) -> &dyn Any { - self - } - fn compression_type(&self) -> Option { None } @@ -593,7 +584,6 @@ impl FileFormat for VortexFormat { ) -> DFResult> { let mut source = file_scan_config .file_source() - .as_any() .downcast_ref::() .cloned() .ok_or_else(|| internal_datafusion_err!("Expected VortexSource"))?; diff --git a/vortex-datafusion/src/persistent/metrics.rs b/vortex-datafusion/src/persistent/metrics.rs index d4c6456d6fa..e3bb1b18868 100644 --- a/vortex-datafusion/src/persistent/metrics.rs +++ b/vortex-datafusion/src/persistent/metrics.rs @@ -73,16 +73,13 @@ impl VortexMetricsFinder { impl ExecutionPlanVisitor for VortexMetricsFinder { type Error = std::convert::Infallible; fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result { - if let Some(exec) = plan.as_any().downcast_ref::() { + if let Some(exec) = plan.downcast_ref::() { // Start with exec metrics or create a new set let mut set = exec.metrics().unwrap_or_default(); // Include our own metrics from VortexSource - if let Some(file_scan) = exec.data_source().as_any().downcast_ref::() - && let Some(scan) = file_scan - .file_source - .as_any() - .downcast_ref::() + if let Some(file_scan) = exec.data_source().downcast_ref::() + && let Some(scan) = file_scan.file_source.downcast_ref::() { for metric in scan .metrics_registry() @@ -240,7 +237,7 @@ mod tests { &mut self, plan: &dyn datafusion_physical_plan::ExecutionPlan, ) -> Result { - if plan.as_any().downcast_ref::().is_some() { + if plan.downcast_ref::().is_some() { self.0 += 1; Ok(false) } else { diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 8a579c8557d..af3df1f72b8 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -314,9 +314,7 @@ impl FileOpener for VortexOpener { let mut scan_builder = ScanBuilder::new(session.clone(), Arc::clone(&layout_reader)); - if let Some(extensions) = file.extensions - && let Some(vortex_plan) = extensions.downcast_ref::() - { + if let Some(vortex_plan) = file.extensions.get::() { scan_builder = vortex_plan.apply_to_builder(scan_builder); } @@ -1149,9 +1147,12 @@ mod tests { let schema = batch.schema(); let mut file = PartitionedFile::new(file_path.to_string(), data_size); - file.extensions = Some(Arc::new(VortexAccessPlan::default().with_selection( - Selection::IncludeByIndex(Buffer::from_iter(vec![1, 3, 5, 7])), - ))); + file.extensions + .insert( + VortexAccessPlan::default().with_selection(Selection::IncludeByIndex( + Buffer::from_iter(vec![1, 3, 5, 7]), + )), + ); let opener = make_test_opener( Arc::clone(&object_store), @@ -1190,9 +1191,12 @@ mod tests { let schema = batch.schema(); let mut file = PartitionedFile::new(file_path.to_string(), data_size); - file.extensions = Some(Arc::new(VortexAccessPlan::default().with_selection( - Selection::ExcludeByIndex(Buffer::from_iter(vec![0, 2, 4, 6, 8])), - ))); + file.extensions + .insert( + VortexAccessPlan::default().with_selection(Selection::ExcludeByIndex( + Buffer::from_iter(vec![0, 2, 4, 6, 8]), + )), + ); let opener = make_test_opener( Arc::clone(&object_store), @@ -1234,9 +1238,8 @@ mod tests { let schema = batch.schema(); let mut file = PartitionedFile::new(file_path.to_string(), data_size); - file.extensions = Some(Arc::new( - VortexAccessPlan::default().with_selection(Selection::All), - )); + file.extensions + .insert(VortexAccessPlan::default().with_selection(Selection::All)); let opener = make_test_opener( Arc::clone(&object_store), diff --git a/vortex-datafusion/src/persistent/sink.rs b/vortex-datafusion/src/persistent/sink.rs index b05cbe84215..51cb60953da 100644 --- a/vortex-datafusion/src/persistent/sink.rs +++ b/vortex-datafusion/src/persistent/sink.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::any::Any; use std::sync::Arc; use arrow_schema::SchemaRef; @@ -69,10 +68,6 @@ impl DisplayAs for VortexSink { #[async_trait] impl DataSink for VortexSink { - fn as_any(&self) -> &dyn Any { - self - } - fn metrics(&self) -> Option { None } diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index aed2024d5d9..cf0c084ab9b 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::any::Any; use std::fmt::Formatter; use std::ops::Range; use std::sync::Arc; @@ -9,6 +8,7 @@ use std::sync::Weak; use datafusion_common::Result as DFResult; use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_datasource::TableSchema; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_scan_config::FileScanConfig; @@ -361,10 +361,6 @@ impl FileSource for VortexSource { )?)) } - fn as_any(&self) -> &dyn Any { - self - } - fn with_batch_size(&self, batch_size: usize) -> Arc { let mut source = self.clone(); source.batch_size = Some(batch_size); @@ -489,6 +485,24 @@ impl FileSource for VortexSource { fn table_schema(&self) -> &TableSchema { &self.table_schema } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&dyn PhysicalExpr) -> DFResult, + ) -> DFResult { + // Visit predicate (filter) expression if present + let mut tnr = TreeNodeRecursion::Continue; + if let Some(predicate) = &self.vortex_predicate { + tnr = tnr.visit_sibling(|| f(predicate.as_ref()))?; + } + + // Visit projection expressions + for proj_expr in &self.projection { + tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?; + } + + Ok(tnr) + } } #[cfg(test)] diff --git a/vortex-datafusion/src/v2/source.rs b/vortex-datafusion/src/v2/source.rs index c51c01bc946..7f43bbee2b5 100644 --- a/vortex-datafusion/src/v2/source.rs +++ b/vortex-datafusion/src/v2/source.rs @@ -67,7 +67,6 @@ //! [`DataSourceRef`]: vortex::scan::DataSourceRef //! [`ScanRequest`]: vortex::scan::ScanRequest -use std::any::Any; use std::fmt; use std::fmt::Formatter; use std::sync::Arc; @@ -82,6 +81,7 @@ use datafusion_common::Statistics; use datafusion_common::arrow::array::AsArray; use datafusion_common::arrow::array::RecordBatch; use datafusion_common::stats::Precision as DFPrecision; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_datasource::source::DataSource; use datafusion_execution::SendableRecordBatchStream; use datafusion_execution::TaskContext; @@ -469,10 +469,6 @@ impl DataSource for VortexDataSource { ))) } - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { write!( f, @@ -509,7 +505,7 @@ impl DataSource for VortexDataSource { EquivalenceProperties::new(Arc::clone(&self.leftover_schema)) } - fn partition_statistics(&self, _partition: Option) -> DFResult { + fn partition_statistics(&self, _partition: Option) -> DFResult> { // FIXME(ngates): this should be adjusted based on filters. See DuckDB for heuristics, // and in the future, store the selectivity stats in the session. let num_rows = estimate_to_df_precision(self.data_source.row_count().as_ref()); @@ -521,11 +517,11 @@ impl DataSource for VortexDataSource { // from the initial schema after try_swapping_with_projection adds computed columns. let column_statistics = self.leftover_statistics.clone(); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows, total_byte_size, column_statistics, - }) + })) } fn with_fetch(&self, limit: Option) -> Option> { @@ -657,6 +653,14 @@ impl DataSource for VortexDataSource { .with_updated_node(Arc::new(this) as _), ) } + + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&dyn PhysicalExpr) -> DFResult, + ) -> DFResult { + // Its unclear to me if this is the desired behavior here. + Ok(TreeNodeRecursion::Continue) + } } /// Convert a Vortex [`Option`] to a DataFusion diff --git a/vortex-datafusion/src/v2/table.rs b/vortex-datafusion/src/v2/table.rs index 9d7ae32713d..f131afca539 100644 --- a/vortex-datafusion/src/v2/table.rs +++ b/vortex-datafusion/src/v2/table.rs @@ -7,7 +7,6 @@ //! [`DataSourceRef`]: vortex::scan::DataSourceRef //! [`TableProvider`]: datafusion_catalog::TableProvider -use std::any::Any; use std::fmt; use std::sync::Arc; @@ -107,10 +106,6 @@ impl VortexTable { #[async_trait] impl TableProvider for VortexTable { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::clone(&self.arrow_schema) }