Add TPC-H to sqllogictests

.github/workflows/ci.yml

@@ -821,8 +821,13 @@ jobs:
         uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
+        with:
+          sync: false
       - name: Run sqllogictest tests
         run: |
+          ./vortex-sqllogictest/slt/tpch/generate_data.sh
           cargo test -p vortex-sqllogictest --test sqllogictests
 
 

Cargo.toml

@@ -99,6 +99,7 @@ async-fs = "2.2.0"
 async-lock = "3.4"
 async-stream = "0.3.6"
 async-trait = "0.1.89"
+bigdecimal = "0.4.8"
 bindgen = "0.72.0"
 bit-vec = "0.8.0"
 bitvec = "1.0.1"

_typos.toml

@@ -8,7 +8,7 @@ extend-ignore-re = [
 ]
 
 [files]
-extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**"]
+extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**", "vortex-sqllogictest/slt/**"]
 
 [type.py]
 extend-ignore-identifiers-re = [

vortex-duckdb/src/convert/scalar.rs

@@ -253,6 +253,9 @@ impl<'a> TryFrom<&'a ValueRef> for Scalar {
             ExtractedValue::HugeInt(_) => {
                 vortex_bail!("DuckDB HugeInt is not yet supported in Vortex");
             }
+            ExtractedValue::UHugeInt(_) => {
+                vortex_bail!("DuckDB UHugeInt is not yet supported in Vortex");
+            }
             ExtractedValue::UTinyInt(v) => Ok(Scalar::primitive(v, Nullable)),
             ExtractedValue::USmallInt(v) => Ok(Scalar::primitive(v, Nullable)),
             ExtractedValue::UInteger(v) => Ok(Scalar::primitive(v, Nullable)),

vortex-duckdb/src/convert/vector.rs

@@ -348,7 +348,7 @@ pub fn flat_vector_to_vortex(vector: &VectorRef, len: usize) -> VortexResult<Arr
             StructArray::try_new(names, children, len, vector.validity_ref(len).to_validity())
                 .map(|a| a.into_array())
         }
-        _ => todo!("missing impl for {type_id:?}"),
+        _ => unimplemented!("missing impl for {type_id:?}"),
     }
 }
 

vortex-duckdb/src/duckdb/logical_type.rs

@@ -166,6 +166,10 @@ impl LogicalType {
     pub fn timestamp_tz() -> Self {
         Self::new(DUCKDB_TYPE::DUCKDB_TYPE_TIMESTAMP_TZ)
     }
+
+    pub fn date() -> Self {
+        Self::new(DUCKDB_TYPE::DUCKDB_TYPE_DATE)
+    }
 }
 
 impl LogicalTypeRef {
@@ -182,6 +186,10 @@ impl LogicalTypeRef {
         }
     }
 
+    pub fn is_decimal(&self) -> bool {
+        matches!(self.as_type_id(), DUCKDB_TYPE::DUCKDB_TYPE_DECIMAL)
+    }
+
     pub fn array_child_type(&self) -> LogicalType {
         unsafe { LogicalType::own(duckdb_array_type_child_type(self.as_ptr())) }
     }

vortex-duckdb/src/duckdb/query_result.rs

@@ -3,11 +3,13 @@
 
 use std::ffi::CStr;
 
+use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
 use vortex::error::vortex_bail;
 use vortex::error::vortex_err;
 
 use crate::cpp;
+use crate::cpp::DUCKDB_TYPE;
 use crate::duckdb::DataChunk;
 use crate::lifetime_wrapper;
 
@@ -71,7 +73,15 @@ impl QueryResultRef {
     /// Get the type of a column by index.
     pub fn column_type(&self, col_idx: usize) -> LogicalType {
         let dtype = unsafe { cpp::duckdb_column_type(self.as_ptr(), col_idx as u64) };
-        LogicalType::new(dtype)
+        if dtype == DUCKDB_TYPE::DUCKDB_TYPE_DECIMAL {
+            let lt = unsafe { cpp::duckdb_column_logical_type(self.as_ptr(), col_idx as u64) };
+            let precision = unsafe { cpp::duckdb_decimal_width(lt) };
+            let scale = unsafe { cpp::duckdb_decimal_scale(lt) };
+
+            LogicalType::decimal_type(precision, scale).vortex_expect("valid decimal")
+        } else {
+            LogicalType::new(dtype)
+        }
     }
 }
 

vortex-duckdb/src/duckdb/value.rs

@@ -59,6 +59,10 @@ impl ValueRef {
             DUCKDB_TYPE::DUCKDB_TYPE_BIGINT => {
                 ExtractedValue::BigInt(unsafe { cpp::duckdb_get_int64(self.as_ptr()) })
             }
+            DUCKDB_TYPE::DUCKDB_TYPE_HUGEINT => {
+                let huge_int = unsafe { cpp::duckdb_get_hugeint(self.as_ptr()) };
+                ExtractedValue::HugeInt(i128_from_parts(huge_int.upper, huge_int.lower))
+            }
             DUCKDB_TYPE::DUCKDB_TYPE_UTINYINT => {
                 ExtractedValue::UTinyInt(unsafe { cpp::duckdb_get_uint8(self.as_ptr()) })
             }
@@ -71,6 +75,10 @@ impl ValueRef {
             DUCKDB_TYPE::DUCKDB_TYPE_UBIGINT => {
                 ExtractedValue::UBigInt(unsafe { cpp::duckdb_get_uint64(self.as_ptr()) })
             }
+            DUCKDB_TYPE::DUCKDB_TYPE_UHUGEINT => {
+                let huge_uint = unsafe { cpp::duckdb_get_uhugeint(self.as_ptr()) };
+                ExtractedValue::UHugeInt(u128_from_parts(huge_uint.upper, huge_uint.lower))
+            }
             DUCKDB_TYPE::DUCKDB_TYPE_FLOAT => {
                 ExtractedValue::Float(unsafe { cpp::duckdb_get_float(self.as_ptr()) })
             }
@@ -149,7 +157,7 @@ impl ValueRef {
                     .collect::<Vec<_>>(),
             ),
             // ...other types remain unimplemented..
-            _ => vortex_panic!("Unsupported DuckDB value type {:?}", self),
+            other => vortex_panic!("Unsupported DuckDB value type {other:?}"),
         }
     }
 }
@@ -264,6 +272,11 @@ pub fn i128_from_parts(high: i64, low: u64) -> i128 {
     ((high as i128) << 64) | (low as i128)
 }
 
+#[inline]
+pub fn u128_from_parts(high: u64, low: u64) -> u128 {
+    ((high as u128) << 64) | (low as u128)
+}
+
 impl<T> TryFrom<Option<T>> for Value
 where
     T: Into<Value> + NativeDType,
@@ -376,6 +389,7 @@ pub enum ExtractedValue {
     USmallInt(u16),
     UInteger(u32),
     UBigInt(u64),
+    UHugeInt(u128),
     Float(f32),
     Double(f64),
     Boolean(bool),
@@ -394,6 +408,7 @@ pub enum ExtractedValue {
 #[cfg(test)]
 mod tests {
     use crate::duckdb::i128_from_parts;
+    use crate::duckdb::u128_from_parts;
 
     #[test]
     fn test_huge_int_from_parts() {
@@ -408,4 +423,17 @@ mod tests {
             (1i128 << 64) + (u64::MAX as i128)
         );
     }
+
+    #[test]
+    fn test_uhuge_int_from_parts() {
+        assert_eq!(u128_from_parts(0, 0), 0u128);
+        assert_eq!(u128_from_parts(0, 34534912), 34534912u128);
+        assert_eq!(u128_from_parts(0, u64::MAX), u64::MAX as u128);
+        assert_eq!(u128_from_parts(u64::MAX, u64::MAX), u128::MAX);
+        assert_eq!(
+            u128_from_parts(1, u64::MAX),
+            (1u128 << 64) + (u64::MAX as u128)
+        );
+        assert_eq!(u128_from_parts(1, 0), 1u128 << 64);
+    }
 }

vortex-duckdb/src/multi_file.rs

@@ -39,6 +39,7 @@ impl DataSourceTableFunction for VortexMultiFileScan {
 
         // Parse the URL and separate the base URL (keep scheme, host, etc.) from the path.
         let glob_url_str = glob_url_parameter.as_string();
+
         let glob_url = match Url::parse(glob_url_str.as_str()) {
             Ok(url) => Ok(url),
             Err(_) => Url::from_file_path(Path::new(glob_url_str.as_str()))

vortex-sqllogictest/Cargo.toml

@@ -16,11 +16,13 @@ version = { workspace = true }
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
+bigdecimal = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 datafusion = { workspace = true }
 datafusion-sqllogictest = { workspace = true }
 futures.workspace = true
 indicatif.workspace = true
+rstest = { workspace = true }
 sqllogictest = "0.28"
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"] }

vortex-sqllogictest/README.md

@@ -6,9 +6,10 @@ Different test files might run in parallel, but within the same file, the file w
 
 ## Running tests
 
-In order to run the tests, the current command is:
+In order to run the tests, you first need to generate TPC-H data (scale factor 0.1), the commands are:
 
 ```shell
+./vortex-sqllogictest/slt/tpch/generate_data.sh
 cargo test -p vortex-sqllogictest --test sqllogictests
 ```
 

vortex-sqllogictest/slt/tpch/.gitignore

@@ -0,0 +1 @@
+/data

vortex-sqllogictest/slt/tpch/datafusion/create.slt.no

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS supplier (
+        s_suppkey  BIGINT,
+        s_name VARCHAR,
+        s_address VARCHAR,
+        s_nationkey BIGINT,
+        s_phone VARCHAR,
+        s_acctbal DECIMAL(15, 2),
+        s_comment VARCHAR,
+        s_rev VARCHAR,
+) STORED AS VORTEX 
+LOCATION 'slt/tpch/data/supplier.vortex';
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS part (
+        p_partkey BIGINT,
+        p_name VARCHAR,
+        p_mfgr VARCHAR,
+        p_brand VARCHAR,
+        p_type VARCHAR,
+        p_size INTEGER,
+        p_container VARCHAR,
+        p_retailprice DECIMAL(15, 2),
+        p_comment VARCHAR,
+        p_rev VARCHAR,
+) STORED AS VORTEX 
+LOCATION 'slt/tpch/data/part.vortex';
+
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS partsupp (
+        ps_partkey BIGINT,
+        ps_suppkey BIGINT,
+        ps_availqty INTEGER,
+        ps_supplycost DECIMAL(15, 2),
+        ps_comment VARCHAR,
+        ps_rev VARCHAR,
+) STORED AS VORTEX 
+LOCATION 'slt/tpch/data/partsupp.vortex';
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS customer (
+        c_custkey BIGINT,
+        c_name VARCHAR,
+        c_address VARCHAR,
+        c_nationkey BIGINT,
+        c_phone VARCHAR,
+        c_acctbal DECIMAL(15, 2),
+        c_mktsegment VARCHAR,
+        c_comment VARCHAR,
+        c_rev VARCHAR,
+) STORED AS VORTEX 
+LOCATION 'slt/tpch/data/customer.vortex';
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS orders (
+        o_orderkey BIGINT,
+        o_custkey BIGINT,
+        o_orderstatus VARCHAR,
+        o_totalprice DECIMAL(15, 2),
+        o_orderdate DATE,
+        o_orderpriority VARCHAR,
+        o_clerk VARCHAR,
+        o_shippriority INTEGER,
+        o_comment VARCHAR,
+        o_rev VARCHAR,
+) STORED AS VORTEX 
+LOCATION 'slt/tpch/data/orders.vortex';
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS lineitem (
+        l_orderkey BIGINT,
+        l_partkey BIGINT,
+        l_suppkey BIGINT,
+        l_linenumber INTEGER,
+        l_quantity DECIMAL(15, 2),
+        l_extendedprice DECIMAL(15, 2),
+        l_discount DECIMAL(15, 2),
+        l_tax DECIMAL(15, 2),
+        l_returnflag VARCHAR,
+        l_linestatus VARCHAR,
+        l_shipdate DATE,
+        l_commitdate DATE,
+        l_receiptdate DATE,
+        l_shipinstruct VARCHAR,
+        l_shipmode VARCHAR,
+        l_comment VARCHAR,
+        l_rev VARCHAR,
+) STORED AS VORTEX 
+LOCATION 'slt/tpch/data/lineitem.vortex';
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS nation (
+        n_nationkey BIGINT,
+        n_name VARCHAR,
+        n_regionkey BIGINT,
+        n_comment VARCHAR,
+        n_rev VARCHAR,
+) STORED AS VORTEX
+LOCATION 'slt/tpch/data/nation.vortex';
+
+statement ok
+CREATE EXTERNAL TABLE IF NOT EXISTS region (
+        r_regionkey BIGINT,
+        r_name VARCHAR,
+        r_comment VARCHAR,
+        r_rev VARCHAR,
+) STORED AS VORTEX
+LOCATION 'slt/tpch/data/region.vortex';

vortex-sqllogictest/slt/tpch/datafusion/tcph.slt

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+include ../../setup.slt
+
+
+include ./create.slt.no
+
+include ../results/*.slt.no
+# include ../drop.slt.no

vortex-sqllogictest/slt/tpch/drop.slt.no

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+statement ok
+DROP TABLE supplier;
+
+statement ok
+DROP TABLE region;
+
+statement ok
+DROP TABLE  nation;
+
+statement ok
+DROP TABLE lineitem;
+
+statement ok
+DROP TABLE orders;
+
+statement ok
+DROP TABLE customer;
+
+statement ok
+DROP TABLE partsupp;
+
+statement ok
+DROP TABLE part;