From 23b0f2a76d143058a35940841c7095464e8b8976 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Fri, 29 May 2026 14:27:04 +0300 Subject: [PATCH 1/3] HIVE-29637: Iceberg: Incorrect Results for NULL Predicate on Partition Column --- .../mr/hive/HiveIcebergStorageHandler.java | 4 +- .../iceberg/mr/hive/IcebergTableUtil.java | 19 +++ .../iceberg_isnull_partition_pruning.q | 38 ++++++ .../iceberg_isnull_partition_pruning.q.out | 112 ++++++++++++++++++ 4 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 3a4b1f1f8d7d..b40672936bf7 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2316,8 +2316,8 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta PartitionData partitionData = IcebergTableUtil.toPartitionData(task.partition(), spec.partitionType()); String partName = spec.partitionToPath(partitionData); - Map partSpecMap = Maps.newLinkedHashMap(); - Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); + Map partSpecMap = + IcebergTableUtil.makeSpecFromName(partName, spec, partitionData); DummyPartition partition = new DummyPartition(hmsTable, partName, partSpecMap); partitions.add(partition); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index fe1a06d32acb..a1788ab4bce9 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -452,6 +452,25 @@ public static void performMetadataDelete(Table icebergTable, String branchName, deleteFiles.deleteFromRowFilter(exp).commit(); } + /** + * Parses an Iceberg partition path into a Hive-compatible spec map. + * Unlike {@link Warehouse#makeSpecFromName}, this correctly represents null partition values + * as {@code null} instead of the literal string "null". + */ + public static Map makeSpecFromName(String partName, + PartitionSpec spec, PartitionData data) { + Map partSpecMap = Maps.newLinkedHashMap(); + Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); + + List fields = spec.fields(); + for (int i = 0; i < fields.size(); i++) { + if (data.get(i, Object.class) == null) { + partSpecMap.put(fields.get(i).name(), null); + } + } + return partSpecMap; + } + public static PartitionData toPartitionData(StructLike key, Types.StructType keyType) { PartitionData keyTemplate = new PartitionData(keyType); return keyTemplate.copyFor(key); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q new file mode 100644 index 000000000000..1275614ae788 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q @@ -0,0 +1,38 @@ +CREATE TABLE sample01 ( + index INT, + date_col DATE, + timestamp_col TIMESTAMP, + str_col VARCHAR(24), + string_col STRING, + double_col DOUBLE, + float_col FLOAT, + decimal_col DECIMAL(9,3), + tinyint_col TINYINT, + smallint_col SMALLINT, + int_col INT, + bigint_col BIGINT, + boolean_col BOOLEAN +); + +INSERT INTO sample01 VALUES +(1003,"1969-10-27","1993-05-17 07:39:58.375409",NULL,"sloppy bronze hare",-181.01933598375618,-181.019336,-999999.999,-128,-32768,-2147483648,-9223372036854775808,false); + +CREATE EXTERNAL TABLE ice01 ( + index INT, + date_col DATE, + timestamp_col TIMESTAMP, + string_col STRING, + double_col DOUBLE, + float_col FLOAT, + decimal_col DECIMAL(9,3), + smallint_col int, + int_col INT, + bigint_col BIGINT, + boolean_col BOOLEAN +) PARTITIONED BY (str_col String, tinyint_col int) +STORED BY iceberg; + +INSERT INTO ice01 PARTITION (str_col, tinyint_col) + SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01; + +SELECT * FROM ice01 WHERE str_col is NULL; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out new file mode 100644 index 000000000000..d881c520f1e7 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out @@ -0,0 +1,112 @@ +PREHOOK: query: CREATE TABLE sample01 ( + index INT, + date_col DATE, + timestamp_col TIMESTAMP, + str_col VARCHAR(24), + string_col STRING, + double_col DOUBLE, + float_col FLOAT, + decimal_col DECIMAL(9,3), + tinyint_col TINYINT, + smallint_col SMALLINT, + int_col INT, + bigint_col BIGINT, + boolean_col BOOLEAN +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sample01 +POSTHOOK: query: CREATE TABLE sample01 ( + index INT, + date_col DATE, + timestamp_col TIMESTAMP, + str_col VARCHAR(24), + string_col STRING, + double_col DOUBLE, + float_col FLOAT, + decimal_col DECIMAL(9,3), + tinyint_col TINYINT, + smallint_col SMALLINT, + int_col INT, + bigint_col BIGINT, + boolean_col BOOLEAN +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sample01 +PREHOOK: query: INSERT INTO sample01 VALUES +(1003,"1969-10-27","1993-05-17 07:39:58.375409",NULL,"sloppy bronze hare",-181.01933598375618,-181.019336,-999999.999,-128,-32768,-2147483648,-9223372036854775808,false) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@sample01 +POSTHOOK: query: INSERT INTO sample01 VALUES +(1003,"1969-10-27","1993-05-17 07:39:58.375409",NULL,"sloppy bronze hare",-181.01933598375618,-181.019336,-999999.999,-128,-32768,-2147483648,-9223372036854775808,false) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@sample01 +POSTHOOK: Lineage: sample01.bigint_col SCRIPT [] +POSTHOOK: Lineage: sample01.boolean_col SCRIPT [] +POSTHOOK: Lineage: sample01.date_col SCRIPT [] +POSTHOOK: Lineage: sample01.decimal_col SCRIPT [] +POSTHOOK: Lineage: sample01.double_col SCRIPT [] +POSTHOOK: Lineage: sample01.float_col SCRIPT [] +POSTHOOK: Lineage: sample01.index SCRIPT [] +POSTHOOK: Lineage: sample01.int_col SCRIPT [] +POSTHOOK: Lineage: sample01.smallint_col SCRIPT [] +POSTHOOK: Lineage: sample01.str_col EXPRESSION [] +POSTHOOK: Lineage: sample01.string_col SCRIPT [] +POSTHOOK: Lineage: sample01.timestamp_col SCRIPT [] +POSTHOOK: Lineage: sample01.tinyint_col SCRIPT [] +PREHOOK: query: CREATE EXTERNAL TABLE ice01 ( + index INT, + date_col DATE, + timestamp_col TIMESTAMP, + string_col STRING, + double_col DOUBLE, + float_col FLOAT, + decimal_col DECIMAL(9,3), + smallint_col int, + int_col INT, + bigint_col BIGINT, + boolean_col BOOLEAN +) PARTITIONED BY (str_col String, tinyint_col int) +STORED BY iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice01 +POSTHOOK: query: CREATE EXTERNAL TABLE ice01 ( + index INT, + date_col DATE, + timestamp_col TIMESTAMP, + string_col STRING, + double_col DOUBLE, + float_col FLOAT, + decimal_col DECIMAL(9,3), + smallint_col int, + int_col INT, + bigint_col BIGINT, + boolean_col BOOLEAN +) PARTITIONED BY (str_col String, tinyint_col int) +STORED BY iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice01 +PREHOOK: query: INSERT INTO ice01 PARTITION (str_col, tinyint_col) + SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01 +PREHOOK: type: QUERY +PREHOOK: Input: default@sample01 +PREHOOK: Output: default@ice01 +POSTHOOK: query: INSERT INTO ice01 PARTITION (str_col, tinyint_col) + SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sample01 +POSTHOOK: Output: default@ice01 +PREHOOK: query: SELECT * FROM ice01 WHERE str_col is NULL +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice01 WHERE str_col is NULL +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1003 1969-10-27 1993-05-17 07:39:58.375409 sloppy bronze hare -181.01933598375618 -181.01933 -999999.999 -32768 -2147483648 -9223372036854775808 false NULL -128 From d82a04d592b578a752e0a076329bf13364c45add Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 2 Jun 2026 11:54:06 +0300 Subject: [PATCH 2/3] review comments #2 --- .../iceberg/mr/hive/IcebergTableUtil.java | 5 ++-- .../iceberg_isnull_partition_pruning.q | 7 ++++- .../iceberg_isnull_partition_pruning.q.out | 28 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index a1788ab4bce9..c5ac4351cb93 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -457,14 +457,13 @@ public static void performMetadataDelete(Table icebergTable, String branchName, * Unlike {@link Warehouse#makeSpecFromName}, this correctly represents null partition values * as {@code null} instead of the literal string "null". */ - public static Map makeSpecFromName(String partName, - PartitionSpec spec, PartitionData data) { + public static Map makeSpecFromName(String partName, PartitionSpec spec, PartitionData data) { Map partSpecMap = Maps.newLinkedHashMap(); Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); List fields = spec.fields(); for (int i = 0; i < fields.size(); i++) { - if (data.get(i, Object.class) == null) { + if (data.get(i) == null) { partSpecMap.put(fields.get(i).name(), null); } } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q index 1275614ae788..2c4ca8453b27 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q @@ -35,4 +35,9 @@ STORED BY iceberg; INSERT INTO ice01 PARTITION (str_col, tinyint_col) SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01; -SELECT * FROM ice01 WHERE str_col is NULL; \ No newline at end of file +INSERT INTO ice01 PARTITION (str_col='null', tinyint_col=1) + VALUES (1004,"1970-01-01","1993-05-17 07:39:58.375409","some string",0.0,0.0,0.000,0,0,0,true); + +SELECT * FROM ice01 WHERE str_col is NULL; +SELECT * FROM ice01 WHERE str_col = 'null'; +SELECT * FROM ice01 WHERE str_col is NOT NULL; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out index d881c520f1e7..274c3e801c67 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out @@ -101,6 +101,16 @@ POSTHOOK: query: INSERT INTO ice01 PARTITION (str_col, tinyint_col) POSTHOOK: type: QUERY POSTHOOK: Input: default@sample01 POSTHOOK: Output: default@ice01 +PREHOOK: query: INSERT INTO ice01 PARTITION (str_col='null', tinyint_col=1) + VALUES (1004,"1970-01-01","1993-05-17 07:39:58.375409","some string",0.0,0.0,0.000,0,0,0,true) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01@tinyint_col=1/str_col=null +POSTHOOK: query: INSERT INTO ice01 PARTITION (str_col='null', tinyint_col=1) + VALUES (1004,"1970-01-01","1993-05-17 07:39:58.375409","some string",0.0,0.0,0.000,0,0,0,true) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01@tinyint_col=1/str_col=null PREHOOK: query: SELECT * FROM ice01 WHERE str_col is NULL PREHOOK: type: QUERY PREHOOK: Input: default@ice01 @@ -110,3 +120,21 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@ice01 POSTHOOK: Output: hdfs://### HDFS PATH ### 1003 1969-10-27 1993-05-17 07:39:58.375409 sloppy bronze hare -181.01933598375618 -181.01933 -999999.999 -32768 -2147483648 -9223372036854775808 false NULL -128 +PREHOOK: query: SELECT * FROM ice01 WHERE str_col = 'null' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice01 WHERE str_col = 'null' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1004 1970-01-01 1993-05-17 07:39:58.375409 some string 0.0 0.0 0.000 0 0 0 true null 1 +PREHOOK: query: SELECT * FROM ice01 WHERE str_col is NOT NULL +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice01 WHERE str_col is NOT NULL +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1004 1970-01-01 1993-05-17 07:39:58.375409 some string 0.0 0.0 0.000 0 0 0 true null 1 From e8c67d57ff6bf9525cc0cb7456db2612e4c9901e Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 2 Jun 2026 22:25:22 +0300 Subject: [PATCH 3/3] review comments #3 --- .../mr/hive/HiveIcebergStorageHandler.java | 3 +- .../iceberg/mr/hive/IcebergTableUtil.java | 10 +- .../iceberg_isnull_partition_pruning.q | 43 ----- .../positive/iceberg_pcr_null_partition.q | 18 ++ .../iceberg_isnull_partition_pruning.q.out | 140 -------------- .../positive/iceberg_pcr_null_partition.q.out | 150 +++++++++++++++ .../ql/optimizer/ppr/PartExprEvalUtils.java | 19 +- .../ql/optimizer/ppr/PartitionPruner.java | 2 +- .../clientpositive/pcr_null_partition.q | 17 ++ .../llap/annotate_stats_part.q.out | 40 ++-- .../llap/dynamic_partition_skip_default.q.out | 11 +- .../llap/pcr_null_partition.q.out | 175 ++++++++++++++++++ 12 files changed, 414 insertions(+), 214 deletions(-) delete mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_pcr_null_partition.q delete mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out create mode 100644 ql/src/test/queries/clientpositive/pcr_null_partition.q create mode 100644 ql/src/test/results/clientpositive/llap/pcr_null_partition.q.out diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b40672936bf7..6da2162df558 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2306,6 +2306,7 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta } Set partitions = Sets.newHashSet(); + String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULT_PARTITION_NAME); try (CloseableIterable tasks = scan.planFiles()) { FluentIterable.from(tasks) @@ -2317,7 +2318,7 @@ public List getPartitionsByExpr(org.apache.hadoop.hive.ql.metadata.Ta String partName = spec.partitionToPath(partitionData); Map partSpecMap = - IcebergTableUtil.makeSpecFromName(partName, spec, partitionData); + IcebergTableUtil.makeSpecFromName(partName, spec, partitionData, defaultPartitionName); DummyPartition partition = new DummyPartition(hmsTable, partName, partSpecMap); partitions.add(partition); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index c5ac4351cb93..0240135c92b7 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -453,18 +453,18 @@ public static void performMetadataDelete(Table icebergTable, String branchName, } /** - * Parses an Iceberg partition path into a Hive-compatible spec map. - * Unlike {@link Warehouse#makeSpecFromName}, this correctly represents null partition values - * as {@code null} instead of the literal string "null". + * Parses an Iceberg partition path into a Hive-compatible spec map, representing null partition + * values with the Hive default partition name. */ - public static Map makeSpecFromName(String partName, PartitionSpec spec, PartitionData data) { + public static Map makeSpecFromName(String partName, PartitionSpec spec, PartitionData data, + String defaultPartitionName) { Map partSpecMap = Maps.newLinkedHashMap(); Warehouse.makeSpecFromName(partSpecMap, new Path(partName), null); List fields = spec.fields(); for (int i = 0; i < fields.size(); i++) { if (data.get(i) == null) { - partSpecMap.put(fields.get(i).name(), null); + partSpecMap.put(fields.get(i).name(), defaultPartitionName); } } return partSpecMap; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q deleted file mode 100644 index 2c4ca8453b27..000000000000 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_isnull_partition_pruning.q +++ /dev/null @@ -1,43 +0,0 @@ -CREATE TABLE sample01 ( - index INT, - date_col DATE, - timestamp_col TIMESTAMP, - str_col VARCHAR(24), - string_col STRING, - double_col DOUBLE, - float_col FLOAT, - decimal_col DECIMAL(9,3), - tinyint_col TINYINT, - smallint_col SMALLINT, - int_col INT, - bigint_col BIGINT, - boolean_col BOOLEAN -); - -INSERT INTO sample01 VALUES -(1003,"1969-10-27","1993-05-17 07:39:58.375409",NULL,"sloppy bronze hare",-181.01933598375618,-181.019336,-999999.999,-128,-32768,-2147483648,-9223372036854775808,false); - -CREATE EXTERNAL TABLE ice01 ( - index INT, - date_col DATE, - timestamp_col TIMESTAMP, - string_col STRING, - double_col DOUBLE, - float_col FLOAT, - decimal_col DECIMAL(9,3), - smallint_col int, - int_col INT, - bigint_col BIGINT, - boolean_col BOOLEAN -) PARTITIONED BY (str_col String, tinyint_col int) -STORED BY iceberg; - -INSERT INTO ice01 PARTITION (str_col, tinyint_col) - SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01; - -INSERT INTO ice01 PARTITION (str_col='null', tinyint_col=1) - VALUES (1004,"1970-01-01","1993-05-17 07:39:58.375409","some string",0.0,0.0,0.000,0,0,0,true); - -SELECT * FROM ice01 WHERE str_col is NULL; -SELECT * FROM ice01 WHERE str_col = 'null'; -SELECT * FROM ice01 WHERE str_col is NOT NULL; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_pcr_null_partition.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_pcr_null_partition.q new file mode 100644 index 000000000000..c77e4500a720 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_pcr_null_partition.q @@ -0,0 +1,18 @@ +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.fetch.task.conversion=none; +set hive.explain.user=false; + +drop table if exists ice_01; +create external table ice_01 (key string, value string) partitioned by (ds string) stored by iceberg; + +insert into ice_01 partition (ds) select 'A', 'V1', '2000-04-08'; +insert into ice_01 partition (ds) select 'B', 'V2', 'null'; +insert into ice_01 partition (ds) select 'C', 'V3', null; + +explain select key, value, ds from ice_01 where ds is null; +select key, value, ds from ice_01 where ds is null; + +explain select key, value, ds from ice_01 where ds is not null; +select key, value, ds from ice_01 where ds is not null order by key; + +select key, value, ds from ice_01 where ds = 'null'; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out deleted file mode 100644 index 274c3e801c67..000000000000 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_isnull_partition_pruning.q.out +++ /dev/null @@ -1,140 +0,0 @@ -PREHOOK: query: CREATE TABLE sample01 ( - index INT, - date_col DATE, - timestamp_col TIMESTAMP, - str_col VARCHAR(24), - string_col STRING, - double_col DOUBLE, - float_col FLOAT, - decimal_col DECIMAL(9,3), - tinyint_col TINYINT, - smallint_col SMALLINT, - int_col INT, - bigint_col BIGINT, - boolean_col BOOLEAN -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@sample01 -POSTHOOK: query: CREATE TABLE sample01 ( - index INT, - date_col DATE, - timestamp_col TIMESTAMP, - str_col VARCHAR(24), - string_col STRING, - double_col DOUBLE, - float_col FLOAT, - decimal_col DECIMAL(9,3), - tinyint_col TINYINT, - smallint_col SMALLINT, - int_col INT, - bigint_col BIGINT, - boolean_col BOOLEAN -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@sample01 -PREHOOK: query: INSERT INTO sample01 VALUES -(1003,"1969-10-27","1993-05-17 07:39:58.375409",NULL,"sloppy bronze hare",-181.01933598375618,-181.019336,-999999.999,-128,-32768,-2147483648,-9223372036854775808,false) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@sample01 -POSTHOOK: query: INSERT INTO sample01 VALUES -(1003,"1969-10-27","1993-05-17 07:39:58.375409",NULL,"sloppy bronze hare",-181.01933598375618,-181.019336,-999999.999,-128,-32768,-2147483648,-9223372036854775808,false) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@sample01 -POSTHOOK: Lineage: sample01.bigint_col SCRIPT [] -POSTHOOK: Lineage: sample01.boolean_col SCRIPT [] -POSTHOOK: Lineage: sample01.date_col SCRIPT [] -POSTHOOK: Lineage: sample01.decimal_col SCRIPT [] -POSTHOOK: Lineage: sample01.double_col SCRIPT [] -POSTHOOK: Lineage: sample01.float_col SCRIPT [] -POSTHOOK: Lineage: sample01.index SCRIPT [] -POSTHOOK: Lineage: sample01.int_col SCRIPT [] -POSTHOOK: Lineage: sample01.smallint_col SCRIPT [] -POSTHOOK: Lineage: sample01.str_col EXPRESSION [] -POSTHOOK: Lineage: sample01.string_col SCRIPT [] -POSTHOOK: Lineage: sample01.timestamp_col SCRIPT [] -POSTHOOK: Lineage: sample01.tinyint_col SCRIPT [] -PREHOOK: query: CREATE EXTERNAL TABLE ice01 ( - index INT, - date_col DATE, - timestamp_col TIMESTAMP, - string_col STRING, - double_col DOUBLE, - float_col FLOAT, - decimal_col DECIMAL(9,3), - smallint_col int, - int_col INT, - bigint_col BIGINT, - boolean_col BOOLEAN -) PARTITIONED BY (str_col String, tinyint_col int) -STORED BY iceberg -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ice01 -POSTHOOK: query: CREATE EXTERNAL TABLE ice01 ( - index INT, - date_col DATE, - timestamp_col TIMESTAMP, - string_col STRING, - double_col DOUBLE, - float_col FLOAT, - decimal_col DECIMAL(9,3), - smallint_col int, - int_col INT, - bigint_col BIGINT, - boolean_col BOOLEAN -) PARTITIONED BY (str_col String, tinyint_col int) -STORED BY iceberg -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ice01 -PREHOOK: query: INSERT INTO ice01 PARTITION (str_col, tinyint_col) - SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01 -PREHOOK: type: QUERY -PREHOOK: Input: default@sample01 -PREHOOK: Output: default@ice01 -POSTHOOK: query: INSERT INTO ice01 PARTITION (str_col, tinyint_col) - SELECT index, date_col, timestamp_col, string_col, double_col, float_col, decimal_col, smallint_col, int_col, bigint_col, boolean_col, str_col, tinyint_col FROM sample01 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@sample01 -POSTHOOK: Output: default@ice01 -PREHOOK: query: INSERT INTO ice01 PARTITION (str_col='null', tinyint_col=1) - VALUES (1004,"1970-01-01","1993-05-17 07:39:58.375409","some string",0.0,0.0,0.000,0,0,0,true) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice01@tinyint_col=1/str_col=null -POSTHOOK: query: INSERT INTO ice01 PARTITION (str_col='null', tinyint_col=1) - VALUES (1004,"1970-01-01","1993-05-17 07:39:58.375409","some string",0.0,0.0,0.000,0,0,0,true) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice01@tinyint_col=1/str_col=null -PREHOOK: query: SELECT * FROM ice01 WHERE str_col is NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@ice01 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ice01 WHERE str_col is NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice01 -POSTHOOK: Output: hdfs://### HDFS PATH ### -1003 1969-10-27 1993-05-17 07:39:58.375409 sloppy bronze hare -181.01933598375618 -181.01933 -999999.999 -32768 -2147483648 -9223372036854775808 false NULL -128 -PREHOOK: query: SELECT * FROM ice01 WHERE str_col = 'null' -PREHOOK: type: QUERY -PREHOOK: Input: default@ice01 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ice01 WHERE str_col = 'null' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice01 -POSTHOOK: Output: hdfs://### HDFS PATH ### -1004 1970-01-01 1993-05-17 07:39:58.375409 some string 0.0 0.0 0.000 0 0 0 true null 1 -PREHOOK: query: SELECT * FROM ice01 WHERE str_col is NOT NULL -PREHOOK: type: QUERY -PREHOOK: Input: default@ice01 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ice01 WHERE str_col is NOT NULL -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice01 -POSTHOOK: Output: hdfs://### HDFS PATH ### -1004 1970-01-01 1993-05-17 07:39:58.375409 some string 0.0 0.0 0.000 0 0 0 true null 1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out new file mode 100644 index 000000000000..fae1fb8c26df --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_pcr_null_partition.q.out @@ -0,0 +1,150 @@ +PREHOOK: query: drop table if exists ice_01 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_01 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_01 (key string, value string) partitioned by (ds string) stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_01 +POSTHOOK: query: create external table ice_01 (key string, value string) partitioned by (ds string) stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_01 +PREHOOK: query: insert into ice_01 partition (ds) select 'A', 'V1', '2000-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_01 +POSTHOOK: query: insert into ice_01 partition (ds) select 'A', 'V1', '2000-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_01 +PREHOOK: query: insert into ice_01 partition (ds) select 'B', 'V2', 'null' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_01 +POSTHOOK: query: insert into ice_01 partition (ds) select 'B', 'V2', 'null' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_01 +PREHOOK: query: insert into ice_01 partition (ds) select 'C', 'V3', null +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_01 +POSTHOOK: query: insert into ice_01 partition (ds) select 'C', 'V3', null +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_01 +PREHOOK: query: explain select key, value, ds from ice_01 where ds is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select key, value, ds from ice_01 where ds is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_01 + filterExpr: ds is null (type: boolean) + Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), null (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key, value, ds from ice_01 where ds is null +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select key, value, ds from ice_01 where ds is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +C V3 NULL +PREHOOK: query: explain select key, value, ds from ice_01 where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select key, value, ds from ice_01 where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_01 + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key, value, ds from ice_01 where ds is not null order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select key, value, ds from ice_01 where ds is not null order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +A V1 2000-04-08 +B V2 null +PREHOOK: query: select key, value, ds from ice_01 where ds = 'null' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select key, value, ds from ice_01 where ds = 'null' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +B V2 null diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java index 91340b1b76ef..11f02e295b8a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java @@ -25,6 +25,7 @@ import java.util.Properties; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.ddl.DDLUtils; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -73,6 +75,9 @@ static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws Hiv throw new HiveException("Internal error : Partition Spec size, " + partSpec.size() + " doesn't match partition key definition size, " + partKeyTypes.length); } + String defaultPartitionName = HiveConf.getVar(SessionState.getSessionConf(), + HiveConf.ConfVars.DEFAULT_PARTITION_NAME); + // Create the row object List partNames = new ArrayList<>(); List partValues = new ArrayList<>(); @@ -82,9 +87,15 @@ static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws Hiv partNames.add(entry.getKey()); ObjectInspector oi = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector (TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i++])); - partValues.add(ObjectInspectorConverters.getConverter( - PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) - .convert(entry.getValue())); + + String partitionValue = entry.getValue(); + if (partitionValue.equals(defaultPartitionName)) { + partValues.add(null); // Null for default partition. + } else { + partValues.add(ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) + .convert(partitionValue)); + } partObjectInspectors.add(oi); } StructObjectInspector partObjectInspector = ObjectInspectorFactory @@ -104,7 +115,7 @@ public static Pair prepareExpr( ExprNodeDesc expr, List partColumnNames, List partColumnTypeInfos) throws HiveException { // Create the row object - List partObjectInspectors = new ArrayList(); + List partObjectInspectors = new ArrayList<>(); for (int i = 0; i < partColumnNames.size(); i++) { partObjectInspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( partColumnTypeInfos.get(i))); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 7574ad5f6d24..8839256f3093 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -563,7 +563,7 @@ public static boolean prunePartitionNames(List partColumnNames, Warehouse.makeValsFromName(partName, values); List convertedValues = new ArrayList<>(values.size()); - for(int i=0; i '2001') (type: boolean) - Select Operator - expressions: state (type: string), locid (type: int) - outputColumnNames: _col0, _col1 - ListSink + Filter Operator + predicate: (year <> '2001') (type: boolean) + Select Operator + expressions: state (type: string), locid (type: int) + outputColumnNames: _col0, _col1 + ListSink PREHOOK: query: explain select * from loc_orc_n4 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_skip_default.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_skip_default.q.out index deefd0129395..ce11a0b74da3 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_skip_default.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_skip_default.q.out @@ -276,8 +276,11 @@ STAGE PLANS: alias: dynamic_part_table filterExpr: ((partcol1 = '1') and (partcol2) IN ('1', '__HIVE_DEFAULT_PARTITION__')) (type: boolean) GatherStats: false - Select Operator - expressions: intcol (type: string) - outputColumnNames: _col0 - ListSink + Filter Operator + isSamplingPred: false + predicate: (partcol2) IN ('1', '__HIVE_DEFAULT_PARTITION__') (type: boolean) + Select Operator + expressions: intcol (type: string) + outputColumnNames: _col0 + ListSink diff --git a/ql/src/test/results/clientpositive/llap/pcr_null_partition.q.out b/ql/src/test/results/clientpositive/llap/pcr_null_partition.q.out new file mode 100644 index 000000000000..ca1d2ea23f32 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/pcr_null_partition.q.out @@ -0,0 +1,175 @@ +PREHOOK: query: drop table if exists pcr_t1 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists pcr_t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create table pcr_t1 (key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@pcr_t1 +POSTHOOK: query: create table pcr_t1 (key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@pcr_t1 +PREHOOK: query: insert into pcr_t1 partition (ds) select 'A', 'V1', '2000-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@pcr_t1 +POSTHOOK: query: insert into pcr_t1 partition (ds) select 'A', 'V1', '2000-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@pcr_t1 +POSTHOOK: Output: default@pcr_t1@ds=2000-04-08 +POSTHOOK: Lineage: pcr_t1 PARTITION(ds=2000-04-08).key SIMPLE [] +POSTHOOK: Lineage: pcr_t1 PARTITION(ds=2000-04-08).value SIMPLE [] +PREHOOK: query: insert into pcr_t1 partition (ds) select 'B', 'V2', 'null' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@pcr_t1 +POSTHOOK: query: insert into pcr_t1 partition (ds) select 'B', 'V2', 'null' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@pcr_t1 +POSTHOOK: Output: default@pcr_t1@ds=null +POSTHOOK: Lineage: pcr_t1 PARTITION(ds=null).key SIMPLE [] +POSTHOOK: Lineage: pcr_t1 PARTITION(ds=null).value SIMPLE [] +PREHOOK: query: insert into pcr_t1 partition (ds) select 'C', 'V3', null +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@pcr_t1 +POSTHOOK: query: insert into pcr_t1 partition (ds) select 'C', 'V3', null +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@pcr_t1 +POSTHOOK: Output: default@pcr_t1@ds=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: pcr_t1 PARTITION(ds=__HIVE_DEFAULT_PARTITION__).key SIMPLE [] +POSTHOOK: Lineage: pcr_t1 PARTITION(ds=__HIVE_DEFAULT_PARTITION__).value SIMPLE [] +PREHOOK: query: explain select key, value, ds from pcr_t1 where ds is null +PREHOOK: type: QUERY +PREHOOK: Input: default@pcr_t1 +PREHOOK: Input: default@pcr_t1@ds=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: explain select key, value, ds from pcr_t1 where ds is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@pcr_t1 +POSTHOOK: Input: default@pcr_t1@ds=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pcr_t1 + filterExpr: ds is null (type: boolean) + Statistics: Num rows: 1 Data size: 171 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), null (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key, value, ds from pcr_t1 where ds is null +PREHOOK: type: QUERY +PREHOOK: Input: default@pcr_t1 +PREHOOK: Input: default@pcr_t1@ds=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select key, value, ds from pcr_t1 where ds is null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@pcr_t1 +POSTHOOK: Input: default@pcr_t1@ds=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +C V3 NULL +PREHOOK: query: explain select key, value, ds from pcr_t1 where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@pcr_t1 +PREHOOK: Input: default@pcr_t1@ds=2000-04-08 +PREHOOK: Input: default@pcr_t1@ds=null +#### A masked pattern was here #### +POSTHOOK: query: explain select key, value, ds from pcr_t1 where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@pcr_t1 +POSTHOOK: Input: default@pcr_t1@ds=2000-04-08 +POSTHOOK: Input: default@pcr_t1@ds=null +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: pcr_t1 + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2 Data size: 710 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 710 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 710 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key, value, ds from pcr_t1 where ds is not null order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@pcr_t1 +PREHOOK: Input: default@pcr_t1@ds=2000-04-08 +PREHOOK: Input: default@pcr_t1@ds=null +#### A masked pattern was here #### +POSTHOOK: query: select key, value, ds from pcr_t1 where ds is not null order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@pcr_t1 +POSTHOOK: Input: default@pcr_t1@ds=2000-04-08 +POSTHOOK: Input: default@pcr_t1@ds=null +#### A masked pattern was here #### +A V1 2000-04-08 +B V2 null +PREHOOK: query: select key, value, ds from pcr_t1 where ds = 'null' +PREHOOK: type: QUERY +PREHOOK: Input: default@pcr_t1 +PREHOOK: Input: default@pcr_t1@ds=null +#### A masked pattern was here #### +POSTHOOK: query: select key, value, ds from pcr_t1 where ds = 'null' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@pcr_t1 +POSTHOOK: Input: default@pcr_t1@ds=null +#### A masked pattern was here #### +B V2 null