[WIP] HIVE-29641: Upgrade Calcite to 1.42.0

druid-handler/pom.xml

@@ -134,14 +134,6 @@
           <groupId>org.codehaus.plexus</groupId>
           <artifactId>plexus-utils</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.ow2.asm</groupId>
-          <artifactId>asm-commons</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.ow2.asm</groupId>
-          <artifactId>asm</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>org.checkerframework</groupId>
           <artifactId>checker-qual</artifactId>
@@ -316,6 +308,14 @@
       <version>${log4j2.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.ow2.asm</groupId>
+      <artifactId>asm</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.ow2.asm</groupId>
+      <artifactId>asm-commons</artifactId>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>
@@ -409,6 +409,18 @@
                 </configuration>
               </execution>
             </executions>
+            <dependencies>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm</artifactId>
+                <version>${asm.version}</version>
+              </dependency>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm-commons</artifactId>
+                <version>${asm.version}</version>
+              </dependency>
+            </dependencies>
           </plugin>
         </plugins>
       </build>

pom.xml

@@ -111,9 +111,10 @@
     <!-- Make sure to sync it with standalone-metastore/pom.xml -->
     <antlr4.version>4.9.3</antlr4.version>
     <apache-directory-server.version>2.0.0-M24</apache-directory-server.version>
+    <asm.version>9.7</asm.version>
     <avro.version>1.12.0</avro.version>
     <bcprov-jdk18on.version>1.82</bcprov-jdk18on.version>
-    <calcite.version>1.33.0</calcite.version>
+    <calcite.version>1.38.0</calcite.version>
     <datanucleus-api-jdo.version>6.0.5</datanucleus-api-jdo.version>
     <datanucleus-core.version>6.0.11</datanucleus-core.version>
     <datanucleus-jdo.version>3.2.0-release</datanucleus-jdo.version>
@@ -358,6 +359,21 @@
         <version>${immutables.version}</version>
         <scope>provided</scope>
       </dependency>
+      <dependency>
+        <groupId>org.ow2.asm</groupId>
+        <artifactId>asm</artifactId>
+        <version>${asm.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.ow2.asm</groupId>
+        <artifactId>asm-commons</artifactId>
+        <version>${asm.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.ow2.asm</groupId>
+        <artifactId>asm-tree</artifactId>
+        <version>${asm.version}</version>
+      </dependency>
       <dependency>
         <groupId>com.sun.jersey</groupId>
         <artifactId>jersey-core</artifactId>

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.calcite.util.ConversionUtil;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -569,6 +570,10 @@ private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException {
     conf = c;
     // turn off calcite rexnode normalization
     System.setProperty("calcite.enable.rexnode.digest.normalize", "false");
+    // update calcite default charset
+    // TODO is this the right place? It's not considered in cases like TestHiveRelJsonReader
+    System.setProperty("calcite.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
+    System.setProperty("calcite.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
     if (doRegisterAllFns) {
       registerAllFunctionsOnce();
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -48,6 +48,8 @@
 import org.apache.calcite.rex.RexDynamicParam;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
+import org.apache.calcite.rex.RexLambdaRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
@@ -1139,6 +1141,16 @@ public Boolean visitSubQuery(RexSubQuery subQuery) {
     public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
       return false;
     }
+
+    @Override
+    public Boolean visitLambda(RexLambda lambda) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitLambdaRef(RexLambdaRef lambdaRef) {
+      return false;
+    }
   }
 
   public static Set<Integer> getInputRefs(RexNode expr) {

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java

@@ -89,7 +89,8 @@ public static RelBuilderFactory proto(final Context context) {
     return new RelBuilderFactory() {
       @Override
       public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) {
-        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED));
+        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED)
+            .withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues
         return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema);
       }
     };

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumAggFunction.java

@@ -108,7 +108,7 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
         break;
       case 2:
         node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, merges);
-        node = rexBuilder.makeAbstractCast(aggregateCall.type, node);
+        node = rexBuilder.makeAbstractCast(aggregateCall.type, node, false);
         break;
       default:
         throw new AssertionError("unexpected count " + merges);
@@ -120,6 +120,7 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
           false,
           false,
           false,
+          List.of(),
           ImmutableList.of(ordinal),
           -1,
           aggregateCall.distinctKeys,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java

@@ -75,8 +75,8 @@ private static AggregateCall replaceAggFunction(RelDataType rowType, AggregateCa
         SqlTypeUtil.projectTypes(rowType, aggCall.getArgList()),
         aggCall.getType());
     return AggregateCall.create(aggFunction, aggCall.isDistinct(), aggCall.isApproximate(), aggCall.ignoreNulls(),
-        aggCall.getArgList(), aggCall.filterArg, aggCall.distinctKeys, aggCall.getCollation(), aggCall.getType(),
-        aggCall.getName());
+        aggCall.rexList, aggCall.getArgList(), aggCall.filterArg, aggCall.distinctKeys, aggCall.getCollation(),
+        aggCall.getType(), aggCall.getName());
   }
 
   @Override

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java

@@ -21,6 +21,7 @@
 import org.apache.calcite.plan.RelOptRuleCall;
 import org.apache.calcite.plan.hep.HepRelVertex;
 import org.apache.calcite.rel.RelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
 
 
@@ -57,7 +58,7 @@ public void onMatch(RelOptRuleCall call) {
       return;
     }
     // The node is the root, release the kraken!
-    node = HiveHepExtractRelNodeRule.execute(node);
+    node = HiveCalciteUtil.stripHepVertices(node);
     call.transformTo(trim(call, node));
     triggered = true;
   }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveHepExtractRelNodeRule.java

@@ -25,19 +25,32 @@
 import org.apache.calcite.plan.hep.HepRelVertex;
 import org.apache.calcite.rel.RelNode;
 
-
 /**
  * The goal of this rule is to extract the RelNode from the
  * HepRelVertex node so rules do tree traversal can be applied correctly.
  * @see HiveFieldTrimmerRule
  * @see org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveAggregateInsertDeleteIncrementalRewritingRule
  */
+// TODO remove
 public class HiveHepExtractRelNodeRule extends RelOptRule {
 
   public static RelNode execute(RelNode node) {
     final HepPlanner tmpPlanner = new HepPlanner(PROGRAM);
     tmpPlanner.setRoot(node);
     return tmpPlanner.findBestExp();
+
+    // TODO cleanup
+    /* RelNode res = node.accept(new RelShuttleImpl() {
+      @Override
+      public RelNode visit(RelNode other) {
+        RelNode innerNode = other.stripped();
+        if (other != innerNode) {
+          return innerNode.accept(this);
+        }
+        return super.visit(other);
+      }
+    });
+    return res; */
   }
 
   private static final HepProgram PROGRAM = new HepProgramBuilder()

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java

@@ -549,9 +549,8 @@ private ImmutableBitSet generateNewGroupset(Aggregate aggregate, ImmutableBitSet
    */
   private Aggregate rewriteGBConstantKeys(Aggregate aggregate, ImmutableBitSet fieldsUsed,
       ImmutableBitSet aggCallFields) {
-    if ((aggregate.getIndicatorCount() > 0)
-        || (aggregate.getGroupSet().isEmpty())
-        || fieldsUsed.contains(aggregate.getGroupSet())) {
+    if (aggregate.getGroupSet().isEmpty()
+        || fieldsUsed.intersects(aggregate.getGroupSet())) {
       return aggregate;
     }
 

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java

@@ -31,10 +31,9 @@
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.tools.RelBuilder;
 import org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rebuild.AlterMaterializedViewRebuildAnalyzer;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
 
-import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveHepExtractRelNodeRule;
-
 /**
  * This rule will perform a rewriting to prepare the plan for incremental
  * view maintenance in case there exist aggregation operator, so we can
@@ -118,7 +117,7 @@ protected IncrementalComputePlanWithDeletedRows createJoinRightInput(RelOptRuleC
     RelNode aggInput = aggregate.getInput();
 
     // Propagate rowIsDeleted column
-    aggInput = HiveHepExtractRelNodeRule.execute(aggInput);
+    aggInput = HiveCalciteUtil.stripHepVertices(aggInput);
     aggInput = new HiveRowIsDeletedPropagator(relBuilder).propagate(aggInput);
 
     // The row schema has two additional columns after propagation: