From 132baf7fcd0336ea5b26f605628a5b8afdb1adc3 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 5 Mar 2026 17:13:02 -0800 Subject: [PATCH 1/5] chore: Array literals tests enable --- .../test/resources/sql-tests/expressions/array/array_append.sql | 2 +- .../test/resources/sql-tests/expressions/array/array_except.sql | 2 +- .../resources/sql-tests/expressions/array/array_intersect.sql | 2 +- .../test/resources/sql-tests/expressions/array/array_max.sql | 2 +- .../test/resources/sql-tests/expressions/array/array_min.sql | 2 +- .../test/resources/sql-tests/expressions/array/array_remove.sql | 2 +- .../test/resources/sql-tests/expressions/array/array_union.sql | 2 +- .../resources/sql-tests/expressions/array/arrays_overlap.sql | 2 +- spark/src/test/resources/sql-tests/expressions/array/size.sql | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql index 2efd13f1c2..6ee215212b 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql @@ -35,5 +35,5 @@ query spark_answer_only SELECT array_append(array(1, 2, 3), val) FROM test_array_append -- literal + literal -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_append(array(1, 2, 3), 4), array_append(array(), 1), array_append(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql index 110f798f97..9bc466c356 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql @@ -35,5 +35,5 @@ query spark_answer_only SELECT array_except(array(1, 2, 3), b) FROM test_array_except -- literal + literal -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_except(array(1, 2, 3), array(2, 3, 4)), array_except(array(1, 2), array()), array_except(array(), array(1)), array_except(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql index 379f16a642..1ec755cf5f 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql @@ -35,5 +35,5 @@ query spark_answer_only SELECT array_intersect(array(1, 2, 3), b) FROM test_array_intersect -- literal + literal -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_intersect(array(1, 2, 3), array(2, 3, 4)), array_intersect(array(1, 2), array(3, 4)), array_intersect(array(), array(1)), array_intersect(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_max.sql b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql index 17c572b952..dd1211598c 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_max.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql @@ -27,5 +27,5 @@ query spark_answer_only SELECT array_max(arr) FROM test_array_max -- literal arguments -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_max(array(1, 2, 3)), array_max(array()), array_max(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_min.sql b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql index eb86d6c15f..da7f8f4d2c 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_min.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql @@ -27,5 +27,5 @@ query spark_answer_only SELECT array_min(arr) FROM test_array_min -- literal arguments -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_min(array(1, 2, 3)), array_min(array()), array_min(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql index bd1af4bc24..f91089d554 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql @@ -35,5 +35,5 @@ query spark_answer_only SELECT array_remove(array(1, 2, 3, 2), val) FROM test_array_remove -- literal + literal -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_remove(array(1, 2, 3, 2), 2), array_remove(array(1, 2, 3), 4), array_remove(array(), 1), array_remove(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql index 4c699ae546..6a52bae60d 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql @@ -35,5 +35,5 @@ query spark_answer_only SELECT array_union(array(1, 2, 3), b) FROM test_array_union -- literal + literal -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT array_union(array(1, 2, 3), array(3, 4, 5)), array_union(array(1, 2), array()), array_union(array(), array(1)), array_union(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql index 6c28224ed6..d628c5ca08 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql @@ -35,5 +35,5 @@ query spark_answer_only SELECT arrays_overlap(array(1, 2, 3), b) FROM test_arrays_overlap -- literal + literal -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)), arrays_overlap(array(1, 2), array(3, 4)), arrays_overlap(array(), array(1)), arrays_overlap(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/size.sql b/spark/src/test/resources/sql-tests/expressions/array/size.sql index a096502bc5..de989337f5 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/size.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/size.sql @@ -27,5 +27,5 @@ query spark_answer_only SELECT size(arr), size(m) FROM test_size -- literal arguments -query ignore(https://github.com/apache/datafusion-comet/issues/3338) +query SELECT size(array(1, 2, 3)), size(array()), size(cast(NULL as array)) From ceadc32b23b11a206f0868f7bffaab41ef57edde Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 6 Mar 2026 16:44:16 -0800 Subject: [PATCH 2/5] chore: Array literals tests enable --- .../scala/org/apache/comet/serde/arrays.scala | 8 ++- .../expressions/array/array_append.sql | 7 +- .../expressions/array/array_except.sql | 10 +-- .../expressions/array/array_intersect.sql | 7 +- .../expressions/array/array_union.sql | 7 +- .../expressions/array/arrays_overlap.sql | 7 +- .../apache/comet/CometSqlFileTestSuite.scala | 71 +++++++++++-------- 7 files changed, 69 insertions(+), 48 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index b7ebb9ba7b..e7721058d3 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -95,7 +95,12 @@ object CometArrayAppend extends CometExpressionSerde[ArrayAppend] { val keyExprProto = exprToProto(expr.children(1), inputs, binding) val arrayAppendScalarExpr = - scalarFunctionExprToProto("array_append", arrayExprProto, keyExprProto) + scalarFunctionExprToProtoWithReturnType( + "array_append", + ArrayType(elementType = elementType), + false, + arrayExprProto, + keyExprProto) val isNotNullExpr = createUnaryExpr( expr, @@ -339,6 +344,7 @@ object CometArrayExcept extends CometExpressionSerde[ArrayExcept] with CometExpr return None } } + val elementType = expr.left.dataType.asInstanceOf[ArrayType].elementType val leftArrayExprProto = exprToProto(expr.left, inputs, binding) val rightArrayExprProto = exprToProto(expr.right, inputs, binding) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql index 6ee215212b..df96240efa 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql @@ -15,6 +15,7 @@ -- specific language governing permissions and limitations -- under the License. +-- Config: spark.comet.expression.ArrayAppend.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true statement @@ -23,15 +24,15 @@ CREATE TABLE test_array_append(arr array, val int) USING parquet statement INSERT INTO test_array_append VALUES (array(1, 2, 3), 4), (array(), 1), (NULL, 1), (array(1, 2), NULL) -query spark_answer_only +query SELECT array_append(arr, val) FROM test_array_append -- column + literal -query spark_answer_only +query SELECT array_append(arr, 99) FROM test_array_append -- literal + column -query spark_answer_only +query SELECT array_append(array(1, 2, 3), val) FROM test_array_append -- literal + literal diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql index 9bc466c356..3b3f45111c 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql @@ -14,7 +14,7 @@ -- KIND, either express or implied. See the License for the -- specific language governing permissions and limitations -- under the License. - +-- Config: spark.comet.expression.ArrayExcept.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true statement @@ -23,17 +23,17 @@ CREATE TABLE test_array_except(a array, b array) USING parquet statement INSERT INTO test_array_except VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, 2), array()), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL)) -query spark_answer_only +query SELECT array_except(a, b) FROM test_array_except -- column + literal -query spark_answer_only +query SELECT array_except(a, array(2, 3)) FROM test_array_except -- literal + column -query spark_answer_only +query SELECT array_except(array(1, 2, 3), b) FROM test_array_except -- literal + literal -query +query ignore(https://github.com/apache/datafusion-comet/issues/3646) SELECT array_except(array(1, 2, 3), array(2, 3, 4)), array_except(array(1, 2), array()), array_except(array(), array(1)), array_except(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql index 1ec755cf5f..d77cccd925 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql @@ -15,6 +15,7 @@ -- specific language governing permissions and limitations -- under the License. +-- Config: spark.comet.expression.ArrayIntersect.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true statement @@ -23,15 +24,15 @@ CREATE TABLE test_array_intersect(a array, b array) USING parquet statement INSERT INTO test_array_intersect VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, 2), array(3, 4)), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) -query spark_answer_only +query SELECT array_intersect(a, b) FROM test_array_intersect -- column + literal -query spark_answer_only +query SELECT array_intersect(a, array(2, 3)) FROM test_array_intersect -- literal + column -query spark_answer_only +query SELECT array_intersect(array(1, 2, 3), b) FROM test_array_intersect -- literal + literal diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql index 6a52bae60d..cbff2dc67a 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql @@ -15,6 +15,7 @@ -- specific language governing permissions and limitations -- under the License. +-- Config: spark.comet.expression.ArrayUnion.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true statement @@ -23,15 +24,15 @@ CREATE TABLE test_array_union(a array, b array) USING parquet statement INSERT INTO test_array_union VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, 2), array()), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3644) SELECT array_union(a, b) FROM test_array_union -- column + literal -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3644) SELECT array_union(a, array(3, 4, 5)) FROM test_array_union -- literal + column -query spark_answer_only +query SELECT array_union(array(1, 2, 3), b) FROM test_array_union -- literal + literal diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql index d628c5ca08..27d28a7402 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql @@ -15,6 +15,7 @@ -- specific language governing permissions and limitations -- under the License. +-- Config: spark.comet.expression.ArraysOverlap.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true statement @@ -23,15 +24,15 @@ CREATE TABLE test_arrays_overlap(a array, b array) USING parquet statement INSERT INTO test_arrays_overlap VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, 2), array(3, 4)), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3645) SELECT arrays_overlap(a, b) FROM test_arrays_overlap -- column + literal -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3645) SELECT arrays_overlap(a, array(3, 4, 5)) FROM test_arrays_overlap -- literal + column -query spark_answer_only +query SELECT arrays_overlap(array(1, 2, 3), b) FROM test_arrays_overlap -- literal + literal diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index 4e3b9e0452..bb1c6b4222 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -83,39 +83,50 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { withTable(file.tables: _*) { file.records.foreach { case SqlStatement(sql, line) => - val location = if (line > 0) s"$relativePath:$line" else relativePath - withClue(s"In SQL file $location, executing statement:\n$sql\n") { - spark.sql(sql) + try { + val location = if (line > 0) s"$relativePath:$line" else relativePath + withClue(s"In SQL file $location, executing statement:\n$sql\n") { + spark.sql(sql) + } + } catch { + case e: Exception => + sys.error(s"Error executing SQL '$sql'. Reason: '${e.getMessage}''") } case SqlQuery(sql, mode, line) => - val location = if (line > 0) s"$relativePath:$line" else relativePath - withClue(s"In SQL file $location, executing query:\n$sql\n") { - mode match { - case CheckCoverageAndAnswer => - checkSparkAnswerAndOperator(sql) - case SparkAnswerOnly => - checkSparkAnswer(sql) - case WithTolerance(tol) => - checkSparkAnswerWithTolerance(sql, tol) - case ExpectFallback(reason) => - checkSparkAnswerAndFallbackReason(sql, reason) - case Ignore(reason) => - logInfo(s"IGNORED query (${reason}): $sql") - case ExpectError(pattern) => - val (sparkError, cometError) = checkSparkAnswerMaybeThrows(spark.sql(sql)) - assert( - sparkError.isDefined, - s"Expected Spark to throw an error matching '$pattern' but query succeeded") - assert( - cometError.isDefined, - s"Expected Comet to throw an error matching '$pattern' but query succeeded") - assert( - sparkError.get.getMessage.contains(pattern), - s"Spark error '${sparkError.get.getMessage}' does not contain '$pattern'") - assert( - cometError.get.getMessage.contains(pattern), - s"Comet error '${cometError.get.getMessage}' does not contain '$pattern'") + try { + val location = if (line > 0) s"$relativePath:$line" else relativePath + withClue(s"In SQL file $location, executing query:\n$sql\n") { + mode match { + case CheckCoverageAndAnswer => + checkSparkAnswerAndOperator(sql) + case SparkAnswerOnly => + checkSparkAnswer(sql) + case WithTolerance(tol) => + checkSparkAnswerWithTolerance(sql, tol) + case ExpectFallback(reason) => + checkSparkAnswerAndFallbackReason(sql, reason) + case Ignore(reason) => + logInfo(s"IGNORED query ($reason): $sql") + case ExpectError(pattern) => + val (sparkError, cometError) = checkSparkAnswerMaybeThrows(spark.sql(sql)) + assert( + sparkError.isDefined, + s"Expected Spark to throw an error matching '$pattern' but query succeeded") + assert( + cometError.isDefined, + s"Expected Comet to throw an error matching '$pattern' but query succeeded") + assert( + sparkError.get.getMessage.contains(pattern), + s"Spark error '${sparkError.get.getMessage}' does not contain '$pattern'") + assert( + cometError.get.getMessage.contains(pattern), + s"Comet error '${cometError.get.getMessage}' does not contain '$pattern'") + } } + + } catch { + case e: Exception => + sys.error(s"Error executing SQL '$sql'. Reason: '${e.getMessage}''") } } } From 7bd9b05e5ba5ad15e91f9ac52fe32637211acd0b Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 7 Mar 2026 09:05:40 -0800 Subject: [PATCH 3/5] chore: Array literals tests enable --- spark/src/main/scala/org/apache/comet/serde/arrays.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index e7721058d3..79e995f2ee 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -344,7 +344,6 @@ object CometArrayExcept extends CometExpressionSerde[ArrayExcept] with CometExpr return None } } - val elementType = expr.left.dataType.asInstanceOf[ArrayType].elementType val leftArrayExprProto = exprToProto(expr.left, inputs, binding) val rightArrayExprProto = exprToProto(expr.right, inputs, binding) From 314f8778c3ca9049f8e381e2983191bc97952194 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 7 Mar 2026 09:41:10 -0800 Subject: [PATCH 4/5] chore: Array literals tests enable --- .../test/resources/sql-tests/expressions/array/array_append.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql index df96240efa..0020af3f3e 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql @@ -15,6 +15,7 @@ -- specific language governing permissions and limitations -- under the License. +-- Config: spark.comet.expression.ArrayInsert.allowIncompatible=true -- Config: spark.comet.expression.ArrayAppend.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true From 0e3890c51885aefca372f2cf016913aa5fc237c3 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 7 Mar 2026 10:30:48 -0800 Subject: [PATCH 5/5] chore: Array literals tests enable --- .../test/scala/org/apache/comet/CometSqlFileTestSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index bb1c6b4222..8e78f9d0b3 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -90,7 +90,7 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } catch { case e: Exception => - sys.error(s"Error executing SQL '$sql'. Reason: '${e.getMessage}''") + throw new RuntimeException(s"Error executing SQL '$sql'", e) } case SqlQuery(sql, mode, line) => try { @@ -126,7 +126,7 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } catch { case e: Exception => - sys.error(s"Error executing SQL '$sql'. Reason: '${e.getMessage}''") + throw new RuntimeException(s"Error executing SQL '$sql'", e) } } }