Apply patches, and fix failing fuzz run

.github/workflows/MainDistributionPipeline.yml

@@ -17,7 +17,7 @@ jobs:
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: sqlsmith
-      duckdb_version: e3509341f681c4cb6f2c22d1f0f4b653ed20644d
+      duckdb_version: main
       ci_tools_version: main
       exclude_archs: ''
 
@@ -28,7 +28,7 @@ jobs:
     secrets: inherit
     with:
       extension_name: sqlsmith
-      duckdb_version: e3509341f681c4cb6f2c22d1f0f4b653ed20644d
+      duckdb_version: main
       ci_tools_version: main
       exclude_archs: ''
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

scripts/fuzzer_helper.py

@@ -22,6 +22,8 @@
 ]
 
 INTERNAL_ERROR_FALSE_POSITIVES = [
+    ".internal",
+    "internal schema",
     "internal use",
     "internal_compress",
     "internal_decompress",

scripts/reduce_sql.py

@@ -53,14 +53,15 @@ def run_shell_command(shell, cmd):
 
 
 # reduce a single statement
-def get_reduced_sql(shell, sql_query):
+def get_reduce_candidates(shell, sql_query):
+    reduce_candidates = []
     reduce_query = get_reduced_query.replace('${QUERY}', sql_query.replace("'", "''"))
     (stdout, stderr, returncode) = run_shell_command(shell, reduce_query)
     if returncode != 0:
+        print(f"Failed to reduce query: {sql_query}")
         print(stdout)
         print(stderr)
-        raise Exception("Failed to reduce query")
-    reduce_candidates = []
+        return(reduce_candidates)
     for line in stdout.split('\n'):
         if len(line) <= 2:
             continue
@@ -77,7 +78,7 @@ def reduce(sql_query, data_load, shell, error_msg, max_time_seconds=300):
     start = time.time()
     while True:
         found_new_candidate = False
-        reduce_candidates = get_reduced_sql(shell, sql_query)
+        reduce_candidates = get_reduce_candidates(shell, sql_query)
         for reduce_candidate in reduce_candidates:
             if reduce_candidate == sql_query:
                 continue
@@ -190,7 +191,7 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds)
     sql_query = queries[query_index]
     while True:
         found_new_candidate = False
-        reduce_candidates = get_reduced_sql(shell, sql_query)
+        reduce_candidates = get_reduce_candidates(shell, sql_query)
         for reduce_candidate in reduce_candidates:
             if reduce_candidate == sql_query:
                 continue

src/include/statement_generator.hpp

@@ -96,8 +96,8 @@ class StatementGenerator {
 
 	void GenerateAllScalar(ScalarFunctionCatalogEntry &scalar_function, vector<string> &result);
 	void GenerateAllAggregate(AggregateFunctionCatalogEntry &aggregate_function, vector<string> &result);
-	string GenerateTestAllTypes(BaseScalarFunction &base_function);
-	string GenerateTestVectorTypes(BaseScalarFunction &base_function);
+	string GenerateTestAllTypes(SimpleFunction &base_function);
+	string GenerateTestVectorTypes(SimpleFunction &base_function);
 	string GenerateCast(const LogicalType &target, const string &source_name, bool add_varchar);
 	bool FunctionArgumentsAlwaysNull(const string &name);
 

src/sqlsmith_extension.cpp

@@ -118,7 +118,7 @@ static void ReduceSQLFunction(ClientContext &context, TableFunctionInput &data_p
 		output.data[0].SetValue(count, Value(entry));
 		count++;
 	}
-	output.SetCardinality(count);
+	output.SetChildCardinality(count);
 }
 
 struct FuzzyDuckFunctionData : public TableFunctionData {

src/statement_generator.cpp

@@ -18,6 +18,7 @@
 #include "duckdb/parser/statement/attach_statement.hpp"
 #include "duckdb/parser/statement/create_statement.hpp"
 #include "duckdb/parser/statement/delete_statement.hpp"
+#include "duckdb/parser/query_node/delete_query_node.hpp"
 #include "duckdb/parser/statement/detach_statement.hpp"
 #include "duckdb/parser/statement/insert_statement.hpp"
 #include "duckdb/parser/statement/multi_statement.hpp"
@@ -189,12 +190,12 @@ unique_ptr<DeleteStatement> StatementGenerator::GenerateDelete() {
 		if (entry.type == CatalogType::TABLE_ENTRY) {
 			auto result = make_uniq<BaseTableRef>();
 			result->table_name = entry.name;
-			delete_statement->table = std::move(result);
+			delete_statement->node->table = std::move(result);
 		} else {
-			delete_statement->table = GenerateTableRef();
+			delete_statement->node->table = GenerateTableRef();
 		}
 	} else {
-		delete_statement->table = GenerateTableRef();
+		delete_statement->node->table = GenerateTableRef();
 	}
 
 	return delete_statement;
@@ -302,7 +303,7 @@ void StatementGenerator::GenerateCTEs(QueryNode &node) {
 	}
 	while (RandomPercentage(20)) {
 		auto cte = make_uniq<CommonTableExpressionInfo>();
-		cte->query = unique_ptr_cast<SQLStatement, SelectStatement>(GenerateSelect());
+		cte->query_node = std::move(unique_ptr_cast<SQLStatement, SelectStatement>(GenerateSelect())->node);
 		for (idx_t i = 0; i < 1 + RandomValue(10); i++) {
 			cte->aliases.push_back(GenerateIdentifier());
 		}
@@ -564,7 +565,7 @@ unique_ptr<TableRef> StatementGenerator::GenerateTableFunctionRef() {
 
 	auto result = make_uniq<TableFunctionRef>();
 	vector<unique_ptr<ParsedExpression>> children;
-	for (idx_t i = 0; i < table_function.arguments.size(); i++) {
+	for (idx_t i = 0; i < table_function.GetArguments().size(); i++) {
 		children.push_back(GenerateConstant());
 	}
 	vector<string> names;
@@ -575,7 +576,7 @@ unique_ptr<TableRef> StatementGenerator::GenerateTableFunctionRef() {
 		auto name = Choose(names);
 		names.erase(std::find(names.begin(), names.end(), name));
 		auto expr = GenerateConstant();
-		expr->alias = name;
+		expr->SetAlias(name);
 		children.push_back(std::move(expr));
 	}
 	result->function = make_uniq<FunctionExpression>(entry.name, std::move(children));
@@ -761,10 +762,12 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateFunction() {
 		auto offset = RandomValue(scalar_entry.functions.Size());
 		auto actual_function = scalar_entry.functions.GetFunctionByOffset(offset);
 		name = scalar_entry.name;
-		arguments = actual_function.arguments;
-		min_parameters = actual_function.arguments.size();
+		for (auto &arg : actual_function.GetSignature().GetParameters()) {
+			arguments.push_back(arg.GetType());
+		}
+		min_parameters = actual_function.GetSignature().GetParameterCount();
 		max_parameters = min_parameters;
-		if (actual_function.varargs.id() != LogicalTypeId::INVALID) {
+		if (actual_function.GetSignature().GetVarArgs().id() != LogicalTypeId::INVALID) {
 			max_parameters += 5;
 		}
 		break;
@@ -775,9 +778,9 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateFunction() {
 		    aggregate_entry.functions.GetFunctionByOffset(RandomValue(aggregate_entry.functions.Size()));
 
 		name = aggregate_entry.name;
-		min_parameters = actual_function.arguments.size();
+		min_parameters = actual_function.GetSignature().GetParameterCount();
 		max_parameters = min_parameters;
-		if (actual_function.varargs.id() != LogicalTypeId::INVALID) {
+		if (actual_function.GetVarArgs().id() != LogicalTypeId::INVALID) {
 			max_parameters += 5;
 		}
 		if (RandomPercentage(10) && !in_window) {
@@ -806,6 +809,9 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateFunction() {
 		max_parameters = min_parameters;
 		break;
 	}
+	case CatalogType::WINDOW_FUNCTION_ENTRY:
+		//	FIXME: Use the function, not a random builtin
+		return GenerateWindowFunction();
 	default:
 		throw InternalException("StatementGenerator::GenerateFunction");
 	}
@@ -940,7 +946,7 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateWindowFunction(optional
 	if (function) {
 		type = ExpressionType::WINDOW_AGGREGATE;
 		name = function->name;
-		min_parameters = function->arguments.size();
+		min_parameters = function->GetSignature().GetParameterCount();
 		max_parameters = min_parameters;
 	} else {
 		name = Choose<string>({"rank", "rank_dense", "percent_rank", "row_number", "first_value", "last_value",
@@ -957,9 +963,12 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateWindowFunction(optional
 		case ExpressionType::WINDOW_NTILE:
 		case ExpressionType::WINDOW_FIRST_VALUE:
 		case ExpressionType::WINDOW_LAST_VALUE:
+			min_parameters = 1;
+			break;
 		case ExpressionType::WINDOW_LEAD:
 		case ExpressionType::WINDOW_LAG:
 			min_parameters = 1;
+			min_parameters = 3;
 			break;
 		case ExpressionType::WINDOW_NTH_VALUE:
 			min_parameters = 2;
@@ -970,7 +979,7 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateWindowFunction(optional
 		max_parameters = min_parameters;
 	}
 	WindowChecker checker(*this);
-	auto result = make_uniq<WindowExpression>(type, INVALID_CATALOG, INVALID_SCHEMA, std::move(name));
+	auto result = make_uniq<WindowExpression>(SYSTEM_CATALOG, DEFAULT_SCHEMA, std::move(name));
 	result->children = GenerateChildren(min_parameters, max_parameters);
 	while (RandomPercentage(50)) {
 		result->partitions.push_back(GenerateExpression());
@@ -1014,15 +1023,6 @@ unique_ptr<ParsedExpression> StatementGenerator::GenerateWindowFunction(optional
 	default:
 		break;
 	}
-	switch (type) {
-	case ExpressionType::WINDOW_LEAD:
-	case ExpressionType::WINDOW_LAG:
-		result->offset_expr = RandomExpression(30);
-		result->default_expr = RandomExpression(30);
-		break;
-	default:
-		break;
-	}
 	return std::move(result);
 }
 
@@ -1279,40 +1279,40 @@ bool StatementGenerator::FunctionArgumentsAlwaysNull(const string &name) {
 
 	return always_null_functions.find(name) != always_null_functions.end();
 }
-string StatementGenerator::GenerateTestAllTypes(BaseScalarFunction &base_function) {
+string StatementGenerator::GenerateTestAllTypes(SimpleFunction &base_function) {
 	auto select = make_uniq<SelectStatement>();
 	auto node = make_uniq<SelectNode>();
 
-	bool always_null = FunctionArgumentsAlwaysNull(base_function.name);
+	bool always_null = FunctionArgumentsAlwaysNull(base_function.GetName());
 
 	vector<unique_ptr<ParsedExpression>> children;
-	for (auto &arg : base_function.arguments) {
+	for (auto &param : base_function.GetSignature().GetParameters()) {
 		// look up the type
 		unique_ptr<ParsedExpression> argument;
 		if (!always_null) {
 			for (auto &test_type : generator_context->test_types) {
-				if (test_type.type.id() == arg.id()) {
+				if (test_type.type.id() == param.GetType().id()) {
 					argument = make_uniq<ColumnRefExpression>(test_type.name);
 				}
 			}
 		}
 		if (!argument) {
-			argument = make_uniq<ConstantExpression>(Value(arg));
+			argument = make_uniq<ConstantExpression>(Value(param.GetType()));
 		}
 		children.push_back(std::move(argument));
 	}
 	auto from_clause = make_uniq<BaseTableRef>();
 	from_clause->table_name = "all_types";
 	node->from_table = std::move(from_clause);
 
-	auto function_expr = make_uniq<FunctionExpression>(base_function.name, std::move(children));
+	auto function_expr = make_uniq<FunctionExpression>(base_function.GetName(), std::move(children));
 	node->select_list.push_back(std::move(function_expr));
 
 	select->node = std::move(node);
 	return select->ToString();
 }
 
-string StatementGenerator::GenerateTestVectorTypes(BaseScalarFunction &base_function) {
+string StatementGenerator::GenerateTestVectorTypes(SimpleFunction &base_function) {
 	auto select = make_uniq<SelectStatement>();
 	auto node = make_uniq<SelectNode>();
 
@@ -1321,17 +1321,17 @@ string StatementGenerator::GenerateTestVectorTypes(BaseScalarFunction &base_func
 	vector<unique_ptr<ParsedExpression>> children;
 	vector<unique_ptr<ParsedExpression>> test_vector_types;
 	vector<string> column_aliases;
-	for (auto &arg : base_function.arguments) {
+	for (auto &param : base_function.GetSignature().GetParameters()) {
 		unique_ptr<ParsedExpression> argument;
 		if (!always_null) {
 			string argument_name = "c" + to_string(column_aliases.size() + 1);
 			column_aliases.push_back(argument_name);
 			argument = make_uniq<ColumnRefExpression>(std::move(argument_name));
 			auto constant_expr = make_uniq<ConstantExpression>(Value());
-			auto cast = make_uniq<CastExpression>(arg, std::move(constant_expr));
+			auto cast = make_uniq<CastExpression>(param.GetType(), std::move(constant_expr));
 			test_vector_types.push_back(std::move(cast));
 		} else {
-			argument = make_uniq<ConstantExpression>(Value(arg));
+			argument = make_uniq<ConstantExpression>(Value(param.GetType()));
 		}
 		children.push_back(std::move(argument));
 	}

src/statement_simplifier.cpp

@@ -8,8 +8,11 @@
 #include "duckdb/parser/expression/list.hpp"
 #include "duckdb/parser/statement/delete_statement.hpp"
 #include "duckdb/parser/statement/insert_statement.hpp"
+#include "duckdb/parser/query_node/insert_query_node.hpp"
 #include "duckdb/parser/statement/prepare_statement.hpp"
 #include "duckdb/parser/statement/update_statement.hpp"
+#include "duckdb/parser/query_node/update_query_node.hpp"
+#include "duckdb/parser/query_node/delete_query_node.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
 #endif
 
@@ -206,7 +209,7 @@ void StatementSimplifier::Simplify(CommonTableExpressionMap &cte) {
 	SimplifyMap(cte.map);
 	for (auto &cte_child : cte.map) {
 		// simplify individual ctes
-		Simplify(cte_child.second->query->node);
+		Simplify(cte_child.second->query_node);
 	}
 }
 
@@ -357,8 +360,6 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptr<ParsedExpression
 		SimplifyChildExpression(expr, window.filter_expr);
 		SimplifyChildExpression(expr, window.start_expr);
 		SimplifyChildExpression(expr, window.end_expr);
-		SimplifyChildExpression(expr, window.offset_expr);
-		SimplifyChildExpression(expr, window.default_expr);
 		SimplifyEnum(window.ignore_nulls, false);
 		SimplifyEnum(window.distinct, false);
 		SimplifyEnum(window.start, WindowBoundary::UNBOUNDED_PRECEDING);
@@ -395,17 +396,19 @@ void StatementSimplifier::Simplify(SelectStatement &stmt) {
 }
 
 void StatementSimplifier::Simplify(InsertStatement &stmt) {
-	Simplify(stmt.cte_map);
-	Simplify(*stmt.select_statement);
-	SimplifyList(stmt.returning_list);
+	Simplify(stmt.node->cte_map);
+	if (stmt.node->select_statement) {
+		Simplify(*stmt.node->select_statement);
+	}
+	SimplifyList(stmt.node->returning_list);
 }
 
 void StatementSimplifier::Simplify(DeleteStatement &stmt) {
-	Simplify(stmt.cte_map);
-	SimplifyOptional(stmt.condition);
-	SimplifyExpression(stmt.condition);
-	SimplifyList(stmt.using_clauses);
-	SimplifyList(stmt.returning_list);
+	Simplify(stmt.node->cte_map);
+	SimplifyOptional(stmt.node->condition);
+	SimplifyExpression(stmt.node->condition);
+	SimplifyList(stmt.node->using_clauses);
+	SimplifyList(stmt.node->returning_list);
 }
 
 void StatementSimplifier::Simplify(UpdateSetInfo &info) {
@@ -432,11 +435,11 @@ void StatementSimplifier::Simplify(PrepareStatement &stmt) {
 }
 
 void StatementSimplifier::Simplify(UpdateStatement &stmt) {
-	Simplify(stmt.cte_map);
-	SimplifyOptional(stmt.from_table);
-	D_ASSERT(stmt.set_info);
-	Simplify(*stmt.set_info);
-	SimplifyList(stmt.returning_list);
+	Simplify(stmt.node->cte_map);
+	SimplifyOptional(stmt.node->from_table);
+	D_ASSERT(stmt.node->set_info);
+	Simplify(*stmt.node->set_info);
+	SimplifyList(stmt.node->returning_list);
 }
 
 void StatementSimplifier::Simplify(SQLStatement &stmt) {