diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index ba34dfd..d847e66 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -17,7 +17,7 @@ jobs: uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: extension_name: sqlsmith - duckdb_version: e3509341f681c4cb6f2c22d1f0f4b653ed20644d + duckdb_version: main ci_tools_version: main exclude_archs: '' @@ -28,7 +28,7 @@ jobs: secrets: inherit with: extension_name: sqlsmith - duckdb_version: e3509341f681c4cb6f2c22d1f0f4b653ed20644d + duckdb_version: main ci_tools_version: main exclude_archs: '' deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index e0ad922..447260f 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -8,12 +8,21 @@ on: - '!main' jobs: + print-vars: + runs-on: ubuntu-latest + steps: + - run: | + echo "github.actor: ${{ github.actor }}" + echo "github.head_ref: ${{ github.head_ref }}" + echo "github.ref_name: ${{ github.ref_name }}" + echo "github.sha: ${{ github.sha }}" + build-duckdb: name: Build DuckDB uses: duckdblabs/duckdb-fuzzer-ci/.github/workflows/build_fuzzer.yml@main with: git_url: ${{ github.actor }}/duckdb-sqlsmith - git_tag: ${{ github.head_ref }} + git_tag: ${{ github.head_ref || github.sha }} timeout-minutes: 120 fuzzer: diff --git a/duckdb b/duckdb index e350934..59b02b7 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit e3509341f681c4cb6f2c22d1f0f4b653ed20644d +Subproject commit 59b02b756b2c63723be7b53b4b4061f8acae602a diff --git a/extension-ci-tools b/extension-ci-tools index 16d89a5..a2da793 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 16d89a59ee14904a62383e83c300432b19d29abc +Subproject commit a2da79398cbc9633a0878a123dd977c54f35d92f diff --git a/scripts/fuzzer_helper.py b/scripts/fuzzer_helper.py index d21ae81..186e83e 100644 --- a/scripts/fuzzer_helper.py +++ b/scripts/fuzzer_helper.py @@ -22,6 +22,8 @@ ] INTERNAL_ERROR_FALSE_POSITIVES = [ + ".internal", + "internal schema", "internal use", "internal_compress", "internal_decompress", diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index 1a0089c..b2fe4c1 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -53,14 +53,15 @@ def run_shell_command(shell, cmd): # reduce a single statement -def get_reduced_sql(shell, sql_query): +def get_reduce_candidates(shell, sql_query): + reduce_candidates = [] reduce_query = get_reduced_query.replace('${QUERY}', sql_query.replace("'", "''")) (stdout, stderr, returncode) = run_shell_command(shell, reduce_query) if returncode != 0: + print(f"Failed to reduce query: {sql_query}") print(stdout) print(stderr) - raise Exception("Failed to reduce query") - reduce_candidates = [] + return(reduce_candidates) for line in stdout.split('\n'): if len(line) <= 2: continue @@ -77,7 +78,7 @@ def reduce(sql_query, data_load, shell, error_msg, max_time_seconds=300): start = time.time() while True: found_new_candidate = False - reduce_candidates = get_reduced_sql(shell, sql_query) + reduce_candidates = get_reduce_candidates(shell, sql_query) for reduce_candidate in reduce_candidates: if reduce_candidate == sql_query: continue @@ -190,7 +191,7 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds) sql_query = queries[query_index] while True: found_new_candidate = False - reduce_candidates = get_reduced_sql(shell, sql_query) + reduce_candidates = get_reduce_candidates(shell, sql_query) for reduce_candidate in reduce_candidates: if reduce_candidate == sql_query: continue diff --git a/src/include/statement_generator.hpp b/src/include/statement_generator.hpp index 2b60c00..639860e 100644 --- a/src/include/statement_generator.hpp +++ b/src/include/statement_generator.hpp @@ -96,8 +96,8 @@ class StatementGenerator { void GenerateAllScalar(ScalarFunctionCatalogEntry &scalar_function, vector &result); void GenerateAllAggregate(AggregateFunctionCatalogEntry &aggregate_function, vector &result); - string GenerateTestAllTypes(BaseScalarFunction &base_function); - string GenerateTestVectorTypes(BaseScalarFunction &base_function); + string GenerateTestAllTypes(SimpleFunction &base_function); + string GenerateTestVectorTypes(SimpleFunction &base_function); string GenerateCast(const LogicalType &target, const string &source_name, bool add_varchar); bool FunctionArgumentsAlwaysNull(const string &name); diff --git a/src/sqlsmith_extension.cpp b/src/sqlsmith_extension.cpp index c5f2274..1542369 100644 --- a/src/sqlsmith_extension.cpp +++ b/src/sqlsmith_extension.cpp @@ -118,7 +118,7 @@ static void ReduceSQLFunction(ClientContext &context, TableFunctionInput &data_p output.data[0].SetValue(count, Value(entry)); count++; } - output.SetCardinality(count); + output.SetChildCardinality(count); } struct FuzzyDuckFunctionData : public TableFunctionData { diff --git a/src/statement_generator.cpp b/src/statement_generator.cpp index 72e8f5b..4c750ce 100644 --- a/src/statement_generator.cpp +++ b/src/statement_generator.cpp @@ -18,6 +18,7 @@ #include "duckdb/parser/statement/attach_statement.hpp" #include "duckdb/parser/statement/create_statement.hpp" #include "duckdb/parser/statement/delete_statement.hpp" +#include "duckdb/parser/query_node/delete_query_node.hpp" #include "duckdb/parser/statement/detach_statement.hpp" #include "duckdb/parser/statement/insert_statement.hpp" #include "duckdb/parser/statement/multi_statement.hpp" @@ -189,12 +190,12 @@ unique_ptr StatementGenerator::GenerateDelete() { if (entry.type == CatalogType::TABLE_ENTRY) { auto result = make_uniq(); result->table_name = entry.name; - delete_statement->table = std::move(result); + delete_statement->node->table = std::move(result); } else { - delete_statement->table = GenerateTableRef(); + delete_statement->node->table = GenerateTableRef(); } } else { - delete_statement->table = GenerateTableRef(); + delete_statement->node->table = GenerateTableRef(); } return delete_statement; @@ -302,7 +303,7 @@ void StatementGenerator::GenerateCTEs(QueryNode &node) { } while (RandomPercentage(20)) { auto cte = make_uniq(); - cte->query = unique_ptr_cast(GenerateSelect()); + cte->query_node = std::move(unique_ptr_cast(GenerateSelect())->node); for (idx_t i = 0; i < 1 + RandomValue(10); i++) { cte->aliases.push_back(GenerateIdentifier()); } @@ -564,7 +565,7 @@ unique_ptr StatementGenerator::GenerateTableFunctionRef() { auto result = make_uniq(); vector> children; - for (idx_t i = 0; i < table_function.arguments.size(); i++) { + for (idx_t i = 0; i < table_function.GetArguments().size(); i++) { children.push_back(GenerateConstant()); } vector names; @@ -575,7 +576,7 @@ unique_ptr StatementGenerator::GenerateTableFunctionRef() { auto name = Choose(names); names.erase(std::find(names.begin(), names.end(), name)); auto expr = GenerateConstant(); - expr->alias = name; + expr->SetAlias(name); children.push_back(std::move(expr)); } result->function = make_uniq(entry.name, std::move(children)); @@ -761,10 +762,12 @@ unique_ptr StatementGenerator::GenerateFunction() { auto offset = RandomValue(scalar_entry.functions.Size()); auto actual_function = scalar_entry.functions.GetFunctionByOffset(offset); name = scalar_entry.name; - arguments = actual_function.arguments; - min_parameters = actual_function.arguments.size(); + for (auto &arg : actual_function.GetSignature().GetParameters()) { + arguments.push_back(arg.GetType()); + } + min_parameters = actual_function.GetSignature().GetParameterCount(); max_parameters = min_parameters; - if (actual_function.varargs.id() != LogicalTypeId::INVALID) { + if (actual_function.GetSignature().GetVarArgs().id() != LogicalTypeId::INVALID) { max_parameters += 5; } break; @@ -775,9 +778,9 @@ unique_ptr StatementGenerator::GenerateFunction() { aggregate_entry.functions.GetFunctionByOffset(RandomValue(aggregate_entry.functions.Size())); name = aggregate_entry.name; - min_parameters = actual_function.arguments.size(); + min_parameters = actual_function.GetSignature().GetParameterCount(); max_parameters = min_parameters; - if (actual_function.varargs.id() != LogicalTypeId::INVALID) { + if (actual_function.GetVarArgs().id() != LogicalTypeId::INVALID) { max_parameters += 5; } if (RandomPercentage(10) && !in_window) { @@ -806,6 +809,9 @@ unique_ptr StatementGenerator::GenerateFunction() { max_parameters = min_parameters; break; } + case CatalogType::WINDOW_FUNCTION_ENTRY: + // FIXME: Use the function, not a random builtin + return GenerateWindowFunction(); default: throw InternalException("StatementGenerator::GenerateFunction"); } @@ -940,7 +946,7 @@ unique_ptr StatementGenerator::GenerateWindowFunction(optional if (function) { type = ExpressionType::WINDOW_AGGREGATE; name = function->name; - min_parameters = function->arguments.size(); + min_parameters = function->GetSignature().GetParameterCount(); max_parameters = min_parameters; } else { name = Choose({"rank", "rank_dense", "percent_rank", "row_number", "first_value", "last_value", @@ -957,9 +963,12 @@ unique_ptr StatementGenerator::GenerateWindowFunction(optional case ExpressionType::WINDOW_NTILE: case ExpressionType::WINDOW_FIRST_VALUE: case ExpressionType::WINDOW_LAST_VALUE: + min_parameters = 1; + break; case ExpressionType::WINDOW_LEAD: case ExpressionType::WINDOW_LAG: min_parameters = 1; + min_parameters = 3; break; case ExpressionType::WINDOW_NTH_VALUE: min_parameters = 2; @@ -970,7 +979,7 @@ unique_ptr StatementGenerator::GenerateWindowFunction(optional max_parameters = min_parameters; } WindowChecker checker(*this); - auto result = make_uniq(type, INVALID_CATALOG, INVALID_SCHEMA, std::move(name)); + auto result = make_uniq(SYSTEM_CATALOG, DEFAULT_SCHEMA, std::move(name)); result->children = GenerateChildren(min_parameters, max_parameters); while (RandomPercentage(50)) { result->partitions.push_back(GenerateExpression()); @@ -1014,15 +1023,6 @@ unique_ptr StatementGenerator::GenerateWindowFunction(optional default: break; } - switch (type) { - case ExpressionType::WINDOW_LEAD: - case ExpressionType::WINDOW_LAG: - result->offset_expr = RandomExpression(30); - result->default_expr = RandomExpression(30); - break; - default: - break; - } return std::move(result); } @@ -1279,25 +1279,25 @@ bool StatementGenerator::FunctionArgumentsAlwaysNull(const string &name) { return always_null_functions.find(name) != always_null_functions.end(); } -string StatementGenerator::GenerateTestAllTypes(BaseScalarFunction &base_function) { +string StatementGenerator::GenerateTestAllTypes(SimpleFunction &base_function) { auto select = make_uniq(); auto node = make_uniq(); - bool always_null = FunctionArgumentsAlwaysNull(base_function.name); + bool always_null = FunctionArgumentsAlwaysNull(base_function.GetName()); vector> children; - for (auto &arg : base_function.arguments) { + for (auto ¶m : base_function.GetSignature().GetParameters()) { // look up the type unique_ptr argument; if (!always_null) { for (auto &test_type : generator_context->test_types) { - if (test_type.type.id() == arg.id()) { + if (test_type.type.id() == param.GetType().id()) { argument = make_uniq(test_type.name); } } } if (!argument) { - argument = make_uniq(Value(arg)); + argument = make_uniq(Value(param.GetType())); } children.push_back(std::move(argument)); } @@ -1305,14 +1305,14 @@ string StatementGenerator::GenerateTestAllTypes(BaseScalarFunction &base_functio from_clause->table_name = "all_types"; node->from_table = std::move(from_clause); - auto function_expr = make_uniq(base_function.name, std::move(children)); + auto function_expr = make_uniq(base_function.GetName(), std::move(children)); node->select_list.push_back(std::move(function_expr)); select->node = std::move(node); return select->ToString(); } -string StatementGenerator::GenerateTestVectorTypes(BaseScalarFunction &base_function) { +string StatementGenerator::GenerateTestVectorTypes(SimpleFunction &base_function) { auto select = make_uniq(); auto node = make_uniq(); @@ -1321,17 +1321,17 @@ string StatementGenerator::GenerateTestVectorTypes(BaseScalarFunction &base_func vector> children; vector> test_vector_types; vector column_aliases; - for (auto &arg : base_function.arguments) { + for (auto ¶m : base_function.GetSignature().GetParameters()) { unique_ptr argument; if (!always_null) { string argument_name = "c" + to_string(column_aliases.size() + 1); column_aliases.push_back(argument_name); argument = make_uniq(std::move(argument_name)); auto constant_expr = make_uniq(Value()); - auto cast = make_uniq(arg, std::move(constant_expr)); + auto cast = make_uniq(param.GetType(), std::move(constant_expr)); test_vector_types.push_back(std::move(cast)); } else { - argument = make_uniq(Value(arg)); + argument = make_uniq(Value(param.GetType())); } children.push_back(std::move(argument)); } diff --git a/src/statement_simplifier.cpp b/src/statement_simplifier.cpp index 4602928..3b46e19 100644 --- a/src/statement_simplifier.cpp +++ b/src/statement_simplifier.cpp @@ -8,8 +8,11 @@ #include "duckdb/parser/expression/list.hpp" #include "duckdb/parser/statement/delete_statement.hpp" #include "duckdb/parser/statement/insert_statement.hpp" +#include "duckdb/parser/query_node/insert_query_node.hpp" #include "duckdb/parser/statement/prepare_statement.hpp" #include "duckdb/parser/statement/update_statement.hpp" +#include "duckdb/parser/query_node/update_query_node.hpp" +#include "duckdb/parser/query_node/delete_query_node.hpp" #include "duckdb/parser/statement/select_statement.hpp" #endif @@ -206,7 +209,7 @@ void StatementSimplifier::Simplify(CommonTableExpressionMap &cte) { SimplifyMap(cte.map); for (auto &cte_child : cte.map) { // simplify individual ctes - Simplify(cte_child.second->query->node); + Simplify(cte_child.second->query_node); } } @@ -357,8 +360,6 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrcte_map); + if (stmt.node->select_statement) { + Simplify(*stmt.node->select_statement); + } + SimplifyList(stmt.node->returning_list); } void StatementSimplifier::Simplify(DeleteStatement &stmt) { - Simplify(stmt.cte_map); - SimplifyOptional(stmt.condition); - SimplifyExpression(stmt.condition); - SimplifyList(stmt.using_clauses); - SimplifyList(stmt.returning_list); + Simplify(stmt.node->cte_map); + SimplifyOptional(stmt.node->condition); + SimplifyExpression(stmt.node->condition); + SimplifyList(stmt.node->using_clauses); + SimplifyList(stmt.node->returning_list); } void StatementSimplifier::Simplify(UpdateSetInfo &info) { @@ -432,11 +435,11 @@ void StatementSimplifier::Simplify(PrepareStatement &stmt) { } void StatementSimplifier::Simplify(UpdateStatement &stmt) { - Simplify(stmt.cte_map); - SimplifyOptional(stmt.from_table); - D_ASSERT(stmt.set_info); - Simplify(*stmt.set_info); - SimplifyList(stmt.returning_list); + Simplify(stmt.node->cte_map); + SimplifyOptional(stmt.node->from_table); + D_ASSERT(stmt.node->set_info); + Simplify(*stmt.node->set_info); + SimplifyList(stmt.node->returning_list); } void StatementSimplifier::Simplify(SQLStatement &stmt) {