diff --git a/.github/workflows/validate-results.yml b/.github/workflows/validate-results.yml new file mode 100644 index 0000000000..273e430b22 --- /dev/null +++ b/.github/workflows/validate-results.yml @@ -0,0 +1,14 @@ +name: "Validate results" + +on: + pull_request: + push: + branches: + - main + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: python3 validate-results.py diff --git a/pandas/results/20260218/c6a.metal.json b/pandas/results/20260218/c6a.metal.json index ead3f84a38..67295d6294 100644 --- a/pandas/results/20260218/c6a.metal.json +++ b/pandas/results/20260218/c6a.metal.json @@ -52,6 +52,7 @@ [38.508, 36.927, 38.915], [37.943, 40.408, 36.671], [39.536, 36.998, 39.238], - [38.46, 37.272, 36.735] + [38.46, 37.272, 36.735], + [null, null, null] ] } diff --git a/snowflake/results/20220701/l.json b/snowflake/results/20220701/l.json index 8614d2f2f5..1fae27df66 100644 --- a/snowflake/results/20220701/l.json +++ b/snowflake/results/20220701/l.json @@ -42,7 +42,7 @@ [0.340,0.296,0.383], [0.695,0.314,0.368], [0.628,0.658,0.637], -[1.511,1.385,1,440], +[1.511,1.385,1.440], [1.390,1.418,1.322], [1.107,0.687,0.537], [1.026,0.737,0.659], diff --git a/validate-results.py b/validate-results.py new file mode 100644 index 0000000000..9fa8c8527f --- /dev/null +++ b/validate-results.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +import sys +from datetime import datetime +from pathlib import Path + + +EXPECTED_QUERIES = 43 +EXPECTED_RUNS = 3 +OUTLIER_SECONDS = 24 * 60 * 60 +DATE_DIR_RE = re.compile(r"^\d{8}$") +ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +SKIP_SYSTEMS = {"hardware", "versions", "gravitons"} + + +def find_result_files(root): + files = [] + for path in root.glob("*/results/*/*.json"): + relative_path = path.relative_to(root) + if relative_path.parts[0] in SKIP_SYSTEMS: + continue + files.append(relative_path) + return sorted(files) + + +def active_result_files(files): + latest = {} + for path in files: + system, _, date_dir, filename = path.parts[:4] + if not DATE_DIR_RE.match(date_dir): + continue + key = (system, filename) + if key not in latest or date_dir > latest[key].parts[2]: + latest[key] = path + return set(latest.values()) + + +def is_iso_date(value): + if not isinstance(value, str) or not ISO_DATE_RE.match(value): + return False + try: + datetime.strptime(value, "%Y-%m-%d") + except ValueError: + return False + return True + + +def add(problems, severity, path, message): + problems.append((severity, str(path), message)) + + +def validate_metadata(path, data, problems): + _, _, date_dir, _ = path.parts[:4] + + if not DATE_DIR_RE.match(date_dir): + add(problems, "error", path, f"result directory date must be YYYYMMDD, got {date_dir!r}") + + if not isinstance(data.get("system"), str) or not data["system"].strip(): + add(problems, "error", path, "system must be a non-empty string") + + if not isinstance(data.get("machine"), str) or not data["machine"].strip(): + add(problems, "error", path, "machine must be a non-empty string") + + if not is_iso_date(data.get("date")): + add(problems, "error", path, "date must be a valid YYYY-MM-DD string") + else: + expected = f"{date_dir[:4]}-{date_dir[4:6]}-{date_dir[6:8]}" + if data["date"] != expected: + add(problems, "warning", path, f"date {data['date']!r} differs from directory date {expected!r}") + + tags = data.get("tags") + if not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags): + add(problems, "error", path, "tags must be an array of strings") + + +def validate_result_matrix(path, data, active, problems): + result = data.get("result") + tags = data.get("tags") if isinstance(data.get("tags"), list) else [] + is_historical = "historical" in tags + severity = "error" if active and not is_historical else "warning" + + if not isinstance(result, list): + add(problems, severity, path, "result must be an array") + return + + if len(result) != EXPECTED_QUERIES: + add(problems, severity, path, f"result must contain {EXPECTED_QUERIES} query rows, got {len(result)}") + + for query_index, row in enumerate(result, 1): + if not isinstance(row, list): + add(problems, severity, path, f"result row {query_index} must be an array") + continue + + if len(row) != EXPECTED_RUNS: + add(problems, severity, path, f"result row {query_index} must contain {EXPECTED_RUNS} timings, got {len(row)}") + + for run_index, value in enumerate(row, 1): + if value is None: + continue + if not isinstance(value, (int, float)): + add(problems, severity, path, f"result row {query_index}, run {run_index} must be a number or null") + continue + if value < 0: + add(problems, severity, path, f"result row {query_index}, run {run_index} must not be negative") + if value > OUTLIER_SECONDS: + add( + problems, + "warning", + path, + f"result row {query_index}, run {run_index} is unusually high: {value} seconds", + ) + + +def validate_file(root, path, active, problems): + try: + with (root / path).open(encoding="utf-8") as handle: + data = json.load(handle) + except json.JSONDecodeError as exc: + add(problems, "error", path, f"invalid JSON: {exc}") + return + + if not isinstance(data, dict): + add(problems, "error", path, "top-level JSON value must be an object") + return + + if "error" in data: + if not isinstance(data["error"], str) or not data["error"].strip(): + add(problems, "error", path, "error entries must contain a non-empty error string") + return + + validate_metadata(path, data, problems) + validate_result_matrix(path, data, active, problems) + + +def main(): + parser = argparse.ArgumentParser(description="Validate main ClickBench result JSON files.") + parser.add_argument("root", nargs="?", default=".", help="repository root") + args = parser.parse_args() + + root = Path(args.root).resolve() + files = find_result_files(root) + active_files = active_result_files(files) + problems = [] + + for path in files: + validate_file(root, path, path in active_files, problems) + + warnings = [problem for problem in problems if problem[0] == "warning"] + errors = [problem for problem in problems if problem[0] == "error"] + + for severity, path, message in problems: + print(f"{severity}: {path}: {message}", file=sys.stderr) + + print(f"Validated {len(files)} result files ({len(active_files)} active).", file=sys.stderr) + if warnings: + print(f"Warnings: {len(warnings)}", file=sys.stderr) + if errors: + print(f"Errors: {len(errors)}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())