From 88bb5a2f6eca18558c93974e1d326ad21914678e Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 5 Mar 2026 13:01:21 +0000 Subject: [PATCH] Revert "chore: migrate CI from spiral AWS account to vortex (#6783)" This reverts commit a33279681523a9c761e6bd6654541478a84a859b. --- .github/runs-on.yml | 4 ++-- .github/workflows/bench-pr.yml | 4 ++-- .github/workflows/bench.yml | 14 +++++++------- .github/workflows/nightly-bench.yml | 4 ++-- .github/workflows/sql-benchmarks.yml | 16 ++++++++-------- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 29bdf810fda..7970bf75e17 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -3,9 +3,9 @@ images: platform: "linux" arch: "x64" name: "vortex-ci-*" - owner: "245040174862" + owner: "375504701696" vortex-ci-arm64: platform: "linux" arch: "arm64" name: "vortex-ci-*" - owner: "245040174862" + owner: "375504701696" diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 99cfd1657a9..7e6cd69b4aa 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -94,7 +94,7 @@ jobs: if: github.event.pull_request.head.repo.fork == false uses: aws-actions/configure-aws-credentials@v5 with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: us-east-1 - name: Install uv @@ -115,7 +115,7 @@ jobs: | jq -r '.workflow_runs[].head_sha' \ ) - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request + python3 scripts/s3-download.py s3://vortex-benchmark-results-database/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz | grep $base_commit_sha > base.json echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c745319fb5e..9743a554eae 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -20,7 +20,7 @@ jobs: - name: Setup AWS CLI uses: aws-actions/configure-aws-credentials@v5 with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: us-east-1 - name: Upload Commit Metadata shell: bash @@ -29,7 +29,7 @@ jobs: sudo apt-get update && sudo apt-get install -y jq bash scripts/commit-json.sh > new-commit.json - bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json + bash scripts/cat-s3.sh vortex-benchmark-results-database commits.json new-commit.json bench: timeout-minutes: 120 @@ -94,13 +94,13 @@ jobs: - name: Setup AWS CLI uses: aws-actions/configure-aws-credentials@v5 with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: us-east-1 - name: Upload Benchmark Results shell: bash run: | - bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json + bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz results.json - name: Alert incident.io if: failure() @@ -137,7 +137,7 @@ jobs: "subcommand": "tpch", "name": "TPC-H SF=1 on S3", "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", "scale_factor": "1.0" }, @@ -154,7 +154,7 @@ jobs: "subcommand": "tpch", "name": "TPC-H SF=10 on S3", "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", "scale_factor": "10.0" }, @@ -184,7 +184,7 @@ jobs: "subcommand": "fineweb", "name": "FineWeb S3", "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact" }, { diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index d207291c16c..adfb94cbce4 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -46,7 +46,7 @@ jobs: "subcommand": "tpch", "name": "TPC-H on S3", "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex", "scale_factor": "10.0" }, @@ -62,7 +62,7 @@ jobs: "subcommand": "tpch", "name": "TPC-H on S3", "local_dir": "vortex-bench/data/tpch/100.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/", "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex", "scale_factor": "100.0" }, diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 336eac4c3c9..f98e7ef87d4 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -35,7 +35,7 @@ on: "subcommand": "tpch", "name": "TPC-H SF=1 on S3", "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", "scale_factor": "1.0" }, @@ -51,7 +51,7 @@ on: "subcommand": "tpch", "name": "TPC-H SF=10 on S3", "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", "scale_factor": "10.0" }, @@ -81,7 +81,7 @@ on: "subcommand": "fineweb", "name": "FineWeb S3", "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", + "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", "scale_factor": "100" }, @@ -161,14 +161,14 @@ jobs: if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false uses: aws-actions/configure-aws-credentials@v5 with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole + role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole aws-region: us-east-1 - name: Upload data if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: - AWS_REGION: "us-east-1" + AWS_REGION: "eu-west-1" run: | aws s3 rm --recursive ${{ matrix.remote_storage }} aws s3 cp --recursive ${{matrix.local_dir}} ${{ matrix.remote_storage }} @@ -200,7 +200,7 @@ jobs: if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: - AWS_REGION: "us-east-1" + AWS_REGION: "eu-west-1" OTEL_SERVICE_NAME: "vortex-bench" OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf" OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}" @@ -231,7 +231,7 @@ jobs: | jq -r '.workflow_runs[].head_sha' \ ) - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request + python3 scripts/s3-download.py s3://vortex-benchmark-results-database/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz | grep $base_commit_sha > base.json echo '# Benchmarks: ${{ matrix.name }}' > comment.md @@ -264,7 +264,7 @@ jobs: if: inputs.mode == 'develop' shell: bash run: | - bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json + bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz results.json - name: Alert incident.io if: failure() && inputs.mode == 'develop'