From 8d66ee8fea02fbde4c940a51277c608cdcb831e5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jan 2026 11:55:51 -0800 Subject: [PATCH 1/5] group commands in build-all-rapids-repos.yml --- .github/workflows/build-all-rapids-repos.yml | 83 +++++++++++++++---- .github/workflows/test-rapids-build-times.yml | 16 ++-- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index bccf5710..eb5e9363 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -77,30 +77,81 @@ jobs: # Verify sccache cache location sccache --show-adv-stats; + function begin_group() { + local blue="34" + echo -e "::group::\e[${blue}m${1:-}\e[0m" + } + + function end_group() { + local name="${1:-}" + local build_status="${2:-0}" + local red="31" + + echo "::endgroup::" + if [ "$build_status" -ne 0 ]; then + echo -e "::error::\e[${red}m ${name} - Failed (⬆️ click above for full log ⬆️)\e[0m" + fi + } + + function run_command() { + local -; + local group="${1:-}" + shift + local command=("$@") + local status=0 + + begin_group "$group" + echo "Working directory: $(pwd)" + echo "Running command: ${command[*]}" + set +e + "${command[@]}" || status=$? + set -e + end_group "$group" "$status" + return "$status" + } + + if test -n "${DISABLE_SCCACHE:+x}"; then + . /opt/devcontainer/bin/update-envvars.sh; + for VAR in RUSTC_WRAPPER CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER CMAKE_CUDA_COMPILER_LAUNCHER; do + reset_envvar "$VAR"; + override_envvar "$VAR" ""; + done + fi + # Clone all the repos - clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules; + run_command "Clone RAPIDS repositories" \ + clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; + + run_command "Create RAPIDS python environment" \ + rapids-post-start-command; # Configure all the C++ libs - time configure-all \ - -j0 \ - -GNinja \ - -Wno-dev \ - -DBUILD_TESTS=ON \ - -DBUILD_BENCHMARKS=ON \ - -DBUILD_PRIMS_BENCH=ON \ - -DBUILD_SHARED_LIBS=ON \ - -DRAFT_COMPILE_LIBRARY=ON \ - -DBUILD_CUGRAPH_MG_TESTS=ON ; + run_command "Configure C++ libraries" bash -c "\ + time configure-all \ + -j0 \ + -GNinja \ + -Wno-dev \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_PRIMS_BENCH=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DRAFT_COMPILE_LIBRARY=ON \ + -DBUILD_CUGRAPH_MG_TESTS=ON 2>&1 \ + | tee -a telemetry-artifacts/build.log"; # Build all the C++ libs - time build-all-cpp -j0; + run_command "Build C++ libraries" bash -c "\ + time build-all-cpp -j0 -v 2>&1 | tee -a telemetry-artifacts/build.log"; # Build all the Python libs - time build-all-python -j0; + run_command "Build Python libraries" bash -c "\ + time build-all-python -j0 -v 2>&1 | tee -a telemetry-artifacts/build.log"; # Print cache and dist stats - sccache --show-adv-stats; + run_command "sccache stats" bash -c "\ + sccache --show-adv-stats | tee -a telemetry-artifacts/sccache-stats.txt"; # Print build times - find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ - | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found + run_command "Build times" bash -c "\ + find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ + | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || :" # Nonfatal if not found diff --git a/.github/workflows/test-rapids-build-times.yml b/.github/workflows/test-rapids-build-times.yml index 57a4cf26..03b1931e 100644 --- a/.github/workflows/test-rapids-build-times.yml +++ b/.github/workflows/test-rapids-build-times.yml @@ -19,14 +19,16 @@ jobs: fail-fast: false matrix: include: - - name: 'sccache: no, dist: no' + - name: 'no sccache' env: | DISABLE_SCCACHE=1 - - name: 'sccache: yes, dist: no' + SCCACHE_NO_CACHE=1 + SCCACHE_NO_DIST_COMPILE=1 + - name: 'recache, local' env: | SCCACHE_RECACHE=1 SCCACHE_NO_DIST_COMPILE=1 - - name: 'sccache: yes, dist: yes' + - name: 'recache, remote' env: | SCCACHE_RECACHE=1 @@ -42,9 +44,9 @@ jobs: fail-fast: false matrix: include: - - name: 'preprocessor cache: no' - env: | - SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0 - - name: 'preprocessor cache: yes' + - name: 'preprocessor cache' env: | SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=1 + - name: 'no preprocessor cache' + env: | + SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0 From 627498386ccc5be67deb6d24e725f48a28ba92bb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jan 2026 14:18:20 -0800 Subject: [PATCH 2/5] don't do set +e --- .github/workflows/build-all-rapids-repos.yml | 52 ++++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index eb5e9363..b3d7747f 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -74,9 +74,6 @@ jobs: SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 ${{ inputs.env }} build_command: | - # Verify sccache cache location - sccache --show-adv-stats; - function begin_group() { local blue="34" echo -e "::group::\e[${blue}m${1:-}\e[0m" @@ -95,19 +92,22 @@ jobs: function run_command() { local -; - local group="${1:-}" - shift - local command=("$@") - local status=0 - - begin_group "$group" - echo "Working directory: $(pwd)" - echo "Running command: ${command[*]}" - set +e - "${command[@]}" || status=$? - set -e - end_group "$group" "$status" - return "$status" + set -euo pipefail; + + local group="${1:-}"; + shift; + local command=("$@"); + local exit_code="0"; + + begin_group "$group"; + + echo "Working directory: $(pwd)"; + echo "Running command: ${command[*]}"; + "${command[@]}" || exit_code=$?; + + end_group "$group" "$exit_code" + + return "$exit_code" } if test -n "${DISABLE_SCCACHE:+x}"; then @@ -127,16 +127,16 @@ jobs: # Configure all the C++ libs run_command "Configure C++ libraries" bash -c "\ - time configure-all \ - -j0 \ - -GNinja \ - -Wno-dev \ - -DBUILD_TESTS=ON \ - -DBUILD_BENCHMARKS=ON \ - -DBUILD_PRIMS_BENCH=ON \ - -DBUILD_SHARED_LIBS=ON \ - -DRAFT_COMPILE_LIBRARY=ON \ - -DBUILD_CUGRAPH_MG_TESTS=ON 2>&1 \ + time configure-all \ + -j0 \ + -GNinja \ + -Wno-dev \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_PRIMS_BENCH=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DRAFT_COMPILE_LIBRARY=ON \ + -DBUILD_CUGRAPH_MG_TESTS=ON 2>&1 \ | tee -a telemetry-artifacts/build.log"; # Build all the C++ libs From f62b810e0aabac3734f56c98d339b58d604c87f4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 27 Jan 2026 14:23:37 -0800 Subject: [PATCH 3/5] use -j in sccache-disabled job --- .github/workflows/build-all-rapids-repos.yml | 7 ++++--- .github/workflows/test-rapids-build-times.yml | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index b3d7747f..5255c449 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -67,6 +67,7 @@ jobs: env: | CONDA_ENV_CREATE_QUIET=1 INCLUDE_REPOS="${{ matrix.libs }}" + PARALLEL_LEVEL=0 SCCACHE_IDLE_TIMEOUT=0 SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_MAX_RETRIES=inf @@ -128,7 +129,7 @@ jobs: # Configure all the C++ libs run_command "Configure C++ libraries" bash -c "\ time configure-all \ - -j0 \ + -j${PARALLEL_LEVEL} \ -GNinja \ -Wno-dev \ -DBUILD_TESTS=ON \ @@ -141,11 +142,11 @@ jobs: # Build all the C++ libs run_command "Build C++ libraries" bash -c "\ - time build-all-cpp -j0 -v 2>&1 | tee -a telemetry-artifacts/build.log"; + time build-all-cpp -j${PARALLEL_LEVEL} -v 2>&1 | tee -a telemetry-artifacts/build.log"; # Build all the Python libs run_command "Build Python libraries" bash -c "\ - time build-all-python -j0 -v 2>&1 | tee -a telemetry-artifacts/build.log"; + time build-all-python -j${PARALLEL_LEVEL} -v 2>&1 | tee -a telemetry-artifacts/build.log"; # Print cache and dist stats run_command "sccache stats" bash -c "\ diff --git a/.github/workflows/test-rapids-build-times.yml b/.github/workflows/test-rapids-build-times.yml index 03b1931e..1873de24 100644 --- a/.github/workflows/test-rapids-build-times.yml +++ b/.github/workflows/test-rapids-build-times.yml @@ -21,9 +21,11 @@ jobs: include: - name: 'no sccache' env: | + PARALLEL_LEVEL= DISABLE_SCCACHE=1 SCCACHE_NO_CACHE=1 SCCACHE_NO_DIST_COMPILE=1 + MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=1 - name: 'recache, local' env: | SCCACHE_RECACHE=1 From 74b042be8cd71c4c28eb91963571f6205e1e2bde Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 28 Jan 2026 18:10:04 -0800 Subject: [PATCH 4/5] remove -v --- .github/workflows/build-all-rapids-repos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 5255c449..0c52e9ce 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -142,11 +142,11 @@ jobs: # Build all the C++ libs run_command "Build C++ libraries" bash -c "\ - time build-all-cpp -j${PARALLEL_LEVEL} -v 2>&1 | tee -a telemetry-artifacts/build.log"; + time build-all-cpp -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log"; # Build all the Python libs run_command "Build Python libraries" bash -c "\ - time build-all-python -j${PARALLEL_LEVEL} -v 2>&1 | tee -a telemetry-artifacts/build.log"; + time build-all-python -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log"; # Print cache and dist stats run_command "sccache stats" bash -c "\ From 67b6bc67ce623a41f16f7b28c8bdfcdc3ae60226 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 29 Jan 2026 11:39:04 -0800 Subject: [PATCH 5/5] add branch and node_type inputs --- .github/workflows/build-all-rapids-repos.yml | 12 ++++++++++-- .github/workflows/test-rapids-build-times.yml | 9 +++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 0c52e9ce..c8a94ae0 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -8,6 +8,14 @@ on: env: type: string required: false + branch: + type: string + required: false + default: main + node_type: + type: string + required: false + default: cpu16 matrix: type: string required: false @@ -56,7 +64,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.9", "13.1"]' - node_type: cpu16 + node_type: ${{ inputs.node_type }} rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 # 1. Prohibit sccache from shutting down automatically @@ -121,7 +129,7 @@ jobs: # Clone all the repos run_command "Clone RAPIDS repositories" \ - clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; + clone-all -j$(nproc) -b ${{ inputs.branch }} -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; run_command "Create RAPIDS python environment" \ rapids-post-start-command; diff --git a/.github/workflows/test-rapids-build-times.yml b/.github/workflows/test-rapids-build-times.yml index 1873de24..53efc451 100644 --- a/.github/workflows/test-rapids-build-times.yml +++ b/.github/workflows/test-rapids-build-times.yml @@ -6,6 +6,11 @@ concurrency: on: workflow_dispatch: + inputs: + branch: + type: string + required: false + default: main jobs: uncached-builds: @@ -13,8 +18,10 @@ jobs: secrets: inherit uses: ./.github/workflows/build-all-rapids-repos.yml with: + branch: ${{ inputs.branch }} env: ${{ matrix.env }} matrix: '{ "include": [{ "libs": "" }] }' + node_type: cpu32 strategy: fail-fast: false matrix: @@ -40,8 +47,10 @@ jobs: secrets: inherit uses: ./.github/workflows/build-all-rapids-repos.yml with: + branch: ${{ inputs.branch }} env: ${{ matrix.env }} matrix: '{ "include": [{ "libs": "" }] }' + node_type: cpu32 strategy: fail-fast: false matrix: