group commands in build-all-rapids-repos.yml

.github/workflows/build-all-rapids-repos.yml

@@ -8,6 +8,14 @@ on:
       env:
         type: string
         required: false
+      branch:
+        type: string
+        required: false
+        default: main
+      node_type:
+        type: string
+        required: false
+        default: cpu16
       matrix:
         type: string
         required: false
@@ -56,7 +64,7 @@ jobs:
     with:
       arch: '["amd64", "arm64"]'
       cuda: '["12.9", "13.1"]'
-      node_type: cpu16
+      node_type: ${{ inputs.node_type }}
       rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN
       timeout-minutes: 720
       # 1. Prohibit sccache from shutting down automatically
@@ -67,40 +75,92 @@ jobs:
       env: |
         CONDA_ENV_CREATE_QUIET=1
         INCLUDE_REPOS="${{ matrix.libs }}"
+        PARALLEL_LEVEL=0
         SCCACHE_IDLE_TIMEOUT=0
         SCCACHE_SERVER_LOG=sccache=debug
         SCCACHE_DIST_MAX_RETRIES=inf
         SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false
         SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1
         ${{ inputs.env }}
       build_command: |
-        # Verify sccache cache location
-        sccache --show-adv-stats;
+        function begin_group() {
+          local blue="34"
+          echo -e "::group::\e[${blue}m${1:-}\e[0m"
+        }
+
+        function end_group() {
+          local name="${1:-}"
+          local build_status="${2:-0}"
+          local red="31"
+
+          echo "::endgroup::"
+          if [ "$build_status" -ne 0 ]; then
+            echo -e "::error::\e[${red}m ${name} - Failed (⬆️ click above for full log ⬆️)\e[0m"
+          fi
+        }
+
+        function run_command() {
+          local -;
+          set -euo pipefail;
+
+          local group="${1:-}";
+          shift;
+          local command=("$@");
+          local exit_code="0";
+
+          begin_group "$group";
+
+          echo "Working directory: $(pwd)";
+          echo "Running command: ${command[*]}";
+          "${command[@]}" || exit_code=$?;
+
+          end_group "$group" "$exit_code"
+
+          return "$exit_code"
+        }
+
+        if test -n "${DISABLE_SCCACHE:+x}"; then
+          . /opt/devcontainer/bin/update-envvars.sh;
+          for VAR in RUSTC_WRAPPER CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER CMAKE_CUDA_COMPILER_LAUNCHER; do
+            reset_envvar "$VAR";
+            override_envvar "$VAR" "";
+          done
+        fi
 
         # Clone all the repos
-        clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules;
+        run_command "Clone RAPIDS repositories" \
+          clone-all -j$(nproc) -b ${{ inputs.branch }} -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env;
+
+        run_command "Create RAPIDS python environment" \
+          rapids-post-start-command;
 
         # Configure all the C++ libs
-        time configure-all            \
-          -j0                         \
-          -GNinja                     \
-          -Wno-dev                    \
-          -DBUILD_TESTS=ON            \
-          -DBUILD_BENCHMARKS=ON       \
-          -DBUILD_PRIMS_BENCH=ON      \
-          -DBUILD_SHARED_LIBS=ON      \
-          -DRAFT_COMPILE_LIBRARY=ON   \
-          -DBUILD_CUGRAPH_MG_TESTS=ON ;
+        run_command "Configure C++ libraries" bash -c "\
+          time configure-all                           \
+            -j${PARALLEL_LEVEL}                        \
+            -GNinja                                    \
+            -Wno-dev                                   \
+            -DBUILD_TESTS=ON                           \
+            -DBUILD_BENCHMARKS=ON                      \
+            -DBUILD_PRIMS_BENCH=ON                     \
+            -DBUILD_SHARED_LIBS=ON                     \
+            -DRAFT_COMPILE_LIBRARY=ON                  \
+            -DBUILD_CUGRAPH_MG_TESTS=ON 2>&1           \
+          | tee -a telemetry-artifacts/build.log";
 
         # Build all the C++ libs
-        time build-all-cpp -j0;
+        run_command "Build C++ libraries" bash -c "\
+          time build-all-cpp -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log";
 
         # Build all the Python libs
-        time build-all-python -j0;
+        run_command "Build Python libraries" bash -c "\
+          time build-all-python -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log";
 
         # Print cache and dist stats
-        sccache --show-adv-stats;
+        run_command "sccache stats" bash -c "\
+          sccache --show-adv-stats | tee -a telemetry-artifacts/sccache-stats.txt";
 
         # Print build times
-        find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \
-          | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found
+        run_command "Build times" bash -c "\
+          find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \
+            | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || :" # Nonfatal if not found

.github/workflows/test-rapids-build-times.yml

@@ -6,27 +6,38 @@ concurrency:
 
 on:
   workflow_dispatch:
+    inputs:
+      branch:
+        type: string
+        required: false
+        default: main
 
 jobs:
   uncached-builds:
     name: ${{ matrix.name }}
     secrets: inherit
     uses: ./.github/workflows/build-all-rapids-repos.yml
     with:
+      branch: ${{ inputs.branch }}
       env: ${{ matrix.env }}
       matrix: '{ "include": [{ "libs": "" }] }'
+      node_type: cpu32
     strategy:
       fail-fast: false
       matrix:
         include:
-          - name: 'sccache: no, dist: no'
+          - name: 'no sccache'
             env: |
+              PARALLEL_LEVEL=
               DISABLE_SCCACHE=1
-          - name: 'sccache: yes, dist: no'
+              SCCACHE_NO_CACHE=1
+              SCCACHE_NO_DIST_COMPILE=1
+              MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=1
+          - name: 'recache, local'
             env: |
               SCCACHE_RECACHE=1
               SCCACHE_NO_DIST_COMPILE=1
-          - name: 'sccache: yes, dist: yes'
+          - name: 'recache, remote'
             env: |
               SCCACHE_RECACHE=1
 
@@ -36,15 +47,17 @@ jobs:
     secrets: inherit
     uses: ./.github/workflows/build-all-rapids-repos.yml
     with:
+      branch: ${{ inputs.branch }}
       env: ${{ matrix.env }}
       matrix: '{ "include": [{ "libs": "" }] }'
+      node_type: cpu32
     strategy:
       fail-fast: false
       matrix:
         include:
-          - name: 'preprocessor cache: no'
-            env: |
-              SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0
-          - name: 'preprocessor cache: yes'
+          - name: 'preprocessor cache'
             env: |
               SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=1
+          - name: 'no preprocessor cache'
+            env: |
+              SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0