diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index bccf5710..c8a94ae0 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -8,6 +8,14 @@ on: env: type: string required: false + branch: + type: string + required: false + default: main + node_type: + type: string + required: false + default: cpu16 matrix: type: string required: false @@ -56,7 +64,7 @@ jobs: with: arch: '["amd64", "arm64"]' cuda: '["12.9", "13.1"]' - node_type: cpu16 + node_type: ${{ inputs.node_type }} rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN timeout-minutes: 720 # 1. Prohibit sccache from shutting down automatically @@ -67,6 +75,7 @@ jobs: env: | CONDA_ENV_CREATE_QUIET=1 INCLUDE_REPOS="${{ matrix.libs }}" + PARALLEL_LEVEL=0 SCCACHE_IDLE_TIMEOUT=0 SCCACHE_SERVER_LOG=sccache=debug SCCACHE_DIST_MAX_RETRIES=inf @@ -74,33 +83,84 @@ jobs: SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 ${{ inputs.env }} build_command: | - # Verify sccache cache location - sccache --show-adv-stats; + function begin_group() { + local blue="34" + echo -e "::group::\e[${blue}m${1:-}\e[0m" + } + + function end_group() { + local name="${1:-}" + local build_status="${2:-0}" + local red="31" + + echo "::endgroup::" + if [ "$build_status" -ne 0 ]; then + echo -e "::error::\e[${red}m ${name} - Failed (⬆️ click above for full log ⬆️)\e[0m" + fi + } + + function run_command() { + local -; + set -euo pipefail; + + local group="${1:-}"; + shift; + local command=("$@"); + local exit_code="0"; + + begin_group "$group"; + + echo "Working directory: $(pwd)"; + echo "Running command: ${command[*]}"; + "${command[@]}" || exit_code=$?; + + end_group "$group" "$exit_code" + + return "$exit_code" + } + + if test -n "${DISABLE_SCCACHE:+x}"; then + . /opt/devcontainer/bin/update-envvars.sh; + for VAR in RUSTC_WRAPPER CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER CMAKE_CUDA_COMPILER_LAUNCHER; do + reset_envvar "$VAR"; + override_envvar "$VAR" ""; + done + fi # Clone all the repos - clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules; + run_command "Clone RAPIDS repositories" \ + clone-all -j$(nproc) -b ${{ inputs.branch }} -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; + + run_command "Create RAPIDS python environment" \ + rapids-post-start-command; # Configure all the C++ libs - time configure-all \ - -j0 \ - -GNinja \ - -Wno-dev \ - -DBUILD_TESTS=ON \ - -DBUILD_BENCHMARKS=ON \ - -DBUILD_PRIMS_BENCH=ON \ - -DBUILD_SHARED_LIBS=ON \ - -DRAFT_COMPILE_LIBRARY=ON \ - -DBUILD_CUGRAPH_MG_TESTS=ON ; + run_command "Configure C++ libraries" bash -c "\ + time configure-all \ + -j${PARALLEL_LEVEL} \ + -GNinja \ + -Wno-dev \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_PRIMS_BENCH=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DRAFT_COMPILE_LIBRARY=ON \ + -DBUILD_CUGRAPH_MG_TESTS=ON 2>&1 \ + | tee -a telemetry-artifacts/build.log"; # Build all the C++ libs - time build-all-cpp -j0; + run_command "Build C++ libraries" bash -c "\ + time build-all-cpp -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log"; # Build all the Python libs - time build-all-python -j0; + run_command "Build Python libraries" bash -c "\ + time build-all-python -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log"; # Print cache and dist stats - sccache --show-adv-stats; + run_command "sccache stats" bash -c "\ + sccache --show-adv-stats | tee -a telemetry-artifacts/sccache-stats.txt"; # Print build times - find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ - | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || : # Nonfatal if not found + run_command "Build times" bash -c "\ + find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ + | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || :" # Nonfatal if not found diff --git a/.github/workflows/test-rapids-build-times.yml b/.github/workflows/test-rapids-build-times.yml index 57a4cf26..53efc451 100644 --- a/.github/workflows/test-rapids-build-times.yml +++ b/.github/workflows/test-rapids-build-times.yml @@ -6,6 +6,11 @@ concurrency: on: workflow_dispatch: + inputs: + branch: + type: string + required: false + default: main jobs: uncached-builds: @@ -13,20 +18,26 @@ jobs: secrets: inherit uses: ./.github/workflows/build-all-rapids-repos.yml with: + branch: ${{ inputs.branch }} env: ${{ matrix.env }} matrix: '{ "include": [{ "libs": "" }] }' + node_type: cpu32 strategy: fail-fast: false matrix: include: - - name: 'sccache: no, dist: no' + - name: 'no sccache' env: | + PARALLEL_LEVEL= DISABLE_SCCACHE=1 - - name: 'sccache: yes, dist: no' + SCCACHE_NO_CACHE=1 + SCCACHE_NO_DIST_COMPILE=1 + MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=1 + - name: 'recache, local' env: | SCCACHE_RECACHE=1 SCCACHE_NO_DIST_COMPILE=1 - - name: 'sccache: yes, dist: yes' + - name: 'recache, remote' env: | SCCACHE_RECACHE=1 @@ -36,15 +47,17 @@ jobs: secrets: inherit uses: ./.github/workflows/build-all-rapids-repos.yml with: + branch: ${{ inputs.branch }} env: ${{ matrix.env }} matrix: '{ "include": [{ "libs": "" }] }' + node_type: cpu32 strategy: fail-fast: false matrix: include: - - name: 'preprocessor cache: no' - env: | - SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0 - - name: 'preprocessor cache: yes' + - name: 'preprocessor cache' env: | SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=1 + - name: 'no preprocessor cache' + env: | + SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=0