diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml new file mode 100644 index 000000000000..664ef27624e4 --- /dev/null +++ b/.github/workflows/yetus-general-check.yml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus General Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: + checks: write + contents: read + pull-requests: write + statuses: write + +jobs: + general-check: + runs-on: ubuntu-latest + timeout-minutes: 600 + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@v4 + with: + path: src + fetch-depth: 0 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@v4 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus General Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + PATCHDIR: "${{ github.workspace }}/yetus-general-check/output" + PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-17" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Hide Old Comments + if: always() + env: + DEBUG: 'true' + run: | + # github-actions[bot] has a known user ID: 41898282 + # Verify: curl -s "https://api.github.com/users/github-actions%5Bbot%5D" | jq .id + export DEBUG="${DEBUG:-false}" + export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export BUILD_BOT_USER_ID="41898282" + export JOB_NAME="HBase-PreCommit-GH-Actions-PR" + export REPO="${{ github.repository }}" + bash src/dev-support/gh_hide_old_comments.sh ${{ github.event.pull_request.number }} || true + + - name: Create Nightlies Directory Structure + # this version of rsync doesn't support --mkdirs so we do this step manually + if: always() + run: | + eval $(ssh-agent -s) + ssh-add - <<< "${{ secrets.NIGHTLIES_RSYNC_KEY }}" + ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} \ + -o StrictHostKeyChecking=no \ + ${{ secrets.NIGHTLIES_RSYNC_USER }}@${{ secrets.NIGHTLIES_RSYNC_HOST }} \ + "mkdir -p ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}" + ssh-agent -k + + - name: Publish to Nightlies + if: always() + uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823 + with: + switches: -avzr --no-o + path: yetus-general-check + remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}/ + remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} + remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} + remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} + remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: yetus-general-check-output + path: ${{ github.workspace }}/yetus-general-check/output + retention-days: 7 diff --git a/dev-support/gh_hide_old_comments.sh b/dev-support/gh_hide_old_comments.sh index 61217cfa241b..abba55a39ad2 100755 --- a/dev-support/gh_hide_old_comments.sh +++ b/dev-support/gh_hide_old_comments.sh @@ -40,25 +40,79 @@ declare CURL="${CURL:-curl}" function fetch_comments { local pr="$1" local comments_file + local page_file + local headers_file local -a curl_args - curl_args=( - --fail - "${GITHUB_AUTH[@]}" - --header 'Accept: application/vnd.github+json' - --header 'X-GitHub-Api-Version: 2022-11-28' - --request GET - --url "${GITHUB_API_URL}/repos/${REPO}/issues/${pr}/comments?per_page=500" - ) - if [ "${DEBUG}" = true ] ; then - curl_args+=(--verbose) - else - curl_args+=(--silent) - fi + local page=1 + local next_url comments_file="$(mktemp "comments_${pr}" 2>/dev/null || mktemp -t "comments_${pr}.XXXXXXXXXX")" || \ { >&2 echo 'cannot create temp file'; exit 1 ;} - "${CURL}" "${curl_args[@]}" > "${comments_file}" + page_file="$(mktemp "page_${pr}" 2>/dev/null || mktemp -t "page_${pr}.XXXXXXXXXX")" || \ + { >&2 echo 'cannot create temp file'; exit 1 ;} + headers_file="$(mktemp "headers_${pr}" 2>/dev/null || mktemp -t "headers_${pr}.XXXXXXXXXX")" || \ + { >&2 echo 'cannot create temp file'; exit 1 ;} + + # cleanup temp files on error + trap 'rm -f "${page_file}" "${headers_file}"; exit 1' ERR + + next_url="${GITHUB_API_URL}/repos/${REPO}/issues/${pr}/comments?per_page=100" + + # start with empty JSON array + echo '[]' > "${comments_file}" + + while [ -n "${next_url}" ] ; do + curl_args=( + --fail + --max-time 30 + "${GITHUB_AUTH[@]}" + --header 'Accept: application/vnd.github+json' + --header 'X-GitHub-Api-Version: 2022-11-28' + --dump-header "${headers_file}" + --request GET + --url "${next_url}" + ) + if [ "${DEBUG}" = true ] ; then + curl_args+=(--verbose) + >&2 echo "Fetching page ${page}: ${next_url}" + else + curl_args+=(--silent) + fi + + if ! "${CURL}" "${curl_args[@]}" > "${page_file}"; then + >&2 echo "Failed to fetch page ${page}: ${next_url}" + rm -f "${page_file}" "${headers_file}" + exit 1 + fi + + if [ "${DEBUG}" = 'true' ] ; then + >&2 echo "Page ${page} returned $(jq length "${page_file}") comments" + fi + + # merge this page into the accumulated results + if ! jq -s '.[0] + .[1]' "${comments_file}" "${page_file}" > "${comments_file}.tmp"; then + >&2 echo "Failed to merge comments from page ${page}" + rm -f "${page_file}" "${headers_file}" "${comments_file}.tmp" + exit 1 + fi + mv "${comments_file}.tmp" "${comments_file}" + + # check for next page in Link header + # Link header format: ; rel="next", ; rel="last" + # Extract URL associated with rel="next" regardless of position + next_url="" + if grep -qi '^link:' "${headers_file}" ; then + next_url=$(grep -i '^link:' "${headers_file}" | tr ',' '\n' | grep 'rel="next"' | sed -n 's/.*<\([^>]*\)>.*/\1/p' || true) + fi + + page=$((page + 1)) + done + + rm -f "${page_file}" "${headers_file}" + trap - ERR + if [ "${DEBUG}" = 'true' ] ; then + >&2 echo "Total comments fetched: $(jq length "${comments_file}")" >&2 cat "${comments_file}" fi echo "${comments_file}" @@ -104,11 +158,16 @@ function identify_most_recent_build_number { local pr="$1" local comments_file="$2" local jq_filter + local url_pattern="${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/" + # GitHub Actions URLs don't have /job/ in them + if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then + url_pattern="${JOB_NAME}/PR-${pr}/(?[0-9]+)/" + fi read -r -d '' jq_filter << EOF || : .[] \ | select(.user.id == ${BUILD_BOT_USER_ID}) \ | .body \ -| capture("${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/") \ +| capture("${url_pattern}") \ | .buildnum EOF @@ -122,10 +181,15 @@ function identify_old_comment_ids { local comments_file="$2" local most_recent_build_number="$3" local jq_filter + local url_pattern="${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/" + # GitHub Actions URLs don't have /job/ in them + if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then + url_pattern="${JOB_NAME}/PR-${pr}/(?[0-9]+)/" + fi read -r -d '' jq_filter << EOF || : .[] \ | select(.user.id == ${BUILD_BOT_USER_ID}) \ -| { node_id, buildnum: (.body | capture("${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/") | .buildnum | tonumber) } \ +| { node_id, buildnum: (.body | capture("${url_pattern}") | .buildnum | tonumber) } \ | select(.buildnum < (${most_recent_build_number} | tonumber)) \ | .node_id EOF diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index bb3a44d47a73..168443384dc6 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -88,6 +88,20 @@ function personality_globals # See HBASE-19902 for how we arrived at 20g. #shellcheck disable=SC2034 DOCKERMEMLIMIT=20g + + # Override BUILD_URL for GitHub Actions to point to nightlies + # instead of the GitHub Actions run page + # The robot sets BUILD_URL to the GHA run page, but we want nightlies URLs + if [[ "${GITHUB_ACTIONS}" == "true" ]] && [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + local pr_number + # GITHUB_REF is a standard GitHub Actions environment variable + # shellcheck disable=SC2153 + pr_number=$(echo "${GITHUB_REF}" | cut -f3 -d/) + #shellcheck disable=SC2034 + BUILD_URL="https://nightlies.apache.org/hbase/HBase-PreCommit-GH-Actions-PR/PR-${pr_number}/${GITHUB_RUN_NUMBER}/" + #shellcheck disable=SC2034 + BUILD_URL_ARTIFACTS="yetus-general-check/output" + fi } ## @description Parse extra arguments required by personalities, if any. @@ -835,6 +849,19 @@ function hbaseanti_patchfile add_test_type spotless +## @description Build artifact URL for GitHub Actions +## @audience private +## @stability evolving +## @replaceable no +if ! declare -f githubactions_artifact_url >/dev/null; then + function githubactions_artifact_url + { + if [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then + echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}" + fi + } +fi + ## @description spotless file filter ## @audience private ## @stability evolving @@ -869,12 +896,13 @@ function spotless_rebuild count=$(${GREP} -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then - add_vote_table -1 spotless "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." - add_footer_table spotless "@@BASE@@/${repostatus}-spotless.txt" + add_vote_table_v2 -1 spotless \ + "@@BASE@@/${repostatus}-spotless.txt" \ + "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." return 1 fi - add_vote_table +1 spotless "${repostatus} has no errors when running spotless:check." + add_vote_table_v2 +1 spotless "" "${repostatus} has no errors when running spotless:check." return 0 } diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index 8604d96760dc..685ae3f186b8 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -31,7 +31,6 @@ declare -i missing_env=0 declare -a required_envs=( # these ENV variables define the required API with Jenkinsfile_GitHub "ARCHIVE_PATTERN_LIST" - "BUILD_URL_ARTIFACTS" "DOCKERFILE" "GITHUB_PASSWORD" "GITHUB_USER" @@ -53,6 +52,12 @@ for required_env in "${required_envs[@]}"; do fi done +# BUILD_URL_ARTIFACTS is required for Jenkins but set in personality for GitHub Actions +if [[ "${GITHUB_ACTIONS}" != "true" ]] && [[ -z "${BUILD_URL_ARTIFACTS}" ]]; then + echo "[ERROR] Required environment variable 'BUILD_URL_ARTIFACTS' is not set." + missing_env=${missing_env}+1 +fi + if [ ${missing_env} -gt 0 ]; then echo "[ERROR] Please set the required environment variables before invoking. If this error is " \ "on Jenkins, then please file a JIRA about the error."