From 935b8a4ecda9537bc92417d87dd4d069f4404970 Mon Sep 17 00:00:00 2001 From: liferoad Date: Wed, 8 Apr 2026 20:21:05 -0400 Subject: [PATCH 1/4] Add PreCommit Java Dataflow workflow (#38094) Split Dataflow worker tests into a separate PreCommit job to reduce Java PreCommit runtime. The StreamingDataflowWorkerTest suite takes >1h, causing the main Java PreCommit to exceed typical CI timeouts. New workflow: beam_PreCommit_Java_Dataflow - Runs only on changes to runners/google-cloud-dataflow-java/worker/** - Triggers on push, PR, schedule (every 6h), workflow_dispatch, and issue comment 'Run Java Dataflow PreCommit' - Runs :runners:google-cloud-dataflow-java:worker:test - Timeout: 240 minutes Fixes #38094 --- .../beam_PreCommit_Java_Dataflow.json | 4 + .../beam_PreCommit_Java_Dataflow.yml | 132 ++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 .github/trigger_files/beam_PreCommit_Java_Dataflow.json create mode 100644 .github/workflows/beam_PreCommit_Java_Dataflow.yml diff --git a/.github/trigger_files/beam_PreCommit_Java_Dataflow.json b/.github/trigger_files/beam_PreCommit_Java_Dataflow.json new file mode 100644 index 000000000000..ffd631c44605 --- /dev/null +++ b/.github/trigger_files/beam_PreCommit_Java_Dataflow.json @@ -0,0 +1,4 @@ +{ + "comment": "Modify this file in a trivial way to cause this test suite to run.", + "modification": 1 +} \ No newline at end of file diff --git a/.github/workflows/beam_PreCommit_Java_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Dataflow.yml new file mode 100644 index 000000000000..f1838211d597 --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Dataflow.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or or in writing, software +# distributed under the License is distributed on an "AS IS" +# BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +name: PreCommit Java Dataflow + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - 'runners/google-cloud-dataflow-java/worker/**' + - '.github/workflows/beam_PreCommit_Java_Dataflow.yml' + pull_request_target: + branches: ['master', 'release-*'] + paths: + - 'runners/google-cloud-dataflow-java/worker/**' + - '.github/trigger_files/beam_PreCommit_Java_Dataflow.json' + issue_comment: + types: [created] + schedule: + - cron: '30 2/6 * * *' + workflow_dispatch: + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Dataflow: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-20.04, main] + strategy: + fail-fast: false + matrix: + job_name: [beam_PreCommit_Java_Dataflow] + job_phrase: [Run Java Dataflow PreCommit] + timeout-minutes: 240 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + (github.event_name == 'schedule' && github.repository == 'apache/beam') || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java Dataflow PreCommit' + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + with: + python-version: default + disable-cache: true + - name: Run Dataflow Worker tests + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :runners:google-cloud-dataflow-java:worker:test + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PenableJacocoReport \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + large_files: true + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' + - name: Archive Jacoco Results + uses: actions/upload-artifact@v4 + with: + name: Jacoco Results + path: '**/build/jacoco/report/**' + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + flags: java \ No newline at end of file From a1a574d87361bda5e113b2e232f5305d1a29d819 Mon Sep 17 00:00:00 2001 From: liferoad Date: Wed, 8 Apr 2026 20:29:00 -0400 Subject: [PATCH 2/4] Add release/trigger_all_tests.json to trigger path per GHA validation --- .github/workflows/beam_PreCommit_Java_Dataflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/beam_PreCommit_Java_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Dataflow.yml index f1838211d597..72f8874c29b7 100644 --- a/.github/workflows/beam_PreCommit_Java_Dataflow.yml +++ b/.github/workflows/beam_PreCommit_Java_Dataflow.yml @@ -28,6 +28,7 @@ on: paths: - 'runners/google-cloud-dataflow-java/worker/**' - '.github/trigger_files/beam_PreCommit_Java_Dataflow.json' + - 'release/trigger_all_tests.json' issue_comment: types: [created] schedule: From 66b313421af00100b88efc7d4a229b2d30b88269 Mon Sep 17 00:00:00 2001 From: liferoad Date: Wed, 8 Apr 2026 20:31:49 -0400 Subject: [PATCH 3/4] Exclude Dataflow worker from main Java PreCommit (#38094) Add !runners/google-cloud-dataflow-java/worker/** to both push and pull_request_target path filters in beam_PreCommit_Java.yml. Without this, the main Java PreCommit still runs the full javaPreCommit (including slow StreamingDataflowWorkerTest) on worker changes, which defeats the purpose of splitting to a separate PreCommit job. --- .github/workflows/beam_PreCommit_Java.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/beam_PreCommit_Java.yml b/.github/workflows/beam_PreCommit_Java.yml index 2dd0ac57dd01..0ed1a50b0cb3 100644 --- a/.github/workflows/beam_PreCommit_Java.yml +++ b/.github/workflows/beam_PreCommit_Java.yml @@ -23,6 +23,7 @@ on: - 'model/**' - 'sdks/java/**' - 'runners/**' + - '!runners/google-cloud-dataflow-java/worker/**' - 'examples/java/**' - 'examples/kotlin/**' - 'release/**' @@ -74,6 +75,7 @@ on: - 'model/**' - 'sdks/java/**' - 'runners/**' + - '!runners/google-cloud-dataflow-java/worker/**' - 'examples/java/**' - 'examples/kotlin/**' - 'release/**' From 0671ed9d947368a90667eb5b30a52f9f700d3798 Mon Sep 17 00:00:00 2001 From: liferoad Date: Wed, 8 Apr 2026 20:38:49 -0400 Subject: [PATCH 4/4] Add beam_PreCommit_Java_Dataflow.yml to workflows/README.md --- .github/workflows/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 448a9e7363a7..4b71805ceb2d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -228,6 +228,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour | [ PreCommit Go ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml) | N/A |`Run Go PreCommit`| [![.github/workflows/beam_PreCommit_Go.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml?query=event%3Aschedule) | | [ PreCommit GoPortable ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_GoPortable.yml) | N/A |`Run GoPortable PreCommit`| [![.github/workflows/beam_PreCommit_GoPortable.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_GoPortable.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_GoPortable.yml?query=event%3Aschedule) | | [ PreCommit Java ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java.yml) | N/A |`Run Java PreCommit`| [![.github/workflows/beam_PreCommit_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java.yml?query=event%3Aschedule) | +| [ PreCommit Java Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Dataflow.yml) | N/A |`Run Java_Dataflow PreCommit`| N/A | | [ PreCommit Java Amazon Web Services2 IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml) | N/A |`Run Java_Amazon-Web-Services2_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Amqp IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml) | N/A |`Run Java_Amqp_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Azure IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml) | N/A |`Run Java_Azure_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml?query=event%3Aschedule) |