From c89c96a6ab116174de069cc804a3cfb0cd04016f Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Wed, 8 Apr 2026 16:22:03 +0800 Subject: [PATCH 1/2] Try to trigger ci failed --- .github/workflows/cluster-it-1c1d.yml | 89 -- .github/workflows/cluster-it-1c1d1a.yml | 60 - .github/workflows/cluster-it-1c3d.yml | 69 -- .github/workflows/compile-check.yml | 55 - .github/workflows/daily-it.yml | 1079 ----------------- .github/workflows/daily-ut.yml | 59 - .github/workflows/dependency-check.yml | 62 - .github/workflows/greeting-ainode.yml | 52 - .github/workflows/multi-language-client.yml | 177 --- .github/workflows/pipe-it.yml | 978 --------------- .github/workflows/sonar-codecov.yml | 89 -- .github/workflows/table-cluster-it-1c1d.yml | 1 + .github/workflows/table-cluster-it-1c3d.yml | 69 -- .github/workflows/todos-check.yml | 54 - .github/workflows/unit-test.yml | 71 -- .github/workflows/vulnerability-check.yml | 50 - .gitignore | 3 + .../exchange/MPPDataExchangeManager.java | 305 +++-- .../exchange/SharedTsBlockQueue.java | 10 +- .../exchange/sink/LocalSinkChannel.java | 42 +- .../exchange/source/LocalSourceHandle.java | 50 +- iotdb-core/relational-grammar/pom.xml | 8 +- 22 files changed, 213 insertions(+), 3219 deletions(-) delete mode 100644 .github/workflows/cluster-it-1c1d.yml delete mode 100644 .github/workflows/cluster-it-1c1d1a.yml delete mode 100644 .github/workflows/cluster-it-1c3d.yml delete mode 100644 .github/workflows/compile-check.yml delete mode 100644 .github/workflows/daily-it.yml delete mode 100644 .github/workflows/daily-ut.yml delete mode 100644 .github/workflows/dependency-check.yml delete mode 100644 .github/workflows/greeting-ainode.yml delete mode 100644 .github/workflows/multi-language-client.yml delete mode 100644 .github/workflows/pipe-it.yml delete mode 100644 .github/workflows/sonar-codecov.yml delete mode 100644 .github/workflows/table-cluster-it-1c3d.yml delete mode 100644 .github/workflows/todos-check.yml delete mode 100644 .github/workflows/unit-test.yml delete mode 100644 .github/workflows/vulnerability-check.yml diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml deleted file mode 100644 index 4ab201450e695..0000000000000 --- a/.github/workflows/cluster-it-1c1d.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Cluster IT - 1C1D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Adjust network dynamic TCP ports range - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: | - netsh int ipv4 set dynamicport tcp start=32768 num=32768 - netsh int ipv4 set dynamicport udp start=32768 num=32768 - netsh int ipv6 set dynamicport tcp start=32768 num=32768 - netsh int ipv6 set dynamicport udp start=32768 num=32768 - - name: Adjust Linux kernel somaxconn - if: ${{ runner.os == 'Linux' }} - shell: bash - run: sudo sysctl -w net.core.somaxconn=65535 - # - name: Adjust Mac kernel somaxconn - # if: ${{ runner.os == 'macOS' }} - # shell: bash - # run: sudo sysctl -w kern.ipc.somaxconn=65535 - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 \ - -pl integration-test \ - -am - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml deleted file mode 100644 index 7ee6ca6ace66f..0000000000000 --- a/.github/workflows/cluster-it-1c1d1a.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Cluster IT - 1C1D1A - -on: - push: - branches: - - master - - 'rel/*' - - 'rc/*' - paths-ignore: - - 'docs/**' - - 'site/**' - pull_request: - branches: - - master - - 'rel/*' - - 'rc/*' - - 'force_ci/**' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - AINode: - strategy: - fail-fast: false - max-parallel: 1 - matrix: - os: [ ubuntu-latest ] - runs-on: [self-hosted, gpu] - - steps: - - uses: actions/checkout@v5 - - name: IT Test - shell: bash - run: | - mvn clean verify \ - -P with-integration-tests,with-ainode \ - -DskipUTs \ - -DintegrationTest.forkCount=1 \ - -pl integration-test,iotdb-core/ainode \ - -am \ - -PAIClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-ainode-${{ matrix.os }} - path: integration-test/target/*-logs - retention-days: 30 diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml deleted file mode 100644 index 035403e1695f4..0000000000000 --- a/.github/workflows/cluster-it-1c3d.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Cluster IT - 1C3D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml deleted file mode 100644 index 48e0571fc7888..0000000000000 --- a/.github/workflows/compile-check.yml +++ /dev/null @@ -1,55 +0,0 @@ -# This workflow will compile IoTDB under jdk8 to check for compatibility issues - -name: Compile Check - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - compile-check: - strategy: - fail-fast: false - matrix: - java: [8] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Compiler Test - shell: bash - run: | - mvn clean package -P with-integration-tests -DskipTests -ntp diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml deleted file mode 100644 index 54b502bcfa58c..0000000000000 --- a/.github/workflows/daily-it.yml +++ /dev/null @@ -1,1079 +0,0 @@ -name: Daily IT - -on: - schedule: - # Run at UTC 19:00 every day (CST 03:00 AM) - - cron: "0 19 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [8, 17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PDailyIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 3 - SingleRegionTableModel: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [8, 17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1 \ - -pl integration-test \ - -am -PTableSimpleIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 3 - PipeSingle: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeAutoBasic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeAutoEnhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTreeManual: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - ] - os: [ubuntu-latest] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2BatchMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2BatchMode - cluster2: HighPerformanceMode - - cluster1: IoTConsensusV2StreamMode - cluster2: LightWeightStandaloneMode - - cluster1: IoTConsensusV2StreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeManual \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeArchVerification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTableArchVerification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTableArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeRegressionConsumer: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - SubscriptionTreeRegressionMisc: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: - [ - ScalableSingleNodeMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTableManualBasic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - PipeDualTableManualEnhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: - [ - LightWeightStandaloneMode, - ScalableSingleNodeMode, - HighPerformanceMode, - IoTConsensusV2BatchMode, - IoTConsensusV2StreamMode, - ] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 diff --git a/.github/workflows/daily-ut.yml b/.github/workflows/daily-ut.yml deleted file mode 100644 index 348276962d2b1..0000000000000 --- a/.github/workflows/daily-ut.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Daily UT - -on: - schedule: - # Run at UTC 19:00 every day (CST 03:00 AM) - - cron: "0 19 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - unit-test: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [8] - os: [ubuntu-latest, windows-latest] - it_task: ["others", "datanode"] - include: - - java: 17 - os: macos-latest - it_task: "datanode" - - java: 17 - os: macos-latest - it_task: "others" - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven - shell: bash - if: ${{ matrix.it_task == 'datanode'}} - run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true - - name: Test Other Modules with Maven - shell: bash - if: ${{ matrix.it_task == 'others'}} - run: | - mvn clean install -DskipTests - mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml deleted file mode 100644 index 4cf6cb202afca..0000000000000 --- a/.github/workflows/dependency-check.yml +++ /dev/null @@ -1,62 +0,0 @@ -# This workflow will check if dependencies have changed (adding new dependencies or removing existing ones) - -name: Dependency Check - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - dependency-check: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Do the dependency check - shell: bash - run: mvn verify -Dmaven.test.skip=true -Dmdep.analyze.skip=true -P enable-sbom-check diff --git a/.github/workflows/greeting-ainode.yml b/.github/workflows/greeting-ainode.yml deleted file mode 100644 index 53deb59f1ce0f..0000000000000 --- a/.github/workflows/greeting-ainode.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: AINode Code Style Check - -on: - push: - branches: - - master - - "rc/*" - paths: - - 'iotdb-core/ainode/**' - pull_request: - branches: - - master - - "rc/*" - paths: - - 'iotdb-core/ainode/**' - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - -jobs: - check-style: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Set up Python 3.10 - uses: actions/setup-python@v6 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip3 install black==25.1.0 isort==6.0.1 - - name: Check code formatting (Black) - run: | - cd iotdb-core/ainode - black --check . - continue-on-error: false - - - name: Check import order (Isort) - run: | - cd iotdb-core/ainode - isort --check-only --profile black . - continue-on-error: false diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml deleted file mode 100644 index 831e3e37639b6..0000000000000 --- a/.github/workflows/multi-language-client.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Multi-Language Client -on: - push: - branches: - - master - - "rc/*" - paths: - - 'pom.xml' - - 'iotdb-client/pom.xml' - - 'iotdb-client/client-py/**' - - 'iotdb-client/client-cpp/**' - - 'example/client-cpp-example/**' - - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' - - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' - - '.github/workflows/multi-language-client.yml' - pull_request: - branches: - - master - - "rc/*" - - 'force_ci/**' - paths: - - 'pom.xml' - - 'iotdb-client/pom.xml' - - 'iotdb-client/client-py/**' - - 'iotdb-client/client-cpp/**' - - 'example/client-cpp-example/**' - - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' - - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' - - '.github/workflows/multi-language-client.yml' - # allow manually run the action: - workflow_dispatch: - - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - -jobs: - cpp: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - os: [ubuntu-22.04, ubuntu-24.04, windows-2022, windows-latest, windows-2025-vs2026, macos-latest] - runs-on: ${{ matrix.os}} - - steps: - - uses: actions/checkout@v5 - - name: Install CPP Dependencies (Ubuntu) - if: runner.os == 'Linux' - shell: bash - run: | - sudo apt-get update - sudo apt-get install libboost-all-dev - sudo apt-get install openssl libssl-dev - - name: Install CPP Dependencies (Mac) - # remove some xcode to release disk space - if: runner.os == 'macOS' - shell: bash - run: | - brew install boost - brew install openssl - sudo rm -rf /Applications/Xcode_14.3.1.app - sudo rm -rf /Applications/Xcode_15.0.1.app - sudo rm -rf /Applications/Xcode_15.1.app - sudo rm -rf /Applications/Xcode_15.2.app - sudo rm -rf /Applications/Xcode_15.3.app - - name: Install CPP Dependencies (Windows) - if: runner.os == 'Windows' - run: | - choco install winflexbison3 - choco install boost-msvc-14.3 - $boost_path = (Get-ChildItem -Path 'C:\local\' -Filter 'boost_*').FullName - echo $boost_path >> $env:GITHUB_PATH - - choco install openssl - $sslPath = (Get-ChildItem 'C:\Program Files\OpenSSL*' -Directory | Select-Object -First 1).FullName - echo "$sslPath\bin" >> $env:GITHUB_PATH - echo "OPENSSL_ROOT_DIR=$sslPath" >> $env:GITHUB_ENV - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Build IoTDB server - shell: bash - run: ./mvnw clean install -pl distribution -am -DskipTests - - name: Test with Maven - shell: bash - # Explicitly using mvnw here as the build requires maven 3.9 and the default installation is older - # Explicitly using "install" instead of package in order to be sure we're using libs built on this machine - # (was causing problems on windows, but could cause problem on linux, when updating the thrift module) - run: | - if [[ "${{ matrix.os }}" == "windows-2025-vs2026" ]]; then - ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Dcmake.generator="Visual Studio 18 2026" - else - ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am - fi - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cpp-IT-${{ runner.os }} - path: distribution/target/apache-iotdb-*-all-bin/apache-iotdb-*-all-bin/logs - retention-days: 1 - - go: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - with: - token: ${{secrets.GITHUB_TOKEN}} - submodules: recursive - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Compile IoTDB Server - run: mvn clean package -pl distribution -am -DskipTests - - name: Integration test - shell: bash - run: | - cd iotdb-client - git clone https://github.com/apache/iotdb-client-go.git - cd iotdb-client-go - make e2e_test_for_parent_git_repo e2e_test_clean_for_parent_git_repo - - python: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - python: ['3.x'] - runs-on: ${{ 'ubuntu-latest' }} - - steps: - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python }} - - uses: actions/checkout@v5 - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Cache pip packages - uses: actions/cache@v5 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: ${{ runner.os }}-pip- - - name: Build IoTDB server distribution zip and python client - run: mvn -B clean install -pl distribution,iotdb-client/client-py -am -DskipTests - - name: Build IoTDB server docker image - run: | - docker build . -f docker/src/main/Dockerfile-1c1d -t "iotdb:dev" - docker images - - name: Install IoTDB python client requirements - run: pip3 install -r iotdb-client/client-py/requirements_dev.txt - - name: Check code style - if: ${{ matrix.python == '3.x'}} - shell: bash - run: black iotdb-client/client-py/ --check --diff - - name: Integration test and test make package - shell: bash - run: | - cd iotdb-client/client-py/ && pytest . - ./release.sh diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml deleted file mode 100644 index 0968e7739a053..0000000000000 --- a/.github/workflows/pipe-it.yml +++ /dev/null @@ -1,978 +0,0 @@ -name: Multi-Cluster IT - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - single: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-basic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-manual: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [HighPerformanceMode] - cluster2: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeManual \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-arch-verification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-table-arch-verification: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTableArchVerification \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-regression-consumer: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - subscription-tree-regression-misc: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal - cluster1: [ScalableSingleNodeMode] - cluster2: [ScalableSingleNodeMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-table-manual-basic: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-table-manual-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [HighPerformanceMode] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - triple: - strategy: - fail-fast: false - max-parallel: 1 - matrix: - java: [ 17 ] - cluster1: [ ScalableSingleNodeMode ] - cluster2: [ ScalableSingleNodeMode ] - cluster3: [ ScalableSingleNodeMode ] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }},${{ matrix.cluster3 }} \ - -pl integration-test \ - -am -PMultiClusterIT3 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then - echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." - fi - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: cluster-log-triple-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}-${{ matrix.cluster3 }} - path: integration-test/target/cluster-logs - retention-days: 30 diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml deleted file mode 100644 index adb943a7f62e7..0000000000000 --- a/.github/workflows/sonar-codecov.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Sonar-Codecov - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "new_*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - PR_NUMBER: ${{ github.event.number }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - codecov: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/iotdb' || github.event_name == 'push' - - steps: - - uses: actions/checkout@v5 - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test - run: | - mvn -B -P with-code-coverage clean package -pl distribution,iotdb-client/cli,iotdb-client/session,iotdb-client/jdbc -am -Dtest.port.closed=true - mvn -B -P with-code-coverage post-integration-test -pl code-coverage - - name: Upload coverage reports to codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./code-coverage/target/jacoco-merged-reports/jacoco.xml - - sonar: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'apache/iotdb' || github.event_name == 'push' - steps: - - uses: actions/checkout@v5 - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: SonarCloud Report - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - run: | - mvn -B -P with-integration-tests,with-code-coverage verify sonar:sonar \ - -Dsonar.organization=apache \ - -Dsonar.projectKey=apache_iotdb \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.token=${{ secrets.SONARCLOUD_TOKEN }} \ - -DskipTests -pl '!distribution,!integration-test' -am diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml index 782bafa4ddbe6..2ff4890bad5e8 100644 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ b/.github/workflows/table-cluster-it-1c1d.yml @@ -77,6 +77,7 @@ jobs: run: | mvn clean verify \ -P with-integration-tests \ + -Dit.test=IoTDBGapFillTableViewIT \ -DskipUTs \ -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \ -pl integration-test \ diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml deleted file mode 100644 index 99a7469500c97..0000000000000 --- a/.github/workflows/table-cluster-it-1c3d.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Table Cluster IT - 1C3D - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - Simple: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [17] - runs-on: [self-hosted, iotdb] - # group: self-hosted - # labels: iotdb - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: IT/UT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ - -pl integration-test \ - -am -PTableClusterIT - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v6 - with: - name: table-cluster-log-java${{ matrix.java }}-${{ runner.os }} - path: integration-test/target/cluster-logs - retention-days: 1 diff --git a/.github/workflows/todos-check.yml b/.github/workflows/todos-check.yml deleted file mode 100644 index fc33b12d3cb58..0000000000000 --- a/.github/workflows/todos-check.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Check TODOs and FIXMEs in Changed Files - -on: - pull_request: - branches: - - master - - 'dev/*' - - 'rel/*' - - "rc/*" - - 'force_ci/**' - paths-ignore: - - 'docs/**' - - 'site/**' - # allow manually run the action: - workflow_dispatch: - -jobs: - todo-check: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Check for TODOs and FIXMEs in changed files - run: | - # Fetch the target branch - git fetch origin $GITHUB_BASE_REF - - git switch -c check_branch - - # Get the diff of the changes - echo Get the diff of the changes - DIFF=$(git diff origin/$GITHUB_BASE_REF check_branch -- . ':(exclude).github/workflows/todos-check.yml') - - if [ -z "$DIFF" ]; then - echo "No changes detected." - exit 0 - fi - - - # Check the diff for TODOs - - # Check the diff for TODOs - echo Check the diff for TODOs - TODOsCOUNT=$(echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | wc -l) - if [ "$TODOsCOUNT" -eq 0 ]; then - echo "No TODOs or FIXMEs found in changed content."; - exit 0 - fi - - echo "TODO or FIXME found in the changes. Please resolve it before merging." - echo "$DIFF" | grep -E '^\+.*(TODO|FIXME)' | tee -a output.log - exit 1 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml deleted file mode 100644 index c53c8d89cfa9a..0000000000000 --- a/.github/workflows/unit-test.yml +++ /dev/null @@ -1,71 +0,0 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: Unit-Test - -on: - push: - branches: - - master - - "rel/*" - - "rc/*" - paths-ignore: - - "docs/**" - - "site/**" - pull_request: - branches: - - master - - "rel/*" - - "rc/*" - - "force_ci/**" - paths-ignore: - - "docs/**" - - "site/**" - # allow manually run the action: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Xms2g -Xmx4g -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - unit-test: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - os: [ubuntu-latest, windows-latest] - it_task: ["others", "datanode"] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: ${{ matrix.java }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Maven packages - uses: actions/cache@v5 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven - shell: bash - if: ${{ matrix.it_task == 'datanode'}} - run: mvn clean integration-test -Dtest.port.closed=true -pl iotdb-core/datanode -am -DskipTests -Diotdb.test.only=true - - name: Test Other Modules with Maven - shell: bash - if: ${{ matrix.it_task == 'others'}} - run: | - mvn clean install -DskipTests - mvn -P get-jar-with-dependencies,with-integration-tests clean test -Dtest.port.closed=true -Diotdb.test.skip=true diff --git a/.github/workflows/vulnerability-check.yml b/.github/workflows/vulnerability-check.yml deleted file mode 100644 index c120bfa5291a1..0000000000000 --- a/.github/workflows/vulnerability-check.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: vulnerability-check -on: - schedule: - # Run at 16:00 UTC every Sunday (Monday 00:00 CST) - - cron: "0 16 * * 0" - workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - MAVEN_ARGS: --batch-mode --no-transfer-progress - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - -jobs: - dependency-check: - if: ${{ github.event_name == 'workflow_dispatch' || github.repository == 'apache/iotdb' }} - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@v5 - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: corretto - java-version: 17 - - - name: Do Maven install - shell: bash - run: mvn $MAVEN_ARGS clean install -DskipTests - - - name: Do the dependency-check:aggregate - shell: bash - run: mvn $MAVEN_ARGS org.owasp:dependency-check-maven:aggregate -DossIndexUsername=${{ secrets.OSS_INDEX_USER }} -DossIndexPassword=${{ secrets.OSS_INDEX_TOKEN }} -DnvdApiKey=${{ secrets.NVD_API_KEY }} - - - name: Generate report date for artifact name - run: | - utc_time="${{ github.run_started_at }}" - target_time=$(TZ=Asia/Shanghai date -d "$utc_time" +"%Y-%m-%d") - echo "REPORT_DATE=$target_time" >> $GITHUB_ENV - - - name: Upload Artifact - uses: actions/upload-artifact@v6 - with: - name: vulnerability-check-result-${{ env.REPORT_DATE }} - path: target/dependency-check-report.html - retention-days: 15 diff --git a/.gitignore b/.gitignore index 2c19b1b3a2cd5..4919c0a69981d 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/gen/ # Relational Grammar ANTLR iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/.antlr/ + +.claude/ +CLAUDE.md \ No newline at end of file diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java index d2caf330d6608..c4d67b657cc4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java @@ -103,12 +103,12 @@ public TGetDataBlockResponse getDataBlock(TGetDataBlockRequest req) throws TExce req.sourceFragmentInstanceId.queryId, req.sourceFragmentInstanceId.fragmentId, req.sourceFragmentInstanceId.instanceId))) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "[ProcessGetTsBlockRequest] sequence ID in [{}, {})", - req.getStartSequenceId(), - req.getEndSequenceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "[ProcessGetTsBlockRequest] sequence ID in [{}, {})", + req.getStartSequenceId(), + req.getEndSequenceId()); + // } TGetDataBlockResponse resp = new TGetDataBlockResponse(new ArrayList<>()); ISinkHandle sinkHandle = shuffleSinkHandles.get(req.getSourceFragmentInstanceId()); if (sinkHandle == null) { @@ -146,20 +146,20 @@ public void onAcknowledgeDataBlockEvent(TAcknowledgeDataBlockEvent e) { e.sourceFragmentInstanceId.queryId, e.sourceFragmentInstanceId.fragmentId, e.sourceFragmentInstanceId.instanceId))) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Received AcknowledgeDataBlockEvent for TsBlocks whose sequence ID are in [{}, {}) from {}.", - e.getStartSequenceId(), - e.getEndSequenceId(), - e.getSourceFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "Received AcknowledgeDataBlockEvent for TsBlocks whose sequence ID are in [{}, {}) from {}.", + e.getStartSequenceId(), + e.getEndSequenceId(), + e.getSourceFragmentInstanceId()); + // } ISinkHandle sinkHandle = shuffleSinkHandles.get(e.getSourceFragmentInstanceId()); if (sinkHandle == null) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "received ACK event but target FragmentInstance[{}] is not found.", - e.getSourceFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "received ACK event but target FragmentInstance[{}] is not found.", + e.getSourceFragmentInstanceId()); + // } return; } // index of the channel must be a SinkChannel @@ -185,20 +185,20 @@ public void onCloseSinkChannelEvent(TCloseSinkChannelEvent e) throws TException e.sourceFragmentInstanceId.queryId, e.sourceFragmentInstanceId.fragmentId, e.sourceFragmentInstanceId.instanceId))) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Closed source handle of ShuffleSinkHandle {}, channel index: {}.", - e.getSourceFragmentInstanceId(), - e.getIndex()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "Closed source handle of ShuffleSinkHandle {}, channel index: {}.", + e.getSourceFragmentInstanceId(), + e.getIndex()); + // } ISinkHandle sinkHandle = shuffleSinkHandles.get(e.getSourceFragmentInstanceId()); if (sinkHandle == null) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "received CloseSinkChannelEvent but target FragmentInstance[{}] is not found.", - e.getSourceFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "received CloseSinkChannelEvent but target FragmentInstance[{}] is not found.", + e.getSourceFragmentInstanceId()); + // } return; } sinkHandle.getChannel(e.getIndex()).close(); @@ -217,13 +217,13 @@ public void onNewDataBlockEvent(TNewDataBlockEvent e) throws TException { long startTime = System.nanoTime(); try (SetThreadName fragmentInstanceName = new SetThreadName(createFullIdFrom(e.targetFragmentInstanceId, e.targetPlanNodeId))) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "New data block event received, for plan node {} of {} from {}.", - e.getTargetPlanNodeId(), - e.getTargetFragmentInstanceId(), - e.getSourceFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "New data block event received, for plan node {} of {} from {}.", + e.getTargetPlanNodeId(), + e.getTargetFragmentInstanceId(), + e.getSourceFragmentInstanceId()); + // } Map sourceHandleMap = sourceHandles.get(e.getTargetFragmentInstanceId()); @@ -237,11 +237,11 @@ public void onNewDataBlockEvent(TNewDataBlockEvent e) throws TException { // may // have already been stopped. For example, in the read whit LimitOperator, the downstream // FragmentInstance may be finished, although the upstream is still working. - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "received NewDataBlockEvent but the downstream FragmentInstance[{}] is not found", - e.getTargetFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "received NewDataBlockEvent but the downstream FragmentInstance[{}] is not found", + e.getTargetFragmentInstanceId()); + // } return; } @@ -258,13 +258,13 @@ public void onNewDataBlockEvent(TNewDataBlockEvent e) throws TException { public void onEndOfDataBlockEvent(TEndOfDataBlockEvent e) throws TException { try (SetThreadName fragmentInstanceName = new SetThreadName(createFullIdFrom(e.targetFragmentInstanceId, e.targetPlanNodeId))) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "End of data block event received, for plan node {} of {} from {}.", - e.getTargetPlanNodeId(), - e.getTargetFragmentInstanceId(), - e.getSourceFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "End of data block event received, for plan node {} of {} from {}.", + e.getTargetPlanNodeId(), + e.getTargetFragmentInstanceId(), + e.getSourceFragmentInstanceId()); + // } Map sourceHandleMap = sourceHandles.get(e.getTargetFragmentInstanceId()); @@ -274,11 +274,11 @@ public void onEndOfDataBlockEvent(TEndOfDataBlockEvent e) throws TException { : (SourceHandle) sourceHandleMap.get(e.getTargetPlanNodeId()); if (sourceHandle == null || sourceHandle.isAborted() || sourceHandle.isFinished()) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "received onEndOfDataBlockEvent but the downstream FragmentInstance[{}] is not found", - e.getTargetFragmentInstanceId()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "received onEndOfDataBlockEvent but the downstream FragmentInstance[{}] is not found", + e.getTargetFragmentInstanceId()); + // } return; } @@ -325,15 +325,14 @@ public SourceHandleListenerImpl(IMPPDataExchangeManagerCallback onFai @Override public void onFinished(ISourceHandle sourceHandle) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ScHListenerOnFinish]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ScHListenerOnFinish]"); + // } Map sourceHandleMap = sourceHandles.get(sourceHandle.getLocalFragmentInstanceId()); if ((sourceHandleMap == null - || sourceHandleMap.remove(sourceHandle.getLocalPlanNodeId()) == null) - && LOGGER.isDebugEnabled()) { - LOGGER.debug("[ScHListenerAlreadyReleased]"); + || sourceHandleMap.remove(sourceHandle.getLocalPlanNodeId()) == null)) { + LOGGER.info("[ScHListenerAlreadyReleased]"); } if (sourceHandleMap != null && sourceHandleMap.isEmpty()) { @@ -343,9 +342,9 @@ public void onFinished(ISourceHandle sourceHandle) { @Override public void onAborted(ISourceHandle sourceHandle) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ScHListenerOnAbort]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ScHListenerOnAbort]"); + // } onFinished(sourceHandle); } @@ -373,16 +372,16 @@ public PipelineSourceHandleListenerImpl( @Override public void onFinished(ISourceHandle sourceHandle) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ScHListenerOnFinish]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ScHListenerOnFinish]"); + // } } @Override public void onAborted(ISourceHandle sourceHandle) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ScHListenerOnAbort]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ScHListenerOnAbort]"); + // } } @Override @@ -409,26 +408,26 @@ public ShuffleSinkListenerImpl( @Override public void onFinish(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ShuffleSinkHandleListenerOnFinish]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ShuffleSinkHandleListenerOnFinish]"); + // } shuffleSinkHandles.remove(sink.getLocalFragmentInstanceId()); context.finished(); } @Override public void onEndOfBlocks(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ShuffleSinkHandleListenerOnEndOfTsBlocks]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ShuffleSinkHandleListenerOnEndOfTsBlocks]"); + // } context.transitionToFlushing(); } @Override public Optional onAborted(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[ShuffleSinkHandleListenerOnAbort]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[ShuffleSinkHandleListenerOnAbort]"); + // } shuffleSinkHandles.remove(sink.getLocalFragmentInstanceId()); return context.getFailureCause(); } @@ -467,24 +466,24 @@ public ISinkChannelListenerImpl( @Override public void onFinish(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[SkHListenerOnFinish]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[SkHListenerOnFinish]"); + // } decrementCnt(); } @Override public void onEndOfBlocks(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[SkHListenerOnEndOfTsBlocks]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[SkHListenerOnEndOfTsBlocks]"); + // } } @Override public Optional onAborted(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[SkHListenerOnAbort]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[SkHListenerOnAbort]"); + // } decrementCnt(); return context.getFailureCause(); } @@ -507,9 +506,9 @@ private void decrementCnt() { private void closeShuffleSinkHandle() { ISinkHandle sinkHandle = shuffleSinkHandles.remove(shuffleSinkHandleId); if (sinkHandle != null) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Close ShuffleSinkHandle: {}", shuffleSinkHandleId); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Close ShuffleSinkHandle: {}", shuffleSinkHandleId); + // } sinkHandle.close(); } } @@ -534,23 +533,23 @@ public PipelineSinkListenerImpl( @Override public void onFinish(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[SkHListenerOnFinish]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[SkHListenerOnFinish]"); + // } } @Override public void onEndOfBlocks(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[SkHListenerOnEndOfTsBlocks]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[SkHListenerOnEndOfTsBlocks]"); + // } } @Override public Optional onAborted(ISink sink) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[SkHListenerOnAbort]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[SkHListenerOnAbort]"); + // } return context.getFailureCause(); } @@ -633,27 +632,27 @@ private synchronized ISinkChannel createLocalSinkChannel( FragmentInstanceContext instanceContext, AtomicInteger cnt) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Create local sink handle to plan node {} of {} for {}", - remotePlanNodeId, - remoteFragmentInstanceId, - localFragmentInstanceId); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "Create local sink handle to plan node {} of {} for {}", + remotePlanNodeId, + remoteFragmentInstanceId, + localFragmentInstanceId); + // } SharedTsBlockQueue queue; Map sourceHandleMap = sourceHandles.get(remoteFragmentInstanceId); LocalSourceHandle localSourceHandle = sourceHandleMap == null ? null : (LocalSourceHandle) sourceHandleMap.get(remotePlanNodeId); if (localSourceHandle != null) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Get SharedTsBlockQueue from local source handle"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Get SharedTsBlockQueue from local source handle"); + // } queue = localSourceHandle.getSharedTsBlockQueue(); } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Create SharedTsBlockQueue"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Create SharedTsBlockQueue"); + // } queue = new SharedTsBlockQueue( localFragmentInstanceId, localPlanNodeId, localMemoryManager, executorService); @@ -672,9 +671,9 @@ private synchronized ISinkChannel createLocalSinkChannel( */ public ISinkChannel createLocalSinkChannelForPipeline( DriverContext driverContext, String planNodeId) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Create local sink handle for {}", driverContext.getDriverTaskID()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Create local sink handle for {}", driverContext.getDriverTaskID()); + // } SharedTsBlockQueue queue = new SharedTsBlockQueue( driverContext.getDriverTaskID().getFragmentInstanceId().toThrift(), @@ -699,13 +698,13 @@ private ISinkChannel createSinkChannel( FragmentInstanceContext instanceContext, AtomicInteger cnt) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Create sink handle to plan node {} of {} for {}", - remotePlanNodeId, - remoteFragmentInstanceId, - localFragmentInstanceId); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "Create sink handle to plan node {} of {} for {}", + remotePlanNodeId, + remoteFragmentInstanceId, + localFragmentInstanceId); + // } return new SinkChannel( remoteEndpoint, @@ -793,9 +792,9 @@ private ISinkChannel createChannelForShuffleSink( */ public ISourceHandle createLocalSourceHandleForPipeline( SharedTsBlockQueue queue, DriverContext context) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Create local source handle for {}", context.getDriverTaskID()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Create local source handle for {}", context.getDriverTaskID()); + // } return new PipelineSourceHandle( queue, new PipelineSourceHandleListenerImpl(context::failed), @@ -819,25 +818,25 @@ public synchronized ISourceHandle createLocalSourceHandleForFragment( + " exists."); } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Create local source handle from {} for plan node {} of {}", - remoteFragmentInstanceId, - localPlanNodeId, - localFragmentInstanceId); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "Create local source handle from {} for plan node {} of {}", + remoteFragmentInstanceId, + localPlanNodeId, + localFragmentInstanceId); + // } SharedTsBlockQueue queue; ISinkHandle sinkHandle = shuffleSinkHandles.get(remoteFragmentInstanceId); if (sinkHandle != null) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Get SharedTsBlockQueue from local sink handle"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Get SharedTsBlockQueue from local sink handle"); + // } queue = ((LocalSinkChannel) (sinkHandle.getChannel(index))).getSharedTsBlockQueue(); } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Create SharedTsBlockQueue"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("Create SharedTsBlockQueue"); + // } queue = new SharedTsBlockQueue( remoteFragmentInstanceId, remotePlanNodeId, localMemoryManager, executorService); @@ -872,13 +871,13 @@ public ISourceHandle createSourceHandle( + " exists."); } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Create source handle from {} for plan node {} of {}", - remoteFragmentInstanceId, - localPlanNodeId, - localFragmentInstanceId); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "Create source handle from {} for plan node {} of {}", + remoteFragmentInstanceId, + localPlanNodeId, + localFragmentInstanceId); + // } SourceHandle sourceHandle = new SourceHandle( @@ -905,9 +904,9 @@ public ISourceHandle createSourceHandle( *

This method should be called when a fragment instance finished in an abnormal state. */ public void forceDeregisterFragmentInstance(TFragmentInstanceId fragmentInstanceId) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartForceReleaseFIDataExchangeResource]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartForceReleaseFIDataExchangeResource]"); + // } ISink sinkHandle = shuffleSinkHandles.get(fragmentInstanceId); if (sinkHandle != null) { sinkHandle.abort(); @@ -916,16 +915,16 @@ public void forceDeregisterFragmentInstance(TFragmentInstanceId fragmentInstance Map planNodeIdToSourceHandle = sourceHandles.get(fragmentInstanceId); if (planNodeIdToSourceHandle != null) { for (Entry entry : planNodeIdToSourceHandle.entrySet()) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[CloseSourceHandle] {}", entry.getKey()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[CloseSourceHandle] {}", entry.getKey()); + // } entry.getValue().abort(); } sourceHandles.remove(fragmentInstanceId); } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndForceReleaseFIDataExchangeResource]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndForceReleaseFIDataExchangeResource]"); + // } } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index e49efabb9869c..e8448477678ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -160,9 +160,9 @@ public void setSourceHandle(LocalSourceHandle sourceHandle) { /** Notify no more TsBlocks will be added to the queue. */ public void setNoMoreTsBlocks(boolean noMoreTsBlocks) { - LOGGER.debug("[SignalNoMoreTsBlockOnQueue]"); + LOGGER.info("[SignalNoMoreTsBlockOnQueue]"); if (closed) { - LOGGER.debug("The queue has been destroyed when calling setNoMoreTsBlocks."); + LOGGER.info("The queue has been destroyed when calling setNoMoreTsBlocks."); return; } this.noMoreTsBlocks = noMoreTsBlocks; @@ -222,9 +222,9 @@ public TsBlock remove() { * the returned future of last invocation completes. */ public ListenableFuture add(TsBlock tsBlock) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[addTsBlock] TsBlock:{}", CommonUtils.toString(tsBlock)); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[addTsBlock] TsBlock:{}", CommonUtils.toString(tsBlock)); + // } if (closed) { // queue may have been closed return immediateVoidFuture(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/LocalSinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/LocalSinkChannel.java index d671f9279900a..002b87ab7a524 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/LocalSinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/LocalSinkChannel.java @@ -148,9 +148,9 @@ public void send(TsBlock tsBlock) { if (queue.hasNoMoreTsBlocks()) { return; } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartSendTsBlockOnLocal]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartSendTsBlockOnLocal]"); + // } synchronized (this) { blocked = queue.add(tsBlock); } @@ -165,9 +165,9 @@ public void send(TsBlock tsBlock) { public void setNoMoreTsBlocks() { synchronized (queue) { synchronized (this) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartSetNoMoreTsBlocksOnLocal]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartSetNoMoreTsBlocksOnLocal]"); + // } if (aborted || closed) { return; } @@ -176,16 +176,16 @@ public void setNoMoreTsBlocks() { } } checkAndInvokeOnFinished(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndSetNoMoreTsBlocksOnLocal]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndSetNoMoreTsBlocksOnLocal]"); + // } } @Override public boolean abort() { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartAbortLocalSinkChannel]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartAbortLocalSinkChannel]"); + // } synchronized (queue) { synchronized (this) { if (aborted || closed) { @@ -200,17 +200,17 @@ public boolean abort() { } } } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndAbortLocalSinkChannel]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndAbortLocalSinkChannel]"); + // } return true; } @Override public boolean close() { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartCloseLocalSinkChannel]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartCloseLocalSinkChannel]"); + // } synchronized (queue) { synchronized (this) { if (aborted || closed) { @@ -224,9 +224,9 @@ public boolean close() { } } } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndCloseLocalSinkChannel]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndCloseLocalSinkChannel]"); + // } return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java index 0dc6b27af04b6..a901dd81f70dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java @@ -124,10 +124,10 @@ public TsBlock receive() { tsBlock = queue.remove(); } if (tsBlock != null) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "[GetTsBlockFromQueue] TsBlock:{} size:{}", currSequenceId, tsBlock.getSizeInBytes()); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info( + "[GetTsBlockFromQueue] TsBlock:{} size:{}", currSequenceId, tsBlock.getSizeInBytes()); + // } currSequenceId++; } checkAndInvokeOnFinished(); @@ -141,9 +141,9 @@ public TsBlock receive() { @Override public ByteBuffer getSerializedTsBlock() throws IoTDBException { TsBlock tsBlock = receive(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[GetSerializedTsBlock] TsBlock:{}", CommonUtils.toString(tsBlock)); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[GetSerializedTsBlock] TsBlock:{}", CommonUtils.toString(tsBlock)); + // } if (tsBlock != null) { long startTime = System.nanoTime(); @@ -195,9 +195,9 @@ public boolean isAborted() { @Override public void abort() { try (SetThreadName sourceHandleName = new SetThreadName(threadName)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartAbortLocalSourceHandle]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartAbortLocalSourceHandle]"); + // } synchronized (queue) { synchronized (this) { if (aborted || closed) { @@ -208,18 +208,18 @@ public void abort() { sourceHandleListener.onAborted(this); } } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndAbortLocalSourceHandle]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndAbortLocalSourceHandle]"); + // } } } @Override public void abort(Throwable t) { try (SetThreadName sourceHandleName = new SetThreadName(threadName)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartAbortLocalSourceHandle]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartAbortLocalSourceHandle]"); + // } synchronized (queue) { synchronized (this) { if (aborted || closed) { @@ -230,18 +230,18 @@ public void abort(Throwable t) { sourceHandleListener.onAborted(this); } } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndAbortLocalSourceHandle]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndAbortLocalSourceHandle]"); + // } } } @Override public void close() { try (SetThreadName sourceHandleName = new SetThreadName(threadName)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[StartCloseLocalSourceHandle]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[StartCloseLocalSourceHandle]"); + // } synchronized (queue) { synchronized (this) { if (aborted || closed) { @@ -252,9 +252,9 @@ public void close() { sourceHandleListener.onFinished(this); } } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[EndCloseLocalSourceHandle]"); - } + // if (LOGGER.isDebugEnabled()) { + LOGGER.info("[EndCloseLocalSourceHandle]"); + // } } } diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index f4fb18bee2c83..76acbf3355057 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -96,8 +96,12 @@ true - **/SqlLexer.java - **/SqlLexer.interp + **/RelationalSqlParser.java + **/RelationalSqlLexer.java + **/RelationalSqlLexer.interp + **/RelationalSql.interp + **/RelationalSqlBaseListener.java + **/RelationalSqlListener.java From 366c00754f22bb865472b08f01c98ec86438fae7 Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Wed, 8 Apr 2026 16:25:10 +0800 Subject: [PATCH 2/2] Change ci yml --- .github/workflows/table-cluster-it-1c1d.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml index 2ff4890bad5e8..f1b2f939e7ebc 100644 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ b/.github/workflows/table-cluster-it-1c1d.yml @@ -77,6 +77,8 @@ jobs: run: | mvn clean verify \ -P with-integration-tests \ + -DfailIfNoTests=false \ + -Dfailsafe.failIfNoSpecifiedTests=false \ -Dit.test=IoTDBGapFillTableViewIT \ -DskipUTs \ -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 \