From ed34717ada04dab7bedaaf0cce572785f8705ad4 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 2 Jun 2026 21:20:15 +0900 Subject: [PATCH 1/2] Fix incomplete Python dev requirements Closes #8082 --- .github/workflows/paimon-python-checks.yml | 28 +++++++++------------- paimon-python/dev/requirements-dev.txt | 20 ++++++++-------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml index 6a88767590e3..a47071f97c9e 100755 --- a/.github/workflows/paimon-python-checks.yml +++ b/.github/workflows/paimon-python-checks.yml @@ -127,25 +127,26 @@ jobs: df -h if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then python -m pip install --upgrade pip==21.3.1 - python --version - python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ else python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' pypaimon-rust==0.2.0 - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + fi + python --version + python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt + if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then + python -m pip install --no-cache-dir ossfs==2021.8.0 + else + python -m pip install --no-cache-dir 'daft>=0.7.6' ossfs==2023.12.0 pylance==0.39.0 maturin[patchelf] if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then - python -m pip install vortex-data==0.70.0 + python -m pip install --no-cache-dir vortex-data==0.70.0 fi fi + python -m pip install --no-cache-dir 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ df -h - name: Build and install tantivy-py from source if: matrix.python-version != '3.6.15' shell: bash run: | - pip install maturin[patchelf] git clone -b support_directory https://github.com/JingsongLi/tantivy-py.git /tmp/tantivy-py cd /tmp/tantivy-py maturin build --release @@ -183,9 +184,8 @@ jobs: shell: bash run: | python -m pip install --upgrade pip + python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ - name: Run lint-python.sh shell: bash run: | @@ -278,13 +278,7 @@ jobs: - name: Install base Python dependencies run: | python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install --no-cache-dir \ - pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \ - fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \ - numpy==1.24.3 pandas==2.0.3 cramjam pytest~=7.0 py4j==0.10.9.9 requests \ - parameterized==0.9.0 packaging - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt - name: Test Ray version compatibility run: | cd paimon-python diff --git a/paimon-python/dev/requirements-dev.txt b/paimon-python/dev/requirements-dev.txt index 9ef88817f726..674e5faee4fb 100644 --- a/paimon-python/dev/requirements-dev.txt +++ b/paimon-python/dev/requirements-dev.txt @@ -15,18 +15,18 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ -# Core dependencies for pypaimon are in requirements.txt +# Core dependencies for pypaimon +-r requirements.txt # Test dependencies for pypaimon are as follows -duckdb==1.3.2 +duckdb==1.3.2; python_version >= "3.8" flake8==4.0.1 -pytest~=7.0 +parameterized==0.8.1; python_version < "3.7" +parameterized==0.9.0; python_version >= "3.7" +py4j==0.10.9.9 +pypaimon-rust==0.2.0; python_version >= "3.10" +pytest<7; python_version < "3.7" +pytest~=7.0; python_version >= "3.7" # merge_into needs Dataset.join (added in Ray 2.50). Python 3.8 has no 2.50 wheel. -ray>=2.10.0; python_version < "3.9" +ray>=2.10.0; python_version >= "3.8" and python_version < "3.9" ray>=2.50.0; python_version >= "3.9" -requests -parameterized -# Vortex 0.71.0 regresses native predicate pushdown on single-row files. -vortex-data==0.70.0; python_version >= "3.11" -# Lumina vector search (optional, for lumina index tests) -lumina-data>=0.1.0 From d7891bacbb0e0ba710250a53ac9b2468d71effac Mon Sep 17 00:00:00 2001 From: AN Long Date: Wed, 3 Jun 2026 00:34:25 +0900 Subject: [PATCH 2/2] Fix pyroaring requirement for BitMap64 --- paimon-python/dev/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paimon-python/dev/requirements.txt b/paimon-python/dev/requirements.txt index 828670f6ddf2..aea14de06a7b 100644 --- a/paimon-python/dev/requirements.txt +++ b/paimon-python/dev/requirements.txt @@ -29,8 +29,8 @@ polars>=0.9,<1; python_version<"3.8" polars>=1,<2; python_version>="3.8" pyarrow>=6,<7; python_version < "3.8" pyarrow>=16,<20; python_version >= "3.8" -pyroaring<=0.3.3; python_version < "3.7" -pyroaring<=0.4.5; python_version == "3.7" +# BitMap64 is required by RoaringBitmap64 and is available since pyroaring 1.0.0. +pyroaring==1.0.0; python_version < "3.8" pyroaring>=1.0.0; python_version >= "3.8" readerwriterlock>=1,<2 requests>=2.21.0,<3