[python] Fix incomplete Python dev requirements

.github/workflows/paimon-python-checks.yml

@@ -127,25 +127,26 @@ jobs:
           df -h
           if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
             python -m pip install --upgrade pip==21.3.1
-            python --version
-            python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
-            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
           else
             python -m pip install --upgrade pip
-            pip install torch --index-url https://download.pytorch.org/whl/cpu
-            python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' pypaimon-rust==0.2.0
-            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
+          fi
+          python --version
+          python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt
+          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
+            python -m pip install --no-cache-dir ossfs==2021.8.0
+          else
+            python -m pip install --no-cache-dir 'daft>=0.7.6' ossfs==2023.12.0 pylance==0.39.0 maturin[patchelf]
             if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then
-              python -m pip install vortex-data==0.70.0
+              python -m pip install --no-cache-dir vortex-data==0.70.0
             fi
           fi
+          python -m pip install --no-cache-dir 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
           df -h
 
       - name: Build and install tantivy-py from source
         if: matrix.python-version != '3.6.15'
         shell: bash
         run: |
-          pip install maturin[patchelf]
           git clone -b support_directory https://github.com/JingsongLi/tantivy-py.git /tmp/tantivy-py
           cd /tmp/tantivy-py
           maturin build --release
@@ -183,9 +184,8 @@ jobs:
         shell: bash
         run: |
             python -m pip install --upgrade pip
+            python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt
             pip install torch --index-url https://download.pytorch.org/whl/cpu
-            python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0
-            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
       - name: Run lint-python.sh
         shell: bash
         run: |
@@ -278,13 +278,7 @@ jobs:
       - name: Install base Python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install torch --index-url https://download.pytorch.org/whl/cpu
-          python -m pip install --no-cache-dir \
-            pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \
-            fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \
-            numpy==1.24.3 pandas==2.0.3 cramjam pytest~=7.0 py4j==0.10.9.9 requests \
-            parameterized==0.9.0 packaging
-          python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
+          python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt
       - name: Test Ray version compatibility
         run: |
           cd paimon-python

paimon-python/dev/requirements-dev.txt

@@ -15,18 +15,18 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
-# Core dependencies for pypaimon are in requirements.txt
+# Core dependencies for pypaimon
+-r requirements.txt
 
 # Test dependencies for pypaimon are as follows
-duckdb==1.3.2
+duckdb==1.3.2; python_version >= "3.8"
 flake8==4.0.1
-pytest~=7.0
+parameterized==0.8.1; python_version < "3.7"
+parameterized==0.9.0; python_version >= "3.7"
+py4j==0.10.9.9
+pypaimon-rust==0.2.0; python_version >= "3.10"
+pytest<7; python_version < "3.7"
+pytest~=7.0; python_version >= "3.7"
 # merge_into needs Dataset.join (added in Ray 2.50). Python 3.8 has no 2.50 wheel.
-ray>=2.10.0; python_version < "3.9"
+ray>=2.10.0; python_version >= "3.8" and python_version < "3.9"
 ray>=2.50.0; python_version >= "3.9"
-requests
-parameterized
-# Vortex 0.71.0 regresses native predicate pushdown on single-row files.
-vortex-data==0.70.0; python_version >= "3.11"
-# Lumina vector search (optional, for lumina index tests)
-lumina-data>=0.1.0

paimon-python/dev/requirements.txt

@@ -29,8 +29,8 @@ polars>=0.9,<1; python_version<"3.8"
 polars>=1,<2; python_version>="3.8"
 pyarrow>=6,<7; python_version < "3.8"
 pyarrow>=16,<20; python_version >= "3.8"
-pyroaring<=0.3.3; python_version < "3.7"
-pyroaring<=0.4.5; python_version == "3.7"
+# BitMap64 is required by RoaringBitmap64 and is available since pyroaring 1.0.0.
+pyroaring==1.0.0; python_version < "3.8"
 pyroaring>=1.0.0; python_version >= "3.8"
 readerwriterlock>=1,<2
 requests>=2.21.0,<3