From 663e4a61b675b1a84c61b60b8cd117c08eaa7a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 11:32:49 +0200 Subject: [PATCH 1/7] GH-49676: [Python][Packaging] Avoid rebuilding all base images by not pulling bases --- dev/tasks/python-wheels/github.windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 77e2a04e3a0a..763116dc4e74 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -70,7 +70,7 @@ jobs: echo on archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 if errorlevel 1 ( - archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 + archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 ) archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 @@ -85,7 +85,7 @@ jobs: cd arrow archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-test if errorlevel 1 ( - archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 + archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 ) archery docker run %TEST_IMAGE_PREFIX%-wheel-windows-test From a0edd40b1a2d03cb17dc9c6d1b9379500ca35531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 11:48:41 +0200 Subject: [PATCH 2/7] Revert --no-pull and split vcpkg installation in two steps to try and reduce layer size --- ...ython-wheel-windows-vs2022-base.dockerfile | 22 ++++++++++++------- dev/tasks/python-wheels/github.windows.yml | 4 ++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index f1bc314d013a..6bf2bd419d6a 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -128,20 +128,26 @@ ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` VCPKG_FEATURE_FLAGS="manifests" COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ -# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains -# ssl related fixes as well as we can patch the vcpkg portfile to support -# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186 -# but we cannot patch those portfiles since vcpkg-tool handles the checkout of -# previous versions => use bundled S3 build + +# We split the dependencies installation into two steps to reduce the size +# if the intermediate image layers. +# Install FS dependencies first to avoid hitting the image size limit of +# the Windows container. GH-49676 RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` --x-manifest-root=arrow/ci/vcpkg ` --x-feature=azure ` - --x-feature=flight ` --x-feature=gcs ` + --x-feature=s3 + +# Install the rest of the dependencies. +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=flight ` --x-feature=json ` --x-feature=opentelemetry ` --x-feature=orc ` - --x-feature=parquet ` - --x-feature=s3 + --x-feature=parquet diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 763116dc4e74..77e2a04e3a0a 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -70,7 +70,7 @@ jobs: echo on archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 if errorlevel 1 ( - archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 + archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1 ) archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 @@ -85,7 +85,7 @@ jobs: cd arrow archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-test if errorlevel 1 ( - archery docker build %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 + archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1 ) archery docker run %TEST_IMAGE_PREFIX%-wheel-windows-test From b45c2874d6ce646d654b2f4c04494c01593303dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 14:53:36 +0200 Subject: [PATCH 3/7] Add opentelemetry to its own Docker image layer --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 6bf2bd419d6a..9671b046696c 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -148,6 +148,11 @@ RUN vcpkg install ` --x-manifest-root=arrow/ci/vcpkg ` --x-feature=flight ` --x-feature=json ` - --x-feature=opentelemetry ` --x-feature=orc ` --x-feature=parquet + +# Install Opentelemetry alone. +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-feature=opentelemetry ` \ No newline at end of file From 4b9fd9ccb1f175025797743abb1e7d0ef34dccc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 15:10:56 +0200 Subject: [PATCH 4/7] Investigate whether the problem is around path length instead of layer size --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 11 +++++++---- ci/scripts/python_wheel_windows_build.bat | 4 ++-- ...indows-static-md-debug.cmake => x64-win-dbg.cmake} | 0 ...dows-static-md-release.cmake => x64-win-rel.cmake} | 0 4 files changed, 9 insertions(+), 6 deletions(-) rename ci/vcpkg/{amd64-windows-static-md-debug.cmake => x64-win-dbg.cmake} (100%) rename ci/vcpkg/{amd64-windows-static-md-release.cmake => x64-win-rel.cmake} (100%) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 9671b046696c..713b393934ca 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,11 +110,11 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin" # started to ship precompiled binaries for the vcpkg-tool. ARG vcpkg COPY ci/vcpkg/*.patch ` - ci/vcpkg/*windows*.cmake ` + ci/vcpkg/*win*.cmake ` arrow/ci/vcpkg/ COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/ -ENV VCPKG_ROOT=C:\\vcpkg -RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` +ENV VCPKG_ROOT=C:\\v +RUN bash arrow/ci/scripts/install_vcpkg.sh /c/v %vcpkg% && ` setx PATH "%PATH%;%VCPKG_ROOT%" # Configure vcpkg and install dependencies @@ -125,8 +125,11 @@ RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` ARG build_type=release ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg ` - VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` VCPKG_FEATURE_FLAGS="manifests" +# Derive a short triplet name to keep paths short and avoid hitting the +# hcsshim::ImportLayer MAX_PATH limit on Windows containers. GH-49676 +RUN if "%build_type%"=="release" (setx VCPKG_DEFAULT_TRIPLET x64-win-rel) ` + else if "%build_type%"=="debug" (setx VCPKG_DEFAULT_TRIPLET x64-win-dbg) COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ # We split the dependencies installation into two steps to reduce the size diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index e094d82861df..e38cdc6eb1c6 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -61,9 +61,9 @@ set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 17 2022 set CMAKE_PLATFORM=x64 -set VCPKG_ROOT=C:\vcpkg +set VCPKG_ROOT=C:\v set VCPKG_FEATURE_FLAGS=-manifests -set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% +if "%CMAKE_BUILD_TYPE%"=="release" (set VCPKG_TARGET_TRIPLET=x64-win-rel) else (set VCPKG_TARGET_TRIPLET=x64-win-dbg) mkdir C:\arrow-build pushd C:\arrow-build diff --git a/ci/vcpkg/amd64-windows-static-md-debug.cmake b/ci/vcpkg/x64-win-dbg.cmake similarity index 100% rename from ci/vcpkg/amd64-windows-static-md-debug.cmake rename to ci/vcpkg/x64-win-dbg.cmake diff --git a/ci/vcpkg/amd64-windows-static-md-release.cmake b/ci/vcpkg/x64-win-rel.cmake similarity index 100% rename from ci/vcpkg/amd64-windows-static-md-release.cmake rename to ci/vcpkg/x64-win-rel.cmake From 90df6a81a75a46eaab7b2d3ebab9aeaaa258ed38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 19:28:06 +0200 Subject: [PATCH 5/7] Revert "Investigate whether the problem is around path length instead of layer size" This reverts commit 4b9fd9ccb1f175025797743abb1e7d0ef34dccc2. --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 11 ++++------- ci/scripts/python_wheel_windows_build.bat | 4 ++-- ...-dbg.cmake => amd64-windows-static-md-debug.cmake} | 0 ...el.cmake => amd64-windows-static-md-release.cmake} | 0 4 files changed, 6 insertions(+), 9 deletions(-) rename ci/vcpkg/{x64-win-dbg.cmake => amd64-windows-static-md-debug.cmake} (100%) rename ci/vcpkg/{x64-win-rel.cmake => amd64-windows-static-md-release.cmake} (100%) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 713b393934ca..9671b046696c 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -110,11 +110,11 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin" # started to ship precompiled binaries for the vcpkg-tool. ARG vcpkg COPY ci/vcpkg/*.patch ` - ci/vcpkg/*win*.cmake ` + ci/vcpkg/*windows*.cmake ` arrow/ci/vcpkg/ COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/ -ENV VCPKG_ROOT=C:\\v -RUN bash arrow/ci/scripts/install_vcpkg.sh /c/v %vcpkg% && ` +ENV VCPKG_ROOT=C:\\vcpkg +RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && ` setx PATH "%PATH%;%VCPKG_ROOT%" # Configure vcpkg and install dependencies @@ -125,11 +125,8 @@ RUN bash arrow/ci/scripts/install_vcpkg.sh /c/v %vcpkg% && ` ARG build_type=release ENV CMAKE_BUILD_TYPE=${build_type} ` VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg ` + VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} ` VCPKG_FEATURE_FLAGS="manifests" -# Derive a short triplet name to keep paths short and avoid hitting the -# hcsshim::ImportLayer MAX_PATH limit on Windows containers. GH-49676 -RUN if "%build_type%"=="release" (setx VCPKG_DEFAULT_TRIPLET x64-win-rel) ` - else if "%build_type%"=="debug" (setx VCPKG_DEFAULT_TRIPLET x64-win-dbg) COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/ # We split the dependencies installation into two steps to reduce the size diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index e38cdc6eb1c6..e094d82861df 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -61,9 +61,9 @@ set CMAKE_INTERPROCEDURAL_OPTIMIZATION=ON set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 17 2022 set CMAKE_PLATFORM=x64 -set VCPKG_ROOT=C:\v +set VCPKG_ROOT=C:\vcpkg set VCPKG_FEATURE_FLAGS=-manifests -if "%CMAKE_BUILD_TYPE%"=="release" (set VCPKG_TARGET_TRIPLET=x64-win-rel) else (set VCPKG_TARGET_TRIPLET=x64-win-dbg) +set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE% mkdir C:\arrow-build pushd C:\arrow-build diff --git a/ci/vcpkg/x64-win-dbg.cmake b/ci/vcpkg/amd64-windows-static-md-debug.cmake similarity index 100% rename from ci/vcpkg/x64-win-dbg.cmake rename to ci/vcpkg/amd64-windows-static-md-debug.cmake diff --git a/ci/vcpkg/x64-win-rel.cmake b/ci/vcpkg/amd64-windows-static-md-release.cmake similarity index 100% rename from ci/vcpkg/x64-win-rel.cmake rename to ci/vcpkg/amd64-windows-static-md-release.cmake From 5479a8d10e0cd3e5015a91ce34f071a91bc47624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 7 Apr 2026 19:29:26 +0200 Subject: [PATCH 6/7] Remove opentelemetry from Windows wheels dependencies base image --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 9671b046696c..5aba02ed9fd7 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -150,9 +150,3 @@ RUN vcpkg install ` --x-feature=json ` --x-feature=orc ` --x-feature=parquet - -# Install Opentelemetry alone. -RUN vcpkg install ` - --clean-after-build ` - --x-install-root=%VCPKG_ROOT%\installed ` - --x-feature=opentelemetry ` \ No newline at end of file From a7c0613e95ff206781cf48e931749b63c90212d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 8 Apr 2026 09:45:01 +0200 Subject: [PATCH 7/7] Move flight to its own layer to see if that helps --- ci/docker/python-wheel-windows-vs2022-base.dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ci/docker/python-wheel-windows-vs2022-base.dockerfile b/ci/docker/python-wheel-windows-vs2022-base.dockerfile index 5aba02ed9fd7..f6f09c3e8fe4 100644 --- a/ci/docker/python-wheel-windows-vs2022-base.dockerfile +++ b/ci/docker/python-wheel-windows-vs2022-base.dockerfile @@ -141,12 +141,19 @@ RUN vcpkg install ` --x-feature=gcs ` --x-feature=s3 -# Install the rest of the dependencies. +# Install Flight separately as it pulls in grpc, protobuf, abseil which +# together produce the largest layer. +RUN vcpkg install ` + --clean-after-build ` + --x-install-root=%VCPKG_ROOT%\installed ` + --x-manifest-root=arrow/ci/vcpkg ` + --x-feature=flight + +# Install other dependencies. RUN vcpkg install ` --clean-after-build ` --x-install-root=%VCPKG_ROOT%\installed ` --x-manifest-root=arrow/ci/vcpkg ` - --x-feature=flight ` --x-feature=json ` --x-feature=orc ` --x-feature=parquet