From 7124a5f592602dd2b067a9536c7881cfff72e6be Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 6 Mar 2026 12:55:11 -0300 Subject: [PATCH 1/2] #1383: Used exaslpm to build template-Exasol-all-python-3.10 flavor --- .../base_test_build_run/Dockerfile | 3 +- .../flavor_base/base_test_deps/Dockerfile | 4 +- .../flavor_base/build_deps/Dockerfile | 23 +---- .../flavor_base/build_run/Dockerfile | 6 +- .../flavor_base/flavor_base_deps/Dockerfile | 5 +- .../flavor_base/language_deps/Dockerfile | 12 +-- .../flavor_base/packages.yml | 69 +++++++++++++ .../flavor_base/security_scan/Dockerfile | 9 +- .../flavor_base/testconfig | 2 - .../flavor_base/udfclient_deps/Dockerfile | 14 +-- .../flavor_customization/Dockerfile | 6 +- .../packages.yml | 96 +++++++++++++++++++ 12 files changed, 186 insertions(+), 63 deletions(-) create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml delete mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/testconfig create mode 100644 flavors/template-Exasol-all-python-3.12/packages.yml diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile index b87c2bfc5..563738244 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile @@ -17,9 +17,10 @@ RUN true # workaround for https://github.com/moby/moby/issues/37965 COPY --from={{base_test_deps}} /etc /etc RUN true # workaround for https://github.com/moby/moby/issues/37965 -COPY --from={{base_test_deps}} /env /env +COPY --from={{base_test_deps}} /build_info /build_info RUN true # workaround for https://github.com/moby/moby/issues/37965 +RUN exaslpm export-variables --out-file /env RUN ldconfig diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile index 3a8ca3f61..959638d89 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile @@ -1,6 +1,6 @@ FROM {{build_deps}} RUN mkdir -p /build_info/packages -COPY base_test_deps/packages /build_info/packages/base_test_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions +COPY base_test_deps_packages.yaml /build_info/packages/base_test_deps_packages.yaml +RUN exaslpm install --package-file /build_info/packages/base_test_deps_packages.yaml --build-step base_test_deps diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile index 7e52ab37e..15d00c843 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile @@ -1,28 +1,15 @@ -FROM ubuntu:24.04 +FROM exasol/script-language-container:exaslpm-ubuntu-24.04 ENV DEBIAN_FRONTEND=noninteractive -ENV ARCHIVE_UBUNTU_PREFIX="" -RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list - COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc COPY scripts /scripts RUN mkdir -p /build_info/packages -COPY build_deps/packages /build_info/packages/build_deps -ENV BAZEL_PACKAGE_VERSION="8.3.1" -ENV BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" -ENV BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/build_deps/apt_get_packages --with-versions +COPY build_deps_packages.yaml /build_info/packages/build_deps_packages.yaml +RUN exaslpm install --package-file /build_info/packages/build_deps_packages.yaml --build-step build_deps -RUN apt-get -y update && \ - curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \ - apt-get install -y "./$BAZEL_PACKAGE_FILE" && \ - rm "$BAZEL_PACKAGE_FILE" && \ - apt-get -y clean && \ - apt-get -y autoremove - RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ tar zxf swig-2.0.4.tar.gz && \ (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ @@ -31,7 +18,3 @@ RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3 RUN locale-gen en_US.UTF-8 && \ update-locale LC_ALL=en_US.UTF-8 && \ ldconfig - -RUN touch /env && \ - echo "export PROTOBUF_BIN=/usr/bin/protoc" >> /env && \ - echo "export JAVA_HOME=/usr/lib/jvm/java-1.17.0-openjdk-amd64/" >> /env diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile index 1d1524f16..7082116f5 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile @@ -15,9 +15,10 @@ RUN true # workaround for https://github.com/moby/moby/issues/37965 COPY --from={{build_deps}} /etc /etc RUN true # workaround for https://github.com/moby/moby/issues/37965 -COPY --from={{build_deps}} /env /env +COPY --from={{build_deps}} /build_info /build_info RUN true # workaround for https://github.com/moby/moby/issues/37965 +RUN exaslpm export-variables --out-file /env RUN ldconfig @@ -36,8 +37,5 @@ WORKDIR /exaudf RUN rm -r /exaudfclient -COPY --from={{build_deps}} /build_info /build_info -RUN true # workaround for https://github.com/moby/moby/issues/37965 - RUN mkdir /conf /buckets diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile index 379c5ffaf..4de8b8ab9 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile @@ -1,8 +1,7 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages -COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions +COPY flavor_base_deps_packages.yaml /build_info/packages/flavor_base_deps_packages.yaml +RUN exaslpm install --package-file /build_info/packages/flavor_base_deps_packages.yaml --build-step flavor_base_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3.12 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile index 0af22d4da..6a4a7d343 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile @@ -1,16 +1,8 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps -COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions - -RUN /scripts/install_scripts/install_python3.12_pip.sh "pip == 25.2" - -COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3.12 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages - -ENV PYTHON3_PREFIX /usr -ENV PYTHON3_VERSION python3.12 +COPY language_deps_packages.yaml /build_info/packages/language_deps_packages.yaml +RUN exaslpm install --package-file /build_info/packages/language_deps_packages.yaml --build-step language_deps RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 #Needed for Bazel build diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml b/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml new file mode 100644 index 000000000..652d77e8a --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml @@ -0,0 +1,69 @@ +build_steps: +- name: build_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: 9.4-3ubuntu6.1 + - name: tar + version: 1.35+dfsg-3build1 + - name: curl + version: 8.5.0-2ubuntu10.7 + - name: openjdk-17-jdk + version: 17.0.18+8-1~24.04.1 + - name: build-essential + version: 12.10ubuntu1 + - name: libpcre3-dev + version: 2:8.39-15build1 + - name: protobuf-compiler + version: 3.21.12-8.2ubuntu0.3 + - name: chrpath + version: 0.16-2build1 + - name: locales + version: 2.39-0ubuntu8.7 + variables: + PROTOBUF_BIN: /usr/bin/protoc + JAVA_HOME: usr/lib/jvm/java-1.17.0-openjdk-{% if platform == 'x86_64' %}amd64{% else %}arm64{% endif %} + - name: install_bazel + tools: + bazel: + version: '8.3.1' + validation_cfg: + version_mandatory: true +- name: base_test_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: gdb + version: 15.0.50.20240403-0ubuntu1 + - name: valgrind + version: 1:3.22.0-0ubuntu3 + - name: binutils + version: 2.42-4ubuntu2.8 + - name: patchelf + version: 0.18.0-1.1build1 + - name: strace + version: 6.8-0ubuntu2 + validation_cfg: + version_mandatory: true +- name: security_scan + phases: + - name: install_deps + apt: + packages: + - name: gnupg + - name: install_trivy + apt: + repos: + trivy: + key_url: https://aquasecurity.github.io/trivy-repo/deb/public.key + entry: deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb + generic main + out_file: trivy.list + packages: + - name: trivy + validation_cfg: + version_mandatory: false +version: 1.0.0 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile index db21d4b92..22e983ad0 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile @@ -2,15 +2,12 @@ FROM {{release}} ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /build_info/packages -COPY security_scan/packages /build_info/packages/security_scan -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages_trivy_deps -RUN curl -s https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \ - /scripts/install_scripts/install_ppa.pl --ppa 'deb https://aquasecurity.github.io/trivy-repo/deb noble main' --out-file trivy.list -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages +COPY security_scan_packages.yaml /build_info/packages/security_scan_packages.yaml +RUN exaslpm install --package-file /build_info/packages/security_scan_packages.yaml --build-step security_scan ENV SECURITY_SCANNERS="trivy" COPY /security_scan/.trivyignore /.trivyignore COPY /security_scan/trivy.rego /trivy.rego COPY /security_scan/trivy.yaml /trivy.yaml -ENTRYPOINT ["/scripts/security_scan/run.sh"] \ No newline at end of file +ENTRYPOINT ["/scripts/security_scan/run.sh"] diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig deleted file mode 100644 index d1bdb8113..000000000 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig +++ /dev/null @@ -1,2 +0,0 @@ -generic_language_tests=python3 -test_folders=python3/all pandas/all pandas/pandas2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile index d8e3e9ea4..0419c24c7 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -1,15 +1,12 @@ -FROM ubuntu:24.04 +FROM exasol/script-language-container:exaslpm-ubuntu-24.04 ENV DEBIAN_FRONTEND=noninteractive -ENV ARCHIVE_UBUNTU_PREFIX="" -RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list - COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc COPY scripts /scripts RUN mkdir -p /build_info/packages -COPY udfclient_deps/packages /build_info/packages/udfclient_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions +COPY udfclient_deps_packages.yaml /build_info/packages/udfclient_deps_packages.yaml +RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages.yaml --build-step udfclient_deps RUN usermod -l exasolution ubuntu RUN groupmod -n exasolution ubuntu @@ -19,8 +16,3 @@ RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,R ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 - -ENV PROTOBUF_LIBRARY_PREFIX=/usr/lib/ -ENV PROTOBUF_INCLUDE_PREFIX=/usr/include/ -ENV ZMQ_LIBRARY_PREFIX=/usr/lib -ENV ZMQ_INCLUDE_PREFIX=/usr/include diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile index f8f16acdc..1cf99264c 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile @@ -16,11 +16,9 @@ FROM {{flavor_base_deps}} RUN mkdir -p /build_info/packages/flavor_customization -COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_customization -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions +COPY flavor_customization_packages.yaml /build_info/packages/flavor_customization_packages.yaml +RUN exaslpm install --package-file /build_info/packages/flavor_customization_packages.yaml --build-step flavor_customization -COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3.12 --with-versions --allow-no-version ########################################################################## diff --git a/flavors/template-Exasol-all-python-3.12/packages.yml b/flavors/template-Exasol-all-python-3.12/packages.yml new file mode 100644 index 000000000..b4efa0f08 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/packages.yml @@ -0,0 +1,96 @@ +build_steps: +- name: language_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: ca-certificates + version: '20240203' + - name: python3.12-dev + version: 3.12.3-1ubuntu0.11 + - name: curl + version: 8.5.0-2ubuntu10.7 + variables: + PYTHON3_PREFIX: /usr + PYTHON3_VERSION: python3.12 + - name: phase_python_binary + tools: + python_binary_path: /usr/bin/python3.12 + - name: install_pip + tools: + pip: + version: '25.2' + needs_break_system_packages: true + - name: install_pip_packages + pip: + packages: + - name: pandas + version: ==2.2.3 + extras: [] + - name: numpy + version: ==1.26.4 + extras: [] + - name: pyarrow + version: ==22.0.0 + extras: [] + install_build_tools_ephemerally: false + validation_cfg: + version_mandatory: true +- name: udfclient_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: 9.4-3ubuntu6.1 + - name: adduser + version: 3.137ubuntu1 + - name: locales + version: 2.39-0ubuntu8.7 + - name: libnss-db + version: 2.2.3pre1-9build3 + - name: libzmq3-dev + version: 4.3.5-1build2 + - name: cppzmq-dev + version: 4.10.0-1build1 + - name: libprotobuf-dev + version: 3.21.12-8.2ubuntu0.3 + - name: libssl-dev + version: 3.0.13-0ubuntu3.7 + variables: + PROTOBUF_LIBRARY_PREFIX: /usr/lib/ + PROTOBUF_INCLUDE_PREFIX: /usr/include/ + ZMQ_LIBRARY_PREFIX: /usr/lib + ZMQ_INCLUDE_PREFIX: /usr/include + validation_cfg: + version_mandatory: true +- name: flavor_base_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: unzip + version: 6.0-28ubuntu4.1 + - name: git + version: 1:2.43.0-1ubuntu7.3 + - name: libcurl4-openssl-dev + version: 8.5.0-2ubuntu10.7 + - name: build-essential + version: 12.10ubuntu1 + validation_cfg: + version_mandatory: true +- name: flavor_customization + phases: + - name: install_apt_packages + apt: + packages: [] +# - name: p7zip-full +# version: 16.02+dfsg-6 + - name: install_pip_packages + pip: + packages: [] +# - name: tensorflow-probability +# version: ==0.9.0 + validation_cfg: + version_mandatory: false +version: 1.0.0 From 2270ba7101fb8c3d7445c7ae79e8a3f22cb3cb6f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 9 Mar 2026 10:07:55 -0300 Subject: [PATCH 2/2] Use exasol/script-language-container:exaslpm-1.0.0-ubuntu-24.04 as Docker base img --- .../flavor_base/build_deps/Dockerfile | 2 +- .../flavor_base/udfclient_deps/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile index 15d00c843..328930734 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile @@ -1,4 +1,4 @@ -FROM exasol/script-language-container:exaslpm-ubuntu-24.04 +FROM exasol/script-language-container:exaslpm-1.0.0-ubuntu-24.04 ENV DEBIAN_FRONTEND=noninteractive COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile index 0419c24c7..78c3c785a 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -1,4 +1,4 @@ -FROM exasol/script-language-container:exaslpm-ubuntu-24.04 +FROM exasol/script-language-container:exaslpm-1.0.0-ubuntu-24.04 ENV DEBIAN_FRONTEND=noninteractive COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc