diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/_activate_current_env.sh b/ext/scripts/install_scripts/_activate_current_env.sh similarity index 93% rename from flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/_activate_current_env.sh rename to ext/scripts/install_scripts/_activate_current_env.sh index 8c5f75ece..99d0714a2 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/_activate_current_env.sh +++ b/ext/scripts/install_scripts/_activate_current_env.sh @@ -1,3 +1,5 @@ +# shellcheck disable=SC2148 + # This script should never be called directly, only sourced: # source _activate_current_env.sh diff --git a/ext/scripts/install_scripts/prepare_conda_env.sh b/ext/scripts/install_scripts/prepare_conda_env.sh new file mode 100755 index 000000000..5b64355e2 --- /dev/null +++ b/ext/scripts/install_scripts/prepare_conda_env.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + + +# Setup MAMBA_ROOT_PREFIX +mkdir -p "$MAMBA_ROOT_PREFIX/conda-meta" && \ +chmod -R a+rwx "$MAMBA_ROOT_PREFIX" && \ + +# Setting environment variables for activating micromamba +{ + echo "export ENV_NAME=$ENV_NAME" + echo "export MAMBA_ROOT_PREFIX=$MAMBA_ROOT_PREFIX" + echo "export MAMBA_EXE=$MAMBA_EXE" + echo "export MAMBA_DOCKERFILE_ACTIVATE=$MAMBA_DOCKERFILE_ACTIVATE" + cat /usr/local/bin/_activate_current_env.sh +} > /tmp/_activate_current_env.sh +cp /tmp/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh +rm /tmp/_activate_current_env.sh + +# Activate micromamba for the bash +echo "source /usr/local/bin/_activate_current_env.sh" >> ~/.bashrc && \ +echo "source /usr/local/bin/_activate_current_env.sh" >> /etc/skel/.bashrc && \ +ln -s /usr/local/bin/_activate_current_env.sh /etc/profile.d/_activate_current_env.sh && \ +chmod -R a+rx /usr/local/bin/_activate_current_env.sh + +# Add conda lib directory to ld.so.conf +echo "$MAMBA_ROOT_PREFIX/lib" > /etc/ld.so.conf.d/conda.conf \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile index e4dcea215..b6b9efe9d 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile @@ -2,6 +2,8 @@ FROM {{base_test_deps}} RUN mkdir /conf /buckets +RUN exaslpm export-variables --out-file /env + RUN ldconfig RUN mkdir /exaudfclient /exaudf @@ -9,10 +11,7 @@ COPY /exaudfclient/ /exaudfclient/ WORKDIR /exaudfclient/ # We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ -ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries +RUN bash -c "source /env && ./build.sh --config no-tty -c dbg --config python --config test-binaries" RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_deps/Dockerfile index 3a8ca3f61..dd98448db 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_deps/Dockerfile @@ -1,6 +1,6 @@ FROM {{build_deps}} RUN mkdir -p /build_info/packages -COPY base_test_deps/packages /build_info/packages/base_test_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions +COPY base_test_deps_packages.yml /build_info/packages/base_test_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/base_test_deps_packages.yml --build-step base_test_deps diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/Dockerfile index 83c1a61bf..cc2e07826 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/Dockerfile @@ -1,13 +1,11 @@ FROM {{language_deps}} RUN mkdir -p /build_info/packages -COPY build_deps/packages /build_info/packages/build_deps -RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba" +COPY build_deps_packages.yml /build_info/packages/build_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/build_deps_packages.yml --build-step build_deps RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ tar zxf swig-2.0.4.tar.gz && \ (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ rm -rf swig-2.0.4 swig-2.0.4.tar.gz - -ENV PROTOBUF_BIN=$MAMBA_ROOT_PREFIX/bin/protoc diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile index 92b8f2343..fa3a6786b 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile @@ -1,13 +1,13 @@ FROM {{build_deps}} +RUN exaslpm export-variables --out-file /env + RUN mkdir /exaudfclient /exaudf COPY exaudfclient/ /exaudfclient/ WORKDIR /exaudfclient/ -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ -ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ -RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary --config fix_conda_ar_tool + +RUN bash -c "source /env && ./build.sh --config no-tty --config optimize --config python --config fast-binary --config fix_conda_ar_tool" RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base @@ -17,8 +17,6 @@ WORKDIR /exaudf RUN rm -r /exaudfclient -COPY --from={{build_deps}} /build_info /build_info -RUN true # workaround for https://github.com/moby/moby/issues/37965 RUN mkdir /conf /buckets diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/Dockerfile index 93ffb34a9..32a950b82 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/conda_deps/Dockerfile @@ -1,32 +1,30 @@ -FROM ubuntu:22.04 +FROM exasol/script-language-container:exaslpm-1.1.0-ubuntu-22.04 ENV DEBIAN_FRONTEND=noninteractive -ENV ARCHIVE_UBUNTU_PREFIX="" -RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc COPY scripts /scripts RUN mkdir -p /build_info/packages -COPY conda_deps/packages /build_info/packages/conda_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/conda_deps/apt_get_packages --with-versions - -ENV ENV_NAME="base" -ENV MAMBA_ROOT_PREFIX="/opt/conda" -ENV MAMBA_EXE="/bin/micromamba" -ENV MAMBA_DOCKERFILE_ACTIVATE=1 - -COPY conda_deps/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh +COPY conda_deps_packages.yml /build_info/packages/conda_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/conda_deps_packages.yml --build-step conda_deps RUN addgroup --gid 1000 exasolution RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" RUN addgroup --gid 500 exausers RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +ENV MAMBA_EXE="/bin/micromamba" +ENV MAMBA_DOCKERFILE_ACTIVATE=1 +ENV ENV_NAME="base" +ENV MAMBA_ROOT_PREFIX="/opt/conda" + +COPY scripts/install_scripts/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh + +RUN /scripts/install_scripts/prepare_conda_env.sh + SHELL ["/bin/bash", "-l", "-c"] -ENV MICROMAMBA_VERSION=0.24.0 -RUN /scripts/install_scripts/install_micromamba.sh "$MICROMAMBA_VERSION" ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/flavor_base_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/flavor_base_deps/Dockerfile index 1124519bb..e8245247f 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/flavor_base_deps/Dockerfile @@ -1,10 +1,8 @@ FROM {{ language_deps }} RUN mkdir -p /build_info/packages -COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions +COPY flavor_base_deps_packages.yml /build_info/packages/flavor_base_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/flavor_base_deps_packages.yml --build-step flavor_base_deps -RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba" -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.10" --with-versions --allow-no-version diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/Dockerfile index c913d8955..1accd73d7 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/Dockerfile @@ -1,10 +1,8 @@ FROM {{udfclient_deps}} RUN mkdir -p /build_info/packages/language_deps -COPY language_deps/packages/ /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/language_deps/conda_packages --channel-file /build_info/packages/language_deps/conda_channels --with-versions --conda-binary /bin/micromamba +COPY language_deps_packages.yml /build_info/packages/language_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/language_deps_packages.yml --build-step language_deps -ENV PYTHON3_PREFIX="$MAMBA_ROOT_PREFIX" -ENV PYTHON3_VERSION=python3.10 diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml new file mode 100644 index 000000000..c396c9033 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml @@ -0,0 +1,68 @@ +build_steps: +- name: build_deps + phases: + - name: install_conda_packages + conda: + channels: + - conda-forge + packages: + - name: protobuf + version: =6.31.1 + - name: pcre + version: '=8.45' + - name: bazel + version: =8.3.1 + - name: gxx_linux-64 + version: =12.4.0 + - name: gcc_linux-64 + version: =12.4.0 + - name: make + version: '=4.3' + - name: chrpath + version: '=0.16' + binary: Mamba + variables: + PROTOBUF_BIN: /opt/conda/bin/protoc + LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/' + CPLUS_INCLUDE_PATH: '$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/' + C_INCLUDE_PATH: '$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/' + validation_cfg: + version_mandatory: true +- name: base_test_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: gdb + version: 12.1-0ubuntu1~22.04.2 + - name: valgrind + version: 1:3.18.1-1ubuntu2 + - name: gdbserver + version: 12.1-0ubuntu1~22.04.2 + - name: binutils + version: 2.38-4ubuntu2.12 + - name: patchelf + version: 0.14.3-1 + - name: strace + version: 5.16-0ubuntu3 + validation_cfg: + version_mandatory: true +- name: security_scan + phases: + - name: install_deps + apt: + packages: + - name: gnupg + - name: install_trivy + apt: + repos: + trivy: + key_url: https://aquasecurity.github.io/trivy-repo/deb/public.key + entry: deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb + generic main + out_file: trivy.list + packages: + - name: trivy + validation_cfg: + version_mandatory: false +version: 1.0.0 diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/security_scan/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/security_scan/Dockerfile index 2147907af..50b4070d7 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/security_scan/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/security_scan/Dockerfile @@ -2,15 +2,12 @@ FROM {{release}} ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /build_info/packages -COPY security_scan/packages /build_info/packages/security_scan -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages_trivy_deps -RUN curl -s https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \ - /scripts/install_scripts/install_ppa.pl --ppa 'deb https://aquasecurity.github.io/trivy-repo/deb bionic main' --out-file trivy.list -RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages +COPY security_scan_packages.yml /build_info/packages/security_scan_packages.yml +RUN exaslpm install --package-file /build_info/packages/security_scan_packages.yml --build-step security_scan ENV SECURITY_SCANNERS="trivy" COPY /security_scan/.trivyignore /.trivyignore COPY /security_scan/trivy.rego /trivy.rego COPY /security_scan/trivy.yaml /trivy.yaml -ENTRYPOINT ["/scripts/security_scan/run.sh"] \ No newline at end of file +ENTRYPOINT ["/scripts/security_scan/run.sh"] diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/testconfig b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/testconfig deleted file mode 100644 index d1bdb8113..000000000 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/testconfig +++ /dev/null @@ -1,2 +0,0 @@ -generic_language_tests=python3 -test_folders=python3/all pandas/all pandas/pandas2 diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/Dockerfile index f367d2b05..51eaf8f10 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/Dockerfile @@ -1,10 +1,6 @@ FROM {{ conda_deps }} -COPY udfclient_deps/packages /build_info/packages/udfclient_deps -RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/udfclient_deps/conda_packages --channel-file /build_info/packages/udfclient_deps/conda_channels --with-versions --conda-binary /bin/micromamba +COPY udfclient_deps_packages.yml /build_info/packages/udfclient_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages.yml --build-step udfclient_deps -ENV PROTOBUF_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib -ENV PROTOBUF_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include -ENV ZMQ_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib -ENV ZMQ_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_customization/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_customization/Dockerfile index 5c80ed8ee..ab40b150d 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_customization/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_customization/Dockerfile @@ -16,11 +16,9 @@ FROM {{flavor_base_deps}} RUN mkdir -p /build_info/packages/flavor_customization -COPY flavor_customization/packages /build_info/packages/flavor_customization -RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_customization/conda_packages --channel-file /build_info/packages/flavor_customization/conda_channels --with-versions --allow-no-version --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba" +COPY flavor_customization_packages.yml /build_info/packages/flavor_customization_packages.yml +RUN exaslpm install --package-file /build_info/packages/flavor_customization_packages.yml --build-step flavor_customization -COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.10" --with-versions --allow-no-version ########################################################################## ########################################################################## diff --git a/flavors/template-Exasol-all-python-3.10-conda/packages.yml b/flavors/template-Exasol-all-python-3.10-conda/packages.yml new file mode 100644 index 000000000..f90d134c7 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.10-conda/packages.yml @@ -0,0 +1,110 @@ +build_steps: +- name: conda_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: 8.32-4.1ubuntu1.2 + - name: locales + version: 2.35-0ubuntu3.13 + - name: curl + version: 7.81.0-1ubuntu1.23 + - name: ca-certificates + version: 20240203~22.04.1 + - name: bzip2 + version: 1.0.8-5build1 + - name: libxml2 + version: 2.9.13+dfsg-1ubuntu0.11 + - name: install_micromamba + tools: + micromamba: + version: 2.5.0-1 + root_prefix: /opt/conda + variables: + MAMBA_ROOT_PREFIX: /opt/conda + validation_cfg: + version_mandatory: true +- name: udfclient_deps + phases: + - name: install_conda_packages + conda: + channels: + - conda-forge + packages: + - name: libprotobuf + version: =6.31.1 + - name: zeromq + version: =4.3.5 + - name: nss + version: '=3.100' + - name: cppzmq + version: =4.10.0 + binary: Micromamba + variables: + PROTOBUF_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib + PROTOBUF_INCLUDE_PREFIX: $MAMBA_ROOT_PREFIX/include + ZMQ_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib + ZMQ_INCLUDE_PREFIX: $MAMBA_ROOT_PREFIX/include + validation_cfg: + version_mandatory: true +- name: language_deps + phases: + - name: install_conda_packages + conda: + channels: + - conda-forge + packages: + - name: python + version: =3.10.4 + - name: numpy + version: =1.26.2 + - name: pandas + version: =2.2.3 + - name: pyarrow + version: =22.0.0 + - name: mamba + version: =2.3.3 + binary: Micromamba + variables: + PYTHON3_PREFIX: $MAMBA_ROOT_PREFIX + PYTHON3_VERSION: python3.10 + - name: configure_tools + tools: + mamba_binary_path: /opt/conda/bin/mamba + python_binary_path: /opt/conda/bin/python3.10 + validation_cfg: + version_mandatory: true +- name: flavor_base_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: tzdata + version: 2025b-0ubuntu0.22.04.1 + - name: install_conda_packages + conda: + channels: + - conda-forge + packages: [] + binary: Mamba + validation_cfg: + version_mandatory: true +- name: flavor_customization + phases: + - name: install_conda_packages + conda: + channels: + - conda-forge + packages: [] +# - name: cuda-compat +# version: =12.9.1 + binary: Mamba + - name: install_pip_packages + pip: + packages: [] +# - name: tensorflow-probability +# version: =0.9.0 + validation_cfg: + version_mandatory: false +version: 1.0.0