Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# shellcheck disable=SC2148

# This script should never be called directly, only sourced:

# source _activate_current_env.sh
Expand Down
30 changes: 30 additions & 0 deletions ext/scripts/install_scripts/prepare_conda_env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash

set -e
set -u
set -o pipefail


# Setup MAMBA_ROOT_PREFIX
mkdir -p "$MAMBA_ROOT_PREFIX/conda-meta" && \
chmod -R a+rwx "$MAMBA_ROOT_PREFIX" && \

# Setting environment variables for activating micromamba
{
echo "export ENV_NAME=$ENV_NAME"
echo "export MAMBA_ROOT_PREFIX=$MAMBA_ROOT_PREFIX"
echo "export MAMBA_EXE=$MAMBA_EXE"
echo "export MAMBA_DOCKERFILE_ACTIVATE=$MAMBA_DOCKERFILE_ACTIVATE"
cat /usr/local/bin/_activate_current_env.sh
} > /tmp/_activate_current_env.sh
cp /tmp/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh
rm /tmp/_activate_current_env.sh

# Activate micromamba for the bash
echo "source /usr/local/bin/_activate_current_env.sh" >> ~/.bashrc && \
echo "source /usr/local/bin/_activate_current_env.sh" >> /etc/skel/.bashrc && \
ln -s /usr/local/bin/_activate_current_env.sh /etc/profile.d/_activate_current_env.sh && \
chmod -R a+rx /usr/local/bin/_activate_current_env.sh

# Add conda lib directory to ld.so.conf
echo "$MAMBA_ROOT_PREFIX/lib" > /etc/ld.so.conf.d/conda.conf
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,16 @@ FROM {{base_test_deps}}

RUN mkdir /conf /buckets

RUN exaslpm export-variables --out-file /env

RUN ldconfig

RUN mkdir /exaudfclient /exaudf
COPY /exaudfclient/ /exaudfclient/

WORKDIR /exaudfclient/
# We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries
RUN bash -c "source /env && ./build.sh --config no-tty -c dbg --config python --config test-binaries"
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM {{build_deps}}

RUN mkdir -p /build_info/packages
COPY base_test_deps/packages /build_info/packages/base_test_deps

RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions
COPY base_test_deps_packages.yml /build_info/packages/base_test_deps_packages.yml
RUN exaslpm install --package-file /build_info/packages/base_test_deps_packages.yml --build-step base_test_deps
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
FROM {{language_deps}}

RUN mkdir -p /build_info/packages
COPY build_deps/packages /build_info/packages/build_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/build_deps/conda_packages --channel-file /build_info/packages/build_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"
COPY build_deps_packages.yml /build_info/packages/build_deps_packages.yml
RUN exaslpm install --package-file /build_info/packages/build_deps_packages.yml --build-step build_deps

RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \
tar zxf swig-2.0.4.tar.gz && \
(cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
rm -rf swig-2.0.4 swig-2.0.4.tar.gz

ENV PROTOBUF_BIN=$MAMBA_ROOT_PREFIX/bin/protoc
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
FROM {{build_deps}}

RUN exaslpm export-variables --out-file /env

RUN mkdir /exaudfclient /exaudf
COPY exaudfclient/ /exaudfclient/

WORKDIR /exaudfclient/
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/
ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/
RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary --config fix_conda_ar_tool

RUN bash -c "source /env && ./build.sh --config no-tty --config optimize --config python --config fast-binary --config fix_conda_ar_tool"
RUN cp -r -L bazel-bin/* /exaudf

WORKDIR /exaudfclient/base
Expand All @@ -17,8 +17,6 @@ WORKDIR /exaudf

RUN rm -r /exaudfclient

COPY --from={{build_deps}} /build_info /build_info
RUN true # workaround for https://github.com/moby/moby/issues/37965


RUN mkdir /conf /buckets
Original file line number Diff line number Diff line change
@@ -1,32 +1,30 @@
FROM ubuntu:22.04
FROM exasol/script-language-container:exaslpm-1.1.0-ubuntu-22.04
ENV DEBIAN_FRONTEND=noninteractive

ENV ARCHIVE_UBUNTU_PREFIX=""
RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list

COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc

COPY scripts /scripts

RUN mkdir -p /build_info/packages
COPY conda_deps/packages /build_info/packages/conda_deps
RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/conda_deps/apt_get_packages --with-versions

ENV ENV_NAME="base"
ENV MAMBA_ROOT_PREFIX="/opt/conda"
ENV MAMBA_EXE="/bin/micromamba"
ENV MAMBA_DOCKERFILE_ACTIVATE=1

COPY conda_deps/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh
COPY conda_deps_packages.yml /build_info/packages/conda_deps_packages.yml
RUN exaslpm install --package-file /build_info/packages/conda_deps_packages.yml --build-step conda_deps

RUN addgroup --gid 1000 exasolution
RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
RUN addgroup --gid 500 exausers
RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone"

ENV MAMBA_EXE="/bin/micromamba"
ENV MAMBA_DOCKERFILE_ACTIVATE=1
ENV ENV_NAME="base"
ENV MAMBA_ROOT_PREFIX="/opt/conda"

COPY scripts/install_scripts/_activate_current_env.sh /usr/local/bin/_activate_current_env.sh

RUN /scripts/install_scripts/prepare_conda_env.sh

SHELL ["/bin/bash", "-l", "-c"]
ENV MICROMAMBA_VERSION=0.24.0
RUN /scripts/install_scripts/install_micromamba.sh "$MICROMAMBA_VERSION"

ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
FROM {{ language_deps }}

RUN mkdir -p /build_info/packages
COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps

RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions
COPY flavor_base_deps_packages.yml /build_info/packages/flavor_base_deps_packages.yml
RUN exaslpm install --package-file /build_info/packages/flavor_base_deps_packages.yml --build-step flavor_base_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_base_deps/conda_packages --channel-file /build_info/packages/flavor_base_deps/conda_channels --with-versions --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"

RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.10" --with-versions --allow-no-version
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
FROM {{udfclient_deps}}

RUN mkdir -p /build_info/packages/language_deps
COPY language_deps/packages/ /build_info/packages/language_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/language_deps/conda_packages --channel-file /build_info/packages/language_deps/conda_channels --with-versions --conda-binary /bin/micromamba
COPY language_deps_packages.yml /build_info/packages/language_deps_packages.yml
RUN exaslpm install --package-file /build_info/packages/language_deps_packages.yml --build-step language_deps

ENV PYTHON3_PREFIX="$MAMBA_ROOT_PREFIX"
ENV PYTHON3_VERSION=python3.10

Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
build_steps:
- name: build_deps
phases:
- name: install_conda_packages
conda:
channels:
- conda-forge
packages:
- name: protobuf
version: =6.31.1
- name: pcre
version: '=8.45'
- name: bazel
version: =8.3.1
- name: gxx_linux-64
version: =12.4.0
- name: gcc_linux-64
version: =12.4.0
- name: make
version: '=4.3'
- name: chrpath
version: '=0.16'
binary: Mamba
variables:
PROTOBUF_BIN: /opt/conda/bin/protoc
LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/'
CPLUS_INCLUDE_PATH: '$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/'
C_INCLUDE_PATH: '$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/'
validation_cfg:
version_mandatory: true
- name: base_test_deps
phases:
- name: install_apt_packages
apt:
packages:
- name: gdb
version: 12.1-0ubuntu1~22.04.2
- name: valgrind
version: 1:3.18.1-1ubuntu2
- name: gdbserver
version: 12.1-0ubuntu1~22.04.2
- name: binutils
version: 2.38-4ubuntu2.12
- name: patchelf
version: 0.14.3-1
- name: strace
version: 5.16-0ubuntu3
validation_cfg:
version_mandatory: true
- name: security_scan
phases:
- name: install_deps
apt:
packages:
- name: gnupg
- name: install_trivy
apt:
repos:
trivy:
key_url: https://aquasecurity.github.io/trivy-repo/deb/public.key
entry: deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb
generic main
out_file: trivy.list
packages:
- name: trivy
validation_cfg:
version_mandatory: false
version: 1.0.0
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,12 @@ FROM {{release}}
ENV DEBIAN_FRONTEND=noninteractive

RUN mkdir -p /build_info/packages
COPY security_scan/packages /build_info/packages/security_scan
RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages_trivy_deps
RUN curl -s https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \
/scripts/install_scripts/install_ppa.pl --ppa 'deb https://aquasecurity.github.io/trivy-repo/deb bionic main' --out-file trivy.list
RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages
COPY security_scan_packages.yml /build_info/packages/security_scan_packages.yml
RUN exaslpm install --package-file /build_info/packages/security_scan_packages.yml --build-step security_scan

ENV SECURITY_SCANNERS="trivy"
COPY /security_scan/.trivyignore /.trivyignore
COPY /security_scan/trivy.rego /trivy.rego
COPY /security_scan/trivy.yaml /trivy.yaml

ENTRYPOINT ["/scripts/security_scan/run.sh"]
ENTRYPOINT ["/scripts/security_scan/run.sh"]

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
FROM {{ conda_deps }}

COPY udfclient_deps/packages /build_info/packages/udfclient_deps

RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/udfclient_deps/conda_packages --channel-file /build_info/packages/udfclient_deps/conda_channels --with-versions --conda-binary /bin/micromamba
COPY udfclient_deps_packages.yml /build_info/packages/udfclient_deps_packages.yml
RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages.yml --build-step udfclient_deps

ENV PROTOBUF_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib
ENV PROTOBUF_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include
ENV ZMQ_LIBRARY_PREFIX=$MAMBA_ROOT_PREFIX/lib
ENV ZMQ_INCLUDE_PREFIX=$MAMBA_ROOT_PREFIX/include
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@ FROM {{flavor_base_deps}}

RUN mkdir -p /build_info/packages/flavor_customization

COPY flavor_customization/packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_conda.pl --file /build_info/packages/flavor_customization/conda_packages --channel-file /build_info/packages/flavor_customization/conda_channels --with-versions --allow-no-version --conda-binary "$MAMBA_ROOT_PREFIX/bin/mamba"
COPY flavor_customization_packages.yml /build_info/packages/flavor_customization_packages.yml
RUN exaslpm install --package-file /build_info/packages/flavor_customization_packages.yml --build-step flavor_customization

COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary "$MAMBA_ROOT_PREFIX/bin/python3.10" --with-versions --allow-no-version

##########################################################################
##########################################################################
Expand Down
Loading
Loading