From 66a56c1ad9f499017a3a0cf2a2216fd013f2683a Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Tue, 29 Apr 2025 12:09:14 +0200 Subject: [PATCH 1/4] SRE-2382 Update for Fedora 41 Signed-off-by: Oksana Salyk --- packaging/Dockerfile.centos.7 | 30 ++++++++++++-- packaging/Dockerfile.mockbuild | 38 ++++++++++++------ packaging/Dockerfile.ubuntu.20.04 | 25 ++++++------ packaging/Makefile_distro_vars.mk | 16 ++++++++ packaging/Makefile_packaging.mk | 15 ++++++- packaging/ccache-stats.patch | 66 +++++++++++++++++++++++++++++++ packaging/get_base_branch | 22 +++++++++++ packaging/rpm_chrootbuild | 65 +++++++++++++++++++----------- 8 files changed, 223 insertions(+), 54 deletions(-) create mode 100644 packaging/ccache-stats.patch create mode 100755 packaging/get_base_branch diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index cdfb7f6..189ea1e 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -5,9 +5,31 @@ # # Pull base image -FROM centos:7 +FROM centos:centos7 LABEL maintainer="daos@daos.groups.io" +# Use local repo server if present +ARG REPO_FILE_URL +RUN set -e; \ + if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -k -f -o daos_ci-centos7-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-centos7-artifactory.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ + mv daos_ci-centos7-artifactory.repo{.tmp,}; \ + fi; \ + yum -y install dnf; \ + yum clean all; \ + dnf --disablerepo \*epel\* -y install epel-release \ + dnf-plugins-core; \ + if [ -n "$REPO_FILE_URL" ]; then \ + dnf -y --quiet config-manager --disable epel; \ + fi; \ + dnf -y update epel-release; \ + dnf -y clean all + # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -15,9 +37,9 @@ ARG UID=1000 #Nothing to do for CentOS # Install basic tools -RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros dnf +RUN dnf install -y epel-release +RUN dnf install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git python-srpm-macros dnf && dnf -y clean all # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index c8bc1a4..6bad37d 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,11 +1,11 @@ # -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -ARG FVERSION=38 +ARG FVERSION=latest FROM fedora:$FVERSION # Needed for later use of FVERSION ARG FVERSION @@ -15,16 +15,19 @@ LABEL maintainer="daos@daos.groups.io" ARG REPO_FILE_URL RUN if [ -n "$REPO_FILE_URL" ]; then \ cd /etc/yum.repos.d/ && \ - curl -f -o daos_ci-fedora-artifactory.repo.tmp \ + curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \ "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - rm -f *.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ mv daos_ci-fedora-artifactory.repo{.tmp,}; \ fi # Install basic tools RUN dnf -y install mock make \ rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools + python-srpm-macros rpmdevtools && \ + dnf -y clean all # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -32,10 +35,12 @@ ARG UID=1000 # Add build user (to keep rpmbuild happy) ENV USER build ENV PASSWD build -RUN useradd -u $UID -ms /bin/bash $USER -RUN echo "$USER:$PASSWD" | chpasswd # add the user to the mock group so it can run mock -RUN usermod -a -G mock $USER +RUN if [ $UID != 0 ]; then \ + useradd -u $UID -ms /bin/bash $USER; \ + echo "$USER:$PASSWD" | chpasswd; \ + usermod -a -G mock $USER; \ + fi ARG CB0 RUN dnf -y upgrade && \ @@ -45,15 +50,24 @@ RUN dnf -y upgrade && \ # https://github.com/rpm-software-management/rpmlint/pull/795 in it # But make sure to patch after dnf upgrade so that an upgraded rpmlint # RPM doesn't wipe out our patch -COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +# Ditto for the patch to zero and display ccache stats +# https://github.com/rpm-software-management/mock/pull/1299 +ARG PACKAGINGDIR=packaging +COPY ${PACKAGINGDIR}/*.patch ./ RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ - if ! patch -p1; then \ + if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then \ exit 1; \ fi; \ rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ - fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ - rm -f rpmlint--ignore-unused-rpmlintrc.patch + fi; \ + if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then \ + if ! patch -p3 < $OLDPWD/ccache-stats.patch; then \ + exit 1; \ + fi; \ + rm -f mockbuild/plugins/__pycache__/ccache.*.pyc; \ + fi); \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch # show the release that was built ARG CACHEBUST diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index ec76bfd..4d05464 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -18,26 +18,23 @@ RUN if [ -n "$REPO_FILE_URL" ]; then \ true > ../sources.list && \ mv daos_ci-ubuntu20.04-artifactory.list.tmp \ daos_ci-ubuntu20.04-artifactory.list; \ + url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \ + else \ + url="https://downloads.linux.hpe.com/SDR/repo/ilorest/"; \ fi; \ cd -; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ mkdir -p /usr/local/share/keyrings/; \ + curl -f -O "$url"GPG-KEY-hprest; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import GPG-KEY-hprest; \ gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg; \ + --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ rm ./temp-keyring.gpg; \ - url_prefix=https://downloads.linux.hpe.com/SDR/; \ - for url in hpPublicKey2048.pub \ - hpPublicKey2048_key1.pub \ - hpePublicKey2048_key1.pub; do \ - curl -f -O "$url_prefix$url"; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import "$(basename $url)"; \ - done; \ + curl -f -O "$REPO_FILE_URL"esad_repo.key; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key; \ gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg + --output /usr/local/share/keyrings/daos-stack-public.gpg # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 6a7f88b..79ae211 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -83,6 +83,22 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64) +VERSION_ID := 15.5 +DISTRO_ID := sl15.5 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.5 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.6-x86_64) +VERSION_ID := 15.6 +DISTRO_ID := sl15.6 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.6 +SED_EXPR := 1p +endif endif ifeq ($(ID),centos) ID = el diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 2fdcaed..75038c1 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -35,7 +35,11 @@ TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project REPO_NAME ?= $(NAME) +ifneq ($(CI_PR_REPOS),) +PR_REPOS ?= $(CI_PR_REPOS) +else PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') +endif LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') @@ -54,7 +58,8 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES) GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') endif -COMMON_RPM_ARGS = --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) +FVERSION ?= latest +COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) DEB_RVERS := $(subst $(DOT),\$(DOT),$(VERSION)) @@ -363,12 +368,14 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) LOCAL_REPOS='$(LOCAL_REPOS)' \ ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ DISTRO_VERSION="$(DISTRO_VERSION)" \ + PACKAGE="$(NAME)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif podman_chrootbuild: if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + --build-arg FVERSION=$(FVERSION) \ -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ -f packaging/Dockerfile.mockbuild .; then \ echo "Container build failed"; \ @@ -386,7 +393,9 @@ podman_chrootbuild: exit 1; \ fi; \ rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ - grep -v -e debuginfo -e debugsource -e src.rpm)' + grep -v -e debuginfo -e debugsource -e src.rpm)'; then \ + exit 1; \ + fi docker_chrootbuild: if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ @@ -419,6 +428,8 @@ packaging_check: --exclude libfabric.spec \ --exclude Makefile \ --exclude README.md \ + --exclude SECURITY.md \ + --exclude LICENSE \ --exclude _topdir \ --exclude \*.tar.\* \ --exclude \*.code-workspace \ diff --git a/packaging/ccache-stats.patch b/packaging/ccache-stats.patch new file mode 100644 index 0000000..26d5eeb --- /dev/null +++ b/packaging/ccache-stats.patch @@ -0,0 +1,66 @@ +From e87d916d7f49ea4949973adf0f09e9e5bf891e03 Mon Sep 17 00:00:00 2001 +From: "Brian J. Murrell" +Date: Tue, 30 Jan 2024 11:03:12 -0500 +Subject: [PATCH 1/2] Show ccache stats at the end of the build + +Zero the ccache stats at the beginning of the build and then display the +ccache stats at the end of the build to see how effective ccache was. + +Signed-off-by: Brian J. Murrell +--- + mock/py/mockbuild/plugins/ccache.py | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py +index 2666ad9fc..1080ffe68 100644 +--- a/mock/py/mockbuild/plugins/ccache.py ++++ b/mock/py/mockbuild/plugins/ccache.py +@@ -35,6 +35,7 @@ def __init__(self, plugins, conf, buildroot): + buildroot.preexisting_deps.append("ccache") + plugins.add_hook("prebuild", self._ccacheBuildHook) + plugins.add_hook("preinit", self._ccachePreInitHook) ++ plugins.add_hook("postbuild", self._ccachePostBuildHook) + buildroot.mounts.add( + BindMountPoint(srcpath=self.ccachePath, bindpath=buildroot.make_chroot_path("/var/tmp/ccache"))) + +@@ -47,6 +48,9 @@ def __init__(self, plugins, conf, buildroot): + @traceLog() + def _ccacheBuildHook(self): + self.buildroot.doChroot(["ccache", "-M", str(self.ccache_opts['max_cache_size'])], shell=False) ++ # zero ccache stats ++ getLog().info("Zero ccache stats:") ++ self.buildroot.doChroot(["ccache", "--zero-stats"], printOutput=True, shell=False) + + # set up the ccache dir. + # we also set a few variables used by ccache to find the shared cache. +@@ -61,3 +65,10 @@ def _ccachePreInitHook(self): + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache')) + file_util.mkdirIfAbsent(self.ccachePath) + self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True) ++ ++ # get some cache stats ++ def _ccachePostBuildHook(self): ++ # show the cache hit stats ++ getLog().info("ccache stats:") ++ self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) +++ + +From bfd3a7e1bb47d28ee60a94cb5985c1f66476475f Mon Sep 17 00:00:00 2001 +From: "Brian J. Murrell" +Date: Tue, 30 Jan 2024 11:17:48 -0500 +Subject: [PATCH 2/2] Remove extraneous line + +Signed-off-by: Brian J. Murrell +--- + mock/py/mockbuild/plugins/ccache.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py +index 1080ffe68..1a20846d3 100644 +--- a/mock/py/mockbuild/plugins/ccache.py ++++ b/mock/py/mockbuild/plugins/ccache.py +@@ -71,4 +71,3 @@ def _ccachePostBuildHook(self): + # show the cache hit stats + getLog().info("ccache stats:") + self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) +-+ diff --git a/packaging/get_base_branch b/packaging/get_base_branch new file mode 100755 index 0000000..75eb90b --- /dev/null +++ b/packaging/get_base_branch @@ -0,0 +1,22 @@ +#!/bin/bash + +# find the base branch of the current branch + +set -eux -o pipefail +IFS=' ' read -r -a add_bases <<< "${1:-}" +origin="${ORIGIN:-origin}" +mapfile -t all_bases < <(echo "master" + git branch -r | sed -ne "/^ $origin\\/release\\/[0-9]/s/^ $origin\\///p") +all_bases+=("${add_bases[@]}") +TARGET="master" +min_diff=-1 +for base in "${all_bases[@]}"; do + git rev-parse --verify "$origin/$base" &> /dev/null || continue + commits_ahead=$(git log --oneline "$origin/$base..HEAD" | wc -l) + if [ "$min_diff" -eq -1 ] || [ "$min_diff" -gt "$commits_ahead" ]; then + TARGET="$base" + min_diff=$commits_ahead + fi +done +echo "$TARGET" +exit 0 diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d122e0e..d6443b7 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,10 +2,14 @@ set -uex -original_cfg_file="/etc/mock/$CHROOT_NAME.cfg" -cfg_file=mock.cfg +cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg + +# Enable mock ccache plugin +cat <> mock.cfg +config_opts['plugin_conf']['ccache_enable'] = True +config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +EOF -cp "$original_cfg_file" "$cfg_file" if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then cat <> mock.cfg @@ -22,7 +26,7 @@ if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then fi # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available -cat <> "$cfg_file" +cat <> mock.cfg config_opts['dnf.conf'] += """ [main] best=0 @@ -33,7 +37,7 @@ EOF repo_adds=() repo_dels=() -echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") @@ -56,7 +60,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" fi fi - curl -sSf "${REPO_FILE_URL}daos_ci-$DISTRO"-mock-artifactory.repo >> "$cfg_file" + curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg repo_adds+=("--enablerepo *-artifactory") fi fi @@ -81,9 +85,9 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ name=$repo:${branch//[@\/]/_}:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ +baseurl=${ARTIFACTS_URL:-${JENKINS_URL:-https://build.hpdd.intel.com/}job/}daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> "$cfg_file" +gpgcheck=False\n" >> mock.cfg done for repo in $JOB_REPOS; do repo_name=${repo##*://} @@ -97,29 +101,46 @@ for repo in $JOB_REPOS; do echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ baseurl=${repo//\//%2F}\n\ -enabled=1\n" >> "$cfg_file" +enabled=1\n" >> mock.cfg done -echo "\"\"\"" >> "$cfg_file" +echo "\"\"\"" >> mock.cfg if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi -bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap -if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then - mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap" +bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap-$(id -u) +if ls -l "$bs_dir"/root_cache/cache.tar.gz; then + mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap/" flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" fi +if ls -l "$bs_dir/ccache-$CHROOT_NAME-$PACKAGE".tar.gz; then + flock "$bs_dir" -c "tar -C / -xzf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz" +fi -# shellcheck disable=SC2086 -eval mock -r "$cfg_file" ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +rc=0 +# shellcheck disable=SC2086,SC2048 +if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ + --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS \ + $RPM_BUILD_OPTIONS "$TARGET"; then + rc=${PIPESTATUS[0]} +fi -date -if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz && - [ -d /scratch/ ]; then - mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/ - if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then - flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" +# Save the ccache +if [ -d /scratch/ ]; then + mkdir -p "$bs_dir"/ + if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then + echo "Failed to save ccache. Plowing onward." + echo "I am $(id)" + fi + if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then + if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then + if ! flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/"; then + echo "Failed to save root_cache. Plowing onward." + echo "I am $(id)" + fi + fi fi fi + +exit "$rc" From 063173a803b8d86bf8f330e7a96342f69cab700d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 1 May 2025 07:02:03 +0200 Subject: [PATCH 2/4] Update all packaging files Signed-off-by: Tomasz Gromadzki --- packaging/Dockerfile.coverity | 3 +- packaging/Dockerfile.mockbuild | 26 +++--- packaging/Dockerfile.ubuntu | 51 ++++++++++++ packaging/Dockerfile.ubuntu.20.04 | 54 +++++-------- packaging/Dockerfile.ubuntu.rolling | 3 +- packaging/debian_chrootbuild | 75 ++++++++++------- packaging/rpm_chrootbuild | 26 +++++- packaging/scripts/repo-helper-fedora.sh | 75 +++++++++++++++++ packaging/scripts/repo-helper-ubuntu.sh | 103 ++++++++++++++++++++++++ 9 files changed, 339 insertions(+), 77 deletions(-) create mode 100644 packaging/Dockerfile.ubuntu create mode 100644 packaging/scripts/repo-helper-fedora.sh create mode 100644 packaging/scripts/repo-helper-ubuntu.sh diff --git a/packaging/Dockerfile.coverity b/packaging/Dockerfile.coverity index 7eed2c3..f8171c9 100755 --- a/packaging/Dockerfile.coverity +++ b/packaging/Dockerfile.coverity @@ -1,12 +1,13 @@ # # Copyright 2018-2020, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build for a Coverity scan. # # Pull base image FROM fedora:latest -MAINTAINER daos-stack +LABEL maintainer="daos-stack "" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 6bad37d..d8d86ac 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,5 +1,6 @@ # # Copyright 2018-2024 Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an RPM # @@ -13,15 +14,16 @@ LABEL maintainer="daos@daos.groups.io" # Use local repo server if present ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/yum.repos.d/ && \ - curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \ - "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - for file in *.repo; do \ - true > $file; \ - done; \ - mv daos_ci-fedora-artifactory.repo{.tmp,}; \ - fi +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to install OS updates basic tools and daos dependencies +# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh # Install basic tools RUN dnf -y install mock make \ @@ -33,13 +35,15 @@ RUN dnf -y install mock make \ ARG UID=1000 # Add build user (to keep rpmbuild happy) -ENV USER build -ENV PASSWD build +ENV USER=build +ENV PASSWD=build # add the user to the mock group so it can run mock RUN if [ $UID != 0 ]; then \ useradd -u $UID -ms /bin/bash $USER; \ echo "$USER:$PASSWD" | chpasswd; \ usermod -a -G mock $USER; \ + mkdir -p /var/cache/mock; \ + chown $USER:root /var/cache/mock; \ fi ARG CB0 diff --git a/packaging/Dockerfile.ubuntu b/packaging/Dockerfile.ubuntu new file mode 100644 index 0000000..1e3813d --- /dev/null +++ b/packaging/Dockerfile.ubuntu @@ -0,0 +1,51 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. +# +# Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO +LABEL org.opencontainers.image.authors="daos@daos.groups.io" +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO + +ARG REPO_FILE_URL +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep chrootbuild happy) +ENV USER=build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index 4d05464..bd5363b 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,54 +1,42 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. # # Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image -FROM ubuntu:20.04 +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO LABEL org.opencontainers.image.authors="daos@daos.groups.io" - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - curl gpg +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/apt/sources.list.d && \ - curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ - "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ - true > ../sources.list && \ - mv daos_ci-ubuntu20.04-artifactory.list.tmp \ - daos_ci-ubuntu20.04-artifactory.list; \ - url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \ - else \ - url="https://downloads.linux.hpe.com/SDR/repo/ilorest/"; \ - fi; \ - cd -; \ - mkdir -p /usr/local/share/keyrings/; \ - curl -f -O "$url"GPG-KEY-hprest; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import GPG-KEY-hprest; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh -# Install basic tools +# Install basic tools - rpmdevtools temporary commented out. RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage \ javahelper locales make patch pbuilder pkg-config \ - python3-dev python3-distro python3-distutils rpm scons wget \ - cmake valgrind rpmdevtools + python3-dev python3-distro python3-distutils rpm scons sudo \ + wget cmake valgrind # rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 # Add build user (to keep chrootbuild happy) -ENV USER build +ENV USER=build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot @@ -56,7 +44,7 @@ RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ fi; \ echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ chmod 0440 /etc/sudoers.d/build; \ visudo -c; \ diff --git a/packaging/Dockerfile.ubuntu.rolling b/packaging/Dockerfile.ubuntu.rolling index 02aca45..ed5d363 100644 --- a/packaging/Dockerfile.ubuntu.rolling +++ b/packaging/Dockerfile.ubuntu.rolling @@ -1,11 +1,12 @@ # # Copyright 2019, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:rolling -Maintainer daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index cc2cc96..5cb2943 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,17 +2,35 @@ set -uex +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" + +# Currently not fully working behind a proxy if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then - echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc + pbuilderrc="./pbuilder_rc.txt" + rm -f "$pbuilderrc" + if [ -n "${HTTPS_PROXY}" ]; then + echo "export http_proxy=\"${HTTPS_PROXY}\"" >> "$pbuilderrc" + else + echo "MIRRORSITE=${ARTIFACTORY_URL}/ubuntu-proxy/ubuntu" > "$pbuilderrc" + fi + #if [ -n "$REPO_FILE_URL" ]; then + # direct="${REPO_FILE_URL##*//}" + # direct="${direct%%/*}" + # echo "no_proxy=\"${direct}\"" >> "$pbuilderrc" + #fi + # shellcheck disable=SC2002 + cat "$pbuilderrc" | sudo tee /root/.pbuilderrc fi # shellcheck disable=SC2086 sudo pbuilder create \ --extrapackages "gnupg ca-certificates" \ - $DISTRO_ID_OPT + $DISTRO_ID_OPT || true # Ignore error status for now. repo_args="" repos_added=() +# currently a bit broken, pbuilder will not accept user provided CAs. for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -32,31 +50,34 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | - sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ - -e 's/signed-by=.*\.gpg/trusted=yes/' | - sed -e ':a; N; $!ba; s/\n/|/g')" -for repo in $JOB_REPOS; do - repo_name=${repo##*://} - repo_name=${repo_name//\//_} - if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then - # don't add duplicates, first found wins - continue - fi - repos_added+=("$repo_name") - repo_args+="|deb ${repo} $VERSION_CODENAME main" -done -# NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. -# After the build is updated to use 22.04, which supports go >= 1.18, it -# should no longer be needed. -repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" -echo "$repo_args" -if [ "$repo_args" = "|" ]; then - repo_args="" -else - #repo_args="--othermirror"${repo_args#|}\"" - repo_args="${repo_args#|}" -fi +# currently broken, builder will not accept internal certs. +# repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | +# sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ +# -e 's/signed-by=.*\.gpg/trusted=yes/' | +# sed -e ':a; N; $!ba; s/\n/|/g')" +#for repo in $JOB_REPOS; do +# repo_name=${repo##*://} +# repo_name=${repo_name//\//_} +# if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then +# # don't add duplicates, first found wins +# continue +# fi +# repos_added+=("$repo_name") +# repo_args+="|deb ${repo} $VERSION_CODENAME main" +#done + +## NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. +## After the build is updated to use 22.04, which supports go >= 1.18, it +## should no longer be needed. +# currently broken - claim is public key not available. +#repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" +#echo "$repo_args" +#if [ "$repo_args" = "|" ]; then +# repo_args="" +#else +# #repo_args="--othermirror"${repo_args#|}\"" +# repo_args="${repo_args#|}" +#fi cd "$DEB_TOP" # shellcheck disable=SC2086 sudo pbuilder update --override-config $DISTRO_ID_OPT ${repo_args:+--othermirror "$repo_args"} diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d6443b7..8d37abc 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,6 +2,11 @@ set -uex +: "${HTTPS_PROXY:=}" +: "${REPO_FILE_URL:=}" +: "${ARCH:=$(arch)}" +: "${REPOSITORY_NAME:=artifactory}" + cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg # Enable mock ccache plugin @@ -11,7 +16,20 @@ config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/cc EOF -if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then +# Optionally add a proxy to mock +if [ -n "$HTTPS_PROXY" ];then + yum_proxy="http://${HTTPS_PROXY##*//}" + echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg +fi + +# No proxy for local mirrors +if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg +fi + +if [[ $CHROOT_NAME == *"epel-8-${ARCH}" ]]; then cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), @@ -21,7 +39,7 @@ EOF fi # Use dnf on CentOS 7 -if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-7-$ARCH" ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi @@ -61,7 +79,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then fi fi curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg - repo_adds+=("--enablerepo *-artifactory") + repo_adds+=("--enablerepo *-${REPOSITORY_NAME}") fi fi @@ -127,7 +145,7 @@ if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ fi # Save the ccache -if [ -d /scratch/ ]; then +if [ -d /scratch/mock ]; then mkdir -p "$bs_dir"/ if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then echo "Failed to save ccache. Plowing onward." diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh new file mode 100644 index 0000000..baa7026 --- /dev/null +++ b/packaging/scripts/repo-helper-fedora.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. + +: "${REPO_FILE_URL:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${FVERSION:=latest}" +: "${REPOSITORY_NAME:=artifactory}" +: "${archive:=}" +if [ "$FVERSION" != "latest" ]; then + archive="-archive" +fi + +# shellcheck disable=SC2120 +disable_repos () { + local repos_dir="$1" + shift + local save_repos + IFS=" " read -r -a save_repos <<< "${*:-} daos_ci-fedora${archive}-${REPOSITORY_NAME}" + if [ -n "$REPO_FILE_URL" ]; then + pushd "$repos_dir" + local repo + for repo in "${save_repos[@]}"; do + mv "$repo".repo{,.tmp} + done + for file in *.repo; do + true > "$file" + done + for repo in "${save_repos[@]}"; do + mv "$repo".repo{.tmp,} + done + popd + fi +} + +# Use local repo server if present +install_curl() { + : +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/etc/pki/ca-trust/source/anchors/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-trust + fi +} + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. + +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + mkdir -p /etc/yum.repos.d + pushd /etc/yum.repos.d/ + curl -k --noproxy '*' -sSf \ + -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ + "{$REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + disable_repos /etc/yum.repos.d/ + popd +fi +dnf -y install dnf-plugins-core +# This does not work in fedora/41 anymore -- needs investigation +# dnf -y config-manager --save --setopt=assumeyes=True +# dnf config-manager --save --setopt=install_weak_deps=False +dnf clean all + +disable_repos /etc/yum.repos.d/ "${save_repos[@]}" diff --git a/packaging/scripts/repo-helper-ubuntu.sh b/packaging/scripts/repo-helper-ubuntu.sh new file mode 100644 index 0000000..20220e9 --- /dev/null +++ b/packaging/scripts/repo-helper-ubuntu.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. +# It will also optionally allow running a /tmp/install script +# for custom packages if present. + +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${REPOSITORY_NAME:=artifactory}" + +disable_repos () { + if [ -e /etc/apt/sources.list.d/ubuntu.sources ];then + mv /etc/apt/sources.list.d/ubuntu.sources \ + etc/apt/sources.list.d/ubuntu.sources.disabled + elif [ -e /etc/apt/sources.list ];then + mv /etc/apt/sources.list \ + etc/apt/sources.list.disabled + fi +} + +# Use local repo server if present +install_curl() { + + if command -v curl; then + echo "found curl!" + return + else + apt-get update + apt-get install curl ca-certificates gpg gpg-agent \ + software-properties-common + fi + + if command -v wget; then + echo "found wget!" + return + fi + # If we don't find one of these, we are basically sunk for using + # a local repository mirror. +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/usr/local/share/ca-certificates/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-certificates + fi +} + +echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt +echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends +if [ -n "$HTTPS_PROXY" ];then + apt_proxy="http://${HTTPS_PROXY##*//}" + echo "Acquire::http::Proxy \"$apt_proxy\";" > \ + /etc/apt/apt.conf.d/local_proxy + if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "Acquire::http::Proxy { $direct DIRECT; };" >> \ + /etc/apt/apt.conf.d/local_proxy + fi +fi + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. +DISTRO_VERSION="${BASE_DISTRO##*:}" +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + # Ubuntu local repo mirror is not working + # curl -k --noproxy '*' -sSf \ + # -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ + # "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" + # disable_repos + # temp hack until we can debug the Ubuntu repos. + REPO_BASE="${REPO_FILE_URL%repo-files/}" + curl -k --noproxy '*' -sSf \ + -o "rpmdevtools_8.10-10_amd64.deb" \ + "${REPO_BASE}daos-stack-deps-ubuntu-${DISTRO_VERSION}-x86_64-stable-local/pool/rpmdevtools_8.10-10_amd64.deb" + apt-get install "./rpmdevtools_8.10-10_amd64.deb" + mkdir -p /usr/local/share/keyrings/ + curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key" + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg +fi + +apt-get update +apt-get upgrade +# add-apt-repository ppa:longsleep/golang-backports +apt-get update +if [ -e /tmp/install.sh ]; then + chmod +x /tmp/install.sh + /tmp/install.sh +fi +apt-get clean all From 9879486d94056dc9c5dd6571548f57e081e35704 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 4 Jul 2025 13:02:35 +0200 Subject: [PATCH 3/4] Centos is no longer supported Signed-off-by: Tomasz Gromadzki --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b9c0d40..3af3240 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,5 +7,5 @@ //@Library(value="pipeline-lib@your_branch") _ /* groovylint-disable-next-line CompileStatic */ -packageBuildingPipelineDAOSTest(['distros' : ['centos7', 'el8', 'el9', 'leap15'], +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'el9', 'leap15'], 'test-tag': 'ior']) From f59a4311f4e48859ccc4161b11a2970c2e9ab641 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 17 Jul 2025 10:00:16 +0200 Subject: [PATCH 4/4] SRE-3205 ci: Update for Fedora 42 - redhat-lsb-core no longer supported by Fedora 42 - Fedora 41 and 42 are not available in archive repo, archive repo is only for EOL releases - new test to verify that packaging works for all supported version of Fedora: latest (default), 41, 42 Signed-off-by: Tomasz Gromadzki --- packaging/Dockerfile.mockbuild | 6 +++--- packaging/scripts/repo-helper-fedora.sh | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index d8d86ac..4d8f243 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -26,9 +26,9 @@ RUN chmod +x /tmp/repo-helper.sh && \ rm -f /tmp/repo-helper.sh # Install basic tools -RUN dnf -y install mock make \ - rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools && \ +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint git \ + python-srpm-macros rpmdevtools && \ dnf -y clean all # use same UID as host and default value of 1000 if not specified diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh index baa7026..6c8375e 100644 --- a/packaging/scripts/repo-helper-fedora.sh +++ b/packaging/scripts/repo-helper-fedora.sh @@ -10,7 +10,11 @@ set -uex : "${REPOSITORY_NAME:=artifactory}" : "${archive:=}" if [ "$FVERSION" != "latest" ]; then - archive="-archive" + if [ "$FVERSION" != "42" ]; then + if [ "$FVERSION" != "41" ]; then + archive="-archive" + fi + fi fi # shellcheck disable=SC2120 @@ -62,7 +66,7 @@ if [ -n "$REPO_FILE_URL" ]; then pushd /etc/yum.repos.d/ curl -k --noproxy '*' -sSf \ -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ - "{$REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + "${REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" disable_repos /etc/yum.repos.d/ popd fi