diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..49b1a2c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Have Release Engineering added as a reviewer to any packaging PR +* @daos-stack/build-and-release-watchers diff --git a/Jenkinsfile b/Jenkinsfile index ea0bc14..96476d8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!/usr/bin/groovy -/* Copyright (C) 2019 Intel Corporation +/* Copyright (C) 2019-2023 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,7 +38,9 @@ */ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -//@Library(value="pipeline-lib@your_branch") _ +@Library(value="pipeline-lib@bmurrell/el9") _ -// no debian/ support yet -packageBuildingPipeline(['distros' : ['centos7', 'centos8', 'leap15']]) +// hdf5 not building for Ubuntu yet +//packageBuildingPipelineDAOSTest(['distros' : ['centos7', 'el8', 'el9', 'leap15', 'ubuntu20.04'], +packageBuildingPipelineDAOSTest(['distros' : ['centos7', 'el8', 'el9', 'leap15'], + 'test-tag': 'ior']) diff --git a/daos-configure.patch b/daos-configure.patch new file mode 100644 index 0000000..01b305d --- /dev/null +++ b/daos-configure.patch @@ -0,0 +1,16 @@ +--- a/configure.ac ++++ b/configure.ac +@@ -307,7 +307,12 @@ + [], [with_daos=no]) + AS_IF([test "x$with_daos" != xno], [ + DAOS="yes" +- LDFLAGS="$LDFLAGS -L$with_daos/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$with_daos/lib64" ++ if test -d $with_daos/lib/x86_64-linux-gnu/; then ++ LIB_DIR=$with_daos/lib/x86_64-linux-gnu ++ else ++ LIB_DIR=$with_daos/lib64 ++ fi ++ LDFLAGS="$LDFLAGS -L$LIB_DIR -Wl,--enable-new-dtags -Wl,-rpath=$LIB_DIR -lmercury" + CPPFLAGS="$CPPFLAGS -I$with_daos/include" + AC_CHECK_HEADERS(gurt/common.h,, [unset DAOS]) + AC_CHECK_HEADERS(daos.h,, [unset DAOS]) diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..3a4d38d --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +ior (3.3.0-19) focal; urgency=medium + + * Initial packaging + + -- Brian J. Murrell Wed, 25 Jan 2023 14:06:11 -0500 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..eec15ea --- /dev/null +++ b/debian/control @@ -0,0 +1,28 @@ +Source: ior +Maintainer: DAOS +Uploaders: Brian J. Murrell +Section: devel +Priority: optional +Build-Depends: debhelper-compat (= 12), + pkg-config, + chrpath, + libhwloc-dev, + libevent-dev, + unzip, + autoconf, automake, + libdaos-dev, + libmercury-dev, + libmpich-dev, + libhdf5-mpich-dev, + libfabric-dev +Standards-Version: 4.4.1.0 +Vcs-Browser: https://github.com/hpc/ior +Vcs-Git: https://github.com/hpc/ior.git +Homepage: https://github.com/hpc/ior.git + +Package: ior +Architecture: any +Multi-Arch: foreign +Depends: ${misc:Depends}, + ${shlibs:Depends} +Description:IOR-HPC diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..11e3e24 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,265 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: IOR +Upstream-Contact: https://github.com/hpc/ior +Source: https://github.com/hpc/ior +Copyright: 2002 University of Chicago + +Files: * +Copyright: 2003 The Regents of the University of California. +License: GPL +Copyright (c) 2003, The Regents of the University of California. +Produced at the Lawrence Livermore National Laboratory. +Written by William Loewe , Tyce McLarty , +and Christopher Morrone . +UCRL-CODE-2003-016. +All rights reserved. + +This file is part of IOR. + +Please also read Our Notice and GNU General Public License. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License (as published by the Free Software +Foundation) version 2, dated June 1991. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the terms and conditions of the GNU General Public +License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + + +OUR NOTICE AND TERMS AND CONDITIONS OF THE GNU GENERAL PUBLIC LICENSE + +Our Preamble Notice + +A. This notice is required to be provided under our contract with the U.S. +Department of Energy (DOE). This work was produced at the University of +California, Lawrence Livermore National Laboratory under Contract No. +W-7405-ENG-48 with the DOE. + +B. Neither the United States Government nor the University of California nor +any of their employees, makes any warranty, express or implied, or assumes any +liability or responsibility for the accuracy, completeness, or usefulness of +any information, apparatus, product, or process disclosed, or represents that +its use would not infringe privately-owned rights. + +C. Also, reference herein to any specific commercial products, process, or +services by trade name, trademark, manufacturer or otherwise does not +necessarily constitute or imply its endorsement, recommendation, or favoring +by the United States Government or the University of California. The views and +opinions of authors expressed herein do not necessarily state or reflect those +of the United States Government or the University of California, and shall not +be used for advertising or product endorsement purposes. + +The precise terms and conditions for copying, distribution and modification +follows. + +GNU Terms and Conditions for Copying, Distribution, and Modification + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program," below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included +without limitation in the term "modification".) Each licensee is addressed as +"you." + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is +not restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, +thus forming a work based on the Program, and copy and distribute such +modifications or work under the terms of Section 1 above, provided that you +also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices stating + that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole + or in part contains or is derived from the Program or any part thereof, + to be licensed as a whole at no charge to all third parties under the terms + of this License. + + c) If the modified program normally reads commands interactively when run, + you must cause it, when started running for such interactive use in the + most ordinary way, to print or display an announcement including an + appropriate copyright notice and a notice that there is no warranty (or + else, saying that you provide a warranty) and that users may redistribute + the program under these conditions, and telling the user how to view a copy + of this License. (Exception: if the Program itself is interactive but does + not normally print such an announcement, your work based on the Program is + not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +work. But when you distribute the same section as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source + code, which must be distributed under the terms of Sections 1 and 2 above + on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to + give any third party, for a charge no more than your cost of physically + performing source distribution, a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed only + for noncommercial distribution and only if you received the program in + object code or executable form with such an offer, in accord with + Subsection b above.) + +The source code for a work means the preferred form the work for making +modifications to it. For an executable work, complete source code means all +the source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code +distributed need not include anything that is normally distributed (in either +source or binary form) with the major components (compiler, kernel, and so on) +of the operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. + +It is not the purpose to this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In +such case, this License incorporates the limitation as if written in the body +of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any later +version," you have the option of following the terms and conditions either of +that version of any later version published by the Free Software Foundation. +If the Program does not specify a version number of this License, you may +choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision to grant permission will be guided by the two goals of preserving +the free status of all derivatives of our free software and or promoting the +sharing and reuse of software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS diff --git a/debian/patches/hdf5-pkg-config.patch b/debian/patches/hdf5-pkg-config.patch new file mode 100644 index 0000000..de7b418 --- /dev/null +++ b/debian/patches/hdf5-pkg-config.patch @@ -0,0 +1,50 @@ +--- a/configure.ac 2023-01-27 15:43:27.815008155 -0500 ++++ b/configure.ac 2023-01-27 15:55:15.980795957 -0500 +@@ -166,6 +166,8 @@ + ]) + + # HDF5 support ++PKG_CHECK_MODULES([HDF5], [hdf5]) ++ + AC_ARG_WITH([hdf5], + [AS_HELP_STRING([--with-hdf5], + [support IO with HDF5 backend @<:@default=no@:>@])], +@@ -173,10 +175,18 @@ + [with_hdf5=no]) + AM_CONDITIONAL([USE_HDF5_AIORI], [test x$with_hdf5 = xyes]) + AM_COND_IF([USE_HDF5_AIORI],[ ++ OLD_CFLAGS="$CFLAGS" ++ OLD_LIBS="$LIBS" ++ CFLAGS="$CFLAGS $HDf5_CFLAGS" ++ LIBS="$LDLIBS $HDF5_LIBS" + AC_DEFINE([USE_HDF5_AIORI], [], [Build HDF5 backend AIORI]) + AC_SEARCH_LIBS([H5Pset_all_coll_metadata_ops], [hdf5]) + AC_CHECK_FUNCS([H5Pset_all_coll_metadata_ops]) ++ LIBS="$OLD_LIBS" ++ CFLAGS="$OLD_CFLAGS" + ]) ++AC_SUBST(HDF5_CFLAGS) ++AC_SUBST(HDF5_LIBS) + + + +--- a/src/Makefile.am ++++ b/src/Makefile.am +@@ -51,7 +51,8 @@ + + if USE_HDF5_AIORI + extraSOURCES += aiori-HDF5.c ++extraCPPFLAGS += $(HDF5_CFLAGS) +-extraLDADD += -lhdf5 -lz ++extraLDADD += -lhdf5 -lz $(HDF5_LIBS) + endif + + if USE_IME_AIORI +--- a/src/test/Makefile.am ++++ b/src/test/Makefile.am +@@ -1,4 +1,4 @@ + AM_LDFLAGS = $(extraLDFLAGS) +-LDADD = ../libaiori.a $(extraLDADD) ++LDADD = ../libaiori.a $(extraLDADD) $(HDF5_LIBS) + + # Add test here diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 0000000..ffe4623 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +hdf5-pkg-config.patch diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..0769d3d --- /dev/null +++ b/debian/rules @@ -0,0 +1,15 @@ +#! /usr/bin/make -f + +export DH_VERBOSE=1 + +DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) + +%: + dh $@ + +override_dh_auto_configure: + dh_auto_configure -- --with-mpiio --with-daos=/usr --with-hdf5 --datadir=/usr/share/doc/ior/ + +override_dh_auto_install-arch: + dh_auto_install -a + chrpath -d debian/ior/usr/bin/* diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/ior.rpmlintrc b/ior.rpmlintrc new file mode 100644 index 0000000..323f97d --- /dev/null +++ b/ior.rpmlintrc @@ -0,0 +1,7 @@ +# This line is mandatory to access the configuration functions +from Config import * + +addFilter('ior.x86_64: E: invalid-locale-man-dir /usr/share/man/mpich-x86_64/man1/mdtest.1.gz') +addFilter('ior.x86_64: E: binary-or-shlib-defines-rpath /usr/lib64/mpich/bin/ior \(RUNPATH: check/lib64:/usr/lib64:/usr/lib64/mpich/lib\)') +addFilter('ior.x86_64: E: binary-or-shlib-defines-rpath /usr/lib64/mpich/bin/md-workbench \(RUNPATH: check/lib64:/usr/lib64:/usr/lib64/mpich/lib\)') +addFilter('ior.x86_64: E: binary-or-shlib-defines-rpath /usr/lib64/mpich/bin/mdtest \(RUNPATH: check/lib64:/usr/lib64:/usr/lib64/mpich/lib\)') diff --git a/ior.spec b/ior.spec index 05da7e0..26679ad 100644 --- a/ior.spec +++ b/ior.spec @@ -8,7 +8,7 @@ Name: ior Version: 3.3.0 -Release: 19%{?commit:.g%{shortcommit}}%{?dist} +Release: 20%{?commit:.g%{shortcommit}}%{?dist} Summary: IOR-HPC @@ -18,6 +18,7 @@ Source0: https://github.com/hpc/%{name}/releases/download/%{version}/%{name}- %if "%{?commit}" != "" Patch1: %{version}..%{commit}.patch %endif +Patch3: daos-configure.patch BuildRequires: mpich-devel BuildRequires: hwloc-devel @@ -26,6 +27,7 @@ BuildRequires: unzip BuildRequires: autoconf, automake BuildRequires: daos-devel BuildRequires: hdf5-mpich-devel%{?_isa} +BuildRequires: mercury-devel %if (0%{?suse_version} >= 1500) BuildRequires: lua-lmod %else @@ -53,14 +55,18 @@ autoreconf %endif %build +export CC=mpicc +export CXX=mpicxx +export FC=mpif90 +export F77=mpif77 +export CFLAGS="$RPM_OPT_FLAGS -fno-strict-aliasing -fPIC" +export CXXFLAGS="$RPM_OPT_FLAGS -fno-strict-aliasing -fPIC" if [ ! -f configure ]; then # probably a git tarball ./bootstrap fi %module_load mpich -echo $PATH -set -x %if (0%{?suse_version} >= 1) %configure --with-mpiio --with-daos=/usr --with-hdf5 %else @@ -71,11 +77,11 @@ set -x %install %module_load mpich %make_install +rm -f %{buildroot}/$MPI_LIB/libaiori.a %if (0%{?suse_version} < 1) cat <> files.mpich $MPI_BIN/* -$MPI_LIB/* $MPI_MAN/man1/* %{_datadir}/doc/ior-mpich/* EOF @@ -94,6 +100,11 @@ EOF %changelog +* Fri May 26 2023 Brian J. Murrell - 3.3.0-20 +- Add BR: mercury-devel +- Remove static library +- Set compiler environment variables + * Fri Mar 18 2022 Brian J. Murrell - 3.3.0-19 - Update to d3574d536643475269d37211e283b49ebd6732d7 diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 55510f7..cdfb7f6 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -1,12 +1,12 @@ # -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image FROM centos:7 -MAINTAINER daos-stack +LABEL maintainer="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 89d494f..c804819 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,20 +1,30 @@ # -# Copyright 2018-2022, Intel Corporation +# Copyright 2018-2022 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -FROM fedora:35 -LABEL maintainer="daos@daos.groups.io>" +FROM fedora:latest +LABEL maintainer="daos@daos.groups.io" -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 +# Use local repo server if present +ARG REPO_FILE_URL +RUN if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -f -o daos_ci-fedora-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ + rm -f *.repo && \ + mv daos_ci-fedora-artifactory.repo{.tmp,}; \ + fi # Install basic tools -RUN dnf -y install mock make \ - rpm-build curl createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools mock-core-configs\ \<\ 37.1 +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint redhat-lsb-core git \ + python-srpm-macros rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 # Add build user (to keep rpmbuild happy) ENV USER build @@ -24,10 +34,24 @@ RUN echo "$USER:$PASSWD" | chpasswd # add the user to the mock group so it can run mock RUN usermod -a -G mock $USER -# mock in Docker needs to use the old-chroot option -RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ - echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg +ARG CB0 +RUN dnf -y upgrade && \ + dnf clean all + +# Monkey-patch rpmlint until a new release is made with +# https://github.com/rpm-software-management/rpmlint/pull/795 in it +# But make sure to patch after dnf upgrade so that an upgraded rpmlint +# RPM doesn't wipe out our patch +COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ + if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ + if ! patch -p1; then \ + exit 1; \ + fi; \ + rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ + fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch +# show the release that was built ARG CACHEBUST -RUN dnf -y upgrade --exclude mock-core-configs && \ - dnf clean all +RUN cat /etc/os-release diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index f956145..ec76bfd 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -7,33 +7,60 @@ FROM ubuntu:20.04 LABEL org.opencontainers.image.authors="daos@daos.groups.io" -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 -ARG REPO_URL="" -ARG REPO_UBUNTU_20_04="" +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl gpg + +ARG REPO_FILE_URL +RUN if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/apt/sources.list.d && \ + curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ + "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ + true > ../sources.list && \ + mv daos_ci-ubuntu20.04-artifactory.list.tmp \ + daos_ci-ubuntu20.04-artifactory.list; \ + fi; \ + cd -; \ + curl -f -O "$REPO_FILE_URL"esad_repo.key; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key; \ + mkdir -p /usr/local/share/keyrings/; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg; \ + rm ./temp-keyring.gpg; \ + url_prefix=https://downloads.linux.hpe.com/SDR/; \ + for url in hpPublicKey2048.pub \ + hpPublicKey2048_key1.pub \ + hpePublicKey2048_key1.pub; do \ + curl -f -O "$url_prefix$url"; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import "$(basename $url)"; \ + done; \ + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ + rm ./temp-keyring.gpg # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ - dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ - make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools -# rpmdevtools -RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - rpmdevtools +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 # Add build user (to keep chrootbuild happy) ENV USER build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot -RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ - echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ - fi; \ - echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ - chmod 0440 /etc/sudoers.d/build; \ - visudo -c; \ +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ sudo -l -U build diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 44006e0..6a7f88b 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -26,7 +26,7 @@ SPECTOOL := spectool # DISTRO_ID (i.e. el7) # DISTRO_BASE (i.e. EL_7) # from the CHROOT_NAME -ifeq ($(CHROOT_NAME),epel-7-x86_64) +ifeq ($(patsubst %epel-7-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) VERSION_ID := 7 DISTRO_ID := el7 @@ -35,15 +35,30 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 7 SED_EXPR := 1s/$(DIST)//p endif -ifeq ($(CHROOT_NAME),epel-8-x86_64) +ifeq ($(patsubst %epel-8-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) VERSION_ID := 8 DISTRO_ID := el8 DISTRO_BASE := EL_8 +ifneq ($(DISTRO_VERSION_EL8),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL8) +endif DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 8 SED_EXPR := 1s/$(DIST)//p endif +ifeq ($(patsubst %epel-9-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 9 +DISTRO_ID := el9 +DISTRO_BASE := EL_9 +ifneq ($(DISTRO_VERSION_EL9),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL9) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 9 +SED_EXPR := 1s/$(DIST)//p +endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 DISTRO_ID := sl15.2 @@ -57,11 +72,31 @@ VERSION_ID := 15.3 DISTRO_ID := sl15.3 DISTRO_BASE := LEAP_15 DISTRO_VERSION ?= $(VERSION_ID) -ORIG_TARGET_VER := 15.2 +ORIG_TARGET_VER := 15.3 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.4-x86_64) +VERSION_ID := 15.4 +DISTRO_ID := sl15.4 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif endif ifeq ($(ID),centos) +ID = el +endif +ifeq ($(ID),rocky) +ID = el +endif +ifeq ($(ID),almalinux) +ID = el +endif +ifeq ($(ID),rhel) +ID = el +endif +ifeq ($(ID),el) DISTRO_ID := el$(VERSION_ID) DISTRO_BASE := $(basename EL_$(VERSION_ID)) DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index b59782e..910d76b 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -6,11 +6,11 @@ # force bash (looking at you Ubuntu) SHELL=/bin/bash -# Put site overrides (i.e. REPOSITORY_URL, DAOS_STACK_*_LOCAL_REPO) in here +# Put site overrides (i.e. DAOS_STACK_*_LOCAL_REPO) in here -include Makefile.local # default to Leap 15 distro for chrootbuild -CHROOT_NAME ?= opensuse-leap-15.2-x86_64 +CHROOT_NAME ?= opensuse-leap-15.3-x86_64 include packaging/Makefile_distro_vars.mk ifeq ($(DEB_NAME),) @@ -19,7 +19,9 @@ endif CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST)) -TOPDIR ?= $(CURDIR) +# this Makefile should always be executed from it's own dir +TOPDIR ?= $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) + BUILD_PREFIX ?= . DOT := . @@ -28,11 +30,6 @@ RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) # some defaults the caller can override PACKAGING_CHECK_DIR ?= ../packaging LOCAL_REPOS ?= true -ifeq ($(ID_LIKE),debian) -DAOS_REPO_TYPE ?= LOCAL -else -DAOS_REPO_TYPE ?= STABLE -endif TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project @@ -42,7 +39,9 @@ PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') +EL_9_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el9: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') +REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') ifneq ($(PKG_GIT_COMMIT),) ifeq ($(GITHUB_PROJECT),) @@ -68,7 +67,7 @@ DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p)) SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) @@ -86,7 +85,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="centos8" \ + el*) distro="$(DISTRO_ID)" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -100,30 +99,35 @@ define distro_map endef define install_repos - IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ - for baseurl in "$${BASES[@]}"; do \ - baseurl="$${baseurl# *}"; \ - $(call install_repo,$$baseurl); \ - done - for repo in $($(DISTRO_BASE)_PR_REPOS) \ - $(PR_REPOS) $(1); do \ - branch="master"; \ - build_number="lastSuccessfulBuild"; \ - if [[ $$repo = *@* ]]; then \ - branch="$${repo#*@}"; \ - repo="$${repo%@*}"; \ - if [[ $$branch = *:* ]]; then \ - build_number="$${branch#*:}"; \ - branch="$${branch%:*}"; \ - fi; \ - fi; \ - $(call distro_map) \ + if [ "$(ID_LIKE)" = "debian" ]; then \ + IFS='|' read -ra BASES <<< "$($(DISTRO_BASE)_LOCAL_REPOS)"; \ + for baseurl in "$${BASES[@]}"; do \ + baseurl="$${baseurl# *}"; \ + $(call install_repo,$$baseurl) \ + done; \ + fi + for repo in $($(DISTRO_BASE)_PR_REPOS) \ + $(PR_REPOS) $(1); do \ + branch="master"; \ + build_number="lastSuccessfulBuild"; \ + if [[ $$repo = *@* ]]; then \ + branch="$${repo#*@}"; \ + repo="$${repo%@*}"; \ + if [[ $$branch = *:* ]]; then \ + build_number="$${branch#*:}"; \ + branch="$${branch%:*}"; \ + fi; \ + fi; \ + $(call distro_map) \ baseurl=$${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$$repo/job/$$branch/; \ - baseurl+=$$build_number/artifact/artifacts/$$distro/; \ - $(call install_repo,$$baseurl); \ + baseurl+=$$build_number/artifact/artifacts/$$distro/; \ + $(call install_repo,$$baseurl) \ done endef +debug: + echo $(EL_7_PR_REPOS) + all: $(TARGETS) %/: @@ -154,28 +158,8 @@ ifeq ($(DL_NAME),) DL_NAME = $(NAME) endif -$(DL_NAME)$(DL_VERSION).linux-amd64.tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) - rm -f ./$(DL_NAME)*.tar{gz,bz*,xz} - $(SPECTOOL) -g $(SPEC) - -$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: $(SPEC) $(CALLING_MAKEFILE) - rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.asc - $(SPECTOOL) -g $(SPEC) - -$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT).sig: $(SPEC) $(CALLING_MAKEFILE) - rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz}.sig - $(SPECTOOL) -g $(SPEC) - -$(DL_NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) - rm -f ./$(DL_NAME)-*.tar.{gz,bz*,xz} - $(SPECTOOL) -g $(SPEC) - -v$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) - rm -f ./v*.tar.{gz,bz*,xz} - $(SPECTOOL) -g $(SPEC) - -$(DL_VERSION).tar.$(SRC_EXT): $(SPEC) $(CALLING_MAKEFILE) - rm -f ./*.tar.{gz,bz*,xz} +$(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE) + # TODO: need to clean up old ones $(SPECTOOL) -g $(SPEC) $(DEB_TOP)/%: % | $(DEB_TOP)/ @@ -291,6 +275,9 @@ $(DEB_TOP)/$(DEB_DSC): $(CALLING_MAKEFILE) $(DEB_BUILD).tar.$(SRC_EXT) \ cd $(DEB_BUILD); dpkg-buildpackage -S --no-sign --no-check-builddeps $(SRPM): $(SPEC) $(SOURCES) + if [ -f bz-1955184_find-requires ]; then \ + chmod 755 bz-1955184_find-requires; \ + fi rpmbuild -bs $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC) srpm: $(SRPM) @@ -299,6 +286,10 @@ $(RPMS): $(SRPM) $(CALLING_MAKEFILE) rpms: $(RPMS) +repo: rpms + rm -rf _topdir/RPMS/repodata/ + createrepo _topdir/RPMS/ + $(DEBS): $(CALLING_MAKEFILE) debs: $(DEBS) @@ -333,66 +324,23 @@ patch: echo "PKG_GIT_COMMIT is not defined" endif -# *_LOCAL_* repos are locally built packages. -# *_GROUP_* repos are a local mirror of a group of upstream repos. -# *_GROUP_* repos may not supply a repomd.xml.key. -ifeq ($(LOCAL_REPOS),true) - ifneq ($(REPOSITORY_URL),) - # group repos are not working in Nexus so we hack in the group members directly below - #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO),) - #DISTRO_REPOS = $(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO) - #$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)/ - #endif - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) - ifeq ($(ID_LIKE),debian) - # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists - # of values with spaces as environment variables - $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] - else - $(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO) - DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not - endif # ifeq ($(ID_LIKE),debian) - ifeq ($(DISTRO_BASE), EL_8) - # hack to use 8.3 non-group repos on EL_8 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-8.3-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-8.3-extras-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-8-x86_64-proxy) - else ifeq ($(DISTRO_BASE), EL_7) - # hack to use 7.9 non-group repos on EL_7 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-7.9-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-extras-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-updates-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-7-x86_64-proxy) - else ifeq ($(DISTRO_BASE), LEAP_15) - # hack to use 15 non-group repos on LEAP_15 - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/opensuse-15.2-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-oss-x86_64-provo-mirror-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-sle-update-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-backports-update-proxy) - else - # debian - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO) - endif # ifeq ($(DISTRO_BASE), *) - endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO) - endif - # group repos are not working in Nexus so we hack in the group members directly above - ifneq ($(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO) - endif - ifneq ($(ID_LIKE),debian) - ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) - $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_INTEL_ONEAPI_REPO) - endif # ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) - endif # ifneq ($(ID_LIKE),debian) - endif # ifneq ($(REPOSITORY_URL),) -endif # ifeq ($(LOCAL_REPOS),true) ifeq ($(ID_LIKE),debian) chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(call distro_map) \ DISTRO="$$distro" \ PR_REPOS="$(PR_REPOS)" \ + REPO_FILES_PR="$(REPO_FILES_PR)" \ DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ JENKINS_URL="$${JENKINS_URL}" \ JOB_REPOS="$(JOB_REPOS)" \ + REPO_FILE_URL="$(REPO_FILE_URL)" \ DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ VERSION_CODENAME="$(VERSION_CODENAME)" \ DEB_TOP="$(DEB_TOP)" \ DEB_DSC="$(DEB_DSC)" \ DISTRO_ID_OPT="$(DISTRO_ID_OPT)" \ + LOCAL_REPOS='$(LOCAL_REPOS)' \ + ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ packaging/debian_chrootbuild else chrootbuild: $(SRPM) $(CALLING_MAKEFILE) @@ -400,26 +348,59 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) DISTRO="$$distro" \ CHROOT_NAME="$(CHROOT_NAME)" \ PR_REPOS="$(PR_REPOS)" \ + REPO_FILES_PR="$(REPO_FILES_PR)" \ DISTRO_BASE_PR_REPOS="$($(DISTRO_BASE)_PR_REPOS)" \ JENKINS_URL="$${JENKINS_URL}" \ JOB_REPOS="$(JOB_REPOS)" \ - DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ + REPO_FILE_URL="$(REPO_FILE_URL)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ - DISTRO_REPOS='$(DISTRO_REPOS)' \ + LOCAL_REPOS='$(LOCAL_REPOS)' \ + ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ + DISTRO_VERSION="$(DISTRO_VERSION)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif +podman_chrootbuild: + if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ + -f packaging/Dockerfile.mockbuild .; then \ + echo "Container build failed"; \ + exit 1; \ + fi + rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log + if ! podman run --rm --privileged -w $(TOPDIR) -v=$(TOPDIR)/..:$(TOPDIR)/.. \ + -it $(subst +,-,$(CHROOT_NAME))-chrootbuild \ + bash -c 'if ! DISTRO_REPOS=false \ + REPO_FILE_URL=$(REPO_FILE_URL) \ + make REPO_FILES_PR=$(REPO_FILES_PR) \ + MOCK_OPTIONS=$(MOCK_OPTIONS) \ + CHROOT_NAME=$(CHROOT_NAME) -C $(CURDIR) chrootbuild; then \ + cat /var/lib/mock/$(CHROOT_NAME)/{result/{root,build},root/builddir/build/BUILD/*/config}.log; \ + exit 1; \ + fi; \ + rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ + grep -v -e debuginfo -e debugsource -e src.rpm)' + docker_chrootbuild: - $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ - -f packaging/Dockerfile.mockbuild . - $(DOCKER) run --privileged=true -w $(TOPDIR) -v=$(TOPDIR):$(TOPDIR) \ - -it chrootbuild bash -c "make -C $(CURDIR) \ - CHROOT_NAME=$(CHROOT_NAME) chrootbuild" + if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ + --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + -f packaging/Dockerfile.mockbuild .; then \ + echo "Container build failed"; \ + exit 1; \ + fi + rm -f /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log + if ! $(DOCKER) run --user=$$(id -u) --privileged=true -w $(TOPDIR) \ + -v=$(TOPDIR):$(TOPDIR) \ + -it chrootbuild bash -c "make -C $(CURDIR) \ + CHROOT_NAME=$(CHROOT_NAME) chrootbuild"; then \ + cat /var/lib/mock/$(CHROOT_NAME)/result/{root,build}.log; \ + exit 1; \ + fi rpmlint: $(SPEC) - rpmlint $< + rpmlint --ignore-unused-rpmlintrc $< packaging_check: if grep -e --repo $(CALLING_MAKEFILE); then \ @@ -439,6 +420,7 @@ packaging_check: --exclude install \ --exclude packaging \ --exclude utils \ + --exclude .vscode \ -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \ exit 1; \ fi @@ -456,6 +438,15 @@ test: $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) dnf -y install $(TEST_PACKAGES) +show_NAME: + @echo '$(NAME)' + +show_DISTRO_ID: + @echo '$(DISTRO_ID)' + +show_distro_map: + @$(call distro_map) echo "$$distro" + show_spec: @echo '$(SPEC)' diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index 5ea4ead..03b232a 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,14 +2,13 @@ set -uex -# shellcheck disable=SC2153 -IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" +if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then + echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc +fi # shellcheck disable=SC2086 -sudo pbuilder create \ - --extrapackages "gnupg ca-certificates" \ - --othermirror \ - "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME universe|deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ $VERSION_CODENAME-updates main universe" \ +sudo pbuilder create \ + --extrapackages "gnupg ca-certificates" \ $DISTRO_ID_OPT repo_args="" @@ -26,9 +25,18 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do fi repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do - repo_args="$repo_args|deb ${repo} $VERSION_CODENAME main" + +repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | + sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ + -e 's/signed-by=.*\.gpg/trusted=yes/' | + sed -e ':a; N; $!ba; s/\n/|/g')" +for repo in $JOB_REPOS; do + repo_args+="|deb ${repo} $VERSION_CODENAME main" done +# NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. +# After the build is updated to use 22.04, which supports go >= 1.18, it +# should no longer be needed. +repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" echo "$repo_args" if [ "$repo_args" = "|" ]; then repo_args="" diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index 8c1008e..bfb49ed 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,17 +2,7 @@ set -uex -# shellcheck disable=SC2153 -IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" -repo_adds=() -repo_dels=() - -if [ -n "${REPOSITORY_URL:-}" ] && [ -n "$DISTRO_REPOS" ]; then - repo_dels+=("--disablerepo=\*") -fi - -: "${WORKSPACE:=$PWD}" -mock_config_dir="$WORKSPACE/mock" +mock_config_dir="${WORKSPACE:-$PWD}/mock" original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" cfg_file="$mock_config_dir/${CHROOT_NAME}.cfg" mkdir -p "$mock_config_dir" @@ -21,12 +11,66 @@ ln -sf /etc/mock/logging.ini "$mock_config_dir/" cp "$original_cfg_file" "$cfg_file" -if [ "${CHROOT_NAME}" == "epel-8-x86_64" ]; then - echo -e "config_opts['module_enable'] = ['javapackages-tools:201801']" \ - >> "$cfg_file" +if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then + cat <> "$cfg_file" +config_opts['module_setup_commands'] = [ + ('enable', 'javapackages-tools:201801'), + ('disable', 'go-toolset') +] +EOF +elif [[ $CHROOT_NAME == *epel-9-x86_64 ]]; then + # DO NOT LAND + # waiting for an answer on https://github.com/rpm-software-management/mock/discussions/1078 + ln -sf /etc/mock/almalinux-9-x86_64.cfg "$mock_config_dir" + ln -sf /etc/mock/rocky-9-x86_64.cfg "$mock_config_dir" +fi + +# Use dnf on CentOS 7 +if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then + MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi +# Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available +cat <> "$cfg_file" +config_opts['dnf.conf'] += """ +[main] +best=0 +""" +EOF + +# shellcheck disable=SC2153 +repo_adds=() +repo_dels=() + echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" + +if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then + repo_dels+=("--disablerepo=\*") + + if [ -n "${REPO_FILE_URL:-}" ]; then + if [ -n "$REPO_FILES_PR" ]; then + if [[ $REPO_FILES_PR = PR-* ]]; then + build_number="lastSuccessfulBuild" + branch="$REPO_FILES_PR" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/" + else + if [ ! -d "$REPO_FILES_PR" ]; then + echo "Could not find your specified directory \"$REPO_FILES_PR\" to read repo-files from" + exit 1 + fi + REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" + fi + fi + curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-mock-artifactory.repo >> "$cfg_file" + repo_adds+=("--enablerepo *-artifactory") + fi +fi + +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -38,25 +82,38 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi - repo_adds+=("--enablerepo $repo:$branch:$build_number") - echo -e "[$repo:$branch:$build_number]\n\ -name=$repo:$branch:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") + echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ +name=$repo:${branch//[@\/]/_}:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done -for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do +for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_adds+=("--enablerepo $repo_name") - echo -e "[${repo_name//@/_}]\n\ + echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ +baseurl=${repo//\//%2F}\n\ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file" +if [ -n "$DISTRO_VERSION" ]; then + releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") +fi # shellcheck disable=SC2086 -eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ - ${repo_dels[*]} ${repo_adds[*]} \ - $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ + ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ + "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" diff --git a/packaging/rpmlint--ignore-unused-rpmlintrc.patch b/packaging/rpmlint--ignore-unused-rpmlintrc.patch new file mode 100644 index 0000000..f24075f --- /dev/null +++ b/packaging/rpmlint--ignore-unused-rpmlintrc.patch @@ -0,0 +1,41 @@ +From 060fbdd03d03b0b9835b861a72f9aeb25347277d Mon Sep 17 00:00:00 2001 +From: Martin Liska +Date: Tue, 25 Jan 2022 14:29:05 +0100 +Subject: [PATCH] Add new option --ignore-unused-rpmlintrc. + +The option skips reporting of unused-rpmlintrc-filter errors. + +Fixes: #794. +--- + rpmlint/cli.py | 2 ++ + rpmlint/lint.py | 3 ++- + test/test_lint.py | 3 ++- + 3 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/rpmlint/cli.py b/rpmlint/cli.py +index 81d5b92b..8a7bc09e 100644 +--- a/rpmlint/cli.py ++++ b/rpmlint/cli.py +@@ -82,6 +82,8 @@ def process_lint_args(argv): + parser.add_argument('-i', '--installed', nargs='+', default='', help='installed packages to be validated by rpmlint') + parser.add_argument('-t', '--time-report', action='store_true', help='print time report for run checks') + parser.add_argument('-T', '--profile', action='store_true', help='print cProfile report') ++ parser.add_argument('--ignore-unused-rpmlintrc', action='store_true', ++ help='Do not report "unused-rpmlintrc-filter" errors') + lint_modes_parser = parser.add_mutually_exclusive_group() + lint_modes_parser.add_argument('-s', '--strict', action='store_true', help='treat all messages as errors') + lint_modes_parser.add_argument('-P', '--permissive', action='store_true', help='treat individual errors as non-fatal') +diff --git a/rpmlint/lint.py b/rpmlint/lint.py +index a98d2c04..1d5e0050 100644 +--- a/rpmlint/lint.py ++++ b/rpmlint/lint.py +@@ -277,7 +277,8 @@ + + # validate used filters in rpmlintrc + if is_last: +- self.output.validate_filters(pkg) ++ if not self.options['ignore_unused_rpmlintrc']: ++ self.output.validate_filters(pkg) + + if spec_checks: + self.specfiles_checked += 1