diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 586458d5..ce05ed4a 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -192,3 +192,34 @@ jobs: with: name: ${{ matrix.triplet }}-Debug path: ${{ env.INSTALL_DIR }}-Debug + + package-macos: + name: Build macOS libraries + runs-on: macos-latest + timeout-minutes: 500 + + strategy: + fail-fast: false + matrix: + arch: [x86_64, arm64] + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Install dependencies + run: | + export ARCH=${{ matrix.arch }} + ./pkg/mac/build-static-library.sh + + - name: Zip artifact + run: | + cd ./pkg/mac/.install + zip -r macos-${{matrix.arch}}.zip ./include/pulsar/* ./lib/* + cp macos-${matrix.arch}.zip ../../../ + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: macos-${matrix.arch}.zip + path: macos-${matrix.arch}.zip diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 03d417fa..904c8d98 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -29,284 +29,23 @@ concurrency: jobs: - wireshark-dissector-build: - name: Build the Wireshark dissector - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - matrix: - os: [ubuntu-20.04, macos-12] - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Install deps (Ubuntu) - if: ${{ startsWith(matrix.os, 'ubuntu') }} - run: | - sudo apt-get update -y - sudo apt-get install -y protobuf-compiler libprotobuf-dev wireshark-dev - - - name: Install deps (macOS) - if: ${{ startsWith(matrix.os, 'macos') }} - run: | - brew update - brew install pkg-config wireshark protobuf - - name: Build wireshark plugin - run: | - cmake -S wireshark -B build-wireshark - cmake --build build-wireshark - - unit-tests: - name: Run unit tests - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Install deps - run: | - sudo apt-get update -y && \ - sudo apt-get install -y \ - libcurl4-openssl-dev \ - protobuf-compiler \ - libprotobuf-dev \ - libboost-dev \ - libboost-program-options-dev \ - libzstd-dev \ - libsnappy-dev \ - libgmock-dev \ - libgtest-dev - - - name: Install gtest-parallel - run: | - sudo curl -o /gtest-parallel https://raw.githubusercontent.com/google/gtest-parallel/master/gtest_parallel.py - - - name: CMake - run: cmake . -DCMAKE_BUILD_TYPE=Debug -DBUILD_PERF_TOOLS=ON - - - name: Check formatting - run: make check-format - - - name: Build - run: | - # Build the libraries first to avoid possible link failures - cmake --build . -j8 --target pulsarShared pulsarStatic - cmake --build . -j8 - - - name: Run unit tests - run: RETRY_FAILED=3 ./run-unit-tests.sh - - cpp20-build: - name: Build with the C++20 standard - runs-on: ubuntu-22.04 - timeout-minutes: 60 - - steps: - - name: checkout - uses: actions/checkout@v3 - - name: Install deps - run: | - sudo apt-get update -y - sudo apt-get install -y libcurl4-openssl-dev libssl-dev \ - protobuf-compiler libprotobuf-dev libboost-dev \ - libboost-dev libboost-program-options-dev \ - libzstd-dev libsnappy-dev libgmock-dev libgtest-dev - - name: CMake - run: cmake -B build -DBUILD_PERF_TOOLS=ON -DCMAKE_CXX_STANDARD=20 - - name: Build - run: | - cmake --build build -j8 --target pulsarShared pulsarStatic - cmake --build build -j8 - - cpp-build-windows: + cpp-build-macos-static: timeout-minutes: 120 - name: Build CPP Client on ${{ matrix.name }} - needs: unit-tests - runs-on: ${{ matrix.os }} - env: - VCPKG_ROOT: '${{ github.workspace }}/vcpkg' - INSTALL_DIR: 'C:\\pulsar-cpp' - strategy: - fail-fast: false - matrix: - include: - - name: 'Windows x64' - os: windows-2019 - triplet: x64-windows-static - suffix: 'windows-win64' - generator: 'Visual Studio 16 2019' - arch: '-A x64' - - name: 'Windows x86' - os: windows-2019 - triplet: x86-windows-static - suffix: 'windows-win32' - generator: 'Visual Studio 16 2019' - arch: '-A Win32' - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Restore vcpkg and its artifacts. - uses: actions/cache@v3 - id: vcpkg-cache - with: - path: | - ${{ env.VCPKG_ROOT }} - vcpkg_installed - !${{ env.VCPKG_ROOT }}/.git - !${{ env.VCPKG_ROOT }}/buildtrees - !${{ env.VCPKG_ROOT }}/packages - !${{ env.VCPKG_ROOT }}/downloads - key: | - ${{ runner.os }}-${{ matrix.triplet}}-${{ hashFiles( 'vcpkg.json' ) }} - - - name: Get vcpkg(windows) - if: ${{ runner.os == 'Windows' && steps.vcpkg-cache.outputs.cache-hit != 'true' }} - run: | - cd ${{ github.workspace }} - mkdir build -force - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - - - name: remove system vcpkg(windows) - if: runner.os == 'Windows' - run: rm -rf "$VCPKG_INSTALLATION_ROOT" - shell: bash - - - name: Install vcpkg packages - run: | - ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }} - - - name: Configure - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake \ - -B ./build-1 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ - -S . - fi - - - name: Install - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake --build ./build-1 --parallel --config Release - cmake --install ./build-1 - fi - - - name: Test examples - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cd win-examples - cmake \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DLINK_STATIC=OFF \ - -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ - -B build-dynamic - cmake --build build-dynamic --config Release - cmake \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DLINK_STATIC=ON \ - -DCMAKE_PREFIX_PATH=${{ env.INSTALL_DIR }} \ - -B build-static - cmake --build build-static --config Release - ./build-static/Release/win-example.exe - fi - - - name: Build (Debug) - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - cmake \ - -B ./build-2 \ - -G "${{ matrix.generator }}" ${{ matrix.arch }} \ - -DBUILD_TESTS=OFF \ - -DVCPKG_TRIPLET="${{ matrix.triplet }}" \ - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" \ - -DCMAKE_BUILD_TYPE=Debug \ - -S . - cmake --build ./build-2 --parallel --config Debug - fi - - package: - name: Build ${{matrix.pkg.name}} ${{matrix.cpu.platform}} - runs-on: ubuntu-22.04 - needs: unit-tests - timeout-minutes: 500 - - strategy: - fail-fast: true - matrix: - pkg: - - { name: 'RPM', type: 'rpm', path: 'pkg/rpm/RPMS' } - - { name: 'Deb', type: 'deb', path: 'pkg/deb/BUILD/DEB' } - - { name: 'Alpine', type: 'apk', path: 'pkg/apk/build' } - cpu: - - { arch: 'x86_64', platform: 'x86_64' } - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Package Pulsar source - run: build-support/generate-source-archive.sh - - - uses: docker/setup-buildx-action@v2 - - run: build-support/copy-deps-versionfile.sh - - - name: Build dependencies Docker image - uses: docker/build-push-action@v3 - with: - context: ./pkg/${{matrix.pkg.type}} - load: true - tags: build:latest - platforms: linux/${{matrix.cpu.platform}} - build-args: PLATFORM=${{matrix.cpu.arch}} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build packages - run: pkg/${{matrix.pkg.type}}/docker-build-${{matrix.pkg.type}}-${{matrix.cpu.platform}}.sh build:latest - - cpp-build-macos: - timeout-minutes: 120 - name: Build CPP Client on macOS + name: Build CPP Client on macOS with static dependencies runs-on: macos-12 - needs: unit-tests steps: - name: checkout uses: actions/checkout@v3 - - name: Install dependencies - run: brew install openssl protobuf boost zstd snappy googletest + - name: Build libraries + run: ./pkg/mac/build-static-library.sh - - name: Configure (default) - shell: bash - run: cmake -B ./build-macos -S . - - - name: Compile - shell: bash + - name: Test static libraries run: | - cmake --build ./build-macos --parallel --config Release - - # Job that will be required to complete and depends on all the other jobs - check-completion: - name: Check Completion - runs-on: ubuntu-latest - needs: [wireshark-dissector-build, unit-tests, cpp20-build, cpp-build-windows, package, cpp-build-macos] - - steps: - - run: true + export PULSAR_DIR=$PWD/pkg/mac/.install + echo "Build with static library" + clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a + ./static.out + echo "Build with dynamic library" + clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar + ./dynamic.out diff --git a/.gitignore b/.gitignore index a569bda0..a1e5224a 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,5 @@ vcpkg_installed/ .tests-container-id.txt Testing .test-token.txt +pkg/mac/.build +pkg/mac/.install diff --git a/CMakeLists.txt b/CMakeLists.txt index c8d6a00b..fb4f1b12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,9 +88,17 @@ find_package(Threads REQUIRED) MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) set(Boost_NO_BOOST_CMAKE ON) + +if (APPLE AND NOT LINK_STATIC) + # The latest Protobuf dependency on macOS requires the C++17 support and + # it could only be found by the CONFIG mode + set(LATEST_PROTOBUF TRUE) +else () + set(LATEST_PROTOBUF FALSE) +endif () + if (NOT CMAKE_CXX_STANDARD) - if (APPLE) - # The latest Protobuf dependency on macOS requires the C++17 support + if (LATEST_PROTOBUF) set(CMAKE_CXX_STANDARD 17) else () set(CMAKE_CXX_STANDARD 11) @@ -143,7 +151,7 @@ find_package(OpenSSL REQUIRED) message("OPENSSL_INCLUDE_DIR: " ${OPENSSL_INCLUDE_DIR}) message("OPENSSL_LIBRARIES: " ${OPENSSL_LIBRARIES}) -if (APPLE) +if (LATEST_PROTOBUF) # See https://github.com/apache/arrow/issues/35987 add_definitions(-DPROTOBUF_USE_DLLS) # Use Config mode to avoid FindProtobuf.cmake does not find the Abseil library @@ -318,7 +326,7 @@ set(COMMON_LIBS ${CMAKE_DL_LIBS} ) -if (APPLE) +if (LATEST_PROTOBUF) # Protobuf_LIBRARIES is empty when finding Protobuf in Config mode set(COMMON_LIBS ${COMMON_LIBS} protobuf::libprotobuf) else () diff --git a/dependencies.yaml b/dependencies.yaml index f206ccc4..18be4383 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -23,5 +23,5 @@ protobuf: 3.20.0 zlib: 1.2.12 zstd: 1.5.2 snappy: 1.1.9 -openssl: 1.1.1q +openssl: 1.1.1v curl: 8.4.0 diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh new file mode 100755 index 00000000..f9ad05f8 --- /dev/null +++ b/pkg/mac/build-static-library.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex +cd `dirname $0` + +pip3 install pyyaml + +MACOSX_DEPLOYMENT_TARGET=11.0 +if [[ -z ${ARCH} ]]; then + ARCH=`uname -m` +fi + +BUILD_DIR=$PWD/.build +INSTALL_DIR=$PWD/.install +PREFIX=$BUILD_DIR/install +mkdir -p $BUILD_DIR +cp -f ../../build-support/dep-version.py $BUILD_DIR/ +cp -f ../../dependencies.yaml $BUILD_DIR/ + +pushd $BUILD_DIR + +BOOST_VERSION=$(./dep-version.py boost) +ZLIB_VERSION=$(./dep-version.py zlib) +OPENSSL_VERSION=$(./dep-version.py openssl) +PROTOBUF_VERSION=$(./dep-version.py protobuf) +ZSTD_VERSION=$(./dep-version.py zstd) +SNAPPY_VERSION=$(./dep-version.py snappy) +CURL_VERSION=$(./dep-version.py curl) + +BOOST_VERSION_=${BOOST_VERSION//./_} +if [ ! -f boost/.done ]; then + echo "Building Boost $BOOST_VERSION" + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz + tar zxf boost_${BOOST_VERSION_}.tar.gz + mkdir -p $PREFIX/include + cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/ + mkdir -p boost + touch boost/.done +else + echo "Using cached Boost $BOOST_VERSION" +fi + +if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then + echo "Building ZLib $ZLIB_VERSION" + curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz + tar zxf zlib-${ZLIB_VERSION}.tar.gz + pushd zlib-$ZLIB_VERSION + CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX + make -j16 + make install + touch .done + popd +else + echo "Using cached ZLib $ZLIB_VERSION" +fi + +OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') +if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then + echo "Building OpenSSL $OPENSSL_VERSION" + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz + tar zxf OpenSSL_$OPENSSL_VERSION_UNDERSCORE.tar.gz + + pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} + if [[ $ARCH = 'arm64' ]]; then + PLATFORM=darwin64-arm64-cc + else + PLATFORM=darwin64-x86_64-cc + fi + CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./Configure --prefix=$PREFIX no-shared no-unit-test $PLATFORM + make -j8 >/dev/null + make install_sw >/dev/null + popd + + touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done +else + echo "Using cached OpenSSL $OPENSSL_VERSION" +fi + +if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then + echo "Building Protobuf $PROTOBUF_VERSION" + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + tar zxf protobuf-cpp-${PROTOBUF_VERSION}.tar.gz + pushd protobuf-${PROTOBUF_VERSION} + pushd cmake/ + cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch x86_64 -arch arm64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=$PREFIX + cmake --build build -j16 --target install + popd + + pushd $PREFIX/lib + mv libprotobuf.a libprotobuf_universal.a + lipo libprotobuf_universal.a -thin ${ARCH} -output libprotobuf.a + popd + + touch .done + popd +else + echo "Using cached Protobuf $PROTOBUF_VERSION" +fi + +if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then + echo "Building ZStd $ZSTD_VERSION" + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz + tar zxf zstd-${ZSTD_VERSION}.tar.gz + pushd zstd-${ZSTD_VERSION} + CFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \ + make -j16 -C lib install-static install-includes + touch .done + popd +else + echo "Using cached ZStd $ZSTD_VERSION" +fi + +if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then + echo "Building Snappy $SNAPPY_VERSION" + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz + tar zxf ${SNAPPY_VERSION}.tar.gz + pushd snappy-${SNAPPY_VERSION} + CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF + make -j16 + make install + touch .done + popd +else + echo "Using cached Snappy $SNAPPY_VERSION" +fi + +if [ ! -f curl-${CURL_VERSION}/.done ]; then + echo "Building LibCurl $CURL_VERSION" + CURL_VERSION_=${CURL_VERSION//./_} + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz + tar zxf curl-${CURL_VERSION}.tar.gz + pushd curl-${CURL_VERSION} + # Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl. + cp -rf $PREFIX/include/openssl include/ + CFLAGS="-I$PREFIX/include -fPIC -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + ./configure --with-ssl=$PREFIX \ + --without-nghttp2 \ + --without-libidn2 \ + --disable-ldap \ + --without-brotli \ + --without-secure-transport \ + --without-librtmp \ + --disable-ipv6 \ + --host=$ARCH-apple-darwin \ + --prefix=$PREFIX + make -j16 install + touch .done + popd +else + echo "Using cached LibCurl $CURL_VERSION" +fi + +popd # pkg/mac +cd ../../ # project root + +cmake -B build-static -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ + -DLINK_STATIC=ON \ + -DBUILD_TESTS=OFF \ + -DBUILD_DYNAMIC_LIB=ON \ + -DBUILD_STATIC_LIB=ON \ + -DCMAKE_OSX_ARCHITECTURES=${ARCH} \ + -DCMAKE_PREFIX_PATH=$PREFIX \ + -DOPENSSL_ROOT_DIR=$PREFIX \ + -DPROTOC_PATH=$PREFIX/bin/protoc \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ + -DCMAKE_BUILD_TYPE=Release +cmake --build build-static -j16 --target install