From a321d86912bd11719965796a01ca1333f6a0f3a7 Mon Sep 17 00:00:00 2001 From: David Zuckerman Date: Mon, 8 Dec 2025 13:20:16 -0800 Subject: [PATCH] adding actions fixed deprecation issues with Docker removed reference to containers.lib image in docker-compose reset secrets in compose.ci moved COA user to environment removing tests for now. will need to be reworked removing tests for now. will need to be reworked updated read me and sftp user moving username to environement variable remove environment for tests, adding tests to build.yml renamed bfs service to app in docker compose Adding /opt/app directory and add artifacts to .gitignore referencing /opt/app as oppose to /opt/app-root/src as base directory adding file processing test removed path for image in container.lib skipping sftp for tests fixed typo for override in compose.ci syntax error in compose.ci build report was named Gobi instead of BFS using newer build and release template, slight syntax change in compose.ci reverting compose.ci to debug DOCKER_APP_IMAGE incorrectly mapped changed compose.ci to use array syntax --- .github/workflows/build.yml | 213 ++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 61 ++++++++++ .gitignore | 3 + .ruby-version | 2 +- Dockerfile | 12 +- Jenkinsfile | 1 - README.md | 20 ++-- docker-compose.ci.yml | 14 +++ docker-compose.yml | 10 +- lib/sftp_bfs.rb | 2 +- spec/bfs_spec.rb | 5 - 11 files changed, 315 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/release.yml delete mode 100644 Jenkinsfile create mode 100644 docker-compose.ci.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..ad28562 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,213 @@ +name: Build / Test / Push + +on: + push: + branches: + - '**' + workflow_call: + workflow_dispatch: + +env: + BUILD_SUFFIX: -build-${{ github.run_id }}_${{ github.run_attempt }} + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + build: + runs-on: ${{ matrix.runner }} + outputs: + build-image-arm: ${{ steps.gen-output.outputs.image-arm64 }} + build-image-x64: ${{ steps.gen-output.outputs.image-x64 }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: build-meta + name: Produce the build image tag + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha,suffix=${{ env.BUILD_SUFFIX }} + + # Build cache is shared among all builds of the same architecture + - id: cache-meta + name: Fetch build cache metadata + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=raw,value=buildcache-${{ runner.arch }} + + - id: get-registry + name: Get the sanitized registry name + run: | + echo "registry=$(echo '${{ steps.build-meta.outputs.tags }}' | cut -f1 -d:)" | tee -a "$GITHUB_OUTPUT" + + - id: set_build_url + name: Set BUILD_URL + run: | + echo "build_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | tee -a "$GITHUB_OUTPUT" + + - id: build + name: Build/push the arch-specific image + uses: docker/build-push-action@v6 + with: + platforms: ${{ matrix.platform }} + build-args: | + BUILD_TIMESTAMP=${{ github.event.repository.updated_at }} + BUILD_URL=${{ steps.set_build_url.outputs.build_url }} + GIT_REF_NAME=${{ github.ref_name }} + GIT_SHA=${{ github.sha }} + GIT_REPOSITORY_URL=${{ github.repositoryUrl }} + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + labels: ${{ steps.build-meta.outputs.labels }} + provenance: mode=max + sbom: true + tags: ${{ steps.get-registry.outputs.registry }} + outputs: type=image,push-by-digest=true,push=true + + - id: gen-output + name: Write arch-specific image digest to outputs + run: | + echo "image-${RUNNER_ARCH,,}=${{ steps.get-registry.outputs.registry }}@${{ steps.build.outputs.digest }}" | tee -a "$GITHUB_OUTPUT" + + merge: + runs-on: ubuntu-latest + needs: + - build + env: + DOCKER_APP_IMAGE_ARM64: ${{ needs.build.outputs.build-image-arm }} + DOCKER_APP_IMAGE_X64: ${{ needs.build.outputs.build-image-x64 }} + outputs: + build-image: ${{ steps.meta.outputs.tags }} + build-image-arm: ${{ needs.build.outputs.build-image-arm }} + build-image-x64: ${{ needs.build.outputs.build-image-x64 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha,suffix=-build-${{ github.run_id }}_${{ github.run_attempt }} + + - name: Push the multi-platform image + run: | + docker buildx imagetools create \ + --tag "$DOCKER_METADATA_OUTPUT_TAGS" \ + "$DOCKER_APP_IMAGE_ARM64" "$DOCKER_APP_IMAGE_X64" + + test: + runs-on: ubuntu-24.04 + needs: merge + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.build-image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup the stack + run: | + docker compose build --quiet + docker compose pull --quiet + docker compose up --wait + docker compose exec -u root app chown -R bfs:bfs artifacts + + - name: Run RSpec + if: ${{ always() }} + run: | + docker compose exec app rspec --format progress --format html --out artifacts/rspec.html + + - name: Copy out artifacts + if: ${{ always() }} + run: | + docker compose cp app:/opt/app/artifacts ./ + docker compose logs > artifacts/docker-compose-services.log + docker compose config > artifacts/docker-compose.merged.yml + + - name: Upload the test report + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: BFS Build Report (${{ github.run_id }}_${{ github.run_attempt }}) + path: artifacts/* + if-no-files-found: error + + push: + runs-on: ubuntu-24.04 + needs: + - merge + - test + env: + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.build-image }} + DOCKER_APP_IMAGE_ARM64: ${{ needs.merge.outputs.build-image-arm }} + DOCKER_APP_IMAGE_X64: ${{ needs.merge.outputs.build-image-x64 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Produce permanent image tags + id: branch-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + + - name: Retag and push the image + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $DOCKER_APP_IMAGE_ARM64 $DOCKER_APP_IMAGE_X64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b9a9c62 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,61 @@ +name: Push Release Tags + +on: + push: + tags: + - '**' + workflow_call: + workflow_dispatch: + +env: + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + retag: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine the sha-based image tag to retag + id: get-base-image + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha + + - name: Verify that the image was previously built + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker manifest inspect "$BASE_IMAGE" + + - name: Produce release tags + id: tag-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + flavor: latest=false + tags: | + type=ref,event=tag + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + + - name: Retag the pulled image + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + "$(echo "$BASE_IMAGE" | cut -f1 -d:)" diff --git a/.gitignore b/.gitignore index b8a2827..300400f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ data/invoicing/pay/processed/* ssh_tests +# Build/test artifacts +artifacts/* + vendor .bundle diff --git a/.ruby-version b/.ruby-version index ef538c2..23887f6 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.1.2 +3.1.7 diff --git a/Dockerfile b/Dockerfile index 9f69c98..98ca861 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM registry.access.redhat.com/ubi8/ruby-31 - USER root # Configure users and groups @@ -7,12 +6,17 @@ RUN groupadd -g 40054 alma && \ useradd -r -s /sbin/nologin -M -u 40054 -g alma alma && \ groupadd -g 40061 bfs && \ usermod -u 40061 -g bfs -G alma -l bfs default && \ - find / -user 1001 -exec chown -h bfs {} \; || true + find / -user 1001 -exec chown -h bfs {} \; || true && \ + mkdir -p /opt/app && \ + chown -R bfs:bfs /opt/app +WORKDIR /opt/app COPY --chown=bfs Gemfile* .ruby-version ./ -RUN bundle install --system +RUN bundle config set force_ruby_platform true +RUN bundle config set system 'true' +RUN bundle install COPY --chown=bfs . . USER bfs -ENTRYPOINT ["/opt/app-root/src/bin/bfs"] +ENTRYPOINT ["/opt/app/bin/bfs"] CMD ["help"] diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 2f65418..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1 +0,0 @@ -dockerComposePipeline() diff --git a/README.md b/README.md index 4f73756..fbe1c6b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A command-line tool for processing BFS .xml files. Input files can be mounted an ## Building the app ```sh -docker-compose build +docker compose build ``` ## Running it @@ -13,38 +13,38 @@ docker-compose build View the CLI tool help/description: ```sh -docker-compose run --rm bfs help +docker compose run --rm bfs help ``` Adds test data to the default watch directory: ```sh -docker-compose run --rm bfs seed +docker compose run --rm bfs seed ``` Run the app in the background. It will continue running, monitoring for .xml files to process every 10s. ```sh -docker-compose up -d -docker-compose logs -f # view processing logs in real time +docker compose up -d +docker compose logs -f # view processing logs in real time ``` Watch a non-standard directory: ```sh -docker-compose run --rm bfs watch /path/in/container # absolute path -docker-compose run --rm bfs watch data/somedir # path relative to /opt/app-root/src +docker compose run --rm bfs watch /path/in/container # absolute path +docker compose run --rm bfs watch data/somedir # path relative to /opt/app-root/src ``` Process a specific file: ```sh -docker-compose run --rm bfs process /abs/path/to/myfile.xml # absolute path -docker-compose run --rm bfs process data/invoicing/pay/somefile.xml # relative path +docker compose run --rm bfs process /abs/path/to/myfile.xml # absolute path +docker compose run --rm bfs process data/invoicing/pay/somefile.xml # relative path ``` Delete previously processed files and error logs: ```sh -docker-compose run --rm bfs clear +docker compose run --rm bfs clear ``` diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..6f908d6 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,14 @@ +services: + app: + build: !reset + image: ${DOCKER_APP_IMAGE} + environment: !override + - SKIP_SFTP=skip_sftp + volumes: !override + - artifacts:/opt/app/artifacts + secrets: !reset + +volumes: + artifacts: + +secrets: !reset diff --git a/docker-compose.yml b/docker-compose.yml index 1f0051b..6677b7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,18 @@ services: - bfs: + app: build: . command: watch --interval 10 - image: containers.lib.berkeley.edu/lap/bfs/development:latest init: true environment: COA_APP_ID: "${COA_APP_ID}" COA_APP_KEY: "${COA_APP_KEY}" + BFS_SFTP_USER: "${BFS_SFTP_USER}" volumes: - - ./:/opt/app-root/src:rw + - ./:/opt/app:rw - ./secrets:/run/secrets:ro secrets: - source: SSH_KEY - target: /opt/app/src/.ssh/id_rsa + target: /opt/app/.ssh/id_rsa uid: "40061" gid: "40061" mode: 0400 @@ -20,5 +20,3 @@ services: secrets: SSH_KEY: file: secrets/SSH_KEY - -version: "3.8" diff --git a/lib/sftp_bfs.rb b/lib/sftp_bfs.rb index b6ad4f5..757bd78 100644 --- a/lib/sftp_bfs.rb +++ b/lib/sftp_bfs.rb @@ -12,7 +12,7 @@ def self.sftp_bfs() begin sftp = Net::SFTP.start( 'ucopmft-in.ucop.edu', - 'cUCB100_library', + ENV['BFS_SFTP_USER'], { append_all_supported_algorithms: true } ) diff --git a/spec/bfs_spec.rb b/spec/bfs_spec.rb index e0d8970..037ecbc 100644 --- a/spec/bfs_spec.rb +++ b/spec/bfs_spec.rb @@ -30,9 +30,4 @@ expect(Pathname.new(bfs_file)).to_not exist end -# it 'refreshes fixture data from incoming' do -# GOBI.refresh! -# expect(Pathname.new(incoming_file)).to exist -# end - end