data_platform/.github/workflows/glue-python-deploy.yaml at main · mbta/data_platform · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
name: Glue (Python) Deploy

permissions:
  id-token: write

on:
  workflow_dispatch:
    inputs:
      env:
        description: "Environment, 'dev' or 'prod'."
        required: true

jobs:
  asdf:
    name: ASDF
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
      # cache the ASDF directory, using the values from .tool-versions
      - name: ASDF cache
        uses: actions/cache@v2
        with:
          path: ~/.asdf
          key: ${{ runner.os }}-asdf-${{ hashFiles('.tool-versions') }}
        id: asdf-cache
      # only run `asdf install` if we didn't hit the cache
      - uses: asdf-vm/actions/install@v1
        if: steps.asdf-cache.outputs.cache-hit != 'true'
      # only install Hex/Rebar if we didn't hit the cache
      - if: steps.asdf-cache.outputs.cache-hit != 'true'
        run: |
          mix local.rebar --force
          mix local.hex --force

  deploy_glue:
    name: Deploy Glue (Python)
    runs-on: ubuntu-latest
    needs: asdf

    steps:
      - name: Checkout repository code
        uses: actions/checkout@v2
      - uses: actions/checkout@v2
      - name: ASDF cache
        uses: actions/cache@v2
        with:
          path: ~/.asdf
          key: ${{ runner.os }}-asdf-${{ hashFiles('.tool-versions') }}
        id: asdf-cache
      - uses: mbta/actions/reshim-asdf@v1
      # The asdf job should have prepared the cache. exit if it didn't for some reason
      - run: exit 1
        if: steps.asdf-cache.outputs.cache-hit != 'true'
      - name: Prepare for installing dependencies and package
        working-directory: ${{github.workspace}}/py_cubic_ingestion
        run: |
          poetry export --without-hashes --output requirements.txt
          poetry build
      - name: Install dependencies and py_cubic_ingestion package into 'libs'
        working-directory: ${{github.workspace}}/py_cubic_ingestion
        run: |
          pip install -r requirements.txt -t ./libs
          pip install dist/py_cubic_ingestion-*.whl -t ./libs

      - name: Setup AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: 'us-east-1'
      - name: Add to py_cubic_ingestion archive
        working-directory: ${{github.workspace}}/py_cubic_ingestion/libs
        run: zip -r ${{github.workspace}}/aws/s3/packages/py_cubic_ingestion.zip ./
      - name: Sync Glue Jobs and package to operations
        env:
          S3_BUCKET_OPERATIONS: ${{github.event.inputs.env == 'prod' && 'mbta-ctd-dataplatform-operations' || 'mbta-ctd-dataplatform-dev-operations'}}
        run: aws s3 sync ${{github.workspace}}/aws/s3/ s3://${{env.S3_BUCKET_OPERATIONS}}/ --exclude "*/empty"
      - uses: mbta/actions/notify-slack-deploy@v1
        if: ${{ !cancelled() }}
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK }}
          job-status: ${{ job.status }}
          custom-message: "(glue-python-${{github.event.inputs.env}})"