diff --git a/.github/actions/asdf-cache/action.yml b/.github/actions/asdf-cache/action.yml new file mode 100644 index 0000000..185b711 --- /dev/null +++ b/.github/actions/asdf-cache/action.yml @@ -0,0 +1,61 @@ +name: "asdf-cache" +description: "asdf cache action from https://github.com/ai/asdf-cache-action/blob/main/action.yml" +inputs: + asdf-version: + description: "asdf version to install" + default: "" + required: false + os: + description: "target os" + default: "linux" + required: false + architecture: + description: "target architecture" + default: "amd64" + required: false +runs: + using: "composite" + steps: + + - name: install asdf + shell: bash + run: | + set -o pipefail + asdf_version="${{ inputs.asdf-version }}" + asdf_version="$(echo "${asdf_version}" | sed 's#v##g')" + if which asdf && [[ "${asdf_version}" == "" || "$(asdf --version | cut -d' ' -f3)" == "v${asdf_version}" ]]; then + echo "asdf: $(asdf --version | cut -d' ' -f3) detected" + else + if [[ "${asdf_version}" == "" ]]; then + asdf_version="$(curl --fail -s "${GITHUB_API_URL}/repos/asdf-vm/asdf/releases/latest" | jq -r '.name')" + fi + asdf_version="$(echo "${asdf_version}" | sed 's#v##g')" + echo "installing asdf ${asdf_version}" + wget -q -O - "https://github.com/asdf-vm/asdf/releases/download/v${asdf_version}/asdf-v${asdf_version}-${{ inputs.os }}-${{ inputs.architecture }}.tar.gz" | tar -zxf - -C /usr/local/bin asdf + fi + + - name: Cache asdf + id: cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: asdf-${{ hashFiles('**/.tool-versions') }} + + - name: install asdf plugins + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + for plugin in $(cat .tool-versions | grep -Ev '^#' | cut -d' ' -f1 | uniq); do + asdf plugin add $plugin + done + + - name: asdf install + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + asdf install + + - name: update path + shell: bash + run: | + echo "${HOME}/.asdf/shims" >> $GITHUB_PATH diff --git a/.github/actions/build-common/action.yml b/.github/actions/build-common/action.yml new file mode 100644 index 0000000..018f8dc --- /dev/null +++ b/.github/actions/build-common/action.yml @@ -0,0 +1,90 @@ +name: "common-build-steps" +description: "regular build steps" +inputs: + python-version: + description: "If set, will use a system python version rather than asdf. Eg. a value of 3.11 would use the latest 3.11.x version. Set to empty string to use asdf versions." + default: "3.11" + required: false + fetch-depth: + description: "git fetch depth" + default: "0" + required: false + + +runs: + using: "composite" + steps: + - name: checkout the calling repo + uses: actions/checkout@v4 + with: + fetch-depth: ${{ inputs.fetch-depth }} + + - name: setup python + if: ${{ inputs.python-version != '' }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: print branch info + shell: bash + run: | + git branch + echo "GITHUB_HEAD_REF=${GITHUB_HEAD_REF}" + echo "GITHUB_BASE_REF=${GITHUB_BASE_REF}" + git log --oneline -n 10 + + - name: clean + shell: bash + run: | + git clean -fdx + + - name: check secrets + uses: ./.github/actions/check-secrets + + - name: merge into base_branch + if: ${{ github.event_name == 'pull_request' }} + shell: bash + run: | + echo base branch "${{ github.base_ref }}" + echo pr branch "${{ github.head_ref }}" + git checkout "${{ github.base_ref }}" + git checkout -b "merging-${{ github.event.number }}" + git merge --ff-only "${{ github.event.pull_request.head.sha }}" + + - name: git reset + shell: bash + run: git reset --hard + + - name: replace asdf python version + if: ${{ inputs.python-version != '' }} + shell: bash + run: sed -i -E 's#^python .*##g' .tool-versions + + - name: Install tools from asdf config + if: ${{ hashFiles('**/.tool-versions') }} + uses: ./.github/actions/asdf-cache + + - name: cache virtualenv + uses: actions/cache@v4 + with: + path: | + .venv + **/.lock-hash + **/requirements.txt + key: ${{ runner.os }}-py-${{ inputs.python-version }}-poetry-${{ hashFiles('./poetry.lock') }} + + - name: fix virtualenv + shell: bash + run: | + if [ -d .venv/bin ]; then + echo fixing .venv + unlink .venv/bin/python3 + py_version="$(ls /opt/hostedtoolcache/Python --color=no | sed 's#/##g' | grep -E '^3.11' | sort -t\. -k3 --numeric | tail -n 1)" + if [ -z "${py_version}" ]; then + ls /opt/hostedtoolcache/Python + echo "could not find a compatible python version for ${py_version}" + exit -1 + fi + ln -s -t .venv/bin "/opt/hostedtoolcache/Python/${py_version}/x64/bin/python3" + find .venv/bin -type f -exec file {} + | awk -F: '/ASCII text/ {print $1}' | xargs grep -lr '.venv' | xargs sed -i -E "s#/.*?/.venv#${GITHUB_WORKSPACE}/.venv#" + fi diff --git a/.github/actions/check-secrets/action.yml b/.github/actions/check-secrets/action.yml new file mode 100644 index 0000000..09d0431 --- /dev/null +++ b/.github/actions/check-secrets/action.yml @@ -0,0 +1,22 @@ +name: "CI stages" +description: "run any standard CI stages found in the root Makefile" + +inputs: + scan-type: + description: secrets scan type recursive/untracked etc. + default: 'recursive' + required: false + +runs: + using: "composite" + steps: + - name: run git-secrets + shell: bash + run: | + export PATH="${PATH}:${{ github.action_path }}" + git secrets --register-aws + if [[ -e ./.gitdisallowed ]]; then + git secrets --add-provider -- grep -Ev '^(#.*|\s*$)' .gitdisallowed || true + git secrets --add --allowed '^.gitdisallowed:[0-9]+:.*' || true + fi + git secrets --scan --${{ inputs.scan-type }} diff --git a/.github/actions/check-secrets/git-secrets b/.github/actions/check-secrets/git-secrets new file mode 100755 index 0000000..a2f16ac --- /dev/null +++ b/.github/actions/check-secrets/git-secrets @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# Copyright 2010-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +NONGIT_OK=1 OPTIONS_SPEC="\ +git secrets --scan [-r|--recursive] [--cached] [--no-index] [--untracked] [...] +git secrets --scan-history +git secrets --install [-f|--force] [] +git secrets --list [--global] +git secrets --add [-a|--allowed] [-l|--literal] [--global] +git secrets --add-provider [--global] [arguments...] +git secrets --register-aws [--global] +git secrets --aws-provider [] +-- +scan Scans for prohibited patterns +scan-history Scans repo for prohibited patterns +install Installs git hooks for Git repository or Git template directory +list Lists secret patterns +add Adds a prohibited or allowed pattern, ensuring to de-dupe with existing patterns +add-provider Adds a secret provider that when called outputs secret patterns on new lines +aws-provider Secret provider that outputs credentials found in an ini file +register-aws Adds common AWS patterns to the git config and scans for ~/.aws/credentials +r,recursive --scan scans directories recursively +cached --scan scans searches blobs registered in the index file +no-index --scan searches files in the current directory that is not managed by Git +untracked In addition to searching in the tracked files in the working tree, --scan also in untracked files +f,force --install overwrites hooks if the hook already exists +l,literal --add and --add-allowed patterns are escaped so that they are literal +a,allowed --add adds an allowed pattern instead of a prohibited pattern +global Uses the --global git config +commit_msg_hook* commit-msg hook (internal only) +pre_commit_hook* pre-commit hook (internal only) +prepare_commit_msg_hook* prepare-commit-msg hook (internal only)" + +# Include the git setup script. This parses and normalized CLI arguments. +. "$(git --exec-path)"/git-sh-setup + +load_patterns() { + git config --get-all secrets.patterns + # Execute each provider and use their output to build up patterns + git config --get-all secrets.providers | while read -r cmd; do + # Only split words on '\n\t ' and strip "\r" from the output to account + # for carriage returns being added on Windows systems. Note that this + # trimming is done before the test to ensure that the string is not empty. + local result="$(export IFS=$'\n\t '; $cmd | tr -d $'\r')" + # Do not add empty lines from providers as that would match everything. + if [ -n "${result}" ]; then + echo "${result}" + fi + done +} + +load_allowed() { + git config --get-all secrets.allowed + local gitallowed="$(git rev-parse --show-toplevel)/.gitallowed" + if [ -e "$gitallowed" ]; then + cat $gitallowed | awk 'NF && $1!~/^#/' + fi +} + +# load patterns and combine them with | +load_combined_patterns() { + local patterns=$(load_patterns) + local combined_patterns='' + for pattern in $patterns; do + combined_patterns=${combined_patterns}${pattern}"|" + done + combined_patterns=${combined_patterns%?} + echo $combined_patterns +} + +# Scans files or a repo using patterns. +scan() { + local files=("${@}") options="" + [ "${SCAN_CACHED}" == 1 ] && options+="--cached" + [ "${SCAN_UNTRACKED}" == 1 ] && options+=" --untracked" + [ "${SCAN_NO_INDEX}" == 1 ] && options+=" --no-index" + # Scan using git-grep if there are no files or if git options are applied. + if [ ${#files[@]} -eq 0 ] || [ ! -z "${options}" ]; then + output=$(git_grep $options "${files[@]}") + else + output=$(regular_grep "${files[@]}") + fi + process_output $? "${output}" +} + +# Scans through history using patterns +scan_history() { + # git log does not support multiple patterns, so we need to combine them + local combined_patterns=$(load_combined_patterns) + [ -z "${combined_patterns}" ] && return 0 + # Looks for differences matching the patterns, reduces the number of revisions to scan + local to_scan=$(git log --all -G"${combined_patterns}" --pretty=%H) + # Scan through revisions with findings to normalize output + output=$(GREP_OPTIONS= LC_ALL=C git grep -nwHEI "${combined_patterns}" $to_scan) + process_output $? "${output}" +} + +# Performs a git-grep, taking into account patterns and options. +# Note: this function returns 1 on success, 0 on error. +git_grep() { + local options="$1"; shift + local files=("${@}") combined_patterns=$(load_combined_patterns) + + [ -z "${combined_patterns}" ] && return 1 + GREP_OPTIONS= LC_ALL=C git grep -nwHEI ${options} "${combined_patterns}" -- "${files[@]}" +} + +# Performs a regular grep, taking into account patterns and recursion. +# Note: this function returns 1 on success, 0 on error. +regular_grep() { + local files=("${@}") patterns=$(load_patterns) action='skip' + [ -z "${patterns}" ] && return 1 + [ ${RECURSIVE} -eq 1 ] && action="recurse" + GREP_OPTIONS= LC_ALL=C grep -d "${action}" -nwHEI "${patterns}" "${files[@]}" +} + +# Process the given status ($1) and output variables ($2). +# Takes into account allowed patterns, and if a bad match is found, +# prints an error message and exits 1. +process_output() { + local status="$1" output="$2" + local allowed=$(load_allowed) + case "$status" in + 0) + [ -z "${allowed}" ] && echo "${output}" >&2 && return 1 + # Determine with a negative grep if the found matches are allowed + echo "${output}" | GREP_OPTIONS= LC_ALL=C grep -Ev "${allowed}" >&2 \ + && return 1 || return 0 + ;; + 1) return 0 ;; + *) exit $status + esac +} + +# Calls the given scanning function at $1, shifts, and passes to it $@. +# Exit 0 if success, otherwise exit 1 with error message. +scan_with_fn_or_die() { + local fn="$1"; shift + $fn "$@" && exit 0 + echo >&2 + echo "[ERROR] Matched one or more prohibited patterns" >&2 + echo >&2 + echo "Possible mitigations:" >&2 + echo "- Mark false positives as allowed using: git config --add secrets.allowed ..." >&2 + echo "- Mark false positives as allowed by adding regular expressions to .gitallowed at repository's root directory" >&2 + echo "- List your configured patterns: git config --get-all secrets.patterns" >&2 + echo "- List your configured allowed patterns: git config --get-all secrets.allowed" >&2 + echo "- List your configured allowed patterns in .gitallowed at repository's root directory" >&2 + echo "- Use --no-verify if this is a one-time false positive" >&2 + exit 1 +} + +# Scans a commit message, passed in the path to a file. +commit_msg_hook() { + scan_with_fn_or_die "scan" "$1" +} + +# Scans all files that are about to be committed. +pre_commit_hook() { + SCAN_CACHED=1 + local files=() file found_match=0 rev="4b825dc642cb6eb9a060e54bf8d69288fbee4904" + # Diff against HEAD if this is not the first commit in the repo. + git rev-parse --verify HEAD >/dev/null 2>&1 && rev="HEAD" + # Filter out deleted files using --diff-filter + while IFS= read -r file; do + [ -n "$file" ] && files+=("$file") + done <<< "$(git diff-index --diff-filter 'ACMU' --name-only --cached $rev --)" + scan_with_fn_or_die "scan" "${files[@]}" +} + +# Determines if merging in a commit will introduce tainted history. +prepare_commit_msg_hook() { + case "$2,$3" in + merge,) + local git_head=$(env | grep GITHEAD) # e.g. GITHEAD_=release/1.43 + local sha="${git_head##*=}" # Get just the SHA + local branch=$(git symbolic-ref HEAD) # e.g. refs/heads/master + local dest="${branch#refs/heads/}" # cut out "refs/heads" + git log "${dest}".."${sha}" -p | scan_with_fn_or_die "scan" - + ;; + esac +} + +install_hook() { + local path="$1" hook="$2" cmd="$3" dest + # Determines the appropriate path for a hook to be installed + if [ -d "${path}/hooks/${hook}.d" ]; then + dest="${path}/hooks/${hook}.d/git-secrets" + else + dest="${path}/hooks/${hook}" + fi + [ -f "${dest}" ] && [ "${FORCE}" -ne 1 ] \ + && die "${dest} already exists. Use -f to force" + echo "#!/usr/bin/env bash" > "${dest}" + echo "git secrets --${cmd} -- \"\$@\"" >> "${dest}" + chmod +x "${dest}" + [ -t 1 ] && command -v tput &> /dev/null && echo -n "$(tput setaf 2)✓$(tput sgr 0) " + echo "Installed ${hook} hook to ${dest}" +} + +install_all_hooks() { + install_hook "$1" "commit-msg" "commit_msg_hook" + install_hook "$1" "pre-commit" "pre_commit_hook" + install_hook "$1" "prepare-commit-msg" "prepare_commit_msg_hook" +} + +# Adds a git config pattern, ensuring to de-dupe +add_config() { + local key="$1"; shift + local value="$@" + if [ ${LITERAL} -eq 1 ]; then + value=$(sed 's/[\.|$(){}?+*^]/\\&/g' <<< "${value}") + fi + if [ ${GLOBAL} -eq 1 ]; then + git config --global --get-all $key | grep -Fq "${value}" && return 1 + git config --global --add "${key}" "${value}" + else + git config --get-all $key | grep -Fq "${value}" && return 1 + git config --add "${key}" "${value}" + fi +} + +register_aws() { + # Reusable regex patterns + local aws="(AWS|aws|Aws)?_?" quote="(\"|')" connect="\s*(:|=>|=)\s*" + local opt_quote="${quote}?" + add_config 'secrets.providers' 'git secrets --aws-provider' + add_config 'secrets.patterns' '(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}' + add_config 'secrets.patterns' "${opt_quote}${aws}(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)${opt_quote}${connect}${opt_quote}[A-Za-z0-9/\+=]{40}${opt_quote}" + add_config 'secrets.patterns' "${opt_quote}${aws}(ACCOUNT|account|Account)_?(ID|id|Id)?${opt_quote}${connect}${opt_quote}[0-9]{4}\-?[0-9]{4}\-?[0-9]{4}${opt_quote}" + add_config 'secrets.allowed' 'AKIAIOSFODNN7EXAMPLE' + add_config 'secrets.allowed' "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + + if [[ $? == 0 ]]; then + echo 'OK' + fi + + exit $? +} + +aws_provider() { + local fi="$1" + [ -z "$fi" ] && fi=~/.aws/credentials + # Find keys and ensure that special characters are escaped. + if [ -f $fi ]; then + awk -F "=" '/aws_access_key_id|aws_secret_access_key/ {print $2}' $fi \ + | tr -d ' "' \ + | sed 's/[]\.|$(){}?+*^]/\\&/g' + fi +} + +# Ensures that the command is what was expected for an option. +assert_option_for_command() { + local expected_command="$1" + local option_name="$2" + if [ "${COMMAND}" != "${expected_command}" ]; then + die "${option_name} can only be supplied with the ${expected_command} subcommand" + fi +} + +declare COMMAND="$1" FORCE=0 RECURSIVE=0 LITERAL=0 GLOBAL=0 ALLOWED=0 +declare SCAN_CACHED=0 SCAN_NO_INDEX=0 SCAN_UNTRACKED=0 + +# Shift off the command name +shift 1 +while [ "$#" -ne 0 ]; do + case "$1" in + -f) + assert_option_for_command "--install" "-f|--force" + FORCE=1 + ;; + -r) + assert_option_for_command "--scan" "-r|--recursive" + RECURSIVE=1 + ;; + -a) + assert_option_for_command "--add" "-a|--allowed" + ALLOWED=1 + ;; + -l) + assert_option_for_command "--add" "-l|--literal" + LITERAL=1 + ;; + --cached) + assert_option_for_command "--scan" "--cached" + SCAN_CACHED=1 + ;; + --no-index) + assert_option_for_command "--scan" "--no-index" + SCAN_NO_INDEX=1 + ;; + --untracked) + assert_option_for_command "--scan" "--untracked" + SCAN_UNTRACKED=1 + ;; + --global) GLOBAL=1 ;; + --) shift; break ;; + esac + shift +done + +# Ensure that recursive is not applied with mutually exclusive options. +if [ ${RECURSIVE} -eq 1 ]; then + if [ ${SCAN_CACHED} -eq 1 ] \ + || [ ${SCAN_NO_INDEX} -eq 1 ] \ + || [ ${SCAN_UNTRACKED} -eq 1 ]; + then + die "-r|--recursive cannot be supplied with --cached, --no-index, or --untracked" + fi +fi + +case "${COMMAND}" in + -h|--help|--) "$0" -h; exit 0 ;; + --add-provider) add_config "secrets.providers" "$@" ;; + --register-aws) register_aws ;; + --aws-provider) aws_provider "$1" ;; + --commit_msg_hook|--pre_commit_hook|--prepare_commit_msg_hook) + ${COMMAND:2} "$@" + ;; + --add) + if [ ${ALLOWED} -eq 1 ]; then + add_config "secrets.allowed" "$1" + else + add_config "secrets.patterns" "$1" + fi + ;; + --scan) scan_with_fn_or_die "scan" "$@" ;; + --scan-history) scan_with_fn_or_die "scan_history" "$@" ;; + --list) + if [ ${GLOBAL} -eq 1 ]; then + git config --global --get-regex secrets.* + else + git config --get-regex secrets.* + fi + ;; + --install) + DIRECTORY="$1" + if [ -z "${DIRECTORY}" ]; then + DIRECTORY=$(git rev-parse --git-dir) || die "Not in a Git repository" + elif [ -d "${DIRECTORY}"/.git ]; then + DIRECTORY="${DIRECTORY}/.git" + fi + mkdir -p "${DIRECTORY}/hooks" || die "Could not create dir: ${DIRECTORY}" + install_all_hooks "${DIRECTORY}" + ;; + *) echo "Unknown option: ${COMMAND}" && "$0" -h ;; +esac diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 0000000..cb6c500 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,31 @@ +name: pull-request +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + + lint: + runs-on: ubuntu-latest + steps: + + - name: checkout the calling repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: common build setup + uses: ./.github/actions/build-common + + - name: tflint --init + if: ${{ hashFiles('.tflint.hcl') }} + run: | + tflint --init + shell: bash + + - name: run lint checks + shell: bash + run: | + make lint-ci diff --git a/.gitignore b/.gitignore index 8bfec31..b95b324 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,10 @@ *vulnerabilities*report*.json *report*json.zip .version - +.idea/ *.code-workspace !project.code-workspace # Please, add your custom content below! -.DS_Store \ No newline at end of file +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6389b61 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,67 @@ +fail_fast: false +exclude: '^.venv/.*' +default_install_hook_types: [pre-commit, pre-push, commit-msg, prepare-commit-msg] +default_stages: [pre-commit] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-ast + - id: check-toml + - id: check-yaml + - id: check-json + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: trailing-whitespace + - id: check-executables-have-shebangs + - id: check-symlinks + - id: destroyed-symlinks + - id: mixed-line-ending + - id: detect-aws-credentials + - id: detect-private-key + - id: fix-byte-order-marker + - id: requirements-txt-fixer + + - repo: local + hooks: + - id: trivy + name: trivy + entry: make tf-trivy + language: system + files: \.tf(vars)?$ + pass_filenames: false + - id: tf-format + name: tf-format + entry: make tf-format + language: system + files: (\.tf|\.tfvars)$ + exclude: \.terraform/.*$ + pass_filenames: false + - id: tf-lint + name: tf-lint + entry: make tf-lint + language: system + files: (\.tf|\.tfvars)$ + exclude: \.terraform/.*$ + pass_filenames: false + - id: shellcheck + name: shellcheck + entry: make shellcheck + language: system + files: (\.sh)$ + pass_filenames: false + - id: secrets + name: git secrets + entry: scripts/check-secrets.sh + language: script + pass_filenames: false + - id: secrets-commit-msg + name: git secrets check commit message + entry: scripts/check-secrets.sh commit-msg + language: system + stages: [commit-msg] + - id: secrets-prep-commit-msg + name: git secrets pre check commit message + entry: scripts/check-secrets.sh commit-msg + language: system + stages: [prepare-commit-msg] diff --git a/.tflint.hcl b/.tflint.hcl new file mode 100644 index 0000000..551aa9f --- /dev/null +++ b/.tflint.hcl @@ -0,0 +1,14 @@ + +plugin "aws" { + enabled = true + version = "0.41.0" + source = "github.com/terraform-linters/tflint-ruleset-aws" +} + +config { + plugin_dir = "~/.tflint.d/plugins" + call_module_type = "local" + ignore_module = { + "does-not-work" = true + } +} diff --git a/.tool-versions b/.tool-versions index 32db55a..a64a6a0 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,7 +1,9 @@ # This file is for you! Please, updated to the versions agreed by your team. -terraform 1.7.0 pre-commit 3.6.0 +terraform 1.7.0 +trivy 0.64.1 +tflint 0.58.1 # ============================================================================== # The section below is reserved for Docker image versions. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a6a8dc9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Contributing + +## dependencies +tools used: +- make +- git +- [asdf version manager](https://asdf-vm.com/guide/getting-started.html) + +## first run ... + +### install project tools +use asdf to ensure required tools are installed ... configured tools are in [.tool-versions](.tool-versions) +```bash +cd ~/work/terraform-aws-backup +for plugin in $(grep -E '^\w+' .tool-versions | cut -d' ' -f1); do asdf plugin add $plugin; done +asdf install +``` + +### setup git-secrets +git secrets scanning uses the awslabs https://github.com/awslabs/git-secrets there are options on how to install but +```bash +# if the command `git secrets` does not work in your repo +# the git-secrets script needs to be added to somewhere in your PATH +# for example if $HOME/.local/bin is in your PATH environment variable +# then: +wget https://raw.githubusercontent.com/awslabs/git-secrets/refs/heads/master/git-secrets -O ~/.local/bin/git-secrets +chmod +x ~/.local/bin/git-secrets +``` + +### install pre-commit hooks +```shell +pre-commit install +``` + + +### secrets +the git-secrets script will try and avoid accidental committing of secrets +patterns are excluded using [.gitdisallowed](.gitdisallowed) and allow listed using [.gitallowed](.gitallowed) +You can check for secrets / test patterns at any time though with +```shell +make check-secrets-all +``` diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ac7103c --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +SHELL:=/bin/bash -o pipefail -O globstar +.SHELLFLAGS = -ec +.PHONY: build dist +.DEFAULT_GOAL := list +make := make --no-print-directory + +list: + @grep '^[^#[:space:]].*:' Makefile + + +guard-%: + @if [[ "${${*}}" == "" ]]; then \ + echo "env var: $* not set"; \ + exit 1; \ + fi + +######################################################################################################################## +## +## Makefile for this project things +## +######################################################################################################################## +pwd := ${PWD} +dirname := $(notdir $(patsubst %/,%,$(CURDIR))) + +tf-lint: + tflint --chdir=modules/aws-backup-source --config "$(pwd)/.tflint.hcl" + tflint --chdir=modules/aws-backup-destination --config "$(pwd)/.tflint.hcl" + tflint --chdir=examples/source --config "$(pwd)/.tflint.hcl" + tflint --chdir=examples/destination --config "$(pwd)/.tflint.hcl" + +tf-format-check: + terraform fmt -check -recursive + +tf-format: + terraform fmt --recursive + +tf-trivy: + trivy conf --exit-code 1 ./ --skip-dirs "**/.terraform" + +shellcheck: + @docker run --rm -i -v ${PWD}:/mnt:ro koalaman/shellcheck -f gcc -e SC1090,SC1091 `find . \( -path "*/.venv/*" -prune -o -path "*/build/*" -prune -o -path "*/dist/*" -prune -o -path "*/.tox/*" -prune \) -o -type f -name '*.sh' -print` + +lint: tf-lint tf-trivy shellcheck +lint-ci: lint + +check-secrets: + scripts/check-secrets.sh + +check-secrets-all: + scripts/check-secrets.sh unstaged + +.env: + echo "LOCALSTACK_PORT=$$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1])')" > .env diff --git a/examples/destination/aws-backups.tf b/examples/destination/aws-backups.tf index 4485b76..fbd53a6 100644 --- a/examples/destination/aws-backups.tf +++ b/examples/destination/aws-backups.tf @@ -1,4 +1,5 @@ -provider "aws" { + +provider "aws" { alias = "source" region = "eu-west-2" } @@ -16,10 +17,7 @@ data "aws_caller_identity" "current" {} locals { # Adjust these as required - project_name = "my-shiny-project" - environment_name = "dev" - - source_account_id = data.aws_arn.source_terraform_role.account + source_account_id = data.aws_arn.source_terraform_role.account destination_account_id = data.aws_caller_identity.current.account_id } @@ -39,7 +37,7 @@ resource "aws_kms_key" "destination_backup_key" { Principal = { AWS = "arn:aws:iam::${local.destination_account_id}:root" } - Action = "kms:*" + Action = "kms:*" Resource = "*" } ] diff --git a/examples/destination/versions.tf b/examples/destination/versions.tf new file mode 100644 index 0000000..49201ad --- /dev/null +++ b/examples/destination/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + + aws = { + source = "hashicorp/aws" + version = "> 5" + } + + } + + required_version = ">= 1.9.5" +} diff --git a/examples/source-bootstrap/permissions.tf b/examples/source-bootstrap/permissions.tf index a4677af..ec14a15 100644 --- a/examples/source-bootstrap/permissions.tf +++ b/examples/source-bootstrap/permissions.tf @@ -85,5 +85,5 @@ resource "aws_iam_policy" "source_account_backup_permissions" { resource "aws_iam_role_policy_attachment" "source_account_backup_permissions" { policy_arn = aws_iam_policy.source_account_backup_permissions.arn - role = var.terraform_apply_role_name + role = var.terraform_apply_role_name } diff --git a/examples/source/aws-backups.tf b/examples/source/aws-backups.tf index b57747b..0dcb594 100644 --- a/examples/source/aws-backups.tf +++ b/examples/source/aws-backups.tf @@ -1,4 +1,4 @@ -provider "aws" { +provider "aws" { alias = "source" region = "eu-west-2" } @@ -14,18 +14,30 @@ data "aws_arn" "destination_vault_arn" { locals { # Adjust these as required - project_name = "my-shiny-project" - environment_name = "dev" - - source_account_id = data.aws_caller_identity.current.account_id + project_name = "my-shiny-project" + environment_name = "dev" + source_account_id = data.aws_caller_identity.current.account_id destination_account_id = data.aws_arn.destination_vault_arn.account } # First, we create an S3 bucket for compliance reports. You may already have a module for creating # S3 buckets with more refined access rules, which you may prefer to use. +# todo: review and remove these ignores +# trivy:ignore:AVD-AWS-0088 +# trivy:ignore:AVD-AWS-0089 +# trivy:ignore:AVD-AWS-0090 +# trivy:ignore:AVD-AWS-0132 resource "aws_s3_bucket" "backup_reports" { - bucket_prefix = "${local.project_name}-backup-reports" + bucket_prefix = "${local.project_name}-backup-reports" +} + +resource "aws_s3_bucket_public_access_block" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.id + ignore_public_acls = true + block_public_acls = true + restrict_public_buckets = true + block_public_policy = true } # Now we have to configure access to the report bucket. @@ -54,7 +66,7 @@ resource "aws_s3_bucket_policy" "backup_reports_policy" { Principal = { AWS = "arn:aws:iam::${local.source_account_id}:role/aws-service-role/reports.backup.amazonaws.com/AWSServiceRoleForBackupReports" }, - Action = "s3:PutObject", + Action = "s3:PutObject", Resource = "${aws_s3_bucket.backup_reports.arn}/*", Condition = { StringEquals = { @@ -73,7 +85,6 @@ resource "aws_s3_bucket_policy" "backup_reports_policy" { # First we need some contextual data data "aws_caller_identity" "current" {} -data "aws_region" "current" {} # Now we can define the key itself resource "aws_kms_key" "backup_notifications" { @@ -89,16 +100,16 @@ resource "aws_kms_key" "backup_notifications" { Principal = { AWS = "arn:aws:iam::${local.source_account_id}:root" } - Action = "kms:*" + Action = "kms:*" Resource = "*" }, { - Effect = "Allow" + Effect = "Allow" Principal = { Service = "sns.amazonaws.com" } - Action = ["kms:GenerateDataKey*", "kms:Decrypt"] - Resource = "*" + Action = ["kms:GenerateDataKey*", "kms:Decrypt"] + Resource = "*" }, { Effect = "Allow" @@ -117,70 +128,70 @@ resource "aws_kms_key" "backup_notifications" { module "source" { source = "../../modules/aws-backup-source" - backup_copy_vault_account_id = local.destination_account_id - backup_copy_vault_arn = data.aws_arn.destination_vault_arn.arn - environment_name = local.environment_name - bootstrap_kms_key_arn = aws_kms_key.backup_notifications.arn - project_name = local.project_name - reports_bucket = aws_s3_bucket.backup_reports.bucket - terraform_role_arns = [data.aws_caller_identity.current.arn] - - backup_plan_config = { - "compliance_resource_types": [ - "S3" - ], - "rules": [ - { - "copy_action": { - "delete_after": 4 - }, - "lifecycle": { - "delete_after": 2 - }, - "name": "daily_kept_for_2_days", - "schedule": "cron(0 0 * * ? *)" - } - ], - "selection_tag": "NHSE-Enable-Backup" - # The selection_tags are optional and can be used to - # provide fine grained resource selection with existing tagging - "selection_tags": [ - { - "key": "Environment" - "value": "myenvironment" - } - ] - } + backup_copy_vault_account_id = local.destination_account_id + backup_copy_vault_arn = data.aws_arn.destination_vault_arn.arn + environment_name = local.environment_name + bootstrap_kms_key_arn = aws_kms_key.backup_notifications.arn + project_name = local.project_name + reports_bucket = aws_s3_bucket.backup_reports.bucket + terraform_role_arns = [data.aws_caller_identity.current.arn] + + backup_plan_config = { + "compliance_resource_types" : [ + "S3" + ], + "rules" : [ + { + "copy_action" : { + "delete_after" : 4 + }, + "lifecycle" : { + "delete_after" : 2 + }, + "name" : "daily_kept_for_2_days", + "schedule" : "cron(0 0 * * ? *)" + } + ], + "selection_tag" : "NHSE-Enable-Backup" + # The selection_tags are optional and can be used to + # provide fine grained resource selection with existing tagging + "selection_tags" : [ + { + "key" : "Environment" + "value" : "myenvironment" + } + ] + } # Note here that we need to explicitly disable DynamoDB and Aurora backups in the source account. # The default config in the module enables backups for all resource types. - backup_plan_config_dynamodb = { - "compliance_resource_types": [ - "DynamoDB" - ], - "rules": [ - ], - "enable": false, - "selection_tag": "NHSE-Enable-Backup" - } - - backup_plan_config_ebsvol = { - "compliance_resource_types": [ - "EBS" - ], - "rules": [ - ], - "enable": false, - "selection_tag": "NHSE-Enable-Backup" - } - - backup_plan_config_aurora = { - "compliance_resource_types": [ - "Aurora" - ], - "rules": [ - ], - "enable": false, - "selection_tag": "NHSE-Enable-Backup" - } + backup_plan_config_dynamodb = { + "compliance_resource_types" : [ + "DynamoDB" + ], + "rules" : [ + ], + "enable" : false, + "selection_tag" : "NHSE-Enable-Backup" + } + + backup_plan_config_ebsvol = { + "compliance_resource_types" : [ + "EBS" + ], + "rules" : [ + ], + "enable" : false, + "selection_tag" : "NHSE-Enable-Backup" + } + + backup_plan_config_aurora = { + "compliance_resource_types" : [ + "Aurora" + ], + "rules" : [ + ], + "enable" : false, + "selection_tag" : "NHSE-Enable-Backup" + } } diff --git a/examples/source/versions.tf b/examples/source/versions.tf new file mode 100644 index 0000000..49201ad --- /dev/null +++ b/examples/source/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + + aws = { + source = "hashicorp/aws" + version = "> 5" + } + + } + + required_version = ">= 1.9.5" +} diff --git a/modules/aws-backup-destination/variables.tf b/modules/aws-backup-destination/variables.tf index 6684b7b..b5f9672 100644 --- a/modules/aws-backup-destination/variables.tf +++ b/modules/aws-backup-destination/variables.tf @@ -20,8 +20,9 @@ variable "account_id" { type = string } +# tflint-ignore: terraform_unused_declarations variable "region" { - description = "The region we should be operating in" + description = "not currently used (deprecated)" type = string default = "eu-west-2" } diff --git a/modules/aws-backup-destination/versions.tf b/modules/aws-backup-destination/versions.tf new file mode 100644 index 0000000..49201ad --- /dev/null +++ b/modules/aws-backup-destination/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + + aws = { + source = "hashicorp/aws" + version = "> 5" + } + + } + + required_version = ">= 1.9.5" +} diff --git a/modules/aws-backup-source/locals.tf b/modules/aws-backup-source/locals.tf index 1941381..5361309 100644 --- a/modules/aws-backup-source/locals.tf +++ b/modules/aws-backup-source/locals.tf @@ -13,5 +13,5 @@ locals { var.backup_plan_config_aurora.enable ? [aws_backup_framework.aurora[0].arn] : [] )) aurora_overrides = var.backup_plan_config_aurora.restore_testing_overrides == null ? null : jsondecode(var.backup_plan_config_aurora.restore_testing_overrides) - terraform_role_arns = var.terraform_role_arns != [] ? var.terraform_role_arns : [var.terraform_role_arn] + terraform_role_arns = length(var.terraform_role_arns) > 0 ? var.terraform_role_arns : [var.terraform_role_arn] } diff --git a/scripts/check-secrets.sh b/scripts/check-secrets.sh new file mode 100755 index 0000000..551bc48 --- /dev/null +++ b/scripts/check-secrets.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +scan_type=${1-pre-commit} + +if ! git secrets -- 1> /dev/null; then + echo "git secrets is not installed" + echo "the git-secrets file needs to be in your PATH" + echo "to install:" + echo " wget https://raw.githubusercontent.com/awslabs/git-secrets/refs/heads/master/git-secrets -O ~/.local/bin/git-secrets && chmod +x ~/.local/bin/git-secrets" + exit 1 +fi + +echo "scan type: ${scan_type}" + +git secrets --register-aws +if [[ -e ./.gitdisallowed ]]; then + git secrets --add-provider -- grep -Ev '^(#.*|\s*$)' .gitdisallowed || true + git secrets --add --allowed '^\.gitdisallowed:[0-9]+:.*' || true +fi + +if { [ "${scan_type}" == "unstaged" ]; } ; then + echo "scanning staged and unstaged files for secrets" + git secrets --scan --recursive + git secrets --scan --untracked +elif { [ "${scan_type}" == "staged" ]; } ; then + echo "scanning staged files for secrets" + git secrets --scan --recursive +elif { [ "${scan_type}" == "commit-msg" ]; } ; then + echo "checking commit msg for secrets" + git secrets --commit_msg_hook -- "${2}" +elif { [ "${scan_type}" == "prep-commit-msg" ]; } ; then + echo "checking commit msg for secrets" + git secrets --prepare_commit_msg_hook -- "${2}" +else + echo "scanning for secrets" + # if staged files exist, this will scan staged files only, otherwise normal scan + git secrets --pre_commit_hook +fi