From 419f6396c404e1339cb8982e3e4b9317715e2c0e Mon Sep 17 00:00:00 2001 From: Juan Jose Jaramillo Date: Wed, 29 Nov 2023 20:35:41 -0800 Subject: [PATCH] feat: Create reusable workflows --- .github/workflows/reusable-changelog.yml | 44 +++ .github/workflows/reusable-release-chart.yml | 65 +++++ .../reusable-release-integration.yml | 266 ++++++++++++++++++ .github/workflows/reusable-security.yaml | 75 +++++ .../workflows/reusable-trigger-prerelease.yml | 55 ++++ 5 files changed, 505 insertions(+) create mode 100644 .github/workflows/reusable-changelog.yml create mode 100644 .github/workflows/reusable-release-chart.yml create mode 100644 .github/workflows/reusable-release-integration.yml create mode 100644 .github/workflows/reusable-security.yaml create mode 100644 .github/workflows/reusable-trigger-prerelease.yml diff --git a/.github/workflows/reusable-changelog.yml b/.github/workflows/reusable-changelog.yml new file mode 100644 index 0000000..62c836e --- /dev/null +++ b/.github/workflows/reusable-changelog.yml @@ -0,0 +1,44 @@ +# This action requires that any PR should touch at +# least one CHANGELOG file. + +name: Reusable changelog + +on: + workflow_call: + +jobs: + changelog-entry: + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.labels.*.name, 'dependencies') && !contains(github.event.pull_request.labels.*.name, 'Skip Changelog') && !startsWith(github.head_ref, 'renovate/')}} + + steps: + - uses: actions/checkout@v4 + - name: Debug Labels + run: | + echo "${{ toJson(github.event.pull_request.labels[*].name) }}" + echo "Should Run: ${{ !contains(github.event.pull_request.labels.*.name, 'dependencies') && !contains(github.event.pull_request.labels.*.name, 'Skip Changelog') && !startsWith(github.head_ref, 'renovate/')}}" + + - name: Check for CHANGELOG file changes + run: | + # Only the latest commit of the feature branch is available + # automatically. To diff with the base branch, we need to + # fetch that too (and we only need its latest commit). + git fetch origin ${{ github.base_ref }} --depth=1 + echo "$(git diff --name-only FETCH_HEAD)" + if [[ $(git diff --name-only FETCH_HEAD | grep --ignore-case CHANGELOG.md) ]] + then + echo "The CHANGELOG file was modified. Looks good!" + else + echo "The CHANGELOG file was not modified." + echo "Please add a CHANGELOG entry to the appropriate header under \"Unreleased\", or add the \"Skip Changelog\" label if not required." + false + fi + + lint-changelog: + runs-on: ubuntu-latest + needs: changelog-entry + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Check if CHANGELOG is valid + uses: newrelic/release-toolkit/validate-markdown@v1 diff --git a/.github/workflows/reusable-release-chart.yml b/.github/workflows/reusable-release-chart.yml new file mode 100644 index 0000000..fa797ba --- /dev/null +++ b/.github/workflows/reusable-release-chart.yml @@ -0,0 +1,65 @@ +name: Reusable Release chart +on: + workflow_call: + secrets: + gh_token: + description: github token + required: true + slack_channel: + description: slack channel for notifications + required: true + slack_token: + description: slack token for slack channel + required: true + +env: + ORIGINAL_REPO_NAME: ${{ github.event.repository.full_name }} + +jobs: + # Sometimes chart-releaser might fetch an outdated index.yaml from gh-pages, causing a WAW hazard on the repo + # This job checks the remote file is up to date with the local one on release + validate-gh-pages-index: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: gh-pages + - name: Download remote index file and check equality + run: | + curl -vsSL https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/index.yaml > index.yaml.remote + LOCAL="$(md5sum < index.yaml)" + REMOTE="$(md5sum < index.yaml.remote)" + echo "$LOCAL" = "$REMOTE" + test "$LOCAL" = "$REMOTE" + chart-release: + runs-on: ubuntu-latest + needs: [ validate-gh-pages-index ] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + - name: Add newrelic repository + run: helm repo add newrelic https://helm-charts.newrelic.com + - name: Release workload charts + uses: helm/chart-releaser-action@v1.6.0 + env: + CR_SKIP_EXISTING: true + CR_TOKEN: ${{ secrets.gh_token }} + + notify-failure: + if: ${{ always() && failure() }} + needs: [validate-gh-pages-index, chart-release] + runs-on: ubuntu-latest + steps: + - name: Notify failure via Slack + uses: archive/github-actions-slack@cb6f1f6bc7bfe991ea956833f9515dac40da14d2 # v2.8.0 + with: + slack-bot-user-oauth-access-token: ${{ secrets.slack_token }} + slack-channel: ${{ secrets.slack_channel }} + slack-text: "❌ `${{ env.ORIGINAL_REPO_NAME }}`: <${{ github.server_url }}/${{ env.ORIGINAL_REPO_NAME }}/actions/runs/${{ github.run_id }}|'Release chart' failed>." diff --git a/.github/workflows/reusable-release-integration.yml b/.github/workflows/reusable-release-integration.yml new file mode 100644 index 0000000..df1ab24 --- /dev/null +++ b/.github/workflows/reusable-release-integration.yml @@ -0,0 +1,266 @@ +name: Reusable Pre-release and Release pipeline + +on: + workflow_call: + inputs: + repo_name: + description: Name of the repo + type: string + required: true + artifact_path: + description: The artifact path + type: string + required: false + enable_helm_chart_release: + description: Whether the release workflow should trigger a helm chart release or not + type: boolean + required: false + default: true + docker_image_name: + description: Docker image name + type: string + required: true + chart_directory: + description: Location of Chart + type: string + required: true + # Usually key is .appVersion + image_name_key: + description: Image name key in chart + type: string + required: false + default: .appVersion + # secrets need to be passed in for reusable workflows + secrets: + dockerhub_username: + description: dockerhub username + required: true + dockerhub_token: + description: dockerhub token + required: true + bot_token: + description: team specific bot token + required: true + slack_channel: + description: slack channel for notifications + required: true + slack_token: + description: slack token for slack channel + required: true + +env: + ORIGINAL_REPO_NAME: ${{ github.event.repository.full_name }} + +jobs: + build: + name: Build integration for + runs-on: ubuntu-latest + strategy: + matrix: + goos: [ linux ] + goarch: [ amd64, arm64, arm ] + steps: + - name: Build env args + run: | + echo "${{ github.event.release.tag_name }}" | grep -E '^[v]?[0-9.]*[0-9]$' + DOCKER_IMAGE_TAG=$(echo "${{ github.event.release.tag_name }}" | sed 's/^v//') + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG" >> $GITHUB_ENV + echo "DATE=`date`" >> $GITHUB_ENV + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + - name: Build integration + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + COMMIT: ${{ github.sha }} + DATE: ${{ env.DATE }} + TAG: ${{ env.DOCKER_IMAGE_TAG }} + run: | + make compile + - name: Upload artifact for docker build step + uses: actions/upload-artifact@v3 + with: + retention-days: 1 + name: ${{ inputs.repo_name }}-${{ matrix.goos }}-${{ matrix.goarch }} + path: ${{ inputs.artifact_path }}${{ inputs.repo_name }}-${{ matrix.goos }}-${{ matrix.goarch }} + + docker-integration: + name: Release docker + needs: [ build ] + runs-on: ubuntu-latest + outputs: + new-version: ${{ steps.set-new-version.outputs.new-version }} + env: + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} + DOCKER_PLATFORMS: "linux/amd64,linux/arm64,linux/arm" # Must be consistent with the matrix from the job above + COMMIT: ${{ github.sha }} + steps: + - name: Generate docker image version from git tag + id: set-new-version + run: | + echo "${{ github.event.release.tag_name }}" | grep -E '^[v]?[0-9.]*[0-9]$' + DOCKER_IMAGE_TAG=$(echo "${{ github.event.release.tag_name }}" | sed 's/^v//') + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG" >> $GITHUB_ENV + echo "DATE=`date`" >> $GITHUB_ENV + echo "new-version=$DOCKER_IMAGE_TAG" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Download all artifacts from build job with bin path + if: ${{ inputs.artifact_path }} + uses: actions/download-artifact@v3 + with: + path: bin + - name: Download all artifacts from build job without bin path + if: ${{ ! inputs.artifact_path }} + uses: actions/download-artifact@v3 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.dockerhub_username }} + password: ${{ secrets.dockerhub_token }} + - name: List files + run: ls -la + - name: Build and load x64 image for security scanning + # We need to build a single-arch image again to be able to --load it into the host + run: | + docker buildx build --load --platform=linux/amd64 \ + -t $DOCKER_IMAGE_NAME:ci-scan \ + . + - name: Build and push docker prerelease image + if: ${{ github.event.release.prerelease }} + run: | + DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG}-pre + docker buildx build --push --platform=$DOCKER_PLATFORMS \ + --build-arg "COMMIT=$COMMIT" \ + --build-arg "DATE=$DATE" \ + --build-arg "TAG=$DOCKER_IMAGE_TAG" \ + -t $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG \ + . + - name: Build and push docker release image + if: ${{ ! github.event.release.prerelease }} + run: | + docker buildx build --push --platform=$DOCKER_PLATFORMS \ + --build-arg "COMMIT=$COMMIT" \ + --build-arg "DATE=$DATE" \ + --build-arg "TAG=$DOCKER_IMAGE_TAG" \ + -t $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG \ + -t $DOCKER_IMAGE_NAME:latest \ + . + open-pr: + name: Update version and appVersion and open pr + needs: [ docker-integration ] + runs-on: ubuntu-latest + # run only for releases (not prereleases) + if: ${{ ! github.event.release.prerelease && inputs.enable_helm_chart_release }} + steps: + - name: Checkout original repo + uses: actions/checkout@v4 + with: + repository: ${{ env.ORIGINAL_REPO_NAME }} + ref: main + + - name: Find new appVersion + id: find-version + run: | + echo "NEW_APP_VERSION=${{ needs.docker-integration.outputs.new-version }}" >> $GITHUB_ENV + echo "new app version: $NEW_APP_VERSION" + + - name: Find current appVersion + id: original_version + run: | + ORIGINAL_APP_VERSION=$(yq eval ${{ inputs.image_name_key }} ${{ inputs.chart_directory }}/Chart.yaml) + echo "original app version: $ORIGINAL_APP_VERSION" + echo "ORIGINAL_APP_VERSION=$ORIGINAL_APP_VERSION" >> $GITHUB_ENV + + - name: Find current helm chart version + run: | + CURRENT_VERSION=$(yq eval '.version' ${{ inputs.chart_directory }}/Chart.yaml) + echo "version: $CURRENT_VERSION" + echo "CURRENT_VERSION=$CURRENT_VERSION" >> $GITHUB_ENV + + - name: Set up Golang + uses: actions/setup-go@v4 + with: + go-version: 1.19.11 + + - name: Checkout version-update.go app + uses: actions/checkout@v4 + with: + repository: newrelic/k8s-metadata-injection + path: tools + ref: main + sparse-checkout: | + version-update.go + sparse-checkout-cone-mode: false + + - name: List files + run: ls -la + + - name: Find next helm chart version + run: | + NEXT_VERSION=$(go run ./tools/src/utils/version-update.go "$CURRENT_VERSION" "$ORIGINAL_APP_VERSION" "$NEW_APP_VERSION") + echo "Next helm chart version: $NEXT_VERSION" + echo "NEXT_VERSION=$NEXT_VERSION" >> $GITHUB_ENV + + - name: Update version helm chart + # fail the workflow if newVersion is "error", otherwise set the new versions and continue with opening pr + run: | + if [ "${NEXT_VERSION}" != 'error' ]; then + echo "new appVersion to set: ${NEW_APP_VERSION}" + echo "new version to set: ${NEXT_VERSION}" + yq eval --inplace "${{ inputs.image_name_key }}=\"${NEW_APP_VERSION}\"" "${{ inputs.chart_directory }}/Chart.yaml" + yq eval --inplace ".version=\"${NEXT_VERSION}\"" "${{ inputs.chart_directory }}/Chart.yaml" + else + echo "Error: newVersion is 'error'." + exit 1 + fi + + - name: Install Helm Docs + run: | + wget https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Linux_x86_64.tar.gz + tar --extract --verbose --file helm-docs_1.11.0_Linux_x86_64.tar.gz + sudo mv helm-docs /usr/local/sbin + + - name: Run Helm Docs + run: | + helm-docs + + - name: Configure Git + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + + - name: Commit Changes + run: | + git checkout -b update-chart-version-${{ github.sha }} + git branch --all + git add ${{ inputs.chart_directory }}/Chart.yaml + git add ${{ inputs.chart_directory }}/README.md + git commit --message="Bump versions and update docs" + + - name: Push Changes + run: git push origin update-chart-version-${{ github.sha }} + + - name: Open pull request + run: | + pr_url=$(gh pr create -B main -H update-chart-version-${{ github.sha }} --label "Skip Changelog" --title 'Bump version and update docs' --body 'Bump version and appVersion and results of running helm docs as part of release automation.') + pr_number=$(basename $pr_url) + gh pr merge $pr_number --squash --admin --delete-branch --body "Merged by k8s agent bot." + env: + GITHUB_TOKEN: ${{ secrets.bot_token }} + + notify-failure: + if: ${{ always() && failure() }} + needs: [docker-integration, open-pr] + runs-on: ubuntu-latest + steps: + - name: Notify failure via Slack + uses: archive/github-actions-slack@cb6f1f6bc7bfe991ea956833f9515dac40da14d2 # v2.8.0 + with: + slack-bot-user-oauth-access-token: ${{ secrets.slack_token }} + slack-channel: ${{ secrets.slack_channel }} + slack-text: "❌ `${{ env.ORIGINAL_REPO_NAME }}`: <${{ github.server_url }}/${{ env.ORIGINAL_REPO_NAME }}/actions/runs/${{ github.run_id }}|'Pre-release and Release pipeline' failed>." diff --git a/.github/workflows/reusable-security.yaml b/.github/workflows/reusable-security.yaml new file mode 100644 index 0000000..cb78987 --- /dev/null +++ b/.github/workflows/reusable-security.yaml @@ -0,0 +1,75 @@ +name: Security Scan + +on: + workflow_call: + # secrets need to be passed in for reusable workflows + secrets: + slack_channel: + description: slack channel for notifications + required: true + slack_token: + description: slack token for slack channel + required: true + +env: + ORIGINAL_REPO_NAME: ${{ github.event.repository.full_name }} + +jobs: + trivy: + name: Trivy security scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@0.14.0 + if: ${{ ! github.event.schedule }} # Do not run inline checks when running periodically + with: + scan-type: fs + ignore-unfixed: true + severity: 'HIGH,CRITICAL' + skip-dirs: 'build' + format: 'table' + + - name: Build integration + env: + GOOS: linux + GOARCH: amd64 + run: | + make compile + + - name: Build and load x64 image + run: | + docker buildx build --load --platform=linux/amd64 --tag trivy-scan:${{ github.sha }} . + + - name: Run Trivy vulnerability scanner sarif output + uses: aquasecurity/trivy-action@0.14.0 + # Upload sarif when running periodically or pushing to main + if: ${{ github.event.schedule || (github.event_name == 'push' && github.ref_name == 'main') }} + with: + image-ref: 'trivy-scan:${{ github.sha }}' + ignore-unfixed: true + severity: 'HIGH,CRITICAL' + skip-dirs: 'build' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + # Upload sarif when running periodically or pushing to main + if: ${{ github.event.schedule || (github.event_name == 'push' && github.ref_name == 'main') }} + with: + sarif_file: 'trivy-results.sarif' + + notify-failure: + if: ${{ always() && failure() && (github.event_name == 'schedule') }} + needs: [trivy] + runs-on: ubuntu-latest + steps: + - name: Notify failure via Slack + uses: archive/github-actions-slack@cb6f1f6bc7bfe991ea956833f9515dac40da14d2 # v2.8.0 + with: + slack-bot-user-oauth-access-token: ${{ secrets.slack_token }} + slack-channel: ${{ secrets.slack_channel }} + slack-text: "❌ `${{ env.ORIGINAL_REPO_NAME }}`: <${{ github.server_url }}/${{ env.ORIGINAL_REPO_NAME }}/actions/runs/${{ github.run_id }}|'Security Scan' failed>." diff --git a/.github/workflows/reusable-trigger-prerelease.yml b/.github/workflows/reusable-trigger-prerelease.yml new file mode 100644 index 0000000..08780fb --- /dev/null +++ b/.github/workflows/reusable-trigger-prerelease.yml @@ -0,0 +1,55 @@ +name: Reusable Trigger Prerelease Creation + +# This workflow triggers a prerelease creation with changelog and the release notes created by the release toolkit. +# This workflow should be triggered merely from the default branch. +# For more details about how to release follow https://github.com/newrelic/coreint-automation/blob/main/docs/release_runbook.md + +on: + workflow_call: + inputs: + bot_email: + description: bot email + type: string + required: true + bot_name: + description: bot name + type: string + required: true + # secrets need to be passed in for reusable workflows + secrets: + bot_token: + description: bot token + required: true + slack_channel: + description: slack channel for notifications + required: true + slack_token: + description: slack token for slack channel + required: true + +env: + ORIGINAL_REPO_NAME: ${{ github.event.repository.full_name }} + +jobs: + prerelease: + uses: newrelic/coreint-automation/.github/workflows/trigger_prerelease.yaml@v1 + with: + rt-included-files: go.mod,go.sum,Dockerfile + bot_email: ${{ inputs.bot_email }} + bot_name: ${{ inputs.bot_name }} + secrets: + bot_token: ${{ secrets.bot_token }} + slack_channel: ${{ secrets.slack_channel }} + slack_token: ${{ secrets.slack_token }} + + notify-failure: + if: ${{ always() && failure() }} + needs: [prerelease] + runs-on: ubuntu-latest + steps: + - name: Notify failure via Slack + uses: archive/github-actions-slack@cb6f1f6bc7bfe991ea956833f9515dac40da14d2 # v2.8.0 + with: + slack-bot-user-oauth-access-token: ${{ secrets.slack_token }} + slack-channel: ${{ secrets.slack_channel }} + slack-text: "❌ `${{ env.ORIGINAL_REPO_NAME }}`: <${{ github.server_url }}/${{ env.ORIGINAL_REPO_NAME }}/actions/runs/${{ github.run_id }}|'Trigger prerelease creation' failed>."