From 843b8b10f1b967c27b725d8c489424cb0a208c5b Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sat, 21 Aug 2021 13:42:17 +0100 Subject: [PATCH] docker builds (#805) * add base * add other images * fix layer refs * fix indent * make two jobs in one workflow * further work on logic * work for branches and tags * try different env use * better debug names * fix shell quoting * rename --- .github/workflows/docker.yml | 74 -------- .github/workflows/runtests.yml | 113 ----------- .github/workflows/test_and_publish.yml | 253 +++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 187 deletions(-) delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/runtests.yml create mode 100644 .github/workflows/test_and_publish.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 1ebef219..00000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Build Docker Images -on: - workflow_dispatch: - inputs: - debug_enabled: - description: 'Run debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' - required: false - default: false -# success_irc_squawk: -# description: 'Squawk to IRC on success' -# required: false -# default: false - push: - tags: - - '2.[0-9][0-9][0-9][0-9][0-9][0-9]' -jobs: - build_docker_images: - name: Build Docker Images - runs-on: ubuntu-latest - defaults: - run: - working-directory: /home/runner/work/netdisco/netdisco-docker - steps: - - name: Check out latest code - uses: actions/checkout@v1 - with: - repository: 'netdisco/netdisco-docker' - ref: 'master' - - name: Get the Tag - id: get_refs - run: | - echo "TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - echo "IMAGE_ROOT=netdisco:${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - - name: Build netdisco-postgresql - id: build_netdisco_postgresql - run: | - cd netdisco-postgresql - docker build -t ${IMAGE_ROOT}-postgresql \ - --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ - --build-arg TAG=${TAG} \ - --build-arg COMMITTISH=${TAG} . - - - name: Retag all images - run: | - for image in postgresql; do - docker tag ${IMAGE_ROOT}-$image netdisco/${IMAGE_ROOT}-$image - docker tag ${IMAGE_ROOT}-$image netdisco/netdisco:latest-$image - done - -# - name: IRC upload failure notification -# uses: Gottox/irc-message-action@v2 -# if: (! github.event.inputs.debug_enabled) && startsWith(github.ref, 'refs/tags/') -# with: -# channel: '#netdisco' -# nickname: github-actions -# message: |- -# 🤖 Failed to upload Netdisco release ${{ steps.get_refs.outputs.RELEASE_TAG }} to CPAN! 😭 -# 👀 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} -# - name: IRC upload success notification -# uses: Gottox/irc-message-action@v2 -# if: (! github.event.inputs.debug_enabled) && startsWith(github.ref, 'refs/tags/') -# with: -# channel: '#netdisco' -# nickname: github-actions -# message: |- -# 🤖 Uploaded Netdisco release ${{ steps.get_refs.outputs.RELEASE_TAG }} to CPAN 🎉 - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: always() && github.event.inputs.debug_enabled && (github.event_name == 'workflow_dispatch') - with: - sudo: true - diff --git a/.github/workflows/runtests.yml b/.github/workflows/runtests.yml deleted file mode 100644 index 9d74749d..00000000 --- a/.github/workflows/runtests.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Test Netdisco -on: - workflow_dispatch: - inputs: - debug_enabled: - description: 'Run debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' - required: false - default: false - success_irc_squawk: - description: 'Squawk to IRC on successful tests' - required: false - default: false - push: - branches: - - master - tags: - - '2.[0-9][0-9][0-9][0-9][0-9][0-9]' -jobs: - test_netdisco: - name: Test and CPAN Upload - runs-on: ubuntu-latest - container: - image: 'netdisco/netdisco:latest-do' - options: '--user root --entrypoint /bin/ash' - volumes: - - '/home/runner/work:/github/workspace' - defaults: - run: - working-directory: /github/workspace/netdisco/netdisco - steps: - - name: Get the Tag and Branch - id: get_refs - run: | - echo ::set-output name=RELEASE_TAG::${GITHUB_REF#refs/tags/} - echo ::set-output name=RELEASE_BRANCH::${GITHUB_REF#refs/heads/} - - - name: Install packages - run: apk add openssh xz tmux bash curl sudo gcc make musl-dev perl-dev unzip - - name: Install fake apt-get - run: echo 'if [ "$1" == "update" ]; then exec apk update; else exec apk add openssh xz; fi' > /usr/local/bin/apt-get && chmod +x /usr/local/bin/apt-get - - name: Add localhost - run: echo "127.0.0.1 localhost" >> /etc/hosts - - - name: Check out latest code - uses: actions/checkout@v1 - - name: Fix owner of checkout - run: chown -R netdisco:netdisco /github/workspace/netdisco/netdisco - - - name: Install Perl deps - run: | - sudo -u netdisco /home/netdisco/bin/localenv cpanm --notest Env::Path Test::Compile CPAN::Uploader - sudo -u netdisco /home/netdisco/bin/localenv curl -L https://raw.githubusercontent.com/rjbs/CPAN-Uploader/4dd4fbad6bc96257c53a50ea2761932795dddf5c/lib/CPAN/Uploader.pm > /home/netdisco/perl5/lib/perl5/CPAN/Uploader.pm - - - name: Run Tests - id: build_and_run_tests - run: | - sudo -u netdisco /home/netdisco/bin/localenv perl ./Build.PL - sudo -u netdisco /home/netdisco/bin/localenv ./Build test --test_files xt/ - continue-on-error: true - - - name: IRC test failure notification - uses: Gottox/irc-message-action@v2 - if: (! github.event.inputs.debug_enabled) && startsWith(github.ref, 'refs/heads/') && (steps.build_and_run_tests.outcome == 'failure') - with: - channel: '#netdisco' - nickname: github-actions - message: |- - 🤖 ${{ github.actor }} pushed to ${{ steps.get_refs.outputs.RELEASE_BRANCH }}: https://github.com/${{ github.repository }}/commit/${{ github.sha }} - 😭 TESTS have FAILED! 👀 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: IRC test success notification - uses: Gottox/irc-message-action@v2 - if: github.event.inputs.success_irc_squawk && startsWith(github.ref, 'refs/heads/') && (steps.build_and_run_tests.outcome == 'success') - with: - channel: '#netdisco' - nickname: github-actions - message: |- - 🤖 ${{ github.actor }} pushed to ${{ steps.get_refs.outputs.RELEASE_BRANCH }}: https://github.com/${{ github.repository }}/commit/${{ github.sha }} (tests PASSED 🎉) - - - name: Upload dist to CPAN - id: upload_to_cpan - if: (! github.event.inputs.debug_enabled) && startsWith(github.ref, 'refs/tags/') && (steps.build_and_run_tests.outcome == 'success') - env: - PAUSE_USERNAME: ${{ secrets.PAUSE_USERNAME }} - PAUSE_PASSWORD: ${{ secrets.PAUSE_PASSWORD }} - run: | - sudo -u netdisco /home/netdisco/bin/localenv ./Build dist - sudo -u netdisco /home/netdisco/bin/localenv cpan-upload --user=$PAUSE_USERNAME --password=$PAUSE_PASSWORD App-Netdisco-*.tar.gz - continue-on-error: true - - - name: IRC upload failure notification - uses: Gottox/irc-message-action@v2 - if: (! github.event.inputs.debug_enabled) && startsWith(github.ref, 'refs/tags/') && (steps.upload_to_cpan.outcome == 'failure') - with: - channel: '#netdisco' - nickname: github-actions - message: |- - 🤖 Failed to upload Netdisco release ${{ steps.get_refs.outputs.RELEASE_TAG }} to CPAN! 😭 - 👀 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - name: IRC upload success notification - uses: Gottox/irc-message-action@v2 - if: (! github.event.inputs.debug_enabled) && startsWith(github.ref, 'refs/tags/') && (steps.upload_to_cpan.outcome == 'success') - with: - channel: '#netdisco' - nickname: github-actions - message: |- - 🤖 Uploaded Netdisco release ${{ steps.get_refs.outputs.RELEASE_TAG }} to CPAN 🎉 - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: always() && github.event.inputs.debug_enabled && (github.event_name == 'workflow_dispatch') - with: - sudo: true - diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml new file mode 100644 index 00000000..0618de61 --- /dev/null +++ b/.github/workflows/test_and_publish.yml @@ -0,0 +1,253 @@ +name: Test and Publish Netdisco +on: + workflow_dispatch: + inputs: + success_irc_squawk: + description: 'Squawk to IRC on successful tests' + required: false + default: false + debug_test_enabled: + description: 'With build and test debug' + required: false + default: false + debug_docker_enabled: + description: 'With docker debug' + required: false + default: false + push: + branches: + - master + tags: + - '2.[0-9][0-9][0-9][0-9][0-9][0-9]' +jobs: + test_netdisco: + name: Test and CPAN Upload + runs-on: ubuntu-latest + container: + image: 'netdisco/netdisco:latest-do' + options: '--user root --entrypoint /bin/ash' + volumes: + - '/home/runner/work:/github/workspace' + defaults: + run: + working-directory: /github/workspace/netdisco/netdisco + steps: + - name: Get the Tag and Branch + id: get_refs + run: | + echo ::set-output name=RELEASE_TAG::${GITHUB_REF#refs/tags/} + echo ::set-output name=RELEASE_BRANCH::${GITHUB_REF#refs/heads/} + + - name: Install packages + run: apk add openssh xz tmux bash curl sudo gcc make musl-dev perl-dev unzip + - name: Install fake apt-get + run: echo 'if [ "$1" == "update" ]; then exec apk update; else exec apk add openssh xz; fi' > /usr/local/bin/apt-get && chmod +x /usr/local/bin/apt-get + - name: Add localhost + run: echo "127.0.0.1 localhost" >> /etc/hosts + + - name: Check out latest code + uses: actions/checkout@v1 + - name: Fix owner of checkout + run: chown -R netdisco:netdisco /github/workspace/netdisco/netdisco + + - name: Install Perl deps + run: | + sudo -u netdisco /home/netdisco/bin/localenv cpanm --notest Env::Path Test::Compile CPAN::Uploader + sudo -u netdisco /home/netdisco/bin/localenv curl -L https://raw.githubusercontent.com/rjbs/CPAN-Uploader/4dd4fbad6bc96257c53a50ea2761932795dddf5c/lib/CPAN/Uploader.pm > /home/netdisco/perl5/lib/perl5/CPAN/Uploader.pm + + - name: Run Tests + id: build_and_run_tests + run: | + sudo -u netdisco /home/netdisco/bin/localenv perl ./Build.PL + sudo -u netdisco /home/netdisco/bin/localenv ./Build test --test_files xt/ + continue-on-error: true + + - name: IRC test failure notification + uses: Gottox/irc-message-action@v2 + if: (steps.build_and_run_tests.outcome == 'failure') && (github.event_name != 'workflow_dispatch') + with: + channel: '#netdisco' + nickname: github-actions + message: |- + 🤖 ${{ github.actor }} pushed to ${{ steps.get_refs.outputs.RELEASE_BRANCH }}: https://github.com/${{ github.repository }}/commit/${{ github.sha }} + 😭 TESTS have FAILED! 👀 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: IRC test success notification + uses: Gottox/irc-message-action@v2 + if: (steps.build_and_run_tests.outcome == 'success') && github.event.inputs.success_irc_squawk + with: + channel: '#netdisco' + nickname: github-actions + message: |- + 🤖 ${{ github.actor }} pushed to ${{ steps.get_refs.outputs.RELEASE_BRANCH }}: https://github.com/${{ github.repository }}/commit/${{ github.sha }} (tests PASSED 🎉) + + - name: Upload dist to CPAN + id: upload_to_cpan + if: (steps.build_and_run_tests.outcome == 'success') && startsWith(github.ref, 'refs/tags/') + env: + PAUSE_USERNAME: ${{ secrets.PAUSE_USERNAME }} + PAUSE_PASSWORD: ${{ secrets.PAUSE_PASSWORD }} + run: | + sudo -u netdisco /home/netdisco/bin/localenv ./Build dist + sudo -u netdisco /home/netdisco/bin/localenv cpan-upload --user=$PAUSE_USERNAME --password=$PAUSE_PASSWORD App-Netdisco-*.tar.gz + continue-on-error: true + + - name: IRC upload failure notification + uses: Gottox/irc-message-action@v2 + if: steps.upload_to_cpan.outcome == 'failure' + with: + channel: '#netdisco' + nickname: github-actions + message: |- + 🤖 Failed to upload Netdisco release ${{ steps.get_refs.outputs.RELEASE_TAG }} to CPAN! 😭 + 👀 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: IRC upload success notification + uses: Gottox/irc-message-action@v2 + if: steps.upload_to_cpan.outcome == 'success' + with: + channel: '#netdisco' + nickname: github-actions + message: |- + 🤖 Uploaded Netdisco release ${{ steps.get_refs.outputs.RELEASE_TAG }} to CPAN 🎉 + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: always() && github.event.inputs.debug_test_enabled + with: + sudo: true + + - name: Fix job exit status + if: always() && (steps.build_and_run_tests.outcome == 'failure') + run: exit 1 + + build_docker_images: + name: Build Docker Images + needs: test_netdisco + if: (github.event_name == 'workflow_dispatch') || startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + defaults: + run: + working-directory: /home/runner/work/netdisco/netdisco-docker + steps: + - name: Check out latest code + uses: actions/checkout@v1 + with: + repository: 'netdisco/netdisco-docker' + ref: 'master' + + - name: Get the Tag + run: | + echo "TAG=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + echo "IMAGE_ROOT=netdisco:$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV + + - name: Build netdisco-postgresql + run: | + cd netdisco-postgresql + docker build -t ${IMAGE_ROOT}-postgresql \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg TAG=${TAG} \ + --build-arg COMMITTISH=${TAG} . + docker tag ${IMAGE_ROOT}-postgresql netdisco/${IMAGE_ROOT}-postgresql + + - name: Build netdisco-base + run: | + cd netdisco-base + docker build -t ${IMAGE_ROOT}-base \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg TAG=${TAG} \ + --build-arg COMMITTISH=${TAG} . + docker tag ${IMAGE_ROOT}-base netdisco/${IMAGE_ROOT}-base + + - name: Build netdisco-backend + run: | + cd netdisco-backend + docker build -t ${IMAGE_ROOT}-backend \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg TAG=${TAG} \ + --build-arg COMMITTISH=${TAG} . + docker tag ${IMAGE_ROOT}-backend netdisco/${IMAGE_ROOT}-backend + + - name: Build netdisco-web + run: | + cd netdisco-web + docker build -t ${IMAGE_ROOT}-web \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg TAG=${TAG} \ + --build-arg COMMITTISH=${TAG} . + docker tag ${IMAGE_ROOT}-web netdisco/${IMAGE_ROOT}-web + + - name: Build netdisco-do + run: | + cd netdisco-do + docker build -t ${IMAGE_ROOT}-do \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg TAG=${TAG} \ + --build-arg COMMITTISH=${TAG} . + docker tag ${IMAGE_ROOT}-do netdisco/${IMAGE_ROOT}-do + + - name: Tag all images latest + run: | + for image in postgresql backend web do; do + docker tag ${IMAGE_ROOT}-$image netdisco/netdisco:latest-$image + done + + - name: Login to DH + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo '${{ env.DOCKER_PASSWORD }}' | docker login -u '${{ env.DOCKER_USERNAME }}' --password-stdin + + - name: Upload to Docker Hub + id: upload_docker_hub + if: startsWith(github.ref, 'refs/tags/') + run: | + for image in postgresql backend web do; do + docker push netdisco/${IMAGE_ROOT}-$image + docker push netdisco/netdisco:latest-$image + done + continue-on-error: true + + - name: Logout from DH + run: docker logout + + - name: Login to GHP + run: | + echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u '${{ github.actor }}' --password-stdin + + - name: Upload to GitHub Packages + id: upload_github_packages + if: startsWith(github.ref, 'refs/tags/') + run: | + for image in postgresql backend web do; do + docker push netdisco/${IMAGE_ROOT}-$image + docker push netdisco/netdisco:latest-$image + done + continue-on-error: true + + - name: Logout from GHP + run: docker logout + + - name: IRC DH upload failure notification + uses: Gottox/irc-message-action@v2 + if: steps.upload_docker_hub.outcome == 'failure' + with: + channel: '#netdisco' + nickname: github-actions + message: |- + 🤖 Failed to publish containers (release $TAG) 😭 + 👀 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: IRC DH upload success notification + uses: Gottox/irc-message-action@v2 + if: steps.upload_docker_hub.outcome == 'success' + with: + channel: '#netdisco' + nickname: github-actions + message: |- + 🤖 Published containers (release: $TAG) 🎉 + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: always() && github.event.inputs.debug_docker_enabled + with: + sudo: true +