From c42e0f9888c70d157c0f76bc36db45c65ec43649 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 17 Feb 2022 22:16:00 -0500 Subject: [PATCH] ci: add do image build via packer --- .github/workflows/build.yml | 130 ++++++++++++---------- .github/workflows/packer.yml | 31 ++++++ dev/packer/digitalocean.json | 78 +++++++++++++ dev/packer/scripts/001-onboot.sh | 15 +++ dev/packer/scripts/010-docker.sh | 19 ++++ dev/packer/scripts/011-docker-compose.sh | 4 + dev/packer/scripts/012-grub-opts.sh | 6 + dev/packer/scripts/013-docker-dns.sh | 4 + dev/packer/scripts/014-ufw-docker.sh | 9 ++ dev/packer/scripts/020-application-tag.sh | 25 +++++ dev/packer/scripts/099-one-click | 21 ++++ dev/packer/scripts/900-cleanup.sh | 44 ++++++++ 12 files changed, 329 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/packer.yml create mode 100644 dev/packer/digitalocean.json create mode 100644 dev/packer/scripts/001-onboot.sh create mode 100644 dev/packer/scripts/010-docker.sh create mode 100644 dev/packer/scripts/011-docker-compose.sh create mode 100644 dev/packer/scripts/012-grub-opts.sh create mode 100644 dev/packer/scripts/013-docker-dns.sh create mode 100644 dev/packer/scripts/014-ufw-docker.sh create mode 100644 dev/packer/scripts/020-application-tag.sh create mode 100644 dev/packer/scripts/099-one-click create mode 100644 dev/packer/scripts/900-cleanup.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 908e7162..da05530b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ on: - main tags: - 'v*' - + env: BASE_DEV_VERSION: 2.5.0 @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: Set Build Variables run: | if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then @@ -32,7 +32,7 @@ jobs: echo "REL_VERSION=v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV fi - + - name: Disable DEV Flag + Set Version run: | sudo apt-get install jq -y @@ -40,20 +40,20 @@ jobs: jq --arg vs "$REL_VERSION_STRICT" -r '. + {dev:false, version:$vs}' pkg-temp.json > package.json rm pkg-temp.json cat package.json - + - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Build and push Docker images uses: docker/build-push-action@v2.9.0 with: @@ -65,7 +65,7 @@ jobs: requarks/wiki:canary-${{ env.REL_VERSION_STRICT }} ghcr.io/requarks/wiki:canary ghcr.io/requarks/wiki:canary-${{ env.REL_VERSION_STRICT }} - + - name: Extract compiled files run: | mkdir -p _dist @@ -81,12 +81,12 @@ jobs: with: name: drop path: wiki-js.tar.gz - + cypress: name: Run Cypress Tests runs-on: ubuntu-latest needs: [build] - + strategy: matrix: dbtype: [postgres, mysql, mariadb, mssql, sqlite] @@ -112,14 +112,14 @@ jobs: chmod u+x dev/cypress/ci-setup.sh dev/cypress/ci-setup.sh docker run --name cypress --ipc=host --shm-size 1G -v $GITHUB_WORKSPACE:/e2e -w /e2e cypress/included:4.9.0 --record --key "$CYPRESS_KEY" --headless --group "$MATRIXENV" --ci-build-id "$REL_VERSION_STRICT-run$GITHUB_RUN_NUMBER.$GITHUB_RUN_ATTEMPT" --tag "$REL_VERSION_STRICT" --config baseUrl=http://172.17.0.1:3000 - + arm: name: ARM Build runs-on: ubuntu-latest needs: [cypress] permissions: packages: write - + strategy: matrix: include: @@ -127,10 +127,10 @@ jobs: docker: arm64 - platform: linux/arm/v7 docker: armv7 - + steps: - uses: actions/checkout@v2 - + - name: Set Version Variables run: | if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then @@ -140,26 +140,26 @@ jobs: echo "Using BRANCH mode: v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV fi - + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - + - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Download a Build Artifact uses: actions/download-artifact@v2.1.0 with: @@ -170,7 +170,7 @@ jobs: run: | mkdir -p build tar -xzf $GITHUB_WORKSPACE/drop/wiki-js.tar.gz -C $GITHUB_WORKSPACE/build --exclude=node_modules - + - name: Build and push Docker images uses: docker/build-push-action@v2.9.0 with: @@ -181,12 +181,12 @@ jobs: tags: | requarks/wiki:canary-${{ matrix.docker }}-${{ env.REL_VERSION_STRICT }} ghcr.io/requarks/wiki:canary-${{ matrix.docker }}-${{ env.REL_VERSION_STRICT }} - + windows: name: Windows Build runs-on: windows-latest needs: [cypress] - + steps: - name: Setup Node.js environment uses: actions/setup-node@v2.5.1 @@ -207,16 +207,16 @@ jobs: - name: Install Dependencies run: yarn --production --frozen-lockfile --non-interactive working-directory: win - + - name: Create Bundle run: tar -czf wiki-js-windows.tar.gz -C $env:GITHUB_WORKSPACE\win . - + - name: Upload a Build Artifact uses: actions/upload-artifact@v2.3.1 with: name: drop-win path: wiki-js-windows.tar.gz - + beta: name: Publish Beta Images runs-on: ubuntu-latest @@ -224,31 +224,26 @@ jobs: needs: [build, arm, windows] permissions: packages: write - + steps: - name: Set Version Variables run: | - if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then - echo "Using TAG mode: $GITHUB_REF_NAME" - echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV - else - echo "Using BRANCH mode: v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" - echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV - fi + echo "Using TAG mode: $GITHUB_REF_NAME" + echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Create and Push Manifests run: | echo "Creating the manifests..." @@ -260,7 +255,7 @@ jobs: docker manifest push -p requarks/wiki:beta-$REL_VERSION_STRICT docker manifest push -p ghcr.io/requarks/wiki:beta-$REL_VERSION_STRICT - + release: name: Publish Release Images runs-on: ubuntu-latest @@ -270,31 +265,26 @@ jobs: permissions: packages: write contents: write - + steps: - name: Set Version Variables run: | - if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then - echo "Using TAG mode: $GITHUB_REF_NAME" - echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV - else - echo "Using BRANCH mode: v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" - echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV - fi + echo "Using TAG mode: $GITHUB_REF_NAME" + echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Create and Push Manifests run: | echo "Fetching semver tool..." @@ -327,19 +317,19 @@ jobs: docker manifest push -p ghcr.io/requarks/wiki:$MAJOR docker manifest push -p ghcr.io/requarks/wiki:$MAJORMINOR docker manifest push -p ghcr.io/requarks/wiki:latest - + - name: Download Linux Build uses: actions/download-artifact@v2.1.0 with: name: drop path: drop - + - name: Download Windows Build uses: actions/download-artifact@v2.1.0 with: name: drop-win path: drop-win - + - name: Generate Changelog id: changelog uses: Requarks/changelog-action@v1 @@ -357,4 +347,30 @@ jobs: body: ${{ steps.changelog.outputs.changes }} token: ${{ github.token }} artifacts: 'drop/wiki-js.tar.gz,drop-win/wiki-js-windows.tar.gz' - + + build-do-image: + name: Build DigitalOcean Image + runs-on: ubuntu-latest + needs: [release] + + steps: + - uses: actions/checkout@v2 + + - name: Set Version Variables + run: | + echo "Using TAG mode: $GITHUB_REF_NAME" + echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV + + - name: Install Packer + run: | + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - + sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + sudo apt-get update && sudo apt-get install packer + + - name: Build Droplet Image + env: + DIGITALOCEAN_API_TOKEN: ${{ secrets.DO_TOKEN }} + WIKI_APP_VERSION: ${{ env.REL_VERSION_STRICT }} + working-directory: dev/packer + run: | + packer build digitalocean.json diff --git a/.github/workflows/packer.yml b/.github/workflows/packer.yml new file mode 100644 index 00000000..4ef42f31 --- /dev/null +++ b/.github/workflows/packer.yml @@ -0,0 +1,31 @@ +name: Build DigitalOcean Image + +on: + workflow_dispatch: + inputs: + version: + description: 'App Version' + required: true + type: string + +jobs: + build-do-image: + name: Build DigitalOcean Image + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Install Packer + run: | + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - + sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + sudo apt-get update && sudo apt-get install packer + + - name: Build Droplet Image + env: + DIGITALOCEAN_API_TOKEN: ${{ secrets.DO_TOKEN }} + WIKI_APP_VERSION: ${{ github.event.inputs.version }} + working-directory: dev/packer + run: | + packer build digitalocean.json diff --git a/dev/packer/digitalocean.json b/dev/packer/digitalocean.json new file mode 100644 index 00000000..da0ccf67 --- /dev/null +++ b/dev/packer/digitalocean.json @@ -0,0 +1,78 @@ +{ + "variables": { + "do_api_token": "{{env `DIGITALOCEAN_API_TOKEN`}}", + "image_name": "wikijs-snapshot-{{timestamp}}", + "apt_packages": "apt-transport-https ca-certificates curl jq linux-image-extra-virtual software-properties-common gnupg-agent openssl ", + "application_name": "Wiki.js", + "application_version": "{{env `WIKI_APP_VERSION`}}", + "docker_compose_version": "1.29.2" + }, + "sensitive-variables": [ + "do_api_token" + ], + "builders": [ + { + "type": "digitalocean", + "api_token": "{{user `do_api_token`}}", + "image": "ubuntu-20-04-x64", + "region": "tor1", + "size": "s-1vcpu-1gb", + "ssh_username": "root", + "snapshot_name": "{{user `image_name`}}" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": [ + "cloud-init status --wait" + ] + }, + { + "type": "file", + "source": "scripts/001-onboot.sh", + "destination": "/var/lib/cloud/scripts/per-instance/001-onboot.sh" + }, + { + "type": "file", + "source": "scripts/099-one-click", + "destination": "/etc/update-motd.d/099-one-click" + }, + { + "type": "shell", + "environment_vars": [ + "DEBIAN_FRONTEND=noninteractive", + "LC_ALL=C", + "LANG=en_US.UTF-8", + "LC_CTYPE=en_US.UTF-8" + ], + "inline": [ + "apt -qqy update", + "apt -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' full-upgrade", + "apt -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' install {{user `apt_packages`}}", + "apt-get -qqy clean" + ] + }, + { + "type": "shell", + "environment_vars": [ + "application_name={{user `application_name`}}", + "application_version={{user `application_version`}}", + "docker_compose_version={{user `docker_compose_version`}}", + "DEBIAN_FRONTEND=noninteractive", + "LC_ALL=C", + "LANG=en_US.UTF-8", + "LC_CTYPE=en_US.UTF-8" + ], + "scripts": [ + "common/scripts/010-docker.sh", + "common/scripts/011-docker-compose.sh", + "common/scripts/012-grub-opts.sh", + "common/scripts/013-docker-dns.sh", + "common/scripts/014-ufw-docker.sh", + "common/scripts/020-application-tag.sh", + "common/scripts/900-cleanup.sh" + ] + } + ] +} diff --git a/dev/packer/scripts/001-onboot.sh b/dev/packer/scripts/001-onboot.sh new file mode 100644 index 00000000..2e4f9e4f --- /dev/null +++ b/dev/packer/scripts/001-onboot.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Scripts in this directory will be executed by cloud-init on the first boot of droplets +# created from your image. Things ike generating passwords, configuration requiring IP address +# or other items that will be unique to each instance should be done in scripts here. + +openssl rand -base64 32 > /etc/wiki/.db-secret + +if [[ -z $DATABASE_URL ]]; then + docker start db +fi +docker start wiki +docker start wiki-update-companion +# docker start nginx-proxy +# docker start watchtower diff --git a/dev/packer/scripts/010-docker.sh b/dev/packer/scripts/010-docker.sh new file mode 100644 index 00000000..54eab61b --- /dev/null +++ b/dev/packer/scripts/010-docker.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt -qqy update +apt -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' install docker-ce docker-ce-cli containerd.io + +systemctl enable docker +systemctl start docker + +mkdir -p /etc/wiki + +docker network create wikinet +docker volume create pgdata +docker create --name=db -e POSTGRES_DB=wiki -e POSTGRES_USER=wiki -e POSTGRES_PASSWORD_FILE=/etc/wiki/.db-secret -v /etc/wiki/.db-secret:/etc/wiki/.db-secret:ro -v pgdata:/var/lib/postgresql/data --restart=unless-stopped -h db --network=wikinet postgres:11 +docker create --name=wiki -e DB_TYPE=postgres -e DB_HOST=db -e DB_PORT=5432 -e DB_PASS_FILE=/etc/wiki/.db-secret -v /etc/wiki/.db-secret:/etc/wiki/.db-secret:ro -e DB_USER=wiki -e DB_NAME=wiki -e UPGRADE_COMPANION=1 --restart=unless-stopped -h wiki --network=wikinet -p 80:3000 -p 443:3443 ghcr.io/requarks/wiki:2 +docker create --name=wiki-update-companion -v /var/run/docker.sock:/var/run/docker.sock:ro --restart=unless-stopped -h wiki-update-companion --network=wikinet requarks/wiki-update-companion:latest +# docker create --name=nginx-proxy -p 80:80 -p 443:443 -e DEFAULT_HOST=wiki.local --network=wikinet -v /var/run/docker.sock:/tmp/docker.sock:ro --restart=unless-stopped jwilder/nginx-proxy +# docker create --name=watchtower --network=wikinet -v /var/run/docker.sock:/var/run/docker.sock --restart=unless-stopped containrrr/watchtower --cleanup --schedule="0 2 * * 6" wiki diff --git a/dev/packer/scripts/011-docker-compose.sh b/dev/packer/scripts/011-docker-compose.sh new file mode 100644 index 00000000..bc856865 --- /dev/null +++ b/dev/packer/scripts/011-docker-compose.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +sudo curl -L "https://github.com/docker/compose/releases/download/${docker_compose_version}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose; +chmod +x /usr/local/bin/docker-compose; diff --git a/dev/packer/scripts/012-grub-opts.sh b/dev/packer/scripts/012-grub-opts.sh new file mode 100644 index 00000000..9a336bb0 --- /dev/null +++ b/dev/packer/scripts/012-grub-opts.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +sed -e 's|GRUB_CMDLINE_LINUX="|GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1|g' \ + -i /etc/default/grub + +update-grub diff --git a/dev/packer/scripts/013-docker-dns.sh b/dev/packer/scripts/013-docker-dns.sh new file mode 100644 index 00000000..80fa11fe --- /dev/null +++ b/dev/packer/scripts/013-docker-dns.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +sed -e 's|#DOCKER_OPTS|DOCKER_OPTS|g' \ + -i /etc/default/docker diff --git a/dev/packer/scripts/014-ufw-docker.sh b/dev/packer/scripts/014-ufw-docker.sh new file mode 100644 index 00000000..48199458 --- /dev/null +++ b/dev/packer/scripts/014-ufw-docker.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +sudo ufw allow ssh +sudo ufw allow http +sudo ufw allow https + +sudo ufw --force enable + +cat /dev/null > /var/log/ufw.log diff --git a/dev/packer/scripts/020-application-tag.sh b/dev/packer/scripts/020-application-tag.sh new file mode 100644 index 00000000..93f8c559 --- /dev/null +++ b/dev/packer/scripts/020-application-tag.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +################################ +## PART: Write the application tag +## +## vi: syntax=sh expandtab ts=4 + +build_date=$(date +%Y-%m-%d) +distro="$(lsb_release -s -i)" +distro_release="$(lsb_release -s -r)" +distro_codename="$(lsb_release -s -c)" +distro_arch="$(uname -m)" + +mkdip -p /var/lib/digitalocean +touch /var/lib/digitalocean/application.info + +cat >> /var/lib/digitalocean/application.info < /root/.bash_history +unset HISTFILE +apt-get -y autoremove +apt-get -y autoclean +find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; +rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +rm -rf /var/lib/cloud/instances/* +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* +touch /etc/ssh/revoked_keys +chmod 600 /etc/ssh/revoked_keys + +# Securely erase the unused portion of the filesystem +GREEN='\033[0;32m' +NC='\033[0m' +printf "\n${GREEN}Writing zeros to the remaining disk space to securely +erase the unused portion of the file system. +Depending on your disk size this may take several minutes. +The secure erase will complete successfully when you see:${NC} + dd: writing to '/zerofile': No space left on device\n +Beginning secure erase now\n" + +dd if=/dev/zero of=/zerofile & + PID=$! + while [ -d /proc/$PID ] + do + printf "." + sleep 5 + done +sync; rm /zerofile; sync +cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp