ci: add do image build via packer

.github/workflows/build.yml

@@ -6,7 +6,7 @@ on:
       - main
     tags:
       - 'v*'
-      
+
 env:
   BASE_DEV_VERSION: 2.5.0
 
@@ -20,7 +20,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    
+
     - name: Set Build Variables
       run: |
         if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then
@@ -32,7 +32,7 @@ jobs:
           echo "REL_VERSION=v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV
           echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV
         fi
-    
+
     - name: Disable DEV Flag + Set Version
       run: |
         sudo apt-get install jq -y
@@ -40,20 +40,20 @@ jobs:
         jq --arg vs "$REL_VERSION_STRICT" -r '. + {dev:false, version:$vs}' pkg-temp.json > package.json
         rm pkg-temp.json
         cat package.json
-    
+
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    
+
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    
+
     - name: Build and push Docker images
       uses: docker/build-push-action@v2.9.0
       with:
@@ -65,7 +65,7 @@ jobs:
           requarks/wiki:canary-${{ env.REL_VERSION_STRICT }}
           ghcr.io/requarks/wiki:canary
           ghcr.io/requarks/wiki:canary-${{ env.REL_VERSION_STRICT }}
-          
+
     - name: Extract compiled files
       run: |
         mkdir -p _dist
@@ -81,12 +81,12 @@ jobs:
       with:
         name: drop
         path: wiki-js.tar.gz
-        
+
   cypress:
     name: Run Cypress Tests
     runs-on: ubuntu-latest
     needs: [build]
-    
+
     strategy:
       matrix:
         dbtype: [postgres, mysql, mariadb, mssql, sqlite]
@@ -112,14 +112,14 @@ jobs:
         chmod u+x dev/cypress/ci-setup.sh
         dev/cypress/ci-setup.sh
         docker run --name cypress --ipc=host --shm-size 1G -v $GITHUB_WORKSPACE:/e2e -w /e2e cypress/included:4.9.0 --record --key "$CYPRESS_KEY" --headless --group "$MATRIXENV" --ci-build-id "$REL_VERSION_STRICT-run$GITHUB_RUN_NUMBER.$GITHUB_RUN_ATTEMPT" --tag "$REL_VERSION_STRICT" --config baseUrl=http://172.17.0.1:3000
-          
+
   arm:
     name: ARM Build
     runs-on: ubuntu-latest
     needs: [cypress]
     permissions:
       packages: write
-      
+
     strategy:
       matrix:
         include:
@@ -127,10 +127,10 @@ jobs:
             docker: arm64
           - platform: linux/arm/v7
             docker: armv7
-    
+
     steps:
     - uses: actions/checkout@v2
-    
+
     - name: Set Version Variables
       run: |
         if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then
@@ -140,26 +140,26 @@ jobs:
           echo "Using BRANCH mode: v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER"
           echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV
         fi
-    
+
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v1
-      
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
-    
+
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    
+
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-        
+
     - name: Download a Build Artifact
       uses: actions/download-artifact@v2.1.0
       with:
@@ -170,7 +170,7 @@ jobs:
       run: |
         mkdir -p build
         tar -xzf $GITHUB_WORKSPACE/drop/wiki-js.tar.gz -C $GITHUB_WORKSPACE/build --exclude=node_modules
-    
+
     - name: Build and push Docker images
       uses: docker/build-push-action@v2.9.0
       with:
@@ -181,12 +181,12 @@ jobs:
         tags: |
           requarks/wiki:canary-${{ matrix.docker }}-${{ env.REL_VERSION_STRICT }}
           ghcr.io/requarks/wiki:canary-${{ matrix.docker }}-${{ env.REL_VERSION_STRICT }}
-          
+
   windows:
     name: Windows Build
     runs-on: windows-latest
     needs: [cypress]
-    
+
     steps:
     - name: Setup Node.js environment
       uses: actions/setup-node@v2.5.1
@@ -207,16 +207,16 @@ jobs:
     - name: Install Dependencies
       run: yarn --production --frozen-lockfile --non-interactive
       working-directory: win
-      
+
     - name: Create Bundle
       run: tar -czf wiki-js-windows.tar.gz -C $env:GITHUB_WORKSPACE\win .
-      
+
     - name: Upload a Build Artifact
       uses: actions/upload-artifact@v2.3.1
       with:
         name: drop-win
         path: wiki-js-windows.tar.gz
-    
+
   beta:
     name: Publish Beta Images
     runs-on: ubuntu-latest
@@ -224,31 +224,26 @@ jobs:
     needs: [build, arm, windows]
     permissions:
       packages: write
-      
+
     steps:
     - name: Set Version Variables
       run: |
-        if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then
+        echo "Using TAG mode: $GITHUB_REF_NAME"
-          echo "Using TAG mode: $GITHUB_REF_NAME"
+        echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV
-          echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV
-        else
-          echo "Using BRANCH mode: v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER"
-          echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV
-        fi
 
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    
+
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    
+
     - name: Create and Push Manifests
       run: |
         echo "Creating the manifests..."
@@ -260,7 +255,7 @@ jobs:
 
         docker manifest push -p requarks/wiki:beta-$REL_VERSION_STRICT
         docker manifest push -p ghcr.io/requarks/wiki:beta-$REL_VERSION_STRICT
-        
+
   release:
     name: Publish Release Images
     runs-on: ubuntu-latest
@@ -270,31 +265,26 @@ jobs:
     permissions:
       packages: write
       contents: write
-      
+
     steps:
     - name: Set Version Variables
       run: |
-        if [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then
+        echo "Using TAG mode: $GITHUB_REF_NAME"
-          echo "Using TAG mode: $GITHUB_REF_NAME"
+        echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV
-          echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV
-        else
-          echo "Using BRANCH mode: v$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER"
-          echo "REL_VERSION_STRICT=$BASE_DEV_VERSION-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV
-        fi
 
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    
+
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v1
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
-        
+
     - name: Create and Push Manifests
       run: |
         echo "Fetching semver tool..."
@@ -327,19 +317,19 @@ jobs:
         docker manifest push -p ghcr.io/requarks/wiki:$MAJOR
         docker manifest push -p ghcr.io/requarks/wiki:$MAJORMINOR
         docker manifest push -p ghcr.io/requarks/wiki:latest
-        
+
     - name: Download Linux Build
       uses: actions/download-artifact@v2.1.0
       with:
         name: drop
         path: drop
-        
+
     - name: Download Windows Build
       uses: actions/download-artifact@v2.1.0
       with:
         name: drop-win
         path: drop-win
-        
+
     - name: Generate Changelog
       id: changelog
       uses: Requarks/changelog-action@v1
@@ -357,4 +347,30 @@ jobs:
         body: ${{ steps.changelog.outputs.changes }}
         token: ${{ github.token }}
         artifacts: 'drop/wiki-js.tar.gz,drop-win/wiki-js-windows.tar.gz'
-    
+
+  build-do-image:
+    name: Build DigitalOcean Image
+    runs-on: ubuntu-latest
+    needs: [release]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set Version Variables
+      run: |
+        echo "Using TAG mode: $GITHUB_REF_NAME"
+        echo "REL_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV
+
+    - name: Install Packer
+      run: |
+        curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
+        sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+        sudo apt-get update && sudo apt-get install packer
+
+    - name: Build Droplet Image
+      env:
+        DIGITALOCEAN_API_TOKEN: ${{ secrets.DO_TOKEN }}
+        WIKI_APP_VERSION: ${{ env.REL_VERSION_STRICT }}
+      working-directory: dev/packer
+      run: |
+        packer build digitalocean.json

.github/workflows/packer.yml

@@ -0,0 +1,31 @@
+name: Build DigitalOcean Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'App Version'
+        required: true
+        type: string
+
+jobs:
+  build-do-image:
+    name: Build DigitalOcean Image
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Packer
+      run: |
+        curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
+        sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+        sudo apt-get update && sudo apt-get install packer
+
+    - name: Build Droplet Image
+      env:
+        DIGITALOCEAN_API_TOKEN: ${{ secrets.DO_TOKEN }}
+        WIKI_APP_VERSION: ${{ github.event.inputs.version }}
+      working-directory: dev/packer
+      run: |
+        packer build digitalocean.json

dev/packer/digitalocean.json

@@ -0,0 +1,78 @@
+{
+  "variables": {
+    "do_api_token": "{{env `DIGITALOCEAN_API_TOKEN`}}",
+    "image_name": "wikijs-snapshot-{{timestamp}}",
+    "apt_packages": "apt-transport-https ca-certificates curl jq linux-image-extra-virtual software-properties-common gnupg-agent openssl ",
+    "application_name": "Wiki.js",
+    "application_version": "{{env `WIKI_APP_VERSION`}}",
+    "docker_compose_version": "1.29.2"
+  },
+  "sensitive-variables": [
+    "do_api_token"
+  ],
+  "builders": [
+    {
+      "type": "digitalocean",
+      "api_token": "{{user `do_api_token`}}",
+      "image": "ubuntu-20-04-x64",
+      "region": "tor1",
+      "size": "s-1vcpu-1gb",
+      "ssh_username": "root",
+      "snapshot_name": "{{user `image_name`}}"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "cloud-init status --wait"
+      ]
+    },
+    {
+      "type": "file",
+      "source": "scripts/001-onboot.sh",
+      "destination": "/var/lib/cloud/scripts/per-instance/001-onboot.sh"
+    },
+    {
+      "type": "file",
+      "source": "scripts/099-one-click",
+      "destination": "/etc/update-motd.d/099-one-click"
+    },
+    {
+      "type": "shell",
+      "environment_vars": [
+        "DEBIAN_FRONTEND=noninteractive",
+        "LC_ALL=C",
+        "LANG=en_US.UTF-8",
+        "LC_CTYPE=en_US.UTF-8"
+      ],
+      "inline": [
+        "apt -qqy update",
+        "apt -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' full-upgrade",
+        "apt -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' install {{user `apt_packages`}}",
+        "apt-get -qqy clean"
+      ]
+    },
+    {
+      "type": "shell",
+      "environment_vars": [
+        "application_name={{user `application_name`}}",
+        "application_version={{user `application_version`}}",
+        "docker_compose_version={{user `docker_compose_version`}}",
+        "DEBIAN_FRONTEND=noninteractive",
+        "LC_ALL=C",
+        "LANG=en_US.UTF-8",
+        "LC_CTYPE=en_US.UTF-8"
+      ],
+      "scripts": [
+        "common/scripts/010-docker.sh",
+        "common/scripts/011-docker-compose.sh",
+        "common/scripts/012-grub-opts.sh",
+        "common/scripts/013-docker-dns.sh",
+        "common/scripts/014-ufw-docker.sh",
+        "common/scripts/020-application-tag.sh",
+        "common/scripts/900-cleanup.sh"
+      ]
+    }
+  ]
+}

dev/packer/scripts/001-onboot.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Scripts in this directory will be executed by cloud-init on the first boot of droplets
+# created from your image.  Things ike generating passwords, configuration requiring IP address
+# or other items that will be unique to each instance should be done in scripts here.
+
+openssl rand -base64 32 > /etc/wiki/.db-secret
+
+if [[ -z $DATABASE_URL ]]; then
+  docker start db
+fi
+docker start wiki
+docker start wiki-update-companion
+# docker start nginx-proxy
+# docker start watchtower

dev/packer/scripts/010-docker.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+apt -qqy update
+apt -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' install docker-ce docker-ce-cli containerd.io
+
+systemctl enable docker
+systemctl start docker
+
+mkdir -p /etc/wiki
+
+docker network create wikinet
+docker volume create pgdata
+docker create --name=db -e POSTGRES_DB=wiki -e POSTGRES_USER=wiki -e POSTGRES_PASSWORD_FILE=/etc/wiki/.db-secret -v /etc/wiki/.db-secret:/etc/wiki/.db-secret:ro -v pgdata:/var/lib/postgresql/data --restart=unless-stopped -h db --network=wikinet postgres:11
+docker create --name=wiki -e DB_TYPE=postgres -e DB_HOST=db -e DB_PORT=5432 -e DB_PASS_FILE=/etc/wiki/.db-secret -v /etc/wiki/.db-secret:/etc/wiki/.db-secret:ro -e DB_USER=wiki -e DB_NAME=wiki -e UPGRADE_COMPANION=1 --restart=unless-stopped -h wiki --network=wikinet -p 80:3000 -p 443:3443 ghcr.io/requarks/wiki:2
+docker create --name=wiki-update-companion -v /var/run/docker.sock:/var/run/docker.sock:ro --restart=unless-stopped -h wiki-update-companion --network=wikinet requarks/wiki-update-companion:latest
+# docker create --name=nginx-proxy -p 80:80 -p 443:443 -e DEFAULT_HOST=wiki.local --network=wikinet -v /var/run/docker.sock:/tmp/docker.sock:ro --restart=unless-stopped jwilder/nginx-proxy
+# docker create --name=watchtower --network=wikinet -v /var/run/docker.sock:/var/run/docker.sock --restart=unless-stopped containrrr/watchtower --cleanup --schedule="0 2 * * 6" wiki

dev/packer/scripts/011-docker-compose.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+sudo curl -L "https://github.com/docker/compose/releases/download/${docker_compose_version}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose;
+chmod +x /usr/local/bin/docker-compose;

dev/packer/scripts/012-grub-opts.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+sed -e 's|GRUB_CMDLINE_LINUX="|GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1|g' \
+    -i /etc/default/grub
+
+update-grub

dev/packer/scripts/013-docker-dns.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+sed -e 's|#DOCKER_OPTS|DOCKER_OPTS|g' \
+    -i /etc/default/docker

dev/packer/scripts/014-ufw-docker.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+sudo ufw allow ssh
+sudo ufw allow http
+sudo ufw allow https
+
+sudo ufw --force enable
+
+cat /dev/null > /var/log/ufw.log

dev/packer/scripts/020-application-tag.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+################################
+## PART: Write the application tag
+##
+## vi: syntax=sh expandtab ts=4
+
+build_date=$(date +%Y-%m-%d)
+distro="$(lsb_release -s  -i)"
+distro_release="$(lsb_release -s  -r)"
+distro_codename="$(lsb_release -s -c)"
+distro_arch="$(uname -m)"
+
+mkdip -p /var/lib/digitalocean
+touch /var/lib/digitalocean/application.info
+
+cat >> /var/lib/digitalocean/application.info <<EOM
+application_name="${application_name}"
+build_date="${build_date}"
+distro="${distro}"
+distro_release="${distro_release}"
+distro_codename="${distro_codename}"
+distro_arch="${distro_arch}"
+application_version="${application_version}"
+EOM

dev/packer/scripts/099-one-click

@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Configured as part of the DigitalOcean 1-Click Image build process
+
+myip=$(hostname -I | awk '{print$1}')
+cat <<EOF
+********************************************************************************
+Welcome to DigitalOcean's 1-Click Docker Droplet.
+To keep this Droplet secure, the UFW firewall is enabled.
+All ports are BLOCKED except 22 (SSH), 80 (Docker) and 443 (Docker).
+* The Docker 1-Click Quickstart guide is available at:
+  https://docs.requarks.io/install/digitalocean
+* You can SSH to this Droplet in a terminal as root: ssh root@$myip
+* Docker is installed and configured per Docker's recommendations:
+  https://docs.docker.com/install/linux/docker-ce/ubuntu/
+* Docker Compose is installed and configured per Docker's recommendations:
+  https://docs.docker.com/compose/install/#install-compose
+For more information, visit https://docs.requarks.io/install/digitalocean
+********************************************************************************
+To delete this message of the day: rm -rf $(readlink -f ${0})
+EOF

dev/packer/scripts/900-cleanup.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Ensure /tmp exists and has the proper permissions before
+# checking for security updates
+# https://github.com/digitalocean/marketplace-partners/issues/94
+if [[ ! -d /tmp ]]; then
+  mkdir /tmp
+fi
+chmod 1777 /tmp
+
+apt-get -y update
+apt-get -y upgrade
+rm -rf /tmp/* /var/tmp/*
+history -c
+cat /dev/null > /root/.bash_history
+unset HISTFILE
+apt-get -y autoremove
+apt-get -y autoclean
+find /var/log -mtime -1 -type f -exec truncate -s 0 {} \;
+rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-????????
+rm -rf /var/lib/cloud/instances/*
+rm -f /root/.ssh/authorized_keys /etc/ssh/*key*
+touch /etc/ssh/revoked_keys
+chmod 600 /etc/ssh/revoked_keys
+
+# Securely erase the unused portion of the filesystem
+GREEN='\033[0;32m'
+NC='\033[0m'
+printf "\n${GREEN}Writing zeros to the remaining disk space to securely
+erase the unused portion of the file system.
+Depending on your disk size this may take several minutes.
+The secure erase will complete successfully when you see:${NC}
+    dd: writing to '/zerofile': No space left on device\n
+Beginning secure erase now\n"
+
+dd if=/dev/zero of=/zerofile &
+  PID=$!
+  while [ -d /proc/$PID ]
+    do
+      printf "."
+      sleep 5
+    done
+sync; rm /zerofile; sync
+cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp