[FL-1533] CI refactoring (#601)

* rename artifacts * add workflow with date * add debug * Add branch name to artifact name and rm genpic job * Add branch name to artifact name for upload job * Add branch restrictions for dev and user branches * Add branch restrictions for dev and user branches * Add cleanup workspace for upload job * Test ignore branch * Test release CI workflow * Rename release CI * Add pre-release trigger * Rm branch restriction for CI * Add release trigger * rm branch restriction for CI * rm genpic job * Github: refactor actions, move linting to parallel actions, separate forlder for artifacts, build cleanup * Github: build date and commit hash on branches and tag on tags * Github: minor artifacts name cleanup * Github: decontaminate previous build leftovers * Github: better decontamination and release build SUFFIX * Github: fetch before decontamitation * Github: change decontamination logic, jump to first commit if submodules are broken * ReadMe: update links to latest firmware and images Co-authored-by: あく <alleteam@gmail.com>
2021-07-24 16:53:02 +03:00 · 2021-07-24 16:53:02 +03:00 · 0c7a8edf51
commit 0c7a8edf51
parent 413e3d5013
6 changed files with 241 additions and 183 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -0,0 +1,180 @@
+name: 'Build'
+
+on: push
+
+env:
+  TARGETS: f6
+
+jobs:
+  build:
+    runs-on: [self-hosted]
+    steps:
+      - name: 'Cleanup workspace'
+        uses: AutoModality/action-clean@v1
+
+      - name: 'Decontaminate previous build leftovers'
+        run: |
+          git submodule status \
+            || git checkout `git rev-list --max-parents=0 HEAD | tail -n 1`
+
+      - name: 'Checkout code'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: 'Docker cache'
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+        with:
+          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
+          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
+
+      - name: 'Build docker image'
+        uses: ./.github/actions/docker
+
+      - name: 'Make artifacts directory'
+        run: |
+          test -d artifacts && rm -rf artifacts || true
+          mkdir artifacts
+
+      - name: 'Generate tag suffix'
+        if: startsWith(github.ref, 'refs/tags/') == true
+        run: echo "SUFFIX=$(git describe --tags --abbrev=0)"  >> $GITHUB_ENV
+
+      - name: 'Generate branch suffix'
+        if: startsWith(github.ref, 'refs/tags/') != true
+        run: echo "SUFFIX=$(date +'%Y-%m-%d')-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: 'Build bootloader in docker'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              make -j$(nproc) -C bootloader TARGET=${TARGET}
+            done
+
+      - name: 'Generate dfu file for bootloader'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              hex2dfu \
+                -i bootloader/.obj/${TARGET}/bootloader.hex \
+                -o bootloader/.obj/${TARGET}/bootloader.dfu
+            done
+
+      - name: 'Build firmware in docker'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              make -j$(nproc) -C firmware TARGET=${TARGET}
+            done
+
+      - name: 'Generate dfu file for firmware'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              hex2dfu \
+                -i firmware/.obj/${TARGET}/firmware.hex \
+                -o firmware/.obj/${TARGET}/firmware.dfu
+            done
+
+      - name: 'Generate full hex file'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              srec_cat \
+                bootloader/.obj/${TARGET}/bootloader.hex -Intel \
+                firmware/.obj/${TARGET}/firmware.hex -Intel \
+                -o firmware/.obj/${TARGET}/full.hex -Intel
+            done
+
+      - name: 'Move upload files'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              mv bootloader/.obj/${TARGET}/bootloader.dfu \
+                artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.dfu
+              mv bootloader/.obj/${TARGET}/bootloader.bin \
+                artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.bin
+              mv bootloader/.obj/${TARGET}/bootloader.elf \
+                artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.elf
+              mv firmware/.obj/${TARGET}/firmware.dfu \
+                artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.dfu
+              mv firmware/.obj/${TARGET}/firmware.bin \
+                artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.bin
+              mv firmware/.obj/${TARGET}/firmware.elf \
+                artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.elf
+            done
+
+      - name: 'Generate full dfu file'
+        uses: ./.github/actions/docker
+        with:
+          run: |
+            for TARGET in ${TARGETS}
+            do
+              hex2dfu \
+                -i firmware/.obj/${TARGET}/full.hex \
+                -o artifacts/flipper-z-${TARGET}-full-${SUFFIX}.dfu
+            done
+
+      - name: 'Full flash asssembly: bootloader as base'
+        run: |
+          for TARGET in ${TARGETS}
+          do
+            cp \
+              artifacts/flipper-z-${TARGET}-bootloader-${SUFFIX}.bin \
+              artifacts/flipper-z-${TARGET}-full-${SUFFIX}.bin
+          done
+
+      - name: 'Full flash asssembly: bootloader padding'
+        run: |
+          for TARGET in ${TARGETS}
+          do
+            truncate -s 32768 artifacts/flipper-z-${TARGET}-full-${SUFFIX}.bin
+          done
+      
+      - name: 'Full flash asssembly: append firmware'
+        run: |
+          for TARGET in ${TARGETS}
+          do
+            cat \
+              artifacts/flipper-z-${TARGET}-firmware-${SUFFIX}.bin \
+              >> artifacts/flipper-z-${TARGET}-full-${SUFFIX}.bin
+          done
+
+      - name: 'Publish artifacts'
+        uses: actions/upload-artifact@v2
+        with:
+          name: artifacts
+          path: artifacts/*
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: 'Upload artifacts to update server'
+        uses: burnett01/rsync-deployments@4.1
+        with:
+          switches: -avzP --delete
+          path: artifacts/
+          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${GITHUB_REF##*/}/"
+          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
+          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
+          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
+          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
+
+      - name: 'Trigger update server reindex'
+        uses: wei/curl@master
+        with:
+          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
+
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -1,177 +0,0 @@
-name: 'CI'
-
-on: push
-
-env:
-  TARGETS: f6
-
-jobs:
-  build:
-    runs-on: [self-hosted]
-    steps:
-      - name: Cleanup workspace
-        uses: AutoModality/action-clean@v1
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Checkout submodules
-        run: git submodule update --init --recursive
-      - name: Checkout submodules
-        run: git submodule sync
-
-      - uses: satackey/action-docker-layer-caching@v0.0.11
-        continue-on-error: true
-        with:
-          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
-          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
-      - name: Build docker image
-        uses: ./.github/actions/docker
-
-      - name: Check syntax
-        uses: ./.github/actions/docker
-        continue-on-error: false
-        with:
-          run: /syntax_check.sh
-
-      - name: Build bootloader in docker
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do make -j$(nproc) -C bootloader TARGET=${TARGET}; done
-
-      - name: Generate dfu file for bootloader
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do hex2dfu -i bootloader/.obj/${TARGET}/bootloader.hex -o bootloader/.obj/${TARGET}/bootloader.dfu; done
-
-      - name: Build firmware in docker
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do make -j$(nproc) -C firmware TARGET=${TARGET}; done
-
-      - name: Generate dfu file for firmware
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do hex2dfu -i firmware/.obj/${TARGET}/firmware.hex -o firmware/.obj/${TARGET}/firmware.dfu; done
-
-      - name: Generate full hex file
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            for TARGET in ${TARGETS}; do
-            srec_cat bootloader/.obj/${TARGET}/bootloader.hex -Intel firmware/.obj/${TARGET}/firmware.hex -Intel -o ${TARGET}_full.hex -Intel
-            done
-
-      - name: Move upload files
-        uses: ./.github/actions/docker
-        with:
-          run: |
-            for TARGET in ${TARGETS}; do
-            mv bootloader/.obj/${TARGET}/bootloader.dfu ${TARGET}_bootloader.dfu
-            mv bootloader/.obj/${TARGET}/bootloader.bin ${TARGET}_bootloader.bin
-            mv bootloader/.obj/${TARGET}/bootloader.elf ${TARGET}_bootloader.elf
-            mv firmware/.obj/${TARGET}/firmware.dfu ${TARGET}_firmware.dfu
-            mv firmware/.obj/${TARGET}/firmware.bin ${TARGET}_firmware.bin
-            mv firmware/.obj/${TARGET}/firmware.elf ${TARGET}_firmware.elf
-            done
-
-      - name: Generate full dfu file
-        uses: ./.github/actions/docker
-        with:
-          run: for TARGET in ${TARGETS}; do hex2dfu -i ${TARGET}_full.hex -o ${TARGET}_full.dfu; done
-
-      - name: Copy bootloader for full.bin 
-        run: for TARGET in ${TARGETS}; do cp ${TARGET}_bootloader.bin ${TARGET}_full.bin; done
-      - name: Truncate full.bin 
-        run: for TARGET in ${TARGETS}; do truncate -s 32768 ${TARGET}_full.bin; done
-      - name: Add firmware.bin at full.bin
-        run: for TARGET in ${TARGETS}; do cat ${TARGET}_firmware.bin >> ${TARGET}_full.bin; done
-
-      - name: Copy last target to full.dfu
-        run: cp ${TARGETS##*" "}_full.dfu full.dfu
-
-      - name: Publish artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: artifacts
-          path: |
-            *_bootloader.dfu
-            *_bootloader.bin
-            *_firmware.dfu
-            *_firmware.bin
-            *_full.dfu
-            *_full.bin
-            full.dfu
-          if-no-files-found: error
-          retention-days: 7
-
-  upload:
-    name: Upload artifacts to external storage
-    needs: build
-    runs-on: [self-hosted]
-    steps:
-
-      - name: Make empty dir
-        run: mkdir -p empty
-
-      - name: Clear upload dir
-        uses: burnett01/rsync-deployments@4.1
-        with:
-          switches: -cvzr --delete
-          path: empty/
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${GITHUB_REF##*/}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
-
-      - name: Generate files list
-        run: ls *_bootloader.dfu *_firmware.dfu *_full.dfu *_bootloader.bin *_firmware.bin *_full.bin *_bootloader.elf *_firmware.elf full.dfu > uploadlist.txt
-
-      - name: Upload artifacts
-        uses: burnett01/rsync-deployments@4.1
-        with:
-          switches: -cvzr --delete --files-from=uploadlist.txt
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${GITHUB_REF##*/}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
-
-      - name: Trigger reindex
-        uses: wei/curl@master
-        with:
-          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
-
-  genpic:
-    name: Generate pic
-    needs: upload
-    if: github.ref == 'refs/heads/release'
-    runs-on: [self-hosted]
-    steps:
-      - name: Set test env
-        run: echo "NUMBER_OF_COMMITS=$(git rev-list --count HEAD)" >> $GITHUB_ENV
-      - name: Test output NUMBER_OF_COMMITS
-        run: echo $NUMBER_OF_COMMITS
-      - name: Test output date
-        run: |
-          GET_DATE=$(date +"%b %d")
-          PREP_DATE=$(echo "${GET_DATE^^}")
-          echo "PREP_DATE=$PREP_DATE" >> $GITHUB_ENV
-      - name: Test output PREP_DATE
-        run: echo $PREP_DATE
-      - name: Gen pic
-        uses: ./.github/actions/docker
-        with:
-          run: convert ./.github/assets/latest-firmware-template.png -font ./.github/assets/Born2bSportyV2.ttf -weight 700 -pointsize 140 -annotate +900+330 "$NUMBER_OF_COMMITS $PREP_DATE" latest-firmware-banner.png
-      - name: Upload pic
-        uses: burnett01/rsync-deployments@4.1
-        with:
-          switches: -avzp --delete
-          path: latest-firmware-banner.png
-          remote_path: "${{ secrets.RSYNC_DEPLOY_BASE_PATH }}/"
-          remote_host: ${{ secrets.RSYNC_DEPLOY_HOST }}
-          remote_port: ${{ secrets.RSYNC_DEPLOY_PORT }}
-          remote_user: ${{ secrets.RSYNC_DEPLOY_USER }}
-          remote_key: ${{ secrets.RSYNC_DEPLOY_KEY }}
--- a/.github/workflows/lint_c.yml
+++ b/.github/workflows/lint_c.yml
@ -0,0 +1,40 @@
+name: 'Lint C/C++ with clang-format'
+
+on: push
+
+env:
+  TARGETS: f6
+
+jobs:
+  lint_c_cpp:
+    runs-on: [self-hosted]
+    steps:
+      - name: 'Cleanup workspace'
+        uses: AutoModality/action-clean@v1
+
+      - name: 'Checkout code'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: 'Checkout submodules: update'
+        run: git submodule update --init --recursive
+
+      - name: 'Checkout submodules: sync'
+        run: git submodule sync
+
+      - name: 'Docker cache'
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+        with:
+          key: docker-cache-${{ hashFiles('docker/**') }}-{hash}
+          restore-keys: docker-cache-${{ hashFiles('docker/**') }}-
+
+      - name: 'Build docker image'
+        uses: ./.github/actions/docker
+
+      - name: 'Check syntax'
+        uses: ./.github/actions/docker
+        continue-on-error: false
+        with:
+          run: /syntax_check.sh
--- a/.github/workflows/lint_python.yml
+++ b/.github/workflows/lint_python.yml
@ -1,4 +1,4 @@
-name: Python Lint
+name: 'Python Lint'

 on: 
  push:
@ -7,7 +7,7 @@ on:
  pull_request:

 jobs:
-  lint:
+  lint_python:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
--- a/.github/workflows/reindex.yml
+++ b/.github/workflows/reindex.yml
@ -0,0 +1,15 @@
+name: 'Reindex'
+
+on:
+  release:
+    types: [prereleased,released]
+
+jobs:
+  reindex:
+    name: 'Reindex updates'
+    runs-on: [self-hosted]
+    steps:
+      - name: Trigger reindex
+        uses: wei/curl@master
+        with:
+          args: -X POST -F 'key=${{ secrets.REINDEX_KEY }}' ${{ secrets.REINDEX_URL }}
--- a/ReadMe.md
+++ b/ReadMe.md
@ -10,12 +10,12 @@ This repo will become completely public closer to the device shipping date.

 # Update firmware

-<a href="https://update.flipperzero.one/release/full.dfu"><img width="300" src="https://update.flipperzero.one/latest-firmware-banner.png" /></a>
+<a href="https://update.flipperzero.one/">Get Latest Firmware from Update Server</a>


 Flipper Zero's firmware consists of three components:

- Core2 firmware - proprietary componenets by ST: FUS + radio stack.
+- Core2 firmware set - proprietary componenets by ST: FUS + radio stack.
 - Core1 Bootloader - controls basic hardware initialization and loads firmware
 - Core1 Firmware - HAL + OS + Drivers + Applications

@ -46,7 +46,7 @@ One liner: `./flash_core1_main.sh`

 ## With USB DFU 

-1. Download latest [Firmware](https://update.flipperzero.one/release/full.dfu)
+1. Download latest [Firmware](https://update.flipperzero.one)

 2. Reboot Flipper to Bootloader 
 - Press and hold `← Left` + `↩ Back` for reset