Splitting units and updater benches (#2165)

* test run, moved updated to separate physical runner/flipper/card
* simplified units, removed re-flashing, moved format to beginning of run
* added reboot requence and mini optimizations
* forgot gitadd, added script modifications, workflow changes
* fixed linter issues
* moved updater to unit bench for speed up
* changes to units, flash (not full) on second update, new fbt GDB thread check
* changed serial of second device
* testing pipelines, added failing unit test
* fixed gdb step
* fixed gdb step v2 electric boogaloo
* fixed gdb step v3, fixed target
* reverted while1 in units, tests complete
* testing colored output
* trying different term setting
* debug outputs for terminal
* fixed typo in SConstruct and another terminal test
* reverted changes, no colored output, for production
* fixed log output to readable format
* fixed linter

Co-authored-by: Konstantin Volkov <k.volkov@flipperdevices.com>
Co-authored-by: あく <alleteam@gmail.com>

.github/workflows/unit_tests.yml

@@ -9,7 +9,7 @@ env:
   FBT_TOOLCHAIN_PATH: /opt
 
 jobs:
-  run_units_on_test_bench:
+  run_units_on_bench:
     runs-on: [self-hosted, FlipperZeroTest]
     steps:
       - name: 'Decontaminate previous build leftovers'
@@ -29,81 +29,38 @@ jobs:
         run: |
           echo "flipper=/dev/ttyACM0" >> $GITHUB_OUTPUT
 
-      - name: 'Flashing target firmware'
-        id: first_full_flash
-        run: |
-          ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
-          source scripts/toolchain/fbtenv.sh
-          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
-
-      - name: 'Validating updater'
-        id: second_full_flash
-        if: success()
-        run: |
-          ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
-          source scripts/toolchain/fbtenv.sh
-          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
-
       - name: 'Flash unit tests firmware'
         id: flashing
         if: success()
         run: |                               
           ./fbt flash OPENOCD_ADAPTER_SERIAL=2A0906016415303030303032 FIRMWARE_APP_SET=unit_tests FORCE=1
 
-      - name: 'Wait for flipper to finish updating'
+      - name: 'Wait for flipper and format ext'
-        id: connect
+        id: format_ext
         if: steps.flashing.outcome == 'success'
         run: |
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
 
-      - name: 'Copy assets and unit tests data to flipper'
+      - name: 'Copy assets and unit data, reboot and wait for flipper'
         id: copy
-        if: steps.connect.outcome == 'success'
+        if: steps.format_ext.outcome == 'success'
         run: |
           source scripts/toolchain/fbtenv.sh
+          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} -f send assets/resources /ext
           python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} -f send assets/unit_tests /ext/unit_tests
+          python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
 
       - name: 'Run units and validate results'
+        id: run_units
         if: steps.copy.outcome == 'success'
         run: |
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testing/units.py ${{steps.device.outputs.flipper}}
 
-      - name: 'Get last release tag'
+      - name: 'Check GDB output'
-        id: release_tag
+        if: failure()
-        if: always()
         run: |
-          echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT
+          ./fbt gdb_trace_all OPENOCD_ADAPTER_SERIAL=2A0906016415303030303032 FIRMWARE_APP_SET=unit_tests FORCE=1
-
-      - name: 'Decontaminate previous build leftovers'
-        if: always()
-        run: |
-          if [ -d .git ]; then
-            git submodule status || git checkout "$(git rev-list --max-parents=0 HEAD | tail -n 1)"
-          fi
-
-      - name: 'Checkout latest release'
-        uses: actions/checkout@v3
-        if: always()
-        with:
-          fetch-depth: 0
-          ref: ${{ steps.release_tag.outputs.tag }}
-
-      - name: 'Flash last release'
-        if: always()
-        run: |
-          ./fbt flash OPENOCD_ADAPTER_SERIAL=2A0906016415303030303032 FIRMWARE_APP_SET=unit_tests FORCE=1
-
-      - name: 'Wait for flipper to finish updating'
-        if: always()
-        run: |
-          source scripts/toolchain/fbtenv.sh
-          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
-
-      - name: 'Format flipper SD card'
-        id: format
-        if: always()
-        run: |
-          source scripts/toolchain/fbtenv.sh
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext

.github/workflows/updater_test.yml

@@ -0,0 +1,77 @@
+name: 'Updater test'
+
+on:
+  pull_request:
+
+env:
+  TARGETS: f7
+  DEFAULT_TARGET: f7
+  FBT_TOOLCHAIN_PATH: /opt
+
+jobs:
+  test_updater_on_bench:
+    runs-on: [self-hosted, FlipperZeroTest] # currently on same bench as units, needs different bench
+    steps:
+      - name: 'Decontaminate previous build leftovers'
+        run: |
+          if [ -d .git ]; then
+            git submodule status || git checkout "$(git rev-list --max-parents=0 HEAD | tail -n 1)"
+          fi
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: 'Get flipper from device manager (mock)'
+        id: device
+        run: |
+          echo "flipper=/dev/ttyACM0" >> $GITHUB_OUTPUT
+
+      - name: 'Flashing target firmware'
+        id: first_full_flash
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+
+      - name: 'Validating updater'
+        id: second_full_flash
+        if: success()
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          ./fbt flash_usb PORT=${{steps.device.outputs.flipper}} FORCE=1
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+
+      - name: 'Get last release tag'
+        id: release_tag
+        if: failure()
+        run: |
+          echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT
+
+      - name: 'Decontaminate previous build leftovers'
+        if: failure()
+        run: |
+          if [ -d .git ]; then
+            git submodule status || git checkout "$(git rev-list --max-parents=0 HEAD | tail -n 1)"
+          fi
+
+      - name: 'Checkout latest release'
+        uses: actions/checkout@v3
+        if: failure()
+        with:
+          fetch-depth: 0
+          ref: ${{ steps.release_tag.outputs.tag }}
+
+      - name: 'Flash last release'
+        if: failure()
+        run: |
+          ./fbt flash OPENOCD_ADAPTER_SERIAL=2A0906016415303030303032 FORCE=1
+
+      - name: 'Wait for flipper and format ext'
+        if: failure()
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext

SConstruct

@@ -194,6 +194,20 @@ firmware_bm_flash = distenv.PhonyTarget(
     ],
 )
 
+gdb_backtrace_all_threads = distenv.PhonyTarget(
+    "gdb_trace_all",
+    "$GDB $GDBOPTS $SOURCES $GDBFLASH",
+    source=firmware_env["FW_ELF"],
+    GDBOPTS="${GDBOPTS_BASE}",
+    GDBREMOTE="${OPENOCD_GDB_PIPE}",
+    GDBFLASH=[
+        "-ex",
+        "thread apply all bt",
+        "-ex",
+        "quit",
+    ],
+)
+
 # Debugging firmware
 firmware_debug = distenv.PhonyTarget(
     "debug",

scripts/power.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+from flipper.app import App
+from flipper.storage import FlipperStorage
+from flipper.utils.cdc import resolve_port
+
+
+class Main(App):
+    # this is basic use without sub-commands, simply to reboot flipper / power it off, not meant as a full CLI wrapper
+    def init(self):
+        self.parser.add_argument("-p", "--port", help="CDC Port", default="auto")
+
+        self.subparsers = self.parser.add_subparsers(help="sub-command help")
+
+        self.parser_power_off = self.subparsers.add_parser(
+            "power_off", help="Power off command, won't return to CLI"
+        )
+        self.parser_power_off.set_defaults(func=self.power_off)
+
+        self.parser_reboot = self.subparsers.add_parser(
+            "reboot", help="Reboot command help"
+        )
+        self.parser_reboot.set_defaults(func=self.reboot)
+
+        self.parser_reboot2dfu = self.subparsers.add_parser(
+            "reboot2dfu", help="Reboot to DFU, won't return to CLI"
+        )
+        self.parser_reboot2dfu.set_defaults(func=self.reboot2dfu)
+
+    def _get_flipper(self):
+        if not (port := resolve_port(self.logger, self.args.port)):
+            return None
+
+        flipper = FlipperStorage(port)
+        flipper.start()
+        return flipper
+
+    def power_off(self):
+        if not (flipper := self._get_flipper()):
+            return 1
+
+        self.logger.debug("Powering off")
+        flipper.send("power off" + "\r")
+        flipper.stop()
+        return 0
+
+    def reboot(self):
+        if not (flipper := self._get_flipper()):
+            return 1
+
+        self.logger.debug("Rebooting")
+        flipper.send("power reboot" + "\r")
+        flipper.stop()
+        return 0
+
+    def reboot2dfu(self):
+        if not (flipper := self._get_flipper()):
+            return 1
+
+        self.logger.debug("Rebooting to DFU")
+        flipper.send("power reboot2dfu" + "\r")
+        flipper.stop()
+
+        return 0
+
+
+if __name__ == "__main__":
+    Main()()

scripts/testing/await_flipper.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
-
+import logging
-import sys, os, time
+import os
+import sys
+import time
 
 
 def flp_serial_by_name(flp_name):
@@ -31,6 +33,12 @@ def main():
     flipper_name = sys.argv[1]
     elapsed = 0
     flipper = flp_serial_by_name(flipper_name)
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        level=logging.INFO,
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    logging.info("Waiting for Flipper to be ready...")
 
     while flipper == "" and elapsed < UPDATE_TIMEOUT:
         elapsed += 1
@@ -38,9 +46,11 @@ def main():
         flipper = flp_serial_by_name(flipper_name)
 
     if flipper == "":
-        print(f"Cannot find {flipper_name} flipper. Guess your flipper swam away")
+        logging.error("Flipper not found!")
         sys.exit(1)
 
+    logging.info(f"Found Flipper at {flipper}")
+
     sys.exit(0)
 
 

scripts/testing/units.py

@@ -1,28 +1,32 @@
 #!/usr/bin/env python3
-
+import logging
-import sys, os
-import serial
 import re
+import sys
 
+import serial
 from await_flipper import flp_serial_by_name
 
 
-LEAK_THRESHOLD = 3000  # added until units are fixed
-
-
 def main():
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        level=logging.INFO,
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    logging.info("Trying to run units on flipper")
     flp_serial = flp_serial_by_name(sys.argv[1])
 
     if flp_serial == "":
-        print("Name or serial port is invalid")
+        logging.error("Flipper not found!")
         sys.exit(1)
 
     with serial.Serial(flp_serial, timeout=1) as flipper:
+        logging.info(f"Found Flipper at {flp_serial}")
         flipper.baudrate = 230400
         flipper.flushOutput()
         flipper.flushInput()
 
-        flipper.timeout = 300
+        flipper.timeout = 180
 
         flipper.read_until(b">: ").decode("utf-8")
         flipper.write(b"unit_tests\r")
@@ -41,9 +45,13 @@ def main():
         status_pattern = re.compile(status_re)
 
         tests, time, leak, status = None, None, None, None
+        total = 0
 
         for line in lines:
-            print(line)
+            logging.info(line)
+            if "()" in line:
+                total += 1
+
             if not tests:
                 tests = re.match(tests_pattern, line)
             if not time:
@@ -53,8 +61,8 @@ def main():
             if not status:
                 status = re.match(status_pattern, line)
 
-        if leak is None or time is None or leak is None or status is None:
+        if None in (tests, time, leak, status):
-            print("Failed to get data. Or output is corrupt")
+            logging.error(f"Failed to parse output: {leak} {time} {leak} {status}")
             sys.exit(1)
 
         leak = int(re.findall(r"[- ]\d+", leak.group(0))[0])
@@ -62,16 +70,18 @@ def main():
         tests = int(re.findall(r"\d+", tests.group(0))[0])
         time = int(re.findall(r"\d+", time.group(0))[0])
 
-        if tests > 0 or leak > LEAK_THRESHOLD or status != "PASSED":
+        if tests > 0 or status != "PASSED":
-            print(f"Got {tests} failed tests.")
+            logging.error(f"Got {tests} failed tests.")
-            print(f"Leaked {leak} bytes.")
+            logging.error(f"Leaked (not failing on this stat): {leak}")
-            print(f"Status by flipper: {status}")
+            logging.error(f"Status: {status}")
-            print(f"Time elapsed {time/1000} seconds.")
+            logging.error(f"Time: {time/1000} seconds")
             sys.exit(1)
 
-        print(
+        logging.info(f"Leaked (not failing on this stat): {leak}")
-            f"Tests ran successfully! Time elapsed {time/1000} seconds. Passed {tests} tests."
+        logging.info(
+            f"Tests ran successfully! Time elapsed {time/1000} seconds. Passed {total} tests."
         )
+
         sys.exit(0)