[FL-1699, FL-1700] Scripts: new radio firmware bundling scheme, manifest for resources. (#700)

* Scripts: new radio firmware bundling scheme, manifest for resources.
* Scripts: add destination address for copro binaries.
* Bootloader: update linker scripts
* Scripts: resource manifest FsTree.

.github/workflows/build.yml

@@ -170,12 +170,7 @@ jobs:
         run: |
           test -d core2_firmware && rm -rf core2_firmware || true
           mkdir core2_firmware
-          cp \
-            lib/STM32CubeWB/package.xml \
-            lib/STM32CubeWB/Projects/STM32WB_Copro_Wireless_Binaries/STM32WB5x/stm32wb5x_FUS_fw.bin \
-            lib/STM32CubeWB/Projects/STM32WB_Copro_Wireless_Binaries/STM32WB5x/stm32wb5x_FUS_fw_for_fus_0_5_3.bin \
-            lib/STM32CubeWB/Projects/STM32WB_Copro_Wireless_Binaries/STM32WB5x/stm32wb5x_BLE_Stack_full_fw.bin \
-            core2_firmware
+          ./scripts/assets.py copro lib/STM32CubeWB core2_firmware STM32WB5x
           tar czpf artifacts/flipper-z-any-core2_firmware-${{steps.names.outputs.suffix}}.tgz core2_firmware
 
       - name: 'Bundle scripts'
@@ -186,6 +181,7 @@ jobs:
       - name: 'Bundle resources'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
+          ./scripts/assets.py manifest assets/resources
           tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources
 
       - name: 'Upload artifacts to update server'

assets/Makefile

@@ -4,7 +4,7 @@ include				$(PROJECT_ROOT)/assets/assets.mk
 
 $(ASSETS): $(ASSETS_SOURCES) $(ASSETS_COMPILLER)
 	@echo "\tASSETS\t" $@
-	@$(ASSETS_COMPILLER) icons -s $(ASSETS_SOURCE_DIR) -o $(ASSETS_COMPILED_DIR)
+	@$(ASSETS_COMPILLER) icons "$(ASSETS_SOURCE_DIR)" "$(ASSETS_COMPILED_DIR)"
 
 clean:
 	@echo "\tCLEAN\t"

bootloader/targets/f6/stm32wb55xx_flash_cm4.ld

@@ -56,7 +56,7 @@ _Min_Stack_Size = 0x400; /* required amount of stack */
 MEMORY
 {
 FLASH (rx)                 : ORIGIN = 0x08000000, LENGTH = 32K
-RAM1 (xrw)                 : ORIGIN = 0x20000004, LENGTH = 0x2FFFC
+RAM1 (xrw)                 : ORIGIN = 0x20000008, LENGTH = 0x2FFF8
 RAM_SHARED (xrw)           : ORIGIN = 0x20030000, LENGTH = 10K
 }
 

bootloader/targets/f7/stm32wb55xx_flash_cm4.ld

@@ -56,7 +56,7 @@ _Min_Stack_Size = 0x400; /* required amount of stack */
 MEMORY
 {
 FLASH (rx)                 : ORIGIN = 0x08000000, LENGTH = 32K
-RAM1 (xrw)                 : ORIGIN = 0x20000004, LENGTH = 0x2FFFC
+RAM1 (xrw)                 : ORIGIN = 0x20000008, LENGTH = 0x2FFF8
 RAM_SHARED (xrw)           : ORIGIN = 0x20030000, LENGTH = 10K
 }
 

scripts/assets.py

@@ -34,13 +34,24 @@ class Main:
         self.parser_icons = self.subparsers.add_parser(
             "icons", help="Process icons and build icon registry"
         )
-        self.parser_icons.add_argument(
-            "-s", "--source-directory", help="Source directory"
-        )
-        self.parser_icons.add_argument(
-            "-o", "--output-directory", help="Output directory"
-        )
+        self.parser_icons.add_argument("source_directory", help="Source directory")
+        self.parser_icons.add_argument("output_directory", help="Output directory")
         self.parser_icons.set_defaults(func=self.icons)
+
+        self.parser_manifest = self.subparsers.add_parser(
+            "manifest", help="Create directory Manifest"
+        )
+        self.parser_manifest.add_argument("local_path", help="local_path")
+        self.parser_manifest.set_defaults(func=self.manifest)
+
+        self.parser_copro = self.subparsers.add_parser(
+            "copro", help="Gather copro binaries for packaging"
+        )
+        self.parser_copro.add_argument("cube_dir", help="Path to Cube folder")
+        self.parser_copro.add_argument("output_dir", help="Path to output folder")
+        self.parser_copro.add_argument("mcu", help="MCU series as in copro folder")
+        self.parser_copro.set_defaults(func=self.copro)
+
         # logging
         self.logger = logging.getLogger()
 
@@ -163,6 +174,42 @@ class Main:
         extension = filename.lower().split(".")[-1]
         return extension in ICONS_SUPPORTED_FORMATS
 
+    def manifest(self):
+        from flipper.manifest import Manifest
+
+        directory_path = os.path.normpath(self.args.local_path)
+        if not os.path.isdir(directory_path):
+            self.logger.error(f'"{directory_path}" is not a directory')
+            exit(255)
+        manifest_file = os.path.join(directory_path, "Manifest")
+        old_manifest = Manifest()
+        if os.path.exists(manifest_file):
+            self.logger.info(
+                f"old manifest is present, loading for compare and removing file"
+            )
+            old_manifest.load(manifest_file)
+            os.unlink(manifest_file)
+        self.logger.info(f'Creating new Manifest for directory "{directory_path}"')
+        new_manifest = Manifest()
+        new_manifest.create(directory_path)
+        new_manifest.save(manifest_file)
+        self.logger.info(f"Comparing new manifest with old")
+        only_in_old, changed, only_in_new = Manifest.compare(old_manifest, new_manifest)
+        for record in only_in_old:
+            self.logger.info(f"Only in old: {record}")
+        for record in changed:
+            self.logger.info(f"Changed: {record}")
+        for record in only_in_new:
+            self.logger.info(f"Only in new: {record}")
+        self.logger.info(f"Complete")
+
+    def copro(self):
+        from flipper.copro import Copro
+
+        copro = Copro(self.args.mcu)
+        copro.loadCubeInfo(self.args.cube_dir)
+        copro.bundle(self.args.output_dir)
+
 
 if __name__ == "__main__":
     Main()()

scripts/flipper/copro.py

@@ -0,0 +1,95 @@
+import logging
+import datetime
+import shutil
+import json
+
+import xml.etree.ElementTree as ET
+from .utils import *
+
+CUBE_COPRO_PATH = "Projects/STM32WB_Copro_Wireless_Binaries"
+
+MANIFEST_TEMPLATE = {
+    "manifest": {"version": 0, "timestamp": 0},
+    "copro": {
+        "fus": {"version": {"major": 1, "minor": 2, "sub": 0}, "files": []},
+        "radio": {
+            "version": {
+                "type": 1,
+                "major": 1,
+                "minor": 12,
+                "sub": 0,
+                "branch": 0,
+                "release": 7,
+            },
+            "files": [],
+        },
+    },
+}
+
+
+class Copro:
+    def __init__(self, mcu):
+        self.mcu = mcu
+        self.version = None
+        self.cube_dir = None
+        self.mcu_copro = None
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def loadCubeInfo(self, cube_dir):
+        if not os.path.isdir(cube_dir):
+            raise Exception(f'"{cube_dir}" doesn\'t exists')
+        self.cube_dir = cube_dir
+        self.mcu_copro = os.path.join(self.cube_dir, CUBE_COPRO_PATH, self.mcu)
+        if not os.path.isdir(self.mcu_copro):
+            raise Exception(f'"{self.mcu_copro}" doesn\'t exists')
+        cube_manifest_file = os.path.join(self.cube_dir, "package.xml")
+        cube_manifest = ET.parse(cube_manifest_file)
+        cube_version = cube_manifest.find("PackDescription")
+        if not cube_version:
+            raise Exception(f"Unknown Cube manifest format")
+        cube_version = cube_version.get("Release")
+        if not cube_version or not cube_version.startswith("FW.WB"):
+            raise Exception(f"Incorrect Cube package or version info")
+        cube_version = cube_version.replace("FW.WB.", "", 1)
+        if cube_version != "1.12.0":
+            raise Exception(f"Unknonwn cube version")
+        self.version = cube_version
+
+    def addFile(self, array, filename, **kwargs):
+        source_file = os.path.join(self.mcu_copro, filename)
+        destination_file = os.path.join(self.output_dir, filename)
+        shutil.copyfile(source_file, destination_file)
+        array.append(
+            {"name": filename, "sha256": file_sha256(destination_file), **kwargs}
+        )
+
+    def bundle(self, output_dir):
+        if not os.path.isdir(output_dir):
+            raise Exception(f'"{output_dir}" doesn\'t exists')
+        self.output_dir = output_dir
+        manifest_file = os.path.join(self.output_dir, "Manifest.json")
+        # Form Manifest
+        manifest = dict(MANIFEST_TEMPLATE)
+        manifest["manifest"]["timestamp"] = timestamp()
+        # Old FUS Update
+        self.addFile(
+            manifest["copro"]["fus"]["files"],
+            "stm32wb5x_FUS_fw_for_fus_0_5_3.bin",
+            condition="==0.5.3",
+            address="0x080EC000",
+        )
+        # New FUS Update
+        self.addFile(
+            manifest["copro"]["fus"]["files"],
+            "stm32wb5x_FUS_fw.bin",
+            condition=">0.5.3",
+            address="0x080EC000",
+        )
+        # BLE Full Stack
+        self.addFile(
+            manifest["copro"]["radio"]["files"],
+            "stm32wb5x_BLE_Stack_full_fw.bin",
+            address="0x080CA000",
+        )
+        # Save manifest to
+        json.dump(manifest, open(manifest_file, "w"))

scripts/flipper/fstree.py

@@ -0,0 +1,76 @@
+from enum import Enum
+from collections import OrderedDict
+
+
+class FsNode:
+    class Type(Enum):
+        File = 0
+        Directory = 1
+
+    def __init__(self, name: str, type: "FsNode.Type", **kwargs):
+        self.name = name
+        self.type = type
+        self.data = kwargs
+        self.parent = None
+        self.children = OrderedDict()
+
+    def addChild(self, node: "FsNode"):
+        self.children[node.name] = node
+        node.parent = self
+
+    def addDirectory(self, path):
+        fragments = path.split("/")
+        name = fragments[-1]
+        fragments = fragments[:-1]
+        parent_node = self.traverse(fragments)
+        if not parent_node:
+            raise Exception(f"No parent node found for: {path}")
+        parent_node.addChild(FsNode(name, FsNode.Type.Directory))
+
+    def addFile(self, path, md5, size):
+        fragments = path.split("/")
+        name = fragments[-1]
+        fragments = fragments[:-1]
+        parent_node = self.traverse(fragments)
+        if not parent_node:
+            raise Exception(f"No parent node found for: {path}")
+        parent_node.addChild(FsNode(name, FsNode.Type.File, md5=md5, size=size))
+
+    def getChild(self, name):
+        return self.children[name]
+
+    def traverse(self, fragments):
+        current = self
+        for fragment in fragments:
+            current = current.getChild(fragment)
+            if not current:
+                break
+        return current
+
+    def getPath(self):
+        fragments = []
+        current = self
+        while current.parent:
+            fragments.append(current.name)
+            current = current.parent
+        return "/".join(reversed(fragments))
+
+    def dump(self):
+        ret = {}
+        ret["name"] = (self.name,)
+        ret["type"] = (self.type,)
+        ret["path"] = (self.getPath(),)
+        if len(self.children):
+            ret["children"] = [node.dump() for node in self.children.values()]
+        return ret
+
+
+def compare_fs_trees(left: FsNode, right: FsNode):
+    # import pprint
+    # pprint.pprint(left.dump())
+    # pprint.pprint(right.dump())
+
+    only_in_left = []
+    changed = []
+    only_in_right = []
+    return [], [], []

scripts/flipper/manifest.py

@@ -0,0 +1,168 @@
+import datetime
+import logging
+import os
+
+from .utils import *
+from .fstree import *
+
+MANIFEST_VERSION = 0
+
+
+class ManifestRecord:
+    tag = None
+
+    @staticmethod
+    def fromLine(line):
+        raise NotImplementedError
+
+    def toLine(self):
+        raise NotImplementedError
+
+    def _unpack(self, manifest, key, type):
+        key, value = manifest.readline().split(":", 1)
+        assert key == key
+        return type(value)
+
+
+MANIFEST_TAGS_RECORDS = {}
+
+
+def addManifestRecord(record: ManifestRecord):
+    assert record.tag
+    MANIFEST_TAGS_RECORDS[record.tag] = record
+
+
+class ManifestRecordVersion(ManifestRecord):
+    tag = "V"
+
+    def __init__(self, version):
+        self.version = version
+
+    @staticmethod
+    def fromLine(line):
+        return ManifestRecordVersion(int(line))
+
+    def toLine(self):
+        return f"{self.tag}:{self.version}\n"
+
+
+addManifestRecord(ManifestRecordVersion)
+
+
+class ManifestRecordTimestamp(ManifestRecord):
+    tag = "T"
+
+    def __init__(self, timestamp: int):
+        self.timestamp = int(timestamp)
+
+    @staticmethod
+    def fromLine(line):
+        return ManifestRecordTimestamp(int(line))
+
+    def toLine(self):
+        return f"{self.tag}:{self.timestamp}\n"
+
+
+addManifestRecord(ManifestRecordTimestamp)
+
+
+class ManifestRecordDirectory(ManifestRecord):
+    tag = "D"
+
+    def __init__(self, path: str):
+        self.path = path
+
+    @staticmethod
+    def fromLine(line):
+        return ManifestRecordDirectory(line)
+
+    def toLine(self):
+        return f"{self.tag}:{self.path}\n"
+
+
+addManifestRecord(ManifestRecordDirectory)
+
+
+class ManifestRecordFile(ManifestRecord):
+    tag = "F"
+
+    def __init__(self, path: str, md5: str, size: int):
+        self.path = path
+        self.md5 = md5
+        self.size = size
+
+    @staticmethod
+    def fromLine(line):
+        data = line.split(":", 3)
+        return ManifestRecordFile(data[2], data[0], data[1])
+
+    def toLine(self):
+        return f"{self.tag}:{self.md5}:{self.size}:{self.path}\n"
+
+
+addManifestRecord(ManifestRecordFile)
+
+
+class Manifest:
+    def __init__(self):
+        self.version = None
+        self.records = []
+        self.records.append(ManifestRecordVersion(MANIFEST_VERSION))
+        self.records.append(ManifestRecordTimestamp(timestamp()))
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def load(self, filename):
+        manifest = open(filename, "r")
+        for line in manifest.readlines():
+            line = line.strip()
+            if len(line) == 0:
+                continue
+            tag, line = line.split(":", 1)
+            record = MANIFEST_TAGS_RECORDS[tag].fromLine(line)
+            self.records.append(record)
+
+    def save(self, filename):
+        manifest = open(filename, "w+")
+        for record in self.records:
+            manifest.write(record.toLine())
+        manifest.close()
+
+    def addDirectory(self, path):
+        self.records.append(ManifestRecordDirectory(path))
+
+    def addFile(self, path, md5, size):
+        self.records.append(ManifestRecordFile(path, md5, size))
+
+    def create(self, directory_path):
+        for root, dirs, files in os.walk(directory_path):
+            relative_root = root.replace(directory_path, "", 1)
+            if relative_root.startswith("/"):
+                relative_root = relative_root[1:]
+            # process directories
+            for dir in dirs:
+                relative_dir_path = os.path.join(relative_root, dir)
+                self.logger.info(f'Adding directory: "{relative_dir_path}"')
+                self.addDirectory(relative_dir_path)
+            # Process files
+            for file in files:
+                relative_file_path = os.path.join(relative_root, file)
+                full_file_path = os.path.join(root, file)
+                self.logger.info(f'Adding file: "{relative_file_path}"')
+                self.addFile(
+                    relative_file_path,
+                    file_md5(full_file_path),
+                    os.path.getsize(full_file_path),
+                )
+
+    def toFsTree(self):
+        root = FsNode("", FsNode.Type.Directory)
+        for record in self.records:
+            if isinstance(record, ManifestRecordDirectory):
+                root.addDirectory(record.path)
+            elif isinstance(record, ManifestRecordFile):
+                root.addFile(record.path, record.md5, record.size)
+        return root
+
+    @staticmethod
+    def compare(left: "Manifest", right: "Manifest"):
+        return compare_fs_trees(left.toFsTree(), right.toFsTree())

scripts/flipper/utils.py

@@ -0,0 +1,27 @@
+import datetime
+import hashlib
+import os
+
+
+def timestamp():
+    return int(datetime.datetime.now().timestamp())
+
+
+def file_hash(path: str, algo: str, block_size: int = 4096):
+    fd = open(path, "rb")
+    h = hashlib.new(algo)
+    while True:
+        data = fd.read(block_size)
+        if len(data) > 0:
+            h.update(data)
+        else:
+            break
+    return h.hexdigest()
+
+
+def file_md5(path, block_size=4096):
+    return file_hash(path, "md5", block_size)
+
+
+def file_sha256(path, block_size=4096):
+    return file_hash(path, "sha256", block_size)