From 4456982e2767f6754587e5981bd87d28bfcded42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=82=E3=81=8F?= Date: Mon, 13 Sep 2021 12:52:50 +0300 Subject: [PATCH] [FL-1699, FL-1700] Scripts: new radio firmware bundling scheme, manifest for resources. (#700) * Scripts: new radio firmware bundling scheme, manifest for resources. * Scripts: add destination address for copro binaries. * Bootloader: update linker scripts * Scripts: resource manifest FsTree. --- .github/workflows/build.yml | 8 +- assets/Makefile | 2 +- .../targets/f6/stm32wb55xx_flash_cm4.ld | 2 +- .../targets/f7/stm32wb55xx_flash_cm4.ld | 2 +- scripts/assets.py | 59 +++++- scripts/flipper/copro.py | 95 ++++++++++ scripts/flipper/fstree.py | 76 ++++++++ scripts/flipper/manifest.py | 168 ++++++++++++++++++ scripts/flipper/utils.py | 27 +++ 9 files changed, 424 insertions(+), 15 deletions(-) create mode 100644 scripts/flipper/copro.py create mode 100644 scripts/flipper/fstree.py create mode 100644 scripts/flipper/manifest.py create mode 100644 scripts/flipper/utils.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70d7e6b9..1bd8c243 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -170,12 +170,7 @@ jobs: run: | test -d core2_firmware && rm -rf core2_firmware || true mkdir core2_firmware - cp \ - lib/STM32CubeWB/package.xml \ - lib/STM32CubeWB/Projects/STM32WB_Copro_Wireless_Binaries/STM32WB5x/stm32wb5x_FUS_fw.bin \ - lib/STM32CubeWB/Projects/STM32WB_Copro_Wireless_Binaries/STM32WB5x/stm32wb5x_FUS_fw_for_fus_0_5_3.bin \ - lib/STM32CubeWB/Projects/STM32WB_Copro_Wireless_Binaries/STM32WB5x/stm32wb5x_BLE_Stack_full_fw.bin \ - core2_firmware + ./scripts/assets.py copro lib/STM32CubeWB core2_firmware STM32WB5x tar czpf artifacts/flipper-z-any-core2_firmware-${{steps.names.outputs.suffix}}.tgz core2_firmware - name: 'Bundle scripts' @@ -186,6 +181,7 @@ jobs: - name: 'Bundle resources' if: ${{ !github.event.pull_request.head.repo.fork }} run: | + ./scripts/assets.py manifest assets/resources tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources - name: 'Upload artifacts to update server' diff --git a/assets/Makefile b/assets/Makefile index ab6fe4f3..1a379af5 100644 --- a/assets/Makefile +++ b/assets/Makefile @@ -4,7 +4,7 @@ include $(PROJECT_ROOT)/assets/assets.mk $(ASSETS): $(ASSETS_SOURCES) $(ASSETS_COMPILLER) @echo "\tASSETS\t" $@ - @$(ASSETS_COMPILLER) icons -s $(ASSETS_SOURCE_DIR) -o $(ASSETS_COMPILED_DIR) + @$(ASSETS_COMPILLER) icons "$(ASSETS_SOURCE_DIR)" "$(ASSETS_COMPILED_DIR)" clean: @echo "\tCLEAN\t" diff --git a/bootloader/targets/f6/stm32wb55xx_flash_cm4.ld b/bootloader/targets/f6/stm32wb55xx_flash_cm4.ld index 41d36b11..fdcda4ac 100644 --- a/bootloader/targets/f6/stm32wb55xx_flash_cm4.ld +++ b/bootloader/targets/f6/stm32wb55xx_flash_cm4.ld @@ -56,7 +56,7 @@ _Min_Stack_Size = 0x400; /* required amount of stack */ MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 32K -RAM1 (xrw) : ORIGIN = 0x20000004, LENGTH = 0x2FFFC +RAM1 (xrw) : ORIGIN = 0x20000008, LENGTH = 0x2FFF8 RAM_SHARED (xrw) : ORIGIN = 0x20030000, LENGTH = 10K } diff --git a/bootloader/targets/f7/stm32wb55xx_flash_cm4.ld b/bootloader/targets/f7/stm32wb55xx_flash_cm4.ld index 41d36b11..fdcda4ac 100644 --- a/bootloader/targets/f7/stm32wb55xx_flash_cm4.ld +++ b/bootloader/targets/f7/stm32wb55xx_flash_cm4.ld @@ -56,7 +56,7 @@ _Min_Stack_Size = 0x400; /* required amount of stack */ MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 32K -RAM1 (xrw) : ORIGIN = 0x20000004, LENGTH = 0x2FFFC +RAM1 (xrw) : ORIGIN = 0x20000008, LENGTH = 0x2FFF8 RAM_SHARED (xrw) : ORIGIN = 0x20030000, LENGTH = 10K } diff --git a/scripts/assets.py b/scripts/assets.py index 154c360f..bb8f9724 100755 --- a/scripts/assets.py +++ b/scripts/assets.py @@ -34,13 +34,24 @@ class Main: self.parser_icons = self.subparsers.add_parser( "icons", help="Process icons and build icon registry" ) - self.parser_icons.add_argument( - "-s", "--source-directory", help="Source directory" - ) - self.parser_icons.add_argument( - "-o", "--output-directory", help="Output directory" - ) + self.parser_icons.add_argument("source_directory", help="Source directory") + self.parser_icons.add_argument("output_directory", help="Output directory") self.parser_icons.set_defaults(func=self.icons) + + self.parser_manifest = self.subparsers.add_parser( + "manifest", help="Create directory Manifest" + ) + self.parser_manifest.add_argument("local_path", help="local_path") + self.parser_manifest.set_defaults(func=self.manifest) + + self.parser_copro = self.subparsers.add_parser( + "copro", help="Gather copro binaries for packaging" + ) + self.parser_copro.add_argument("cube_dir", help="Path to Cube folder") + self.parser_copro.add_argument("output_dir", help="Path to output folder") + self.parser_copro.add_argument("mcu", help="MCU series as in copro folder") + self.parser_copro.set_defaults(func=self.copro) + # logging self.logger = logging.getLogger() @@ -163,6 +174,42 @@ class Main: extension = filename.lower().split(".")[-1] return extension in ICONS_SUPPORTED_FORMATS + def manifest(self): + from flipper.manifest import Manifest + + directory_path = os.path.normpath(self.args.local_path) + if not os.path.isdir(directory_path): + self.logger.error(f'"{directory_path}" is not a directory') + exit(255) + manifest_file = os.path.join(directory_path, "Manifest") + old_manifest = Manifest() + if os.path.exists(manifest_file): + self.logger.info( + f"old manifest is present, loading for compare and removing file" + ) + old_manifest.load(manifest_file) + os.unlink(manifest_file) + self.logger.info(f'Creating new Manifest for directory "{directory_path}"') + new_manifest = Manifest() + new_manifest.create(directory_path) + new_manifest.save(manifest_file) + self.logger.info(f"Comparing new manifest with old") + only_in_old, changed, only_in_new = Manifest.compare(old_manifest, new_manifest) + for record in only_in_old: + self.logger.info(f"Only in old: {record}") + for record in changed: + self.logger.info(f"Changed: {record}") + for record in only_in_new: + self.logger.info(f"Only in new: {record}") + self.logger.info(f"Complete") + + def copro(self): + from flipper.copro import Copro + + copro = Copro(self.args.mcu) + copro.loadCubeInfo(self.args.cube_dir) + copro.bundle(self.args.output_dir) + if __name__ == "__main__": Main()() diff --git a/scripts/flipper/copro.py b/scripts/flipper/copro.py new file mode 100644 index 00000000..bb1b40f7 --- /dev/null +++ b/scripts/flipper/copro.py @@ -0,0 +1,95 @@ +import logging +import datetime +import shutil +import json + +import xml.etree.ElementTree as ET +from .utils import * + +CUBE_COPRO_PATH = "Projects/STM32WB_Copro_Wireless_Binaries" + +MANIFEST_TEMPLATE = { + "manifest": {"version": 0, "timestamp": 0}, + "copro": { + "fus": {"version": {"major": 1, "minor": 2, "sub": 0}, "files": []}, + "radio": { + "version": { + "type": 1, + "major": 1, + "minor": 12, + "sub": 0, + "branch": 0, + "release": 7, + }, + "files": [], + }, + }, +} + + +class Copro: + def __init__(self, mcu): + self.mcu = mcu + self.version = None + self.cube_dir = None + self.mcu_copro = None + self.logger = logging.getLogger(self.__class__.__name__) + + def loadCubeInfo(self, cube_dir): + if not os.path.isdir(cube_dir): + raise Exception(f'"{cube_dir}" doesn\'t exists') + self.cube_dir = cube_dir + self.mcu_copro = os.path.join(self.cube_dir, CUBE_COPRO_PATH, self.mcu) + if not os.path.isdir(self.mcu_copro): + raise Exception(f'"{self.mcu_copro}" doesn\'t exists') + cube_manifest_file = os.path.join(self.cube_dir, "package.xml") + cube_manifest = ET.parse(cube_manifest_file) + cube_version = cube_manifest.find("PackDescription") + if not cube_version: + raise Exception(f"Unknown Cube manifest format") + cube_version = cube_version.get("Release") + if not cube_version or not cube_version.startswith("FW.WB"): + raise Exception(f"Incorrect Cube package or version info") + cube_version = cube_version.replace("FW.WB.", "", 1) + if cube_version != "1.12.0": + raise Exception(f"Unknonwn cube version") + self.version = cube_version + + def addFile(self, array, filename, **kwargs): + source_file = os.path.join(self.mcu_copro, filename) + destination_file = os.path.join(self.output_dir, filename) + shutil.copyfile(source_file, destination_file) + array.append( + {"name": filename, "sha256": file_sha256(destination_file), **kwargs} + ) + + def bundle(self, output_dir): + if not os.path.isdir(output_dir): + raise Exception(f'"{output_dir}" doesn\'t exists') + self.output_dir = output_dir + manifest_file = os.path.join(self.output_dir, "Manifest.json") + # Form Manifest + manifest = dict(MANIFEST_TEMPLATE) + manifest["manifest"]["timestamp"] = timestamp() + # Old FUS Update + self.addFile( + manifest["copro"]["fus"]["files"], + "stm32wb5x_FUS_fw_for_fus_0_5_3.bin", + condition="==0.5.3", + address="0x080EC000", + ) + # New FUS Update + self.addFile( + manifest["copro"]["fus"]["files"], + "stm32wb5x_FUS_fw.bin", + condition=">0.5.3", + address="0x080EC000", + ) + # BLE Full Stack + self.addFile( + manifest["copro"]["radio"]["files"], + "stm32wb5x_BLE_Stack_full_fw.bin", + address="0x080CA000", + ) + # Save manifest to + json.dump(manifest, open(manifest_file, "w")) diff --git a/scripts/flipper/fstree.py b/scripts/flipper/fstree.py new file mode 100644 index 00000000..c42b4aa6 --- /dev/null +++ b/scripts/flipper/fstree.py @@ -0,0 +1,76 @@ +from enum import Enum +from collections import OrderedDict + + +class FsNode: + class Type(Enum): + File = 0 + Directory = 1 + + def __init__(self, name: str, type: "FsNode.Type", **kwargs): + self.name = name + self.type = type + self.data = kwargs + self.parent = None + self.children = OrderedDict() + + def addChild(self, node: "FsNode"): + self.children[node.name] = node + node.parent = self + + def addDirectory(self, path): + fragments = path.split("/") + name = fragments[-1] + fragments = fragments[:-1] + parent_node = self.traverse(fragments) + if not parent_node: + raise Exception(f"No parent node found for: {path}") + parent_node.addChild(FsNode(name, FsNode.Type.Directory)) + + def addFile(self, path, md5, size): + fragments = path.split("/") + name = fragments[-1] + fragments = fragments[:-1] + parent_node = self.traverse(fragments) + if not parent_node: + raise Exception(f"No parent node found for: {path}") + parent_node.addChild(FsNode(name, FsNode.Type.File, md5=md5, size=size)) + + def getChild(self, name): + return self.children[name] + + def traverse(self, fragments): + current = self + for fragment in fragments: + current = current.getChild(fragment) + if not current: + break + return current + + def getPath(self): + fragments = [] + current = self + while current.parent: + fragments.append(current.name) + current = current.parent + return "/".join(reversed(fragments)) + + def dump(self): + ret = {} + ret["name"] = (self.name,) + ret["type"] = (self.type,) + ret["path"] = (self.getPath(),) + if len(self.children): + ret["children"] = [node.dump() for node in self.children.values()] + return ret + + +def compare_fs_trees(left: FsNode, right: FsNode): + # import pprint + # pprint.pprint(left.dump()) + # pprint.pprint(right.dump()) + + only_in_left = [] + changed = [] + only_in_right = [] + return [], [], [] diff --git a/scripts/flipper/manifest.py b/scripts/flipper/manifest.py new file mode 100644 index 00000000..878e4c02 --- /dev/null +++ b/scripts/flipper/manifest.py @@ -0,0 +1,168 @@ +import datetime +import logging +import os + +from .utils import * +from .fstree import * + +MANIFEST_VERSION = 0 + + +class ManifestRecord: + tag = None + + @staticmethod + def fromLine(line): + raise NotImplementedError + + def toLine(self): + raise NotImplementedError + + def _unpack(self, manifest, key, type): + key, value = manifest.readline().split(":", 1) + assert key == key + return type(value) + + +MANIFEST_TAGS_RECORDS = {} + + +def addManifestRecord(record: ManifestRecord): + assert record.tag + MANIFEST_TAGS_RECORDS[record.tag] = record + + +class ManifestRecordVersion(ManifestRecord): + tag = "V" + + def __init__(self, version): + self.version = version + + @staticmethod + def fromLine(line): + return ManifestRecordVersion(int(line)) + + def toLine(self): + return f"{self.tag}:{self.version}\n" + + +addManifestRecord(ManifestRecordVersion) + + +class ManifestRecordTimestamp(ManifestRecord): + tag = "T" + + def __init__(self, timestamp: int): + self.timestamp = int(timestamp) + + @staticmethod + def fromLine(line): + return ManifestRecordTimestamp(int(line)) + + def toLine(self): + return f"{self.tag}:{self.timestamp}\n" + + +addManifestRecord(ManifestRecordTimestamp) + + +class ManifestRecordDirectory(ManifestRecord): + tag = "D" + + def __init__(self, path: str): + self.path = path + + @staticmethod + def fromLine(line): + return ManifestRecordDirectory(line) + + def toLine(self): + return f"{self.tag}:{self.path}\n" + + +addManifestRecord(ManifestRecordDirectory) + + +class ManifestRecordFile(ManifestRecord): + tag = "F" + + def __init__(self, path: str, md5: str, size: int): + self.path = path + self.md5 = md5 + self.size = size + + @staticmethod + def fromLine(line): + data = line.split(":", 3) + return ManifestRecordFile(data[2], data[0], data[1]) + + def toLine(self): + return f"{self.tag}:{self.md5}:{self.size}:{self.path}\n" + + +addManifestRecord(ManifestRecordFile) + + +class Manifest: + def __init__(self): + self.version = None + self.records = [] + self.records.append(ManifestRecordVersion(MANIFEST_VERSION)) + self.records.append(ManifestRecordTimestamp(timestamp())) + self.logger = logging.getLogger(self.__class__.__name__) + + def load(self, filename): + manifest = open(filename, "r") + for line in manifest.readlines(): + line = line.strip() + if len(line) == 0: + continue + tag, line = line.split(":", 1) + record = MANIFEST_TAGS_RECORDS[tag].fromLine(line) + self.records.append(record) + + def save(self, filename): + manifest = open(filename, "w+") + for record in self.records: + manifest.write(record.toLine()) + manifest.close() + + def addDirectory(self, path): + self.records.append(ManifestRecordDirectory(path)) + + def addFile(self, path, md5, size): + self.records.append(ManifestRecordFile(path, md5, size)) + + def create(self, directory_path): + for root, dirs, files in os.walk(directory_path): + relative_root = root.replace(directory_path, "", 1) + if relative_root.startswith("/"): + relative_root = relative_root[1:] + # process directories + for dir in dirs: + relative_dir_path = os.path.join(relative_root, dir) + self.logger.info(f'Adding directory: "{relative_dir_path}"') + self.addDirectory(relative_dir_path) + # Process files + for file in files: + relative_file_path = os.path.join(relative_root, file) + full_file_path = os.path.join(root, file) + self.logger.info(f'Adding file: "{relative_file_path}"') + self.addFile( + relative_file_path, + file_md5(full_file_path), + os.path.getsize(full_file_path), + ) + + def toFsTree(self): + root = FsNode("", FsNode.Type.Directory) + for record in self.records: + if isinstance(record, ManifestRecordDirectory): + root.addDirectory(record.path) + elif isinstance(record, ManifestRecordFile): + root.addFile(record.path, record.md5, record.size) + return root + + @staticmethod + def compare(left: "Manifest", right: "Manifest"): + return compare_fs_trees(left.toFsTree(), right.toFsTree()) diff --git a/scripts/flipper/utils.py b/scripts/flipper/utils.py new file mode 100644 index 00000000..056d4502 --- /dev/null +++ b/scripts/flipper/utils.py @@ -0,0 +1,27 @@ +import datetime +import hashlib +import os + + +def timestamp(): + return int(datetime.datetime.now().timestamp()) + + +def file_hash(path: str, algo: str, block_size: int = 4096): + fd = open(path, "rb") + h = hashlib.new(algo) + while True: + data = fd.read(block_size) + if len(data) > 0: + h.update(data) + else: + break + return h.hexdigest() + + +def file_md5(path, block_size=4096): + return file_hash(path, "md5", block_size) + + +def file_sha256(path, block_size=4096): + return file_hash(path, "sha256", block_size)