From e8499e4edeaedfc9ad22bc6153b7d598d7c1a7d2 Mon Sep 17 00:00:00 2001 From: hedger Date: Tue, 19 Apr 2022 11:03:28 +0300 Subject: [PATCH] [FL-2477] Updater support for resource bundles (#1131) * Resource unpacking core * Added more fields to manifest; updated dist scripts * Python linter fixes * Parsing manifest before unpacking * Updated pipelines for separate resource build * Removed raw path formatting * Visual progress for resource extraction * Renamed update status enum Co-authored-by: Aleksandr Kutuzov --- .github/workflows/build.yml | 4 +- Makefile | 12 +- .../updater/scenes/updater_scene_main.c | 2 +- applications/updater/util/update_task.c | 3 +- applications/updater/util/update_task.h | 3 +- .../updater/util/update_task_workers.c | 117 +++++++++++++++--- lib/toolbox/tar/tar_archive.c | 44 ++++++- lib/toolbox/tar/tar_archive.h | 7 ++ lib/update_util/update_manifest.c | 59 +++++++-- lib/update_util/update_manifest.h | 3 + scripts/dist.py | 8 ++ scripts/update.py | 45 ++++++- 12 files changed, 260 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f158efbb..b1063e0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,8 +76,7 @@ jobs: with: run: | set -e - make -C assets clean - make -C assets + make assets_manifest git diff --quiet || ( echo "Assets recompilation required."; exit 255 ) - name: 'Build the firmware in docker' @@ -118,7 +117,6 @@ jobs: - name: 'Bundle resources' if: ${{ !github.event.pull_request.head.repo.fork }} run: | - ./scripts/assets.py manifest assets/resources tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources - name: 'Bundle core2 firmware' diff --git a/Makefile b/Makefile index 06486f85..d90c4e73 100644 --- a/Makefile +++ b/Makefile @@ -92,9 +92,19 @@ updater_clean: updater_debug: @$(MAKE) -C $(PROJECT_ROOT)/firmware -j$(NPROCS) RAM_EXEC=1 debug +.PHONY: updater_package_bin +updater_package_bin: firmware_all updater + @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)" + .PHONY: updater_package updater_package: firmware_all updater - @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)" + @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) -a assets/resources --bundlever "$(VERSION_STRING)" + +.PHONY: assets_manifest +assets_manifest: + @$(MAKE) -C $(PROJECT_ROOT)/assets clean + @$(MAKE) -C $(PROJECT_ROOT)/assets + @$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources .PHONY: flash_radio flash_radio: diff --git a/applications/updater/scenes/updater_scene_main.c b/applications/updater/scenes/updater_scene_main.c index d7b28a9d..31a212e7 100644 --- a/applications/updater/scenes/updater_scene_main.c +++ b/applications/updater/scenes/updater_scene_main.c @@ -73,7 +73,7 @@ bool updater_scene_main_on_event(void* context, SceneManagerEvent event) { case UpdaterCustomEventRetryUpdate: if(!update_task_is_running(updater->update_task) && - (update_task_get_state(updater->update_task)->stage != UpdateTaskStageComplete)) + (update_task_get_state(updater->update_task)->stage != UpdateTaskStageCompleted)) update_task_start(updater->update_task); consumed = true; break; diff --git a/applications/updater/util/update_task.c b/applications/updater/util/update_task.c index 8ca5aad2..04405bcf 100644 --- a/applications/updater/util/update_task.c +++ b/applications/updater/util/update_task.c @@ -19,7 +19,8 @@ static const char* update_task_stage_descr[] = { [UpdateTaskStageRadioCommit] = "Applying radio stack", [UpdateTaskStageLfsBackup] = "Backing up LFS", [UpdateTaskStageLfsRestore] = "Restoring LFS", - [UpdateTaskStageComplete] = "Complete", + [UpdateTaskStageAssetsUpdate] = "Updating assets", + [UpdateTaskStageCompleted] = "Completed!", [UpdateTaskStageError] = "Error", }; diff --git a/applications/updater/util/update_task.h b/applications/updater/util/update_task.h index 32dea989..25650de8 100644 --- a/applications/updater/util/update_task.h +++ b/applications/updater/util/update_task.h @@ -23,7 +23,8 @@ typedef enum { UpdateTaskStageRadioCommit, UpdateTaskStageLfsBackup, UpdateTaskStageLfsRestore, - UpdateTaskStageComplete, + UpdateTaskStageAssetsUpdate, + UpdateTaskStageCompleted, UpdateTaskStageError, } UpdateTaskStage; diff --git a/applications/updater/util/update_task_workers.c b/applications/updater/util/update_task_workers.c index fb1e86be..2a22692b 100644 --- a/applications/updater/util/update_task_workers.c +++ b/applications/updater/util/update_task_workers.c @@ -8,6 +8,7 @@ #include #include #include +#include #define CHECK_RESULT(x) \ if(!(x)) { \ @@ -19,6 +20,8 @@ /* Written into DFU file by build pipeline */ #define FLIPPER_ZERO_DFU_DEVICE_CODE 0xFFFF +#define EXT_PATH "/ext" + static const DfuValidationParams flipper_dfu_params = { .device = FLIPPER_ZERO_DFU_DEVICE_CODE, .product = STM_DFU_PRODUCT_ID, @@ -85,7 +88,7 @@ int32_t update_task_worker_flash_writer(void* context) { CHECK_RESULT(dfu_file_process_targets(&page_task, update_task->file, valid_targets)); } - update_task_set_progress(update_task, UpdateTaskStageComplete, 100); + update_task_set_progress(update_task, UpdateTaskStageCompleted, 100); furi_hal_rtc_set_boot_mode(FuriHalRtcBootModePostUpdate); @@ -99,6 +102,95 @@ int32_t update_task_worker_flash_writer(void* context) { return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED; } +static bool update_task_pre_update(UpdateTask* update_task) { + bool success = false; + string_t backup_file_path; + string_init(backup_file_path); + path_concat( + string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path); + + update_task->state.total_stages = 1; + update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0); + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops + if((success = lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) { + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate); + } + + string_clear(backup_file_path); + return success; +} + +typedef struct { + UpdateTask* update_task; + int32_t total_files, processed_files; +} TarUnpackProgress; + +static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) { + UNUSED(name); + UNUSED(is_directory); + TarUnpackProgress* unpack_progress = context; + unpack_progress->processed_files++; + update_task_set_progress( + unpack_progress->update_task, + UpdateTaskStageProgress, + unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1)); + return true; +} + +static bool update_task_post_update(UpdateTask* update_task) { + bool success = false; + + string_t file_path; + string_init(file_path); + + update_task->state.total_stages = 2; + + do { + CHECK_RESULT(update_task_parse_manifest(update_task)); + path_concat( + string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path); + + bool unpack_resources = !string_empty_p(update_task->manifest->resource_bundle); + if(unpack_resources) { + update_task->state.total_stages++; + } + + update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0); + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); + + CHECK_RESULT(lfs_backup_unpack(update_task->storage, string_get_cstr(file_path))); + + if(unpack_resources) { + TarUnpackProgress progress = { + .update_task = update_task, + .total_files = 0, + .processed_files = 0, + }; + update_task_set_progress(update_task, UpdateTaskStageAssetsUpdate, 0); + + path_concat( + string_get_cstr(update_task->update_path), + string_get_cstr(update_task->manifest->resource_bundle), + file_path); + + update_task_set_progress(update_task, UpdateTaskStageProgress, 0); + TarArchive* archive = tar_archive_alloc(update_task->storage); + tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress); + success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ); + if(success) { + progress.total_files = tar_archive_get_entries_count(archive); + if(progress.total_files > 0) { + tar_archive_unpack_to(archive, EXT_PATH); + } + } + tar_archive_free(archive); + } + } while(false); + + string_clear(file_path); + return success; +} + int32_t update_task_worker_backup_restore(void* context) { furi_assert(context); UpdateTask* update_task = context; @@ -112,37 +204,22 @@ int32_t update_task_worker_backup_restore(void* context) { } update_task->state.current_stage_idx = 0; - update_task->state.total_stages = 1; if(!update_operation_get_current_package_path(update_task->storage, update_task->update_path)) { return UPDATE_TASK_FAILED; } - string_t backup_file_path; - string_init(backup_file_path); - path_concat( - string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path); - if(boot_mode == FuriHalRtcBootModePreUpdate) { - update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0); - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops - if((success = - lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) { - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate); - } + success = update_task_pre_update(update_task); } else if(boot_mode == FuriHalRtcBootModePostUpdate) { - update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0); - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); - success = lfs_backup_unpack(update_task->storage, string_get_cstr(backup_file_path)); + success = update_task_post_update(update_task); } if(success) { - update_task_set_progress(update_task, UpdateTaskStageComplete, 100); + update_task_set_progress(update_task, UpdateTaskStageCompleted, 100); } else { update_task_set_progress(update_task, UpdateTaskStageError, update_task->state.progress); } - string_clear(backup_file_path); - return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED; -} +} \ No newline at end of file diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c index 7be68bd8..e3cc1818 100644 --- a/lib/toolbox/tar/tar_archive.c +++ b/lib/toolbox/tar/tar_archive.c @@ -15,6 +15,8 @@ typedef struct TarArchive { Storage* storage; mtar_t tar; + tar_unpack_file_cb unpack_cb; + void* unpack_cb_context; } TarArchive; /* API WRAPPER */ @@ -51,6 +53,7 @@ TarArchive* tar_archive_alloc(Storage* storage) { furi_check(storage); TarArchive* archive = malloc(sizeof(TarArchive)); archive->storage = storage; + archive->unpack_cb = NULL; return archive; } @@ -92,6 +95,28 @@ void tar_archive_free(TarArchive* archive) { } } +void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context) { + furi_assert(archive); + archive->unpack_cb = callback; + archive->unpack_cb_context = context; +} + +static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) { + UNUSED(tar); + UNUSED(header); + int32_t* counter = param; + (*counter)++; + return 0; +} + +int32_t tar_archive_get_entries_count(TarArchive* archive) { + int32_t counter = 0; + if(mtar_foreach(&archive->tar, tar_archive_entry_counter, &counter) != MTAR_ESUCCESS) { + counter = -1; + } + return counter; +} + bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath) { furi_assert(archive); return (mtar_write_dir_header(&archive->tar, dirpath) == MTAR_ESUCCESS); @@ -142,14 +167,25 @@ typedef struct { static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) { TarArchiveDirectoryOpParams* op_params = param; + TarArchive* archive = op_params->archive; string_t fname; + bool skip_entry = false; + if(archive->unpack_cb) { + skip_entry = !archive->unpack_cb( + header->name, header->type == MTAR_TDIR, archive->unpack_cb_context); + } + + if(skip_entry) { + FURI_LOG_W(TAG, "filter: skipping entry \"%s\"", header->name); + return 0; + } + if(header->type == MTAR_TDIR) { string_init(fname); path_concat(op_params->work_dir, header->name, fname); - bool create_res = - storage_simply_mkdir(op_params->archive->storage, string_get_cstr(fname)); + bool create_res = storage_simply_mkdir(archive->storage, string_get_cstr(fname)); string_clear(fname); return create_res ? 0 : -1; } @@ -162,7 +198,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, string_init(fname); path_concat(op_params->work_dir, header->name, fname); FURI_LOG_I(TAG, "Extracting %d bytes to '%s'", header->size, header->name); - File* out_file = storage_file_alloc(op_params->archive->storage); + File* out_file = storage_file_alloc(archive->storage); uint8_t* readbuf = malloc(FILE_BLOCK_SIZE); bool failed = false; @@ -303,4 +339,4 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch free(name); storage_file_free(directory); return success; -} \ No newline at end of file +} diff --git a/lib/toolbox/tar/tar_archive.h b/lib/toolbox/tar/tar_archive.h index fe3e248e..a8976181 100644 --- a/lib/toolbox/tar/tar_archive.h +++ b/lib/toolbox/tar/tar_archive.h @@ -34,6 +34,13 @@ bool tar_archive_add_file( bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const char* path_prefix); +int32_t tar_archive_get_entries_count(TarArchive* archive); + +/* Optional per-entry callback on unpacking - return false to skip entry */ +typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context); + +void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context); + /* Low-level API */ bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath); diff --git a/lib/update_util/update_manifest.c b/lib/update_util/update_manifest.c index 2ae52f17..5ec942c8 100644 --- a/lib/update_util/update_manifest.c +++ b/lib/update_util/update_manifest.c @@ -4,12 +4,24 @@ #include #include +#define MANIFEST_KEY_INFO "Info" +#define MANIFEST_KEY_TARGET "Target" +#define MANIFEST_KEY_LOADER_FILE "Loader" +#define MANIFEST_KEY_LOADER_CRC "Loader CRC" +#define MANIFEST_KEY_DFU_FILE "Firmware" +#define MANIFEST_KEY_RADIO_FILE "Radio" +#define MANIFEST_KEY_RADIO_ADDRESS "Radio address" +#define MANIFEST_KEY_RADIO_VERSION "Radio version" +#define MANIFEST_KEY_RADIO_CRC "Radio CRC" +#define MANIFEST_KEY_ASSETS_FILE "Assets" + UpdateManifest* update_manifest_alloc() { UpdateManifest* update_manifest = malloc(sizeof(UpdateManifest)); string_init(update_manifest->version); string_init(update_manifest->firmware_dfu_image); string_init(update_manifest->radio_image); string_init(update_manifest->staged_loader_file); + string_init(update_manifest->resource_bundle); update_manifest->target = 0; update_manifest->valid = false; return update_manifest; @@ -21,6 +33,7 @@ void update_manifest_free(UpdateManifest* update_manifest) { string_clear(update_manifest->firmware_dfu_image); string_clear(update_manifest->radio_image); string_clear(update_manifest->staged_loader_file); + string_clear(update_manifest->resource_bundle); free(update_manifest); } @@ -36,21 +49,47 @@ static bool string_init(filetype); update_manifest->valid = flipper_format_read_header(flipper_file, filetype, &version) && - flipper_format_read_string(flipper_file, "Info", update_manifest->version) && - flipper_format_read_uint32(flipper_file, "Target", &update_manifest->target, 1) && - flipper_format_read_string(flipper_file, "Loader", update_manifest->staged_loader_file) && + flipper_format_read_string(flipper_file, MANIFEST_KEY_INFO, update_manifest->version) && + flipper_format_read_uint32( + flipper_file, MANIFEST_KEY_TARGET, &update_manifest->target, 1) && + flipper_format_read_string( + flipper_file, MANIFEST_KEY_LOADER_FILE, update_manifest->staged_loader_file) && flipper_format_read_hex( flipper_file, - "Loader CRC", + MANIFEST_KEY_LOADER_CRC, (uint8_t*)&update_manifest->staged_loader_crc, sizeof(uint32_t)); string_clear(filetype); - /* Optional fields - we can have dfu, radio, or both */ - flipper_format_read_string(flipper_file, "Firmware", update_manifest->firmware_dfu_image); - flipper_format_read_string(flipper_file, "Radio", update_manifest->radio_image); - flipper_format_read_hex( - flipper_file, "Radio address", (uint8_t*)&update_manifest->radio_address, sizeof(uint32_t)); + if(update_manifest->valid) { + /* Optional fields - we can have dfu, radio, or both */ + flipper_format_read_string( + flipper_file, MANIFEST_KEY_DFU_FILE, update_manifest->firmware_dfu_image); + flipper_format_read_string( + flipper_file, MANIFEST_KEY_RADIO_FILE, update_manifest->radio_image); + flipper_format_read_hex( + flipper_file, + MANIFEST_KEY_RADIO_ADDRESS, + (uint8_t*)&update_manifest->radio_address, + sizeof(uint32_t)); + flipper_format_read_hex( + flipper_file, + MANIFEST_KEY_RADIO_VERSION, + (uint8_t*)&update_manifest->radio_version, + sizeof(uint32_t)); + flipper_format_read_hex( + flipper_file, + MANIFEST_KEY_RADIO_CRC, + (uint8_t*)&update_manifest->radio_crc, + sizeof(uint32_t)); + flipper_format_read_string( + flipper_file, MANIFEST_KEY_ASSETS_FILE, update_manifest->resource_bundle); + + update_manifest->valid = + (!string_empty_p(update_manifest->firmware_dfu_image) || + !string_empty_p(update_manifest->radio_image) || + !string_empty_p(update_manifest->resource_bundle)); + } return update_manifest->valid; } @@ -83,4 +122,4 @@ bool update_manifest_init_mem( flipper_format_free(flipper_file); return update_manifest->valid; -} +} \ No newline at end of file diff --git a/lib/update_util/update_manifest.h b/lib/update_util/update_manifest.h index 7b60f58a..7d4757c4 100644 --- a/lib/update_util/update_manifest.h +++ b/lib/update_util/update_manifest.h @@ -21,6 +21,9 @@ typedef struct { string_t firmware_dfu_image; string_t radio_image; uint32_t radio_address; + uint32_t radio_version; + uint32_t radio_crc; + string_t resource_bundle; bool valid; } UpdateManifest; diff --git a/scripts/dist.py b/scripts/dist.py index 172ecc01..541c386e 100755 --- a/scripts/dist.py +++ b/scripts/dist.py @@ -18,6 +18,7 @@ class Main(App): self.parser_copy.add_argument("-t", dest="target", required=True) self.parser_copy.add_argument("-p", dest="projects", nargs="+", required=True) self.parser_copy.add_argument("-s", dest="suffix", required=True) + self.parser_copy.add_argument("-a", dest="assets", required=False) self.parser_copy.add_argument( "--bundlever", dest="version", @@ -83,6 +84,13 @@ class Main(App): "-stage", self.get_dist_filepath(self.get_project_filename("updater", "bin")), ] + if self.args.assets: + bundle_args.extend( + ( + "-a", + self.args.assets, + ) + ) self.logger.info( f"Use this directory to self-update your Flipper:\n\t{bundle_dir}" ) diff --git a/scripts/update.py b/scripts/update.py index 7e49aad0..187c9288 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -3,12 +3,16 @@ from flipper.app import App from flipper.utils.fff import FlipperFormatFile from os.path import basename, join, exists -from os import makedirs +import os import shutil import zlib +import tarfile class Main(App): + # No compression, plain tar + ASSET_TAR_MODE = "w:" + def init(self): self.subparsers = self.parser.add_subparsers(help="sub-command help") @@ -20,24 +24,41 @@ class Main(App): self.parser_generate.add_argument("-d", dest="directory", required=True) self.parser_generate.add_argument("-v", dest="version", required=True) self.parser_generate.add_argument("-t", dest="target", required=True) - self.parser_generate.add_argument("-dfu", dest="dfu", required=True) + self.parser_generate.add_argument("-dfu", dest="dfu", required=False) + self.parser_generate.add_argument("-a", dest="assets", required=False) self.parser_generate.add_argument("-stage", dest="stage", required=True) - self.parser_generate.add_argument("-radio", dest="radiobin", required=False) + self.parser_generate.add_argument( + "-radio", dest="radiobin", default="", required=False + ) self.parser_generate.add_argument( "-radioaddr", dest="radioaddr", required=False ) + self.parser_generate.add_argument( + "-radiover", dest="radioversion", required=False + ) self.parser_generate.set_defaults(func=self.generate) def generate(self): stage_basename = basename(self.args.stage) dfu_basename = basename(self.args.dfu) + radiobin_basename = basename(self.args.radiobin) + assets_basename = "" if not exists(self.args.directory): - makedirs(self.args.directory) + os.makedirs(self.args.directory) shutil.copyfile(self.args.stage, join(self.args.directory, stage_basename)) shutil.copyfile(self.args.dfu, join(self.args.directory, dfu_basename)) + if radiobin_basename: + shutil.copyfile( + self.args.radiobin, join(self.args.directory, radiobin_basename) + ) + if self.args.assets: + assets_basename = "assets.tar" + self.package_assets( + self.args.assets, join(self.args.directory, assets_basename) + ) file = FlipperFormatFile() file.setHeader("Flipper firmware upgrade configuration", 1) @@ -47,12 +68,24 @@ class Main(App): file.writeComment("little-endian hex!") file.writeKey("Loader CRC", self.int2ffhex(self.crc(self.args.stage))) file.writeKey("Firmware", dfu_basename) - file.writeKey("Radio", self.args.radiobin or "") + file.writeKey("Radio", radiobin_basename or "") file.writeKey("Radio address", self.int2ffhex(self.args.radioaddr or 0)) - file.save("%s/update.fuf" % self.args.directory) + file.writeKey("Radio version", self.int2ffhex(self.args.radioversion or 0)) + if radiobin_basename: + file.writeKey("Radio CRC", self.int2ffhex(self.crc(self.args.radiobin))) + else: + file.writeKey("Radio CRC", self.int2ffhex(0)) + file.writeKey("Assets", assets_basename) + file.save(join(self.args.directory, "update.fuf")) return 0 + def package_assets(self, srcdir: str, dst_name: str): + with tarfile.open( + dst_name, self.ASSET_TAR_MODE, format=tarfile.USTAR_FORMAT + ) as tarball: + tarball.add(srcdir, arcname="") + @staticmethod def int2ffhex(value: int): hexstr = "%08X" % value