liz/flipperzero-firmware
liz/flipperzero-firmware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[FL-2477] Updater support for resource bundles (#1131)

Browse Source 
* Resource unpacking core
* Added more fields to manifest; updated dist scripts
* Python linter fixes
* Parsing manifest before unpacking
* Updated pipelines for separate resource build
* Removed raw path formatting
* Visual progress for resource extraction
* Renamed update status enum

Co-authored-by: Aleksandr Kutuzov <alleteam@gmail.com>
This commit is contained in:
hedger 2022-04-19 11:03:28 +03:00 committed by GitHub
parent 1623134a82
commit e8499e4ede
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 260 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build.yml vendored
View File

@ -76,8 +76,7 @@ jobs:
with:
run: |
set -e
make -C assets clean
make -C assets
make assets_manifest
git diff --quiet || ( echo "Assets recompilation required."; exit 255 )
- name: 'Build the firmware in docker'
@ -118,7 +117,6 @@ jobs:
- name: 'Bundle resources'
if: ${{ !github.event.pull_request.head.repo.fork }}
run: |
./scripts/assets.py manifest assets/resources
tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources
- name: 'Bundle core2 firmware'

12
Makefile
View File

@ -92,9 +92,19 @@ updater_clean:
updater_debug:
@$(MAKE) -C $(PROJECT_ROOT)/firmware -j$(NPROCS) RAM_EXEC=1 debug
.PHONY: updater_package_bin
updater_package_bin: firmware_all updater
@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)"
.PHONY: updater_package
updater_package: firmware_all updater
@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)"
@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) -a assets/resources --bundlever "$(VERSION_STRING)"
.PHONY: assets_manifest
assets_manifest:
@$(MAKE) -C $(PROJECT_ROOT)/assets clean
@$(MAKE) -C $(PROJECT_ROOT)/assets
@$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources
.PHONY: flash_radio
flash_radio:

2
applications/updater/scenes/updater_scene_main.c
View File

@ -73,7 +73,7 @@ bool updater_scene_main_on_event(void* context, SceneManagerEvent event) {
case UpdaterCustomEventRetryUpdate:
if(!update_task_is_running(updater->update_task) &&
(update_task_get_state(updater->update_task)->stage != UpdateTaskStageComplete))
(update_task_get_state(updater->update_task)->stage != UpdateTaskStageCompleted))
update_task_start(updater->update_task);
consumed = true;
break;

3
applications/updater/util/update_task.c
View File

@ -19,7 +19,8 @@ static const char* update_task_stage_descr[] = {
[UpdateTaskStageRadioCommit] = "Applying radio stack",
[UpdateTaskStageLfsBackup] = "Backing up LFS",
[UpdateTaskStageLfsRestore] = "Restoring LFS",
[UpdateTaskStageComplete] = "Complete",
[UpdateTaskStageAssetsUpdate] = "Updating assets",
[UpdateTaskStageCompleted] = "Completed!",
[UpdateTaskStageError] = "Error",
};

3
applications/updater/util/update_task.h
View File

@ -23,7 +23,8 @@ typedef enum {
UpdateTaskStageRadioCommit,
UpdateTaskStageLfsBackup,
UpdateTaskStageLfsRestore,
UpdateTaskStageComplete,
UpdateTaskStageAssetsUpdate,
UpdateTaskStageCompleted,
UpdateTaskStageError,
} UpdateTaskStage;

115
applications/updater/util/update_task_workers.c
View File

@ -8,6 +8,7 @@
#include <update_util/dfu_file.h>
#include <update_util/lfs_backup.h>
#include <update_util/update_operation.h>
#include <toolbox/tar/tar_archive.h>
#define CHECK_RESULT(x) \
if(!(x)) { \
@ -19,6 +20,8 @@
/* Written into DFU file by build pipeline */
#define FLIPPER_ZERO_DFU_DEVICE_CODE 0xFFFF
#define EXT_PATH "/ext"
static const DfuValidationParams flipper_dfu_params = {
.device = FLIPPER_ZERO_DFU_DEVICE_CODE,
.product = STM_DFU_PRODUCT_ID,
@ -85,7 +88,7 @@ int32_t update_task_worker_flash_writer(void* context) {
CHECK_RESULT(dfu_file_process_targets(&page_task, update_task->file, valid_targets));
}
update_task_set_progress(update_task, UpdateTaskStageComplete, 100);
update_task_set_progress(update_task, UpdateTaskStageCompleted, 100);
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModePostUpdate);
@ -99,6 +102,95 @@ int32_t update_task_worker_flash_writer(void* context) {
return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED;
}
static bool update_task_pre_update(UpdateTask* update_task) {
bool success = false;
string_t backup_file_path;
string_init(backup_file_path);
path_concat(
string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path);
update_task->state.total_stages = 1;
update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0);
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops
if((success = lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) {
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate);
}
string_clear(backup_file_path);
return success;
}
typedef struct {
UpdateTask* update_task;
int32_t total_files, processed_files;
} TarUnpackProgress;
static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) {
UNUSED(name);
UNUSED(is_directory);
TarUnpackProgress* unpack_progress = context;
unpack_progress->processed_files++;
update_task_set_progress(
unpack_progress->update_task,
UpdateTaskStageProgress,
unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1));
return true;
}
static bool update_task_post_update(UpdateTask* update_task) {
bool success = false;
string_t file_path;
string_init(file_path);
update_task->state.total_stages = 2;
do {
CHECK_RESULT(update_task_parse_manifest(update_task));
path_concat(
string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path);
bool unpack_resources = !string_empty_p(update_task->manifest->resource_bundle);
if(unpack_resources) {
update_task->state.total_stages++;
}
update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0);
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal);
CHECK_RESULT(lfs_backup_unpack(update_task->storage, string_get_cstr(file_path)));
if(unpack_resources) {
TarUnpackProgress progress = {
.update_task = update_task,
.total_files = 0,
.processed_files = 0,
};
update_task_set_progress(update_task, UpdateTaskStageAssetsUpdate, 0);
path_concat(
string_get_cstr(update_task->update_path),
string_get_cstr(update_task->manifest->resource_bundle),
file_path);
update_task_set_progress(update_task, UpdateTaskStageProgress, 0);
TarArchive* archive = tar_archive_alloc(update_task->storage);
tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress);
success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ);
if(success) {
progress.total_files = tar_archive_get_entries_count(archive);
if(progress.total_files > 0) {
tar_archive_unpack_to(archive, EXT_PATH);
}
}
tar_archive_free(archive);
}
} while(false);
string_clear(file_path);
return success;
}
int32_t update_task_worker_backup_restore(void* context) {
furi_assert(context);
UpdateTask* update_task = context;
@ -112,37 +204,22 @@ int32_t update_task_worker_backup_restore(void* context) {
}
update_task->state.current_stage_idx = 0;
update_task->state.total_stages = 1;
if(!update_operation_get_current_package_path(update_task->storage, update_task->update_path)) {
return UPDATE_TASK_FAILED;
}
string_t backup_file_path;
string_init(backup_file_path);
path_concat(
string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path);
if(boot_mode == FuriHalRtcBootModePreUpdate) {
update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0);
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops
if((success =
lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) {
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate);
}
success = update_task_pre_update(update_task);
} else if(boot_mode == FuriHalRtcBootModePostUpdate) {
update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0);
furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal);
success = lfs_backup_unpack(update_task->storage, string_get_cstr(backup_file_path));
success = update_task_post_update(update_task);
}
if(success) {
update_task_set_progress(update_task, UpdateTaskStageComplete, 100);
update_task_set_progress(update_task, UpdateTaskStageCompleted, 100);
} else {
update_task_set_progress(update_task, UpdateTaskStageError, update_task->state.progress);
}
string_clear(backup_file_path);
return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED;
}

42
lib/toolbox/tar/tar_archive.c
View File

@ -15,6 +15,8 @@
typedef struct TarArchive {
Storage* storage;
mtar_t tar;
tar_unpack_file_cb unpack_cb;
void* unpack_cb_context;
} TarArchive;
/* API WRAPPER */
@ -51,6 +53,7 @@ TarArchive* tar_archive_alloc(Storage* storage) {
furi_check(storage);
TarArchive* archive = malloc(sizeof(TarArchive));
archive->storage = storage;
archive->unpack_cb = NULL;
return archive;
}
@ -92,6 +95,28 @@ void tar_archive_free(TarArchive* archive) {
}
}
void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context) {
furi_assert(archive);
archive->unpack_cb = callback;
archive->unpack_cb_context = context;
}
static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) {
UNUSED(tar);
UNUSED(header);
int32_t* counter = param;
(*counter)++;
return 0;
}
int32_t tar_archive_get_entries_count(TarArchive* archive) {
int32_t counter = 0;
if(mtar_foreach(&archive->tar, tar_archive_entry_counter, &counter) != MTAR_ESUCCESS) {
counter = -1;
}
return counter;
}
bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath) {
furi_assert(archive);
return (mtar_write_dir_header(&archive->tar, dirpath) == MTAR_ESUCCESS);
@ -142,14 +167,25 @@ typedef struct {
static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) {
TarArchiveDirectoryOpParams* op_params = param;
TarArchive* archive = op_params->archive;
string_t fname;
bool skip_entry = false;
if(archive->unpack_cb) {
skip_entry = !archive->unpack_cb(
header->name, header->type == MTAR_TDIR, archive->unpack_cb_context);
}
if(skip_entry) {
FURI_LOG_W(TAG, "filter: skipping entry \"%s\"", header->name);
return 0;
}
if(header->type == MTAR_TDIR) {
string_init(fname);
path_concat(op_params->work_dir, header->name, fname);
bool create_res =
storage_simply_mkdir(op_params->archive->storage, string_get_cstr(fname));
bool create_res = storage_simply_mkdir(archive->storage, string_get_cstr(fname));
string_clear(fname);
return create_res ? 0 : -1;
}
@ -162,7 +198,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header,
string_init(fname);
path_concat(op_params->work_dir, header->name, fname);
FURI_LOG_I(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
File* out_file = storage_file_alloc(op_params->archive->storage);
File* out_file = storage_file_alloc(archive->storage);
uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
bool failed = false;

7
lib/toolbox/tar/tar_archive.h
View File

@ -34,6 +34,13 @@ bool tar_archive_add_file(
bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const char* path_prefix);
int32_t tar_archive_get_entries_count(TarArchive* archive);
/* Optional per-entry callback on unpacking - return false to skip entry */
typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context);
void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context);
/* Low-level API */
bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath);

53
lib/update_util/update_manifest.c
View File

@ -4,12 +4,24 @@
#include <flipper_format/flipper_format.h>
#include <flipper_format/flipper_format_i.h>
#define MANIFEST_KEY_INFO "Info"
#define MANIFEST_KEY_TARGET "Target"
#define MANIFEST_KEY_LOADER_FILE "Loader"
#define MANIFEST_KEY_LOADER_CRC "Loader CRC"
#define MANIFEST_KEY_DFU_FILE "Firmware"
#define MANIFEST_KEY_RADIO_FILE "Radio"
#define MANIFEST_KEY_RADIO_ADDRESS "Radio address"
#define MANIFEST_KEY_RADIO_VERSION "Radio version"
#define MANIFEST_KEY_RADIO_CRC "Radio CRC"
#define MANIFEST_KEY_ASSETS_FILE "Assets"
UpdateManifest* update_manifest_alloc() {
UpdateManifest* update_manifest = malloc(sizeof(UpdateManifest));
string_init(update_manifest->version);
string_init(update_manifest->firmware_dfu_image);
string_init(update_manifest->radio_image);
string_init(update_manifest->staged_loader_file);
string_init(update_manifest->resource_bundle);
update_manifest->target = 0;
update_manifest->valid = false;
return update_manifest;
@ -21,6 +33,7 @@ void update_manifest_free(UpdateManifest* update_manifest) {
string_clear(update_manifest->firmware_dfu_image);
string_clear(update_manifest->radio_image);
string_clear(update_manifest->staged_loader_file);
string_clear(update_manifest->resource_bundle);
free(update_manifest);
}
@ -36,21 +49,47 @@ static bool
string_init(filetype);
update_manifest->valid =
flipper_format_read_header(flipper_file, filetype, &version) &&
flipper_format_read_string(flipper_file, "Info", update_manifest->version) &&
flipper_format_read_uint32(flipper_file, "Target", &update_manifest->target, 1) &&
flipper_format_read_string(flipper_file, "Loader", update_manifest->staged_loader_file) &&
flipper_format_read_string(flipper_file, MANIFEST_KEY_INFO, update_manifest->version) &&
flipper_format_read_uint32(
flipper_file, MANIFEST_KEY_TARGET, &update_manifest->target, 1) &&
flipper_format_read_string(
flipper_file, MANIFEST_KEY_LOADER_FILE, update_manifest->staged_loader_file) &&
flipper_format_read_hex(
flipper_file,
"Loader CRC",
MANIFEST_KEY_LOADER_CRC,
(uint8_t*)&update_manifest->staged_loader_crc,
sizeof(uint32_t));
string_clear(filetype);
if(update_manifest->valid) {
/* Optional fields - we can have dfu, radio, or both */
flipper_format_read_string(flipper_file, "Firmware", update_manifest->firmware_dfu_image);
flipper_format_read_string(flipper_file, "Radio", update_manifest->radio_image);
flipper_format_read_string(
flipper_file, MANIFEST_KEY_DFU_FILE, update_manifest->firmware_dfu_image);
flipper_format_read_string(
flipper_file, MANIFEST_KEY_RADIO_FILE, update_manifest->radio_image);
flipper_format_read_hex(
flipper_file, "Radio address", (uint8_t*)&update_manifest->radio_address, sizeof(uint32_t));
flipper_file,
MANIFEST_KEY_RADIO_ADDRESS,
(uint8_t*)&update_manifest->radio_address,
sizeof(uint32_t));
flipper_format_read_hex(
flipper_file,
MANIFEST_KEY_RADIO_VERSION,
(uint8_t*)&update_manifest->radio_version,
sizeof(uint32_t));
flipper_format_read_hex(
flipper_file,
MANIFEST_KEY_RADIO_CRC,
(uint8_t*)&update_manifest->radio_crc,
sizeof(uint32_t));
flipper_format_read_string(
flipper_file, MANIFEST_KEY_ASSETS_FILE, update_manifest->resource_bundle);
update_manifest->valid =
(!string_empty_p(update_manifest->firmware_dfu_image) ||
!string_empty_p(update_manifest->radio_image) ||
!string_empty_p(update_manifest->resource_bundle));
}
return update_manifest->valid;
}

3
lib/update_util/update_manifest.h
View File

@ -21,6 +21,9 @@ typedef struct {
string_t firmware_dfu_image;
string_t radio_image;
uint32_t radio_address;
uint32_t radio_version;
uint32_t radio_crc;
string_t resource_bundle;
bool valid;
} UpdateManifest;

8
scripts/dist.py
View File

@ -18,6 +18,7 @@ class Main(App):
self.parser_copy.add_argument("-t", dest="target", required=True)
self.parser_copy.add_argument("-p", dest="projects", nargs="+", required=True)
self.parser_copy.add_argument("-s", dest="suffix", required=True)
self.parser_copy.add_argument("-a", dest="assets", required=False)
self.parser_copy.add_argument(
"--bundlever",
dest="version",
@ -83,6 +84,13 @@ class Main(App):
"-stage",
self.get_dist_filepath(self.get_project_filename("updater", "bin")),
]
if self.args.assets:
bundle_args.extend(
(
"-a",
self.args.assets,
)
)
self.logger.info(
f"Use this directory to self-update your Flipper:\n\t{bundle_dir}"
)

45
scripts/update.py
View File

@ -3,12 +3,16 @@
from flipper.app import App
from flipper.utils.fff import FlipperFormatFile
from os.path import basename, join, exists
from os import makedirs
import os
import shutil
import zlib
import tarfile
class Main(App):
# No compression, plain tar
ASSET_TAR_MODE = "w:"
def init(self):
self.subparsers = self.parser.add_subparsers(help="sub-command help")
@ -20,24 +24,41 @@ class Main(App):
self.parser_generate.add_argument("-d", dest="directory", required=True)
self.parser_generate.add_argument("-v", dest="version", required=True)
self.parser_generate.add_argument("-t", dest="target", required=True)
self.parser_generate.add_argument("-dfu", dest="dfu", required=True)
self.parser_generate.add_argument("-dfu", dest="dfu", required=False)
self.parser_generate.add_argument("-a", dest="assets", required=False)
self.parser_generate.add_argument("-stage", dest="stage", required=True)
self.parser_generate.add_argument("-radio", dest="radiobin", required=False)
self.parser_generate.add_argument(
"-radio", dest="radiobin", default="", required=False
)
self.parser_generate.add_argument(
"-radioaddr", dest="radioaddr", required=False
)
self.parser_generate.add_argument(
"-radiover", dest="radioversion", required=False
)
self.parser_generate.set_defaults(func=self.generate)
def generate(self):
stage_basename = basename(self.args.stage)
dfu_basename = basename(self.args.dfu)
radiobin_basename = basename(self.args.radiobin)
assets_basename = ""
if not exists(self.args.directory):
makedirs(self.args.directory)
os.makedirs(self.args.directory)
shutil.copyfile(self.args.stage, join(self.args.directory, stage_basename))
shutil.copyfile(self.args.dfu, join(self.args.directory, dfu_basename))
if radiobin_basename:
shutil.copyfile(
self.args.radiobin, join(self.args.directory, radiobin_basename)
)
if self.args.assets:
assets_basename = "assets.tar"
self.package_assets(
self.args.assets, join(self.args.directory, assets_basename)
)
file = FlipperFormatFile()
file.setHeader("Flipper firmware upgrade configuration", 1)
@ -47,12 +68,24 @@ class Main(App):
file.writeComment("little-endian hex!")
file.writeKey("Loader CRC", self.int2ffhex(self.crc(self.args.stage)))
file.writeKey("Firmware", dfu_basename)
file.writeKey("Radio", self.args.radiobin or "")
file.writeKey("Radio", radiobin_basename or "")
file.writeKey("Radio address", self.int2ffhex(self.args.radioaddr or 0))
file.save("%s/update.fuf" % self.args.directory)
file.writeKey("Radio version", self.int2ffhex(self.args.radioversion or 0))
if radiobin_basename:
file.writeKey("Radio CRC", self.int2ffhex(self.crc(self.args.radiobin)))
else:
file.writeKey("Radio CRC", self.int2ffhex(0))
file.writeKey("Assets", assets_basename)
file.save(join(self.args.directory, "update.fuf"))
return 0
def package_assets(self, srcdir: str, dst_name: str):
with tarfile.open(
dst_name, self.ASSET_TAR_MODE, format=tarfile.USTAR_FORMAT
) as tarball:
tarball.add(srcdir, arcname="")
@staticmethod
def int2ffhex(value: int):
hexstr = "%08X" % value