[FL-2831] Resources cleanup in updater (#1796)

* updater: remove files from existing resources Manifest file before deploying new resources
* toolbox: tar: single file extraction API

Co-authored-by: あく <alleteam@gmail.com>
This commit is contained in:
hedger 2022-09-28 21:13:12 +04:00 committed by GitHub
parent e25b424188
commit f8b532f063
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 281 additions and 46 deletions

View File

@ -10,7 +10,7 @@ extern "C" {
#include <stdbool.h>
#include <m-string.h>
#define UPDATE_DELAY_OPERATION_OK 300
#define UPDATE_DELAY_OPERATION_OK 10
#define UPDATE_DELAY_OPERATION_ERROR INT_MAX
typedef enum {

View File

@ -9,6 +9,7 @@
#include <update_util/dfu_file.h>
#include <update_util/lfs_backup.h>
#include <update_util/update_operation.h>
#include <update_util/resources/manifest.h>
#include <toolbox/tar/tar_archive.h>
#include <toolbox/crc32_calc.h>
@ -50,10 +51,46 @@ static bool update_task_resource_unpack_cb(const char* name, bool is_directory,
update_task_set_progress(
unpack_progress->update_task,
UpdateTaskStageProgress,
unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1));
/* For this stage, last 70% of progress = extraction */
30 + (unpack_progress->processed_files * 70) / (unpack_progress->total_files + 1));
return true;
}
static void
update_task_cleanup_resources(UpdateTask* update_task, uint32_t n_approx_file_entries) {
ResourceManifestReader* manifest_reader = resource_manifest_reader_alloc(update_task->storage);
do {
FURI_LOG_I(TAG, "Cleaning up old manifest");
if(!resource_manifest_reader_open(manifest_reader, EXT_PATH("Manifest"))) {
FURI_LOG_W(TAG, "No existing manifest");
break;
}
/* We got # of entries in TAR file. Approx 1/4th is dir entries, we skip them */
n_approx_file_entries = n_approx_file_entries * 3 / 4 + 1;
uint32_t n_processed_files = 0;
ResourceManifestEntry* entry_ptr = NULL;
while((entry_ptr = resource_manifest_reader_next(manifest_reader))) {
if(entry_ptr->type == ResourceManifestEntryTypeFile) {
update_task_set_progress(
update_task,
UpdateTaskStageProgress,
/* For this stage, first 30% of progress = cleanup */
(n_processed_files++ * 30) / (n_approx_file_entries + 1));
string_t file_path;
string_init(file_path);
path_concat(STORAGE_EXT_PATH_PREFIX, string_get_cstr(entry_ptr->name), file_path);
FURI_LOG_D(TAG, "Removing %s", string_get_cstr(file_path));
storage_simply_remove(update_task->storage, string_get_cstr(file_path));
string_clear(file_path);
}
}
} while(false);
resource_manifest_reader_free(manifest_reader);
}
static bool update_task_post_update(UpdateTask* update_task) {
bool success = false;
@ -88,6 +125,8 @@ static bool update_task_post_update(UpdateTask* update_task) {
progress.total_files = tar_archive_get_entries_count(archive);
if(progress.total_files > 0) {
update_task_cleanup_resources(update_task, progress.total_files);
CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL));
}
}

View File

@ -308,7 +308,7 @@ bool update_task_validate_optionbytes(UpdateTask* update_task) {
}
}
} else {
FURI_LOG_I(
FURI_LOG_D(
TAG,
"OB MATCH: #%d: real %08X == %08X (exp.)",
idx,

View File

@ -2269,6 +2269,7 @@ Function,+,tar_archive_get_entries_count,int32_t,TarArchive*
Function,+,tar_archive_open,_Bool,"TarArchive*, const char*, TarOpenMode"
Function,+,tar_archive_set_file_callback,void,"TarArchive*, tar_unpack_file_cb, void*"
Function,+,tar_archive_store_data,_Bool,"TarArchive*, const char*, const uint8_t*, const int32_t"
Function,+,tar_archive_unpack_file,_Bool,"TarArchive*, const char*, const char*"
Function,+,tar_archive_unpack_to,_Bool,"TarArchive*, const char*, Storage_name_converter"
Function,-,tempnam,char*,"const char*, const char*"
Function,+,text_box_alloc,TextBox*,

1 entry status name type params
2269 Function + tar_archive_open _Bool TarArchive*, const char*, TarOpenMode
2270 Function + tar_archive_set_file_callback void TarArchive*, tar_unpack_file_cb, void*
2271 Function + tar_archive_store_data _Bool TarArchive*, const char*, const uint8_t*, const int32_t
2272 Function + tar_archive_unpack_file _Bool TarArchive*, const char*, const char*
2273 Function + tar_archive_unpack_to _Bool TarArchive*, const char*, Storage_name_converter
2274 Function - tempnam char* const char*, const char*
2275 Function + text_box_alloc TextBox*

View File

@ -6,6 +6,9 @@ void furi_hal_cortex_init_early() {
CoreDebug->DEMCR |= CoreDebug_DEMCR_TRCENA_Msk;
DWT->CTRL |= DWT_CTRL_CYCCNTENA_Msk;
DWT->CYCCNT = 0U;
/* Enable instruction prefetch */
SET_BIT(FLASH->ACR, FLASH_ACR_PRFTEN);
}
void furi_hal_cortex_delay_us(uint32_t microseconds) {

View File

@ -168,7 +168,44 @@ typedef struct {
Storage_name_converter converter;
} TarArchiveDirectoryOpParams;
static bool archive_extract_current_file(TarArchive* archive, const char* dst_path) {
mtar_t* tar = &archive->tar;
File* out_file = storage_file_alloc(archive->storage);
uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
bool success = true;
uint8_t n_tries = FILE_OPEN_NTRIES;
do {
while(n_tries-- > 0) {
if(storage_file_open(out_file, dst_path, FSAM_WRITE, FSOM_CREATE_ALWAYS)) {
break;
}
FURI_LOG_W(TAG, "Failed to open '%s', reties: %d", dst_path, n_tries);
storage_file_close(out_file);
furi_delay_ms(FILE_OPEN_RETRY_DELAY);
}
if(!storage_file_is_open(out_file)) {
success = false;
break;
}
while(!mtar_eof_data(tar)) {
int32_t readcnt = mtar_read_data(tar, readbuf, FILE_BLOCK_SIZE);
if(!readcnt || !storage_file_write(out_file, readbuf, readcnt)) {
success = false;
break;
}
}
} while(false);
storage_file_free(out_file);
free(readbuf);
return success;
}
static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) {
UNUSED(tar);
TarArchiveDirectoryOpParams* op_params = param;
TarArchive* archive = op_params->archive;
@ -199,58 +236,22 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header,
return 0;
}
string_init(full_extracted_fname);
FURI_LOG_D(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
string_t converted_fname;
string_init_set(converted_fname, header->name);
if(op_params->converter) {
op_params->converter(converted_fname);
}
string_init(full_extracted_fname);
path_concat(op_params->work_dir, string_get_cstr(converted_fname), full_extracted_fname);
bool success = archive_extract_current_file(archive, string_get_cstr(full_extracted_fname));
string_clear(converted_fname);
FURI_LOG_D(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
File* out_file = storage_file_alloc(archive->storage);
uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
bool failed = false;
uint8_t n_tries = FILE_OPEN_NTRIES;
do {
while(n_tries-- > 0) {
if(storage_file_open(
out_file,
string_get_cstr(full_extracted_fname),
FSAM_WRITE,
FSOM_CREATE_ALWAYS)) {
break;
}
FURI_LOG_W(
TAG,
"Failed to open '%s', reties: %d",
string_get_cstr(full_extracted_fname),
n_tries);
storage_file_close(out_file);
furi_delay_ms(FILE_OPEN_RETRY_DELAY);
}
if(!storage_file_is_open(out_file)) {
failed = true;
break;
}
while(!mtar_eof_data(tar)) {
int32_t readcnt = mtar_read_data(tar, readbuf, FILE_BLOCK_SIZE);
if(!readcnt || !storage_file_write(out_file, readbuf, readcnt)) {
failed = true;
break;
}
}
} while(false);
storage_file_free(out_file);
free(readbuf);
string_clear(full_extracted_fname);
return failed ? -1 : 0;
return success ? 0 : -1;
}
bool tar_archive_unpack_to(
@ -369,3 +370,16 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch
storage_file_free(directory);
return success;
}
bool tar_archive_unpack_file(
TarArchive* archive,
const char* archive_fname,
const char* destination) {
furi_assert(archive);
furi_assert(archive_fname);
furi_assert(destination);
if(mtar_find(&archive->tar, archive_fname) != MTAR_ESUCCESS) {
return false;
}
return archive_extract_current_file(archive, destination);
}

View File

@ -41,6 +41,11 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch
int32_t tar_archive_get_entries_count(TarArchive* archive);
bool tar_archive_unpack_file(
TarArchive* archive,
const char* archive_fname,
const char* destination);
/* Optional per-entry callback on unpacking - return false to skip entry */
typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context);

View File

@ -0,0 +1,115 @@
#include "manifest.h"
#include <toolbox/stream/buffered_file_stream.h>
#include <toolbox/hex.h>
struct ResourceManifestReader {
Storage* storage;
Stream* stream;
string_t linebuf;
ResourceManifestEntry entry;
};
ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage) {
ResourceManifestReader* resource_manifest =
(ResourceManifestReader*)malloc(sizeof(ResourceManifestReader));
resource_manifest->storage = storage;
resource_manifest->stream = buffered_file_stream_alloc(resource_manifest->storage);
memset(&resource_manifest->entry, 0, sizeof(ResourceManifestEntry));
string_init(resource_manifest->entry.name);
string_init(resource_manifest->linebuf);
return resource_manifest;
}
void resource_manifest_reader_free(ResourceManifestReader* resource_manifest) {
furi_assert(resource_manifest);
string_clear(resource_manifest->linebuf);
string_clear(resource_manifest->entry.name);
buffered_file_stream_close(resource_manifest->stream);
stream_free(resource_manifest->stream);
free(resource_manifest);
}
bool resource_manifest_reader_open(ResourceManifestReader* resource_manifest, const char* filename) {
furi_assert(resource_manifest);
return buffered_file_stream_open(
resource_manifest->stream, filename, FSAM_READ, FSOM_OPEN_EXISTING);
}
/* Read entries in format of
* F:<hash>:<size>:<name>
* D:<name>
*/
ResourceManifestEntry* resource_manifest_reader_next(ResourceManifestReader* resource_manifest) {
furi_assert(resource_manifest);
string_reset(resource_manifest->entry.name);
resource_manifest->entry.type = ResourceManifestEntryTypeUnknown;
resource_manifest->entry.size = 0;
memset(resource_manifest->entry.hash, 0, sizeof(resource_manifest->entry.hash));
do {
if(!stream_read_line(resource_manifest->stream, resource_manifest->linebuf)) {
return NULL;
}
/* Trim end of line */
string_strim(resource_manifest->linebuf);
char type_code = string_get_char(resource_manifest->linebuf, 0);
switch(type_code) {
case 'F':
resource_manifest->entry.type = ResourceManifestEntryTypeFile;
break;
case 'D':
resource_manifest->entry.type = ResourceManifestEntryTypeDirectory;
break;
default: /* Skip other entries - version, timestamp, etc */
continue;
};
if(resource_manifest->entry.type == ResourceManifestEntryTypeFile) {
/* Parse file entry
F:<hash>:<size>:<name> */
/* Remove entry type code */
string_right(resource_manifest->linebuf, 2);
if(string_search_char(resource_manifest->linebuf, ':') !=
sizeof(resource_manifest->entry.hash) * 2) {
/* Invalid hash */
continue;
}
/* Read hash */
hex_chars_to_uint8(
string_get_cstr(resource_manifest->linebuf), resource_manifest->entry.hash);
/* Remove hash */
string_right(
resource_manifest->linebuf, sizeof(resource_manifest->entry.hash) * 2 + 1);
resource_manifest->entry.size = atoi(string_get_cstr(resource_manifest->linebuf));
/* Remove size */
size_t offs = string_search_char(resource_manifest->linebuf, ':');
string_right(resource_manifest->linebuf, offs + 1);
string_set(resource_manifest->entry.name, resource_manifest->linebuf);
} else if(resource_manifest->entry.type == ResourceManifestEntryTypeDirectory) {
/* Parse directory entry
D:<name> */
/* Remove entry type code */
string_right(resource_manifest->linebuf, 2);
string_set(resource_manifest->entry.name, resource_manifest->linebuf);
}
return &resource_manifest->entry;
} while(true);
return NULL;
}

View File

@ -0,0 +1,58 @@
#pragma once
#include <storage/storage.h>
#include <m-string.h>
#include <stdbool.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
ResourceManifestEntryTypeUnknown = 0,
ResourceManifestEntryTypeDirectory,
ResourceManifestEntryTypeFile,
} ResourceManifestEntryType;
typedef struct {
ResourceManifestEntryType type;
string_t name;
uint32_t size;
uint8_t hash[16];
} ResourceManifestEntry;
typedef struct ResourceManifestReader ResourceManifestReader;
/**
* @brief Initialize resource manifest reader
* @param storage Storage API pointer
* @return allocated object
*/
ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage);
/**
* @brief Release resource manifest reader
* @param resource_manifest allocated object
*/
void resource_manifest_reader_free(ResourceManifestReader* resource_manifest);
/**
* @brief Initialize resource manifest reader iteration
* @param resource_manifest allocated object
* @param filename manifest file name
* @return true if file opened
*/
bool resource_manifest_reader_open(ResourceManifestReader* resource_manifest, const char* filename);
/**
* @brief Read next file/dir entry from manifest
* @param resource_manifest allocated object
* @return entry or NULL if end of file
*/
ResourceManifestEntry* resource_manifest_reader_next(ResourceManifestReader* resource_manifest);
#ifdef __cplusplus
} // extern "C"
#endif