[FL-2274] Inventing streams and moving FFF to them (#981)

* Streams: string stream
* String stream: updated insert/delete api
* Streams: generic stream interface and string stream implementation
* Streams: helpers for insert and delete_and_insert
* FFF: now compatible with streams
* MinUnit: introduced tests with arguments
* FFF: stream access violation
* Streams: copy data between streams
* Streams: file stream
* FFF: documentation
* FFStream: documentation
* FFF: alloc as file
* MinUnit: support for nested tests
* Streams: changed delete_and_insert, now it returns success flag. Added ability dump stream inner parameters and data to cout.
* FFF: simplified file open function
* Streams: unit tests
* FFF: tests
* Streams: declare cache_size constant as define, to allow variable modified arrays
* FFF: lib moved to a separate folder
* iButton: new FFF
* RFID: new FFF
* Animations: new FFF
* IR: new FFF
* NFC: new FFF
* Flipper file format: delete lib
* U2F: new FFF
* Subghz: new FFF and streams
* Streams: read line
* Streams: split
* FuriCore: implement memset with extra asserts
* FuriCore: implement extra heap asserts without inventing memset
* Scene manager: protected access to the scene id stack with a size check
* NFC worker: dirty fix for issue where hal_nfc was busy on app start
* Furi: update allocator to erase memory on allocation. Replace furi_alloc with malloc.
* FuriCore: cleanup memmgr code.
* Furi HAL: furi_hal_init is split into critical and non-critical parts. The critical part is currently clock and console.
* Memmgr: added ability to track allocations and deallocations through console.
* FFStream: some speedup
* Streams, FF: minor fixes
* Tests: restore
* File stream: a slightly more thread-safe version of file_stream_delete_and_insert

Co-authored-by: Aleksandr Kutuzov <alleteam@gmail.com>

applications/nfc/helpers/nfc_emv_parser.c

@@ -1,5 +1,5 @@
 #include "nfc_emv_parser.h"
-#include <lib/flipper_file/flipper_file.h>
+#include <flipper_format/flipper_format.h>
 
 static const char* nfc_resources_header = "Flipper EMV resources";
 static const uint32_t nfc_resources_file_version = 1;
@@ -10,25 +10,25 @@ static bool nfc_emv_parser_search_data(
     string_t key,
     string_t data) {
     bool parsed = false;
-    FlipperFile* file = flipper_file_alloc(storage);
+    FlipperFormat* file = flipper_format_file_alloc(storage);
     string_t temp_str;
     string_init(temp_str);
 
     do {
         // Open file
-        if(!flipper_file_open_existing(file, file_name)) break;
+        if(!flipper_format_file_open_existing(file, file_name)) break;
         // Read file header and version
         uint32_t version = 0;
-        if(!flipper_file_read_header(file, temp_str, &version)) break;
+        if(!flipper_format_read_header(file, temp_str, &version)) break;
         if(string_cmp_str(temp_str, nfc_resources_header) ||
            (version != nfc_resources_file_version))
             break;
-        if(!flipper_file_read_string(file, string_get_cstr(key), data)) break;
+        if(!flipper_format_read_string(file, string_get_cstr(key), data)) break;
         parsed = true;
     } while(false);
 
     string_clear(temp_str);
-    flipper_file_free(file);
+    flipper_format_free(file);
     return parsed;
 }
 

applications/nfc/nfc.c

@@ -20,7 +20,7 @@ void nfc_tick_event_callback(void* context) {
 }
 
 Nfc* nfc_alloc() {
-    Nfc* nfc = furi_alloc(sizeof(Nfc));
+    Nfc* nfc = malloc(sizeof(Nfc));
 
     nfc->worker = nfc_worker_alloc();
     nfc->view_dispatcher = view_dispatcher_alloc();

applications/nfc/nfc_device.c

@@ -1,14 +1,14 @@
 #include "nfc_device.h"
 #include "nfc_types.h"
 
-#include <lib/toolbox/path.h>
-#include <lib/flipper_file/flipper_file.h>
+#include <toolbox/path.h>
+#include <flipper_format/flipper_format.h>
 
 static const char* nfc_file_header = "Flipper NFC device";
 static const uint32_t nfc_file_version = 2;
 
 NfcDevice* nfc_device_alloc() {
-    NfcDevice* nfc_dev = furi_alloc(sizeof(NfcDevice));
+    NfcDevice* nfc_dev = malloc(sizeof(NfcDevice));
     nfc_dev->storage = furi_record_open("storage");
     nfc_dev->dialogs = furi_record_open("dialogs");
     return nfc_dev;
@@ -56,7 +56,7 @@ bool nfc_device_parse_format_string(NfcDevice* dev, string_t format_string) {
     return false;
 }
 
-static bool nfc_device_save_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
+static bool nfc_device_save_mifare_ul_data(FlipperFormat* file, NfcDevice* dev) {
     bool saved = false;
     MifareUlData* data = &dev->dev_data.mf_ul_data;
     string_t temp_str;
@@ -64,22 +64,23 @@ static bool nfc_device_save_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
 
     // Save Mifare Ultralight specific data
     do {
-        if(!flipper_file_write_comment_cstr(file, "Mifare Ultralight specific data")) break;
-        if(!flipper_file_write_hex(file, "Signature", data->signature, sizeof(data->signature)))
+        if(!flipper_format_write_comment_cstr(file, "Mifare Ultralight specific data")) break;
+        if(!flipper_format_write_hex(file, "Signature", data->signature, sizeof(data->signature)))
             break;
-        if(!flipper_file_write_hex(
+        if(!flipper_format_write_hex(
                file, "Mifare version", (uint8_t*)&data->version, sizeof(data->version)))
             break;
         // Write conters and tearing flags data
         bool counters_saved = true;
         for(uint8_t i = 0; i < 3; i++) {
             string_printf(temp_str, "Counter %d", i);
-            if(!flipper_file_write_uint32(file, string_get_cstr(temp_str), &data->counter[i], 1)) {
+            if(!flipper_format_write_uint32(
+                   file, string_get_cstr(temp_str), &data->counter[i], 1)) {
                 counters_saved = false;
                 break;
             }
             string_printf(temp_str, "Tearing %d", i);
-            if(!flipper_file_write_hex(file, string_get_cstr(temp_str), &data->tearing[i], 1)) {
+            if(!flipper_format_write_hex(file, string_get_cstr(temp_str), &data->tearing[i], 1)) {
                 counters_saved = false;
                 break;
             }
@@ -87,11 +88,11 @@ static bool nfc_device_save_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
         if(!counters_saved) break;
         // Write pages data
         uint32_t pages_total = data->data_size / 4;
-        if(!flipper_file_write_uint32(file, "Pages total", &pages_total, 1)) break;
+        if(!flipper_format_write_uint32(file, "Pages total", &pages_total, 1)) break;
         bool pages_saved = true;
         for(uint16_t i = 0; i < data->data_size; i += 4) {
             string_printf(temp_str, "Page %d", i / 4);
-            if(!flipper_file_write_hex(file, string_get_cstr(temp_str), &data->data[i], 4)) {
+            if(!flipper_format_write_hex(file, string_get_cstr(temp_str), &data->data[i], 4)) {
                 pages_saved = false;
                 break;
             }
@@ -104,7 +105,7 @@ static bool nfc_device_save_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
     return saved;
 }
 
-bool nfc_device_load_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
+bool nfc_device_load_mifare_ul_data(FlipperFormat* file, NfcDevice* dev) {
     bool parsed = false;
     MifareUlData* data = &dev->dev_data.mf_ul_data;
     string_t temp_str;
@@ -112,22 +113,22 @@ bool nfc_device_load_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
 
     do {
         // Read signature
-        if(!flipper_file_read_hex(file, "Signature", data->signature, sizeof(data->signature)))
+        if(!flipper_format_read_hex(file, "Signature", data->signature, sizeof(data->signature)))
             break;
         // Read Mifare version
-        if(!flipper_file_read_hex(
+        if(!flipper_format_read_hex(
                file, "Mifare version", (uint8_t*)&data->version, sizeof(data->version)))
             break;
         // Read counters and tearing flags
         bool counters_parsed = true;
         for(uint8_t i = 0; i < 3; i++) {
             string_printf(temp_str, "Counter %d", i);
-            if(!flipper_file_read_uint32(file, string_get_cstr(temp_str), &data->counter[i], 1)) {
+            if(!flipper_format_read_uint32(file, string_get_cstr(temp_str), &data->counter[i], 1)) {
                 counters_parsed = false;
                 break;
             }
             string_printf(temp_str, "Tearing %d", i);
-            if(!flipper_file_read_hex(file, string_get_cstr(temp_str), &data->tearing[i], 1)) {
+            if(!flipper_format_read_hex(file, string_get_cstr(temp_str), &data->tearing[i], 1)) {
                 counters_parsed = false;
                 break;
             }
@@ -135,12 +136,12 @@ bool nfc_device_load_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
         if(!counters_parsed) break;
         // Read pages
         uint32_t pages = 0;
-        if(!flipper_file_read_uint32(file, "Pages total", &pages, 1)) break;
+        if(!flipper_format_read_uint32(file, "Pages total", &pages, 1)) break;
         data->data_size = pages * 4;
         bool pages_parsed = true;
         for(uint16_t i = 0; i < pages; i++) {
             string_printf(temp_str, "Page %d", i);
-            if(!flipper_file_read_hex(file, string_get_cstr(temp_str), &data->data[i * 4], 4)) {
+            if(!flipper_format_read_hex(file, string_get_cstr(temp_str), &data->data[i * 4], 4)) {
                 pages_parsed = false;
                 break;
             }
@@ -153,28 +154,28 @@ bool nfc_device_load_mifare_ul_data(FlipperFile* file, NfcDevice* dev) {
     return parsed;
 }
 
-static bool nfc_device_save_bank_card_data(FlipperFile* file, NfcDevice* dev) {
+static bool nfc_device_save_bank_card_data(FlipperFormat* file, NfcDevice* dev) {
     bool saved = false;
     NfcEmvData* data = &dev->dev_data.emv_data;
     uint32_t data_temp = 0;
 
     do {
         // Write Bank card specific data
-        if(!flipper_file_write_comment_cstr(file, "Bank card specific data")) break;
-        if(!flipper_file_write_hex(file, "AID", data->aid, data->aid_len)) break;
-        if(!flipper_file_write_string_cstr(file, "Name", data->name)) break;
-        if(!flipper_file_write_hex(file, "Number", data->number, data->number_len)) break;
+        if(!flipper_format_write_comment_cstr(file, "Bank card specific data")) break;
+        if(!flipper_format_write_hex(file, "AID", data->aid, data->aid_len)) break;
+        if(!flipper_format_write_string_cstr(file, "Name", data->name)) break;
+        if(!flipper_format_write_hex(file, "Number", data->number, data->number_len)) break;
         if(data->exp_mon) {
             uint8_t exp_data[2] = {data->exp_mon, data->exp_year};
-            if(!flipper_file_write_hex(file, "Exp data", exp_data, sizeof(exp_data))) break;
+            if(!flipper_format_write_hex(file, "Exp data", exp_data, sizeof(exp_data))) break;
         }
         if(data->country_code) {
             data_temp = data->country_code;
-            if(!flipper_file_write_uint32(file, "Country code", &data_temp, 1)) break;
+            if(!flipper_format_write_uint32(file, "Country code", &data_temp, 1)) break;
         }
         if(data->currency_code) {
             data_temp = data->currency_code;
-            if(!flipper_file_write_uint32(file, "Currency code", &data_temp, 1)) break;
+            if(!flipper_format_write_uint32(file, "Currency code", &data_temp, 1)) break;
         }
         saved = true;
     } while(false);
@@ -182,7 +183,7 @@ static bool nfc_device_save_bank_card_data(FlipperFile* file, NfcDevice* dev) {
     return saved;
 }
 
-bool nfc_device_load_bank_card_data(FlipperFile* file, NfcDevice* dev) {
+bool nfc_device_load_bank_card_data(FlipperFormat* file, NfcDevice* dev) {
     bool parsed = false;
     NfcEmvData* data = &dev->dev_data.emv_data;
     memset(data, 0, sizeof(NfcEmvData));
@@ -192,25 +193,25 @@ bool nfc_device_load_bank_card_data(FlipperFile* file, NfcDevice* dev) {
 
     do {
         // Load essential data
-        if(!flipper_file_get_value_count(file, "AID", &data_cnt)) break;
+        if(!flipper_format_get_value_count(file, "AID", &data_cnt)) break;
         data->aid_len = data_cnt;
-        if(!flipper_file_read_hex(file, "AID", data->aid, data->aid_len)) break;
-        if(!flipper_file_read_string(file, "Name", temp_str)) break;
+        if(!flipper_format_read_hex(file, "AID", data->aid, data->aid_len)) break;
+        if(!flipper_format_read_string(file, "Name", temp_str)) break;
         strlcpy(data->name, string_get_cstr(temp_str), sizeof(data->name));
-        if(!flipper_file_get_value_count(file, "Number", &data_cnt)) break;
+        if(!flipper_format_get_value_count(file, "Number", &data_cnt)) break;
         data->number_len = data_cnt;
-        if(!flipper_file_read_hex(file, "Number", data->number, data->number_len)) break;
+        if(!flipper_format_read_hex(file, "Number", data->number, data->number_len)) break;
         parsed = true;
         // Load optional data
         uint8_t exp_data[2] = {};
-        if(flipper_file_read_hex(file, "Exp data", exp_data, 2)) {
+        if(flipper_format_read_hex(file, "Exp data", exp_data, 2)) {
             data->exp_mon = exp_data[0];
             data->exp_year = exp_data[1];
         }
-        if(flipper_file_read_uint32(file, "Country code", &data_cnt, 1)) {
+        if(flipper_format_read_uint32(file, "Country code", &data_cnt, 1)) {
             data->country_code = data_cnt;
         }
-        if(flipper_file_read_uint32(file, "Currency code", &data_cnt, 1)) {
+        if(flipper_format_read_uint32(file, "Currency code", &data_cnt, 1)) {
             data->currency_code = data_cnt;
         }
     } while(false);
@@ -233,7 +234,7 @@ static bool nfc_device_save_file(
     furi_assert(dev);
 
     bool saved = false;
-    FlipperFile* file = flipper_file_alloc(dev->storage);
+    FlipperFormat* file = flipper_format_file_alloc(dev->storage);
     NfcDeviceCommonData* data = &dev->dev_data.nfc_data;
     string_t temp_str;
     string_init(temp_str);
@@ -244,21 +245,21 @@ static bool nfc_device_save_file(
         // First remove nfc device file if it was saved
         string_printf(temp_str, "%s/%s%s", folder, dev_name, extension);
         // Open file
-        if(!flipper_file_open_always(file, string_get_cstr(temp_str))) break;
+        if(!flipper_format_file_open_always(file, string_get_cstr(temp_str))) break;
         // Write header
-        if(!flipper_file_write_header_cstr(file, nfc_file_header, nfc_file_version)) break;
+        if(!flipper_format_write_header_cstr(file, nfc_file_header, nfc_file_version)) break;
         // Write nfc device type
-        if(!flipper_file_write_comment_cstr(
+        if(!flipper_format_write_comment_cstr(
                file, "Nfc device type can be UID, Mifare Ultralight, Bank card"))
             break;
         nfc_device_prepare_format_string(dev, temp_str);
-        if(!flipper_file_write_string(file, "Device type", temp_str)) break;
+        if(!flipper_format_write_string(file, "Device type", temp_str)) break;
         // Write UID, ATQA, SAK
-        if(!flipper_file_write_comment_cstr(file, "UID, ATQA and SAK are common for all formats"))
+        if(!flipper_format_write_comment_cstr(file, "UID, ATQA and SAK are common for all formats"))
             break;
-        if(!flipper_file_write_hex(file, "UID", data->uid, data->uid_len)) break;
-        if(!flipper_file_write_hex(file, "ATQA", data->atqa, 2)) break;
-        if(!flipper_file_write_hex(file, "SAK", &data->sak, 1)) break;
+        if(!flipper_format_write_hex(file, "UID", data->uid, data->uid_len)) break;
+        if(!flipper_format_write_hex(file, "ATQA", data->atqa, 2)) break;
+        if(!flipper_format_write_hex(file, "SAK", &data->sak, 1)) break;
         // Save more data if necessary
         if(dev->format == NfcDeviceSaveFormatMifareUl) {
             if(!nfc_device_save_mifare_ul_data(file, dev)) break;
@@ -272,8 +273,7 @@ static bool nfc_device_save_file(
         dialog_message_show_storage_error(dev->dialogs, "Can not save\nkey file");
     }
     string_clear(temp_str);
-    flipper_file_close(file);
-    flipper_file_free(file);
+    flipper_format_free(file);
     return saved;
 }
 
@@ -288,7 +288,7 @@ bool nfc_device_save_shadow(NfcDevice* dev, const char* dev_name) {
 
 static bool nfc_device_load_data(NfcDevice* dev, string_t path) {
     bool parsed = false;
-    FlipperFile* file = flipper_file_alloc(dev->storage);
+    FlipperFormat* file = flipper_format_file_alloc(dev->storage);
     NfcDeviceCommonData* data = &dev->dev_data.nfc_data;
     uint32_t data_cnt = 0;
     string_t temp_str;
@@ -304,26 +304,26 @@ static bool nfc_device_load_data(NfcDevice* dev, string_t path) {
             storage_common_stat(dev->storage, string_get_cstr(temp_str), NULL) == FSE_OK;
         // Open shadow file if it exists. If not - open original
         if(dev->shadow_file_exist) {
-            if(!flipper_file_open_existing(file, string_get_cstr(temp_str))) break;
+            if(!flipper_format_file_open_existing(file, string_get_cstr(temp_str))) break;
         } else {
-            if(!flipper_file_open_existing(file, string_get_cstr(path))) break;
+            if(!flipper_format_file_open_existing(file, string_get_cstr(path))) break;
         }
         // Read and verify file header
         uint32_t version = 0;
-        if(!flipper_file_read_header(file, temp_str, &version)) break;
+        if(!flipper_format_read_header(file, temp_str, &version)) break;
         if(string_cmp_str(temp_str, nfc_file_header) || (version != nfc_file_version)) {
             depricated_version = true;
             break;
         }
         // Read Nfc device type
-        if(!flipper_file_read_string(file, "Device type", temp_str)) break;
+        if(!flipper_format_read_string(file, "Device type", temp_str)) break;
         if(!nfc_device_parse_format_string(dev, temp_str)) break;
         // Read and parse UID, ATQA and SAK
-        if(!flipper_file_get_value_count(file, "UID", &data_cnt)) break;
+        if(!flipper_format_get_value_count(file, "UID", &data_cnt)) break;
         data->uid_len = data_cnt;
-        if(!flipper_file_read_hex(file, "UID", data->uid, data->uid_len)) break;
-        if(!flipper_file_read_hex(file, "ATQA", data->atqa, 2)) break;
-        if(!flipper_file_read_hex(file, "SAK", &data->sak, 1)) break;
+        if(!flipper_format_read_hex(file, "UID", data->uid, data->uid_len)) break;
+        if(!flipper_format_read_hex(file, "ATQA", data->atqa, 2)) break;
+        if(!flipper_format_read_hex(file, "SAK", &data->sak, 1)) break;
         // Parse other data
         if(dev->format == NfcDeviceSaveFormatMifareUl) {
             if(!nfc_device_load_mifare_ul_data(file, dev)) break;
@@ -342,8 +342,7 @@ static bool nfc_device_load_data(NfcDevice* dev, string_t path) {
     }
 
     string_clear(temp_str);
-    flipper_file_close(file);
-    flipper_file_free(file);
+    flipper_format_free(file);
     return parsed;
 }
 

applications/nfc/nfc_worker.c

@@ -8,18 +8,17 @@
 /***************************** NFC Worker API *******************************/
 
 NfcWorker* nfc_worker_alloc() {
-    NfcWorker* nfc_worker = furi_alloc(sizeof(NfcWorker));
+    NfcWorker* nfc_worker = malloc(sizeof(NfcWorker));
     // Worker thread attributes
     nfc_worker->thread_attr.name = "NfcWorker";
     nfc_worker->thread_attr.stack_size = 8192;
     nfc_worker->callback = NULL;
     nfc_worker->context = NULL;
     // Initialize rfal
-    if(!furi_hal_nfc_is_busy()) {
-        nfc_worker_change_state(nfc_worker, NfcWorkerStateReady);
-    } else {
-        nfc_worker_change_state(nfc_worker, NfcWorkerStateBroken);
+    while(furi_hal_nfc_is_busy()) {
+        osDelay(10);
     }
+    nfc_worker_change_state(nfc_worker, NfcWorkerStateReady);
 
     return nfc_worker;
 }
@@ -41,7 +40,7 @@ void nfc_worker_start(
     void* context) {
     furi_assert(nfc_worker);
     furi_assert(dev_data);
-    while(nfc_worker->state != NfcWorkerStateReady) {
+    while(furi_hal_nfc_is_busy()) {
         osDelay(10);
     }
 

applications/nfc/views/bank_card.c

@@ -7,7 +7,7 @@ struct BankCard {
 };
 
 BankCard* bank_card_alloc() {
-    BankCard* bank_card = furi_alloc(sizeof(BankCard));
+    BankCard* bank_card = malloc(sizeof(BankCard));
     bank_card->widget = widget_alloc();
     return bank_card;
 }