[FL-2274] Inventing streams and moving FFF to them (#981)

* Streams: string stream
* String stream: updated insert/delete api
* Streams: generic stream interface and string stream implementation
* Streams: helpers for insert and delete_and_insert
* FFF: now compatible with streams
* MinUnit: introduced tests with arguments
* FFF: stream access violation
* Streams: copy data between streams
* Streams: file stream
* FFF: documentation
* FFStream: documentation
* FFF: alloc as file
* MinUnit: support for nested tests
* Streams: changed delete_and_insert, now it returns success flag. Added ability dump stream inner parameters and data to cout.
* FFF: simplified file open function
* Streams: unit tests
* FFF: tests
* Streams: declare cache_size constant as define, to allow variable modified arrays
* FFF: lib moved to a separate folder
* iButton: new FFF
* RFID: new FFF
* Animations: new FFF
* IR: new FFF
* NFC: new FFF
* Flipper file format: delete lib
* U2F: new FFF
* Subghz: new FFF and streams
* Streams: read line
* Streams: split
* FuriCore: implement memset with extra asserts
* FuriCore: implement extra heap asserts without inventing memset
* Scene manager: protected access to the scene id stack with a size check
* NFC worker: dirty fix for issue where hal_nfc was busy on app start
* Furi: update allocator to erase memory on allocation. Replace furi_alloc with malloc.
* FuriCore: cleanup memmgr code.
* Furi HAL: furi_hal_init is split into critical and non-critical parts. The critical part is currently clock and console.
* Memmgr: added ability to track allocations and deallocations through console.
* FFStream: some speedup
* Streams, FF: minor fixes
* Tests: restore
* File stream: a slightly more thread-safe version of file_stream_delete_and_insert

Co-authored-by: Aleksandr Kutuzov <alleteam@gmail.com>

core/furi/memmgr.c

@@ -20,26 +20,17 @@ void* realloc(void* ptr, size_t size) {
         return NULL;
     }
 
-    void* p;
-    p = pvPortMalloc(size);
-    if(p) {
-        // TODO implement secure realloc
-        // insecure, but will do job in our case
-        if(ptr != NULL) {
-            memcpy(p, ptr, size);
-            vPortFree(ptr);
-        }
+    void* p = pvPortMalloc(size);
+    if(ptr != NULL) {
+        memcpy(p, ptr, size);
+        vPortFree(ptr);
     }
+
     return p;
 }
 
 void* calloc(size_t count, size_t size) {
-    void* ptr = pvPortMalloc(count * size);
-    if(ptr) {
-        // zero the memory
-        memset(ptr, 0, count * size);
-    }
-    return ptr;
+    return pvPortMalloc(count * size);
 }
 
 char* strdup(const char* s) {
@@ -49,13 +40,8 @@ char* strdup(const char* s) {
     }
 
     size_t siz = strlen(s) + 1;
-    char* y = malloc(siz);
-
-    if(y != NULL) {
-        memcpy(y, s, siz);
-    } else {
-        return NULL;
-    }
+    char* y = pvPortMalloc(siz);
+    memcpy(y, s, siz);
 
     return y;
 }
@@ -68,27 +54,18 @@ size_t memmgr_get_minimum_free_heap(void) {
     return xPortGetMinimumEverFreeHeapSize();
 }
 
-void* furi_alloc(size_t size) {
-    void* p = malloc(size);
-    furi_check(p);
-    return memset(p, 0, size);
-}
-
 void* __wrap__malloc_r(struct _reent* r, size_t size) {
-    void* pointer = malloc(size);
-    return pointer;
+    return pvPortMalloc(size);
 }
 
 void __wrap__free_r(struct _reent* r, void* ptr) {
-    free(ptr);
+    vPortFree(ptr);
 }
 
 void* __wrap__calloc_r(struct _reent* r, size_t count, size_t size) {
-    void* pointer = calloc(count, size);
-    return pointer;
+    return calloc(count, size);
 }
 
 void* __wrap__realloc_r(struct _reent* r, void* ptr, size_t size) {
-    void* pointer = realloc(ptr, size);
-    return pointer;
-}
+    return realloc(ptr, size);
+}

core/furi/memmgr.h

@@ -29,16 +29,6 @@ size_t memmgr_get_free_heap(void);
  */
 size_t memmgr_get_minimum_free_heap(void);
 
-/** Allocate memory from heap
- *
- * @note       performs memset with 0, will crash system if not enough memory
- *
- * @param[in]  size  bytes to allocate
- *
- * @return     pointer to allocated memory
- */
-void* furi_alloc(size_t size);
-
 #ifdef __cplusplus
 }
 #endif

core/furi/memmgr_heap.c

@@ -38,6 +38,10 @@
 #include "check.h"
 #include <stdlib.h>
 #include <cmsis_os2.h>
+#include <stm32wbxx.h>
+#include <furi_hal_console.h>
+#include <furi/common_defines.h>
+#include <furi_hal_task.h>
 
 /* Defining MPU_WRAPPERS_INCLUDED_FROM_API_FILE prevents task.h from redefining
 all the API functions to use the MPU wrappers.  That should only be done when
@@ -235,23 +239,115 @@ void memmgr_heap_printf_free_blocks() {
 
     //osKernelUnlock();
 }
+
+#ifdef HEAP_PRINT_DEBUG
+char* ultoa(unsigned long num, char* str, int radix) {
+    char temp[33]; // at radix 2 the string is at most 32 + 1 null long.
+    int temp_loc = 0;
+    int digit;
+    int str_loc = 0;
+
+    //construct a backward string of the number.
+    do {
+        digit = (unsigned long)num % ((unsigned long)radix);
+        if(digit < 10)
+            temp[temp_loc++] = digit + '0';
+        else
+            temp[temp_loc++] = digit - 10 + 'A';
+        num = ((unsigned long)num) / ((unsigned long)radix);
+    } while((unsigned long)num > 0);
+
+    temp_loc--;
+
+    //now reverse the string.
+    while(temp_loc >= 0) { // while there are still chars
+        str[str_loc++] = temp[temp_loc--];
+    }
+    str[str_loc] = 0; // add null termination.
+
+    return str;
+}
+
+static void print_heap_init() {
+    char tmp_str[33];
+    size_t heap_start = (size_t)&__heap_start__;
+    size_t heap_end = (size_t)&__heap_end__;
+
+    // {PHStart|heap_start|heap_end}
+    FURI_CRITICAL_ENTER();
+    furi_hal_console_puts("{PHStart|");
+    ultoa(heap_start, tmp_str, 16);
+    furi_hal_console_puts(tmp_str);
+    furi_hal_console_puts("|");
+    ultoa(heap_end, tmp_str, 16);
+    furi_hal_console_puts(tmp_str);
+    furi_hal_console_puts("}\r\n");
+    FURI_CRITICAL_EXIT();
+}
+
+static void print_heap_malloc(void* ptr, size_t size) {
+    char tmp_str[33];
+    const char* name = osThreadGetName(osThreadGetId());
+
+    // {thread name|m|address|size}
+    FURI_CRITICAL_ENTER();
+    furi_hal_console_puts("{");
+    furi_hal_console_puts(name);
+    furi_hal_console_puts("|m|0x");
+    ultoa((unsigned long)ptr, tmp_str, 16);
+    furi_hal_console_puts(tmp_str);
+    furi_hal_console_puts("|");
+    utoa(size, tmp_str, 10);
+    furi_hal_console_puts(tmp_str);
+    furi_hal_console_puts("}\r\n");
+    FURI_CRITICAL_EXIT();
+}
+
+static void print_heap_free(void* ptr) {
+    char tmp_str[33];
+    const char* name = osThreadGetName(osThreadGetId());
+
+    // {thread name|f|address}
+    FURI_CRITICAL_ENTER();
+    furi_hal_console_puts("{");
+    furi_hal_console_puts(name);
+    furi_hal_console_puts("|f|0x");
+    ultoa((unsigned long)ptr, tmp_str, 16);
+    furi_hal_console_puts(tmp_str);
+    furi_hal_console_puts("}\r\n");
+    FURI_CRITICAL_EXIT();
+}
+#endif
 /*-----------------------------------------------------------*/
 
 void* pvPortMalloc(size_t xWantedSize) {
     BlockLink_t *pxBlock, *pxPreviousBlock, *pxNewBlockLink;
     void* pvReturn = NULL;
+    size_t to_wipe = xWantedSize;
+
+#ifdef HEAP_PRINT_DEBUG
+    BlockLink_t* print_heap_block = NULL;
+#endif
+
+    /* If this is the first call to malloc then the heap will require
+        initialisation to setup the list of free blocks. */
+    if(pxEnd == NULL) {
+#ifdef HEAP_PRINT_DEBUG
+        print_heap_init();
+#endif
+
+        vTaskSuspendAll();
+        {
+            prvHeapInit();
+            memmgr_heap_init();
+        }
+        (void)xTaskResumeAll();
+    } else {
+        mtCOVERAGE_TEST_MARKER();
+    }
 
     vTaskSuspendAll();
     {
-        /* If this is the first call to malloc then the heap will require
-        initialisation to setup the list of free blocks. */
-        if(pxEnd == NULL) {
-            prvHeapInit();
-            memmgr_heap_init();
-        } else {
-            mtCOVERAGE_TEST_MARKER();
-        }
-
         /* Check the requested block size is not so large that the top bit is
         set.  The top bit of the block size member of the BlockLink_t structure
         is used to determine who owns the block - the application or the
@@ -330,6 +426,10 @@ void* pvPortMalloc(size_t xWantedSize) {
                     by the application and has no "next" block. */
                     pxBlock->xBlockSize |= xBlockAllocatedBit;
                     pxBlock->pxNextFreeBlock = NULL;
+
+#ifdef HEAP_PRINT_DEBUG
+                    print_heap_block = pxBlock;
+#endif
                 } else {
                     mtCOVERAGE_TEST_MARKER();
                 }
@@ -344,6 +444,10 @@ void* pvPortMalloc(size_t xWantedSize) {
     }
     (void)xTaskResumeAll();
 
+#ifdef HEAP_PRINT_DEBUG
+    print_heap_malloc(print_heap_block, print_heap_block->xBlockSize & ~xBlockAllocatedBit);
+#endif
+
 #if(configUSE_MALLOC_FAILED_HOOK == 1)
     {
         if(pvReturn == NULL) {
@@ -356,6 +460,9 @@ void* pvPortMalloc(size_t xWantedSize) {
 #endif
 
     configASSERT((((size_t)pvReturn) & (size_t)portBYTE_ALIGNMENT_MASK) == 0);
+
+    furi_check(pvReturn);
+    pvReturn = memset(pvReturn, 0, to_wipe);
     return pvReturn;
 }
 /*-----------------------------------------------------------*/
@@ -382,8 +489,17 @@ void vPortFree(void* pv) {
                 allocated. */
                 pxLink->xBlockSize &= ~xBlockAllocatedBit;
 
+#ifdef HEAP_PRINT_DEBUG
+                print_heap_free(pxLink);
+#endif
+
                 vTaskSuspendAll();
                 {
+                    furi_assert((size_t)pv >= SRAM_BASE);
+                    furi_assert((size_t)pv < SRAM_BASE + 1024 * 256);
+                    furi_assert((pxLink->xBlockSize - xHeapStructSize) < 1024 * 256);
+                    furi_assert((int32_t)(pxLink->xBlockSize - xHeapStructSize) >= 0);
+
                     /* Add this block to the list of free blocks. */
                     xFreeBytesRemaining += pxLink->xBlockSize;
                     traceFREE(pv, pxLink->xBlockSize);
@@ -397,6 +513,10 @@ void vPortFree(void* pv) {
         } else {
             mtCOVERAGE_TEST_MARKER();
         }
+    } else {
+#ifdef HEAP_PRINT_DEBUG
+        print_heap_free(pv);
+#endif
     }
 }
 /*-----------------------------------------------------------*/

core/furi/pubsub.c

@@ -18,7 +18,7 @@ struct FuriPubSub {
 };
 
 FuriPubSub* furi_pubsub_alloc() {
-    FuriPubSub* pubsub = furi_alloc(sizeof(FuriPubSub));
+    FuriPubSub* pubsub = malloc(sizeof(FuriPubSub));
 
     pubsub->mutex = osMutexNew(NULL);
     furi_assert(pubsub->mutex);

core/furi/record.c

@@ -24,7 +24,7 @@ typedef struct {
 static FuriRecord* furi_record = NULL;
 
 void furi_record_init() {
-    furi_record = furi_alloc(sizeof(FuriRecord));
+    furi_record = malloc(sizeof(FuriRecord));
     furi_record->mutex = osMutexNew(NULL);
     furi_check(furi_record->mutex);
     FuriRecordDataDict_init(furi_record->records);

core/furi/stdglue.c

@@ -67,7 +67,7 @@ static ssize_t stdout_write(void* _cookie, const char* data, size_t size) {
 }
 
 void furi_stdglue_init() {
-    furi_stdglue = furi_alloc(sizeof(FuriStdglue));
+    furi_stdglue = malloc(sizeof(FuriStdglue));
     // Init outputs structures
     furi_stdglue->mutex = osMutexNew(NULL);
     furi_check(furi_stdglue->mutex);

core/furi/thread.c

@@ -56,7 +56,7 @@ void furi_thread_body(void* context) {
 }
 
 FuriThread* furi_thread_alloc() {
-    FuriThread* thread = furi_alloc(sizeof(FuriThread));
+    FuriThread* thread = malloc(sizeof(FuriThread));
 
     return thread;
 }