[FL-3097] fbt, faploader: minimal app module implementation (#2420)

* fbt, faploader: minimal app module implementation
* faploader, libs: moved API hashtable core to flipper_application
* example: compound api
* lib: flipper_application: naming fixes, doxygen comments
* fbt: changed `requires` manifest field behavior for app extensions
* examples: refactored plugin apps; faploader: changed new API naming; fbt: changed PLUGIN app type meaning
* loader: dropped support for debug apps & plugin menus
* moved applications/plugins -> applications/external
* Restored x bit on chiplist_convert.py
* git: fixed free-dap submodule path
* pvs: updated submodule paths
* examples: example_advanced_plugins.c: removed potential memory leak on errors
* examples: example_plugins: refined requires
* fbt: not deploying app modules for debug/sample apps; extra validation for .PLUGIN-type apps
* apps: removed cdefines for external apps
* fbt: moved ext app path definition
* fbt: reworked fap_dist handling; f18: synced api_symbols.csv
* fbt: removed resources_paths for extapps
* scripts: reworked storage
* scripts: reworked runfap.py & selfupdate.py to use new api
* wip: fal runner
* fbt: moved file packaging into separate module
* scripts: storage: fixes
* scripts: storage: minor fixes for new api
* fbt: changed internal artifact storage details for external apps
* scripts: storage: additional fixes and better error reporting; examples: using APP_DATA_PATH()
* fbt, scripts: reworked launch_app to deploy plugins; moved old runfap.py to distfap.py
* fbt: extra check for plugins descriptors
* fbt: additional checks in emitter
* fbt: better info message on SDK rebuild
* scripts: removed requirements.txt
* loader: removed remnants of plugins & debug menus
* post-review fixes

applications/examples/application.fam

@@ -1,3 +1,4 @@
+# Placeholder
 App(
     appid="example_apps",
     name="Example apps bundle",

applications/examples/example_plugins/application.fam

@@ -0,0 +1,31 @@
+App(
+    appid="example_plugins",
+    name="Example: App w/plugin",
+    apptype=FlipperAppType.EXTERNAL,
+    entry_point="example_plugins_app",
+    stack_size=2 * 1024,
+    fap_category="Examples",
+)
+
+App(
+    appid="example_plugins_multi",
+    name="Example: App w/plugins",
+    apptype=FlipperAppType.EXTERNAL,
+    entry_point="example_plugins_multi_app",
+    stack_size=2 * 1024,
+    fap_category="Examples",
+)
+
+App(
+    appid="example_plugin1",
+    apptype=FlipperAppType.PLUGIN,
+    entry_point="example_plugin1_ep",
+    requires=["example_plugins", "example_plugins_multi"],
+)
+
+App(
+    appid="example_plugin2",
+    apptype=FlipperAppType.PLUGIN,
+    entry_point="example_plugin2_ep",
+    requires=["example_plugins_multi"],
+)

applications/examples/example_plugins/example_plugins.c

@@ -0,0 +1,70 @@
+/* 
+ * An example of a plugin host application.
+ * Loads a single plugin and calls its methods.
+ */
+
+#include "plugin_interface.h"
+
+#include <furi.h>
+
+#include <flipper_application/flipper_application.h>
+#include <loader/firmware_api/firmware_api.h>
+#include <storage/storage.h>
+
+#define TAG "example_plugins"
+
+int32_t example_plugins_app(void* p) {
+    UNUSED(p);
+
+    FURI_LOG_I(TAG, "Starting");
+
+    Storage* storage = furi_record_open(RECORD_STORAGE);
+
+    FlipperApplication* app = flipper_application_alloc(storage, firmware_api_interface);
+
+    do {
+        FlipperApplicationPreloadStatus preload_res =
+            flipper_application_preload(app, APP_DATA_PATH("plugins/example_plugin1.fal"));
+
+        if(preload_res != FlipperApplicationPreloadStatusSuccess) {
+            FURI_LOG_E(TAG, "Failed to preload plugin");
+            break;
+        }
+
+        if(!flipper_application_is_plugin(app)) {
+            FURI_LOG_E(TAG, "Plugin file is not a library");
+            break;
+        }
+
+        FlipperApplicationLoadStatus load_status = flipper_application_map_to_memory(app);
+        if(load_status != FlipperApplicationLoadStatusSuccess) {
+            FURI_LOG_E(TAG, "Failed to load plugin file");
+            break;
+        }
+
+        const FlipperAppPluginDescriptor* app_descriptor =
+            flipper_application_plugin_get_descriptor(app);
+
+        FURI_LOG_I(
+            TAG,
+            "Loaded plugin for appid '%s', API %lu",
+            app_descriptor->appid,
+            app_descriptor->ep_api_version);
+
+        furi_check(app_descriptor->ep_api_version == PLUGIN_API_VERSION);
+        furi_check(strcmp(app_descriptor->appid, PLUGIN_APP_ID) == 0);
+
+        const ExamplePlugin* plugin = app_descriptor->entry_point;
+
+        FURI_LOG_I(TAG, "Plugin name: %s", plugin->name);
+        FURI_LOG_I(TAG, "Plugin method1: %d", plugin->method1());
+        FURI_LOG_I(TAG, "Plugin method2(7,8): %d", plugin->method2(7, 8));
+        FURI_LOG_I(TAG, "Plugin method2(1337,228): %d", plugin->method2(1337, 228));
+    } while(false);
+    flipper_application_free(app);
+
+    furi_record_close(RECORD_STORAGE);
+    FURI_LOG_I(TAG, "Goodbye!");
+
+    return 0;
+}

applications/examples/example_plugins/example_plugins_multi.c

@@ -0,0 +1,43 @@
+/*
+ * An example of an advanced plugin host application.
+ * It uses PluginManager to load all plugins from a directory
+ */
+
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+#include <flipper_application/plugins/plugin_manager.h>
+#include <loader/firmware_api/firmware_api.h>
+
+#include <furi.h>
+
+#define TAG "example_plugins"
+
+int32_t example_plugins_multi_app(void* p) {
+    UNUSED(p);
+
+    FURI_LOG_I(TAG, "Starting");
+
+    PluginManager* manager =
+        plugin_manager_alloc(PLUGIN_APP_ID, PLUGIN_API_VERSION, firmware_api_interface);
+
+    if(plugin_manager_load_all(manager, APP_DATA_PATH("plugins")) != PluginManagerErrorNone) {
+        FURI_LOG_E(TAG, "Failed to load all libs");
+        return 0;
+    }
+
+    uint32_t plugin_count = plugin_manager_get_count(manager);
+    FURI_LOG_I(TAG, "Loaded %lu plugin(s)", plugin_count);
+
+    for(uint32_t i = 0; i < plugin_count; i++) {
+        const ExamplePlugin* plugin = plugin_manager_get_ep(manager, i);
+        FURI_LOG_I(TAG, "plugin name: %s", plugin->name);
+        FURI_LOG_I(TAG, "plugin method1: %d", plugin->method1());
+        FURI_LOG_I(TAG, "plugin method2(7,8): %d", plugin->method2(7, 8));
+    }
+
+    plugin_manager_free(manager);
+    FURI_LOG_I(TAG, "Goodbye!");
+
+    return 0;
+}

applications/examples/example_plugins/plugin1.c

@@ -0,0 +1,32 @@
+/* A simple plugin implementing example_plugins application's plugin interface */
+
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+
+static int example_plugin1_method1() {
+    return 42;
+}
+
+static int example_plugin1_method2(int arg1, int arg2) {
+    return arg1 + arg2;
+}
+
+/* Actual implementation of app<>plugin interface */
+static const ExamplePlugin example_plugin1 = {
+    .name = "Demo App Plugin 1",
+    .method1 = &example_plugin1_method1,
+    .method2 = &example_plugin1_method2,
+};
+
+/* Plugin descriptor to comply with basic plugin specification */
+static const FlipperAppPluginDescriptor example_plugin1_descriptor = {
+    .appid = PLUGIN_APP_ID,
+    .ep_api_version = PLUGIN_API_VERSION,
+    .entry_point = &example_plugin1,
+};
+
+/* Plugin entry point - must return a pointer to const descriptor  */
+const FlipperAppPluginDescriptor* example_plugin1_ep() {
+    return &example_plugin1_descriptor;
+}

applications/examples/example_plugins/plugin2.c

@@ -0,0 +1,32 @@
+/* Second plugin implementing example_plugins application's plugin interface */
+
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+
+static int example_plugin2_method1() {
+    return 1337;
+}
+
+static int example_plugin2_method2(int arg1, int arg2) {
+    return arg1 - arg2;
+}
+
+/* Actual implementation of app<>plugin interface */
+static const ExamplePlugin example_plugin2 = {
+    .name = "Demo App Plugin 2",
+    .method1 = &example_plugin2_method1,
+    .method2 = &example_plugin2_method2,
+};
+
+/* Plugin descriptor to comply with basic plugin specification */
+static const FlipperAppPluginDescriptor example_plugin2_descriptor = {
+    .appid = PLUGIN_APP_ID,
+    .ep_api_version = PLUGIN_API_VERSION,
+    .entry_point = &example_plugin2,
+};
+
+/* Plugin entry point - must return a pointer to const descriptor */
+const FlipperAppPluginDescriptor* example_plugin2_ep() {
+    return &example_plugin2_descriptor;
+}

applications/examples/example_plugins/plugin_interface.h

@@ -0,0 +1,12 @@
+#pragma once
+
+/* Common interface between a plugin and host applicaion */
+
+#define PLUGIN_APP_ID "example_plugins"
+#define PLUGIN_API_VERSION 1
+
+typedef struct {
+    const char* name;
+    int (*method1)();
+    int (*method2)(int, int);
+} ExamplePlugin;

applications/examples/example_plugins_advanced/app_api.c

@@ -0,0 +1,25 @@
+#include "app_api.h"
+
+/* Actual implementation of app's API and its private state */
+
+static uint32_t accumulator = 0;
+
+void app_api_accumulator_set(uint32_t value) {
+    accumulator = value;
+}
+
+uint32_t app_api_accumulator_get() {
+    return accumulator;
+}
+
+void app_api_accumulator_add(uint32_t value) {
+    accumulator += value;
+}
+
+void app_api_accumulator_sub(uint32_t value) {
+    accumulator -= value;
+}
+
+void app_api_accumulator_mul(uint32_t value) {
+    accumulator *= value;
+}

applications/examples/example_plugins_advanced/app_api.h

@@ -0,0 +1,25 @@
+#pragma once
+
+/* 
+ * This file contains an API that is internally implemented by the application
+ * It is also exposed to plugins to allow them to use the application's API.
+ */
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void app_api_accumulator_set(uint32_t value);
+
+uint32_t app_api_accumulator_get();
+
+void app_api_accumulator_add(uint32_t value);
+
+void app_api_accumulator_sub(uint32_t value);
+
+void app_api_accumulator_mul(uint32_t value);
+
+#ifdef __cplusplus
+}
+#endif

applications/examples/example_plugins_advanced/app_api_interface.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <flipper_application/api_hashtable/api_hashtable.h>
+
+/* 
+ * Resolver interface with private application's symbols. 
+ * Implementation is contained in app_api_table.c
+ */
+extern const ElfApiInterface* const application_api_interface;

applications/examples/example_plugins_advanced/app_api_table.cpp

@@ -0,0 +1,27 @@
+#include <flipper_application/api_hashtable/api_hashtable.h>
+#include <flipper_application/api_hashtable/compilesort.hpp>
+
+/* 
+ * This file contains an implementation of a symbol table 
+ * with private app's symbols. It is used by composite API resolver
+ * to load plugins that use internal application's APIs.
+ */
+#include "app_api_table_i.h"
+
+static_assert(!has_hash_collisions(app_api_table), "Detected API method hash collision!");
+
+constexpr HashtableApiInterface applicaton_hashtable_api_interface{
+    {
+        .api_version_major = 0,
+        .api_version_minor = 0,
+        /* generic resolver using pre-sorted array */
+        .resolver_callback = &elf_resolve_from_hashtable,
+    },
+    /* pointers to application's API table boundaries */
+    .table_cbegin = app_api_table.cbegin(),
+    .table_cend = app_api_table.cend(),
+};
+
+/* Casting to generic resolver to use in Composite API resolver */
+extern "C" const ElfApiInterface* const application_api_interface =
+    &applicaton_hashtable_api_interface;

applications/examples/example_plugins_advanced/app_api_table_i.h

@@ -0,0 +1,13 @@
+#include "app_api.h"
+
+/* 
+ * A list of app's private functions and objects to expose for plugins.
+ * It is used to generate a table of symbols for import resolver to use.
+ * TBD: automatically generate this table from app's header files
+ */
+static constexpr auto app_api_table = sort(create_array_t<sym_entry>(
+    API_METHOD(app_api_accumulator_set, void, (uint32_t)),
+    API_METHOD(app_api_accumulator_get, uint32_t, ()),
+    API_METHOD(app_api_accumulator_add, void, (uint32_t)),
+    API_METHOD(app_api_accumulator_sub, void, (uint32_t)),
+    API_METHOD(app_api_accumulator_mul, void, (uint32_t))));

applications/examples/example_plugins_advanced/application.fam

@@ -0,0 +1,24 @@
+App(
+    appid="example_advanced_plugins",
+    name="Example: advanced plugins",
+    apptype=FlipperAppType.EXTERNAL,
+    entry_point="example_advanced_plugins_app",
+    stack_size=2 * 1024,
+    fap_category="Examples",
+)
+
+App(
+    appid="advanced_plugin1",
+    apptype=FlipperAppType.PLUGIN,
+    entry_point="advanced_plugin1_ep",
+    requires=["example_advanced_plugins"],
+    sources=["plugin1.c"],
+)
+
+App(
+    appid="advanced_plugin2",
+    apptype=FlipperAppType.PLUGIN,
+    entry_point="advanced_plugin2_ep",
+    requires=["example_advanced_plugins"],
+    sources=["plugin2.c"],
+)

applications/examples/example_plugins_advanced/example_advanced_plugins.c

@@ -0,0 +1,48 @@
+#include "app_api.h"
+#include "plugin_interface.h"
+#include "app_api_interface.h"
+
+#include <flipper_application/flipper_application.h>
+#include <flipper_application/plugins/plugin_manager.h>
+#include <flipper_application/plugins/composite_resolver.h>
+
+#include <loader/firmware_api/firmware_api.h>
+
+#define TAG "example_advanced_plugins"
+
+int32_t example_advanced_plugins_app(void* p) {
+    UNUSED(p);
+
+    FURI_LOG_I(TAG, "Starting");
+
+    CompositeApiResolver* resolver = composite_api_resolver_alloc();
+    composite_api_resolver_add(resolver, firmware_api_interface);
+    composite_api_resolver_add(resolver, application_api_interface);
+
+    PluginManager* manager = plugin_manager_alloc(
+        PLUGIN_APP_ID, PLUGIN_API_VERSION, composite_api_resolver_get(resolver));
+
+    do {
+        if(plugin_manager_load_all(manager, APP_DATA_PATH("plugins")) != PluginManagerErrorNone) {
+            FURI_LOG_E(TAG, "Failed to load all libs");
+            break;
+        }
+
+        uint32_t plugin_count = plugin_manager_get_count(manager);
+        FURI_LOG_I(TAG, "Loaded libs: %lu", plugin_count);
+
+        for(uint32_t i = 0; i < plugin_count; i++) {
+            const AdvancedPlugin* plugin = plugin_manager_get_ep(manager, i);
+            FURI_LOG_I(TAG, "plugin name: %s. Calling methods", plugin->name);
+            plugin->method1(228);
+            plugin->method2();
+            FURI_LOG_I(TAG, "Accumulator: %lu", app_api_accumulator_get());
+        }
+    } while(0);
+
+    plugin_manager_free(manager);
+    composite_api_resolver_free(resolver);
+    FURI_LOG_I(TAG, "Goodbye!");
+
+    return 0;
+}

applications/examples/example_plugins_advanced/plugin1.c

@@ -0,0 +1,40 @@
+/*
+ * This plugin uses both firmware's API interface and private application headers.
+ * It can be loaded by a plugin manager that uses CompoundApiInterface,
+ * which combines both interfaces.
+ */
+
+#include "app_api.h"
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+#include <furi.h>
+
+static void advanced_plugin1_method1(int arg1) {
+    /* This function is implemented inside host application */
+    app_api_accumulator_add(arg1);
+}
+
+static void advanced_plugin1_method2() {
+    /* Accumulator value is stored inside host application */
+    FURI_LOG_I("TEST", "Plugin 1, accumulator: %lu", app_api_accumulator_get());
+}
+
+/* Actual implementation of app<>plugin interface */
+static const AdvancedPlugin advanced_plugin1 = {
+    .name = "Advanced Plugin 1",
+    .method1 = &advanced_plugin1_method1,
+    .method2 = &advanced_plugin1_method2,
+};
+
+/* Plugin descriptor to comply with basic plugin specification */
+static const FlipperAppPluginDescriptor advanced_plugin1_descriptor = {
+    .appid = PLUGIN_APP_ID,
+    .ep_api_version = PLUGIN_API_VERSION,
+    .entry_point = &advanced_plugin1,
+};
+
+/* Plugin entry point - must return a pointer to const descriptor */
+const FlipperAppPluginDescriptor* advanced_plugin1_ep() {
+    return &advanced_plugin1_descriptor;
+}

applications/examples/example_plugins_advanced/plugin2.c

@@ -0,0 +1,40 @@
+/*
+ * This plugin uses both firmware's API interface and private application headers.
+ * It can be loaded by a plugin manager that uses CompoundApiInterface,
+ * which combines both interfaces.
+ */
+
+#include "app_api.h"
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+#include <furi.h>
+
+static void advanced_plugin2_method1(int arg1) {
+    /* This function is implemented inside host application */
+    app_api_accumulator_mul(arg1);
+}
+
+static void advanced_plugin2_method2() {
+    /* Accumulator value is stored inside host application */
+    FURI_LOG_I("TEST", "Plugin 2, accumulator: %lu", app_api_accumulator_get());
+}
+
+/* Actual implementation of app<>plugin interface */
+static const AdvancedPlugin advanced_plugin2 = {
+    .name = "Advanced Plugin 2",
+    .method1 = &advanced_plugin2_method1,
+    .method2 = &advanced_plugin2_method2,
+};
+
+/* Plugin descriptor to comply with basic plugin specification */
+static const FlipperAppPluginDescriptor advanced_plugin2_descriptor = {
+    .appid = PLUGIN_APP_ID,
+    .ep_api_version = PLUGIN_API_VERSION,
+    .entry_point = &advanced_plugin2,
+};
+
+/* Plugin entry point - must return a pointer to const descriptor */
+const FlipperAppPluginDescriptor* advanced_plugin2_ep() {
+    return &advanced_plugin2_descriptor;
+}

applications/examples/example_plugins_advanced/plugin_interface.h

@@ -0,0 +1,12 @@
+#pragma once
+
+/* Common interface between a plugin and host applicaion */
+
+#define PLUGIN_APP_ID "example_plugins_advanced"
+#define PLUGIN_API_VERSION 1
+
+typedef struct {
+    const char* name;
+    void (*method1)(int);
+    void (*method2)();
+} AdvancedPlugin;