RPC: Screen streaming & Input injection (#794)

* RPC: Screen stream * Move callback to rpc_screen, implement graceful stop * RPC: Implement input injection * Update protobuf submodule * Gui: thread safe frame buffer callback setter. * RPC: Keep gui record open Co-authored-by: あく <alleteam@gmail.com>
2021-11-01 19:26:37 +03:00 · 2021-11-01 19:26:37 +03:00 · 86bced5b2c
commit 86bced5b2c
parent 22a4bac448
17 changed files with 564 additions and 50 deletions
--- a/applications/gui/gui.c
+++ b/applications/gui/gui.c
@ -258,8 +258,7 @@ void gui_cli_screen_stream_callback(uint8_t* data, size_t size, void* context) {
 void gui_cli_screen_stream(Cli* cli, string_t args, void* context) {
    furi_assert(context);
    Gui* gui = context;
-    gui_set_framebuffer_callback_context(gui, gui);
-    gui_set_framebuffer_callback(gui, gui_cli_screen_stream_callback);
+    gui_set_framebuffer_callback(gui, gui_cli_screen_stream_callback, gui);
    gui_redraw(gui);

    // Wait for control events
@ -279,8 +278,7 @@ void gui_cli_screen_stream(Cli* cli, string_t args, void* context) {
        }
    }

-    gui_set_framebuffer_callback(gui, NULL);
-    gui_set_framebuffer_callback_context(gui, NULL);
+    gui_set_framebuffer_callback(gui, NULL, NULL);
 }

 void gui_add_view_port(Gui* gui, ViewPort* view_port, GuiLayer layer) {
@ -387,14 +385,12 @@ void gui_view_port_send_to_back(Gui* gui, ViewPort* view_port) {
    gui_unlock(gui);
 }

-void gui_set_framebuffer_callback(Gui* gui, GuiCanvasCommitCallback callback) {
+void gui_set_framebuffer_callback(Gui* gui, GuiCanvasCommitCallback callback, void* context) {
    furi_assert(gui);
+    gui_lock(gui);
    gui->canvas_callback = callback;
-}
-
-void gui_set_framebuffer_callback_context(Gui* gui, void* context) {
-    furi_assert(gui);
    gui->canvas_callback_context = context;
+    gui_unlock(gui);
 }

 Gui* gui_alloc() {
--- a/applications/gui/gui.h
+++ b/applications/gui/gui.h
@ -73,15 +73,9 @@ void gui_view_port_send_to_back(Gui* gui, ViewPort* view_port);
 *
 * @param      gui       Gui instance
 * @param      callback  GuiCanvasCommitCallback
+ * @param      context   GuiCanvasCommitCallback context
 */
-void gui_set_framebuffer_callback(Gui* gui, GuiCanvasCommitCallback callback);
-
-/** Set gui canvas commit callback context
- *
- * @param      gui      Gui instance
- * @param      context  pointer to context
- */
-void gui_set_framebuffer_callback_context(Gui* gui, void* context);
+void gui_set_framebuffer_callback(Gui* gui, GuiCanvasCommitCallback callback, void* context);

 #ifdef __cplusplus
 }
--- a/applications/rpc/rpc.c
+++ b/applications/rpc/rpc.c
@ -45,6 +45,10 @@ static RpcSystemCallbacks rpc_systems[] = {
        .alloc = rpc_system_app_alloc,
        .free = NULL,
    },
+    {
+        .alloc = rpc_system_gui_alloc,
+        .free = rpc_system_gui_free,
+    },
 };

 struct RpcSession {
@ -161,10 +165,10 @@ void rpc_print_message(const PB_Main* message) {
    case PB_Main_stop_session_tag:
        string_cat_printf(str, "\tstop_session {\r\n");
        break;
-    case PB_Main_app_start_tag: {
+    case PB_Main_app_start_request_tag: {
        string_cat_printf(str, "\tapp_start {\r\n");
-        const char* name = message->content.app_start.name;
-        const char* args = message->content.app_start.args;
+        const char* name = message->content.app_start_request.name;
+        const char* args = message->content.app_start_request.args;
        if(name) {
            string_cat_printf(str, "\t\tname: %s\r\n", name);
        }
@ -260,6 +264,22 @@ void rpc_print_message(const PB_Main* message) {
        string_cat_printf(str, "\tlist_response {\r\n");
        rpc_sprintf_msg_file(str, "\t\t", msg_file, msg_file_count);
    }
+    case PB_Main_gui_start_screen_stream_request_tag:
+        string_cat_printf(str, "\tstart_screen_stream {\r\n");
+        break;
+    case PB_Main_gui_stop_screen_stream_request_tag:
+        string_cat_printf(str, "\tstop_screen_stream {\r\n");
+        break;
+    case PB_Main_gui_screen_stream_frame_tag:
+        string_cat_printf(str, "\tscreen_stream_frame {\r\n");
+        break;
+    case PB_Main_gui_send_input_event_request_tag:
+        string_cat_printf(str, "\tsend_input_event {\r\n");
+        string_cat_printf(
+            str, "\t\tkey: %d\r\n", message->content.gui_send_input_event_request.key);
+        string_cat_printf(
+            str, "\t\type: %d\r\n", message->content.gui_send_input_event_request.type);
+        break;
    }
    string_cat_printf(str, "\t}\r\n}\r\n");
    printf("%s", string_get_cstr(str));
--- a/applications/rpc/rpc_app.c
+++ b/applications/rpc/rpc_app.c
@ -9,13 +9,13 @@ void rpc_system_app_start_process(const PB_Main* request, void* context) {
    Rpc* rpc = context;
    furi_assert(rpc);
    furi_assert(request);
-    furi_assert(request->which_content == PB_Main_app_start_tag);
+    furi_assert(request->which_content == PB_Main_app_start_request_tag);
    PB_CommandStatus result = PB_CommandStatus_ERROR_APP_CANT_START;

    Loader* loader = furi_record_open("loader");
-    const char* app_name = request->content.app_start.name;
+    const char* app_name = request->content.app_start_request.name;
    if(app_name) {
-        const char* app_args = request->content.app_start.args;
+        const char* app_args = request->content.app_start_request.args;
        LoaderStatus status = loader_start(loader, app_name, app_args);
        if(status == LoaderStatusErrorAppStarted) {
            result = PB_CommandStatus_ERROR_APP_SYSTEM_LOCKED;
@ -70,7 +70,7 @@ void* rpc_system_app_alloc(Rpc* rpc) {
    };

    rpc_handler.message_handler = rpc_system_app_start_process;
-    rpc_add_handler(rpc, PB_Main_app_start_tag, &rpc_handler);
+    rpc_add_handler(rpc, PB_Main_app_start_request_tag, &rpc_handler);

    rpc_handler.message_handler = rpc_system_app_lock_status_process;
    rpc_add_handler(rpc, PB_Main_app_lock_status_request_tag, &rpc_handler);
--- a/applications/rpc/rpc_gui.c
+++ b/applications/rpc/rpc_gui.c
@ -0,0 +1,153 @@
+#include "flipper.pb.h"
+#include "rpc_i.h"
+#include "gui.pb.h"
+#include <gui/gui_i.h>
+
+typedef struct {
+    Rpc* rpc;
+    Gui* gui;
+} RpcGuiSystem;
+
+void rpc_system_gui_screen_frame_callback(uint8_t* data, size_t size, void* context) {
+    furi_assert(data);
+    furi_assert(size == 1024);
+    furi_assert(context);
+
+    RpcGuiSystem* rpc_gui = context;
+
+    PB_Main* frame = furi_alloc(sizeof(PB_Main));
+
+    frame->which_content = PB_Main_gui_screen_stream_frame_tag;
+    frame->command_status = PB_CommandStatus_OK;
+    frame->content.gui_screen_stream_frame.data = furi_alloc(PB_BYTES_ARRAY_T_ALLOCSIZE(size));
+    uint8_t* buffer = frame->content.gui_screen_stream_frame.data->bytes;
+    uint16_t* frame_size_msg = &frame->content.gui_screen_stream_frame.data->size;
+    *frame_size_msg = size;
+    memcpy(buffer, data, size);
+
+    rpc_send_and_release(rpc_gui->rpc, frame);
+
+    free(frame);
+}
+
+void rpc_system_gui_start_screen_stream_process(const PB_Main* request, void* context) {
+    furi_assert(request);
+    furi_assert(context);
+    RpcGuiSystem* rpc_gui = context;
+
+    rpc_send_and_release_empty(rpc_gui->rpc, request->command_id, PB_CommandStatus_OK);
+
+    gui_set_framebuffer_callback(rpc_gui->gui, rpc_system_gui_screen_frame_callback, context);
+}
+
+void rpc_system_gui_stop_screen_stream_process(const PB_Main* request, void* context) {
+    furi_assert(request);
+    furi_assert(context);
+    RpcGuiSystem* rpc_gui = context;
+
+    rpc_send_and_release_empty(rpc_gui->rpc, request->command_id, PB_CommandStatus_OK);
+
+    gui_set_framebuffer_callback(rpc_gui->gui, NULL, NULL);
+}
+
+void rpc_system_gui_send_input_event_request_process(const PB_Main* request, void* context) {
+    furi_assert(request);
+    furi_assert(request->which_content == PB_Main_gui_send_input_event_request_tag);
+    furi_assert(context);
+    RpcGuiSystem* rpc_gui = context;
+
+    InputEvent event;
+
+    bool invalid = false;
+
+    switch(request->content.gui_send_input_event_request.key) {
+    case PB_Gui_InputKey_UP:
+        event.key = InputKeyUp;
+        break;
+    case PB_Gui_InputKey_DOWN:
+        event.key = InputKeyDown;
+        break;
+    case PB_Gui_InputKey_RIGHT:
+        event.key = InputKeyRight;
+        break;
+    case PB_Gui_InputKey_LEFT:
+        event.key = InputKeyLeft;
+        break;
+    case PB_Gui_InputKey_OK:
+        event.key = InputKeyOk;
+        break;
+    case PB_Gui_InputKey_BACK:
+        event.key = InputKeyBack;
+        break;
+    default:
+        // Invalid key
+        invalid = true;
+        break;
+    }
+
+    switch(request->content.gui_send_input_event_request.type) {
+    case PB_Gui_InputType_PRESS:
+        event.type = InputTypePress;
+        break;
+    case PB_Gui_InputType_RELEASE:
+        event.type = InputTypeRelease;
+        break;
+    case PB_Gui_InputType_SHORT:
+        event.type = InputTypeShort;
+        break;
+    case PB_Gui_InputType_LONG:
+        event.type = InputTypeLong;
+        break;
+    case PB_Gui_InputType_REPEAT:
+        event.type = InputTypeRepeat;
+        break;
+    default:
+        // Invalid type
+        invalid = true;
+        break;
+    }
+
+    if(invalid) {
+        rpc_send_and_release_empty(
+            rpc_gui->rpc, request->command_id, PB_CommandStatus_ERROR_INVALID_PARAMETERS);
+        return;
+    }
+
+    PubSub* input_events = furi_record_open("input_events");
+    furi_check(input_events);
+    notify_pubsub(input_events, &event);
+    furi_record_close("input_events");
+    rpc_send_and_release_empty(rpc_gui->rpc, request->command_id, PB_CommandStatus_OK);
+}
+
+void* rpc_system_gui_alloc(Rpc* rpc) {
+    furi_assert(rpc);
+
+    RpcGuiSystem* rpc_gui = furi_alloc(sizeof(RpcGuiSystem));
+    rpc_gui->gui = furi_record_open("gui");
+    rpc_gui->rpc = rpc;
+
+    RpcHandler rpc_handler = {
+        .message_handler = NULL,
+        .decode_submessage = NULL,
+        .context = rpc_gui,
+    };
+
+    rpc_handler.message_handler = rpc_system_gui_start_screen_stream_process;
+    rpc_add_handler(rpc, PB_Main_gui_start_screen_stream_request_tag, &rpc_handler);
+
+    rpc_handler.message_handler = rpc_system_gui_stop_screen_stream_process;
+    rpc_add_handler(rpc, PB_Main_gui_stop_screen_stream_request_tag, &rpc_handler);
+
+    rpc_handler.message_handler = rpc_system_gui_send_input_event_request_process;
+    rpc_add_handler(rpc, PB_Main_gui_send_input_event_request_tag, &rpc_handler);
+
+    return NULL;
+}
+
+void rpc_system_gui_free(void* ctx) {
+    RpcGuiSystem* rpc_gui = ctx;
+    gui_set_framebuffer_callback(rpc_gui->gui, NULL, NULL);
+    furi_record_close("gui");
+    free(rpc_gui);
+}
--- a/applications/rpc/rpc_i.h
+++ b/applications/rpc/rpc_i.h
@ -24,6 +24,8 @@ void* rpc_system_status_alloc(Rpc* rpc);
 void* rpc_system_storage_alloc(Rpc* rpc);
 void rpc_system_storage_free(void* ctx);
 void* rpc_system_app_alloc(Rpc* rpc);
+void* rpc_system_gui_alloc(Rpc* rpc);
+void rpc_system_gui_free(void* ctx);

 void rpc_print_message(const PB_Main* message);
 void rpc_cli_command_start_session(Cli* cli, string_t args, void* context);
--- a/applications/tests/rpc/rpc_test.c
+++ b/applications/tests/rpc/rpc_test.c
@ -1158,23 +1158,23 @@ static void test_app_create_request(
    request->command_id = command_id;
    request->command_status = PB_CommandStatus_OK;
    request->cb_content.funcs.encode = NULL;
-    request->which_content = PB_Main_app_start_tag;
+    request->which_content = PB_Main_app_start_request_tag;
    request->has_next = false;

    if(app_name) {
        char* msg_app_name = furi_alloc(strlen(app_name) + 1);
        strcpy(msg_app_name, app_name);
-        request->content.app_start.name = msg_app_name;
+        request->content.app_start_request.name = msg_app_name;
    } else {
-        request->content.app_start.name = NULL;
+        request->content.app_start_request.name = NULL;
    }

    if(app_args) {
        char* msg_app_args = furi_alloc(strlen(app_args) + 1);
        strcpy(msg_app_args, app_args);
-        request->content.app_start.args = msg_app_args;
+        request->content.app_start_request.args = msg_app_args;
    } else {
-        request->content.app_start.args = NULL;
+        request->content.app_start_request.args = NULL;
    }
 }

--- a/assets/compiled/application.pb.c
+++ b/assets/compiled/application.pb.c
@ -6,7 +6,7 @@
 #error Regenerate this file with the current version of nanopb generator.
 #endif

-PB_BIND(PB_App_Start, PB_App_Start, AUTO)
+PB_BIND(PB_App_StartRequest, PB_App_StartRequest, AUTO)


 PB_BIND(PB_App_LockStatusRequest, PB_App_LockStatusRequest, AUTO)
--- a/assets/compiled/application.pb.h
+++ b/assets/compiled/application.pb.h
@ -14,10 +14,10 @@ typedef struct _PB_App_LockStatusRequest {
    char dummy_field;
 } PB_App_LockStatusRequest;

-typedef struct _PB_App_Start { 
+typedef struct _PB_App_StartRequest { 
    char *name; 
    char *args; 
-} PB_App_Start;
+} PB_App_StartRequest;

 typedef struct _PB_App_LockStatusResponse { 
    bool locked; 
@ -29,24 +29,24 @@ extern "C" {
 #endif

 /* Initializer values for message structs */
-#define PB_App_Start_init_default                {NULL, NULL}
+#define PB_App_StartRequest_init_default         {NULL, NULL}
 #define PB_App_LockStatusRequest_init_default    {0}
 #define PB_App_LockStatusResponse_init_default   {0}
-#define PB_App_Start_init_zero                   {NULL, NULL}
+#define PB_App_StartRequest_init_zero            {NULL, NULL}
 #define PB_App_LockStatusRequest_init_zero       {0}
 #define PB_App_LockStatusResponse_init_zero      {0}

 /* Field tags (for use in manual encoding/decoding) */
-#define PB_App_Start_name_tag                    1
-#define PB_App_Start_args_tag                    2
+#define PB_App_StartRequest_name_tag             1
+#define PB_App_StartRequest_args_tag             2
 #define PB_App_LockStatusResponse_locked_tag     1

 /* Struct field encoding specification for nanopb */
-#define PB_App_Start_FIELDLIST(X, a) \
+#define PB_App_StartRequest_FIELDLIST(X, a) \
 X(a, POINTER,  SINGULAR, STRING,   name,              1) \
 X(a, POINTER,  SINGULAR, STRING,   args,              2)
-#define PB_App_Start_CALLBACK NULL
-#define PB_App_Start_DEFAULT NULL
+#define PB_App_StartRequest_CALLBACK NULL
+#define PB_App_StartRequest_DEFAULT NULL

 #define PB_App_LockStatusRequest_FIELDLIST(X, a) \

@ -58,17 +58,17 @@ X(a, STATIC,   SINGULAR, BOOL,     locked,            1)
 #define PB_App_LockStatusResponse_CALLBACK NULL
 #define PB_App_LockStatusResponse_DEFAULT NULL

-extern const pb_msgdesc_t PB_App_Start_msg;
+extern const pb_msgdesc_t PB_App_StartRequest_msg;
 extern const pb_msgdesc_t PB_App_LockStatusRequest_msg;
 extern const pb_msgdesc_t PB_App_LockStatusResponse_msg;

 /* Defines for backwards compatibility with code written before nanopb-0.4.0 */
-#define PB_App_Start_fields &PB_App_Start_msg
+#define PB_App_StartRequest_fields &PB_App_StartRequest_msg
 #define PB_App_LockStatusRequest_fields &PB_App_LockStatusRequest_msg
 #define PB_App_LockStatusResponse_fields &PB_App_LockStatusResponse_msg

 /* Maximum encoded size of messages (where known) */
-/* PB_App_Start_size depends on runtime parameters */
+/* PB_App_StartRequest_size depends on runtime parameters */
 #define PB_App_LockStatusRequest_size            0
 #define PB_App_LockStatusResponse_size           2

--- a/assets/compiled/flipper.pb.h
+++ b/assets/compiled/flipper.pb.h
@ -7,6 +7,7 @@
 #include "storage.pb.h"
 #include "status.pb.h"
 #include "application.pb.h"
+#include "gui.pb.h"

 #if PB_PROTO_HEADER_VERSION != 40
 #error Regenerate this file with the current version of nanopb generator.
@ -15,12 +16,14 @@
 /* Enum definitions */
 typedef enum _PB_CommandStatus { 
    PB_CommandStatus_OK = 0, 
+    /* *< Common Errors */
    PB_CommandStatus_ERROR = 1, /* *< Unknown error */
    PB_CommandStatus_ERROR_DECODE = 2, /* *< Command can't be decoded successfully - command_id in response may be wrong! */
    PB_CommandStatus_ERROR_NOT_IMPLEMENTED = 3, /* *< Command succesfully decoded, but not implemented (deprecated or not yet implemented) */
    PB_CommandStatus_ERROR_BUSY = 4, /* *< Somebody took global lock, so not all commands are available */
    PB_CommandStatus_ERROR_CONTINUOUS_COMMAND_INTERRUPTED = 14, /* *< Not received has_next == 0 */
    PB_CommandStatus_ERROR_INVALID_PARAMETERS = 15, /* *< not provided (or provided invalid) crucial parameters to perform rpc */
+    /* *< Storage Errors */
    PB_CommandStatus_ERROR_STORAGE_NOT_READY = 5, /* *< FS not ready */
    PB_CommandStatus_ERROR_STORAGE_EXIST = 6, /* *< File/Dir alrady exist */
    PB_CommandStatus_ERROR_STORAGE_NOT_EXIST = 7, /* *< File/Dir does not exist */
@ -31,6 +34,7 @@ typedef enum _PB_CommandStatus {
    PB_CommandStatus_ERROR_STORAGE_NOT_IMPLEMENTED = 12, /* *< Functon not implemented */
    PB_CommandStatus_ERROR_STORAGE_ALREADY_OPEN = 13, /* *< File/Dir already opened */
    PB_CommandStatus_ERROR_STORAGE_DIR_NOT_EMPTY = 18, /* *< Directory, you're going to remove is not empty */
+    /* *< Application Errors */
    PB_CommandStatus_ERROR_APP_CANT_START = 16, /* *< Can't start app - internal error */
    PB_CommandStatus_ERROR_APP_SYSTEM_LOCKED = 17 /* *< Another app is running */
 } PB_CommandStatus;
@ -66,10 +70,14 @@ typedef struct _PB_Main {
        PB_Storage_MkdirRequest storage_mkdir_request;
        PB_Storage_Md5sumRequest storage_md5sum_request;
        PB_Storage_Md5sumResponse storage_md5sum_response;
-        PB_App_Start app_start;
+        PB_App_StartRequest app_start_request;
        PB_App_LockStatusRequest app_lock_status_request;
        PB_App_LockStatusResponse app_lock_status_response;
        PB_StopSession stop_session;
+        PB_Gui_StartScreenStreamRequest gui_start_screen_stream_request;
+        PB_Gui_StopScreenStreamRequest gui_stop_screen_stream_request;
+        PB_Gui_ScreenStreamFrame gui_screen_stream_frame;
+        PB_Gui_SendInputEventRequest gui_send_input_event_request;
    } content; 
 } PB_Main;

@ -108,10 +116,14 @@ extern "C" {
 #define PB_Main_storage_mkdir_request_tag        13
 #define PB_Main_storage_md5sum_request_tag       14
 #define PB_Main_storage_md5sum_response_tag      15
-#define PB_Main_app_start_tag                    16
+#define PB_Main_app_start_request_tag            16
 #define PB_Main_app_lock_status_request_tag      17
 #define PB_Main_app_lock_status_response_tag     18
 #define PB_Main_stop_session_tag                 19
+#define PB_Main_gui_start_screen_stream_request_tag 20
+#define PB_Main_gui_stop_screen_stream_request_tag 21
+#define PB_Main_gui_screen_stream_frame_tag      22
+#define PB_Main_gui_send_input_event_request_tag 23

 /* Struct field encoding specification for nanopb */
 #define PB_Empty_FIELDLIST(X, a) \
@ -140,10 +152,14 @@ X(a, STATIC,   ONEOF,    MSG_W_CB, (content,storage_delete_request,content.stora
 X(a, STATIC,   ONEOF,    MSG_W_CB, (content,storage_mkdir_request,content.storage_mkdir_request),  13) \
 X(a, STATIC,   ONEOF,    MSG_W_CB, (content,storage_md5sum_request,content.storage_md5sum_request),  14) \
 X(a, STATIC,   ONEOF,    MSG_W_CB, (content,storage_md5sum_response,content.storage_md5sum_response),  15) \
-X(a, STATIC,   ONEOF,    MSG_W_CB, (content,app_start,content.app_start),  16) \
+X(a, STATIC,   ONEOF,    MSG_W_CB, (content,app_start_request,content.app_start_request),  16) \
 X(a, STATIC,   ONEOF,    MSG_W_CB, (content,app_lock_status_request,content.app_lock_status_request),  17) \
 X(a, STATIC,   ONEOF,    MSG_W_CB, (content,app_lock_status_response,content.app_lock_status_response),  18) \
-X(a, STATIC,   ONEOF,    MSG_W_CB, (content,stop_session,content.stop_session),  19)
+X(a, STATIC,   ONEOF,    MSG_W_CB, (content,stop_session,content.stop_session),  19) \
+X(a, STATIC,   ONEOF,    MSG_W_CB, (content,gui_start_screen_stream_request,content.gui_start_screen_stream_request),  20) \
+X(a, STATIC,   ONEOF,    MSG_W_CB, (content,gui_stop_screen_stream_request,content.gui_stop_screen_stream_request),  21) \
+X(a, STATIC,   ONEOF,    MSG_W_CB, (content,gui_screen_stream_frame,content.gui_screen_stream_frame),  22) \
+X(a, STATIC,   ONEOF,    MSG_W_CB, (content,gui_send_input_event_request,content.gui_send_input_event_request),  23)
 #define PB_Main_CALLBACK NULL
 #define PB_Main_DEFAULT NULL
 #define PB_Main_content_empty_MSGTYPE PB_Empty
@ -158,10 +174,14 @@ X(a, STATIC,   ONEOF,    MSG_W_CB, (content,stop_session,content.stop_session),
 #define PB_Main_content_storage_mkdir_request_MSGTYPE PB_Storage_MkdirRequest
 #define PB_Main_content_storage_md5sum_request_MSGTYPE PB_Storage_Md5sumRequest
 #define PB_Main_content_storage_md5sum_response_MSGTYPE PB_Storage_Md5sumResponse
-#define PB_Main_content_app_start_MSGTYPE PB_App_Start
+#define PB_Main_content_app_start_request_MSGTYPE PB_App_StartRequest
 #define PB_Main_content_app_lock_status_request_MSGTYPE PB_App_LockStatusRequest
 #define PB_Main_content_app_lock_status_response_MSGTYPE PB_App_LockStatusResponse
 #define PB_Main_content_stop_session_MSGTYPE PB_StopSession
+#define PB_Main_content_gui_start_screen_stream_request_MSGTYPE PB_Gui_StartScreenStreamRequest
+#define PB_Main_content_gui_stop_screen_stream_request_MSGTYPE PB_Gui_StopScreenStreamRequest
+#define PB_Main_content_gui_screen_stream_frame_MSGTYPE PB_Gui_ScreenStreamFrame
+#define PB_Main_content_gui_send_input_event_request_MSGTYPE PB_Gui_SendInputEventRequest

 extern const pb_msgdesc_t PB_Empty_msg;
 extern const pb_msgdesc_t PB_StopSession_msg;
@ -175,9 +195,9 @@ extern const pb_msgdesc_t PB_Main_msg;
 /* Maximum encoded size of messages (where known) */
 #define PB_Empty_size                            0
 #define PB_StopSession_size                      0
-#if defined(PB_Storage_ListRequest_size) && defined(PB_Storage_ListResponse_size) && defined(PB_Storage_ReadRequest_size) && defined(PB_Storage_ReadResponse_size) && defined(PB_Storage_WriteRequest_size) && defined(PB_Storage_DeleteRequest_size) && defined(PB_Storage_MkdirRequest_size) && defined(PB_Storage_Md5sumRequest_size) && defined(PB_App_Start_size)
+#if defined(PB_Storage_ListRequest_size) && defined(PB_Storage_ListResponse_size) && defined(PB_Storage_ReadRequest_size) && defined(PB_Storage_ReadResponse_size) && defined(PB_Storage_WriteRequest_size) && defined(PB_Storage_DeleteRequest_size) && defined(PB_Storage_MkdirRequest_size) && defined(PB_Storage_Md5sumRequest_size) && defined(PB_App_StartRequest_size) && defined(PB_Gui_ScreenStreamFrame_size)
 #define PB_Main_size                             (10 + sizeof(union PB_Main_content_size_union))
-union PB_Main_content_size_union {char f7[(6 + PB_Storage_ListRequest_size)]; char f8[(6 + PB_Storage_ListResponse_size)]; char f9[(6 + PB_Storage_ReadRequest_size)]; char f10[(6 + PB_Storage_ReadResponse_size)]; char f11[(6 + PB_Storage_WriteRequest_size)]; char f12[(6 + PB_Storage_DeleteRequest_size)]; char f13[(6 + PB_Storage_MkdirRequest_size)]; char f14[(6 + PB_Storage_Md5sumRequest_size)]; char f16[(7 + PB_App_Start_size)]; char f0[36];};
+union PB_Main_content_size_union {char f7[(6 + PB_Storage_ListRequest_size)]; char f8[(6 + PB_Storage_ListResponse_size)]; char f9[(6 + PB_Storage_ReadRequest_size)]; char f10[(6 + PB_Storage_ReadResponse_size)]; char f11[(6 + PB_Storage_WriteRequest_size)]; char f12[(6 + PB_Storage_DeleteRequest_size)]; char f13[(6 + PB_Storage_MkdirRequest_size)]; char f14[(6 + PB_Storage_Md5sumRequest_size)]; char f16[(7 + PB_App_StartRequest_size)]; char f22[(7 + PB_Gui_ScreenStreamFrame_size)]; char f0[36];};
 #endif

 #ifdef __cplusplus
--- a/assets/compiled/gui.pb.c
+++ b/assets/compiled/gui.pb.c
@ -0,0 +1,23 @@
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.4.5 */
+
+#include "gui.pb.h"
+#if PB_PROTO_HEADER_VERSION != 40
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+PB_BIND(PB_Gui_StartScreenStreamRequest, PB_Gui_StartScreenStreamRequest, AUTO)
+
+
+PB_BIND(PB_Gui_StopScreenStreamRequest, PB_Gui_StopScreenStreamRequest, AUTO)
+
+
+PB_BIND(PB_Gui_ScreenStreamFrame, PB_Gui_ScreenStreamFrame, AUTO)
+
+
+PB_BIND(PB_Gui_SendInputEventRequest, PB_Gui_SendInputEventRequest, AUTO)
+
+
+
+
+
--- a/assets/compiled/gui.pb.h
+++ b/assets/compiled/gui.pb.h
@ -0,0 +1,121 @@
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.4.5 */
+
+#ifndef PB_PB_GUI_GUI_PB_H_INCLUDED
+#define PB_PB_GUI_GUI_PB_H_INCLUDED
+#include <pb.h>
+
+#if PB_PROTO_HEADER_VERSION != 40
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+/* Enum definitions */
+typedef enum _PB_Gui_InputKey { 
+    PB_Gui_InputKey_UP = 0, 
+    PB_Gui_InputKey_DOWN = 1, 
+    PB_Gui_InputKey_RIGHT = 2, 
+    PB_Gui_InputKey_LEFT = 3, 
+    PB_Gui_InputKey_OK = 4, 
+    PB_Gui_InputKey_BACK = 5 
+} PB_Gui_InputKey;
+
+typedef enum _PB_Gui_InputType { 
+    PB_Gui_InputType_PRESS = 0, /* *< Press event, emitted after debounce */
+    PB_Gui_InputType_RELEASE = 1, /* *< Release event, emitted after debounce */
+    PB_Gui_InputType_SHORT = 2, /* *< Short event, emitted after InputTypeRelease done withing INPUT_LONG_PRESS interval */
+    PB_Gui_InputType_LONG = 3, /* *< Long event, emmited after INPUT_LONG_PRESS interval, asynchronouse to InputTypeRelease */
+    PB_Gui_InputType_REPEAT = 4 /* *< Repeat event, emmited with INPUT_REPEATE_PRESS period after InputTypeLong event */
+} PB_Gui_InputType;
+
+/* Struct definitions */
+typedef struct _PB_Gui_ScreenStreamFrame { 
+    pb_bytes_array_t *data; 
+} PB_Gui_ScreenStreamFrame;
+
+typedef struct _PB_Gui_StartScreenStreamRequest { 
+    char dummy_field;
+} PB_Gui_StartScreenStreamRequest;
+
+typedef struct _PB_Gui_StopScreenStreamRequest { 
+    char dummy_field;
+} PB_Gui_StopScreenStreamRequest;
+
+typedef struct _PB_Gui_SendInputEventRequest { 
+    PB_Gui_InputKey key; 
+    PB_Gui_InputType type; 
+} PB_Gui_SendInputEventRequest;
+
+
+/* Helper constants for enums */
+#define _PB_Gui_InputKey_MIN PB_Gui_InputKey_UP
+#define _PB_Gui_InputKey_MAX PB_Gui_InputKey_BACK
+#define _PB_Gui_InputKey_ARRAYSIZE ((PB_Gui_InputKey)(PB_Gui_InputKey_BACK+1))
+
+#define _PB_Gui_InputType_MIN PB_Gui_InputType_PRESS
+#define _PB_Gui_InputType_MAX PB_Gui_InputType_REPEAT
+#define _PB_Gui_InputType_ARRAYSIZE ((PB_Gui_InputType)(PB_Gui_InputType_REPEAT+1))
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Initializer values for message structs */
+#define PB_Gui_StartScreenStreamRequest_init_default {0}
+#define PB_Gui_StopScreenStreamRequest_init_default {0}
+#define PB_Gui_ScreenStreamFrame_init_default    {NULL}
+#define PB_Gui_SendInputEventRequest_init_default {_PB_Gui_InputKey_MIN, _PB_Gui_InputType_MIN}
+#define PB_Gui_StartScreenStreamRequest_init_zero {0}
+#define PB_Gui_StopScreenStreamRequest_init_zero {0}
+#define PB_Gui_ScreenStreamFrame_init_zero       {NULL}
+#define PB_Gui_SendInputEventRequest_init_zero   {_PB_Gui_InputKey_MIN, _PB_Gui_InputType_MIN}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define PB_Gui_ScreenStreamFrame_data_tag        1
+#define PB_Gui_SendInputEventRequest_key_tag     1
+#define PB_Gui_SendInputEventRequest_type_tag    2
+
+/* Struct field encoding specification for nanopb */
+#define PB_Gui_StartScreenStreamRequest_FIELDLIST(X, a) \
+
+#define PB_Gui_StartScreenStreamRequest_CALLBACK NULL
+#define PB_Gui_StartScreenStreamRequest_DEFAULT NULL
+
+#define PB_Gui_StopScreenStreamRequest_FIELDLIST(X, a) \
+
+#define PB_Gui_StopScreenStreamRequest_CALLBACK NULL
+#define PB_Gui_StopScreenStreamRequest_DEFAULT NULL
+
+#define PB_Gui_ScreenStreamFrame_FIELDLIST(X, a) \
+X(a, POINTER,  SINGULAR, BYTES,    data,              1)
+#define PB_Gui_ScreenStreamFrame_CALLBACK NULL
+#define PB_Gui_ScreenStreamFrame_DEFAULT NULL
+
+#define PB_Gui_SendInputEventRequest_FIELDLIST(X, a) \
+X(a, STATIC,   SINGULAR, UENUM,    key,               1) \
+X(a, STATIC,   SINGULAR, UENUM,    type,              2)
+#define PB_Gui_SendInputEventRequest_CALLBACK NULL
+#define PB_Gui_SendInputEventRequest_DEFAULT NULL
+
+extern const pb_msgdesc_t PB_Gui_StartScreenStreamRequest_msg;
+extern const pb_msgdesc_t PB_Gui_StopScreenStreamRequest_msg;
+extern const pb_msgdesc_t PB_Gui_ScreenStreamFrame_msg;
+extern const pb_msgdesc_t PB_Gui_SendInputEventRequest_msg;
+
+/* Defines for backwards compatibility with code written before nanopb-0.4.0 */
+#define PB_Gui_StartScreenStreamRequest_fields &PB_Gui_StartScreenStreamRequest_msg
+#define PB_Gui_StopScreenStreamRequest_fields &PB_Gui_StopScreenStreamRequest_msg
+#define PB_Gui_ScreenStreamFrame_fields &PB_Gui_ScreenStreamFrame_msg
+#define PB_Gui_SendInputEventRequest_fields &PB_Gui_SendInputEventRequest_msg
+
+/* Maximum encoded size of messages (where known) */
+/* PB_Gui_ScreenStreamFrame_size depends on runtime parameters */
+#define PB_Gui_SendInputEventRequest_size        4
+#define PB_Gui_StartScreenStreamRequest_size     0
+#define PB_Gui_StopScreenStreamRequest_size      0
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
--- a/assets/compiled/input.pb.c
+++ b/assets/compiled/input.pb.c
@ -0,0 +1,14 @@
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.4.5 */
+
+#include "input.pb.h"
+#if PB_PROTO_HEADER_VERSION != 40
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+PB_BIND(PB_Input_SendEventRequest, PB_Input_SendEventRequest, AUTO)
+
+
+
+
+
--- a/assets/compiled/input.pb.h
+++ b/assets/compiled/input.pb.h
@ -0,0 +1,78 @@
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.4.5 */
+
+#ifndef PB_PB_INPUT_INPUT_PB_H_INCLUDED
+#define PB_PB_INPUT_INPUT_PB_H_INCLUDED
+#include <pb.h>
+
+#if PB_PROTO_HEADER_VERSION != 40
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+/* Enum definitions */
+typedef enum _PB_Input_Key { 
+    PB_Input_Key_UP = 0, 
+    PB_Input_Key_DOWN = 1, 
+    PB_Input_Key_RIGHT = 2, 
+    PB_Input_Key_LEFT = 3, 
+    PB_Input_Key_OK = 4, 
+    PB_Input_Key_BACK = 5 
+} PB_Input_Key;
+
+typedef enum _PB_Input_Type { 
+    PB_Input_Type_PRESS = 0, /* *< Press event, emitted after debounce */
+    PB_Input_Type_RELEASE = 1, /* *< Release event, emitted after debounce */
+    PB_Input_Type_SHORT = 2, /* *< Short event, emitted after InputTypeRelease done withing INPUT_LONG_PRESS interval */
+    PB_Input_Type_LONG = 3, /* *< Long event, emmited after INPUT_LONG_PRESS interval, asynchronouse to InputTypeRelease */
+    PB_Input_Type_REPEAT = 4 /* *< Repeat event, emmited with INPUT_REPEATE_PRESS period after InputTypeLong event */
+} PB_Input_Type;
+
+/* Struct definitions */
+typedef struct _PB_Input_SendEventRequest { 
+    PB_Input_Key key; 
+    PB_Input_Type type; 
+} PB_Input_SendEventRequest;
+
+
+/* Helper constants for enums */
+#define _PB_Input_Key_MIN PB_Input_Key_UP
+#define _PB_Input_Key_MAX PB_Input_Key_BACK
+#define _PB_Input_Key_ARRAYSIZE ((PB_Input_Key)(PB_Input_Key_BACK+1))
+
+#define _PB_Input_Type_MIN PB_Input_Type_PRESS
+#define _PB_Input_Type_MAX PB_Input_Type_REPEAT
+#define _PB_Input_Type_ARRAYSIZE ((PB_Input_Type)(PB_Input_Type_REPEAT+1))
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Initializer values for message structs */
+#define PB_Input_SendEventRequest_init_default   {_PB_Input_Key_MIN, _PB_Input_Type_MIN}
+#define PB_Input_SendEventRequest_init_zero      {_PB_Input_Key_MIN, _PB_Input_Type_MIN}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define PB_Input_SendEventRequest_key_tag        1
+#define PB_Input_SendEventRequest_type_tag       2
+
+/* Struct field encoding specification for nanopb */
+#define PB_Input_SendEventRequest_FIELDLIST(X, a) \
+X(a, STATIC,   SINGULAR, UENUM,    key,               1) \
+X(a, STATIC,   SINGULAR, UENUM,    type,              2)
+#define PB_Input_SendEventRequest_CALLBACK NULL
+#define PB_Input_SendEventRequest_DEFAULT NULL
+
+extern const pb_msgdesc_t PB_Input_SendEventRequest_msg;
+
+/* Defines for backwards compatibility with code written before nanopb-0.4.0 */
+#define PB_Input_SendEventRequest_fields &PB_Input_SendEventRequest_msg
+
+/* Maximum encoded size of messages (where known) */
+#define PB_Input_SendEventRequest_size           4
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
--- a/assets/compiled/screen.pb.c
+++ b/assets/compiled/screen.pb.c
@ -0,0 +1,18 @@
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.4.5 */
+
+#include "screen.pb.h"
+#if PB_PROTO_HEADER_VERSION != 40
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+PB_BIND(PB_Screen_StartStreamRequest, PB_Screen_StartStreamRequest, AUTO)
+
+
+PB_BIND(PB_Screen_StopStreamRequest, PB_Screen_StopStreamRequest, AUTO)
+
+
+PB_BIND(PB_Screen_StreamFrame, PB_Screen_StreamFrame, AUTO)
+
+
+
--- a/assets/compiled/screen.pb.h
+++ b/assets/compiled/screen.pb.h
@ -0,0 +1,75 @@
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.4.5 */
+
+#ifndef PB_PB_SCREEN_SCREEN_PB_H_INCLUDED
+#define PB_PB_SCREEN_SCREEN_PB_H_INCLUDED
+#include <pb.h>
+
+#if PB_PROTO_HEADER_VERSION != 40
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+/* Struct definitions */
+typedef struct _PB_Screen_StartStreamRequest { 
+    char dummy_field;
+} PB_Screen_StartStreamRequest;
+
+typedef struct _PB_Screen_StopStreamRequest { 
+    char dummy_field;
+} PB_Screen_StopStreamRequest;
+
+typedef struct _PB_Screen_StreamFrame { 
+    pb_bytes_array_t *data; 
+} PB_Screen_StreamFrame;
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Initializer values for message structs */
+#define PB_Screen_StartStreamRequest_init_default {0}
+#define PB_Screen_StopStreamRequest_init_default {0}
+#define PB_Screen_StreamFrame_init_default       {NULL}
+#define PB_Screen_StartStreamRequest_init_zero   {0}
+#define PB_Screen_StopStreamRequest_init_zero    {0}
+#define PB_Screen_StreamFrame_init_zero          {NULL}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define PB_Screen_StreamFrame_data_tag           1
+
+/* Struct field encoding specification for nanopb */
+#define PB_Screen_StartStreamRequest_FIELDLIST(X, a) \
+
+#define PB_Screen_StartStreamRequest_CALLBACK NULL
+#define PB_Screen_StartStreamRequest_DEFAULT NULL
+
+#define PB_Screen_StopStreamRequest_FIELDLIST(X, a) \
+
+#define PB_Screen_StopStreamRequest_CALLBACK NULL
+#define PB_Screen_StopStreamRequest_DEFAULT NULL
+
+#define PB_Screen_StreamFrame_FIELDLIST(X, a) \
+X(a, POINTER,  SINGULAR, BYTES,    data,              1)
+#define PB_Screen_StreamFrame_CALLBACK NULL
+#define PB_Screen_StreamFrame_DEFAULT NULL
+
+extern const pb_msgdesc_t PB_Screen_StartStreamRequest_msg;
+extern const pb_msgdesc_t PB_Screen_StopStreamRequest_msg;
+extern const pb_msgdesc_t PB_Screen_StreamFrame_msg;
+
+/* Defines for backwards compatibility with code written before nanopb-0.4.0 */
+#define PB_Screen_StartStreamRequest_fields &PB_Screen_StartStreamRequest_msg
+#define PB_Screen_StopStreamRequest_fields &PB_Screen_StopStreamRequest_msg
+#define PB_Screen_StreamFrame_fields &PB_Screen_StreamFrame_msg
+
+/* Maximum encoded size of messages (where known) */
+/* PB_Screen_StreamFrame_size depends on runtime parameters */
+#define PB_Screen_StartStreamRequest_size        0
+#define PB_Screen_StopStreamRequest_size         0
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
--- a/assets/protobuf
+++ b/assets/protobuf
@ -1 +1 @@
-Subproject commit 021ba48abb64d25c7094da13b752fe37d4bf6007
+Subproject commit 76f43b8c6510306d40c006b696d9d1b14a252dc1