liz/flipperzero-firmware
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[FL-3097] fbt, faploader: minimal app module implementation (#2420)

Browse Source 
* fbt, faploader: minimal app module implementation
* faploader, libs: moved API hashtable core to flipper_application
* example: compound api
* lib: flipper_application: naming fixes, doxygen comments
* fbt: changed `requires` manifest field behavior for app extensions
* examples: refactored plugin apps; faploader: changed new API naming; fbt: changed PLUGIN app type meaning
* loader: dropped support for debug apps & plugin menus
* moved applications/plugins -> applications/external
* Restored x bit on chiplist_convert.py
* git: fixed free-dap submodule path
* pvs: updated submodule paths
* examples: example_advanced_plugins.c: removed potential memory leak on errors
* examples: example_plugins: refined requires
* fbt: not deploying app modules for debug/sample apps; extra validation for .PLUGIN-type apps
* apps: removed cdefines for external apps
* fbt: moved ext app path definition
* fbt: reworked fap_dist handling; f18: synced api_symbols.csv
* fbt: removed resources_paths for extapps
* scripts: reworked storage
* scripts: reworked runfap.py & selfupdate.py to use new api
* wip: fal runner
* fbt: moved file packaging into separate module
* scripts: storage: fixes
* scripts: storage: minor fixes for new api
* fbt: changed internal artifact storage details for external apps
* scripts: storage: additional fixes and better error reporting; examples: using APP_DATA_PATH()
* fbt, scripts: reworked launch_app to deploy plugins; moved old runfap.py to distfap.py
* fbt: extra check for plugins descriptors
* fbt: additional checks in emitter
* fbt: better info message on SDK rebuild
* scripts: removed requirements.txt
* loader: removed remnants of plugins & debug menus
* post-review fixes
This commit is contained in:
hedger
2023-03-14 18:29:28 +04:00
committed by GitHub
parent 4bd3dca16f
commit 53435579b3
376 changed files with 2041 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

348
scripts/flipper/storage.py
View File

@@ -4,6 +4,9 @@ import serial
import time
import hashlib
import math
import logging
import posixpath
import enum
def timing(func):
@@ -25,12 +28,47 @@ def timing(func):
return wrapper
class StorageErrorCode(enum.Enum):
OK = "OK"
NOT_READY = "filesystem not ready"
EXIST = "file/dir already exist"
NOT_EXIST = "file/dir not exist"
INVALID_PARAMETER = "invalid parameter"
DENIED = "access denied"
INVALID_NAME = "invalid name/path"
INTERNAL = "internal error"
NOT_IMPLEMENTED = "function not implemented"
ALREADY_OPEN = "file is already open"
UNKNOWN = "unknown error"
@property
def is_error(self):
return self != self.OK
@classmethod
def from_value(cls, s: str | bytes):
if isinstance(s, bytes):
s = s.decode("ascii")
for code in cls:
if code.value == s:
return code
return cls.UNKNOWN
class FlipperStorageException(Exception):
def __init__(self, message):
super().__init__(f"Storage error: {message}")
def __init__(self, path: str, error_code: StorageErrorCode):
super().__init__(f"Storage error: path '{path}': {error_code.value}")
class BufferedRead:
def __init__(self, stream):
self.buffer = bytearray()
self.stream = stream
def until(self, eol="\n", cut_eol=True):
def until(self, eol: str = "\n", cut_eol: bool = True):
eol = eol.encode("ascii")
while True:
# search in buffer
@@ -59,9 +97,15 @@ class FlipperStorage:
self.port.timeout = 2
self.port.baudrate = 115200 # Doesn't matter for VCP
self.read = BufferedRead(self.port)
self.last_error = ""
self.chunk_size = chunk_size
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.stop()
def start(self):
self.port.open()
self.port.reset_input_buffer()
@@ -71,37 +115,34 @@ class FlipperStorage:
# And read buffer until we get prompt
self.read.until(self.CLI_PROMPT)
def stop(self):
def stop(self) -> None:
self.port.close()
def send(self, line):
def send(self, line: str) -> None:
self.port.write(line.encode("ascii"))
def send_and_wait_eol(self, line):
def send_and_wait_eol(self, line: str):
self.send(line)
return self.read.until(self.CLI_EOL)
def send_and_wait_prompt(self, line):
def send_and_wait_prompt(self, line: str):
self.send(line)
return self.read.until(self.CLI_PROMPT)
def has_error(self, data):
"""Is data has error"""
if data.find(b"Storage error") != -1:
return True
else:
return False
def has_error(self, data: bytes | str) -> bool:
"""Is data an error message"""
return data.find(b"Storage error:") != -1
def get_error(self, data):
def get_error(self, data: bytes) -> StorageErrorCode:
"""Extract error text from data and print it"""
error, error_text = data.decode("ascii").split(": ")
return error_text.strip()
_, error_text = data.decode("ascii").split(": ")
return StorageErrorCode.from_value(error_text.strip())
def list_tree(self, path="/", level=0):
def list_tree(self, path: str = "/", level: int = 0):
"""List files and dirs on Flipper"""
path = path.replace("//", "/")
self.send_and_wait_eol('storage list "' + path + '"\r')
self.send_and_wait_eol(f'storage list "{path}"\r')
data = self.read.until(self.CLI_PROMPT)
lines = data.split(b"\r\n")
@@ -139,7 +180,7 @@ class FlipperStorage:
# Something wrong, pass
pass
def walk(self, path="/"):
def walk(self, path: str = "/"):
dirs = []
nondirs = []
walk_dirs = []
@@ -181,14 +222,15 @@ class FlipperStorage:
# Something wrong, pass
pass
# topdown walk, yield before recursy
# topdown walk, yield before recursing
yield path, dirs, nondirs
for new_path in walk_dirs:
yield from self.walk(new_path)
def send_file(self, filename_from, filename_to):
def send_file(self, filename_from: str, filename_to: str):
"""Send file from local device to Flipper"""
self.remove(filename_to)
if self.exist_file(filename_to):
self.remove(filename_to)
with open(filename_from, "rb") as file:
filesize = os.fstat(file.fileno()).st_size
@@ -203,9 +245,9 @@ class FlipperStorage:
self.send_and_wait_eol(f'storage write_chunk "{filename_to}" {size}\r')
answer = self.read.until(self.CLI_EOL)
if self.has_error(answer):
self.last_error = self.get_error(answer)
last_error = self.get_error(answer)
self.read.until(self.CLI_PROMPT)
return False
raise FlipperStorageException(filename_to, last_error)
self.port.write(filedata)
self.read.until(self.CLI_PROMPT)
@@ -218,9 +260,8 @@ class FlipperStorage:
)
sys.stdout.flush()
print()
return True
def read_file(self, filename):
def read_file(self, filename: str):
"""Receive file from Flipper, and get filedata (bytes)"""
buffer_size = self.chunk_size
self.send_and_wait_eol(
@@ -229,9 +270,10 @@ class FlipperStorage:
answer = self.read.until(self.CLI_EOL)
filedata = bytearray()
if self.has_error(answer):
self.last_error = self.get_error(answer)
last_error = self.get_error(answer)
self.read.until(self.CLI_PROMPT)
return filedata
raise FlipperStorageException(filename, last_error)
# return filedata
size = int(answer.split(b": ")[1])
read_size = 0
@@ -251,121 +293,89 @@ class FlipperStorage:
self.read.until(self.CLI_PROMPT)
return filedata
def receive_file(self, filename_from, filename_to):
def receive_file(self, filename_from: str, filename_to: str):
"""Receive file from Flipper to local storage"""
with open(filename_to, "wb") as file:
data = self.read_file(filename_from)
if not data:
return False
else:
file.write(data)
return True
file.write(data)
def exist(self, path):
"""Is file or dir exist on Flipper"""
self.send_and_wait_eol('storage stat "' + path + '"\r')
answer = self.read.until(self.CLI_EOL)
def exist(self, path: str):
"""Does file or dir exist on Flipper"""
self.send_and_wait_eol(f'storage stat "{path}"\r')
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
return True
return not self.has_error(response)
def exist_dir(self, path):
"""Is dir exist on Flipper"""
self.send_and_wait_eol('storage stat "' + path + '"\r')
answer = self.read.until(self.CLI_EOL)
def exist_dir(self, path: str):
"""Does dir exist on Flipper"""
self.send_and_wait_eol(f'storage stat "{path}"\r')
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
if self.has_error(response):
error_code = self.get_error(response)
if error_code in (
StorageErrorCode.NOT_EXIST,
StorageErrorCode.INVALID_NAME,
):
return False
raise FlipperStorageException(path, error_code)
return True
def exist_file(self, path: str):
"""Does file exist on Flipper"""
self.send_and_wait_eol(f'storage stat "{path}"\r')
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
if answer.find(b"Directory") != -1:
return True
elif answer.find(b"Storage") != -1:
return True
else:
return False
return response.find(b"File, size:") != -1
def exist_file(self, path):
"""Is file exist on Flipper"""
self.send_and_wait_eol('storage stat "' + path + '"\r')
answer = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
def _check_no_error(self, response, path=None):
if self.has_error(response):
raise FlipperStorageException(self.get_error(response))
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
if answer.find(b"File, size:") != -1:
return True
else:
return False
def size(self, path):
def size(self, path: str):
"""file size on Flipper"""
self.send_and_wait_eol('storage stat "' + path + '"\r')
answer = self.read.until(self.CLI_EOL)
self.send_and_wait_eol(f'storage stat "{path}"\r')
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
if answer.find(b"File, size:") != -1:
size = int(
"".join(
ch
for ch in answer.split(b": ")[1].decode("ascii")
if ch.isdigit()
)
self._check_no_error(response, path)
if response.find(b"File, size:") != -1:
size = int(
"".join(
ch
for ch in response.split(b": ")[1].decode("ascii")
if ch.isdigit()
)
return size
else:
self.last_error = "access denied"
return -1
)
return size
raise FlipperStorageException("Not a file")
def mkdir(self, path):
def mkdir(self, path: str):
"""Create a directory on Flipper"""
self.send_and_wait_eol('storage mkdir "' + path + '"\r')
answer = self.read.until(self.CLI_EOL)
self.send_and_wait_eol(f'storage mkdir "{path}"\r')
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
return True
self._check_no_error(response, path)
def format_ext(self):
"""Create a directory on Flipper"""
"""Format external storage on Flipper"""
self.send_and_wait_eol("storage format /ext\r")
self.send_and_wait_eol("y\r")
answer = self.read.until(self.CLI_EOL)
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
self._check_no_error(response, "/ext")
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
return True
def remove(self, path):
def remove(self, path: str):
"""Remove file or directory on Flipper"""
self.send_and_wait_eol('storage remove "' + path + '"\r')
answer = self.read.until(self.CLI_EOL)
self.send_and_wait_eol(f'storage remove "{path}"\r')
response = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
self._check_no_error(response, path)
if self.has_error(answer):
self.last_error = self.get_error(answer)
return False
else:
return True
def hash_local(self, filename):
def hash_local(self, filename: str):
"""Hash of local file"""
hash_md5 = hashlib.md5()
with open(filename, "rb") as f:
@@ -373,14 +383,112 @@ class FlipperStorage:
hash_md5.update(chunk)
return hash_md5.hexdigest()
def hash_flipper(self, filename):
def hash_flipper(self, filename: str):
"""Get hash of file on Flipper"""
self.send_and_wait_eol('storage md5 "' + filename + '"\r')
hash = self.read.until(self.CLI_EOL)
self.read.until(self.CLI_PROMPT)
self._check_no_error(hash, filename)
return hash.decode("ascii")
if self.has_error(hash):
self.last_error = self.get_error(hash)
return ""
class FlipperStorageOperations:
def __init__(self, storage):
self.storage: FlipperStorage = storage
self.logger = logging.getLogger("FStorageOps")
def send_file_to_storage(
self, flipper_file_path: str, local_file_path: str, force: bool = False
):
self.logger.debug(
f"* send_file_to_storage: {local_file_path}->{flipper_file_path}, {force=}"
)
exists = self.storage.exist_file(flipper_file_path)
do_upload = not exists
if exists:
hash_local = self.storage.hash_local(local_file_path)
hash_flipper = self.storage.hash_flipper(flipper_file_path)
self.logger.debug(f"hash check: local {hash_local}, flipper {hash_flipper}")
do_upload = force or (hash_local != hash_flipper)
if do_upload:
self.logger.info(f'Sending "{local_file_path}" to "{flipper_file_path}"')
self.storage.send_file(local_file_path, flipper_file_path)
# make directory with exist check
def mkpath(self, flipper_dir_path: str):
path_components, dirs_to_create = flipper_dir_path.split("/"), []
while not self.storage.exist_dir(dir_path := "/".join(path_components)):
self.logger.debug(f'"{dir_path}" does not exist, will create')
dirs_to_create.append(path_components.pop())
for dir_to_create in reversed(dirs_to_create):
path_components.append(dir_to_create)
self.storage.mkdir("/".join(path_components))
# send file or folder recursively
def recursive_send(self, flipper_path: str, local_path: str, force: bool = False):
if not os.path.exists(local_path):
raise FlipperStorageException(f'"{local_path}" does not exist')
if os.path.isdir(local_path):
# create parent dir
self.mkpath(flipper_path)
for dirpath, dirnames, filenames in os.walk(local_path):
self.logger.debug(f'Processing directory "{os.path.normpath(dirpath)}"')
dirnames.sort()
filenames.sort()
rel_path = os.path.relpath(dirpath, local_path)
# create subdirs
for dirname in dirnames:
flipper_dir_path = os.path.join(flipper_path, rel_path, dirname)
flipper_dir_path = os.path.normpath(flipper_dir_path).replace(
os.sep, "/"
)
self.mkpath(flipper_dir_path)
# send files
for filename in filenames:
flipper_file_path = os.path.join(flipper_path, rel_path, filename)
flipper_file_path = os.path.normpath(flipper_file_path).replace(
os.sep, "/"
)
local_file_path = os.path.normpath(os.path.join(dirpath, filename))
self.send_file_to_storage(flipper_file_path, local_file_path, force)
else:
return hash.decode("ascii")
self.mkpath(posixpath.dirname(flipper_path))
self.send_file_to_storage(flipper_path, local_path, force)
def recursive_receive(self, flipper_path: str, local_path: str):
if self.storage.exist_dir(flipper_path):
for dirpath, dirnames, filenames in self.storage.walk(flipper_path):
self.logger.debug(
f'Processing directory "{os.path.normpath(dirpath)}"'.replace(
os.sep, "/"
)
)
dirnames.sort()
filenames.sort()
rel_path = os.path.relpath(dirpath, flipper_path)
for dirname in dirnames:
local_dir_path = os.path.join(local_path, rel_path, dirname)
local_dir_path = os.path.normpath(local_dir_path)
os.makedirs(local_dir_path, exist_ok=True)
for filename in filenames:
local_file_path = os.path.join(local_path, rel_path, filename)
local_file_path = os.path.normpath(local_file_path)
flipper_file_path = os.path.normpath(
os.path.join(dirpath, filename)
).replace(os.sep, "/")
self.logger.info(
f'Receiving "{flipper_file_path}" to "{local_file_path}"'
)
self.storage.receive_file(flipper_file_path, local_file_path)
else:
self.logger.info(f'Receiving "{flipper_path}" to "{local_path}"')
self.storage.receive_file(flipper_path, local_path)