From 76d38e832e2da86457d0114beb20b88b6dc24221 Mon Sep 17 00:00:00 2001 From: hedger Date: Thu, 29 Sep 2022 15:00:22 +0400 Subject: [PATCH] fbt: reproducible manifest builds & improvements (#1801) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fbt: reproducible manifest builds, less rebuild on small updates; scripts: assets: using timestamp from commandline af available * fbt: added app import validation for launch_app & single app build targets * fbt: COMSTR for app imports validation * docs: minor fixes * docs: markdown fix * vscode: comments for RTOS startup Co-authored-by: あく --- .vscode/example/launch.json | 3 ++- assets/SConscript | 6 +++++- documentation/AppManifests.md | 4 ++-- firmware.scons | 1 + scripts/assets.py | 9 ++++++++- scripts/flipper/assets/manifest.py | 4 ++-- site_scons/extapps.scons | 3 ++- site_scons/fbt/appmanifest.py | 2 +- site_scons/fbt/version.py | 5 +++++ site_scons/site_tools/fbt_extapps.py | 21 +++++++++++++++++---- 10 files changed, 45 insertions(+), 13 deletions(-) diff --git a/.vscode/example/launch.json b/.vscode/example/launch.json index 7cb2542d..f4df9659 100644 --- a/.vscode/example/launch.json +++ b/.vscode/example/launch.json @@ -28,13 +28,14 @@ "servertype": "openocd", "device": "stlink", "svdFile": "./debug/STM32WB55_CM4.svd", + // If you're debugging early in the boot process, before OS scheduler is running, + // you have to comment out the following line. "rtos": "FreeRTOS", "configFiles": [ "interface/stlink.cfg", "./debug/stm32wbx.cfg", ], "postAttachCommands": [ - // "attach 1", // "compare-sections", "source debug/flipperapps.py", // "source debug/FreeRTOS/FreeRTOS.py", diff --git a/assets/SConscript b/assets/SConscript index 47713d1a..a0b3b13a 100644 --- a/assets/SConscript +++ b/assets/SConscript @@ -1,8 +1,11 @@ Import("env") +from fbt.version import get_git_commit_unix_timestamp + assetsenv = env.Clone( tools=["fbt_assets"], FW_LIB_NAME="assets", + GIT_UNIX_TIMESTAMP=get_git_commit_unix_timestamp(), ) assetsenv.ApplyLibFlags() @@ -90,10 +93,11 @@ if assetsenv["IS_BASE_FIRMWARE"]: "#/assets/resources/Manifest", assetsenv.GlobRecursive("*", "resources", exclude="Manifest"), action=Action( - '${PYTHON3} "${ASSETS_COMPILER}" manifest "${TARGET.dir.posix}"', + '${PYTHON3} "${ASSETS_COMPILER}" manifest "${TARGET.dir.posix}" --timestamp=${GIT_UNIX_TIMESTAMP}', "${RESMANIFESTCOMSTR}", ), ) + assetsenv.Precious(resources) assetsenv.AlwaysBuild(resources) assetsenv.Clean( resources, diff --git a/documentation/AppManifests.md b/documentation/AppManifests.md index 5b14b7dd..0a4f5e9b 100644 --- a/documentation/AppManifests.md +++ b/documentation/AppManifests.md @@ -43,8 +43,8 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio The following parameters are used only for [FAPs](./AppsOnSDCard.md): -* **sources**: list of file name masks, used for gathering sources within app folder. Default value of ["\*.c\*"] includes C and CPP source files. -* **fap_version**: string, 2 numbers in form of "x.y": application version to be embedded within .fap file. +* **sources**: list of strings, file name masks, used for gathering sources within app folder. Default value of `["*.c*"]` includes C and CPP source files. +* **fap_version**: tuple, 2 numbers in form of (x,y): application version to be embedded within .fap file. Default value is (0,1), meanig version "0.1". * **fap_icon**: name of a .png file, 1-bit color depth, 10x10px, to be embedded within .fap file. * **fap_libs**: list of extra libraries to link application against. Provides access to extra functions that are not exported as a part of main firmware at expense of increased .fap file size and RAM consumption. * **fap_category**: string, may be empty. App subcategory, also works as path of FAP within apps folder in the file system. diff --git a/firmware.scons b/firmware.scons index 530634ef..0970541e 100644 --- a/firmware.scons +++ b/firmware.scons @@ -96,6 +96,7 @@ if not env["VERBOSE"]: SDKSYM_GENERATOR_COMSTR="\tSDKSYM\t${TARGET}", APPMETA_COMSTR="\tAPPMETA\t${TARGET}", APPMETAEMBED_COMSTR="\tFAP\t${TARGET}", + APPCHECK_COMSTR="\tAPPCHK\t${SOURCE}", ) diff --git a/scripts/assets.py b/scripts/assets.py index 6f941013..9b4ee5b6 100755 --- a/scripts/assets.py +++ b/scripts/assets.py @@ -39,6 +39,13 @@ class Main(App): "manifest", help="Create directory Manifest" ) self.parser_manifest.add_argument("local_path", help="local_path") + self.parser_manifest.add_argument( + "--timestamp", + help="timestamp value to embed", + default=0, + type=int, + required=False, + ) self.parser_manifest.set_defaults(func=self.manifest) self.parser_copro = self.subparsers.add_parser( @@ -213,7 +220,7 @@ class Main(App): self.logger.info( f'Creating temporary Manifest for directory "{directory_path}"' ) - new_manifest = Manifest() + new_manifest = Manifest(self.args.timestamp) new_manifest.create(directory_path) self.logger.info(f"Comparing new manifest with existing") diff --git a/scripts/flipper/assets/manifest.py b/scripts/flipper/assets/manifest.py index 840761d7..a8f6855a 100644 --- a/scripts/flipper/assets/manifest.py +++ b/scripts/flipper/assets/manifest.py @@ -106,11 +106,11 @@ addManifestRecord(ManifestRecordFile) class Manifest: - def __init__(self): + def __init__(self, timestamp_value=None): self.version = None self.records = [] self.records.append(ManifestRecordVersion(MANIFEST_VERSION)) - self.records.append(ManifestRecordTimestamp(timestamp())) + self.records.append(ManifestRecordTimestamp(timestamp_value or timestamp())) self.logger = logging.getLogger(self.__class__.__name__) def load(self, filename): diff --git a/site_scons/extapps.scons b/site_scons/extapps.scons index 4cb5e35c..66002915 100644 --- a/site_scons/extapps.scons +++ b/site_scons/extapps.scons @@ -86,12 +86,13 @@ if appenv["FORCE"]: Alias(appenv["FIRMWARE_BUILD_CFG"] + "_extapps", extapps["compact"].values()) if appsrc := appenv.subst("$APPSRC"): - app_manifest, fap_file = appenv.GetExtAppFromPath(appsrc) + app_manifest, fap_file, app_validator = appenv.GetExtAppFromPath(appsrc) appenv.PhonyTarget( "launch_app", '${PYTHON3} scripts/runfap.py ${SOURCE} --fap_dst_dir "/ext/apps/${FAP_CATEGORY}"', source=fap_file, FAP_CATEGORY=app_manifest.fap_category, ) + appenv.Alias("launch_app", app_validator) Return("extapps") diff --git a/site_scons/fbt/appmanifest.py b/site_scons/fbt/appmanifest.py index a0ab5e7c..a1132eea 100644 --- a/site_scons/fbt/appmanifest.py +++ b/site_scons/fbt/appmanifest.py @@ -38,7 +38,7 @@ class FlipperApplication: sdk_headers: List[str] = field(default_factory=list) # .fap-specific sources: List[str] = field(default_factory=lambda: ["*.c*"]) - fap_version: Tuple[int] = field(default_factory=lambda: (0, 0)) + fap_version: Tuple[int] = field(default_factory=lambda: (0, 1)) fap_icon: Optional[str] = None fap_libs: List[str] = field(default_factory=list) fap_category: str = "" diff --git a/site_scons/fbt/version.py b/site_scons/fbt/version.py index 1b1c166f..e7fe2eda 100644 --- a/site_scons/fbt/version.py +++ b/site_scons/fbt/version.py @@ -3,6 +3,11 @@ import datetime from functools import cache +@cache +def get_git_commit_unix_timestamp(): + return int(subprocess.check_output(["git", "show", "-s", "--format=%ct"])) + + @cache def get_fast_git_version_id(): try: diff --git a/site_scons/site_tools/fbt_extapps.py b/site_scons/site_tools/fbt_extapps.py index 4b584530..fec24071 100644 --- a/site_scons/site_tools/fbt_extapps.py +++ b/site_scons/site_tools/fbt_extapps.py @@ -37,7 +37,15 @@ def BuildAppElf(env, app): APP=app, ) - env.Depends(app_elf_augmented, [env["SDK_DEFINITION"], env.Value(app)]) + manifest_vals = vars(app) + manifest_vals = { + k: v for k, v in manifest_vals.items() if k not in ("_appdir", "_apppath") + } + + env.Depends( + app_elf_augmented, + [env["SDK_DEFINITION"], env.Value(manifest_vals)], + ) if app.fap_icon: env.Depends( app_elf_augmented, @@ -47,6 +55,7 @@ def BuildAppElf(env, app): app_elf_import_validator = env.ValidateAppImports(app_elf_augmented) env.AlwaysBuild(app_elf_import_validator) + env.Alias(app_alias, app_elf_import_validator) return (app_elf_augmented, app_elf_raw, app_elf_import_validator) @@ -100,9 +109,13 @@ def GetExtAppFromPath(env, app_dir): app_elf = env["_extapps"]["compact"].get(app.appid, None) if not app_elf: - raise UserError(f"No external app found for {app.appid}") + raise UserError( + f"Application {app.appid} is not configured for building as external" + ) - return (app, app_elf[0]) + app_validator = env["_extapps"]["validators"].get(app.appid, None) + + return (app, app_elf[0], app_validator[0]) def generate(env, **kw): @@ -138,7 +151,7 @@ def generate(env, **kw): ), Action( validate_app_imports, - None, # "$APPCHECK_COMSTR", + "$APPCHECK_COMSTR", ), ], suffix=".impsyms",