diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1304c5d7..8fb67ed1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,14 +56,14 @@ jobs: - name: 'Bundle scripts' if: ${{ !github.event.pull_request.head.repo.fork }} run: | - tar czpf artifacts/flipper-z-any-scripts-${SUFFIX}.tgz scripts + tar czpf artifacts/flipper-z-any-scripts-${SUFFIX}.tgz scripts debug - name: 'Build the firmware' run: | set -e for TARGET in ${TARGETS}; do FBT_TOOLCHAIN_PATH=/runner/_work ./fbt TARGET_HW="$(echo "${TARGET}" | sed 's/f//')" \ - updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }} + copro_dist updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }} done - name: 'Move upload files' @@ -74,17 +74,6 @@ jobs: mv dist/${TARGET}-*/* artifacts/ done - - name: 'Bundle self-update package' - if: ${{ !github.event.pull_request.head.repo.fork }} - run: | - set -e - for UPDATEBUNDLE in artifacts/*/; do - BUNDLE_NAME="$(echo "$UPDATEBUNDLE" | cut -d'/' -f2)" - echo Packaging "${BUNDLE_NAME}" - tar czpf "artifacts/flipper-z-${BUNDLE_NAME}.tgz" -C artifacts "${BUNDLE_NAME}" - rm -rf "artifacts/${BUNDLE_NAME}" - done - - name: "Check for uncommitted changes" run: | git diff --exit-code @@ -97,8 +86,7 @@ jobs: - name: 'Bundle core2 firmware' if: ${{ !github.event.pull_request.head.repo.fork }} run: | - FBT_TOOLCHAIN_PATH=/runner/_work ./fbt copro_dist - tar czpf "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz" -C assets core2_firmware + cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz" - name: 'Copy .map file' run: | diff --git a/SConstruct b/SConstruct index 5ad2ac3c..74fa5667 100644 --- a/SConstruct +++ b/SConstruct @@ -7,7 +7,6 @@ # construction of certain targets behind command-line options. import os -import subprocess DefaultEnvironment(tools=[]) @@ -15,17 +14,22 @@ EnsurePythonVersion(3, 8) # Progress(["OwO\r", "owo\r", "uwu\r", "owo\r"], interval=15) - # This environment is created only for loading options & validating file/dir existence fbt_variables = SConscript("site_scons/commandline.scons") -cmd_environment = Environment(tools=[], variables=fbt_variables) -Help(fbt_variables.GenerateHelpText(cmd_environment)) +cmd_environment = Environment( + toolpath=["#/scripts/fbt_tools"], + tools=[ + ("fbt_help", {"vars": fbt_variables}), + ], + variables=fbt_variables, +) # Building basic environment - tools, utility methods, cross-compilation # settings, gcc flags for Cortex-M4, basic builders and more coreenv = SConscript( "site_scons/environ.scons", exports={"VAR_ENV": cmd_environment}, + toolpath=["#/scripts/fbt_tools"], ) SConscript("site_scons/cc.scons", exports={"ENV": coreenv}) @@ -35,41 +39,13 @@ coreenv["ROOT_DIR"] = Dir(".") # Create a separate "dist" environment and add construction envs to it distenv = coreenv.Clone( - tools=["fbt_dist", "openocd", "blackmagic", "jflash"], - OPENOCD_GDB_PIPE=[ - "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}" + tools=[ + "fbt_dist", + "fbt_debugopts", + "openocd", + "blackmagic", + "jflash", ], - GDBOPTS_BASE=[ - "-ex", - "target extended-remote ${GDBREMOTE}", - "-ex", - "set confirm off", - "-ex", - "set pagination off", - ], - GDBOPTS_BLACKMAGIC=[ - "-ex", - "monitor swdp_scan", - "-ex", - "monitor debug_bmp enable", - "-ex", - "attach 1", - "-ex", - "set mem inaccessible-by-default off", - ], - GDBPYOPTS=[ - "-ex", - "source debug/FreeRTOS/FreeRTOS.py", - "-ex", - "source debug/flipperapps.py", - "-ex", - "source debug/PyCortexMDebug/PyCortexMDebug.py", - "-ex", - "svd_load ${SVD_FILE}", - "-ex", - "compare-sections", - ], - JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash", ENV=os.environ, ) @@ -166,7 +142,7 @@ basic_dist = distenv.DistCommand("fw_dist", distenv["DIST_DEPENDS"]) distenv.Default(basic_dist) dist_dir = distenv.GetProjetDirName() -plugin_dist = [ +fap_dist = [ distenv.Install( f"#/dist/{dist_dir}/apps/debug_elf", firmware_env["FW_EXTAPPS"]["debug"].values(), @@ -176,9 +152,9 @@ plugin_dist = [ for dist_entry in firmware_env["FW_EXTAPPS"]["dist"].values() ), ] -Depends(plugin_dist, firmware_env["FW_EXTAPPS"]["validators"].values()) -Alias("plugin_dist", plugin_dist) -# distenv.Default(plugin_dist) +Depends(fap_dist, firmware_env["FW_EXTAPPS"]["validators"].values()) +Alias("fap_dist", fap_dist) +# distenv.Default(fap_dist) plugin_resources_dist = list( distenv.Install(f"#/assets/resources/apps/{dist_entry[0]}", dist_entry[1]) @@ -189,9 +165,10 @@ distenv.Depends(firmware_env["FW_RESOURCES"], plugin_resources_dist) # Target for bundling core2 package for qFlipper copro_dist = distenv.CoproBuilder( - distenv.Dir("assets/core2_firmware"), + "#/build/core2_firmware.tgz", [], ) +distenv.AlwaysBuild(copro_dist) distenv.Alias("copro_dist", copro_dist) firmware_flash = distenv.AddOpenOCDFlashTarget(firmware_env) diff --git a/applications/examples/example_images/example_images.c b/applications/examples/example_images/example_images.c index 48fa5e77..b00818cd 100644 --- a/applications/examples/example_images/example_images.c +++ b/applications/examples/example_images/example_images.c @@ -4,6 +4,8 @@ #include #include +/* Magic happens here -- this file is generated by fbt. + * Just set fap_icon_assets in application.fam and #include {APPID}_icons.h */ #include "example_images_icons.h" typedef struct { diff --git a/documentation/AppManifests.md b/documentation/AppManifests.md index 4fec9d22..c7c73110 100644 --- a/documentation/AppManifests.md +++ b/documentation/AppManifests.md @@ -30,7 +30,7 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio | METAPACKAGE | Does not define any code to be run, used for declaring dependencies and application bundles | * **name**: Name that is displayed in menus. -* **entry_point**: C function to be used as application's entry point. +* **entry_point**: C function to be used as application's entry point. Note that C++ function names are mangled, so you need to wrap them in `extern "C"` in order to use them as entry points. * **flags**: Internal flags for system apps. Do not use. * **cdefines**: C preprocessor definitions to declare globally for other apps when current application is included in active build configuration. * **requires**: List of application IDs to also include in build configuration, when current application is referenced in list of applications to build. @@ -55,7 +55,7 @@ The following parameters are used only for [FAPs](./AppsOnSDCard.md): * **fap_author**: string, may be empty. Application's author. * **fap_weburl**: string, may be empty. Application's homepage. * **fap_icon_assets**: string. If present, defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details. -* **fap_extbuild**: provides support for parts of application sources to be build by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list. +* **fap_extbuild**: provides support for parts of application sources to be built by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list. Note that commands are executed at the firmware root folder's root, and all intermediate files must be placed in a application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**. Example for building an app from Rust sources: diff --git a/documentation/AppsOnSDCard.md b/documentation/AppsOnSDCard.md index aff8314d..4acb3ec3 100644 --- a/documentation/AppsOnSDCard.md +++ b/documentation/AppsOnSDCard.md @@ -2,7 +2,7 @@ [fbt](./fbt.md) has support for building applications as FAP files. FAP are essentially .elf executables with extra metadata and resources bundled in. -FAPs are built with `faps` **`fbt`** target. They can also be deployed to `dist` folder with `plugin_dist` **`fbt`** target. +FAPs are built with `faps` target. They can also be deployed to `dist` folder with `fap_dist` target. FAPs do not depend on being run on a specific firmware version. Compatibility is determined by the FAP's metadata, which includes the required [API version](#api-versioning). @@ -15,7 +15,7 @@ To build your application as a FAP, just create a folder with your app's source * To build your application, run `./fbt fap_{APPID}`, where APPID is your application's ID in its manifest. * To build your app, then upload it over USB & run it on Flipper, use `./fbt launch_app APPSRC=applications/path/to/app`. This command is configured in default [VSCode profile](../.vscode/ReadMe.md) as "Launch App on Flipper" build action (Ctrl+Shift+B menu). - * To build all FAPs, run `./fbt plugin_dist`. + * To build all FAPs, run `./fbt faps` or `./fbt fap_dist`. ## FAP assets diff --git a/documentation/fbt.md b/documentation/fbt.md index e20d4317..3fac7ce7 100644 --- a/documentation/fbt.md +++ b/documentation/fbt.md @@ -43,7 +43,7 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option. ### High-level (what you most likely need) - `fw_dist` - build & publish firmware to `dist` folder. This is a default target, when no other are specified -- `plugin_dist` - build external plugins & publish to `dist` folder +- `fap_dist` - build external plugins & publish to `dist` folder - `updater_package`, `updater_minpackage` - build self-update package. Minimal version only inclues firmware's DFU file; full version also includes radio stack & resources for SD card - `copro_dist` - bundle Core2 FUS+stack binaries for qFlipper - `flash` - flash attached device with OpenOCD over ST-Link @@ -56,6 +56,7 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option. - `get_blackmagic` - output blackmagic address in gdb remote format. Useful for IDE integration - `lint`, `format` - run clang-format on C source code to check and reformat it according to `.clang-format` specs - `lint_py`, `format_py` - run [black](https://black.readthedocs.io/en/stable/index.html) on Python source code, build system files & application manifests +- `cli` - start Flipper CLI session over USB ### Firmware targets diff --git a/firmware.scons b/firmware.scons index dd13b6b3..d28309d0 100644 --- a/firmware.scons +++ b/firmware.scons @@ -3,7 +3,7 @@ Import("ENV", "fw_build_meta") from SCons.Errors import UserError import itertools -from fbt.util import ( +from fbt_extra.util import ( should_gen_cdb_and_link_dir, link_elf_dir_as_latest, ) @@ -141,6 +141,10 @@ else: if extra_int_apps := GetOption("extra_int_apps"): fwenv.Append(APPS=extra_int_apps.split(",")) + +if fwenv["FAP_EXAMPLES"]: + fwenv.Append(APPDIRS=[("applications/examples", False)]) + fwenv.LoadApplicationManifests() fwenv.PrepareApplicationsBuild() @@ -316,10 +320,13 @@ if fwenv["IS_BASE_FIRMWARE"]: "-D__inline__=inline", ], ) - Depends(sdk_source, (fwenv["SDK_HEADERS"], fwenv["FW_ASSETS_HEADERS"])) + # Depends(sdk_source, (fwenv["SDK_HEADERS"], fwenv["FW_ASSETS_HEADERS"])) + Depends(sdk_source, fwenv.ProcessSdkDepends("sdk_origin.d")) - sdk_tree = fwenv.SDKTree("sdk/sdk.opts", "sdk_origin") - AlwaysBuild(sdk_tree) + fwenv["SDK_DIR"] = fwenv.Dir("sdk") + sdk_tree = fwenv.SDKTree(fwenv["SDK_DIR"], "sdk_origin") + fw_artifacts.append(sdk_tree) + # AlwaysBuild(sdk_tree) Alias("sdk_tree", sdk_tree) sdk_apicheck = fwenv.SDKSymUpdater(fwenv.subst("$SDK_DEFINITION"), "sdk_origin") @@ -329,7 +336,7 @@ if fwenv["IS_BASE_FIRMWARE"]: Alias("sdk_check", sdk_apicheck) sdk_apisyms = fwenv.SDKSymGenerator( - "assets/compiled/symbols.h", fwenv.subst("$SDK_DEFINITION") + "assets/compiled/symbols.h", fwenv["SDK_DEFINITION"] ) Alias("api_syms", sdk_apisyms) diff --git a/firmware/targets/f7/application-ext.ld b/firmware/targets/f7/application_ext.ld similarity index 100% rename from firmware/targets/f7/application-ext.ld rename to firmware/targets/f7/application_ext.ld diff --git a/site_scons/fbt/__init__.py b/scripts/fbt/__init__.py similarity index 100% rename from site_scons/fbt/__init__.py rename to scripts/fbt/__init__.py diff --git a/site_scons/fbt/appmanifest.py b/scripts/fbt/appmanifest.py similarity index 100% rename from site_scons/fbt/appmanifest.py rename to scripts/fbt/appmanifest.py diff --git a/site_scons/fbt/elfmanifest.py b/scripts/fbt/elfmanifest.py similarity index 100% rename from site_scons/fbt/elfmanifest.py rename to scripts/fbt/elfmanifest.py diff --git a/site_scons/fbt/sdk.py b/scripts/fbt/sdk.py similarity index 100% rename from site_scons/fbt/sdk.py rename to scripts/fbt/sdk.py diff --git a/site_scons/fbt/util.py b/scripts/fbt/util.py similarity index 58% rename from site_scons/fbt/util.py rename to scripts/fbt/util.py index 8d8af518..baa4ddfe 100644 --- a/site_scons/fbt/util.py +++ b/scripts/fbt/util.py @@ -41,25 +41,3 @@ def link_dir(target_path, source_path, is_windows): def single_quote(arg_list): return " ".join(f"'{arg}'" if " " in arg else str(arg) for arg in arg_list) - - -def link_elf_dir_as_latest(env, elf_node): - elf_dir = elf_node.Dir(".") - latest_dir = env.Dir("#build/latest") - print(f"Setting {elf_dir} as latest built dir (./build/latest/)") - return link_dir(latest_dir.abspath, elf_dir.abspath, env["PLATFORM"] == "win32") - - -def should_gen_cdb_and_link_dir(env, requested_targets): - explicitly_building_updater = False - # Hacky way to check if updater-related targets were requested - for build_target in requested_targets: - if "updater" in str(build_target): - explicitly_building_updater = True - - is_updater = not env["IS_BASE_FIRMWARE"] - # If updater is explicitly requested, link to the latest updater - # Otherwise, link to firmware - return (is_updater and explicitly_building_updater) or ( - not is_updater and not explicitly_building_updater - ) diff --git a/site_scons/fbt/version.py b/scripts/fbt/version.py similarity index 100% rename from site_scons/fbt/version.py rename to scripts/fbt/version.py diff --git a/site_scons/site_tools/blackmagic.py b/scripts/fbt_tools/blackmagic.py similarity index 100% rename from site_scons/site_tools/blackmagic.py rename to scripts/fbt_tools/blackmagic.py diff --git a/site_scons/site_tools/ccache.py b/scripts/fbt_tools/ccache.py similarity index 100% rename from site_scons/site_tools/ccache.py rename to scripts/fbt_tools/ccache.py diff --git a/site_scons/site_tools/crosscc.py b/scripts/fbt_tools/crosscc.py similarity index 100% rename from site_scons/site_tools/crosscc.py rename to scripts/fbt_tools/crosscc.py diff --git a/site_scons/site_tools/fbt_apps.py b/scripts/fbt_tools/fbt_apps.py similarity index 100% rename from site_scons/site_tools/fbt_apps.py rename to scripts/fbt_tools/fbt_apps.py diff --git a/site_scons/site_tools/fbt_assets.py b/scripts/fbt_tools/fbt_assets.py similarity index 100% rename from site_scons/site_tools/fbt_assets.py rename to scripts/fbt_tools/fbt_assets.py diff --git a/scripts/fbt_tools/fbt_debugopts.py b/scripts/fbt_tools/fbt_debugopts.py new file mode 100644 index 00000000..3e7b0701 --- /dev/null +++ b/scripts/fbt_tools/fbt_debugopts.py @@ -0,0 +1,41 @@ +def generate(env, **kw): + env.SetDefault( + OPENOCD_GDB_PIPE=[ + "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}" + ], + GDBOPTS_BASE=[ + "-ex", + "target extended-remote ${GDBREMOTE}", + "-ex", + "set confirm off", + "-ex", + "set pagination off", + ], + GDBOPTS_BLACKMAGIC=[ + "-ex", + "monitor swdp_scan", + "-ex", + "monitor debug_bmp enable", + "-ex", + "attach 1", + "-ex", + "set mem inaccessible-by-default off", + ], + GDBPYOPTS=[ + "-ex", + "source debug/FreeRTOS/FreeRTOS.py", + "-ex", + "source debug/flipperapps.py", + "-ex", + "source debug/PyCortexMDebug/PyCortexMDebug.py", + "-ex", + "svd_load ${SVD_FILE}", + "-ex", + "compare-sections", + ], + JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash", + ) + + +def exists(env): + return True diff --git a/site_scons/site_tools/fbt_dist.py b/scripts/fbt_tools/fbt_dist.py similarity index 98% rename from site_scons/site_tools/fbt_dist.py rename to scripts/fbt_tools/fbt_dist.py index 2b5c83df..853013e9 100644 --- a/site_scons/site_tools/fbt_dist.py +++ b/scripts/fbt_tools/fbt_dist.py @@ -136,7 +136,6 @@ def generate(env): "CoproBuilder": Builder( action=Action( [ - Mkdir("$TARGET"), '${PYTHON3} "${ROOT_DIR.abspath}/scripts/assets.py" ' "copro ${COPRO_CUBE_DIR} " "${TARGET} ${COPRO_MCU_FAMILY} " @@ -145,7 +144,7 @@ def generate(env): '--stack_file="${COPRO_STACK_BIN}" ' "--stack_addr=${COPRO_STACK_ADDR} ", ], - "", + "\tCOPRO\t${TARGET}", ) ), } diff --git a/site_scons/site_tools/fbt_extapps.py b/scripts/fbt_tools/fbt_extapps.py similarity index 96% rename from site_scons/site_tools/fbt_extapps.py rename to scripts/fbt_tools/fbt_extapps.py index a1fa7714..34fb942a 100644 --- a/site_scons/site_tools/fbt_extapps.py +++ b/scripts/fbt_tools/fbt_extapps.py @@ -6,12 +6,10 @@ import SCons.Warnings import os import pathlib from fbt.elfmanifest import assemble_manifest_data -from fbt.appmanifest import FlipperManifestException +from fbt.appmanifest import FlipperApplication, FlipperManifestException from fbt.sdk import SdkCache import itertools -from site_scons.fbt.appmanifest import FlipperApplication - def BuildAppElf(env, app): ext_apps_work_dir = env.subst("$EXT_APPS_WORK_DIR") @@ -111,7 +109,7 @@ def BuildAppElf(env, app): ) app_elf_raw = app_env.Program( - os.path.join(app_work_dir, f"{app.appid}_d"), + os.path.join(ext_apps_work_dir, f"{app.appid}_d"), app_sources, APP_ENTRY=app.entry_point, ) @@ -180,7 +178,7 @@ def validate_app_imports(target, source, env): if unresolved_syms: SCons.Warnings.warn( SCons.Warnings.LinkWarning, - f"{source[0].path}: app won't run. Unresolved symbols: {unresolved_syms}", + f"\033[93m{source[0].path}: app won't run. Unresolved symbols: \033[95m{unresolved_syms}\033[0m", ) diff --git a/scripts/fbt_tools/fbt_help.py b/scripts/fbt_tools/fbt_help.py new file mode 100644 index 00000000..0475f51b --- /dev/null +++ b/scripts/fbt_tools/fbt_help.py @@ -0,0 +1,44 @@ +targets_help = """Configuration variables: +""" + +tail_help = """ + +TASKS: +Building: + firmware_all, fw_dist: + Build firmware; create distribution package + faps, fap_dist: + Build all FAP apps + fap_{APPID}, launch_app APPSRC={APPID}: + Build FAP app with appid={APPID}; upload & start it over USB + +Flashing & debugging: + flash, flash_blackmagic, jflash: + Flash firmware to target using debug probe + flash_usb, flash_usb_full: + Install firmware using self-update package + debug, debug_other, blackmagic: + Start GDB + +Other: + cli: + Open a Flipper CLI session over USB + firmware_cdb, updater_cdb: + Generate сompilation_database.json + lint, lint_py: + run linters + format, format_py: + run code formatters + +For more targets & info, see documentation/fbt.md +""" + + +def generate(env, **kw): + vars = kw["vars"] + basic_help = vars.GenerateHelpText(env) + env.Help(targets_help + basic_help + tail_help) + + +def exists(env): + return True diff --git a/site_scons/site_tools/fbt_sdk.py b/scripts/fbt_tools/fbt_sdk.py similarity index 71% rename from site_scons/site_tools/fbt_sdk.py rename to scripts/fbt_tools/fbt_sdk.py index f6c2d452..ed0abdff 100644 --- a/site_scons/site_tools/fbt_sdk.py +++ b/scripts/fbt_tools/fbt_sdk.py @@ -9,10 +9,32 @@ from SCons.Util import LogicalLines import os.path import posixpath import pathlib +import json from fbt.sdk import SdkCollector, SdkCache +def ProcessSdkDepends(env, filename): + try: + with open(filename, "r") as fin: + lines = LogicalLines(fin).readlines() + except IOError: + return [] + + _, depends = lines[0].split(":", 1) + depends = depends.split() + depends.pop(0) # remove the .c file + depends = list( + # Don't create dependency on non-existing files + # (e.g. when they were renamed since last build) + filter( + lambda file: file.exists(), + (env.File(f"#{path}") for path in depends), + ) + ) + return depends + + def prebuild_sdk_emitter(target, source, env): target.append(env.ChangeFileExtension(target[0], ".d")) target.append(env.ChangeFileExtension(target[0], ".i.c")) @@ -25,6 +47,25 @@ def prebuild_sdk_create_origin_file(target, source, env): sdk_c.write("\n".join(f"#include <{h.path}>" for h in env["SDK_HEADERS"])) +class SdkMeta: + def __init__(self, env): + self.env = env + + def save_to(self, json_manifest_path: str): + meta_contents = { + "sdk_symbols": self.env["SDK_DEFINITION"].name, + "cc_args": self._wrap_scons_vars("$CCFLAGS $_CCCOMCOM"), + "cpp_args": self._wrap_scons_vars("$CXXFLAGS $CCFLAGS $_CCCOMCOM"), + "linker_args": self._wrap_scons_vars("$LINKFLAGS"), + } + with open(json_manifest_path, "wt") as f: + json.dump(meta_contents, f, indent=4) + + def _wrap_scons_vars(self, vars: str): + expanded_vars = self.env.subst(vars, target=Entry("dummy")) + return expanded_vars.replace("\\", "/") + + class SdkTreeBuilder: def __init__(self, env, target, source) -> None: self.env = env @@ -34,8 +75,9 @@ class SdkTreeBuilder: self.header_depends = [] self.header_dirs = [] - self.target_sdk_dir = env.subst("f${TARGET_HW}_sdk") - self.sdk_deploy_dir = target[0].Dir(self.target_sdk_dir) + self.target_sdk_dir_name = env.subst("f${TARGET_HW}_sdk") + self.sdk_root_dir = target[0].Dir(".") + self.sdk_deploy_dir = self.sdk_root_dir.Dir(self.target_sdk_dir_name) def _parse_sdk_depends(self): deps_file = self.source[0] @@ -50,7 +92,7 @@ class SdkTreeBuilder: ) def _generate_sdk_meta(self): - filtered_paths = [self.target_sdk_dir] + filtered_paths = [self.target_sdk_dir_name] full_fw_paths = list( map( os.path.normpath, @@ -62,17 +104,18 @@ class SdkTreeBuilder: for dir in full_fw_paths: if dir in sdk_dirs: filtered_paths.append( - posixpath.normpath(posixpath.join(self.target_sdk_dir, dir)) + posixpath.normpath(posixpath.join(self.target_sdk_dir_name, dir)) ) sdk_env = self.env.Clone() sdk_env.Replace(CPPPATH=filtered_paths) - with open(self.target[0].path, "wt") as f: - cmdline_options = sdk_env.subst( - "$CCFLAGS $_CCCOMCOM", target=Entry("dummy") - ) - f.write(cmdline_options.replace("\\", "/")) - f.write("\n") + meta = SdkMeta(sdk_env) + meta.save_to(self.target[0].path) + + def emitter(self, target, source, env): + target_folder = target[0] + target = [target_folder.File("sdk.opts")] + return target, source def _create_deploy_commands(self): dirs_to_create = set( @@ -81,13 +124,17 @@ class SdkTreeBuilder: actions = [ Delete(self.sdk_deploy_dir), Mkdir(self.sdk_deploy_dir), + Copy( + self.sdk_root_dir, + self.env["SDK_DEFINITION"], + ), ] actions += [Mkdir(d) for d in dirs_to_create] actions += [ - Copy( - self.sdk_deploy_dir.File(h).path, - h, + Action( + Copy(self.sdk_deploy_dir.File(h).path, h), + # f"Copy {h} to {self.sdk_deploy_dir}", ) for h in self.header_depends ] @@ -108,6 +155,11 @@ def deploy_sdk_tree(target, source, env, for_signature): return sdk_tree.generate_actions() +def deploy_sdk_tree_emitter(target, source, env): + sdk_tree = SdkTreeBuilder(env, target, source) + return sdk_tree.emitter(target, source, env) + + def gen_sdk_data(sdk_cache: SdkCache): api_def = [] api_def.extend( @@ -165,6 +217,7 @@ def generate_sdk_symbols(source, target, env): def generate(env, **kw): + env.AddMethod(ProcessSdkDepends) env.Append( BUILDERS={ "SDKPrebuilder": Builder( @@ -183,6 +236,7 @@ def generate(env, **kw): ), "SDKTree": Builder( generator=deploy_sdk_tree, + emitter=deploy_sdk_tree_emitter, src_suffix=".d", ), "SDKSymUpdater": Builder( diff --git a/site_scons/site_tools/fbt_version.py b/scripts/fbt_tools/fbt_version.py similarity index 100% rename from site_scons/site_tools/fbt_version.py rename to scripts/fbt_tools/fbt_version.py diff --git a/site_scons/site_tools/fwbin.py b/scripts/fbt_tools/fwbin.py similarity index 100% rename from site_scons/site_tools/fwbin.py rename to scripts/fbt_tools/fwbin.py diff --git a/site_scons/site_tools/gdb.py b/scripts/fbt_tools/gdb.py similarity index 100% rename from site_scons/site_tools/gdb.py rename to scripts/fbt_tools/gdb.py diff --git a/site_scons/site_tools/jflash.py b/scripts/fbt_tools/jflash.py similarity index 100% rename from site_scons/site_tools/jflash.py rename to scripts/fbt_tools/jflash.py diff --git a/site_scons/site_tools/objdump.py b/scripts/fbt_tools/objdump.py similarity index 100% rename from site_scons/site_tools/objdump.py rename to scripts/fbt_tools/objdump.py diff --git a/site_scons/site_tools/openocd.py b/scripts/fbt_tools/openocd.py similarity index 100% rename from site_scons/site_tools/openocd.py rename to scripts/fbt_tools/openocd.py diff --git a/site_scons/site_tools/python3.py b/scripts/fbt_tools/python3.py similarity index 100% rename from site_scons/site_tools/python3.py rename to scripts/fbt_tools/python3.py diff --git a/site_scons/site_tools/sconsmodular.py b/scripts/fbt_tools/sconsmodular.py similarity index 100% rename from site_scons/site_tools/sconsmodular.py rename to scripts/fbt_tools/sconsmodular.py diff --git a/site_scons/site_tools/sconsrecursiveglob.py b/scripts/fbt_tools/sconsrecursiveglob.py similarity index 100% rename from site_scons/site_tools/sconsrecursiveglob.py rename to scripts/fbt_tools/sconsrecursiveglob.py diff --git a/site_scons/site_tools/strip.py b/scripts/fbt_tools/strip.py similarity index 100% rename from site_scons/site_tools/strip.py rename to scripts/fbt_tools/strip.py diff --git a/scripts/flipper/assets/copro.py b/scripts/flipper/assets/copro.py index 0c78e889..33c3ac23 100644 --- a/scripts/flipper/assets/copro.py +++ b/scripts/flipper/assets/copro.py @@ -1,10 +1,9 @@ import logging -import datetime -import shutil import json -from os.path import basename - +from io import BytesIO +import tarfile import xml.etree.ElementTree as ET + from flipper.utils import * from flipper.assets.coprobin import CoproBinary, get_stack_type @@ -51,20 +50,19 @@ class Copro: raise Exception(f"Unsupported cube version") self.version = cube_version + @staticmethod + def _getFileName(name): + return os.path.join("core2_firmware", name) + def addFile(self, array, filename, **kwargs): source_file = os.path.join(self.mcu_copro, filename) - destination_file = os.path.join(self.output_dir, filename) - shutil.copyfile(source_file, destination_file) - array.append( - {"name": filename, "sha256": file_sha256(destination_file), **kwargs} - ) + self.output_tar.add(source_file, arcname=self._getFileName(filename)) + array.append({"name": filename, "sha256": file_sha256(source_file), **kwargs}) + + def bundle(self, output_file, stack_file_name, stack_type, stack_addr=None): + self.output_tar = tarfile.open(output_file, "w:gz") - def bundle(self, output_dir, stack_file_name, stack_type, stack_addr=None): - if not os.path.isdir(output_dir): - raise Exception(f'"{output_dir}" doesn\'t exists') - self.output_dir = output_dir stack_file = os.path.join(self.mcu_copro, stack_file_name) - manifest_file = os.path.join(self.output_dir, "Manifest.json") # Form Manifest manifest = dict(MANIFEST_TEMPLATE) manifest["manifest"]["timestamp"] = timestamp() @@ -105,6 +103,10 @@ class Copro: stack_file_name, address=f"0x{stack_addr:X}", ) - # Save manifest to - with open(manifest_file, "w", newline="\n") as file: - json.dump(manifest, file) + + # Save manifest + manifest_data = json.dumps(manifest, indent=4).encode("utf-8") + info = tarfile.TarInfo(self._getFileName("Manifest.json")) + info.size = len(manifest_data) + self.output_tar.addfile(info, BytesIO(manifest_data)) + self.output_tar.close() diff --git a/scripts/guruguru.py b/scripts/guruguru.py index 3560bcd9..c227e17e 100755 --- a/scripts/guruguru.py +++ b/scripts/guruguru.py @@ -17,7 +17,7 @@ class Main(App): async def rebuild(self, line): self.clearConsole() self.logger.info(f"Triggered by: {line}") - proc = await asyncio.create_subprocess_exec("make") + proc = await asyncio.create_subprocess_exec("./fbt") await proc.wait() await asyncio.sleep(1) self.is_building = False diff --git a/scripts/sconsdist.py b/scripts/sconsdist.py index 1e95ee2f..45ae03e3 100644 --- a/scripts/sconsdist.py +++ b/scripts/sconsdist.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 from flipper.app import App -from os.path import join, exists -from os import makedirs +from os.path import join, exists, relpath +from os import makedirs, walk from update import Main as UpdateMain import shutil +import zipfile +import tarfile class ProjectDir: @@ -17,6 +19,8 @@ class ProjectDir: class Main(App): + DIST_FILE_PREFIX = "flipper-z-" + def init(self): self.subparsers = self.parser.add_subparsers(help="sub-command help") @@ -45,9 +49,13 @@ class Main(App): def get_project_filename(self, project, filetype): # Temporary fix project_name = project.project - if project_name == "firmware" and filetype != "elf": - project_name = "full" - return f"flipper-z-{self.target}-{project_name}-{self.args.suffix}.{filetype}" + if project_name == "firmware": + if filetype == "zip": + project_name = "sdk" + elif filetype != "elf": + project_name = "full" + + return f"{self.DIST_FILE_PREFIX}{self.target}-{project_name}-{self.args.suffix}.{filetype}" def get_dist_filepath(self, filename): return join(self.output_dir_path, filename) @@ -56,10 +64,28 @@ class Main(App): obj_directory = join("build", project.dir) for filetype in ("elf", "bin", "dfu", "json"): - shutil.copyfile( - join(obj_directory, f"{project.project}.{filetype}"), - self.get_dist_filepath(self.get_project_filename(project, filetype)), - ) + if exists(src_file := join(obj_directory, f"{project.project}.{filetype}")): + shutil.copyfile( + src_file, + self.get_dist_filepath( + self.get_project_filename(project, filetype) + ), + ) + if exists(sdk_folder := join(obj_directory, "sdk")): + with zipfile.ZipFile( + self.get_dist_filepath(self.get_project_filename(project, "zip")), + "w", + zipfile.ZIP_DEFLATED, + ) as zf: + for root, dirs, files in walk(sdk_folder): + for file in files: + zf.write( + join(root, file), + relpath( + join(root, file), + sdk_folder, + ), + ) def copy(self): self.projects = dict( @@ -103,9 +129,8 @@ class Main(App): ) if self.args.version: - bundle_dir = join( - self.output_dir_path, f"{self.target}-update-{self.args.suffix}" - ) + bundle_dir_name = f"{self.target}-update-{self.args.suffix}" + bundle_dir = join(self.output_dir_path, bundle_dir_name) bundle_args = [ "generate", "-d", @@ -131,10 +156,24 @@ class Main(App): ) ) bundle_args.extend(self.other_args) - self.logger.info( - f"Use this directory to self-update your Flipper:\n\t{bundle_dir}" - ) - return UpdateMain(no_exit=True)(bundle_args) + + if (bundle_result := UpdateMain(no_exit=True)(bundle_args)) == 0: + self.logger.info( + f"Use this directory to self-update your Flipper:\n\t{bundle_dir}" + ) + + # Create tgz archive + with tarfile.open( + join( + self.output_dir_path, + f"{self.DIST_FILE_PREFIX}{bundle_dir_name}.tgz", + ), + "w:gz", + compresslevel=9, + ) as tar: + tar.add(bundle_dir, arcname=bundle_dir_name) + + return bundle_result return 0 diff --git a/site_scons/commandline.scons b/site_scons/commandline.scons index 765af08f..324dfe33 100644 --- a/site_scons/commandline.scons +++ b/site_scons/commandline.scons @@ -81,46 +81,41 @@ vars.AddVariables( help="Enable debug tools to be built", default=False, ), -) - -vars.Add( - "DIST_SUFFIX", - help="Suffix for binaries in build output for dist targets", - default="local", -) - -vars.Add( - "UPDATE_VERSION_STRING", - help="Version string for updater package", - default="${DIST_SUFFIX}", -) - - -vars.Add( - "COPRO_CUBE_VERSION", - help="Cube version", - default="", -) - -vars.Add( - "COPRO_STACK_ADDR", - help="Core2 Firmware address", - default="0", -) - -vars.Add( - "COPRO_STACK_BIN", - help="Core2 Firmware file name", - default="", -) - -vars.Add( - "COPRO_DISCLAIMER", - help="Value to pass to bundling script to confirm dangerous operations", - default="", -) - -vars.AddVariables( + BoolVariable( + "FAP_EXAMPLES", + help="Enable example applications to be built", + default=False, + ), + ( + "DIST_SUFFIX", + "Suffix for binaries in build output for dist targets", + "local", + ), + ( + "UPDATE_VERSION_STRING", + "Version string for updater package", + "${DIST_SUFFIX}", + ), + ( + "COPRO_CUBE_VERSION", + "Cube version", + "", + ), + ( + "COPRO_STACK_ADDR", + "Core2 Firmware address", + "0", + ), + ( + "COPRO_STACK_BIN", + "Core2 Firmware file name", + "", + ), + ( + "COPRO_DISCLAIMER", + "Value to pass to bundling script to confirm dangerous operations", + "", + ), PathVariable( "COPRO_OB_DATA", help="Path to OB reference data", @@ -161,86 +156,75 @@ vars.AddVariables( validator=PathVariable.PathAccept, default="", ), -) - -vars.Add( - "FBT_TOOLCHAIN_VERSIONS", - help="Whitelisted toolchain versions (leave empty for no check)", - default=tuple(), -) - -vars.Add( - "OPENOCD_OPTS", - help="Options to pass to OpenOCD", - default="", -) - -vars.Add( - "BLACKMAGIC", - help="Blackmagic probe location", - default="auto", -) - -vars.Add( - "UPDATE_SPLASH", - help="Directory name with slideshow frames to render after installing update package", - default="update_default", -) - -vars.Add( - "LOADER_AUTOSTART", - help="Application name to automatically run on Flipper boot", - default="", -) - - -vars.Add( - "FIRMWARE_APPS", - help="Map of (configuration_name->application_list)", - default={ - "default": ( - # Svc - "basic_services", - # Apps - "main_apps", - "system_apps", - # Settings - "settings_apps", - # Plugins - # "basic_plugins", - # Debug - # "debug_apps", - ) - }, -) - -vars.Add( - "FIRMWARE_APP_SET", - help="Application set to use from FIRMWARE_APPS", - default="default", -) - -vars.Add( - "APPSRC", - help="Application source directory for app to build & upload", - default="", -) - -# List of tuples (directory, add_to_global_include_path) -vars.Add( - "APPDIRS", - help="Directories to search for firmware components & external apps", - default=[ - ("applications", False), - ("applications/services", True), - ("applications/main", True), - ("applications/settings", False), - ("applications/system", False), - ("applications/debug", False), - ("applications/plugins", False), - ("applications/examples", False), - ("applications_user", False), - ], + ( + "FBT_TOOLCHAIN_VERSIONS", + "Whitelisted toolchain versions (leave empty for no check)", + tuple(), + ), + ( + "OPENOCD_OPTS", + "Options to pass to OpenOCD", + "", + ), + ( + "BLACKMAGIC", + "Blackmagic probe location", + "auto", + ), + ( + "UPDATE_SPLASH", + "Directory name with slideshow frames to render after installing update package", + "update_default", + ), + ( + "LOADER_AUTOSTART", + "Application name to automatically run on Flipper boot", + "", + ), + ( + "FIRMWARE_APPS", + "Map of (configuration_name->application_list)", + { + "default": ( + # Svc + "basic_services", + # Apps + "main_apps", + "system_apps", + # Settings + "settings_apps", + # Plugins + # "basic_plugins", + # Debug + # "debug_apps", + ) + }, + ), + ( + "FIRMWARE_APP_SET", + "Application set to use from FIRMWARE_APPS", + "default", + ), + ( + "APPSRC", + "Application source directory for app to build & upload", + "", + ), + # List of tuples (directory, add_to_global_include_path) + ( + "APPDIRS", + "Directories to search for firmware components & external apps", + [ + ("applications", False), + ("applications/services", True), + ("applications/main", True), + ("applications/settings", False), + ("applications/system", False), + ("applications/debug", False), + ("applications/plugins", False), + ("applications_user", False), + ], + ), ) Return("vars") diff --git a/site_scons/environ.scons b/site_scons/environ.scons index 3e0c6bea..2d082838 100644 --- a/site_scons/environ.scons +++ b/site_scons/environ.scons @@ -1,6 +1,5 @@ -import SCons from SCons.Platform import TempFileMunge -from fbt import util +from fbt.util import tempfile_arg_esc_func, single_quote, wrap_tempfile import os import multiprocessing @@ -13,14 +12,18 @@ forward_os_env = { } # Proxying CI environment to child processes & scripts variables_to_forward = [ + # CI/CD variables "WORKFLOW_BRANCH_OR_TAG", "DIST_SUFFIX", + # Python & other tools "HOME", "APPDATA", "PYTHONHOME", "PYTHONNOUSERSITE", "TMP", "TEMP", + # Colors for tools + "TERM", ] if proxy_env := GetOption("proxy_env"): variables_to_forward.extend(proxy_env.split(",")) @@ -79,7 +82,7 @@ if not coreenv["VERBOSE"]: SetOption("num_jobs", multiprocessing.cpu_count()) # Avoiding re-scan of all sources on every startup SetOption("implicit_cache", True) -SetOption("implicit_deps_unchanged", True) +# SetOption("implicit_deps_unchanged", True) # More aggressive caching SetOption("max_drift", 1) # Random task queue - to discover isses with build logic faster @@ -87,10 +90,10 @@ SetOption("max_drift", 1) # Setting up temp file parameters - to overcome command line length limits -coreenv["TEMPFILEARGESCFUNC"] = util.tempfile_arg_esc_func -util.wrap_tempfile(coreenv, "LINKCOM") -util.wrap_tempfile(coreenv, "ARCOM") +coreenv["TEMPFILEARGESCFUNC"] = tempfile_arg_esc_func +wrap_tempfile(coreenv, "LINKCOM") +wrap_tempfile(coreenv, "ARCOM") -coreenv["SINGLEQUOTEFUNC"] = util.single_quote +coreenv["SINGLEQUOTEFUNC"] = single_quote Return("coreenv") diff --git a/site_scons/extapps.scons b/site_scons/extapps.scons index c976fbbe..9edc6b5c 100644 --- a/site_scons/extapps.scons +++ b/site_scons/extapps.scons @@ -21,7 +21,7 @@ appenv = ENV.Clone( ) appenv.Replace( - LINKER_SCRIPT="application-ext", + LINKER_SCRIPT="application_ext", ) appenv.AppendUnique( @@ -106,6 +106,7 @@ appenv.PhonyTarget("firmware_extapps", appenv.Action(legacy_app_build_stub, None Alias("faps", extapps["compact"].values()) +Alias("faps", extapps["validators"].values()) if appsrc := appenv.subst("$APPSRC"): app_manifest, fap_file, app_validator = appenv.GetExtAppFromPath(appsrc) diff --git a/site_scons/fbt_extra/util.py b/site_scons/fbt_extra/util.py new file mode 100644 index 00000000..aa3d50b6 --- /dev/null +++ b/site_scons/fbt_extra/util.py @@ -0,0 +1,23 @@ +from fbt.util import link_dir + + +def link_elf_dir_as_latest(env, elf_node): + elf_dir = elf_node.Dir(".") + latest_dir = env.Dir("#build/latest") + print(f"Setting {elf_dir} as latest built dir (./build/latest/)") + return link_dir(latest_dir.abspath, elf_dir.abspath, env["PLATFORM"] == "win32") + + +def should_gen_cdb_and_link_dir(env, requested_targets): + explicitly_building_updater = False + # Hacky way to check if updater-related targets were requested + for build_target in requested_targets: + if "updater" in str(build_target): + explicitly_building_updater = True + + is_updater = not env["IS_BASE_FIRMWARE"] + # If updater is explicitly requested, link to the latest updater + # Otherwise, link to firmware + return (is_updater and explicitly_building_updater) or ( + not is_updater and not explicitly_building_updater + )