[FL-2052] New build system based on scons (#1269)

site_scons/site_tools/ccache.py

@@ -0,0 +1,14 @@
+def exists():
+    return True
+
+
+def generate(env):
+    if ccache := env.WhereIs("ccache"):
+        env["CCACHE"] = "ccache"
+        env["CC_NOCACHE"] = env["CC"]
+        env["CC"] = "$CCACHE $CC_NOCACHE"
+        # Tricky place: linking is done with CXX
+        # Using ccache breaks it
+        env["LINK"] = env["CXX"]
+        env["CXX_NOCACHE"] = env["CXX"]
+        env["CXX"] = "$CCACHE $CXX_NOCACHE"

site_scons/site_tools/crosscc.py

@@ -0,0 +1,72 @@
+from SCons.Tool import asm
+from SCons.Tool import gcc
+from SCons.Tool import gxx
+from SCons.Tool import ar
+from SCons.Tool import gnulink
+import strip
+import gdb
+import objdump
+
+from SCons.Action import _subproc
+import subprocess
+
+
+def prefix_commands(env, command_prefix, cmd_list):
+    for command in cmd_list:
+        if command in env:
+            env[command] = command_prefix + env[command]
+
+
+def _get_tool_version(env, tool):
+    verstr = "version unknown"
+    proc = _subproc(
+        env,
+        env.subst("${%s} --version" % tool),
+        stdout=subprocess.PIPE,
+        stderr="devnull",
+        stdin="devnull",
+        universal_newlines=True,
+        error="raise",
+        shell=True,
+    )
+    if proc:
+        verstr = proc.stdout.readline()
+        proc.communicate()
+    return verstr
+
+
+def generate(env, **kw):
+    for orig_tool in (asm, gcc, gxx, ar, gnulink, strip, gdb, objdump):
+        orig_tool.generate(env)
+    env.SetDefault(
+        TOOLCHAIN_PREFIX=kw.get("toolchain_prefix"),
+    )
+    prefix_commands(
+        env,
+        env.subst("$TOOLCHAIN_PREFIX"),
+        [
+            "AR",
+            "AS",
+            "CC",
+            "CXX",
+            "OBJCOPY",
+            "RANLIB",
+            "STRIP",
+            "GDB",
+            "GDBPY",
+            "OBJDUMP",
+        ],
+    )
+    # Call CC to check version
+    if whitelisted_versions := kw.get("versions", ()):
+        cc_version = _get_tool_version(env, "CC")
+        # print("CC version =", cc_version)
+        # print(list(filter(lambda v: v in cc_version, whitelisted_versions)))
+        if not any(filter(lambda v: v in cc_version, whitelisted_versions)):
+            raise Exception(
+                f"Toolchain version is not supported. Allowed: {whitelisted_versions}, toolchain: {cc_version} "
+            )
+
+
+def exists(env):
+    return True

site_scons/site_tools/fbt_apps.py

@@ -0,0 +1,61 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+
+import SCons
+from fbt.appmanifest import FlipperAppType, AppManager, ApplicationsCGenerator
+
+# Adding objects for application management to env
+#  AppManager env["APPMGR"] - loads all manifests; manages list of known apps
+#  AppBuildset env["APPBUILD"] - contains subset of apps, filtered for current config
+
+
+def LoadApplicationManifests(env):
+    appmgr = env["APPMGR"] = AppManager()
+    for entry in env.Glob("#/applications/*"):
+        if isinstance(entry, SCons.Node.FS.Dir) and not str(entry).startswith("."):
+            appmgr.load_manifest(entry.File("application.fam").abspath, entry.name)
+
+
+def PrepareApplicationsBuild(env):
+    env["APPBUILD"] = env["APPMGR"].filter_apps(env["APPS"])
+    env["APPBUILD_DUMP"] = env.Action(
+        DumpApplicationConfig,
+        "\tINFO\t",
+    )
+
+
+def DumpApplicationConfig(target, source, env):
+    print(f"Loaded {len(env['APPMGR'].known_apps)} app definitions.")
+    print("Firmware modules configuration:")
+    for apptype in FlipperAppType:
+        app_sublist = env["APPBUILD"].get_apps_of_type(apptype)
+        if app_sublist:
+            print(
+                f"{apptype.value}:\n\t",
+                ", ".join(app.appid for app in app_sublist),
+            )
+
+
+def build_apps_c(target, source, env):
+    target_file_name = target[0].path
+
+    gen = ApplicationsCGenerator(env["APPBUILD"])
+    with open(target_file_name, "w") as file:
+        file.write(gen.generate())
+
+
+def generate(env):
+    env.AddMethod(LoadApplicationManifests)
+    env.AddMethod(PrepareApplicationsBuild)
+
+    env.Append(
+        BUILDERS={
+            "ApplicationsC": Builder(
+                action=Action(build_apps_c, "${APPSCOMSTR}"),
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/fbt_assets.py

@@ -0,0 +1,151 @@
+import SCons
+
+from SCons.Builder import Builder
+from SCons.Action import Action
+from SCons.Node.FS import File
+
+import os
+import subprocess
+
+
+def icons_emitter(target, source, env):
+    target = [
+        "compiled/assets_icons.c",
+        "compiled/assets_icons.h",
+    ]
+    return target, source
+
+
+def proto_emitter(target, source, env):
+    out_path = target[0].path
+    target = []
+    for src in source:
+        basename = os.path.splitext(src.name)[0]
+        target.append(env.File(f"compiled/{basename}.pb.c"))
+        target.append(env.File(f"compiled/{basename}.pb.h"))
+    return target, source
+
+
+def dolphin_emitter(target, source, env):
+    res_root_dir = source[0].Dir(env["DOLPHIN_RES_TYPE"])
+    source = [res_root_dir]
+    source.extend(
+        env.GlobRecursive("*.*", res_root_dir),
+    )
+
+    target_base_dir = target[0]
+    env.Replace(_DOLPHIN_OUT_DIR=target[0])
+
+    if env["DOLPHIN_RES_TYPE"] == "external":
+        target = []
+        target.extend(
+            map(
+                lambda node: target_base_dir.File(
+                    res_root_dir.rel_path(node).replace(".png", ".bm")
+                ),
+                filter(lambda node: isinstance(node, SCons.Node.FS.File), source),
+            )
+        )
+    else:
+        asset_basename = f"assets_dolphin_{env['DOLPHIN_RES_TYPE']}"
+        target = [
+            target_base_dir.File(asset_basename + ".c"),
+            target_base_dir.File(asset_basename + ".h"),
+        ]
+
+    return target, source
+
+
+def _invoke_git(args, source_dir):
+    cmd = ["git"]
+    cmd.extend(args)
+    return (
+        subprocess.check_output(cmd, cwd=source_dir, stderr=subprocess.STDOUT)
+        .strip()
+        .decode()
+    )
+
+
+def proto_ver_generator(target, source, env):
+    target_file = target[0]
+    src_dir = source[0].dir.abspath
+    try:
+        git_fetch = _invoke_git(
+            ["fetch", "--tags"],
+            source_dir=src_dir,
+        )
+    except (subprocess.CalledProcessError, EnvironmentError) as e:
+        # Not great, not terrible
+        print("Git: fetch failed")
+
+    try:
+        git_describe = _invoke_git(
+            ["describe", "--tags", "--abbrev=0"],
+            source_dir=src_dir,
+        )
+    except (subprocess.CalledProcessError, EnvironmentError) as e:
+        print("Git: describe failed")
+        Exit("git error")
+
+    # print("describe=", git_describe)
+    git_major, git_minor = git_describe.split(".")
+    version_file_data = (
+        "#pragma once",
+        f"#define PROTOBUF_MAJOR_VERSION {git_major}",
+        f"#define PROTOBUF_MINOR_VERSION {git_minor}",
+        "",
+    )
+    with open(str(target_file), "wt") as file:
+        file.write("\n".join(version_file_data))
+
+
+def generate(env):
+    env.SetDefault(
+        ASSETS_COMPILER="${ROOT_DIR.abspath}/scripts/assets.py",
+        NANOPB_COMPILER="${ROOT_DIR.abspath}/lib/nanopb/generator/nanopb_generator.py",
+    )
+
+    env.Append(
+        BUILDERS={
+            "IconBuilder": Builder(
+                action=Action(
+                    "${PYTHON3} ${ASSETS_COMPILER} icons ${SOURCE.posix} ${TARGET.dir.posix}",
+                    "${ICONSCOMSTR}",
+                ),
+                emitter=icons_emitter,
+            ),
+            "ProtoBuilder": Builder(
+                action=Action(
+                    "${PYTHON3} ${NANOPB_COMPILER} -q -I${SOURCE.dir.posix} -D${TARGET.dir.posix} ${SOURCES.posix}",
+                    "${PROTOCOMSTR}",
+                ),
+                emitter=proto_emitter,
+                suffix=".pb.c",
+                src_suffix=".proto",
+            ),
+            "DolphinSymBuilder": Builder(
+                action=Action(
+                    '${PYTHON3} ${ASSETS_COMPILER} dolphin -s dolphin_${DOLPHIN_RES_TYPE} "${SOURCE}" "${_DOLPHIN_OUT_DIR}"',
+                    "${DOLPHINCOMSTR}",
+                ),
+                emitter=dolphin_emitter,
+            ),
+            "DolphinExtBuilder": Builder(
+                action=Action(
+                    '${PYTHON3} ${ASSETS_COMPILER} dolphin "${SOURCE}" "${_DOLPHIN_OUT_DIR}"',
+                    "${DOLPHINCOMSTR}",
+                ),
+                emitter=dolphin_emitter,
+            ),
+            "ProtoVerBuilder": Builder(
+                action=Action(
+                    proto_ver_generator,
+                    "${PBVERCOMSTR}",
+                ),
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/fbt_dist.py

@@ -0,0 +1,104 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+from SCons.Script import Mkdir
+
+
+def get_variant_dirname(env, project=None):
+    parts = [f"f{env['TARGET_HW']}"]
+    if project:
+        parts.append(project)
+
+    suffix = ""
+    if env["DEBUG"]:
+        suffix += "D"
+    if env["COMPACT"]:
+        suffix += "C"
+    if suffix:
+        parts.append(suffix)
+
+    return "-".join(parts)
+
+
+def create_fw_build_targets(env, configuration_name):
+    flavor = get_variant_dirname(env, configuration_name)
+    build_dir = env.Dir("build").Dir(flavor).abspath
+    return env.SConscript(
+        "firmware.scons",
+        variant_dir=build_dir,
+        duplicate=0,
+        exports={
+            "ENV": env,
+            "fw_build_meta": {
+                "type": configuration_name,
+                "flavor": flavor,
+                "build_dir": build_dir,
+            },
+        },
+    )
+
+
+def AddFwProject(env, base_env, fw_type, fw_env_key):
+    project_env = env[fw_env_key] = create_fw_build_targets(base_env, fw_type)
+    env.Append(
+        DIST_PROJECTS=[
+            project_env["FW_FLAVOR"],
+        ],
+        DIST_DEPENDS=[
+            project_env["FW_ARTIFACTS"],
+        ],
+    )
+    return project_env
+
+
+def AddDebugTarget(env, targetenv, force_flash=True):
+    pseudo_name = f"debug.{targetenv.subst('$FIRMWARE_BUILD_CFG')}.pseudo"
+    debug_target = env.GDBPy(
+        pseudo_name,
+        targetenv["FW_ELF"],
+        GDBPYOPTS='-ex "source debug/FreeRTOS/FreeRTOS.py" '
+        '-ex "source debug/PyCortexMDebug/PyCortexMDebug.py" '
+        '-ex "svd_load ${SVD_FILE}" '
+        '-ex "compare-sections"',
+    )
+    if force_flash:
+        env.Depends(debug_target, targetenv["FW_FLASH"])
+    env.Pseudo(pseudo_name)
+    env.AlwaysBuild(debug_target)
+    return debug_target
+
+
+def generate(env):
+    env.AddMethod(AddFwProject)
+    env.AddMethod(AddDebugTarget)
+    env.SetDefault(
+        COPRO_MCU_FAMILY="STM32WB5x",
+    )
+    env.Append(
+        BUILDERS={
+            "DistBuilder": Builder(
+                action=Action(
+                    '${PYTHON3} ${ROOT_DIR.abspath}/scripts/sconsdist.py copy -p ${DIST_PROJECTS} -s "${DIST_SUFFIX}" ${DIST_EXTRA}',
+                    "${DISTCOMSTR}",
+                ),
+            ),
+            "CoproBuilder": Builder(
+                action=Action(
+                    [
+                        Mkdir("$TARGET"),
+                        "${PYTHON3} ${ROOT_DIR.abspath}/scripts/assets.py "
+                        "copro ${COPRO_CUBE_DIR} "
+                        "${TARGET} ${COPRO_MCU_FAMILY} "
+                        "--cube_ver=${COPRO_CUBE_VERSION} "
+                        "--stack_type=${COPRO_STACK_TYPE} "
+                        '--stack_file="${COPRO_STACK_BIN}" '
+                        "--stack_addr=${COPRO_STACK_ADDR} ",
+                    ],
+                    "",
+                )
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/fbt_extapps.py

@@ -0,0 +1,30 @@
+import os
+
+
+def BuildAppElf(env, app):
+    work_dir = env.subst("$EXT_APPS_WORK_DIR")
+    app_target_name = os.path.join(work_dir, app.appid)
+    app_alias = f"{env['FIRMWARE_BUILD_CFG']}_{app.appid}"
+    app_elf = env.Program(
+        app_target_name,
+        env.GlobRecursive("*.c*", os.path.join(work_dir, app._appdir)),
+        APP_ENTRY=app.entry_point,
+    )
+    app_elf_dump = env.ObjDump(app_target_name)
+    env.Alias(f"{app_alias}_list", app_elf_dump)
+
+    app_stripped_elf = env.ELFStripper(
+        os.path.join(env.subst("$PLUGIN_ELF_DIR"), app.appid), app_elf
+    )
+    env.Alias(app_alias, app_stripped_elf)
+    return app_stripped_elf
+
+
+def generate(env, **kw):
+    env.SetDefault(EXT_APPS_WORK_DIR=kw.get("EXT_APPS_WORK_DIR", ".extapps"))
+    env.VariantDir(env.subst("$EXT_APPS_WORK_DIR"), ".", duplicate=False)
+    env.AddMethod(BuildAppElf)
+
+
+def exists(env):
+    return True

site_scons/site_tools/fbt_version.py

@@ -0,0 +1,29 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+
+
+def version_emitter(target, source, env):
+    target_dir = target[0]
+    target = [
+        target_dir.File("version.inc.h"),
+        target_dir.File("version.json"),
+    ]
+    return target, source
+
+
+def generate(env):
+    env.Append(
+        BUILDERS={
+            "VersionBuilder": Builder(
+                action=Action(
+                    "${PYTHON3} ${ROOT_DIR.abspath}/scripts/version.py generate -t ${TARGET_HW} -o ${TARGET.dir.posix} --dir ${ROOT_DIR}",
+                    "${VERSIONCOMSTR}",
+                ),
+                emitter=version_emitter,
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/fwbin.py

@@ -0,0 +1,52 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+import SCons
+
+__OBJCOPY_ARM_BIN = "arm-none-eabi-objcopy"
+
+
+def generate(env):
+    env.SetDefault(
+        BIN2DFU="${ROOT_DIR.abspath}/scripts/bin2dfu.py",
+        OBJCOPY=__OBJCOPY_ARM_BIN,  # FIXME
+    )
+    env.Append(
+        BUILDERS={
+            "HEXBuilder": Builder(
+                action=Action(
+                    '${OBJCOPY} -O ihex "${SOURCE}" "${TARGET}"',
+                    "${HEXCOMSTR}",
+                ),
+                suffix=".hex",
+                src_suffix=".elf",
+            ),
+            "BINBuilder": Builder(
+                action=Action(
+                    '${OBJCOPY} -O binary -S "${SOURCE}" "${TARGET}"',
+                    "${BINCOMSTR}",
+                ),
+                suffix=".bin",
+                src_suffix=".elf",
+            ),
+            "DFUBuilder": Builder(
+                action=Action(
+                    '${PYTHON3} ${BIN2DFU} -i "${SOURCE}" -o "${TARGET}" -a ${IMAGE_BASE_ADDRESS} -l "Flipper Zero F${TARGET_HW}"',
+                    "${DFUCOMSTR}",
+                ),
+                suffix=".dfu",
+                src_suffix=".bin",
+            ),
+        }
+    )
+
+
+def exists(env):
+    try:
+        return env["OBJCOPY"]
+    except KeyError:
+        pass
+
+    if objcopy := env.WhereIs(__OBJCOPY_ARM_BIN):
+        return objcopy
+
+    raise SCons.Errors.StopError("Could not detect objcopy for arm")

site_scons/site_tools/gdb.py

@@ -0,0 +1,33 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+
+
+def generate(env):
+    env.SetDefault(
+        GDB="gdb",
+        GDBPY="gdb-py",
+        GDBOPTS="",
+        GDBPYOPTS="",
+        GDBCOM="$GDB $GDBOPTS $SOURCES",  # no $TARGET
+        GDBPYCOM="$GDBPY $GDBOPTS $GDBPYOPTS $SOURCES",  # no $TARGET
+    )
+    env.Append(
+        BUILDERS={
+            "GDB": Builder(
+                action=Action(
+                    "${GDBCOM}",
+                    "${GDBCOMSTR}",
+                ),
+            ),
+            "GDBPy": Builder(
+                action=Action(
+                    "${GDBPYCOM}",
+                    "${GDBPYCOMSTR}",
+                ),
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/objdump.py

@@ -0,0 +1,26 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+
+
+def generate(env):
+    env.SetDefault(
+        OBJDUMP="objdump",
+        OBJDUMPFLAGS=[],
+        OBJDUMPCOM="$OBJDUMP $OBJDUMPFLAGS -S $SOURCES > $TARGET",
+    )
+    env.Append(
+        BUILDERS={
+            "ObjDump": Builder(
+                action=Action(
+                    "${OBJDUMPCOM}",
+                    "${OBJDUMPCOMSTR}",
+                ),
+                suffix=".lst",
+                src_suffix=".elf",
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/openocd.py

@@ -0,0 +1,48 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+from SCons.Defaults import Touch
+import SCons
+
+__OPENOCD_BIN = "openocd"
+
+_oocd_action = Action(
+    "${OPENOCD} ${OPENOCD_OPTS} ${OPENOCD_COMMAND}",
+    "${OOCDCOMSTR}",
+)
+
+
+def generate(env):
+    env.SetDefault(
+        OPENOCD=__OPENOCD_BIN,
+        OPENOCD_OPTS="",
+        OPENOCD_COMMAND="",
+        OOCDCOMSTR="",
+    )
+
+    env.Append(
+        BUILDERS={
+            "OOCDFlashCommand": Builder(
+                action=[
+                    _oocd_action,
+                    Touch("${TARGET}"),
+                ],
+                suffix=".flash",
+                src_suffix=".bin",
+            ),
+            "OOCDCommand": Builder(
+                action=_oocd_action,
+            ),
+        }
+    )
+
+
+def exists(env):
+    try:
+        return env["OPENOCD"]
+    except KeyError:
+        pass
+
+    if openocd := env.WhereIs(__OPENOCD_BIN):
+        return openocd
+
+    raise SCons.Errors.StopError("Could not detect openocd")

site_scons/site_tools/python3.py

@@ -0,0 +1,13 @@
+def generate(env):
+    py_name = "python3"
+    if env["PLATFORM"] == "win32":
+        # On Windows, Python 3 executable is usually just "python"
+        py_name = "python"
+
+    env.SetDefault(
+        PYTHON3=py_name,
+    )
+
+
+def exists(env):
+    return True

site_scons/site_tools/sconsmodular.py

@@ -0,0 +1,38 @@
+import posixpath
+import os
+
+
+def BuildModule(env, module):
+    src_dir = str(env.Dir(".").srcdir or os.getcwd())
+    module_sconscript = posixpath.join(src_dir, module, "SConscript")
+    if not os.path.exists(module_sconscript):
+        module_sconscript = posixpath.join(src_dir, f"{module}.scons")
+        if not os.path.exists(module_sconscript):
+            print(f"Cannot build module {module}: scons file not found")
+            Exit(2)
+
+    return env.SConscript(
+        module_sconscript,
+        variant_dir=posixpath.join(env.subst("$BUILD_DIR"), module),
+        duplicate=0,
+    )
+
+
+def BuildModules(env, modules):
+    result = []
+    for module in modules:
+        build_res = env.BuildModule(module)
+        # print("module ", module, build_res)
+        if build_res is None:
+            continue
+        result.append(build_res)
+    return result
+
+
+def generate(env):
+    env.AddMethod(BuildModule)
+    env.AddMethod(BuildModules)
+
+
+def exists(env):
+    return True

site_scons/site_tools/sconsrecursiveglob.py

@@ -0,0 +1,25 @@
+import SCons
+
+
+def GlobRecursive(env, pattern, node=".", exclude=None):
+    results = []
+    if isinstance(node, str):
+        node = env.Dir(node)
+    for f in node.glob("*", source=True, exclude=exclude):
+        if isinstance(f, SCons.Node.FS.Dir):
+            results += env.GlobRecursive(pattern, f, exclude)
+    results += node.glob(
+        pattern,
+        source=True,
+        exclude=exclude,
+    )
+    # print(f"Glob for {pattern} from {node}: {results}")
+    return results
+
+
+def generate(env):
+    env.AddMethod(GlobRecursive)
+
+
+def exists(env):
+    return True

site_scons/site_tools/strip.py

@@ -0,0 +1,26 @@
+from SCons.Builder import Builder
+from SCons.Action import Action
+
+
+def generate(env):
+    env.SetDefault(
+        STRIP="strip",
+        STRIPFLAGS=[],
+        STRIPCOM="$STRIP $STRIPFLAGS $SOURCES -o $TARGET",
+    )
+    env.Append(
+        BUILDERS={
+            "ELFStripper": Builder(
+                action=Action(
+                    "${STRIPCOM}",
+                    "${STRIPCOMSTR}",
+                ),
+                suffix=".elf",
+                src_suffix=".elf",
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True