From 2d6c2886ae429b634736fe9da6f48caef155d2e3 Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 7 Nov 2022 18:54:41 +0400 Subject: [PATCH] fbt: compile_db fixes (#1981) * fbt: forked compilation_db tool * fbt: fixes for static analysis * pvs-studio: ignoring more generic warnings * fbt: util: added extract_abs_dir * vscode: added fap-set-debug-elf-root for debug configurations --- .github/workflows/pvs_studio.yml | 2 +- .gitignore | 2 + .pvsconfig | 1 + .vscode/example/launch.json | 4 + SConstruct | 8 +- firmware.scons | 5 +- scripts/fbt/util.py | 12 +- scripts/fbt_tools/compilation_db.py | 278 ++++++++++++++++++++++++++++ scripts/fbt_tools/fbt_extapps.py | 3 +- 9 files changed, 307 insertions(+), 8 deletions(-) create mode 100644 scripts/fbt_tools/compilation_db.py diff --git a/.github/workflows/pvs_studio.yml b/.github/workflows/pvs_studio.yml index f28fad20..9de493a4 100644 --- a/.github/workflows/pvs_studio.yml +++ b/.github/workflows/pvs_studio.yml @@ -57,7 +57,7 @@ jobs: - name: 'Generate compile_comands.json' run: | - FBT_TOOLCHAIN_PATH=/runner/_work ./fbt COMPACT=1 version_json proto_ver icons firmware_cdb dolphin_internal dolphin_blocking + FBT_TOOLCHAIN_PATH=/runner/_work ./fbt COMPACT=1 version_json proto_ver icons firmware_cdb dolphin_internal dolphin_blocking _fap_icons - name: 'Static code analysis' run: | diff --git a/.gitignore b/.gitignore index 38a31bf0..61c594ed 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ openocd.log # PVS Studio temporary files .PVS-Studio/ PVS-Studio.log + +.gdbinit diff --git a/.pvsconfig b/.pvsconfig index d17eaa5a..5f1ffb7c 100644 --- a/.pvsconfig +++ b/.pvsconfig @@ -5,6 +5,7 @@ //-V:BPTREE_DEF2:779,1086,557,773,512 //-V:DICT_DEF2:779,524,776,760,1044,1001,729,590,568,747,685 //-V:ALGO_DEF:1048,747,1044 +//-V:TUPLE_DEF2:524,590,1001,760 # Non-severe malloc/null pointer deref warnings //-V::522:2,3 diff --git a/.vscode/example/launch.json b/.vscode/example/launch.json index c8b0c601..5c46d397 100644 --- a/.vscode/example/launch.json +++ b/.vscode/example/launch.json @@ -38,6 +38,7 @@ "postAttachCommands": [ // "compare-sections", "source debug/flipperapps.py", + "fap-set-debug-elf-root build/latest/.extapps", // "source debug/FreeRTOS/FreeRTOS.py", // "svd_load debug/STM32WB55_CM4.svd" ] @@ -59,6 +60,7 @@ "set confirm off", "set mem inaccessible-by-default off", "source debug/flipperapps.py", + "fap-set-debug-elf-root build/latest/.extapps", // "compare-sections", ] // "showDevDebugOutput": "raw", @@ -76,6 +78,7 @@ "rtos": "FreeRTOS", "postAttachCommands": [ "source debug/flipperapps.py", + "fap-set-debug-elf-root build/latest/.extapps", ] // "showDevDebugOutput": "raw", }, @@ -95,6 +98,7 @@ ], "postAttachCommands": [ "source debug/flipperapps.py", + "fap-set-debug-elf-root build/latest/.extapps", ], // "showDevDebugOutput": "raw", }, diff --git a/SConstruct b/SConstruct index 67eac382..3ee89f64 100644 --- a/SConstruct +++ b/SConstruct @@ -200,7 +200,9 @@ firmware_debug = distenv.PhonyTarget( source=firmware_env["FW_ELF"], GDBOPTS="${GDBOPTS_BASE}", GDBREMOTE="${OPENOCD_GDB_PIPE}", - FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT"), + FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT").replace( + "\\", "/" + ), ) distenv.Depends(firmware_debug, firmware_flash) @@ -210,7 +212,9 @@ distenv.PhonyTarget( source=firmware_env["FW_ELF"], GDBOPTS="${GDBOPTS_BASE} ${GDBOPTS_BLACKMAGIC}", GDBREMOTE="${BLACKMAGIC_ADDR}", - FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT"), + FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT").replace( + "\\", "/" + ), ) # Debug alien elf diff --git a/firmware.scons b/firmware.scons index 6feb73ac..6a923747 100644 --- a/firmware.scons +++ b/firmware.scons @@ -264,7 +264,10 @@ fw_artifacts.extend( fwcdb = fwenv.CompilationDatabase() # without filtering, both updater & firmware commands would be generated in same file -fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*")) +fwenv.Replace( + COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*"), + COMPILATIONDB_SRCPATH_FILTER="*.c*", +) AlwaysBuild(fwcdb) Precious(fwcdb) NoClean(fwcdb) diff --git a/scripts/fbt/util.py b/scripts/fbt/util.py index f5404458..b8e9c592 100644 --- a/scripts/fbt/util.py +++ b/scripts/fbt/util.py @@ -43,12 +43,18 @@ def single_quote(arg_list): return " ".join(f"'{arg}'" if " " in arg else str(arg) for arg in arg_list) -def extract_abs_dir_path(node): +def extract_abs_dir(node): if isinstance(node, SCons.Node.FS.EntryProxy): node = node.get() for repo_dir in node.get_all_rdirs(): if os.path.exists(repo_dir.abspath): - return repo_dir.abspath + return repo_dir - raise StopError(f"Can't find absolute path for {node.name}") + +def extract_abs_dir_path(node): + abs_dir_node = extract_abs_dir(node) + if abs_dir_node is None: + raise StopError(f"Can't find absolute path for {node.name}") + + return abs_dir_node.abspath diff --git a/scripts/fbt_tools/compilation_db.py b/scripts/fbt_tools/compilation_db.py new file mode 100644 index 00000000..17ff6aaa --- /dev/null +++ b/scripts/fbt_tools/compilation_db.py @@ -0,0 +1,278 @@ +""" +Implements the ability for SCons to emit a compilation database for the MongoDB project. See +http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation +database is, and why you might want one. The only user visible entry point here is +'env.CompilationDatabase'. This method takes an optional 'target' to name the file that +should hold the compilation database, otherwise, the file defaults to compile_commands.json, +which is the name that most clang tools search for by default. +""" + +# Copyright 2020 MongoDB Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import json +import itertools +import fnmatch +import SCons + +from SCons.Tool.cxx import CXXSuffixes +from SCons.Tool.cc import CSuffixes +from SCons.Tool.asm import ASSuffixes, ASPPSuffixes + +# TODO: Is there a better way to do this than this global? Right now this exists so that the +# emitter we add can record all of the things it emits, so that the scanner for the top level +# compilation database can access the complete list, and also so that the writer has easy +# access to write all of the files. But it seems clunky. How can the emitter and the scanner +# communicate more gracefully? +__COMPILATION_DB_ENTRIES = [] + + +# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even +# integrate with the cache, but there doesn't seem to be much call for it. +class __CompilationDbNode(SCons.Node.Python.Value): + def __init__(self, value): + SCons.Node.Python.Value.__init__(self, value) + self.Decider(changed_since_last_build_node) + + +def changed_since_last_build_node(child, target, prev_ni, node): + """Dummy decider to force always building""" + return True + + +def make_emit_compilation_DB_entry(comstr): + """ + Effectively this creates a lambda function to capture: + * command line + * source + * target + :param comstr: unevaluated command line + :return: an emitter which has captured the above + """ + user_action = SCons.Action.Action(comstr) + + def emit_compilation_db_entry(target, source, env): + """ + This emitter will be added to each c/c++ object build to capture the info needed + for clang tools + :param target: target node(s) + :param source: source node(s) + :param env: Environment for use building this node + :return: target(s), source(s) + """ + + dbtarget = __CompilationDbNode(source) + + entry = env.__COMPILATIONDB_Entry( + target=dbtarget, + source=[], + __COMPILATIONDB_UOUTPUT=target, + __COMPILATIONDB_USOURCE=source, + __COMPILATIONDB_UACTION=user_action, + __COMPILATIONDB_ENV=env, + ) + + # TODO: Technically, these next two lines should not be required: it should be fine to + # cache the entries. However, they don't seem to update properly. Since they are quick + # to re-generate disable caching and sidestep this problem. + env.AlwaysBuild(entry) + env.NoCache(entry) + + __COMPILATION_DB_ENTRIES.append(dbtarget) + + return target, source + + return emit_compilation_db_entry + + +def compilation_db_entry_action(target, source, env, **kw): + """ + Create a dictionary with evaluated command line, target, source + and store that info as an attribute on the target + (Which has been stored in __COMPILATION_DB_ENTRIES array + :param target: target node(s) + :param source: source node(s) + :param env: Environment for use building this node + :param kw: + :return: None + """ + + command = env["__COMPILATIONDB_UACTION"].strfunction( + target=env["__COMPILATIONDB_UOUTPUT"], + source=env["__COMPILATIONDB_USOURCE"], + env=env["__COMPILATIONDB_ENV"], + ) + + entry = { + "directory": env.Dir("#").abspath, + "command": command, + "file": env["__COMPILATIONDB_USOURCE"][0], + "output": env["__COMPILATIONDB_UOUTPUT"][0], + } + + target[0].write(entry) + + +def write_compilation_db(target, source, env): + entries = [] + + use_abspath = env["COMPILATIONDB_USE_ABSPATH"] in [True, 1, "True", "true"] + use_path_filter = env.subst("$COMPILATIONDB_PATH_FILTER") + use_srcpath_filter = env.subst("$COMPILATIONDB_SRCPATH_FILTER") + + for s in __COMPILATION_DB_ENTRIES: + entry = s.read() + source_file = entry["file"] + output_file = entry["output"] + + if source_file.rfile().srcnode().exists(): + source_file = source_file.rfile().srcnode() + + if use_abspath: + source_file = source_file.abspath + output_file = output_file.abspath + else: + source_file = source_file.path + output_file = output_file.path + + # print("output_file, path_filter", output_file, use_path_filter) + if use_path_filter and not fnmatch.fnmatch(output_file, use_path_filter): + continue + + if use_srcpath_filter and not fnmatch.fnmatch(source_file, use_srcpath_filter): + continue + + path_entry = { + "directory": entry["directory"], + "command": entry["command"], + "file": source_file, + "output": output_file, + } + + entries.append(path_entry) + + with open(target[0].path, "w") as output_file: + json.dump( + entries, output_file, sort_keys=True, indent=4, separators=(",", ": ") + ) + + +def scan_compilation_db(node, env, path): + return __COMPILATION_DB_ENTRIES + + +def compilation_db_emitter(target, source, env): + """fix up the source/targets""" + + # Someone called env.CompilationDatabase('my_targetname.json') + if not target and len(source) == 1: + target = source + + # Default target name is compilation_db.json + if not target: + target = [ + "compile_commands.json", + ] + + # No source should have been passed. Drop it. + if source: + source = [] + + return target, source + + +def generate(env, **kwargs): + static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + + env.SetDefault( + COMPILATIONDB_COMSTR=kwargs.get( + "COMPILATIONDB_COMSTR", "Building compilation database $TARGET" + ), + COMPILATIONDB_USE_ABSPATH=False, + COMPILATIONDB_PATH_FILTER="", + COMPILATIONDB_SRCPATH_FILTER="", + ) + + components_by_suffix = itertools.chain( + itertools.product( + CSuffixes, + [ + (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"), + (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"), + ], + ), + itertools.product( + CXXSuffixes, + [ + (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"), + (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"), + ], + ), + itertools.product( + ASSuffixes, + [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")], + [(shared_obj, SCons.Defaults.SharedObjectEmitter, "$ASCOM")], + ), + itertools.product( + ASPPSuffixes, + [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASPPCOM")], + [(shared_obj, SCons.Defaults.SharedObjectEmitter, "$ASPPCOM")], + ), + ) + + for entry in components_by_suffix: + suffix = entry[0] + builder, base_emitter, command = entry[1] + + # Assumes a dictionary emitter + emitter = builder.emitter.get(suffix, False) + if emitter: + # We may not have tools installed which initialize all or any of + # cxx, cc, or assembly. If not skip resetting the respective emitter. + builder.emitter[suffix] = SCons.Builder.ListEmitter( + [ + emitter, + make_emit_compilation_DB_entry(command), + ] + ) + + env.Append( + BUILDERS={ + "__COMPILATIONDB_Entry": SCons.Builder.Builder( + action=SCons.Action.Action(compilation_db_entry_action, None), + ), + "CompilationDatabase": SCons.Builder.Builder( + action=SCons.Action.Action( + write_compilation_db, "$COMPILATIONDB_COMSTR" + ), + target_scanner=SCons.Scanner.Scanner( + function=scan_compilation_db, node_class=None + ), + emitter=compilation_db_emitter, + suffix="json", + ), + } + ) + + +def exists(env): + return True diff --git a/scripts/fbt_tools/fbt_extapps.py b/scripts/fbt_tools/fbt_extapps.py index f1906191..a4116e51 100644 --- a/scripts/fbt_tools/fbt_extapps.py +++ b/scripts/fbt_tools/fbt_extapps.py @@ -57,11 +57,12 @@ def BuildAppElf(env, app): ) if app.fap_icon_assets: - app_env.CompileIcons( + fap_icons = app_env.CompileIcons( app_env.Dir(app_work_dir), app._appdir.Dir(app.fap_icon_assets), icon_bundle_name=f"{app.appid}_icons", ) + app_env.Alias("_fap_icons", fap_icons) private_libs = []