[FL-2052] New build system based on scons (#1269)

This commit is contained in:
hedger
2022-06-26 15:00:03 +03:00
committed by GitHub
parent c79fb61909
commit f3b1475ede
179 changed files with 3986 additions and 5196 deletions

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3
from flipper.app import App
from flipper.assets.icon import file2image
import logging
import argparse
@@ -90,31 +91,8 @@ class Main(App):
self.parser_dolphin.set_defaults(func=self.dolphin)
def _icon2header(self, file):
output = subprocess.check_output(["convert", file, "xbm:-"])
assert output
f = io.StringIO(output.decode().strip())
width = int(f.readline().strip().split(" ")[2])
height = int(f.readline().strip().split(" ")[2])
data = f.read().strip().replace("\n", "").replace(" ", "").split("=")[1][:-1]
data_bin_str = data[1:-1].replace(",", " ").replace("0x", "")
data_bin = bytearray.fromhex(data_bin_str)
# Encode icon data with LZSS
data_encoded_str = subprocess.check_output(
["heatshrink", "-e", "-w8", "-l4"], input=data_bin
)
assert data_encoded_str
data_enc = bytearray(data_encoded_str)
data_enc = bytearray([len(data_enc) & 0xFF, len(data_enc) >> 8]) + data_enc
# Use encoded data only if its lenght less than original, including header
if len(data_enc) < len(data_bin) + 1:
data = (
"{0x01,0x00,"
+ "".join("0x{:02x},".format(byte) for byte in data_enc)
+ "}"
)
else:
data = "{0x00," + data[1:]
return width, height, data
image = file2image(file)
return image.width, image.height, image.data_as_carray()
def _iconIsSupported(self, filename):
extension = filename.lower().split(".")[-1]
@@ -122,7 +100,11 @@ class Main(App):
def icons(self):
self.logger.debug(f"Converting icons")
icons_c = open(os.path.join(self.args.output_directory, "assets_icons.c"), "w")
icons_c = open(
os.path.join(self.args.output_directory, "assets_icons.c"),
"w",
newline="\n",
)
icons_c.write(ICONS_TEMPLATE_C_HEADER)
icons = []
# Traverse icons tree, append image data to source file
@@ -204,12 +186,19 @@ class Main(App):
)
)
icons_c.write("\n")
icons_c.close()
# Create Public Header
self.logger.debug(f"Creating header")
icons_h = open(os.path.join(self.args.output_directory, "assets_icons.h"), "w")
icons_h = open(
os.path.join(self.args.output_directory, "assets_icons.h"),
"w",
newline="\n",
)
icons_h.write(ICONS_TEMPLATE_H_HEADER)
for name, width, height, frame_rate, frame_count in icons:
icons_h.write(ICONS_TEMPLATE_H_ICON_NAME.format(name=name))
icons_h.close()
self.logger.debug(f"Done")
return 0

View File

@@ -62,7 +62,8 @@ class Main(App):
data += struct.pack("<I", dwCRC)
open(self.args.output, "wb").write(data)
with open(self.args.output, "wb") as file:
file.write(data)
return 0

View File

@@ -1,77 +0,0 @@
#!/usr/bin/env python3
from flipper.app import App
import json
import pathlib
class Main(App):
def init(self):
self.subparsers = self.parser.add_subparsers(help="sub-command help")
# generate
self.parser_generate = self.subparsers.add_parser(
"generate", help="Check source code format and file names"
)
self.parser_generate.add_argument("-p", dest="path", required=True)
self.parser_generate.set_defaults(func=self.generate)
def parse_sources(self, path, source_path, flags_path):
flags = ""
with open(path + "/" + flags_path) as f:
for line in f:
if line.strip():
flags += line.strip() + " "
local_path = str(pathlib.Path().resolve())
data = []
with open(path + "/" + source_path) as f:
for line in f:
if line.strip():
file = line.strip()
data.append(
{
"directory": local_path,
"command": flags + "-c " + file,
"file": file,
}
)
return data
def generate(self):
DB_SOURCE = [
{
"name": "ASM",
"source": "db.asm_source.list",
"flags": "db.asm_flags.list",
},
{"name": "C", "source": "db.c_source.list", "flags": "db.c_flags.list"},
{
"name": "CPP",
"source": "db.cpp_source.list",
"flags": "db.cpp_flags.list",
},
]
path = self.args.path
out_data = []
out_path = path + "/" + "compile_commands.json"
out_file = open(out_path, mode="w")
for record in DB_SOURCE:
self.logger.info(
f"Processing {record['name']} ({record['source']}, {record['flags']})"
)
data = self.parse_sources(path, record["source"], record["flags"])
out_data += data
self.logger.info(f"Saving")
json.dump(out_data, out_file, indent=2)
self.logger.info(f"Compilation DB written to " + out_path)
return 0
if __name__ == "__main__":
Main()()

View File

@@ -106,4 +106,5 @@ class Copro:
address=f"0x{stack_addr:X}",
)
# Save manifest to
json.dump(manifest, open(manifest_file, "w"))
with open(manifest_file, "w", newline="\n") as file:
json.dump(manifest, file)

View File

@@ -227,12 +227,19 @@ class DolphinBubbleAnimation:
(frame, os.path.join(animation_directory, f"frame_{index}.bm"))
)
pool = multiprocessing.Pool()
pool.map(_convert_image_to_bm, to_pack)
if ImageTools.is_processing_slow():
pool = multiprocessing.Pool()
pool.map(_convert_image_to_bm, to_pack)
else:
for image in to_pack:
_convert_image_to_bm(image)
def process(self):
pool = multiprocessing.Pool()
self.frames = pool.map(_convert_image, self.frames)
if ImageTools.is_processing_slow():
pool = multiprocessing.Pool()
self.frames = pool.map(_convert_image, self.frames)
else:
self.frames = list(_convert_image(frame) for frame in self.frames)
class DolphinManifest:
@@ -295,7 +302,8 @@ class DolphinManifest:
def _renderTemplate(self, template_filename: str, output_filename: str, **kwargs):
template = Templite(filename=template_filename)
output = template.render(**kwargs)
open(output_filename, "w").write(output)
with open(output_filename, "w", newline="\n") as file:
file.write(output)
def save2code(self, output_directory: str, symbol_name: str):
# Process frames

View File

@@ -15,8 +15,13 @@ class Image:
self.data = data
def write(self, filename):
file = open(filename, "wb")
file.write(self.data)
with open(filename, "wb") as file:
file.write(self.data)
def data_as_carray(self):
return (
"{" + "".join("0x{:02x},".format(img_byte) for img_byte in self.data) + "}"
)
def is_file_an_icon(filename):
@@ -24,8 +29,62 @@ def is_file_an_icon(filename):
return extension in ICONS_SUPPORTED_FORMATS
class ImageTools:
__pil_unavailable = False
__hs2_unavailable = False
@staticmethod
def is_processing_slow():
try:
from PIL import Image, ImageOps
import heatshrink2
return False
except ImportError as e:
return True
def __init__(self):
self.logger = logging.getLogger()
def png2xbm(self, file):
if self.__pil_unavailable:
return subprocess.check_output(["convert", file, "xbm:-"])
try:
from PIL import Image, ImageOps
except ImportError as e:
self.__pil_unavailable = True
self.logger.info("pillow module is missing, using convert cli util")
return self.png2xbm(file)
with Image.open(file) as im:
with io.BytesIO() as output:
bw = im.convert("1")
bw = ImageOps.invert(bw)
bw.save(output, format="XBM")
return output.getvalue()
def xbm2hs(self, data):
if self.__hs2_unavailable:
return subprocess.check_output(
["heatshrink", "-e", "-w8", "-l4"], input=data
)
try:
import heatshrink2
except ImportError as e:
self.__hs2_unavailable = True
self.logger.info("heatshrink2 module is missing, using heatshrink cli util")
return self.xbm2hs(data)
return heatshrink2.compress(data, window_sz2=8, lookahead_sz2=4)
__tools = ImageTools()
def file2image(file):
output = subprocess.check_output(["convert", file, "xbm:-"])
output = __tools.png2xbm(file)
assert output
# Extract data from text
@@ -38,9 +97,7 @@ def file2image(file):
data_bin = bytearray.fromhex(data_str)
# Encode icon data with LZSS
data_encoded_str = subprocess.check_output(
["heatshrink", "-e", "-w8", "-l4"], input=data_bin
)
data_encoded_str = __tools.xbm2hs(data_bin)
assert data_encoded_str

View File

@@ -1,6 +1,8 @@
import datetime
import logging
import os
import posixpath
from pathlib import Path
from flipper.utils import *
from flipper.utils.fstree import *
@@ -112,20 +114,19 @@ class Manifest:
self.logger = logging.getLogger(self.__class__.__name__)
def load(self, filename):
manifest = open(filename, "r")
for line in manifest.readlines():
line = line.strip()
if len(line) == 0:
continue
tag, line = line.split(":", 1)
record = MANIFEST_TAGS_RECORDS[tag].fromLine(line)
self.records.append(record)
with open(filename, "r") as manifest:
for line in manifest.readlines():
line = line.strip()
if len(line) == 0:
continue
tag, line = line.split(":", 1)
record = MANIFEST_TAGS_RECORDS[tag].fromLine(line)
self.records.append(record)
def save(self, filename):
manifest = open(filename, "w+")
for record in self.records:
manifest.write(record.toLine())
manifest.close()
with open(filename, "w+", newline="\n") as manifest:
for record in self.records:
manifest.write(record.toLine())
def addDirectory(self, path):
self.records.append(ManifestRecordDirectory(path))
@@ -138,20 +139,22 @@ class Manifest:
dirs.sort()
files.sort()
relative_root = root.replace(directory_path, "", 1)
if relative_root:
relative_root = Path(relative_root).as_posix()
if relative_root.startswith("/"):
relative_root = relative_root[1:]
# process directories
for dir in dirs:
relative_dir_path = os.path.join(relative_root, dir)
for dirname in dirs:
relative_dir_path = posixpath.join(relative_root, dirname)
self.logger.debug(f'Adding directory: "{relative_dir_path}"')
self.addDirectory(relative_dir_path)
# Process files
for file in files:
relative_file_path = os.path.join(relative_root, file)
relative_file_path = posixpath.join(relative_root, file)
if file in ignore_files:
self.logger.info(f'Skipping file "{relative_file_path}"')
continue
full_file_path = os.path.join(root, file)
full_file_path = posixpath.join(root, file)
self.logger.debug(f'Adding file: "{relative_file_path}"')
self.addFile(
relative_file_path,

View File

@@ -189,33 +189,33 @@ class FlipperStorage:
"""Send file from local device to Flipper"""
self.remove(filename_to)
file = open(filename_from, "rb")
filesize = os.fstat(file.fileno()).st_size
with open(filename_from, "rb") as file:
filesize = os.fstat(file.fileno()).st_size
buffer_size = 512
while True:
filedata = file.read(buffer_size)
size = len(filedata)
if size == 0:
break
buffer_size = 512
while True:
filedata = file.read(buffer_size)
size = len(filedata)
if size == 0:
break
self.send_and_wait_eol(f'storage write_chunk "{filename_to}" {size}\r')
answer = self.read.until(self.CLI_EOL)
if self.has_error(answer):
self.last_error = self.get_error(answer)
self.send_and_wait_eol(f'storage write_chunk "{filename_to}" {size}\r')
answer = self.read.until(self.CLI_EOL)
if self.has_error(answer):
self.last_error = self.get_error(answer)
self.read.until(self.CLI_PROMPT)
return False
self.port.write(filedata)
self.read.until(self.CLI_PROMPT)
file.close()
return False
self.port.write(filedata)
self.read.until(self.CLI_PROMPT)
percent = str(math.ceil(file.tell() / filesize * 100))
total_chunks = str(math.ceil(filesize / buffer_size))
current_chunk = str(math.ceil(file.tell() / buffer_size))
sys.stdout.write(f"\r{percent}%, chunk {current_chunk} of {total_chunks}")
sys.stdout.flush()
file.close()
percent = str(math.ceil(file.tell() / filesize * 100))
total_chunks = str(math.ceil(filesize / buffer_size))
current_chunk = str(math.ceil(file.tell() / buffer_size))
sys.stdout.write(
f"\r{percent}%, chunk {current_chunk} of {total_chunks}"
)
sys.stdout.flush()
print()
return True

View File

@@ -8,14 +8,14 @@ def timestamp():
def file_hash(path: str, algo: str, block_size: int = 4096):
fd = open(path, "rb")
h = hashlib.new(algo)
while True:
data = fd.read(block_size)
if len(data) > 0:
h.update(data)
else:
break
with open(path, "rb") as fd:
while True:
data = fd.read(block_size)
if len(data) > 0:
h.update(data)
else:
break
return h.hexdigest()

View File

@@ -99,5 +99,5 @@ class FlipperFormatFile:
self.lines = file.readlines()
def save(self, filename: str):
with open(filename, "w") as file:
with open(filename, "w", newline="\n") as file:
file.write("\n".join(self.lines))

View File

@@ -117,6 +117,7 @@ class Main(App):
if dry_run:
if len(bad) > 0:
self.logger.error(f"Found {len(bad)} incorrectly named files")
self.logger.info(bad)
return False
else:
# Replace occurances in text files

View File

@@ -40,10 +40,10 @@ class Main(App):
def before(self):
self.logger.info(f"Loading Option Bytes data")
file_path = os.path.join(os.path.dirname(sys.argv[0]), "ob.data")
file = open(file_path, "r")
for line in file.readlines():
k, v, o = line.split(":")
self.ob[k.strip()] = v.strip(), o.strip()
with open(file_path, "r") as file:
for line in file.readlines():
k, v, o = line.split(":")
self.ob[k.strip()] = v.strip(), o.strip()
def check(self):
self.logger.info(f"Checking Option Bytes")

View File

@@ -149,8 +149,10 @@ class Main(App):
self.logger.info(f"Generating OTP")
self._processFirstArgs()
self._processSecondArgs()
open(f"{self.args.file}_first.bin", "wb").write(self._packFirst())
open(f"{self.args.file}_second.bin", "wb").write(self._packSecond())
with open(f"{self.args.file}_first.bin", "wb") as file:
file.write(self._packFirst())
with open(f"{self.args.file}_second.bin", "wb") as file:
file.write(self._packSecond())
self.logger.info(
f"Generated files: {self.args.file}_first.bin and {self.args.file}_second.bin"
)
@@ -166,9 +168,9 @@ class Main(App):
try:
self.logger.info(f"Packing binary data")
file = open(filename, "wb")
file.write(self._packFirst())
file.close()
with open(filename, "wb") as file:
file.write(self._packFirst())
self.logger.info(f"Flashing OTP")
cp = CubeProgrammer(self._getCubeParams())
cp.flashBin("0x1FFF7000", filename)
@@ -190,9 +192,9 @@ class Main(App):
try:
self.logger.info(f"Packing binary data")
file = open(filename, "wb")
file.write(self._packSecond())
file.close()
with open(filename, "wb") as file:
file.write(self._packSecond())
self.logger.info(f"Flashing OTP")
cp = CubeProgrammer(self._getCubeParams())
cp.flashBin("0x1FFF7010", filename)
@@ -215,10 +217,10 @@ class Main(App):
try:
self.logger.info(f"Packing binary data")
file = open(filename, "wb")
file.write(self._packFirst())
file.write(self._packSecond())
file.close()
with open(filename, "wb") as file:
file.write(self._packFirst())
file.write(self._packSecond())
self.logger.info(f"Flashing OTP")
cp = CubeProgrammer(self._getCubeParams())
cp.flashBin("0x1FFF7000", filename)

7
scripts/requirements.txt Normal file
View File

@@ -0,0 +1,7 @@
pyserial==3.5
heatshrink2==0.11.0
Pillow==9.1.1
grpcio==1.47.0
grpcio-tools==1.47.0
protobuf==3.20.1
python3-protobuf==2.5.0

69
scripts/dist.py → scripts/sconsdist.py Executable file → Normal file
View File

@@ -7,6 +7,15 @@ from update import Main as UpdateMain
import shutil
class ProjectDir:
def __init__(self, project_dir):
self.dir = project_dir
parts = project_dir.split("-")
self.target = parts[0]
self.project = parts[1]
self.flavor = parts[2] if len(parts) > 2 else ""
class Main(App):
def init(self):
self.subparsers = self.parser.add_subparsers(help="sub-command help")
@@ -15,8 +24,7 @@ class Main(App):
"copy", help="Copy firmware binaries & metadata"
)
self.parser_copy.add_argument("-t", dest="target", required=True)
self.parser_copy.add_argument("-p", dest="projects", nargs="+", required=True)
self.parser_copy.add_argument("-p", dest="project", nargs="+", required=True)
self.parser_copy.add_argument("-s", dest="suffix", required=True)
self.parser_copy.add_argument("-r", dest="resources", required=False)
self.parser_copy.add_argument(
@@ -36,40 +44,67 @@ class Main(App):
def get_project_filename(self, project, filetype):
# Temporary fix
if project == "firmware" and filetype != "elf":
project = "full"
return f"flipper-z-{self.args.target}-{project}-{self.args.suffix}.{filetype}"
project_name = project.project
if project_name == "firmware" and filetype != "elf":
project_name = "full"
return f"flipper-z-{self.target}-{project_name}-{self.args.suffix}.{filetype}"
def get_dist_filepath(self, filename):
return join(self.output_dir_path, filename)
def copy_single_project(self, project):
target_project = f"{self.args.target}-{project}"
obj_directory = join("firmware", ".obj", target_project)
obj_directory = join("build", project.dir)
for filetype in ("elf", "bin", "dfu", "json"):
shutil.copyfile(
join(obj_directory, f"{project}.{filetype}"),
join(obj_directory, f"{project.project}.{filetype}"),
self.get_dist_filepath(self.get_project_filename(project, filetype)),
)
def copy(self):
self.output_dir_path = join("dist", self.args.target)
self.projects = dict(
map(
lambda pd: (pd.project, pd),
map(ProjectDir, self.args.project),
)
)
project_targets = set(map(lambda p: p.target, self.projects.values()))
if len(project_targets) != 1:
self.logger.error(f"Cannot mix targets {project_targets}!")
return 1
self.target = project_targets.pop()
project_flavors = set(map(lambda p: p.flavor, self.projects.values()))
if len(project_flavors) != 1:
self.logger.error(f"Cannot mix flavors {project_flavors}!")
return 2
self.flavor = project_flavors.pop()
dist_dir_components = [self.target]
if self.flavor:
dist_dir_components.append(self.flavor)
self.output_dir_path = join("dist", "-".join(dist_dir_components))
if exists(self.output_dir_path) and not self.args.noclean:
shutil.rmtree(self.output_dir_path)
try:
shutil.rmtree(self.output_dir_path)
except Exception as ex:
pass
if not exists(self.output_dir_path):
makedirs(self.output_dir_path)
for project in self.args.projects:
for project in self.projects.values():
self.copy_single_project(project)
self.logger.info(
f"Firmware binaries can be found at:\n\t{self.output_dir_path}"
)
if self.args.version:
bundle_dir = join(
self.output_dir_path, f"{self.args.target}-update-{self.args.suffix}"
self.output_dir_path, f"{self.target}-update-{self.args.suffix}"
)
bundle_args = [
"generate",
@@ -78,11 +113,15 @@ class Main(App):
"-v",
self.args.version,
"-t",
self.args.target,
self.target,
"--dfu",
self.get_dist_filepath(self.get_project_filename("firmware", "dfu")),
self.get_dist_filepath(
self.get_project_filename(self.projects["firmware"], "dfu")
),
"--stage",
self.get_dist_filepath(self.get_project_filename("updater", "bin")),
self.get_dist_filepath(
self.get_project_filename(self.projects["updater"], "bin")
),
]
if self.args.resources:
bundle_args.extend(

View File

@@ -293,7 +293,8 @@ class Main:
with tempfile.TemporaryDirectory() as tmpdirname:
send_file_name = os.path.join(tmpdirname, "send")
receive_file_name = os.path.join(tmpdirname, "receive")
open(send_file_name, "w").write("A" * self.args.file_size)
with open(send_file_name, "w") as fout:
fout.write("A" * self.args.file_size)
storage = FlipperStorage(self.args.port)
storage.start()
if storage.exist_file(self.args.flipper_path):

131
scripts/version.py Normal file
View File

@@ -0,0 +1,131 @@
#!/usb/bin/env python3
from flipper.app import App
import subprocess
import os
import json
from datetime import date
class GitVersion:
def __init__(self, source_dir):
self.source_dir = source_dir
def get_version_info(self):
commit = self._exec_git("rev-parse --short HEAD") or "unknown"
dirty = False
try:
self._exec_git("diff --quiet")
except subprocess.CalledProcessError as e:
if e.returncode == 1:
dirty = True
# If WORKFLOW_BRANCH_OR_TAG is set in environment, is has precedence
# (set by CI)
branch = (
os.environ.get("WORKFLOW_BRANCH_OR_TAG", None)
or self._exec_git("rev-parse --abbrev-ref HEAD")
or "unknown"
)
branch_num = self._exec_git("rev-list --count HEAD") or "n/a"
try:
version = self._exec_git("describe --tags --abbrev=0 --exact-match")
except subprocess.CalledProcessError:
version = "unknown"
return {
"GIT_COMMIT": commit,
"GIT_BRANCH": branch,
"GIT_BRANCH_NUM": branch_num,
"VERSION": version,
"BUILD_DIRTY": dirty and 1 or 0,
}
def _exec_git(self, args):
cmd = ["git"]
cmd.extend(args.split(" "))
return (
subprocess.check_output(cmd, cwd=self.source_dir, stderr=subprocess.STDOUT)
.strip()
.decode()
)
class Main(App):
def init(self):
self.subparsers = self.parser.add_subparsers(help="sub-command help")
# generate
self.parser_generate = self.subparsers.add_parser(
"generate", help="Generate version header"
)
self.parser_generate.add_argument("-o", dest="outdir", required=True)
self.parser_generate.add_argument(
"-t",
dest="target",
type=int,
help="hardware target",
required=True,
)
self.parser_generate.add_argument("--dir", dest="sourcedir", required=True)
self.parser_generate.set_defaults(func=self.generate)
def generate(self):
current_info = GitVersion(self.args.sourcedir).get_version_info()
current_info.update(
{
"BUILD_DATE": date.today().strftime("%d-%m-%Y"),
"TARGET": self.args.target,
}
)
version_values = []
for key in current_info:
val = current_info[key]
if isinstance(val, str):
val = f'"{val}"'
version_values.append(f"#define {key} {val}")
new_version_info_fmt = "\n".join(version_values) + "\n"
current_version_info = None
version_header_name = os.path.join(self.args.outdir, "version.inc.h")
version_json_name = os.path.join(self.args.outdir, "version.json")
try:
with open(version_header_name, "r") as file:
current_version_info = file.read()
except EnvironmentError as e:
if self.args.debug:
print(e)
if current_version_info != new_version_info_fmt:
if self.args.debug:
print("old: ", current_version_info)
print("new: ", new_version_info_fmt)
with open(version_header_name, "w", newline="\n") as file:
file.write(new_version_info_fmt)
# os.utime("../lib/toolbox/version.c", None)
print("Version information updated")
else:
if self.args.debug:
print("Version information hasn't changed")
version_json = {
"firmware_build_date": current_info["BUILD_DATE"],
"firmware_commit": current_info["GIT_COMMIT"],
"firmware_branch": current_info["GIT_BRANCH"],
"firmware_target": current_info["TARGET"],
}
with open(version_json_name, "w", newline="\n") as file:
json.dump(version_json, file, indent=4)
return 0
if __name__ == "__main__":
Main()()