Trigger amap from another repo (#2171)

* Add s3 upload, repository dispatch
* Add trigger
* Fix map file upload
* Debug
* Add event file upload to s3
* fix triggering
* Fix upload process
* fix build.yml

Co-authored-by: あく <alleteam@gmail.com>
This commit is contained in:
Max Andreev 2022-12-26 23:22:22 +03:00 committed by GitHub
parent 9192520c70
commit 1c926cf8a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 246 deletions

View File

@ -1,103 +0,0 @@
name: 'Analyze .map file with Amap'
on:
push:
branches:
- dev
- "release*"
tags:
- '*'
pull_request:
env:
TARGETS: f7
FBT_TOOLCHAIN_PATH: /opt
jobs:
amap_analyse:
if: ${{ !github.event.pull_request.head.repo.fork }}
runs-on: [self-hosted,FlipperZeroMacShell]
timeout-minutes: 15
steps:
- name: 'Wait Build workflow'
uses: fountainhead/action-wait-for-check@v1.0.0
id: wait-for-build
with:
token: ${{ secrets.GITHUB_TOKEN }}
checkName: 'main'
ref: ${{ github.event.pull_request.head.sha || github.sha }}
intervalSeconds: 20
- name: 'Check Build workflow status'
if: steps.wait-for-build.outputs.conclusion == 'failure'
run: |
exit 1
- name: 'Decontaminate previous build leftovers'
run: |
if [ -d .git ]; then
git submodule status || git checkout "$(git rev-list --max-parents=0 HEAD | tail -n 1)"
fi
- name: 'Checkout code'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: 'Get commit details'
run: |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then
TYPE="pull"
elif [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
TYPE="tag"
else
TYPE="other"
fi
python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE"
- name: 'Make artifacts directory'
run: |
rm -rf artifacts
mkdir artifacts
- name: 'Download build artifacts'
run: |
mkdir -p ~/.ssh
ssh-keyscan -p ${{ secrets.RSYNC_DEPLOY_PORT }} -H ${{ secrets.RSYNC_DEPLOY_HOST }} > ~/.ssh/known_hosts
echo "${{ secrets.RSYNC_DEPLOY_KEY }}" > deploy_key;
chmod 600 ./deploy_key;
rsync -avzP \
-e 'ssh -p ${{ secrets.RSYNC_DEPLOY_PORT }} -i ./deploy_key' \
${{ secrets.RSYNC_DEPLOY_USER }}@${{ secrets.RSYNC_DEPLOY_HOST }}:"${{ secrets.RSYNC_DEPLOY_BASE_PATH }}${BRANCH_NAME}/" artifacts/;
rm ./deploy_key;
- name: 'Make .map file analyze'
run: |
cd artifacts/
/Applications/amap/Contents/MacOS/amap -f "flipper-z-f7-firmware-${SUFFIX}.elf.map"
- name: 'Upload report to DB'
run: |
source scripts/toolchain/fbtenv.sh
get_size()
{
SECTION="$1";
arm-none-eabi-size \
-A artifacts/flipper-z-f7-firmware-$SUFFIX.elf \
| grep "^$SECTION" | awk '{print $2}'
}
export BSS_SIZE="$(get_size ".bss")"
export TEXT_SIZE="$(get_size ".text")"
export RODATA_SIZE="$(get_size ".rodata")"
export DATA_SIZE="$(get_size ".data")"
export FREE_FLASH_SIZE="$(get_size ".free_flash")"
python3 -m pip install mariadb==1.1.4
python3 scripts/amap_mariadb_insert.py \
${{ secrets.AMAP_MARIADB_USER }} \
${{ secrets.AMAP_MARIADB_PASSWORD }} \
${{ secrets.AMAP_MARIADB_HOST }} \
${{ secrets.AMAP_MARIADB_PORT }} \
${{ secrets.AMAP_MARIADB_DATABASE }} \
artifacts/flipper-z-f7-firmware-$SUFFIX.elf.map.all

View File

@ -30,11 +30,6 @@ jobs:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: 'Make artifacts directory'
run: |
rm -rf artifacts
mkdir artifacts
- name: 'Get commit details'
id: names
run: |
@ -46,6 +41,15 @@ jobs:
TYPE="other"
fi
python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE"
echo random_hash=$(openssl rand -base64 40 | shasum -a 256 | awk '{print $1}') >> $GITHUB_OUTPUT
echo "event_type=$TYPE" >> $GITHUB_OUTPUT
- name: 'Make artifacts directory'
run: |
rm -rf artifacts
rm -rf map_analyser_files
mkdir artifacts
mkdir map_analyser_files
- name: 'Bundle scripts'
if: ${{ !github.event.pull_request.head.repo.fork }}
@ -82,9 +86,30 @@ jobs:
run: |
cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz"
- name: 'Copy .map file'
- name: 'Copy map analyser files'
run: |
cp build/f7-firmware-*/firmware.elf.map "artifacts/flipper-z-f7-firmware-${SUFFIX}.elf.map"
cp build/f7-firmware-*/firmware.elf.map map_analyser_files/firmware.elf.map
cp build/f7-firmware-*/firmware.elf map_analyser_files/firmware.elf
cp ${{ github.event_path }} map_analyser_files/event.json
- name: 'Upload map analyser files to storage'
uses: keithweaver/aws-s3-github-action@v1.0.0
with:
source: map_analyser_files/
destination: "s3://${{ secrets.MAP_REPORT_AWS_BUCKET }}/${{steps.names.outputs.random_hash}}"
aws_access_key_id: "${{ secrets.MAP_REPORT_AWS_ACCESS_KEY }}"
aws_secret_access_key: "${{ secrets.MAP_REPORT_AWS_SECRET_KEY }}"
aws_region: "${{ secrets.MAP_REPORT_AWS_REGION }}"
flags: --recursive
- name: 'Trigger map file reporter'
uses: peter-evans/repository-dispatch@v2
with:
repository: flipperdevices/flipper-map-reporter
token: ${{ secrets.REPOSITORY_DISPATCH_TOKEN }}
event-type: map-file-analyse
client-payload: '{"random_hash": "${{steps.names.outputs.random_hash}}", "event_type": "${{steps.names.outputs.event_type}}"}'
- name: 'Upload artifacts to update server'
if: ${{ !github.event.pull_request.head.repo.fork }}

View File

@ -1,136 +0,0 @@
#!/usr/bin/env python3
from datetime import datetime
import argparse
import mariadb
import sys
import os
def parseArgs():
parser = argparse.ArgumentParser()
parser.add_argument("db_user", help="MariaDB user")
parser.add_argument("db_pass", help="MariaDB password")
parser.add_argument("db_host", help="MariaDB hostname")
parser.add_argument("db_port", type=int, help="MariaDB port")
parser.add_argument("db_name", help="MariaDB database")
parser.add_argument("report_file", help="Report file(.map.all)")
args = parser.parse_args()
return args
def mariadbConnect(args):
try:
conn = mariadb.connect(
user=args.db_user,
password=args.db_pass,
host=args.db_host,
port=args.db_port,
database=args.db_name,
)
except mariadb.Error as e:
print(f"Error connecting to MariaDB: {e}")
sys.exit(1)
return conn
def parseEnv():
outArr = []
outArr.append(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
outArr.append(os.getenv("COMMIT_HASH", default=None))
outArr.append(os.getenv("COMMIT_MSG", default=None))
outArr.append(os.getenv("BRANCH_NAME", default=None))
outArr.append(os.getenv("BSS_SIZE", default=None))
outArr.append(os.getenv("TEXT_SIZE", default=None))
outArr.append(os.getenv("RODATA_SIZE", default=None))
outArr.append(os.getenv("DATA_SIZE", default=None))
outArr.append(os.getenv("FREE_FLASH_SIZE", default=None))
outArr.append(os.getenv("PULL_ID", default=None))
outArr.append(os.getenv("PULL_NAME", default=None))
return outArr
def createTables(cur, conn):
headerTable = "CREATE TABLE IF NOT EXISTS `header` ( \
`id` int(10) unsigned NOT NULL AUTO_INCREMENT, \
`datetime` datetime NOT NULL, \
`commit` varchar(40) NOT NULL, \
`commit_msg` text NOT NULL, \
`branch_name` text NOT NULL, \
`bss_size` int(10) unsigned NOT NULL, \
`text_size` int(10) unsigned NOT NULL, \
`rodata_size` int(10) unsigned NOT NULL, \
`data_size` int(10) unsigned NOT NULL, \
`free_flash_size` int(10) unsigned NOT NULL, \
`pullrequest_id` int(10) unsigned DEFAULT NULL, \
`pullrequest_name` text DEFAULT NULL, \
PRIMARY KEY (`id`), \
KEY `header_id_index` (`id`) )"
dataTable = "CREATE TABLE IF NOT EXISTS `data` ( \
`header_id` int(10) unsigned NOT NULL, \
`id` int(10) unsigned NOT NULL AUTO_INCREMENT, \
`section` text NOT NULL, \
`address` text NOT NULL, \
`size` int(10) unsigned NOT NULL, \
`name` text NOT NULL, \
`lib` text NOT NULL, \
`obj_name` text NOT NULL, \
PRIMARY KEY (`id`), \
KEY `data_id_index` (`id`), \
KEY `data_header_id_index` (`header_id`), \
CONSTRAINT `data_header_id_foreign` FOREIGN KEY (`header_id`) REFERENCES `header` (`id`) )"
cur.execute(headerTable)
cur.execute(dataTable)
conn.commit()
def insertHeader(data, cur, conn):
query = "INSERT INTO `header` ( \
datetime, commit, commit_msg, branch_name, bss_size, text_size, \
rodata_size, data_size, free_flash_size, pullrequest_id, pullrequest_name) \
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
cur.execute(query, data)
conn.commit()
return cur.lastrowid
def parseFile(fileObj, headerID):
arr = []
fileLines = fileObj.readlines()
for line in fileLines:
lineArr = []
tempLineArr = line.split("\t")
lineArr.append(headerID)
lineArr.append(tempLineArr[0]) # section
lineArr.append(int(tempLineArr[2], 16)) # address hex
lineArr.append(int(tempLineArr[3])) # size
lineArr.append(tempLineArr[4]) # name
lineArr.append(tempLineArr[5]) # lib
lineArr.append(tempLineArr[6]) # obj_name
arr.append(tuple(lineArr))
return arr
def insertData(data, cur, conn):
query = "INSERT INTO `data` ( \
header_id, section, address, size, \
name, lib, obj_name) \
VALUES (?, ?, ?, ?, ? ,?, ?)"
cur.executemany(query, data)
conn.commit()
def main():
args = parseArgs()
dbConn = mariadbConnect(args)
reportFile = open(args.report_file)
dbCurs = dbConn.cursor()
createTables(dbCurs, dbConn)
headerID = insertHeader(parseEnv(), dbCurs, dbConn)
insertData(parseFile(reportFile, headerID), dbCurs, dbConn)
reportFile.close()
dbCurs.close()
if __name__ == "__main__":
main()