diff --git a/.github/workflows/perf_emulation_handle_results.yml b/.github/workflows/perf_emulation_handle_results.yml index 1d6ddc5f8d..498fe2bfde 100644 --- a/.github/workflows/perf_emulation_handle_results.yml +++ b/.github/workflows/perf_emulation_handle_results.yml @@ -97,7 +97,6 @@ jobs: --new input/results*.json \ --output comment.md fi - - name: Comment PR if: ${{ github.event.workflow_run.event == 'pull_request' }} uses: actions/github-script@v7 @@ -106,28 +105,49 @@ jobs: const fs = require('fs'); const commentPath = 'comment.md'; const prPath = 'pr_number'; - + if (!fs.existsSync(commentPath)) { throw new Error('Error: comment.md not found! Exiting.'); } - if (!fs.existsSync(prPath)) { throw new Error('Error: pr_number not found! Exiting.'); } - + const commentBody = fs.readFileSync(commentPath, 'utf8').trim(); - if (commentBody.length == 0) { - console.log("Not generating a comment as the comment body is empty"); - process.exit(0); - } const prNumber = Number(fs.readFileSync(prPath, 'utf8').trim()); - - github.rest.issues.createComment({ + + // Try to find if a comment already exists so we avoid spamming the PR with comments + const { data: comments } = await github.rest.issues.listComments({ issue_number: prNumber, owner: context.repo.owner, repo: context.repo.repo, - body: commentBody }); + const existingComment = comments.find(comment => + comment.body.includes(':robot: This comment was automatically generated by a bot.') + ); + + try { + // Now we either edit the already existing comment or we generate a new one + if (existingComment) { + await github.rest.issues.updateComment({ + comment_id: existingComment.id, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody + }); + console.log(`Updated existing comment (ID: ${existingComment.id})`); + } else { + await github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody + }); + console.log('Created new comment'); + } + } catch (error) { + console.error("Error:", error.message); + } - name: Serialize Results if: ${{ github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'master' }} diff --git a/scripts/perf/benchmark_results_comment.py b/scripts/perf/benchmark_results_comment.py index 7297d45540..f8d7ddbc72 100644 --- a/scripts/perf/benchmark_results_comment.py +++ b/scripts/perf/benchmark_results_comment.py @@ -110,24 +110,19 @@ comparing different LVGL features and configurations, but may not correlate dire physical hardware. The measurements are intended for comparative analysis only. """ -DATA_KEYS = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"] - -def generate_table(results: list[dict], prev_results: list[dict]) -> list[dict]: - return [ - {key: ((scene_results[key] - prev_scene_results[key])) for key in DATA_KEYS} - for scene_results, prev_scene_results in zip(results, prev_results) - ] - - -def format_table(results: list[dict], delta: list[dict]) -> str: +def format_table(results: list[dict], prev_results: list[dict]): table = "| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) |\n" table += "|------------|------------|---------|--------------|----------------|--------------|\n" + data_keys = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"] - for scene_results, delta_p in zip(results, delta): + for scene_results, prev_scene_results in zip(results, prev_results): + delta_p = { + key: ((scene_results[key] - prev_scene_results[key])) for key in data_keys + } table += f"| {scene_results['scene_name']} |" - for key in DATA_KEYS: + for key in data_keys: if delta_p[key] == 0: table += f" {scene_results[key]} |" else: @@ -169,9 +164,6 @@ def main(): results_paths = args.new output_path = args.output - print(f"Previous results: {previous_results_paths}") - print(f"New results: {results_paths}") - previous_results_map: dict[str, list[dict]] = {} if previous_results_paths: for results_path in previous_results_paths: @@ -194,7 +186,6 @@ def main(): comment += "We've run benchmarks in an emulated environment." comment += " Here are the results:\n\n" - diff_found = False for result_path, result in new_results.items(): mpk_path = result_path.replace(".json", ".mpk") @@ -205,7 +196,6 @@ def main(): prev_results = previous_results_map.get(mpk_path, []) if len(prev_results) > 0: - print(f"Found previous results for {result_path} in {mpk_path}") prev_scenes = prev_results[-1]["scenes"] prev_all_scene_avg = [ [ @@ -220,26 +210,17 @@ def main(): # the previous aswell # In this case, the difference will always be zero and we won't # add any new information to the result table - print(f"Previous results not found for {result_path}") - diff_found = True # Generate a real comment since we got new results prev_results = result prev_all_scene_avg = new_all_scene_avg - delta_all_scene_avg = generate_table(new_all_scene_avg, prev_all_scene_avg) - delta_results_avg = generate_table(result, prev_results) - - diff_found = diff_found or any( - value != 0 for dic in delta_results_avg for value in dic.values() - ) - _, image_type, config = result_path.replace(".json", "").split("-") comment += f"#### ARM Emulated {image_type} - {config}\n\n" - comment += format_table(new_all_scene_avg, delta_all_scene_avg) + comment += format_table(new_all_scene_avg, prev_all_scene_avg) comment += "\n
" comment += "\n" comment += "\nDetailed Results Per Scene" comment += "\n\n\n" - comment += format_table(result, delta_results_avg) + comment += format_table(result, prev_results) comment += "\n\n
\n\n" comment += DISCLAIMER @@ -248,10 +229,8 @@ def main(): comment += "\n\n" comment += ":robot: This comment was automatically generated by a bot." - # Create the file but don't write anything to it if we haven't found a difference with open(output_path, "w") as f: - if diff_found: - f.write(comment) + f.write(comment) if __name__ == "__main__": diff --git a/scripts/perf/tests/benchmark_results_comment/no_diff/expected.md b/scripts/perf/tests/benchmark_results_comment/no_diff/expected.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json b/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json deleted file mode 100644 index 31a0b23f4b..0000000000 --- a/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json +++ /dev/null @@ -1 +0,0 @@ -[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.mpk b/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.mpk deleted file mode 100644 index ba089eadb9..0000000000 Binary files a/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.mpk and /dev/null differ diff --git a/scripts/perf/tests/benchmark_results_comment/test.sh b/scripts/perf/tests/benchmark_results_comment/test.sh index b03f5d9cd3..724dad1dc2 100755 --- a/scripts/perf/tests/benchmark_results_comment/test.sh +++ b/scripts/perf/tests/benchmark_results_comment/test.sh @@ -9,12 +9,9 @@ SCRIPT_PATH=$(dirname $SCRIPT) cd $SCRIPT_PATH python3 ../../benchmark_results_comment.py --new no_mpk/results*.json -o no_mpk/actual.md python3 ../../benchmark_results_comment.py --previous normal/results*.mpk --new normal/results*.json -o normal/actual.md -python3 ../../benchmark_results_comment.py --previous no_diff/results*.mpk --new no_diff/results*.json -o no_diff/actual.md cd $ORIGINAL_PWD cmp $SCRIPT_PATH/no_mpk/expected.md $SCRIPT_PATH/no_mpk/actual.md cmp $SCRIPT_PATH/normal/expected.md $SCRIPT_PATH/normal/actual.md -cmp $SCRIPT_PATH/no_diff/expected.md $SCRIPT_PATH/no_diff/actual.md -echo "[TEST] Passed"