diff --git a/.github/workflows/perf_emulation_handle_results.yml b/.github/workflows/perf_emulation_handle_results.yml index 27c3604c23..ce64acdb2d 100644 --- a/.github/workflows/perf_emulation_handle_results.yml +++ b/.github/workflows/perf_emulation_handle_results.yml @@ -116,6 +116,10 @@ jobs: } const commentBody = fs.readFileSync(commentPath, 'utf8').trim(); + if (commentBody.length == 0) { + console.log("Not generating a comment as the comment body is empty"); + process.exit(0); + } const prNumber = Number(fs.readFileSync(prPath, 'utf8').trim()); github.rest.issues.createComment({ diff --git a/scripts/perf/benchmark_results_comment.py b/scripts/perf/benchmark_results_comment.py index f76d46457c..a009fd3340 100644 --- a/scripts/perf/benchmark_results_comment.py +++ b/scripts/perf/benchmark_results_comment.py @@ -109,19 +109,24 @@ comparing different LVGL features and configurations, but may not correlate dire physical hardware. The measurements are intended for comparative analysis only. """ +DATA_KEYS = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"] -def format_table(results: list[dict], prev_results: list[dict]): + +def generate_table(results: list[dict], prev_results: list[dict]) -> list[dict]: + return [ + {key: ((scene_results[key] - prev_scene_results[key])) for key in DATA_KEYS} + for scene_results, prev_scene_results in zip(results, prev_results) + ] + + +def format_table(results: list[dict], delta: list[dict]) -> str: table = "| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) |\n" table += "|------------|------------|---------|--------------|----------------|--------------|\n" - data_keys = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"] - for scene_results, prev_scene_results in zip(results, prev_results): - delta_p = { - key: ((scene_results[key] - prev_scene_results[key])) for key in data_keys - } + for scene_results, delta_p in zip(results, delta): table += f"| {scene_results['scene_name']} |" - for key in data_keys: + for key in DATA_KEYS: if delta_p[key] == 0: table += f" {scene_results[key]} |" else: @@ -163,6 +168,9 @@ def main(): results_paths = args.new output_path = args.output + print(f"Previous results: {previous_results_paths}") + print(f"New results: {results_paths}") + previous_results_map: dict[str, list[dict]] = {} if previous_results_paths: for results_path in previous_results_paths: @@ -183,6 +191,7 @@ def main(): comment += "We've run benchmarks in an emulated environment." comment += " Here are the results:\n\n" + diff_found = False for result_path, result in new_results.items(): mpk_path = result_path.replace(".json", ".mpk") @@ -193,6 +202,7 @@ def main(): prev_results = previous_results_map.get(mpk_path, []) if len(prev_results) > 0: + print(f"Found previous results for {result_path} in {mpk_path}") prev_scenes = prev_results[-1]["scenes"] prev_all_scene_avg = [ [ @@ -207,17 +217,26 @@ def main(): # the previous aswell # In this case, the difference will always be zero and we won't # add any new information to the result table + print(f"Previous results not found for {result_path}") + diff_found = True # Generate a real comment since we got new results prev_results = result prev_all_scene_avg = new_all_scene_avg + delta_all_scene_avg = generate_table(new_all_scene_avg, prev_all_scene_avg) + delta_results_avg = generate_table(result, prev_results) + + diff_found = diff_found or any( + value != 0 for dic in delta_results_avg for value in dic.values() + ) + _, image_type, config = result_path.replace(".json", "").split("-") comment += f"#### ARM Emulated {image_type} - {config}\n\n" - comment += format_table(new_all_scene_avg, prev_all_scene_avg) + comment += format_table(new_all_scene_avg, delta_all_scene_avg) comment += "\n
" comment += "\n" comment += "\nDetailed Results Per Scene" comment += "\n\n\n" - comment += format_table(result, prev_results) + comment += format_table(result, delta_results_avg) comment += "\n\n
\n\n" comment += DISCLAIMER @@ -226,8 +245,10 @@ def main(): comment += "\n\n" comment += ":robot: This comment was automatically generated by a bot." + # Create the file but don't write anything to it if we haven't found a difference with open(output_path, "w") as f: - f.write(comment) + if diff_found: + f.write(comment) if __name__ == "__main__": diff --git a/scripts/perf/tests/benchmark_results_comment/no_diff/expected.md b/scripts/perf/tests/benchmark_results_comment/no_diff/expected.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json b/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json new file mode 100644 index 0000000000..31a0b23f4b --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.mpk b/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.mpk new file mode 100644 index 0000000000..ba089eadb9 Binary files /dev/null and b/scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.mpk differ diff --git a/scripts/perf/tests/benchmark_results_comment/test.sh b/scripts/perf/tests/benchmark_results_comment/test.sh index 724dad1dc2..b03f5d9cd3 100755 --- a/scripts/perf/tests/benchmark_results_comment/test.sh +++ b/scripts/perf/tests/benchmark_results_comment/test.sh @@ -9,9 +9,12 @@ SCRIPT_PATH=$(dirname $SCRIPT) cd $SCRIPT_PATH python3 ../../benchmark_results_comment.py --new no_mpk/results*.json -o no_mpk/actual.md python3 ../../benchmark_results_comment.py --previous normal/results*.mpk --new normal/results*.json -o normal/actual.md +python3 ../../benchmark_results_comment.py --previous no_diff/results*.mpk --new no_diff/results*.json -o no_diff/actual.md cd $ORIGINAL_PWD cmp $SCRIPT_PATH/no_mpk/expected.md $SCRIPT_PATH/no_mpk/actual.md cmp $SCRIPT_PATH/normal/expected.md $SCRIPT_PATH/normal/actual.md +cmp $SCRIPT_PATH/no_diff/expected.md $SCRIPT_PATH/no_diff/actual.md +echo "[TEST] Passed"