ci: find and edit existing comment for benchmark results (#8531)

scripts/perf/benchmark_results_comment.py

@@ -110,24 +110,19 @@ comparing different LVGL features and configurations, but may not correlate dire
 physical hardware. The measurements are intended for comparative analysis only.
 """
 
-DATA_KEYS = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"]
 
-
-def generate_table(results: list[dict], prev_results: list[dict]) -> list[dict]:
-    return [
-        {key: ((scene_results[key] - prev_scene_results[key])) for key in DATA_KEYS}
-        for scene_results, prev_scene_results in zip(results, prev_results)
-    ]
-
-
-def format_table(results: list[dict], delta: list[dict]) -> str:
+def format_table(results: list[dict], prev_results: list[dict]):
 
     table = "| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) |\n"
     table += "|------------|------------|---------|--------------|----------------|--------------|\n"
+    data_keys = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"]
 
-    for scene_results, delta_p in zip(results, delta):
+    for scene_results, prev_scene_results in zip(results, prev_results):
+        delta_p = {
+            key: ((scene_results[key] - prev_scene_results[key])) for key in data_keys
+        }
         table += f"| {scene_results['scene_name']} |"
-        for key in DATA_KEYS:
+        for key in data_keys:
             if delta_p[key] == 0:
                 table += f" {scene_results[key]} |"
             else:
@@ -169,9 +164,6 @@ def main():
     results_paths = args.new
     output_path = args.output
 
-    print(f"Previous results: {previous_results_paths}")
-    print(f"New results: {results_paths}")
-
     previous_results_map: dict[str, list[dict]] = {}
     if previous_results_paths:
         for results_path in previous_results_paths:
@@ -194,7 +186,6 @@ def main():
     comment += "We've run benchmarks in an emulated environment."
     comment += " Here are the results:\n\n"
 
-    diff_found = False
     for result_path, result in new_results.items():
         mpk_path = result_path.replace(".json", ".mpk")
 
@@ -205,7 +196,6 @@ def main():
         prev_results = previous_results_map.get(mpk_path, [])
 
         if len(prev_results) > 0:
-            print(f"Found previous results for {result_path} in {mpk_path}")
             prev_scenes = prev_results[-1]["scenes"]
             prev_all_scene_avg = [
                 [
@@ -220,26 +210,17 @@ def main():
             # the previous aswell
             # In this case, the difference will always be zero and we won't
             # add any new information to the result table
-            print(f"Previous results not found for {result_path}")
-            diff_found = True  # Generate a real comment since we got new results
             prev_results = result
             prev_all_scene_avg = new_all_scene_avg
 
-        delta_all_scene_avg = generate_table(new_all_scene_avg, prev_all_scene_avg)
-        delta_results_avg = generate_table(result, prev_results)
-
-        diff_found = diff_found or any(
-            value != 0 for dic in delta_results_avg for value in dic.values()
-        )
-
         _, image_type, config = result_path.replace(".json", "").split("-")
         comment += f"#### ARM Emulated {image_type} - {config}\n\n"
-        comment += format_table(new_all_scene_avg, delta_all_scene_avg)
+        comment += format_table(new_all_scene_avg, prev_all_scene_avg)
         comment += "\n<details>"
         comment += "\n<summary>"
         comment += "\nDetailed Results Per Scene"
         comment += "\n</summary>\n\n"
-        comment += format_table(result, delta_results_avg)
+        comment += format_table(result, prev_results)
         comment += "\n\n</details>\n\n"
 
     comment += DISCLAIMER
@@ -248,10 +229,8 @@ def main():
     comment += "\n\n"
     comment += ":robot: This comment was automatically generated by a bot."
 
-    # Create the file but don't write anything to it if we haven't found a difference
     with open(output_path, "w") as f:
-        if diff_found:
-            f.write(comment)
+        f.write(comment)
 
 
 if __name__ == "__main__":

scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json

@@ -1 +0,0 @@
-[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}]

scripts/perf/tests/benchmark_results_comment/test.sh

@@ -9,12 +9,9 @@ SCRIPT_PATH=$(dirname $SCRIPT)
 cd $SCRIPT_PATH 
 python3 ../../benchmark_results_comment.py --new no_mpk/results*.json -o no_mpk/actual.md
 python3 ../../benchmark_results_comment.py --previous normal/results*.mpk --new normal/results*.json -o normal/actual.md
-python3 ../../benchmark_results_comment.py --previous no_diff/results*.mpk --new no_diff/results*.json -o no_diff/actual.md
 cd $ORIGINAL_PWD
 
 
 cmp $SCRIPT_PATH/no_mpk/expected.md $SCRIPT_PATH/no_mpk/actual.md
 cmp $SCRIPT_PATH/normal/expected.md $SCRIPT_PATH/normal/actual.md
-cmp $SCRIPT_PATH/no_diff/expected.md $SCRIPT_PATH/no_diff/actual.md 
 
-echo "[TEST] Passed"