ci: disable benchmark results comment if there's no diff (#8515)

scripts/perf/benchmark_results_comment.py

@@ -109,19 +109,24 @@ comparing different LVGL features and configurations, but may not correlate dire
 physical hardware. The measurements are intended for comparative analysis only.
 """
 
+DATA_KEYS = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"]
 
-def format_table(results: list[dict], prev_results: list[dict]):
+
+def generate_table(results: list[dict], prev_results: list[dict]) -> list[dict]:
+    return [
+        {key: ((scene_results[key] - prev_scene_results[key])) for key in DATA_KEYS}
+        for scene_results, prev_scene_results in zip(results, prev_results)
+    ]
+
+
+def format_table(results: list[dict], delta: list[dict]) -> str:
 
     table = "| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) |\n"
     table += "|------------|------------|---------|--------------|----------------|--------------|\n"
-    data_keys = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"]
 
-    for scene_results, prev_scene_results in zip(results, prev_results):
-        delta_p = {
-            key: ((scene_results[key] - prev_scene_results[key])) for key in data_keys
-        }
+    for scene_results, delta_p in zip(results, delta):
         table += f"| {scene_results['scene_name']} |"
-        for key in data_keys:
+        for key in DATA_KEYS:
             if delta_p[key] == 0:
                 table += f" {scene_results[key]} |"
             else:
@@ -163,6 +168,9 @@ def main():
     results_paths = args.new
     output_path = args.output
 
+    print(f"Previous results: {previous_results_paths}")
+    print(f"New results: {results_paths}")
+
     previous_results_map: dict[str, list[dict]] = {}
     if previous_results_paths:
         for results_path in previous_results_paths:
@@ -183,6 +191,7 @@ def main():
     comment += "We've run benchmarks in an emulated environment."
     comment += " Here are the results:\n\n"
 
+    diff_found = False
     for result_path, result in new_results.items():
         mpk_path = result_path.replace(".json", ".mpk")
 
@@ -193,6 +202,7 @@ def main():
         prev_results = previous_results_map.get(mpk_path, [])
 
         if len(prev_results) > 0:
+            print(f"Found previous results for {result_path} in {mpk_path}")
             prev_scenes = prev_results[-1]["scenes"]
             prev_all_scene_avg = [
                 [
@@ -207,17 +217,26 @@ def main():
             # the previous aswell
             # In this case, the difference will always be zero and we won't
             # add any new information to the result table
+            print(f"Previous results not found for {result_path}")
+            diff_found = True  # Generate a real comment since we got new results
             prev_results = result
             prev_all_scene_avg = new_all_scene_avg
 
+        delta_all_scene_avg = generate_table(new_all_scene_avg, prev_all_scene_avg)
+        delta_results_avg = generate_table(result, prev_results)
+
+        diff_found = diff_found or any(
+            value != 0 for dic in delta_results_avg for value in dic.values()
+        )
+
         _, image_type, config = result_path.replace(".json", "").split("-")
         comment += f"#### ARM Emulated {image_type} - {config}\n\n"
-        comment += format_table(new_all_scene_avg, prev_all_scene_avg)
+        comment += format_table(new_all_scene_avg, delta_all_scene_avg)
         comment += "\n<details>"
         comment += "\n<summary>"
         comment += "\nDetailed Results Per Scene"
         comment += "\n</summary>\n\n"
-        comment += format_table(result, prev_results)
+        comment += format_table(result, delta_results_avg)
         comment += "\n\n</details>\n\n"
 
     comment += DISCLAIMER
@@ -226,8 +245,10 @@ def main():
     comment += "\n\n"
     comment += ":robot: This comment was automatically generated by a bot."
 
+    # Create the file but don't write anything to it if we haven't found a difference
     with open(output_path, "w") as f:
-        f.write(comment)
+        if diff_found:
+            f.write(comment)
 
 
 if __name__ == "__main__":

scripts/perf/tests/benchmark_results_comment/no_diff/results-32b-lv_conf_perf32b.json

@@ -0,0 +1 @@
+[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}]

scripts/perf/tests/benchmark_results_comment/test.sh

@@ -9,9 +9,12 @@ SCRIPT_PATH=$(dirname $SCRIPT)
 cd $SCRIPT_PATH 
 python3 ../../benchmark_results_comment.py --new no_mpk/results*.json -o no_mpk/actual.md
 python3 ../../benchmark_results_comment.py --previous normal/results*.mpk --new normal/results*.json -o normal/actual.md
+python3 ../../benchmark_results_comment.py --previous no_diff/results*.mpk --new no_diff/results*.json -o no_diff/actual.md
 cd $ORIGINAL_PWD
 
 
 cmp $SCRIPT_PATH/no_mpk/expected.md $SCRIPT_PATH/no_mpk/actual.md
 cmp $SCRIPT_PATH/normal/expected.md $SCRIPT_PATH/normal/actual.md
+cmp $SCRIPT_PATH/no_diff/expected.md $SCRIPT_PATH/no_diff/actual.md 
 
+echo "[TEST] Passed"