From 47977f33d1ba480715eb7924e85f40b930b6d8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Costa?= Date: Thu, 26 Jun 2025 08:09:23 +0200 Subject: [PATCH] ci: emulated perf workflow (#7949) --- .github/workflows/perf_emulation.yml | 103 +++++++ .../perf_emulation_handle_results.yml | 133 ++++++++ configs/ci/perf/lv_conf_perf32b.defaults | 83 +++++ configs/ci/perf/lv_conf_perf64b.defaults | 83 +++++ scripts/perf/benchmark_results_comment.py | 234 ++++++++++++++ scripts/perf/filter_docker_benchmark_logs.py | 101 ++++++ scripts/perf/lvperf_dependencies.sh | 9 + scripts/perf/serialize_results.py | 151 +++++++++ .../no_mpk/expected.md | 83 +++++ .../no_mpk/results-32b-lv_conf_perf32b.json | 1 + .../no_mpk/results-64b-lv_conf_perf64b.json | 1 + .../normal/expected.md | 83 +++++ .../normal/results-32b-lv_conf_perf32b.json | 1 + .../normal/results-32b-lv_conf_perf32b.mpk | Bin 0 -> 2965 bytes .../normal/results-64b-lv_conf_perf64b.json | 1 + .../normal/results-64b-lv_conf_perf64b.mpk | Bin 0 -> 2965 bytes .../tests/benchmark_results_comment/test.sh | 17 ++ .../docker-logs-example.txt | 205 +++++++++++++ .../tests/filter_docker_logs/expected.json | 1 + scripts/perf/tests/filter_docker_logs/test.sh | 15 + .../results-32b-lv_conf_perf32b.mpk | Bin 0 -> 4415 bytes .../results-32b-lv_conf_perf32b_a.mpk | Bin 0 -> 1451 bytes .../results-32b-lv_conf_perf32b_b.mpk | Bin 0 -> 2965 bytes .../results-64b-lv_conf_perf64b.mpk | Bin 0 -> 4415 bytes .../input/results-32b-lv_conf_perf32b.json | 1 + .../input/results-32b-lv_conf_perf32b.mpk | Bin 0 -> 2965 bytes .../input/results-32b-lv_conf_perf32b_a.json | 1 + .../input/results-32b-lv_conf_perf32b_b.mpk | Bin 0 -> 2965 bytes .../input/results-64b-lv_conf_perf64b.json | 1 + .../input/results-64b-lv_conf_perf64b.mpk | Bin 0 -> 2965 bytes scripts/perf/tests/serialize_results/test.sh | 43 +++ tests/README.md | 116 ++++--- tests/benchmark_emu.py | 289 ++++++++++++++++++ 33 files changed, 1720 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/perf_emulation.yml create mode 100644 .github/workflows/perf_emulation_handle_results.yml create mode 100644 configs/ci/perf/lv_conf_perf32b.defaults create mode 100644 configs/ci/perf/lv_conf_perf64b.defaults create mode 100644 scripts/perf/benchmark_results_comment.py create mode 100644 scripts/perf/filter_docker_benchmark_logs.py create mode 100755 scripts/perf/lvperf_dependencies.sh create mode 100644 scripts/perf/serialize_results.py create mode 100644 scripts/perf/tests/benchmark_results_comment/no_mpk/expected.md create mode 100644 scripts/perf/tests/benchmark_results_comment/no_mpk/results-32b-lv_conf_perf32b.json create mode 100644 scripts/perf/tests/benchmark_results_comment/no_mpk/results-64b-lv_conf_perf64b.json create mode 100644 scripts/perf/tests/benchmark_results_comment/normal/expected.md create mode 100644 scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.json create mode 100644 scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.mpk create mode 100644 scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.json create mode 100644 scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.mpk create mode 100755 scripts/perf/tests/benchmark_results_comment/test.sh create mode 100644 scripts/perf/tests/filter_docker_logs/docker-logs-example.txt create mode 100644 scripts/perf/tests/filter_docker_logs/expected.json create mode 100755 scripts/perf/tests/filter_docker_logs/test.sh create mode 100644 scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b.mpk create mode 100644 scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b_a.mpk create mode 100644 scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b_b.mpk create mode 100644 scripts/perf/tests/serialize_results/expected_output/results-64b-lv_conf_perf64b.mpk create mode 100644 scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b.json create mode 100644 scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b.mpk create mode 100644 scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_a.json create mode 100644 scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_b.mpk create mode 100644 scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.json create mode 100644 scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.mpk create mode 100755 scripts/perf/tests/serialize_results/test.sh create mode 100755 tests/benchmark_emu.py diff --git a/.github/workflows/perf_emulation.yml b/.github/workflows/perf_emulation.yml new file mode 100644 index 0000000000..bae459c49c --- /dev/null +++ b/.github/workflows/perf_emulation.yml @@ -0,0 +1,103 @@ +name: Emulated Performance Test + +on: + push: + branches: 'master' + pull_request: + branches: 'master' + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + check_scripts: + runs-on: ubuntu-24.04 + name: ARM Emulated Benchmark - Script Check + strategy: + fail-fast: false + matrix: + test_script: + - scripts/perf/tests/filter_docker_logs/test.sh + # These scripts aren't executed in this workflow directly. Instead, they're run + # by the `Emulated Performance Test Results Handler` workflow. + # That workflow runs in the context of the `master` branch, so this workflow ensures + # the scripts are tested in the PR context before merging. Helping to prevent CI failures in `master` + - scripts/perf/tests/benchmark_results_comment/test.sh + - scripts/perf/tests/serialize_results/test.sh + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check Script + run: | + pip3 install msgpack==1.1.0 + ./${{ matrix.test_script }} + + run_benchmark: + runs-on: ubuntu-24.04 + name: ARM Emulated Benchmark ${{ matrix.image_type }} - ${{matrix.config }} + needs: check_scripts + strategy: + fail-fast: false + matrix: + include: + - image_type: 32b + image_version : | + main@sha256:9c4587f5852be856aed4548fcb8b505290410de837bb87fb2ba8e1b778f10bec + config : lv_conf_perf32b + - image_type: 64b + image_version: | + main@sha256:7e4c0a883dec21e2e9c027fe21188b2b009e46a1b3eb9f02499ca02ed7914e4a + config : lv_conf_perf64b + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Generate lv_conf.h + run: | + cp lv_conf_template.h configs/ci/perf/lv_conf_perf.h + python scripts/generate_lv_conf.py \ + --template lv_conf_template.h \ + --config configs/ci/perf/lv_conf_perf.h \ + --defaults configs/ci/perf/${{matrix.config}}.defaults + - name: Run Benchmark Demo + run: | + # Mounts the current lvgl source code into the container and runs the demo benchmark + # To add additional build dependencies, modify the `lvperf_dependencies.sh` script + docker run -t --privileged \ + --name so3-lvperf \ + -v /dev:/dev \ + -v $(pwd)/configs/ci/perf/lv_conf_perf.h:/so3/usr/lib/lv_conf.h \ + -v $(pwd):/so3/usr/lib/lvgl \ + -v $(pwd)/scripts/perf/lvperf_dependencies.sh:/so3/install_dependencies.sh \ + ghcr.io/smartobjectoriented/so3-lvperf${{matrix.image_type}}:${{matrix.image_version}} + + - name: Collect Logs + run: | + sudo cat $(docker inspect --format='{{.LogPath}}' so3-lvperf) | python scripts/perf/filter_docker_benchmark_logs.py results-${{matrix.image_type}}-${{matrix.config}}.json + + - name: Upload Results + uses: actions/upload-artifact@v4 + with: + name: results-${{matrix.image_type}}-${{matrix.config}} + path: results-${{matrix.image_type}}-${{matrix.config}}.json + if-no-files-found: error + + save_pr_number: + runs-on: ubuntu-24.04 + if: ${{ github.event_name == 'pull_request' }} + name: ARM Emulated Benchmark - Save PR Number + needs: run_benchmark + steps: + - name: Save PR number + run: | + echo ${{ github.event.number }} > pr_number + + - name: Upload PR number as artifact + uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number diff --git a/.github/workflows/perf_emulation_handle_results.yml b/.github/workflows/perf_emulation_handle_results.yml new file mode 100644 index 0000000000..67b7006049 --- /dev/null +++ b/.github/workflows/perf_emulation_handle_results.yml @@ -0,0 +1,133 @@ +name: Emulated Performance Test Results Handler + +on: + workflow_run: + workflows: [Emulated Performance Test] + types: + - completed + +concurrency: + group: ${{ github.event.workflow_run.event }}-${{ github.event.workflow_run.head_branch }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + +jobs: + check_scripts: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-24.04 + name: ARM Emulated Benchmark - Script Check + strategy: + fail-fast: false + matrix: + test_script: + - scripts/perf/tests/benchmark_results_comment/test.sh + - scripts/perf/tests/serialize_results/test.sh + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check Script + run: | + pip3 install msgpack==1.1.0 + ./${{ matrix.test_script }} + + handle_results: + needs: check_scripts + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-24.04 + name: ARM Emulated Benchmark - Handle Results + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download Results + uses: dawidd6/action-download-artifact@v9 + with: + workflow: perf_emulation.yml + path: artifacts + + - name: Move JSON files to a single folder + run: | + mkdir input + find artifacts -name "*.json" -exec mv {} input/ \; + + - name: Collect 'master' Results + uses: robinraju/release-downloader@v1 + continue-on-error: true # The release may not exist yet + with: + preRelease: true + tag: emulated-benchmark-latest + fileName: results*.mpk + + - name: Move PR data files to current folder + if: ${{ github.event.workflow_run.event == 'pull_request' }} + run: | + mv artifacts/pr_number/pr_number . + + - name: Prepare Comment + if: ${{ github.event.workflow_run.event == 'pull_request' }} + run: | + pip3 install msgpack==1.1.0 + if ls results*.mpk 1> /dev/null 2>&1; then + python3 scripts/perf/benchmark_results_comment.py \ + --previous results*.mpk \ + --new input/results*.json \ + --output comment.md + else + echo "No previous results found, generating comment without comparison." + python3 scripts/perf/benchmark_results_comment.py \ + --new input/results*.json \ + --output comment.md + fi + + - name: Comment PR + if: ${{ github.event.workflow_run.event == 'pull_request' }} + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const commentPath = 'comment.md'; + const prPath = 'pr_number'; + + if (!fs.existsSync(commentPath)) { + throw new Error('Error: comment.md not found! Exiting.'); + } + + if (!fs.existsSync(prPath)) { + throw new Error('Error: pr_number not found! Exiting.'); + } + + const commentBody = fs.readFileSync(commentPath, 'utf8').trim(); + const prNumber = Number(fs.readFileSync(prPath, 'utf8').trim()); + + github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody + }); + + - name: Serialize Results + if: ${{ github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'master' }} + run: | + # Here the input folder already exists from a previous step + pip3 install msgpack==1.1.0 + mkdir output + find . -maxdepth 1 \( -name "results*.mpk" \) -exec mv -t input {} + + python scripts/perf/serialize_results.py --input input --output output --commit-hash ${{ github.sha }} + + - name: Store Results in Benchmark Release + if: ${{ github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'master' }} + uses: softprops/action-gh-release@v2 + with: + name: Emulated Benchmark Latest + files: output/results*.mpk + tag_name: emulated-benchmark-latest + prerelease: true + diff --git a/configs/ci/perf/lv_conf_perf32b.defaults b/configs/ci/perf/lv_conf_perf32b.defaults new file mode 100644 index 0000000000..b5543a5d3a --- /dev/null +++ b/configs/ci/perf/lv_conf_perf32b.defaults @@ -0,0 +1,83 @@ +LV_COLOR_DEPTH 32 +LV_USE_STDLIB_MALLOC LV_STDLIB_BUILTIN +LV_USE_STDLIB_STRING LV_STDLIB_BUILTIN +LV_USE_STDLIB_SPRINTF LV_STDLIB_BUILTIN +LV_MEM_SIZE (128 * 1024U) +LV_DEF_REFR_PERIOD 30 +LV_USE_LOG 1 +LV_LOG_PRINTF 1 +LV_USE_ASSERT_NULL 1 +LV_USE_ASSERT_MALLOC 1 +LV_FONT_MONTSERRAT_8 1 +LV_FONT_MONTSERRAT_10 1 +LV_FONT_MONTSERRAT_12 1 +LV_FONT_MONTSERRAT_14 1 +LV_FONT_MONTSERRAT_16 1 +LV_FONT_MONTSERRAT_18 1 +LV_FONT_MONTSERRAT_20 1 +LV_FONT_MONTSERRAT_22 1 +LV_FONT_MONTSERRAT_24 1 +LV_FONT_MONTSERRAT_26 1 +LV_FONT_MONTSERRAT_28 1 +LV_FONT_MONTSERRAT_30 1 +LV_FONT_MONTSERRAT_32 1 +LV_FONT_MONTSERRAT_34 1 +LV_FONT_MONTSERRAT_36 1 +LV_FONT_MONTSERRAT_38 1 +LV_FONT_MONTSERRAT_40 1 +LV_FONT_MONTSERRAT_42 1 +LV_FONT_MONTSERRAT_44 1 +LV_FONT_MONTSERRAT_46 1 +LV_FONT_MONTSERRAT_48 1 +LV_WIDGETS_HAS_DEFAULT_VALUE 1 +LV_USE_ANIMIMG 1 +LV_USE_ARC 1 +LV_USE_BAR 1 +LV_USE_BUTTON 1 +LV_USE_BUTTONMATRIX 1 +LV_USE_CALENDAR 1 +LV_USE_CANVAS 1 +LV_USE_CHART 1 +LV_USE_CHECKBOX 1 +LV_USE_DROPDOWN 1 +LV_USE_IMAGE 1 +LV_USE_IMAGEBUTTON 1 +LV_USE_KEYBOARD 1 +LV_USE_LABEL 1 +LV_LABEL_TEXT_SELECTION 1 +LV_LABEL_LONG_TXT_HINT 1 +LV_LABEL_WAIT_CHAR_COUNT 3 +LV_USE_LED 1 +LV_USE_LINE 1 +LV_USE_LIST 1 +LV_USE_MENU 1 +LV_USE_MSGBOX 1 +LV_USE_ROLLER 1 +LV_USE_SCALE 1 +LV_USE_SLIDER 1 +LV_USE_SPAN 1 +LV_SPAN_SNIPPET_STACK_SIZE 64 +LV_USE_SPINBOX 1 +LV_USE_SPINNER 1 +LV_USE_SWITCH 1 +LV_USE_TABLE 1 +LV_USE_TABVIEW 1 +LV_USE_TEXTAREA 1 +LV_TEXTAREA_DEF_PWD_SHOW_TIME 1500 +LV_USE_TILEVIEW 1 +LV_USE_WIN 1 +LV_USE_THEME_DEFAULT 1 +LV_THEME_DEFAULT_DARK 0 +LV_THEME_DEFAULT_GROW 1 +LV_THEME_DEFAULT_TRANSITION_TIME 80 +LV_USE_THEME_SIMPLE 1 +LV_USE_THEME_MONO 1 +LV_USE_FLEX 1 +LV_USE_GRID 1 +LV_USE_SYSMON 1 +LV_USE_PERF_MONITOR 1 +LV_USE_MEM_MONITOR 1 +LV_USE_OBSERVER 1 +LV_USE_DEMO_WIDGETS 1 +LV_USE_DEMO_BENCHMARK 1 +LV_USE_DEMO_STRESS 1 diff --git a/configs/ci/perf/lv_conf_perf64b.defaults b/configs/ci/perf/lv_conf_perf64b.defaults new file mode 100644 index 0000000000..b5543a5d3a --- /dev/null +++ b/configs/ci/perf/lv_conf_perf64b.defaults @@ -0,0 +1,83 @@ +LV_COLOR_DEPTH 32 +LV_USE_STDLIB_MALLOC LV_STDLIB_BUILTIN +LV_USE_STDLIB_STRING LV_STDLIB_BUILTIN +LV_USE_STDLIB_SPRINTF LV_STDLIB_BUILTIN +LV_MEM_SIZE (128 * 1024U) +LV_DEF_REFR_PERIOD 30 +LV_USE_LOG 1 +LV_LOG_PRINTF 1 +LV_USE_ASSERT_NULL 1 +LV_USE_ASSERT_MALLOC 1 +LV_FONT_MONTSERRAT_8 1 +LV_FONT_MONTSERRAT_10 1 +LV_FONT_MONTSERRAT_12 1 +LV_FONT_MONTSERRAT_14 1 +LV_FONT_MONTSERRAT_16 1 +LV_FONT_MONTSERRAT_18 1 +LV_FONT_MONTSERRAT_20 1 +LV_FONT_MONTSERRAT_22 1 +LV_FONT_MONTSERRAT_24 1 +LV_FONT_MONTSERRAT_26 1 +LV_FONT_MONTSERRAT_28 1 +LV_FONT_MONTSERRAT_30 1 +LV_FONT_MONTSERRAT_32 1 +LV_FONT_MONTSERRAT_34 1 +LV_FONT_MONTSERRAT_36 1 +LV_FONT_MONTSERRAT_38 1 +LV_FONT_MONTSERRAT_40 1 +LV_FONT_MONTSERRAT_42 1 +LV_FONT_MONTSERRAT_44 1 +LV_FONT_MONTSERRAT_46 1 +LV_FONT_MONTSERRAT_48 1 +LV_WIDGETS_HAS_DEFAULT_VALUE 1 +LV_USE_ANIMIMG 1 +LV_USE_ARC 1 +LV_USE_BAR 1 +LV_USE_BUTTON 1 +LV_USE_BUTTONMATRIX 1 +LV_USE_CALENDAR 1 +LV_USE_CANVAS 1 +LV_USE_CHART 1 +LV_USE_CHECKBOX 1 +LV_USE_DROPDOWN 1 +LV_USE_IMAGE 1 +LV_USE_IMAGEBUTTON 1 +LV_USE_KEYBOARD 1 +LV_USE_LABEL 1 +LV_LABEL_TEXT_SELECTION 1 +LV_LABEL_LONG_TXT_HINT 1 +LV_LABEL_WAIT_CHAR_COUNT 3 +LV_USE_LED 1 +LV_USE_LINE 1 +LV_USE_LIST 1 +LV_USE_MENU 1 +LV_USE_MSGBOX 1 +LV_USE_ROLLER 1 +LV_USE_SCALE 1 +LV_USE_SLIDER 1 +LV_USE_SPAN 1 +LV_SPAN_SNIPPET_STACK_SIZE 64 +LV_USE_SPINBOX 1 +LV_USE_SPINNER 1 +LV_USE_SWITCH 1 +LV_USE_TABLE 1 +LV_USE_TABVIEW 1 +LV_USE_TEXTAREA 1 +LV_TEXTAREA_DEF_PWD_SHOW_TIME 1500 +LV_USE_TILEVIEW 1 +LV_USE_WIN 1 +LV_USE_THEME_DEFAULT 1 +LV_THEME_DEFAULT_DARK 0 +LV_THEME_DEFAULT_GROW 1 +LV_THEME_DEFAULT_TRANSITION_TIME 80 +LV_USE_THEME_SIMPLE 1 +LV_USE_THEME_MONO 1 +LV_USE_FLEX 1 +LV_USE_GRID 1 +LV_USE_SYSMON 1 +LV_USE_PERF_MONITOR 1 +LV_USE_MEM_MONITOR 1 +LV_USE_OBSERVER 1 +LV_USE_DEMO_WIDGETS 1 +LV_USE_DEMO_BENCHMARK 1 +LV_USE_DEMO_STRESS 1 diff --git a/scripts/perf/benchmark_results_comment.py b/scripts/perf/benchmark_results_comment.py new file mode 100644 index 0000000000..f76d46457c --- /dev/null +++ b/scripts/perf/benchmark_results_comment.py @@ -0,0 +1,234 @@ +""" +This script takes json and mpk input files and creates a PR comment in markdown format. + +An mpk (msgpack) file contains the benchmark result history for a specific config. +Unpacked, this file will have the following format: +```json +[ + { + "commit_hash": "" + "scenes": [ + { + "scene_name": "", + "avg_cpu": 0, + "avg_fps": 0, + "avg_time": 0, + "render_time": 0, + "flush_time": 0, + }, + ... + ] + }, + ... +] +``` + +A json file contains the benchmark result for a specific commit: + +```json +[ + { + "scene_name": "", + "avg_cpu": 0, + "avg_fps": 0, + "avg_time": 0, + "render_time": 0, + "flush_time": 0, + } + ... +] +``` + +This script reads the previous and new benchmark results, compares them and creates a comment. + +Example comment: + +``` +Hi :wave:, thank you for your PR! + +We've run benchmarks in an emulated environment. Here are the results: + +#### ARM Emulated 32b - lv_conf_perf32b + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| All scenes avg. | 20 | 24 | 7 | 7 | 0 | + +
+ +Detailed Results Per Scene + + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| Empty screen | 11 | 25 | 0 | 0 | 0 | +| Moving wallpaper | 1 | 25 | 0 | 0 | 0 | +| Single rectangle | 0 | 25 | 0 | 0 | 0 | +| Multiple rectangles | 0 | 25 | 0 | 0 | 0 | +| Multiple RGB images | 0 | 25 | 0 | 0 | 0 | +| Multiple ARGB images | 22 (-1)| 25 | 1 | 1 | 0 | +| Rotated ARGB images | 47 (-1)| 24 | 20 | 20 | 0 | +| Multiple labels | 2 (-2)| 25 | 0 | 0 | 0 | +| Screen sized text | 30 (+1)| 24 (-1)| 11 (-1)| 11 (-1)| 0 | +| Multiple arcs | 19 (+4)| 24 | 7 | 7 | 0 | +| Containers | 1 (-1)| 25 | 0 | 0 | 0 | +| Containers with overlay | 87 (-2)| 21 | 44 | 44 | 0 | +| Containers with opa | 23 (+1)| 25 | 4 | 4 | 0 | +| Containers with opa_layer | 22 (+1)| 25 | 8 | 8 | 0 | +| Containers with scrolling | 25 | 25 | 10 | 10 | 0 | +| Widgets demo | 34 | 24 (-1)| 13 | 13 | 0 | +| All scenes avg. | 20 | 24 | 7 | 7 | 0 | + + +
+ + +Disclaimer: These benchmarks were run in an emulated environment using QEMU with instruction counting mode. +The timing values represent relative performance metrics within this specific virtualized setup and should +not be interpreted as absolute real-world performance measurements. Values are deterministic and useful for +comparing different LVGL features and configurations, but may not correlate directly with performance on +physical hardware. The measurements are intended for comparative analysis only. + + +--- + +:robot: This comment was automatically generated by a bot. +``` +""" + +import argparse +import json +import msgpack + + +DISCLAIMER = """ +Disclaimer: These benchmarks were run in an emulated environment using QEMU with instruction counting mode. +The timing values represent relative performance metrics within this specific virtualized setup and should +not be interpreted as absolute real-world performance measurements. Values are deterministic and useful for +comparing different LVGL features and configurations, but may not correlate directly with performance on +physical hardware. The measurements are intended for comparative analysis only. +""" + + +def format_table(results: list[dict], prev_results: list[dict]): + + table = "| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) |\n" + table += "|------------|------------|---------|--------------|----------------|--------------|\n" + data_keys = ["avg_cpu", "avg_fps", "avg_time", "render_time", "flush_time"] + + for scene_results, prev_scene_results in zip(results, prev_results): + delta_p = { + key: ((scene_results[key] - prev_scene_results[key])) for key in data_keys + } + table += f"| {scene_results['scene_name']} |" + for key in data_keys: + if delta_p[key] == 0: + table += f" {scene_results[key]} |" + else: + table += f" {scene_results[key]} ({delta_p[key]:+})|" + table += "\n" + + return table + + +def main(): + parser = argparse.ArgumentParser( + description="Process previous and new results, and output a comment file." + ) + + parser.add_argument( + "--previous", + type=str, + nargs="+", + required=False, + help="Path to the previous results file (supports multiple, e.g., results*.mpk)", + ) + parser.add_argument( + "--new", + type=str, + nargs="+", + required=True, + help="Paths to new results files (supports multiple, e.g., results*.json)", + ) + parser.add_argument( + "-o", + "--output", + type=str, + required=True, + help="Output file path (e.g., comment.md)", + ) + + args = parser.parse_args() + previous_results_paths = args.previous + results_paths = args.new + output_path = args.output + + previous_results_map: dict[str, list[dict]] = {} + if previous_results_paths: + for results_path in previous_results_paths: + _, image_type, config = results_path.replace(".mpk", "").split("-") + + with open(results_path, "rb") as f: + previousb = f.read() + rs: list = msgpack.unpackb(previousb) + previous_results_map[results_path] = rs + + new_results: dict[str, list[dict]] = {} + for results_path in results_paths: + with open(results_path, "r") as f: + r: list[dict] = json.load(f) + new_results[results_path] = r + + comment = "Hi :wave:, thank you for your PR!\n\n" + comment += "We've run benchmarks in an emulated environment." + comment += " Here are the results:\n\n" + + for result_path, result in new_results.items(): + mpk_path = result_path.replace(".json", ".mpk") + + new_all_scene_avg = [ + scene for scene in result if scene["scene_name"] == "All scenes avg." + ] + + prev_results = previous_results_map.get(mpk_path, []) + + if len(prev_results) > 0: + prev_scenes = prev_results[-1]["scenes"] + prev_all_scene_avg = [ + [ + scene + for scene in prev_scenes + if scene["scene_name"] == "All scenes avg." + ][0] + ] + prev_results = prev_scenes + else: + # If there are no previous results, we use the current result as + # the previous aswell + # In this case, the difference will always be zero and we won't + # add any new information to the result table + prev_results = result + prev_all_scene_avg = new_all_scene_avg + + _, image_type, config = result_path.replace(".json", "").split("-") + comment += f"#### ARM Emulated {image_type} - {config}\n\n" + comment += format_table(new_all_scene_avg, prev_all_scene_avg) + comment += "\n
" + comment += "\n" + comment += "\nDetailed Results Per Scene" + comment += "\n\n\n" + comment += format_table(result, prev_results) + comment += "\n\n
\n\n" + + comment += DISCLAIMER + comment += "\n\n" + comment += "---" + comment += "\n\n" + comment += ":robot: This comment was automatically generated by a bot." + + with open(output_path, "w") as f: + f.write(comment) + + +if __name__ == "__main__": + main() diff --git a/scripts/perf/filter_docker_benchmark_logs.py b/scripts/perf/filter_docker_benchmark_logs.py new file mode 100644 index 0000000000..85bf15bbf2 --- /dev/null +++ b/scripts/perf/filter_docker_benchmark_logs.py @@ -0,0 +1,101 @@ +""" +This script parses a docker logs file for the lv_benchmark summary results + +The input looks something like: +``` +{"log":"SO3: starting the initial process ...\r\n","stream":"stdout","time":"2025-03-14T20:32:17.712134853Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.712146014Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.712150624Z"} +{"log":"Starting LVGL Benchmark\r\n","stream":"stdout","time":"2025-03-14T20:32:17.816299617Z"} +{"log":"Warning: The guest is now late by 0.0 to 1.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:32:19.129030996Z"} +{"log":"Warning: The guest is now late by 1.0 to 2.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:32:40.699491404Z"} +{"log":"Warning: The guest is now late by 2.0 to 3.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:02.966355127Z"} +{"log":"Warning: The guest is now late by 3.0 to 4.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:30.06251689Z"} +{"log":"Warning: The guest is now late by 4.0 to 5.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:38.253535255Z"} +{"log":"Warning: The guest is now late by 5.0 to 6.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:51.010250882Z"} +{"log":"Benchmark Summary (9.3.0 dev)\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.855785347Z"} +{"log":"Name, Avg. CPU, Avg. FPS, Avg. time, render time, flush time\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.855997881Z"} +{"log":"Empty screen, 11%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.857828373Z"} +{"log":"Moving wallpaper, 2%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.858455924Z"} +{"log":"Single rectangle, 0%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.859125276Z"} +{"log":"Multiple rectangles, 0%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.859847899Z"} +{"log":"Multiple RGB images, 0%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.860658884Z"} +{"log":"Multiple ARGB images, 23%, 25, 1, 1, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.86156999Z"} +{"log":"Rotated ARGB images, 48%, 24, 20, 20, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.862511197Z"} +{"log":"Multiple labels, 3%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.863549175Z"} +{"log":"Screen sized text, 30%, 24, 11, 11, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.864642275Z"} +{"log":"Multiple arcs, 18%, 24, 7, 7, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.865801475Z"} +{"log":"Containers, 3%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.867039657Z"} +{"log":"Containers with overlay, 88%, 21, 44, 44, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.868379231Z"} +{"log":"Containers with opa, 10%, 24, 3, 3, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.869736765Z"} +{"log":"Containers with opa_layer, 22%, 24, 8, 8, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.871269383Z"} +{"log":"Containers with scrolling, 25%, 25, 10, 10, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.872899952Z"} +{"log":"Widgets demo, 34%, 25, 13, 13, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.874447759Z"} +{"log":"All scenes avg.,19%, 24, 7, 7, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.875031879Z"} +{"log":"LVGL Benchmark Over\r\n","stream":"stdout","time":"2025-03-14T20:33:57.933458479Z"} +```` + +Outpus a json file with the format: +```json +[ + { + "scene_name": "", + "avg_cpu": 0, + "avg_fps": 0, + "avg_time": 0, + "render_time": 0, + "flush_time": 0, + } + ... +] +``` +""" + +import sys +import json + + +def main(): + if len(sys.argv) < 2: + print("Missing output path") + return + + output_path = sys.argv[1] + found_start_of_results = False + found_header = False + results = [] + for line in sys.stdin: + if "Benchmark Summary" in line: + found_start_of_results = True + continue + if not found_start_of_results: + continue + + fmt_line: str = json.loads(line)["log"].strip() + cols = fmt_line.split(",") + if len(cols) < 6: + if fmt_line != "LVGL Benchmark Over": + print(f"Warning found invalid log line '{fmt_line}'. Skipping...") + continue + + if not found_header: + found_header = True + continue + + results.append( + { + "scene_name": cols[0], + "avg_cpu": int(cols[1].replace("%", "")), + "avg_fps": int(cols[2]), + "avg_time": int(cols[3]), + "render_time": int(cols[4]), + "flush_time": int(cols[5]), + } + ) + + with open(output_path, "w") as f: + json.dump(results, f) + + +if __name__ == "__main__": + main() diff --git a/scripts/perf/lvperf_dependencies.sh b/scripts/perf/lvperf_dependencies.sh new file mode 100755 index 0000000000..2019c84524 --- /dev/null +++ b/scripts/perf/lvperf_dependencies.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# This script installs extra dependencies for the lvperf Docker images used in the CI pipeline. +# The images are based on Alpine Linux and support runtime installation of dependencies, +# allowing you to extend functionality without rebuilding the image. +# +# For guidance on how dependencies are typically added, refer to the Dockerfiles: +# - https://github.com/smartobjectoriented/so3/blob/main/docker/Dockerfile.lvperf_32b +# - https://github.com/smartobjectoriented/so3/blob/main/docker/Dockerfile.lvperf_64b diff --git a/scripts/perf/serialize_results.py b/scripts/perf/serialize_results.py new file mode 100644 index 0000000000..85fd9d3d27 --- /dev/null +++ b/scripts/perf/serialize_results.py @@ -0,0 +1,151 @@ +""" +This script takes json and mpk input files and combines them + +An mpk (msgpack) file contains the benchmark result history for a specific config. +Unpacked, this file will have the following format: +```json +[ + { + "commit_hash": "" + "scenes": [ + { + "scene_name": "", + "avg_cpu": 0, + "avg_fps": 0, + "avg_time": 0, + "render_time": 0, + "flush_time": 0, + } + ... + ] + }, + ... +] +``` + +A json file contains the benchmark result for a specific commit: + +```json +[ + { + "scene_name": "", + "avg_cpu": 0, + "avg_fps": 0, + "avg_time": 0, + "render_time": 0, + "flush_time": 0, + } + ... +] +``` + +This script combines the json files with the mpk files and outputs new mpk files + +The files are expected to follow the format 'results--.[mpk|json]' + +There are 3 possibilities when handling these files + +1. The pair (`file1.mpk` `file1.json`) exists: Normal operation, we have a history file and some new results. + In this case we take the information inside the `file1.json` and append it to the `file1.mpk` before exporting it + +2. `file1.json` exists but `file1.mpk` doesn't: Happens when a new config and/or image type are added and there's no history yet + In this case we create a new `file1.mpk` with a single history entry. + +3. `file1.mpk` exists but `file1.json` doesn't: Happens when an old config isn't tested anymore for any reason. + In this case we simpyl copy the old `file1.mpk` +""" + +import argparse +import os +import shutil +import msgpack +import json + + +def save_mpk(mpk_path, mpk_data): + + with open(mpk_path, "wb") as f: + f.write(mpk_data) + + +def generate_new_mpk(json_path, output_path, commit_hash): + + with open(json_path, "r") as f: + json_data: list = json.load(f) + + mpk_data = msgpack.packb([{"commit_hash": commit_hash, "scenes": json_data}]) + save_mpk(output_path, mpk_data) + + +def append_json_to_mpk(mpk_path, json_path, output_path, commit_hash): + with open(mpk_path, "rb") as f: + mpk_data: list = msgpack.unpackb(f.read()) + + with open(json_path, "r") as f: + json_data = json.load(f) + + mpk_data.append({"commit_hash": commit_hash, "scenes": json_data}) + + save_mpk(output_path, msgpack.packb(mpk_data)) + + +def path(folder, filename): + return os.path.join(folder, filename) + + +def main(): + parser = argparse.ArgumentParser( + description="Merge new results into previous results" + ) + + parser.add_argument("--commit-hash", type=str, required=True, help="Commit Hash") + parser.add_argument( + "--input", + type=str, + required=True, + help="Input Folder", + ) + parser.add_argument( + "--output", + type=str, + required=True, + help="Output Folder", + ) + + args = parser.parse_args() + input_folder: str = args.input + output_folder: str = args.output + commit_hash: str = args.commit_hash + + input_files = os.listdir(input_folder) + input_mpk: set[str] = {file for file in input_files if ".mpk" in file} + input_json: list[str] = [file for file in input_files if ".json" in file] + + for json_file in input_json: + mpk_file = json_file.replace(".json", ".mpk") + json_path = path(input_folder, json_file) + output_path = path(output_folder, mpk_file) + + if mpk_file not in input_mpk: + # First time this config is being run + print( + f"Couldn't find pair for {json_file} - ({mpk_file} not found). Generating new mpk file" + ) + generate_new_mpk(json_path, output_path, commit_hash) + else: + print(f"Found pair ({mpk_file} {json_file}). Combining") + mpk_path = path(input_folder, mpk_file) + + append_json_to_mpk(mpk_path, json_path, output_path, commit_hash) + input_mpk.remove(mpk_file) + + # Keep old mpk files + for mpk_file in input_mpk: + print(f"Couldn't find new reults to add to {mpk_file}. Copying it") + mpk_path = path(input_folder, mpk_file) + output_path = path(output_folder, mpk_file) + shutil.copy2(mpk_path, output_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/perf/tests/benchmark_results_comment/no_mpk/expected.md b/scripts/perf/tests/benchmark_results_comment/no_mpk/expected.md new file mode 100644 index 0000000000..0686f4ec82 --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/no_mpk/expected.md @@ -0,0 +1,83 @@ +Hi :wave:, thank you for your PR! + +We've run benchmarks in an emulated environment. Here are the results: + +#### ARM Emulated 32b - lv_conf_perf32b + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| All scenes avg. | 20 | 24 | 7 | 7 | 0 | + +
+ +Detailed Results Per Scene + + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| Empty screen | 11 | 25 | 0 | 0 | 0 | +| Moving wallpaper | 1 | 25 | 0 | 0 | 0 | +| Single rectangle | 0 | 25 | 0 | 0 | 0 | +| Multiple rectangles | 0 | 25 | 0 | 0 | 0 | +| Multiple RGB images | 0 | 25 | 0 | 0 | 0 | +| Multiple ARGB images | 22 | 25 | 1 | 1 | 0 | +| Rotated ARGB images | 47 | 24 | 20 | 20 | 0 | +| Multiple labels | 2 | 25 | 0 | 0 | 0 | +| Screen sized text | 30 | 24 | 11 | 11 | 0 | +| Multiple arcs | 19 | 24 | 7 | 7 | 0 | +| Containers | 1 | 25 | 0 | 0 | 0 | +| Containers with overlay | 87 | 21 | 44 | 44 | 0 | +| Containers with opa | 23 | 25 | 4 | 4 | 0 | +| Containers with opa_layer | 22 | 25 | 8 | 8 | 0 | +| Containers with scrolling | 25 | 25 | 10 | 10 | 0 | +| Widgets demo | 34 | 24 | 13 | 13 | 0 | +| All scenes avg. | 20 | 24 | 7 | 7 | 0 | + + +
+ +#### ARM Emulated 64b - lv_conf_perf64b + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| All scenes avg. | 16 | 24 | 6 | 6 | 0 | + +
+ +Detailed Results Per Scene + + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| Empty screen | 11 | 25 | 0 | 0 | 0 | +| Moving wallpaper | 0 | 25 | 0 | 0 | 0 | +| Single rectangle | 0 | 25 | 0 | 0 | 0 | +| Multiple rectangles | 0 | 25 | 0 | 0 | 0 | +| Multiple RGB images | 0 | 25 | 0 | 0 | 0 | +| Multiple ARGB images | 1 | 25 | 0 | 0 | 0 | +| Rotated ARGB images | 23 | 25 | 10 | 10 | 0 | +| Multiple labels | 3 | 24 | 0 | 0 | 0 | +| Screen sized text | 23 | 24 | 10 | 10 | 0 | +| Multiple arcs | 15 | 24 | 6 | 6 | 0 | +| Containers | 3 | 25 | 0 | 0 | 0 | +| Containers with overlay | 90 | 23 | 41 | 41 | 0 | +| Containers with opa | 19 | 25 | 4 | 4 | 0 | +| Containers with opa_layer | 13 | 24 | 5 | 5 | 0 | +| Containers with scrolling | 25 | 25 | 10 | 10 | 0 | +| Widgets demo | 33 | 25 | 13 | 13 | 0 | +| All scenes avg. | 16 | 24 | 6 | 6 | 0 | + + +
+ + +Disclaimer: These benchmarks were run in an emulated environment using QEMU with instruction counting mode. +The timing values represent relative performance metrics within this specific virtualized setup and should +not be interpreted as absolute real-world performance measurements. Values are deterministic and useful for +comparing different LVGL features and configurations, but may not correlate directly with performance on +physical hardware. The measurements are intended for comparative analysis only. + + +--- + +:robot: This comment was automatically generated by a bot. \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/no_mpk/results-32b-lv_conf_perf32b.json b/scripts/perf/tests/benchmark_results_comment/no_mpk/results-32b-lv_conf_perf32b.json new file mode 100644 index 0000000000..31a0b23f4b --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/no_mpk/results-32b-lv_conf_perf32b.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/no_mpk/results-64b-lv_conf_perf64b.json b/scripts/perf/tests/benchmark_results_comment/no_mpk/results-64b-lv_conf_perf64b.json new file mode 100644 index 0000000000..f7fd438c4e --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/no_mpk/results-64b-lv_conf_perf64b.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 23, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 3, "avg_fps": 24, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 23, "avg_fps": 24, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 15, "avg_fps": 24, "avg_time": 6, "render_time": 6, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 3, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 90, "avg_fps": 23, "avg_time": 41, "render_time": 41, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 19, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 13, "avg_fps": 24, "avg_time": 5, "render_time": 5, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 33, "avg_fps": 25, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 16, "avg_fps": 24, "avg_time": 6, "render_time": 6, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/normal/expected.md b/scripts/perf/tests/benchmark_results_comment/normal/expected.md new file mode 100644 index 0000000000..9c4bc06c44 --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/normal/expected.md @@ -0,0 +1,83 @@ +Hi :wave:, thank you for your PR! + +We've run benchmarks in an emulated environment. Here are the results: + +#### ARM Emulated 32b - lv_conf_perf32b + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| All scenes avg. | 20 | 24 | 7 | 7 | 0 | + +
+ +Detailed Results Per Scene + + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| Empty screen | 11 | 25 | 0 | 0 | 0 | +| Moving wallpaper | 1 | 25 | 0 | 0 | 0 | +| Single rectangle | 0 | 25 | 0 | 0 | 0 | +| Multiple rectangles | 0 | 25 | 0 | 0 | 0 | +| Multiple RGB images | 0 | 25 | 0 | 0 | 0 | +| Multiple ARGB images | 22 (-1)| 25 | 1 | 1 | 0 | +| Rotated ARGB images | 47 (-1)| 24 | 20 | 20 | 0 | +| Multiple labels | 2 (-2)| 25 | 0 | 0 | 0 | +| Screen sized text | 30 (+1)| 24 (-1)| 11 (-1)| 11 (-1)| 0 | +| Multiple arcs | 19 (+4)| 24 | 7 | 7 | 0 | +| Containers | 1 (-1)| 25 | 0 | 0 | 0 | +| Containers with overlay | 87 (-2)| 21 | 44 | 44 | 0 | +| Containers with opa | 23 (+1)| 25 | 4 | 4 | 0 | +| Containers with opa_layer | 22 (+1)| 25 | 8 | 8 | 0 | +| Containers with scrolling | 25 | 25 | 10 | 10 | 0 | +| Widgets demo | 34 | 24 (-1)| 13 | 13 | 0 | +| All scenes avg. | 20 | 24 | 7 | 7 | 0 | + + +
+ +#### ARM Emulated 64b - lv_conf_perf64b + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| All scenes avg. | 16 | 24 | 6 | 6 | 0 | + +
+ +Detailed Results Per Scene + + +| Scene Name | Avg CPU (%) | Avg FPS | Avg Time (ms) | Render Time (ms) | Flush Time (ms) | +|------------|------------|---------|--------------|----------------|--------------| +| Empty screen | 11 | 25 | 0 | 0 | 0 | +| Moving wallpaper | 0 | 25 | 0 | 0 | 0 | +| Single rectangle | 0 | 25 | 0 | 0 | 0 | +| Multiple rectangles | 0 | 25 | 0 | 0 | 0 | +| Multiple RGB images | 0 | 25 | 0 | 0 | 0 | +| Multiple ARGB images | 1 | 25 | 0 | 0 | 0 | +| Rotated ARGB images | 23 | 25 | 10 | 10 | 0 | +| Multiple labels | 3 | 24 | 0 | 0 | 0 | +| Screen sized text | 23 | 24 | 10 | 10 | 0 | +| Multiple arcs | 15 | 24 | 6 | 6 | 0 | +| Containers | 3 (+1)| 25 (+1)| 0 | 0 | 0 | +| Containers with overlay | 90 (+2)| 23 | 41 | 41 | 0 | +| Containers with opa | 19 | 25 | 4 | 4 | 0 | +| Containers with opa_layer | 13 (-2)| 24 (-1)| 5 (-1)| 5 (-1)| 0 | +| Containers with scrolling | 25 | 25 (-1)| 10 | 10 | 0 | +| Widgets demo | 33 | 25 | 13 | 13 | 0 | +| All scenes avg. | 16 | 24 | 6 | 6 | 0 | + + +
+ + +Disclaimer: These benchmarks were run in an emulated environment using QEMU with instruction counting mode. +The timing values represent relative performance metrics within this specific virtualized setup and should +not be interpreted as absolute real-world performance measurements. Values are deterministic and useful for +comparing different LVGL features and configurations, but may not correlate directly with performance on +physical hardware. The measurements are intended for comparative analysis only. + + +--- + +:robot: This comment was automatically generated by a bot. \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.json b/scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.json new file mode 100644 index 0000000000..31a0b23f4b --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.mpk b/scripts/perf/tests/benchmark_results_comment/normal/results-32b-lv_conf_perf32b.mpk new file mode 100644 index 0000000000000000000000000000000000000000..617676f3b3012cdc9d0e69eed9cae9ec57641672 GIT binary patch literal 2965 zcmeI!PiqrF7zXe}11(L93aM>DVesUk$eP{RZI7*pr(W8FcnY&K?mHz4`o+hfPFF(wwZ znd5?EVP-$=2Cf(2EoNC3_I!`wF1EbWoI(uw_r|@q z(RA3JErK-Pct8C{gb_5(GKd*ummkIfALfJ0wXS{KOGaVbM~8&-l%$X;v3YIrKzbZd z22_ygN{Nl@ij%z|7h$TK`ENw`U+khVB7OLE^mCDPr>fqs?!T$esA-zfWIqu^07EsX zrO@?a@EslPn9z$NDknbhV(%>lUiBn>I+&eEl!vcnQ~^gq&FFz5%ZzP3W80MRLy?hW zv`Bcj$}Q;Jf{7bHOJYI77_!2;t*oo=_DgA@!%z%RGJ=egW3}g33SKSdvZ;sMG=-d0 z;FKtIRiS0tmX5Ye=+_#wC)1YKy@p=Y>BYHeSxgD%vIA7=jcV#8onA80=PyIn2azKN zQKGa*Rc%GrR!rK(4(GBpr;UnaM?O(tM}g((G#{s*ntYquzUvXpf&gRY`r9t}1UxGH zo4JnTxbj!K>)kYY472~Pf27IR)R8Ig;s4R(533tr)-T_(>GIV#`5nd0bM^FBOHGsC YQdT*W`CdSOrO9jF0yj+_!5?h$A6dYybN~PV literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.json b/scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.json new file mode 100644 index 0000000000..f7fd438c4e --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 23, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 3, "avg_fps": 24, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 23, "avg_fps": 24, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 15, "avg_fps": 24, "avg_time": 6, "render_time": 6, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 3, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 90, "avg_fps": 23, "avg_time": 41, "render_time": 41, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 19, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 13, "avg_fps": 24, "avg_time": 5, "render_time": 5, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 33, "avg_fps": 25, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 16, "avg_fps": 24, "avg_time": 6, "render_time": 6, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.mpk b/scripts/perf/tests/benchmark_results_comment/normal/results-64b-lv_conf_perf64b.mpk new file mode 100644 index 0000000000000000000000000000000000000000..083e68591542e5d54c831d4d8c6392ab5b7857fc GIT binary patch literal 2965 zcmeH}O>fgM7=R1=IBco_aTwdl+}H($CQjIKpo!Bi+X2L-a^g3wCFeu2+cAzi><928 z0ChtH6$$ZEkoYmUo%|uPi0%U-YteK@1X<2c}YsBG_iGM;=bZ>KoumC zOh(rxj&}!KveIRW8-(t?c!pR;dT@Q{S7W+er{22i8y)Wzl1QNF0_a*Dy5cVUO4Guh z8C6bV;AZX%8u;LnJL${*bVnk=-YQiIZ=_x$<7knyIes#Zq)Lq8ox0(&JLTo3DUY5P zxg;!y%CK&0)@8q&ZWWNpf5RmqbJFYpAtdbfJq) z(C-&OyUNyn?rn{>YqaE|mu5z*FcqAu259SbjkeaaE&u6DMtdLyqL3Au_E6KTXxkNS zH;JBYbFQj6DJoKpe5`?Y>%ivi^jpcd5O^*hnL zJ#&8rYllR0$v4cQb`vGHqW`eu FzW{r!teOA- literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/benchmark_results_comment/test.sh b/scripts/perf/tests/benchmark_results_comment/test.sh new file mode 100755 index 0000000000..724dad1dc2 --- /dev/null +++ b/scripts/perf/tests/benchmark_results_comment/test.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +ORIGINAL_PWD=$(pwd) +SCRIPT=$(readlink -f $0) +SCRIPT_PATH=$(dirname $SCRIPT) + +cd $SCRIPT_PATH +python3 ../../benchmark_results_comment.py --new no_mpk/results*.json -o no_mpk/actual.md +python3 ../../benchmark_results_comment.py --previous normal/results*.mpk --new normal/results*.json -o normal/actual.md +cd $ORIGINAL_PWD + + +cmp $SCRIPT_PATH/no_mpk/expected.md $SCRIPT_PATH/no_mpk/actual.md +cmp $SCRIPT_PATH/normal/expected.md $SCRIPT_PATH/normal/actual.md + diff --git a/scripts/perf/tests/filter_docker_logs/docker-logs-example.txt b/scripts/perf/tests/filter_docker_logs/docker-logs-example.txt new file mode 100644 index 0000000000..545812676e --- /dev/null +++ b/scripts/perf/tests/filter_docker_logs/docker-logs-example.txt @@ -0,0 +1,205 @@ +{"log":"Found existing rootfs image in /persistence/rootfs.fat.virt32.\r\n","stream":"stdout","time":"2025-03-14T20:32:14.707303831Z"} +{"log":"Found existing filesystem image in /persistence/sdcard.img.virt32.\r\n","stream":"stdout","time":"2025-03-14T20:32:14.713651764Z"} +{"log":"Platform is virt32\r\n","stream":"stdout","time":"2025-03-14T20:32:14.716038006Z"} +{"log":"Starting Release build\r\n","stream":"stdout","time":"2025-03-14T20:32:14.717227898Z"} +{"log":"Not searching for unused variables given on the command line.\r\n","stream":"stdout","time":"2025-03-14T20:32:14.745430299Z"} +{"log":"\u001b[0mSystem is unknown to cmake, create:\r\n","stream":"stdout","time":"2025-03-14T20:32:14.748807599Z"} +{"log":"Platform/SO3_usr to use this system, please post your config file on discourse.cmake.org so it can be added to cmake\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:14.74881708Z"} +{"log":"\u001b[0mYour CMakeCache.txt file was copied to CopyOfCMakeCache.txt. Please post that file on discourse.cmake.org.\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:14.749184826Z"} +{"log":"-- Configuring done (0.1s)\r\n","stream":"stdout","time":"2025-03-14T20:32:14.847438284Z"} +{"log":"-- Generating done (0.3s)\r\n","stream":"stdout","time":"2025-03-14T20:32:15.176738283Z"} +{"log":"-- Build files have been written to: /so3/usr/build\r\n","stream":"stdout","time":"2025-03-14T20:32:15.177012138Z"} +{"log":"[ 0%] Built target slv\r\n","stream":"stdout","time":"2025-03-14T20:32:15.230938018Z"} +{"log":"[ 16%] Built target c\r\n","stream":"stdout","time":"2025-03-14T20:32:15.259557577Z"} +{"log":"[ 16%] \u001b[32m\u001b[1mLinking C executable sh.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:15.273128278Z"} +{"log":"[ 16%] \u001b[32m\u001b[1mLinking C executable ls.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:15.273371483Z"} +{"log":"[ 16%] \u001b[32m\u001b[1mLinking C executable mydev_test.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:15.273527115Z"} +{"log":"[ 17%] \u001b[32m\u001b[1mLinking C executable time.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:15.273681218Z"} +{"log":"[ 17%] \u001b[32m\u001b[1mLinking C executable ping.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:15.274979851Z"} +{"log":"[ 17%] \u001b[32m\u001b[1mLinking C executable more.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:15.275235356Z"} +{"log":"arm-none-eabi-ld: warning: mydev_test.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:15.286865573Z"} +{"log":"arm-none-eabi-ld: warning: time.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:15.287133237Z"} +{"log":"arm-none-eabi-ld: warning: ls.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:15.287408832Z"} +{"log":"arm-none-eabi-ld: warning: more.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:15.288070604Z"} +{"log":"arm-none-eabi-ld: warning: ping.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:15.288727656Z"} +{"log":"arm-none-eabi-ld: warning: sh.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:15.289373017Z"} +{"log":"[ 17%] Built target time.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:15.294744583Z"} +{"log":"[ 17%] Built target mydev_test.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:15.295257472Z"} +{"log":"[ 17%] Built target ls.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:15.295839172Z"} +{"log":"[ 17%] Built target more.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:15.295856623Z"} +{"log":"[ 18%] Built target ping.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:15.296507264Z"} +{"log":"[ 18%] Built target sh.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:15.29684107Z"} +{"log":"[ 51%] Built target lvgl\r\n","stream":"stdout","time":"2025-03-14T20:32:15.679676932Z"} +{"log":"[ 55%] Built target lvgl_thorvg\r\n","stream":"stdout","time":"2025-03-14T20:32:15.700285078Z"} +{"log":"[ 74%] Built target lvgl_examples\r\n","stream":"stdout","time":"2025-03-14T20:32:16.438310259Z"} +{"log":"[ 99%] Built target lvgl_demos\r\n","stream":"stdout","time":"2025-03-14T20:32:16.548772835Z"} +{"log":"[ 99%] \u001b[32m\u001b[1mLinking C executable lvgl_benchmark.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:16.567263714Z"} +{"log":"[100%] \u001b[32m\u001b[1mLinking C executable lvgl_demo.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:16.567286834Z"} +{"log":"[100%] \u001b[32m\u001b[1mLinking C executable lvgl_perf.elf\u001b[0m\r\n","stream":"stdout","time":"2025-03-14T20:32:16.568622548Z"} +{"log":"arm-none-eabi-ld: warning: lvgl_perf.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:16.596923961Z"} +{"log":"arm-none-eabi-ld: warning: lvgl_benchmark.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:16.599814023Z"} +{"log":"arm-none-eabi-ld: warning: lvgl_demo.elf has a LOAD segment with RWX permissions\r\n","stream":"stdout","time":"2025-03-14T20:32:16.599829503Z"} +{"log":"[100%] Built target lvgl_perf.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:16.611878347Z"} +{"log":"[100%] Built target lvgl_benchmark.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:16.61540439Z"} +{"log":"[100%] Built target lvgl_demo.elf\r\n","stream":"stdout","time":"2025-03-14T20:32:16.615420571Z"} +{"log":"/so3/usr\r\n","stream":"stdout","time":"2025-03-14T20:32:16.624743276Z"} +{"log":"Installing out\r\n","stream":"stdout","time":"2025-03-14T20:32:16.625245525Z"} +{"log":"Installing \r\n","stream":"stdout","time":"2025-03-14T20:32:16.625911907Z"} +{"log":"FIT description: Kernel and rootfs components for virt32 environment\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931786439Z"} +{"log":"Created: Fri Mar 14 20:32:16 2025\r\n","stream":"stdout","time":"2025-03-14T20:32:16.93183975Z"} +{"log":" Image 0 (so3)\r\n","stream":"stdout","time":"2025-03-14T20:32:16.93184516Z"} +{"log":" Description: SO3 OS kernel\r\n","stream":"stdout","time":"2025-03-14T20:32:16.9318507Z"} +{"log":" Created: Fri Mar 14 20:32:16 2025\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931855601Z"} +{"log":" Type: Kernel Image\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931860341Z"} +{"log":" Compression: uncompressed\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931864181Z"} +{"log":" Data Size: 233544 Bytes = 228.07 KiB = 0.22 MiB\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931867961Z"} +{"log":" Architecture: ARM\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931871821Z"} +{"log":" OS: Linux\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931875611Z"} +{"log":" Load Address: 0x41008000\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931880061Z"} +{"log":" Entry Point: 0x41008000\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931884561Z"} +{"log":" Image 1 (lvperf_fdt)\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931888791Z"} +{"log":" Description: Flattened Device Tree blob\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931893221Z"} +{"log":" Created: Fri Mar 14 20:32:16 2025\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931897651Z"} +{"log":" Type: Flat Device Tree\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931902121Z"} +{"log":" Compression: uncompressed\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931905941Z"} +{"log":" Data Size: 1454 Bytes = 1.42 KiB = 0.00 MiB\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931910192Z"} +{"log":" Architecture: ARM\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931914132Z"} +{"log":" Load Address: 0x44a00000\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931917882Z"} +{"log":" Image 2 (ramfs)\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931921612Z"} +{"log":" Description: SO3 environment minimal rootfs\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931925382Z"} +{"log":" Created: Fri Mar 14 20:32:16 2025\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931931202Z"} +{"log":" Type: RAMDisk Image\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931941882Z"} +{"log":" Compression: uncompressed\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931949552Z"} +{"log":" Data Size: 17825792 Bytes = 17408.00 KiB = 17.00 MiB\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931969373Z"} +{"log":" Architecture: ARM\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931974473Z"} +{"log":" OS: Linux\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931979023Z"} +{"log":" Load Address: 0x44c00000\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931982833Z"} +{"log":" Entry Point: unavailable\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931986583Z"} +{"log":" Default Configuration: 'so3_lvperf_ramfs'\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931990363Z"} +{"log":" Configuration 0 (so3_lvperf_ramfs)\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931994313Z"} +{"log":" Description: SO3 kernel image including device tree\r\n","stream":"stdout","time":"2025-03-14T20:32:16.931998103Z"} +{"log":" Kernel: so3\r\n","stream":"stdout","time":"2025-03-14T20:32:16.932001933Z"} +{"log":" Init Ramdisk: ramfs\r\n","stream":"stdout","time":"2025-03-14T20:32:16.932005693Z"} +{"log":" FDT: lvperf_fdt\r\n","stream":"stdout","time":"2025-03-14T20:32:16.932009463Z"} +{"log":"\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.141214715Z"} +{"log":"\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.141238796Z"} +{"log":"U-Boot 2022.04 (Mar 12 2025 - 21:28:47 +0000)\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.141320337Z"} +{"log":"\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.141343368Z"} +{"log":"DRAM: 1 GiB\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.142811464Z"} +{"log":"Core: 42 devices, 11 uclasses, devicetree: board\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.187948537Z"} +{"log":"Flash: 64 MiB\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.360854793Z"} +{"log":"Loading Environment from Flash... *** Warning - bad CRC, using default environment\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.364120671Z"} +{"log":"\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.364133852Z"} +{"log":"In: pl011@9000000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.369442616Z"} +{"log":"Out: pl011@9000000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.369521737Z"} +{"log":"Err: pl011@9000000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.369630069Z"} +{"log":"Net: No ethernet found.\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.375946722Z"} +{"log":"Hit any key to stop autoboot: 0 \r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.377595221Z"} +{"log":"99 bytes read in 0 ms\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.38541157Z"} +{"log":"## Warning: defaulting to text format\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.38593644Z"} +{"log":"## Info: input data size = 734 = 0x2DE\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.386195084Z"} +{"log":"18062712 bytes read in 6 ms (2.8 GiB/s)\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.397864982Z"} +{"log":"## Loading kernel from FIT Image at 70000000 ...\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.398653806Z"} +{"log":" Using 'so3_lvperf_ramfs' configuration\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.399347568Z"} +{"log":" Verifying Hash Integrity ... OK\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.399790356Z"} +{"log":" Trying 'so3' kernel subimage\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.400054021Z"} +{"log":" Description: SO3 OS kernel\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.400250634Z"} +{"log":" Created: 2025-03-14 20:32:16 UTC\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.400707812Z"} +{"log":" Type: Kernel Image\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.400981297Z"} +{"log":" Compression: uncompressed\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.40115414Z"} +{"log":" Data Start: 0x700000cc\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.401441475Z"} +{"log":" Data Size: 233544 Bytes = 228.1 KiB\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.401621409Z"} +{"log":" Architecture: ARM\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.401834352Z"} +{"log":" OS: Linux\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.401982385Z"} +{"log":" Load Address: 0x41008000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.40225652Z"} +{"log":" Entry Point: 0x41008000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.402359142Z"} +{"log":" Verifying Hash Integrity ... OK\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.402934512Z"} +{"log":"## Loading ramdisk from FIT Image at 70000000 ...\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.403726946Z"} +{"log":" Using 'so3_lvperf_ramfs' configuration\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.404009111Z"} +{"log":" Verifying Hash Integrity ... OK\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.404358757Z"} +{"log":" Trying 'ramfs' ramdisk subimage\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.40451006Z"} +{"log":" Description: SO3 environment minimal rootfs\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.404692053Z"} +{"log":" Created: 2025-03-14 20:32:16 UTC\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.404854176Z"} +{"log":" Type: RAMDisk Image\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.404994809Z"} +{"log":" Compression: uncompressed\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.405112601Z"} +{"log":" Data Start: 0x70039808\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.405224613Z"} +{"log":" Data Size: 17825792 Bytes = 17 MiB\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.405373045Z"} +{"log":" Architecture: ARM\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.405475037Z"} +{"log":" OS: Linux\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.405540908Z"} +{"log":" Load Address: 0x44c00000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.40564666Z"} +{"log":" Entry Point: unavailable\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.405794883Z"} +{"log":" Verifying Hash Integrity ... OK\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.406170159Z"} +{"log":" Loading ramdisk from 0x70039808 to 0x44c00000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.406457395Z"} +{"log":"## Loading fdt from FIT Image at 70000000 ...\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.417841727Z"} +{"log":" Using 'so3_lvperf_ramfs' configuration\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.418055241Z"} +{"log":" Verifying Hash Integrity ... OK\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.418439928Z"} +{"log":" Trying 'lvperf_fdt' fdt subimage\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.41857305Z"} +{"log":" Description: Flattened Device Tree blob\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.418728843Z"} +{"log":" Created: 2025-03-14 20:32:16 UTC\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.418925947Z"} +{"log":" Type: Flat Device Tree\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.419084659Z"} +{"log":" Compression: uncompressed\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.419173371Z"} +{"log":" Data Start: 0x700391c8\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.419289543Z"} +{"log":" Data Size: 1454 Bytes = 1.4 KiB\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.419426266Z"} +{"log":" Architecture: ARM\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.419506947Z"} +{"log":" Load Address: 0x44a00000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.419641259Z"} +{"log":" Verifying Hash Integrity ... OK\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.420025906Z"} +{"log":" Loading fdt from 0x700391c8 to 0x44a00000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.420210119Z"} +{"log":" Booting using the fdt blob at 0x44a00000\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.420773569Z"} +{"log":" Loading Kernel Image\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.42136626Z"} +{"log":" Using Device Tree in place at 44a00000, end 44a035ad\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.422625712Z"} +{"log":"\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.424265281Z"} +{"log":"Starting kernel ...\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.424312452Z"} +{"log":"\r\r\n","stream":"stdout","time":"2025-03-14T20:32:17.424321252Z"} +{"log":"setup_arch: CPU control register (CR) = c5387d\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429470984Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429544555Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429552336Z"} +{"log":"********** Smart Object Oriented SO3 Operating System **********\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429699368Z"} +{"log":"Copyright (c) 2014-2023 REDS Institute, HEIG-VD, Yverdon\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429856331Z"} +{"log":"Version 2023.6.0\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429931402Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429942853Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429949053Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.429958813Z"} +{"log":"Now bootstraping the kernel ...\r\n","stream":"stdout","time":"2025-03-14T20:32:17.430034164Z"} +{"log":"SO3: allocating a kernel heap of 33554404 bytes at address c0044000.\r\n","stream":"stdout","time":"2025-03-14T20:32:17.673343683Z"} +{"log":"memory_init: Device tree virt addr: c3a00000\r\n","stream":"stdout","time":"2025-03-14T20:32:17.673549157Z"} +{"log":"memory_init: relocating the device tree from 0xc3a00000 to 0xc21a4000 (size of 4224 bytes)\r\n","stream":"stdout","time":"2025-03-14T20:32:17.673910253Z"} +{"log":"SO3 Memory information:\r\n","stream":"stdout","time":"2025-03-14T20:32:17.674259929Z"} +{"log":" - Memory size : 536870912 bytes\r\n","stream":"stdout","time":"2025-03-14T20:32:17.674530184Z"} +{"log":" - Available pages: 122458 (1de5a)\r\n","stream":"stdout","time":"2025-03-14T20:32:17.674673837Z"} +{"log":" - Kernel size without frame table is: 35282944 (0x21a6000) bytes, 33 MB / 0x21a6 PFNs\r\n","stream":"stdout","time":"2025-03-14T20:32:17.675009963Z"} +{"log":" - Kernel size including frame table is: 36265984 (0x2296000) bytes, 34 MB / 0x2296 PFNs\r\n","stream":"stdout","time":"2025-03-14T20:32:17.677971645Z"} +{"log":" - Number of available page frames: 0x1de5a\r\n","stream":"stdout","time":"2025-03-14T20:32:17.678144519Z"} +{"log":" - Frame table size is: 979664 bytes meaning 240 (0xf0) page frames\r\n","stream":"stdout","time":"2025-03-14T20:32:17.678424433Z"} +{"log":" - Page frame number of the first available page: 0x431a6\r\n","stream":"stdout","time":"2025-03-14T20:32:17.678619197Z"} +{"log":"so3: rootfs in RAM detected (ramdev enabled) with size of 17825792 bytes...\r\n","stream":"stdout","time":"2025-03-14T20:32:17.68218061Z"} +{"log":"calibrate_delay: calibrating...done. jiffies_ref = a2be8\r\n","stream":"stdout","time":"2025-03-14T20:32:17.704849544Z"} +{"log":"SO3: starting the initial process ...\r\n","stream":"stdout","time":"2025-03-14T20:32:17.712134853Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.712146014Z"} +{"log":"\r\n","stream":"stdout","time":"2025-03-14T20:32:17.712150624Z"} +{"log":"Starting LVGL Benchmark\r\n","stream":"stdout","time":"2025-03-14T20:32:17.816299617Z"} +{"log":"Warning: The guest is now late by 0.0 to 1.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:32:19.129030996Z"} +{"log":"Warning: The guest is now late by 1.0 to 2.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:32:40.699491404Z"} +{"log":"Warning: The guest is now late by 2.0 to 3.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:02.966355127Z"} +{"log":"Warning: The guest is now late by 3.0 to 4.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:30.06251689Z"} +{"log":"Warning: The guest is now late by 4.0 to 5.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:38.253535255Z"} +{"log":"Warning: The guest is now late by 5.0 to 6.0 seconds\r\n","stream":"stdout","time":"2025-03-14T20:33:51.010250882Z"} +{"log":"Benchmark Summary (9.3.0 dev)\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.855785347Z"} +{"log":"Name, Avg. CPU, Avg. FPS, Avg. time, render time, flush time\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.855997881Z"} +{"log":"Empty screen, 11%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.857828373Z"} +{"log":"Moving wallpaper, 2%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.858455924Z"} +{"log":"Single rectangle, 0%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.859125276Z"} +{"log":"Multiple rectangles, 0%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.859847899Z"} +{"log":"Multiple RGB images, 0%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.860658884Z"} +{"log":"Multiple ARGB images, 23%, 25, 1, 1, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.86156999Z"} +{"log":"Rotated ARGB images, 48%, 24, 20, 20, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.862511197Z"} +{"log":"Multiple labels, 3%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.863549175Z"} +{"log":"Screen sized text, 30%, 24, 11, 11, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.864642275Z"} +{"log":"Multiple arcs, 18%, 24, 7, 7, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.865801475Z"} +{"log":"Containers, 3%, 25, 0, 0, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.867039657Z"} +{"log":"Containers with overlay, 88%, 21, 44, 44, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.868379231Z"} +{"log":"Containers with opa, 10%, 24, 3, 3, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.869736765Z"} +{"log":"Containers with opa_layer, 22%, 24, 8, 8, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.871269383Z"} +{"log":"Containers with scrolling, 25%, 25, 10, 10, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.872899952Z"} +{"log":"Widgets demo, 34%, 25, 13, 13, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.874447759Z"} +{"log":"All scenes avg.,19%, 24, 7, 7, 0\r\r\n","stream":"stdout","time":"2025-03-14T20:33:57.875031879Z"} +{"log":"LVGL Benchmark Over\r\n","stream":"stdout","time":"2025-03-14T20:33:57.933458479Z"} diff --git a/scripts/perf/tests/filter_docker_logs/expected.json b/scripts/perf/tests/filter_docker_logs/expected.json new file mode 100644 index 0000000000..e3d1438c47 --- /dev/null +++ b/scripts/perf/tests/filter_docker_logs/expected.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 23, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 48, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 3, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 18, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 3, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 88, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 10, "avg_fps": 24, "avg_time": 3, "render_time": 3, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 24, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 25, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/filter_docker_logs/test.sh b/scripts/perf/tests/filter_docker_logs/test.sh new file mode 100755 index 0000000000..89e087f99f --- /dev/null +++ b/scripts/perf/tests/filter_docker_logs/test.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +ORIGINAL_PWD=$(pwd) +SCRIPT=$(readlink -f $0) +SCRIPT_PATH=$(dirname $SCRIPT) + +cd $SCRIPT_PATH +cat docker-logs-example.txt | python3 ../../filter_docker_benchmark_logs.py actual.json +cd $ORIGINAL_PWD + + +cmp $SCRIPT_PATH/expected.json $SCRIPT_PATH/actual.json + diff --git a/scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b.mpk b/scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b.mpk new file mode 100644 index 0000000000000000000000000000000000000000..005677ed6593e9b31fa933d89c2634336721f864 GIT binary patch literal 4415 zcmeH~%Wl&^6owOs)T(I(h!m&9Qcis+)FEv5Ll?ZsLn>f;^e{|0Ie0=oo*a>~###pMwgTeA$xA=t2Pko3fZzk^ zwOdZqt&a|Fl&pOmKH;7ZmE#x!3h~*GzRS8v=j<%gzHEm*FKDBE!dOHihy}4aF>yEL zF+edmoD5HbSecYK*zPj!MKUv)A#~@}bL9D?4bwxvWl6V+)H}udSLGU2RW%yygq(1o zs06AB=)J7)4H?={K~J-&jM%`kyVnHpqMh`0chr$6@!qCUIlSSbM)w6W&sdi;)>Rol zXBkQCWC?E;aSJkTL51tT2m?;M0OE{wU9c{C+wVCG?R&g~!XCto9Edx=7U0#aF6(m0 zbydhg0UQwlT@}zgZB2&ORL~zK&{mo@ANLaUqKsY~8=Z%J8irw}|LRR^Zt?&|@2)Fi^1GsC@;UrJO#YyF@nw1VmQ}-7Zt@!fH_qtE zTT6+_uL)LOlkqm7OEG!LE-*K_50_=~PsDaMn!16_`ObZ%OkFZ()0_IKP))9_>2~gk z-KE~S9~LuCvvc1r;?zshIi{ZY%kn%^?}?DJ+__8M^|LZ{Npt?=J9pmjf6>&x0ft=r APyhe` literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b_a.mpk b/scripts/perf/tests/serialize_results/expected_output/results-32b-lv_conf_perf32b_a.mpk new file mode 100644 index 0000000000000000000000000000000000000000..ba089eadb92c8dfb3fbdf74f5db7563628b77494 GIT binary patch literal 1451 zcmb`{J5K^Z5C?EDBoGLN5g#;WV@)(Zdo;$11`5i{ayJ~C>^-u(fLL1C`z3q;1`Q_0 z@8XB?fb8BT$0mw)b65QGpUmvs-;V6S^N6rIn0KCW%RW3hJ~=&m;5PWczf$XW!+B}> z*n{z<7m5MmHUsbDVesUk$eP{RZI7*pr(W8FcnY&K?mHz4`o+hfPFF(wwZ znd5?EVP-$=2Cf(2EoNC3_I!`wF1EbWoI(uw_r|@q z(RA3JErK-Pct8C{gb_5(GKd*ummkIfALfJ0wXS{KOGaVbM~8&-l%$X;v3YIrKzbZd z22_ygN{Nl@ij%z|7h$TK`ENw`U+khVB7OLE^mCDPr>fqs?!T$esA-zfWIqu^07EsX zrO@?a@EslPn9z$NDknbhV(%>lUiBn>I+&eEl!vcnQ~^gq&FFz5%ZzP3W80MRLy?hW zv`Bcj$}Q;Jf{7bHOJYI77_!2;t*oo=_DgA@!%z%RGJ=egW3}g33SKSdvZ;sMG=-d0 z;FKtIRiS0tmX5Ye=+_#wC)1YKy@p=Y>BYHeSxgD%vIA7=jcV#8onA80=PyIn2azKN zQKGa*Rc%GrR!rK(4(GBpr;UnaM?O(tM}g((G#{s*ntYquzUvXpf&gRY`r9t}1UxGH zo4JnTxbj!K>)kYY472~Pf27IR)R8Ig;s4R(533tr)-T_(>GIV#`5nd0bM^FBOHGsC YQdT*W`CdSOrO9jF0yj+_!5?h$A6dYybN~PV literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/serialize_results/expected_output/results-64b-lv_conf_perf64b.mpk b/scripts/perf/tests/serialize_results/expected_output/results-64b-lv_conf_perf64b.mpk new file mode 100644 index 0000000000000000000000000000000000000000..2fab3588d452020df0b2cd3eb288a4d9fe73c45d GIT binary patch literal 4415 zcmeH~O>fgM7{?22I}BBTIE-;JH+Df;6DRCA5aP7Ub^vjy9Q#Qd$$3%Yc8uc=`vQCi zUb>-yiiG$mNPHOFPJR(tM7Omdq}-fD`sr^!_Rmk=9v;&yNn+9Mld}Kgq35_56NkFY z^TBg4b2mGYA4K>ua~uZ;VZd+)JHfk>LJH;2)}5o#bl6Qv0w>RsTpXa1769pcGVFC} zKDap@M0vUXVe&@A3ABz2NEsAo52J_=%Kq8drhVSYhH=_M`-JnH(azM$>V?u zpn^oP@yj+;1r0y$kN7FMHD+iOTqmj4I%@P&LvS4RW@OpL9l2P@V8rHQ|aq<(0ZA zkDg|!AaM$XW?fgTi=LZrhK2TH(MQ=33Qi8xoL?#MnqI~YJLE=P$Z-Xn69v7ipeF5- z4PB~(em@7=m1)g!Z!5H^(Xx$Ro*6C0lyNQ{pi-|Zv`IZ%`A=TNtOue*43bRI9w?d# z+O~tX>w=zcb1tnpF)EUde58Q4tHAp0^qa}Is2lnL!7Pd}W`4NkLr5T?(%;PYJkOV3 z-A>S$Jca4q)tJ07`Jzp3yf9k_^e2tVm?xf@eY{<4~$8!f8m)_)l0+|S3DVesUk$eP{RZI7*pr(W8FcnY&K?mHz4`o+hfPFF(wwZ znd5?EVP-$=2Cf(2EoNC3_I!`wF1EbWoI(uw_r|@q z(RA3JErK-Pct8C{gb_5(GKd*ummkIfALfJ0wXS{KOGaVbM~8&-l%$X;v3YIrKzbZd z22_ygN{Nl@ij%z|7h$TK`ENw`U+khVB7OLE^mCDPr>fqs?!T$esA-zfWIqu^07EsX zrO@?a@EslPn9z$NDknbhV(%>lUiBn>I+&eEl!vcnQ~^gq&FFz5%ZzP3W80MRLy?hW zv`Bcj$}Q;Jf{7bHOJYI77_!2;t*oo=_DgA@!%z%RGJ=egW3}g33SKSdvZ;sMG=-d0 z;FKtIRiS0tmX5Ye=+_#wC)1YKy@p=Y>BYHeSxgD%vIA7=jcV#8onA80=PyIn2azKN zQKGa*Rc%GrR!rK(4(GBpr;UnaM?O(tM}g((G#{s*ntYquzUvXpf&gRY`r9t}1UxGH zo4JnTxbj!K>)kYY472~Pf27IR)R8Ig;s4R(533tr)-T_(>GIV#`5nd0bM^FBOHGsC YQdT*W`CdSOrO9jF0yj+_!5?h$A6dYybN~PV literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_a.json b/scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_a.json new file mode 100644 index 0000000000..31a0b23f4b --- /dev/null +++ b/scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_a.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 22, "avg_fps": 25, "avg_time": 1, "render_time": 1, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 47, "avg_fps": 24, "avg_time": 20, "render_time": 20, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 2, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 30, "avg_fps": 24, "avg_time": 11, "render_time": 11, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 19, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 87, "avg_fps": 21, "avg_time": 44, "render_time": 44, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 23, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 22, "avg_fps": 25, "avg_time": 8, "render_time": 8, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 34, "avg_fps": 24, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 20, "avg_fps": 24, "avg_time": 7, "render_time": 7, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_b.mpk b/scripts/perf/tests/serialize_results/input/results-32b-lv_conf_perf32b_b.mpk new file mode 100644 index 0000000000000000000000000000000000000000..617676f3b3012cdc9d0e69eed9cae9ec57641672 GIT binary patch literal 2965 zcmeI!PiqrF7zXe}11(L93aM>DVesUk$eP{RZI7*pr(W8FcnY&K?mHz4`o+hfPFF(wwZ znd5?EVP-$=2Cf(2EoNC3_I!`wF1EbWoI(uw_r|@q z(RA3JErK-Pct8C{gb_5(GKd*ummkIfALfJ0wXS{KOGaVbM~8&-l%$X;v3YIrKzbZd z22_ygN{Nl@ij%z|7h$TK`ENw`U+khVB7OLE^mCDPr>fqs?!T$esA-zfWIqu^07EsX zrO@?a@EslPn9z$NDknbhV(%>lUiBn>I+&eEl!vcnQ~^gq&FFz5%ZzP3W80MRLy?hW zv`Bcj$}Q;Jf{7bHOJYI77_!2;t*oo=_DgA@!%z%RGJ=egW3}g33SKSdvZ;sMG=-d0 z;FKtIRiS0tmX5Ye=+_#wC)1YKy@p=Y>BYHeSxgD%vIA7=jcV#8onA80=PyIn2azKN zQKGa*Rc%GrR!rK(4(GBpr;UnaM?O(tM}g((G#{s*ntYquzUvXpf&gRY`r9t}1UxGH zo4JnTxbj!K>)kYY472~Pf27IR)R8Ig;s4R(533tr)-T_(>GIV#`5nd0bM^FBOHGsC YQdT*W`CdSOrO9jF0yj+_!5?h$A6dYybN~PV literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.json b/scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.json new file mode 100644 index 0000000000..f7fd438c4e --- /dev/null +++ b/scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.json @@ -0,0 +1 @@ +[{"scene_name": "Empty screen", "avg_cpu": 11, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Moving wallpaper", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Single rectangle", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple rectangles", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple RGB images", "avg_cpu": 0, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Multiple ARGB images", "avg_cpu": 1, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Rotated ARGB images", "avg_cpu": 23, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Multiple labels", "avg_cpu": 3, "avg_fps": 24, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Screen sized text", "avg_cpu": 23, "avg_fps": 24, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Multiple arcs", "avg_cpu": 15, "avg_fps": 24, "avg_time": 6, "render_time": 6, "flush_time": 0}, {"scene_name": "Containers", "avg_cpu": 3, "avg_fps": 25, "avg_time": 0, "render_time": 0, "flush_time": 0}, {"scene_name": "Containers with overlay", "avg_cpu": 90, "avg_fps": 23, "avg_time": 41, "render_time": 41, "flush_time": 0}, {"scene_name": "Containers with opa", "avg_cpu": 19, "avg_fps": 25, "avg_time": 4, "render_time": 4, "flush_time": 0}, {"scene_name": "Containers with opa_layer", "avg_cpu": 13, "avg_fps": 24, "avg_time": 5, "render_time": 5, "flush_time": 0}, {"scene_name": "Containers with scrolling", "avg_cpu": 25, "avg_fps": 25, "avg_time": 10, "render_time": 10, "flush_time": 0}, {"scene_name": "Widgets demo", "avg_cpu": 33, "avg_fps": 25, "avg_time": 13, "render_time": 13, "flush_time": 0}, {"scene_name": "All scenes avg.", "avg_cpu": 16, "avg_fps": 24, "avg_time": 6, "render_time": 6, "flush_time": 0}] \ No newline at end of file diff --git a/scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.mpk b/scripts/perf/tests/serialize_results/input/results-64b-lv_conf_perf64b.mpk new file mode 100644 index 0000000000000000000000000000000000000000..083e68591542e5d54c831d4d8c6392ab5b7857fc GIT binary patch literal 2965 zcmeH}O>fgM7=R1=IBco_aTwdl+}H($CQjIKpo!Bi+X2L-a^g3wCFeu2+cAzi><928 z0ChtH6$$ZEkoYmUo%|uPi0%U-YteK@1X<2c}YsBG_iGM;=bZ>KoumC zOh(rxj&}!KveIRW8-(t?c!pR;dT@Q{S7W+er{22i8y)Wzl1QNF0_a*Dy5cVUO4Guh z8C6bV;AZX%8u;LnJL${*bVnk=-YQiIZ=_x$<7knyIes#Zq)Lq8ox0(&JLTo3DUY5P zxg;!y%CK&0)@8q&ZWWNpf5RmqbJFYpAtdbfJq) z(C-&OyUNyn?rn{>YqaE|mu5z*FcqAu259SbjkeaaE&u6DMtdLyqL3Au_E6KTXxkNS zH;JBYbFQj6DJoKpe5`?Y>%ivi^jpcd5O^*hnL zJ#&8rYllR0$v4cQb`vGHqW`eu FzW{r!teOA- literal 0 HcmV?d00001 diff --git a/scripts/perf/tests/serialize_results/test.sh b/scripts/perf/tests/serialize_results/test.sh new file mode 100755 index 0000000000..4b8b881a7e --- /dev/null +++ b/scripts/perf/tests/serialize_results/test.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +set -e + +ORIGINAL_PWD=$(pwd) +SCRIPT=$(readlink -f $0) +SCRIPT_PATH=$(dirname $SCRIPT) + +cd $SCRIPT_PATH +rm -rf output +mkdir output +python3 ../../serialize_results.py --input input --output output --commit-hash abc123456 +cd $ORIGINAL_PWD + + +OUTPUT_DIR=$SCRIPT_PATH/output +EXPECTED_DIR=$SCRIPT_PATH/expected_output + +OUTPUT_COUNT=$(ls -1q "$OUTPUT_DIR" | wc -l) +EXPECTED_COUNT=$(ls -1q "$EXPECTED_DIR" | wc -l) + +if [ "$OUTPUT_COUNT" -ne "$EXPECTED_COUNT" ]; then + echo "[TEST] Mismatch in number of files: Expected $EXPECTED_COUNT. Found $OUTPUT_COUNT" + exit 1 +fi + +for file in "$EXPECTED_DIR"/*; do + filename=$(basename "$file") + output_file="$OUTPUT_DIR/$filename" + + if [ ! -f "$output_file" ]; then + echo "[TEST] Missing file: $filename in output directory" + exit 1 + fi + + if ! cmp -s "$file" "$output_file"; then + echo "[TEST] File mismatch: $filename" + exit 1 + fi +done + +echo "[TEST] Test Passed" +exit 0 diff --git a/tests/README.md b/tests/README.md index 13c26151a2..4f003d7448 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,21 @@ # Tests for LVGL -The tests in the folder can be run locally and automatically by GitHub CI. +## Test types available + +- **Unit Tests**: Standard functional tests in `src/test_cases/` with screenshot comparison capabilities +- **Performance Tests**: ARM-emulated benchmarks in `src/test_cases_perf/` running on QEMU/SO3 environment +- **Emulated Benchmarks**: Automated `lv_demo_benchmark` runs in ARM emulation to prevent performance regressions + +All of the tests are automatically ran in LVGL's CI. + +## Quick start + +- **Local Testing**: Run `./tests/main.py test` (after `scripts/install-prerequisites.sh`) +- **Docker Testing**: Build with `docker build . -f tests/Dockerfile -t lvgl_test_env` then run +- **Performance Testing**: Use `./tests/perf.py test` (requires Docker + Linux) +- **Benchmark Testing**: Use `./tests/benchmark_emu.py run` for emulated performance benchmarks (requires Docker + Linux) + +--- ## Running locally @@ -44,43 +59,9 @@ docker run --rm -it -v $(pwd):/work lvgl_test_env "./tests/main.py" This ensures you are testing in a consistent environment with the same dependencies as the CI pipeline. -## Performance Tests - -### Requirements - -- **Docker** -- **Linux host machine** (WSL *may* work but is untested) - -### Running Tests - -The performance tests are run inside a Docker container that launches an ARM emulated environment using QEMU to ensure consistent timing across machines. Each test runs on a lightweight ARM-based OS ([SO3](https://github.com/smartobjectoriented/so3)) within this emulated environment. - -To run the tests: - -```bash -./perf.py [--clean] [--auto-clean] [--test-suite ] [--build-options