[ci] Fix scheduled failure reporting and monthly BSP builds (#11921)

* [ci] Fix scheduled failure reporting and monthly BSP builds

* [ci] Preserve partial monitoring results and abnormal job details

---------

Co-authored-by: Rbb666 <zhangbingru@rt-thread.com>
This commit is contained in:
CYFSandRbb666 authored and GitHub committed 2026-11-05 14:56:59 +08:00
1 parent 964dd80240
commit 8b3c2db64b
6 files changed
+160 -97

No files matched your search

+22 -16
View File
@@ -15,7 +15,7 @@ name: manual_trigger_scons_build_all
# toolchain主要用来选择默认安装的toolchain
on:
# Runs at 16:00 UTC (BeiJing 00:00) on the 1st of every month
# Runs at 16:00 UTC on the 1st (00:00 Beijing time on the 2nd).
schedule:
- cron: '0 16 1 * *'
@@ -52,7 +52,10 @@ permissions:
jobs:
build:
runs-on: ubuntu-22.04
name: ${{ github.event.inputs.bsp_options }}
name: ${{ inputs.bsp_options || 'all' }}
env:
RTT_BSP: ${{ inputs.bsp_options || 'all' }}
RTT_TOOL_CHAIN: ${{ inputs.bsp_tool_chain || 'sourcery-arm' }}
steps:
- uses: actions/checkout@main
- name: Set up Python
@@ -72,7 +75,7 @@ jobs:
echo "RTT_CC=gcc" >> $GITHUB_ENV
- name: Install Arm ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'sourcery-arm' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'sourcery-arm' && success() }}
shell: bash
run: |
wget -q https://github.com/RT-Thread/toolchains-ci/releases/download/v1.3/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2
@@ -81,7 +84,7 @@ jobs:
echo "RTT_EXEC_PATH=/opt/gcc-arm-none-eabi-10-2020-q4-major/bin" >> $GITHUB_ENV
- name: Install LLVM-Arm ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'llvm-arm' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'llvm-arm' && success() }}
shell: bash
run: |
wget -q https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-16.0.0/LLVMEmbeddedToolchainForArm-16.0.0-Linux-x86_64.tar.gz
@@ -92,7 +95,7 @@ jobs:
echo "RTT_EXEC_PATH=/opt/LLVMEmbeddedToolchainForArm-16.0.0-Linux-x86_64/bin" >> $GITHUB_ENV
- name: Install AArch64 ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'sourcery-aarch64' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'sourcery-aarch64' && success() }}
shell: bash
run: |
wget -q https://github.com/RT-Thread/toolchains-ci/releases/download/v1.6/gcc-arm-10.2-2020.11-x86_64-aarch64-none-elf.tar.xz
@@ -101,7 +104,7 @@ jobs:
echo "RTT_EXEC_PATH=/opt/gcc-arm-10.2-2020.11-x86_64-aarch64-none-elf/bin" >> $GITHUB_ENV
- name: Install Mips ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'sourcery-mips' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'sourcery-mips' && success() }}
shell: bash
run: |
wget -q https://github.com/RT-Thread/toolchains-ci/releases/download/v1.1/mips-2016.05-7-mips-sde-elf-i686-pc-linux-gnu.tar.bz2
@@ -110,7 +113,7 @@ jobs:
echo "RTT_EXEC_PATH=/opt/mips-2016.05/bin" >> $GITHUB_ENV
- name: Install Riscv64-unknown-elf ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'sourcery-riscv64-unknown-elf' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'sourcery-riscv64-unknown-elf' && success() }}
run: |
wget -q https://github.com/RT-Thread/toolchains-ci/releases/download/v1.4/riscv64-unknown-elf-toolchain-10.2.0-2020.12.8-x86_64-linux-ubuntu14.tar.gz
sudo tar zxf riscv64-unknown-elf-toolchain-10.2.0-2020.12.8-x86_64-linux-ubuntu14.tar.gz -C /opt
@@ -118,7 +121,7 @@ jobs:
echo "RTT_EXEC_PATH=/opt/riscv64-unknown-elf-toolchain-10.2.0-2020.12.8-x86_64-linux-ubuntu14/bin" >> $GITHUB_ENV
- name: Install Riscv-none-embed ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'sourcery-riscv-none-embed' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'sourcery-riscv-none-embed' && success() }}
run: |
wget -q https://github.com/RT-Thread/toolchains-ci/releases/download/v1.5/xpack-riscv-none-embed-gcc-8.3.0-2.3-linux-x64.tar.gz
sudo tar zxf xpack-riscv-none-embed-gcc-8.3.0-2.3-linux-x64.tar.gz -C /opt
@@ -126,7 +129,7 @@ jobs:
echo "RTT_EXEC_PATH=/opt/xpack-riscv-none-embed-gcc-8.3.0-2.3/bin" >> $GITHUB_ENV
- name: Install riscv32-esp-elf ToolChains
if: ${{ github.event.inputs.bsp_tool_chain == 'sourcery-riscv32-esp32' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'sourcery-riscv32-esp32' && success() }}
run: |
wget -q https://github.com/espressif/crosstool-NG/releases/download/esp-15.2.0_20251204/riscv32-esp-elf-15.2.0_20251204-x86_64-linux-gnu.tar.xz
sudo tar xf riscv32-esp-elf-15.2.0_20251204-x86_64-linux-gnu.tar.xz -C /opt
@@ -135,21 +138,24 @@ jobs:
echo "RTT_EXEC_PATH=/opt/riscv32-esp-elf/bin" >> $GITHUB_ENV
- name: Install GCC Tools
if: ${{ github.event.inputs.bsp_tool_chain == 'gcc' && success() }}
if: ${{ env.RTT_TOOL_CHAIN == 'gcc' && success() }}
run: |
sudo apt-get -qq install libsdl2-dev
- name: Bsp Scons Compile
if: ${{ success() }}
shell: bash
env:
RTT_BSP: ${{ github.event.inputs.bsp_options }}
RTT_TOOL_CHAIN: ${{ github.event.inputs.bsp_tool_chain}}
run: |
source ~/.env/env.sh
python tools/ci/manual_bsp_build_all.py $RTT_BSP
python tools/ci/manual_bsp_build_all.py "$RTT_BSP"
if [ -s failed_bsp_list.txt ]; then
echo "::error::Some BSP builds failed; see failed_bsp.log"
exit 1
fi
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: dist_file
path: ${{ github.workspace }}/failed_bsp.log
path: ${{ github.workspace }}/failed_bsp.log
if-no-files-found: ignore
+29 -7
View File
@@ -1,4 +1,4 @@
name: Weekly CI Scheduler
name: Daily CI Scheduler
on:
# Runs at 08:00 Beijing time every day
@@ -18,7 +18,12 @@ env:
jobs:
trigger-and-monitor:
name: Trigger and Monitor CIs
if: github.repository_owner == 'RT-Thread'
runs-on: ubuntu-latest
timeout-minutes: 300
permissions:
contents: read
actions: write
outputs:
failed_workflows: ${{ steps.collect-results.outputs.failed_workflows }}
total_workflows: ${{ steps.collect-results.outputs.total_workflows }}
@@ -27,8 +32,10 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
timeout-minutes: 5
- name: Install Python dependencies
timeout-minutes: 5
run: |
python -m pip install --upgrade pip
pip install requests
@@ -41,6 +48,7 @@ jobs:
- name: Trigger CI workflows directly
id: trigger-ci
timeout-minutes: 5
run: |
python tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py
env:
@@ -49,6 +57,8 @@ jobs:
- name: Wait for workflows to appear
id: wait-for-workflows
timeout-minutes: 10
if: ${{ !cancelled() && steps.start-time.outcome == 'success' }}
run: |
echo "Waiting for workflows to appear in API..."
python tools/ci/scheduled-ci-trigger/wait_for_workflows.py "${{ steps.start-time.outputs.start_time }}"
@@ -58,6 +68,9 @@ jobs:
- name: Monitor CI workflows
id: monitor-ci
# Leave time for reporting even if monitoring is interrupted.
timeout-minutes: 245
if: ${{ !cancelled() && steps.start-time.outcome == 'success' }}
run: |
python tools/ci/scheduled-ci-trigger/monitor_workflows.py "${{ steps.start-time.outputs.start_time }}"
env:
@@ -66,11 +79,13 @@ jobs:
- name: Collect monitoring results
id: collect-results
timeout-minutes: 1
if: ${{ always() && steps.start-time.outcome == 'success' }}
run: |
echo "Checking for monitoring results..."
if [ -f "monitoring_results.json" ]; then
echo "monitoring_results.json found"
FAILED_COUNT=$(python -c "import json; data=json.load(open('monitoring_results.json')); print(len([w for w in data if w.get('conclusion') == 'failure']))")
FAILED_COUNT=$(python -c "import json; data=json.load(open('monitoring_results.json')); print(len([w for w in data if w.get('conclusion') != 'success']))")
TOTAL_COUNT=$(python -c "import json; data=json.load(open('monitoring_results.json')); print(len(data))")
echo "failed_workflows=$FAILED_COUNT" >> $GITHUB_OUTPUT
echo "total_workflows=$TOTAL_COUNT" >> $GITHUB_OUTPUT
@@ -81,10 +96,12 @@ jobs:
echo "failed_workflows=0" >> $GITHUB_OUTPUT
echo "total_workflows=0" >> $GITHUB_OUTPUT
echo "has_results=false" >> $GITHUB_OUTPUT
exit 1
fi
- name: Generate detailed report
if: steps.collect-results.outputs.has_results == 'true' && steps.collect-results.outputs.failed_workflows != '0'
timeout-minutes: 1
if: ${{ always() && steps.collect-results.outputs.has_results == 'true' && steps.collect-results.outputs.failed_workflows != '0' }}
id: generate-report
run: |
echo "Generating detailed report..."
@@ -92,7 +109,8 @@ jobs:
echo "Report generation completed"
- name: Upload report artifact
if: steps.collect-results.outputs.has_results == 'true' && steps.collect-results.outputs.failed_workflows != '0'
timeout-minutes: 5
if: ${{ always() && steps.collect-results.outputs.has_results == 'true' && steps.collect-results.outputs.failed_workflows != '0' }}
uses: actions/upload-artifact@v4
with:
name: ci-failure-report
@@ -104,8 +122,12 @@ jobs:
create-discussion:
name: Create Discussion Report
needs: trigger-and-monitor
if: needs.trigger-and-monitor.outputs.has_results == 'true' && needs.trigger-and-monitor.outputs.failed_workflows != '0'
if: ${{ !cancelled() && needs.trigger-and-monitor.outputs.has_results == 'true' && needs.trigger-and-monitor.outputs.failed_workflows != '0' }}
runs-on: ubuntu-latest
permissions:
contents: read
actions: read
discussions: write
steps:
- name: Checkout repository
@@ -117,7 +139,7 @@ jobs:
name: ci-failure-report
- name: Create Discussion
uses: actions/github-script@v6
uses: actions/github-script@v8
env:
DISCUSSION_CATEGORY: ${{ env.DISCUSSION_CATEGORY }}
with:
@@ -197,4 +219,4 @@ jobs:
body: reportContent // 使用清理后的内容(无第一行)
});
console.log('Discussion created successfully:', result.createDiscussion.discussion.url);
console.log('Discussion created successfully:', result.createDiscussion.discussion.url);
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import json
import os
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from typing import List, Dict, Any
def load_monitoring_results() -> List[Dict[str, Any]]:
@@ -17,7 +17,7 @@ def load_monitoring_results() -> List[Dict[str, Any]]:
return []
def get_beijing_time() -> datetime:
return datetime.utcnow() + timedelta(hours=8)
return datetime.now(timezone(timedelta(hours=8)))
def format_time(dt: datetime) -> str:
return dt.strftime("%Y-%m-%d %H:%M")
@@ -45,7 +45,7 @@ def generate_report():
if not results:
return
failed_workflows = [r for r in results if r.get('conclusion') == 'failure']
failed_workflows = [r for r in results if r.get('conclusion') != 'success']
if not failed_workflows:
print("No failed workflows to report")
return
@@ -55,15 +55,15 @@ def generate_report():
# 时间范围
created_times = [
datetime.fromisoformat(r["created_at"].replace("Z", "+00:00")) + timedelta(hours=8)
for r in failed_workflows
datetime.fromisoformat(r["created_at"].replace("Z", "+00:00")).astimezone(now.tzinfo)
for r in failed_workflows if r.get("created_at")
]
updated_times = [
datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")) + timedelta(hours=8)
for r in failed_workflows
datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")).astimezone(now.tzinfo)
for r in failed_workflows if r.get("updated_at")
]
start_time = min(created_times)
end_time = max(updated_times)
start_time = min(created_times, default=now)
end_time = max(updated_times, default=now)
total = len(results)
failed_count = len(failed_workflows)
@@ -91,26 +91,34 @@ def generate_report():
details = wf.get("failure_details", [])
report += f"**📌 Run-{run_id}** | [{name}]({html_url})\n"
report += f"**运行结果 | Conclusion**: `{wf.get('conclusion', 'unknown')}`\n"
if wf.get("error"):
report += f"**错误信息 | Error**: {wf['error']}\n"
if not details:
report += "└─ 无失败作业详情 | No details of failed jobs\n\n"
report += "└─ 无异常作业详情 | No details of affected jobs\n\n"
continue
failed_jobs = [j for j in details if j.get("steps")]
for i, job in enumerate(failed_jobs):
for i, job in enumerate(details):
job_name = job["name"]
steps = job["steps"]
job_prefix = "└─" if i == len(failed_jobs) - 1 else "├─"
report += f"{job_prefix} **失败作业 | Failed Job**: {job_name}\n"
steps = job.get("steps", [])
job_status = job.get("conclusion", "failure") or job.get("status") or "unknown"
job_prefix = "└─" if i == len(details) - 1 else "├─"
report += f"{job_prefix} **异常作业 | Affected Job**: {job_name} (`{job_status}`)\n"
if not steps:
report += " └─ 无异常步骤详情 | No details of affected steps\n"
for j, step in enumerate(steps):
step_name = step["name"]
step_num = step["number"]
error_type = classify_error(step_name, job_name)
step_conclusion = step.get("conclusion", "failure")
step_status = step_conclusion or step.get("status") or "unknown"
step_prefix = " └─" if j == len(steps) - 1 else " ├─"
report += f"{step_prefix} **失败步骤 | Failed Step**: {step_name} (Step {step_num})\n"
indent = " " if j == len(steps) - 1 else " │ "
report += f"{indent}**错误类型 | Error Type**: `{error_type}`\n"
report += f"{step_prefix} **异常步骤 | Affected Step**: {step_name} (Step {step_num}, `{step_status}`)\n"
if step_conclusion == "failure":
error_type = classify_error(step_name, job_name)
indent = " " if j == len(steps) - 1 else " │ "
report += f"{indent}**错误类型 | Error Type**: `{error_type}`\n"
report += "\n"
# === Team Collaboration & Support ===
@@ -132,4 +140,4 @@ def generate_report():
print(f"Error writing report: {e}")
if __name__ == "__main__":
generate_report()
generate_report()
File diff suppressed because it is too large. Load diff
@@ -3,6 +3,7 @@ import os
import json
import requests
import time
import sys
from datetime import datetime, timezone
def trigger_workflow_directly(workflow_name, github_token, repo):
@@ -31,7 +32,7 @@ def trigger_workflow_directly(workflow_name, github_token, repo):
try:
print(f"Triggering workflow: {workflow_name} (ID: {workflow_id})")
response = requests.post(dispatch_url, headers=headers, json=dispatch_data) # 🔧 修复:现在 dispatch_url 已定义
response = requests.post(dispatch_url, headers=headers, json=dispatch_data, timeout=30)
if response.status_code == 204:
print(f"✓ Successfully triggered workflow: {workflow_name}")
@@ -53,7 +54,7 @@ def get_workflow_id(github_token, repo, workflow_name):
}
url = f"https://api.github.com/repos/{repo}/actions/workflows"
response = requests.get(url, headers=headers)
response = requests.get(url, headers=headers, params={"per_page": 100}, timeout=30)
if response.status_code == 200:
workflows = response.json()["workflows"]
@@ -83,7 +84,11 @@ def main():
success_count = 0
for i, workflow in enumerate(workflows):
success = trigger_workflow_directly(workflow, github_token, repo)
try:
success = trigger_workflow_directly(workflow, github_token, repo)
except requests.RequestException as error:
print(f"Failed to trigger {workflow}: {error}")
success = False
if success:
success_count += 1
@@ -93,6 +98,7 @@ def main():
time.sleep(10)
print(f"Triggering completed: {success_count}/{len(workflows)} successful")
return 0 if workflows and success_count == len(workflows) else 1
if __name__ == "__main__":
main()
sys.exit(main())
@@ -29,8 +29,8 @@ def wait_for_workflows_to_appear(github_token, repo, workflow_names, start_time,
workflow_id = get_workflow_id(github_token, repo, workflow_name)
if not workflow_id:
print(f"Workflow {workflow_name} not found, skipping")
found_workflows.add(workflow_name)
print(f"Workflow {workflow_name} not found")
all_found = False
continue
# 检查是否有新的运行
@@ -60,7 +60,7 @@ def get_workflow_id(github_token, repo, workflow_name):
}
url = f"https://api.github.com/repos/{repo}/actions/workflows"
response = requests.get(url, headers=headers)
response = requests.get(url, headers=headers, params={"per_page": 100}, timeout=30)
if response.status_code == 200:
workflows = response.json()["workflows"]
@@ -77,9 +77,9 @@ def get_recent_runs(github_token, repo, workflow_id, start_time):
}
url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/runs"
params = {"per_page": 5}
params = {"per_page": 10, "event": "workflow_dispatch", "branch": "master"}
response = requests.get(url, headers=headers, params=params)
response = requests.get(url, headers=headers, params=params, timeout=30)
if response.status_code != 200:
return []
@@ -107,7 +107,8 @@ def main():
success = wait_for_workflows_to_appear(github_token, repo, workflows, start_time)
if not success:
print("Proceeding anyway, some workflows may not be detected...")
print("Some workflows did not start within the waiting period")
return 0 if success else 1
if __name__ == "__main__":
main()
sys.exit(main())