ardupilot/Tools/scripts/board_list.py

#!/usr/bin/env python3

'''
list of boards for build_binaries.py and custom build server

AP_FLAKE8_CLEAN
'''

import os
import re
import sys
import fnmatch
from collections.abc import Collection
from typing import Optional

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../libraries/AP_HAL/hwdef/scripts'))
import hwdef  # noqa:E402

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../libraries/AP_HAL_ChibiOS/hwdef/scripts'))
import chibios_hwdef  # noqa:E402

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../libraries/AP_HAL_Linux/hwdef/scripts'))
import linux_hwdef  # noqa:E402


class Board(object):
    def __init__(self, name : str, directory : str = None) -> None:
        self.name = name
        self.is_ap_periph = False
        self.toolchain = 'arm-none-eabi'  # FIXME: try to remove this?
        self.autobuild_targets = [
            'antennatracker',
            'blimp',
            'copter',
            'heli',
            'plane',
            'rover',
            'sub',
        ]
        SITL_toolchain = {
            "SITL_x86_64_linux_gnu": "x86_64-linux-gnu",
            "SITL_arm_linux_gnueabihf": "arm-linux-gnueabihf",
        }
        if name in SITL_toolchain:
            self.toolchain = SITL_toolchain[name]
        self.directory = directory

    def __repr__(self) -> str:
        return f"Board({self.name})"

    def get_HWDef(self) -> Optional[hwdef.HWDef]:
        '''return a HWDef object for the board'''
        if self.directory is None:
            # eg. SITL_x86_64_linux_gnu - to be fixed!
            return
        hwdef_filepath = os.path.join(self.directory, "hwdef.dat")
        if not os.path.exists(hwdef_filepath):
            return None

        cls = self.hwdef_class()
        hwdef = cls(hwdef=[hwdef_filepath], quiet=True)
        hwdef.process_hwdefs()
        return hwdef

    def hwdef_class(self) -> type:
        if "AP_HAL_ChibiOS" in self.directory:
            return chibios_hwdef.ChibiOSHWDef
        if "AP_HAL_Linux" in self.directory:
            return linux_hwdef.LinuxHWDef
        raise ValueError(f"Unknown hwdef dir {self.directory}")

    def assert_ok_no_bootloader_hwdef_for_board(self) -> None:
        cls = self.hwdef_class()
        if cls == linux_hwdef.LinuxHWDef:
            # Linux boards don't have bootloaders
            return

        non_bootloader_hwdef = self.get_HWDef()
        if non_bootloader_hwdef is None:
            raise ValueError("Must have either bootloader hwdef or main hwdef file or we shouldn't get here")
        other = non_bootloader_hwdef.get_config(
            "USE_BOOTLOADER_FROM_BOARD",
            default=None,
            required=False,
        )
        if other is None:
            if self.name in BoardList().bootloader_blacklist:
                # it's OK we don't have a bootloader for this board
                return
            raise ValueError(f"Unable to find bootloader in hwdef {self.directory}")
        if self.name in BoardList().bootloader_blacklist:
            raise ValueError(f"{self.name} is not supposed to have a bootloader but does (via USE_BOOTLOADER_FROM_BOARD)")

        # make sure that other board has a bootloader:
        other_board = BoardList().boards_by_name[other]

        # recurse; this assumes get_HWDef_bootloader does the same
        # assertion we are in.
        other_board.get_HWDef_bootloader()

    def get_HWDef_bootloader(self) -> Optional[hwdef.HWDef]:
        '''return a HWDef object for the board - bootloader edition!'''
        if self.directory is None:
            # eg. SITL_x86_64_linux_gnu - to be fixed!
            return
        hwdef_filepath = os.path.join(self.directory, "hwdef-bl.dat")
        if not os.path.exists(hwdef_filepath):
            self.assert_ok_no_bootloader_hwdef_for_board()
            return None
        if self.name in BoardList().bootloader_blacklist:
            raise ValueError(f"Able to get bootloader hwdef for board {self.name} in blacklist")

        cls = self.hwdef_class()
        hwdef = cls(hwdef=[hwdef_filepath], quiet=True)
        hwdef.process_hwdefs()
        return hwdef

    def uses_hwdef(self, filepath) -> bool:
        if self.directory is None:
            return False
        try:
            hwdef = self.get_HWDef()
        except FileNotFoundError:
            return False
        return hwdef.uses_filepath(filepath)

    def uses_hwdef_bootloader(self, filepath) -> bool:
        if self.directory is None:
            return False
        try:
            non_bootloader_hwdef = self.get_HWDef()
        except FileNotFoundError:
            return False
        if isinstance(non_bootloader_hwdef, linux_hwdef.LinuxHWDef):
            # this can be removed once we move get_config to HWDef!
            return False
        other = non_bootloader_hwdef.get_config(
            "USE_BOOTLOADER_FROM_BOARD",
            default=None,
            required=False,
        )
        if other is not None:
            bl = BoardList()
            for b in bl.boards:
                if b.name == other:
                    return b.uses_hwdef_bootloader(filepath)
            raise ValueError(f"Bad USE_BOOTLOADER_FROM_BOARD {other} {sorted(bl.boards, key=lambda x : x.name)}")

        hwdef = self.get_HWDef_bootloader()
        return hwdef.uses_filepath(filepath)


def in_boardlist(boards : Collection[str], board : str) -> bool:
    '''return true if board is in a collection of wildcard patterns'''
    for pattern in boards:
        if fnmatch.fnmatch(board, pattern):
            return True
    return False


class BoardList(object):

    def set_hwdef_dir(self):
        # work out wheer the hwdef files exist.  This file
        # (board_list.py) is copied into place on the autotest server,
        # so it isn't always in the same relative position to the
        # hwdef directories!
        found = False
        for relpath_bit in [
                os.path.join("..", "..", "libraries"),
                'libraries',
        ]:
            probe = os.path.join(
                os.path.dirname(os.path.realpath(__file__)),
                relpath_bit, "AP_HAL_ChibiOS", "hwdef"
            )
            if os.path.exists(probe):
                found = True
                break

        if not found:
            raise ValueError("Did not find hwdef_dir")

        realpath = os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            relpath_bit
        )

        self.hwdef_dir = []
        for haldir in 'AP_HAL_ChibiOS', 'AP_HAL_Linux':
            self.hwdef_dir.append(os.path.join(realpath, haldir, "hwdef"))

    def __init__(self):
        self.set_hwdef_dir()

        # no hwdefs for Linux boards - yet?
        self.boards = [
            Board("SITL_x86_64_linux_gnu"),
            Board("SITL_arm_linux_gnueabihf"),
        ]

        # TODO: move this information into hwdefs:
        self.bootloader_blacklist = set([
            'CubeOrange-SimOnHardWare',
            'CubeOrangePlus-SimOnHardWare',
            'CubeRedSecondary-IO',
            'fmuv2',  # no hwdef-bl.dat
            'iomcu',
            'iomcu-dshot',
            'iomcu-f103',
            'iomcu-f103-dshot',
            'iomcu-f103-8MHz-dshot',
            'iomcu_f103_8MHz',
            'luminousbee4',  # no hwdef-bl.dat
            'skyviper-v2450',  # no hwdef-bl.dat
            'skyviper-f412-rev1',  # no hwdef-bl.dat
            'skyviper-journey',  # no hwdef-bl.dat
            'Pixhawk1-1M-bdshot',  # no hwdef-bl.dat
            'RADIX2HD',  # closed-source bootloader
            'kha_eth',  # no hwdef-bl.dat
        ])

        for hwdef_dir in self.hwdef_dir:
            self.add_hwdefs_from_hwdef_dir(hwdef_dir)

        self.boards_by_name = {}
        for board in self.boards:
            self.boards_by_name[board.name] = board

    def add_hwdefs_from_hwdef_dir(self, hwdef_dir):
        for adir in os.listdir(hwdef_dir):
            if adir is None:
                continue
            if not os.path.isdir(os.path.join(hwdef_dir, adir)):
                continue
            if adir in ["scripts", "common", "STM32CubeConf"]:
                continue
            directory = os.path.join(hwdef_dir, adir)
            filepath = os.path.join(directory, "hwdef.dat")
            if not os.path.exists(filepath):
                filepath = os.path.join(directory, "hwdef-bl.dat")
                if not os.path.exists(filepath):
                    continue

            # FIXME: we really should be using hwdef.py to parse
            # these, but it's too slow.  We use board_list in some
            # places we can't afford to be slow.
            text = self.read_hwdef(filepath)

            board = Board(adir, directory=directory)
            self.boards.append(board)
            board_toolchain_set = False
            for line in text:
                if re.match(r"^\s*env AP_PERIPH 1", line):
                    board.is_ap_periph = 1
                if re.match(r"^\s*env AP_PERIPH_HEAVY 1", line):
                    board.is_ap_periph = 1

                # a hwdef can specify which vehicles this target is valid for:
                match = re.match(r"AUTOBUILD_TARGETS\s*(.*)", line)
                if match is not None:
                    mname = match.group(1)
                    if mname.lower() == 'none':
                        board.autobuild_targets = []
                    else:
                        board.autobuild_targets = [
                            x.rstrip().lstrip().lower() for x in mname.split(",")
                        ]

                m = re.match(r"\s*env\s*TOOLCHAIN\s*([-\w]+)\s*", line)
                if m is not None:
                    board.toolchain = m.group(1)
                    board_toolchain_set = True
                    if board.toolchain == 'native':
                        board.toolchain = None

            # toolchain not in hwdef; make up some defaults:
            if not board_toolchain_set:
                if "Linux" in hwdef_dir:
                    board.toolchain = 'arm-linux-gnueabihf'
                elif "ChibiOS" in hwdef_dir:
                    board.toolchain = 'arm-none-eabi'
                else:
                    raise ValueError(f"Unable to determine toolchain for {adir}")

    def read_hwdef(self, filepath):
        fh = open(filepath)
        ret = []
        text = fh.readlines()
        for line in text:
            m = re.match(r"^\s*include\s+(.+)\s*$", line)
            if m is not None:
                ret += self.read_hwdef(os.path.join(os.path.dirname(filepath), m.group(1)))
            else:
                ret += [line]
        return ret

    def find_autobuild_boards(self, build_target=None, skip : Collection[str] = None):
        ret = []
        for board in self.boards:
            if board.is_ap_periph:
                continue
            ret.append(board.name)

        # these were missing in the original list for unknown reasons.
        # Omitting them for backwards-compatability here - but we
        # should probably have a line in the hwdef indicating they
        # shouldn't be auto-built...
        if skip is None:
            skip = [
                # IOMCU:
                "iomcu",
                'iomcu_f103_8MHz',

                # bdshot
                "fmuv3-bdshot",

                # renamed to KakuteH7Mini-Nand
                "KakuteH7Miniv2",

                # renamed to AtomRCF405NAVI
                "AtomRCF405"

                # other
                "crazyflie2",
                "CubeOrange-joey",
                "luminousbee4",
                "MazzyStarDrone",
                "omnibusf4pro-one",
                "skyviper-f412-rev1",
                "SkystarsH7HD",
                "*-ODID",
                "*-ODID-heli",
            ]

        ret = filter(lambda x : not in_boardlist(skip, x), ret)

        # if the caller has supplied a vehicle to limit to then we do that here:
        if build_target is not None:
            # Slow down: n^2 algorithm ahead
            newret = []
            for x in ret:
                for b in self.boards:
                    if b.name.lower() != x.lower():
                        continue
                    if build_target.lower() not in [y.lower() for y in b.autobuild_targets]:
                        continue
                    newret.append(x)
            ret = newret

        return sorted(list(ret))

    def find_ap_periph_boards(self, skip : Collection[str] = None):
        if skip is None:
            skip = [
                "CubeOrange-periph-heavy",
                "f103-HWESC",
                "f103-Trigger",
                "G4-ESC",
            ]
        ret = []
        for x in self.boards:
            if not x.is_ap_periph:
                continue
            if in_boardlist(skip, x.name):
                continue
            ret.append(x.name)
        return sorted(list(ret))


AUTOBUILD_BOARDS = BoardList().find_autobuild_boards()
AP_PERIPH_BOARDS = BoardList().find_ap_periph_boards()

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='list boards to build')

    parser.add_argument('target')
    parser.add_argument('--per-line', action='store_true', default=False, help='list one per line for use with xargs')
    args = parser.parse_args()
    board_list = BoardList()
    target = args.target
    if target == "AP_Periph":
        blist = board_list.find_ap_periph_boards()
    else:
        blist = board_list.find_autobuild_boards(target)
    blist = sorted(blist)
    if args.per_line:
        for b in blist:
            print(b)
    else:
        print(blist)