From 69052cd2ea3049e2df71665c7b2b1ffefbb11706 Mon Sep 17 00:00:00 2001 From: Victor Wheeler Date: Thu, 20 Mar 2025 06:54:45 -0600 Subject: [PATCH] docs(doc_builder.py): overhaul, remove bugs, clean up, modularize (#7913) --- docs/.gitignore | 1 + docs/Doxyfile | 62 +- docs/README.md | 36 +- docs/announce.py | 80 ++ docs/api_doc_builder.py | 770 ++++++++++++ docs/build.py | 207 ++-- docs/doxygen_config.py | 429 +++++++ docs/{doc_builder.py => doxygen_xml.py} | 1318 +++++++++++---------- docs/make.bat | 2 +- docs/src/details/index.rst | 1 - scripts/gen_json/gen_json.py | 64 +- scripts/gen_json/pycparser_monkeypatch.py | 45 +- 12 files changed, 2168 insertions(+), 847 deletions(-) create mode 100644 docs/announce.py create mode 100644 docs/api_doc_builder.py create mode 100644 docs/doxygen_config.py rename docs/{doc_builder.py => doxygen_xml.py} (56%) diff --git a/docs/.gitignore b/docs/.gitignore index aaac5cea88..3c34fa6145 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,4 @@ build/ +doxygen/ intermediate/ src/_static/built_lv_examples diff --git a/docs/Doxyfile b/docs/Doxyfile index 1eaca9456c..e5a4d727d6 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "LVGL" +PROJECT_NAME = LVGL # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version @@ -58,7 +58,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = . +OUTPUT_DIRECTORY = doxygen # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and @@ -118,16 +118,16 @@ REPEAT_BRIEF = YES # the entity):The $name class, The $name widget, The $name file, is, provides, # specifies, contains, represents, a, an and the. -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ the # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then @@ -724,7 +724,7 @@ CITE_BIB_FILES = # messages are off. # The default value is: NO. -QUIET = YES +QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by doxygen. If WARNINGS is set to YES @@ -790,7 +790,7 @@ WARN_LOGFILE = doxygen_warnings.txt # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = <> +INPUT = ../src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -815,11 +815,11 @@ INPUT_ENCODING = UTF-8 # *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, # *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. -FILE_PATTERNS = *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ +FILE_PATTERNS = *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. @@ -850,14 +850,14 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = */libs/barcode/code* \ - */libs/freetype/ft* \ - */libs/gif/gif* \ - */libs/lodepng/lode* \ - */libs/qrcode/qr* \ - */libs/thorvg/* \ - */libs/tiny_ttf/stb* \ - */libs/tjpgd/tjp* \ +EXCLUDE_PATTERNS = */libs/barcode/code* \ + */libs/freetype/ft* \ + */libs/gif/gif* \ + */libs/lodepng/lode* \ + */libs/qrcode/qr* \ + */libs/thorvg/* \ + */libs/tiny_ttf/stb* \ + */libs/tjpgd/tjp* \ */others/vg_lite_tvg/vg* # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names @@ -1099,7 +1099,7 @@ GENERATE_HTML = YES # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_OUTPUT = doxygen_html +HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). @@ -1915,7 +1915,7 @@ MAN_LINKS = NO # captures the structure of the code including all documentation. # The default value is: NO. -GENERATE_XML = YES +GENERATE_XML = NO # The XML_OUTPUT tag is used to specify where the XML pages will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1932,7 +1932,7 @@ XML_OUTPUT = xml # The default value is: YES. # This tag requires that the tag GENERATE_XML is set to YES. -XML_PROGRAMLISTING = YES +XML_PROGRAMLISTING = NO #--------------------------------------------------------------------------- # Configuration options related to the DOCBOOK output @@ -2068,7 +2068,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = DOXYGEN LV_CONF_PATH="<>" +PREDEFINED = DOXYGEN LV_CONF_PATH="../../lv_conf.h" # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The diff --git a/docs/README.md b/docs/README.md index 230a689d78..511af422a0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -42,7 +42,7 @@ or if you are on a Unix like OS: python3 build.py html -Intermediate files are prepared in `./docs/intermediate/` and the final documentation will appear in `./docs/build/html/`. +Intermediate files are prepared in `./docs/intermediate/` and the final documentation will appear in `./docs/build/html/`. (Both of these directories can be overridden using environment variables. See documentation in `build.py` for details.) If the list of document source files has changed (names or paths): @@ -66,13 +66,36 @@ The below are some rules to follow when updating any of the `.rst` files located ### What to Name Your `.rst` File -The documentation-generation logic uses the stem of the file name (i.e. "event" from file name "event.rst") and compares this with code-element names found by Doxygen. If a match is found, then it appends hyperlinks to the API pages that contain those code elements (names of macros, enum/struct/union types, variables, namespaces, typedefs and functions). +The directory structure under the `./docs/src/` directory, and the filenames of the `.rst` files govern the eventual URLs that are generated in the HTML output. These directories are organized so as to reflect the nature of the content. Example: the `.rst` files under `./docs/src/intro` contain introductory material—detailed reference material would not go there, but instead in an appropriate subdirectory of `./docs/src/details/`. It is expected that the content and location of any new documents added would be in alignment with this directory structure, and placed and named according to their content. Additionally, to be linked into the eventual generated documentation, the stem of the new filename would need to appear in at least one (normally *only one*) `.. toctree::` directive, normally in an `index.rst` file in the directory where it will appear in the table of contents (TOC). -If this is appropriate for the .RST file you are creating, ensure the stem of the file name matches the beginning part of the code-element name you want it to be associated with. +Other than that, there are no restrictions on filenames. Previous linking of filenames to generated API links has been removed and replaced by a better scheme. For sake of illustration, let's say you are creating (or enhancing) documentation related to the `lv_scale_t` data type (one of the LVGL Widgets): if you want the doc-build logic to generate appropriate links to LVGL API pages, place an API section at the end of your document (it must be at the end) like this: -If this is *not* appropriate for the .RST file you are creating, ensure the stem of the file name DOES NOT match any code-element names found in the LVGL header files under the ./src/ directory. +```rst +API +*** +``` -In alignment with the above, use a file name stem that is appropriate to the topic being covered. +and then, if you want the API-link-generation logic to generate hyperlinks to API pages based on an ***exact, case-sensitive string match*** with specific C symbols, follow it with a reStructuredText comment using this syntax: + +```rst +.. API equals: lv_scale_t, lv_scale_create +``` + +What follows the colon is a comma- or space-separated list of exact C symbols documented somewhere in the `lvgl/src/` directory. If the list is long, it can be wrapped to subsequent lines, though continuation lines must be all indented at the same level. The list ends with the first blank line after this pseudo-directive. + +If you instead want the API-link-generation logic to simply include links to code that ***starts with a specific string*** use this syntax instead. The format of the list is the same as for `.. API equals:`: + +```rst +.. API startswith: lv_scale, lv_obj_set_style +``` + +You can also manually link to API pages, in which case the API-link-generation logic will see that you have already added links and will not repeat them. + +```rst +:ref:`lv_scale_h` +``` + +Note that the period before the `h` is replaced with an underscore (`_`). The naming of this reference (`lv_scale_h`) will generate a hyperlink to the documentation extracted by Doxygen from the `lvgl/src/widgets/scale/lv_scale.h` file. ### Text Format @@ -234,7 +257,7 @@ To create a bulleted list, do the following: lines to align with item text like this. - If you want to include a code block under a list item, it must be intended to align with the list item like this: - + .. code-block: python <=== blank line here is important # this is some code @@ -286,4 +309,3 @@ For such examples, simply use reStructuredText literal markup like this: ``lv_obj_set_layout(&widget, LV_LAYOUT_FLEX);`` ``lv_obj_set_layout(widget, ...)`` - diff --git a/docs/announce.py b/docs/announce.py new file mode 100644 index 0000000000..e700fa77b7 --- /dev/null +++ b/docs/announce.py @@ -0,0 +1,80 @@ +"""announce.py +Manage logging announcements to `stdout` + +It is the designer's intention that: + +1. The variable `__file__` be passed as the first argument in + `announce()` and `announce_start()`. + (Unfortunately, there is no way this author knows of yet to + have this module know what Python module is importing it. So + this is a hold-over requirement until that need is fulfilled.) + +2. `announce_start()` and `announce_finish()` should be used + in pairs like this: + + announce_start(__file__, 'something is running...') + # do something that takes a while here + announce_finish() + +3. If this is used in a module that sometimes has a need to + not have anything output to STDOUT, when that is known, + call `announce_set_silent_mode()`. To turn "silent mode" + off, call `announce_set_silent_mode(False)`. + +""" +import os +import datetime + +__all__ = ('announce', 'announce_start', 'announce_finish', 'announce_set_silent_mode') +_announce_start_time: datetime.datetime +_announce_silent_mode: bool = False + + +def _announce(file: str, args: tuple, start=False): + if _announce_silent_mode: + return + + _args = [] + + for arg in args: + # Avoid the single quotes `repr()` puts around strings. + if type(arg) is str: + _args.append(arg) + else: + _args.append(repr(arg)) + + if start: + _end = '' + else: + _end = '\n' + + print(f'{os.path.basename(file)}: ', ' '.join(_args), end=_end, flush=True) + + +def announce(file: str, *args, start=False, finish=False): + global _announce_start_time + _announce_start_time = None + _announce(file, args) + + +def announce_start(file: str, *args, start=False, finish=False): + global _announce_start_time + _announce_start_time = datetime.datetime.now() + _announce(file, args, start=True) + + +def announce_finish(): + # Just output line ending to terminate output for `announce_start()`. + global _announce_start_time + if _announce_start_time is not None: + if not _announce_silent_mode: + print(' Elapsed: ', datetime.datetime.now() - _announce_start_time, flush=True) + _announce_start_time = None + else: + if not _announce_silent_mode: + print(flush=True) + + +def announce_set_silent_mode(mode=True): + global _announce_silent_mode + _announce_silent_mode = mode diff --git a/docs/api_doc_builder.py b/docs/api_doc_builder.py new file mode 100644 index 0000000000..64c6f112df --- /dev/null +++ b/docs/api_doc_builder.py @@ -0,0 +1,770 @@ +"""api_doc_builder.py + +Create and provide links to API pages in LVGL doc build. + +Uses doxygen_xml.py module to: + +- Prep and run Doxygen +- make Doxygen XML output available, and +- make Doxygen-documented symbols from the C code available. +""" +import os +import re +import doxygen_xml +from announce import * + +old_html_files = {} +EMIT_WARNINGS = True +rst_section_line_char = '=' + +# Multi-line match ``API + newline + \*\*\* + whitespace``. +# NB: the ``\s*`` at the end forces the regex to match the whitespace +# at the end including all \r\n's. This will match UP TO: +# - the next non-blank character (could be many blank lines), or +# - to the end of the file, whichever comes first. +_re_api_section_sep = re.compile(r'(?mi)^API *\r?\n^\*\*\*\s*') + +# Regex to identify '.. API equals: lv_obj_t, lv_array_t' directives. +_re_api_equals = re.compile(r'(?mi)^\s*\.\.\s+API\s+equals:\s*([\w,\s]+)\r\n\s*') + +# Regex to identify '.. API startswith: lv_obj, lv_array' directives. +_re_api_startswith = re.compile(r'(?mi)^\s*\.\.\s+API\s+startswith:\s*([\w,\s]+)\r\n\s*') + +# Regex to match comma and whitespace list-item separators on multiple lines. +_re_multi_line_comma_sep = re.compile(r'(?m)[,\s]+') + +# Regex to identify editor-added hyperlinks: :ref:`lv_obj_h` +_re_editor_added_hyperlink = re.compile(r'^\s*:ref:`(\w+)`') + +# Separator to mark place where this script added hyperlinks. +_auto_gen_sep = '.. Autogenerated' + +# List of symbol dictionaries +_defines = {} +_enums = {} +_variables = {} +_namespaces = {} +_structs = {} +_unions = {} +_typedefs = {} +_functions = {} + +_symbol_dict_list = [ + _defines, + _enums, + _variables, + _namespaces, + _structs, + _unions, + _typedefs, + _functions +] + + +def old_clean_name(nme): + """Strip beginning "_lv" and ending "_t".""" + # Handle error: + # AttributeError: 'NoneType' object has no attribute 'startswith' + if nme is None: + return nme + + if nme.startswith('_lv_'): + nme = nme[4:] + elif nme.startswith('lv_'): + nme = nme[3:] + + if nme.endswith('_t'): + nme = nme[:-2] + + return nme + + +# Definitions: +# - "section" => The name "abc_def" has 2 sections. +# - N = number of sections in `item_name`. +# After removing leading '_lv_', 'lv_' and trailing '_t' from `obj_name`, +# do the remaining first N "sections" of `obj_name` match `item_name` +# (case sensitive)? +def old_is_name_match(item_name, obj_name): + # Handle error: + # AttributeError: 'NoneType' object has no attribute 'split' + if obj_name is None: + return False + + sect_count = item_name.count('_') + 1 + + obj_name = obj_name.split('_') + + # Reject (False) if `obj_name` doesn't have as many sections as `item_name`. + if len(obj_name) < sect_count: + return False + + obj_name = '_'.join(obj_name[:sect_count]) + + return item_name == obj_name + + +def old_get_includes(name1, name2, obj, includes): + name2 = old_clean_name(name2) + + if not old_is_name_match(name1, name2): + return + + if obj.parent is not None and hasattr(obj.parent, 'header_file'): + header_file = obj.parent.header_file + elif hasattr(obj, 'header_file'): + header_file = obj.header_file + else: + return + + if not header_file: + return + + if header_file not in old_html_files: + return + + includes.add((header_file, old_html_files[header_file])) + + +def _conditionally_add_hyperlink(obj, genned_link_set: set, exclude_set: set): + """ + Add hyperlink names to `link_set` if: + - not in `exclude_set`, and + - not already in `link_set`. + :param obj: "thing" from dictionary with matching symbol. + These are objects instantiated from classes + in `doxygen_xml` module such as + STRUCT, FUNCTION, DEFINE, ENUMVALUE, etc. + :param genned_link_set: Set in which to accumulate link names + :param exclude_set: Set with link names not to add to `link_set` + :return: + """ + if obj.file_name is not None: + link_name = os.path.basename(obj.file_name).replace('.', '_') + if link_name not in genned_link_set: + if link_name not in exclude_set: + genned_link_set.add(link_name) + + +def _add_startswith_matches(strings: [str], genned_link_set, editor_link_set): + """ + Add set of hyperlinks to `genned_link_set` that are not already in + `editor_link_set`, for C symbols that start with strings in `strings`. + + :param strings: List of strings to match against + :param genned_link_set: Generated link set + :param editor_link_set: Hyperlinks added by editor + :return: n/a + """ + for partial_symbol in strings: + for symbol_dict in _symbol_dict_list: + for key in symbol_dict: + if key is None: + # Dictionary `enums` has a key `None` which contains + # all enumvalues from all unnamed enums, and each + # enumvalue has a `file_name` field. + enum_values_list = symbol_dict[None].members + for enum_val in enum_values_list: + if enum_val.name.startswith(partial_symbol): + _conditionally_add_hyperlink(enum_val, genned_link_set, editor_link_set) + else: + if key.startswith(partial_symbol): + obj = symbol_dict[key] + _conditionally_add_hyperlink(obj, genned_link_set, editor_link_set) + + +def _add_exact_matches(symbols: [str], genned_link_set, editor_link_set): + """ + Add set of hyperlinks to `genned_link_set` that are not already in + `editor_link_set`, for exact C symbol matches. + + :param symbols: List of C symbols to match against + :param genned_link_set: Generated link set + :param editor_link_set: Hyperlinks added by editor + :return: n/a + """ + for symbol in symbols: + for symbol_dict in _symbol_dict_list: + if symbol in symbol_dict: + obj = symbol_dict[symbol] + _conditionally_add_hyperlink(obj, genned_link_set, editor_link_set) + + +def _hyperlink_sort_value(init_value: str): + if init_value.endswith('_h'): + result = init_value[:-2] + else: + result = init_value + + return result + + +def _process_end_of_eligible_doc(b: str, rst_file: str) -> (str, str, int): + """ + Edit end section after API section heading. + + 3. Initialize: + - links_added_count = 0 + - editor_link_set = set() + - genned_link_set = set() + - C = '' # string for generated hyperlinks + 4. Remove `_auto_gen_sep` and everything after it: + - new_B = B.split(_auto_gen_sep, 1)[0] + 5. With `new_B, add any editor-added hyperlinks to set: + `editor_link_set`. + 6. If `_re_api_equals` match present: + - build list of symbols + - compute list of hyperlinks from symbols + - add hyperlinks to `genned_link_set` if not in `editor_link_set`. + 7. If `_re_api_startswith` match present: + - build list of symbols + - compute list of hyperlinks from symbols + - add hyperlinks to `genned_link_set` if not in `editor_link_set`. + 8. Lacking either of the above custom directives, use the lower-case + stem of the filename and prefix it with "lv_" and try an + "API startswith" search with it. + 9. If len(genned_link_set) > 0: + - `C` = _auto_gen_sep + '\n\n' + `genned_link_set` + (with a blank line between each). + 10. Return tuple: (new_B, C, links_added_count). + + :param b: End of document after API section heading + whitespace. + :return: Tuple: (new_B, C, links_added_count) + """ + # 3. Initialize: + editor_link_set = set() + genned_link_set = set() + c = '' + api_directives_found_count = 0 + + # 4. Remove `_auto_gen_sep` and everything after it: + new_b = b.split(_auto_gen_sep, 1)[0] + + # 5. With `new_B, add any editor-added hyperlinks to set: + # `editor_link_set`. + for line in new_b.splitlines(): + match = _re_editor_added_hyperlink.match(line) + if match is not None: + editor_link_set.add(match[1]) + + # 6. If `_re_api_equals` present: + # - build list of symbols + # - compute list of hyperlinks from symbols + # - add hyperlinks to `genned_link_set` if not in `editor_link_set`. + match = _re_api_equals.search(new_b) + if match is not None: + api_directives_found_count += 1 + comma_sep_list = match[1].strip() + symbols = _re_multi_line_comma_sep.split(comma_sep_list) + _add_exact_matches(symbols, genned_link_set, editor_link_set) + + # 7. If `_re_api_startswith` present: + # - build list of symbols + # - compute list of hyperlinks from symbols + # - add hyperlinks to `genned_link_set` if not in `editor_link_set`. + match = _re_api_startswith.search(new_b) + if match is not None: + api_directives_found_count += 1 + comma_sep_list = match[1].strip() + symbols = _re_multi_line_comma_sep.split(comma_sep_list) + _add_startswith_matches(symbols, genned_link_set, editor_link_set) + + # 8. Lacking either of the above custom directives, use the lower-case + # stem of the filename and prefix it with "lv_" and try an + # "API startswith" with it. + if api_directives_found_count == 0: + base = os.path.basename(rst_file) + stem = os.path.splitext(base)[0] + _add_startswith_matches(['lv_' + stem], genned_link_set, editor_link_set) + + # 9. If len(genned_link_set) > 0: + # - `C` = _auto_gen_sep + '\n\n' + `genned_link_set` + # (with a blank line between each). + links_added_count = len(genned_link_set) + + if links_added_count > 0: + c = _auto_gen_sep + '\n\n' + for link_name in sorted(genned_link_set, key=_hyperlink_sort_value): + c += ':ref:`' + link_name + '`\n\n' + + # 10. Return tuple: (new_B, C, links_added_count). + return new_b, c, links_added_count + + +def _process_one_file(rst_file: str): + """ + Add applicable API hyperlinks to one file. + + Eligible + An `.rst` file is eligible if it contains an API section at + its end. This can happen also in `index.rst` files when + they head a single subject for which an API section is + appropriate there and not in the sub-docs. So `index.rst` + files are included, whereas they were not included previously. + + Algorithm: + ---------- + A. Doc editors may have already added a set of hyperlinks of their + own. This routine takes note of and does not duplicate what is + already there. + + B. Doc editors may also have added specifications for this routine + that look like this: + + .. API equals: lv_obj_t, lv_arc_t, lv_barcode_t, + lv_win_t, lv_list_t, + lv_button_t + + .. API startswith: lv_obj, lv_arc, lv_barcode, + lv_win, lv_list, + lv_button + + as directives for this routine to build a set of applicable + hyperlinks. + + C. Lacking any of the above custom directives, use the lower-case + stem of the filename and prefix it with "lv_" and try an + "API startswith" search with it. + + Any hyperlinks added by this routine are prefixed with the + reStructuredText comment defined by the `_auto_gen_sep` + variable, normally: + + .. Autogenerated + + If `rst_file` is eligible, edit after API section heading such that: + - any editor-added hyperlinks are retained at the top of the list; + - `_auto_gen_sep` (looked for in case a source file ends up having it; + anything after it is replaced); + - applicable hyperlinks added such that they do not repeat those + added by editors of `.rst` file. + + Steps to Implement: + ------------------- + 0. links_added_count = 0 + 1. Determine if eligible. + - If not, skip to step 12. + - If so, continue. + 2. Split doc into 2 parts: + A. beginning through API section heading and subsequent + whitespace including subsequent blank lines; + B. anything after that which may include editor-added hyperlinks. + 3-10. new_B, C, links_added_count = _process_end_of_eligible_doc() + 11. Write `A` + `new_B` + `C` back to `rst_file`. + 12. Return links_added_count. + + :param rst_file: Full path to `.rst` file in question. + It may or may not be eligible. + :return: Number of links added. + """ + links_added_count = 0 + + with open(rst_file, 'rb') as f: + try: + rst_contents = f.read().decode('utf-8') + except UnicodeDecodeError: + announce(__file__, f'Error: UnicodeDecodeError in [{rst_file}].') + raise + + eligible_match = _re_api_section_sep.search(rst_contents) + + if eligible_match is not None: + # Eligible (API section found). + i = eligible_match.end() + # Split just after the API section heading + whitespace. + a = rst_contents[:i] + b = rst_contents[i:] + new_b, c, links_added_count = _process_end_of_eligible_doc(b, rst_file) + + if links_added_count > 0: + rst_contents = a + new_b + c + + with open(rst_file, 'wb') as f: + f.write(rst_contents.encode('utf-8')) + + return links_added_count + + +def _build_one_local_dictionary(local_dict, remote_dict): + """ + Remove '_' prefix in symbols beginning with '_lv' to make + symbols like `lv_obj_t` actually connect with the struct + in `lv_obj_private.h`, and not the typedef in `lv_types.h`. + + :param local_dict: Local (adjusted) symbol dictionary + :param remote_dict: Dictionary from `doxygen_xml` module + :return: n/a + """ + for symbol in remote_dict: + # Note: symbol `None` is actually a valid symbol in the + # `enums` dictionary, containing all enumvalue symbols + # for enums without names. + if symbol is None or not symbol.startswith('_lv'): + loc_symbol = symbol + else: + # Remove '_' prefix. + loc_symbol = symbol[1:] + + local_dict[loc_symbol] = remote_dict[symbol] + + +def _build_local_symbol_dictionaries(): + """ + Build "work-around" dictionaries so that a symbol like `lv_obj_t` + actually connects with the struct in `lv_obj_private.h`, and not + the typedef in `lv_types.h`. + + :return: n/a + """ + _build_one_local_dictionary(_defines, doxygen_xml.defines) + _build_one_local_dictionary(_enums, doxygen_xml.enums) + _build_one_local_dictionary(_variables, doxygen_xml.variables) + _build_one_local_dictionary(_namespaces, doxygen_xml.namespaces) + _build_one_local_dictionary(_structs, doxygen_xml.structures) + _build_one_local_dictionary(_unions, doxygen_xml.unions) + _build_one_local_dictionary(_typedefs, doxygen_xml.typedefs) + _build_one_local_dictionary(_functions, doxygen_xml.functions) + + +def _add_hyperlinks_to_eligible_files(intermediate_dir: str, + new_algorithm: bool, + *doc_rel_paths: [str]): + """ + Add applicable hyperlinks to eligible docs found joining + `intermediate_dir` with each relative path in `doc_rel_paths`. + + See API-link algorithm documented under `_process_one_file()`. + + :param intermediate_dir: Top directory where hyperlinks are to be added. + :param doc_rel_paths: Tuple of relative paths from `intermediate_dir` to + walk to find docs eligible for API hyperlinks. + :return: + """ + if new_algorithm: + # Populate local symbol dictionary set with + # symbols WITHOUT any '_' prefixes. + _build_local_symbol_dictionaries() + + # Build `.rst` file list. + file_list = [] + + for rel_path in doc_rel_paths: + top_dir = os.path.join(intermediate_dir, rel_path) + for dir_bep, sub_dirs, files in os.walk(top_dir, topdown=False): + for file in files: + if file.lower().endswith('.rst'): + file_list.append(os.path.join(dir_bep, file)) + + total_eligible_doc_count = 0 + total_links_added_count = 0 + + # For each `.rst` file, add appropriate API hyperlinks. + for rst_file in file_list: + links_added_count = _process_one_file(rst_file) + + if links_added_count > 0: + total_links_added_count += links_added_count + total_eligible_doc_count += 1 + # announce(__file__, f'Eligible doc: [{rst_file}].') + + announce(__file__, f'Docs eligible for API hyperlinks: {total_eligible_doc_count:>4}') + announce(__file__, f'API hyperlinks added : {total_links_added_count:>4}') + else: + for folder in doc_rel_paths: + # Fetch a list of '.rst' files excluding 'index.rst'. + rst_files = list( + (os.path.splitext(item)[0], os.path.join(folder, item)) + for item in os.listdir(folder) + if item.endswith('.rst') and 'index.rst' not in item + ) + + # For each .RST file in that directory... + for stem, path in rst_files: + # Start with an empty set. + html_includes = set() + + # Build `html_includes` set as a list of tuples containing + # (name, html_file). Example: "draw.rst" has `stem` == 'draw', + # and generates a list of tuples from .H files where matching + # C-code-element names were found. Example: + # {('lv_draw_line', 'draw\\lv_draw_line.html'), + # ('lv_draw_sdl', 'draw\\sdl\\lv_draw_sdl.html'), + # ('lv_draw_sw_blend_to_i1', 'draw\\sw\\blend\\lv_draw_sw_blend_to_i1.html'), + # etc.} + for symbol_dict in ( + doxygen_xml.defines, + doxygen_xml.enums, + doxygen_xml.variables, + doxygen_xml.namespaces, + doxygen_xml.structures, + doxygen_xml.unions, + doxygen_xml.typedefs, + doxygen_xml.functions + ): + for key, obj in symbol_dict.items(): + old_get_includes(stem, key, obj, html_includes) + + if html_includes: + # Convert `html_includes` set to a list of strings containing the + # Sphinx hyperlink syntax "link references". Example from above: + # [':ref:`lv_draw_line_h`\n', + # ':ref:`lv_draw_sdl_h`\n', + # ':ref:`lv_draw_sw_blend_to_i1_h`\n', + # etc.] + html_includes = list( + ':ref:`{0}_h`\n'.format(inc) + for inc, _ in html_includes + ) + + # Convert that list to a single string of Sphinx hyperlink + # references with blank lines between them. + # :ref:`lv_draw_line_h` + # + # :ref:`lv_draw_sdl_h` + # + # :ref:`lv_draw_sw_blend_to_i1_h` + # + # etc. + output = ('\n'.join(html_includes)) + '\n' + + # Append that string to the source .RST file being processed. + with open(path, 'rb') as f: + try: + data = f.read().decode('utf-8') + except UnicodeDecodeError: + print(path) + raise + + data = data.split(_auto_gen_sep, 1)[0] + + data += f'{_auto_gen_sep}\n\n' + data += output + + with open(path, 'wb') as f: + f.write(data.encode('utf-8')) + + +def _create_rst_files_for_dir(src_root_dir_len: int, + src_dir_bep: str, + elig_h_files: [str], + elig_sub_dirs: [str], + out_root_dir: str): + """ + - Create `index.rst` file and add its top section. + - For each file in `elig_h_files`: + - Create one `.rst` file. + - Add reference to it in `index.rst`. + - For each subdir in `elig_sub_dirs`: + - add reference "sub_dir_name/index" in `index.rst`. + + :param src_root_dir_len: Length of source-root path string, used with `out_root_dir` to build paths + :param src_dir_bep: Directory currently *being processed* + :param elig_h_files: Eligible `.h` files directly contained in `src_dir_bep` + :param elig_sub_dirs: List of sub-dirs that contained eligible `.h` files + :param out_root_dir: Root of output directory, used with to build paths. + :return: n/a + """ + indent = ' ' + sub_path = src_dir_bep[src_root_dir_len:] + out_dir = str(os.path.join(out_root_dir, sub_path)) + + # Ensure dir exists. Multiple dirs MAY have to be created + # since `.rst` files are created in bottom-up sequence. + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + + # For top-level directory only... (the last index.rst created, + # since they are created in bottom-up sequence) + if len(sub_path) == 0 and out_dir.endswith(os.sep): + # Trim trailing slash from `out_dir`. + out_dir = out_dir[:-1] + + # index.rst + with open(os.path.join(out_dir, 'index.rst'), 'w') as f: + subdir_stem = os.path.split(out_dir)[-1] + section_line = (rst_section_line_char * len(subdir_stem)) + '\n' + f.write(section_line) + f.write(subdir_stem + '\n') + f.write(section_line) + f.write('\n') + f.write('.. toctree::\n :maxdepth: 2\n\n') + + # One entry per `.rst` file + for h_file in elig_h_files: + filename = os.path.basename(h_file) + stem = os.path.splitext(filename)[0] + f.write(indent + stem + '\n') + + # One entry per eligible subdirectory. + for sub_dir in elig_sub_dirs: + stem = os.path.split(sub_dir)[-1] + f.write(indent + stem + '/index\n') + + # One .rst file per h_file + for h_file in elig_h_files: + filename = os.path.basename(h_file) + stem = os.path.splitext(filename)[0] + rst_file = os.path.join(out_dir, stem + '.rst') + html_file = os.path.join(sub_path, stem + '.html') + old_html_files[stem] = html_file + + with open(rst_file, 'w') as f: + # Sphinx link target. + f.write(f'.. _{stem}_h:\n\n') + # Doc title. + section_line = (rst_section_line_char * len(filename)) + '\n' + f.write(section_line) + f.write(filename + '\n') + f.write(section_line) + f.write('\n') + # Content for `breathe`. + f.write(f'.. doxygenfile:: {filename}\n') + f.write(' :project: lvgl\n\n') + + +def _recursively_create_api_rst_files(depth: int, + src_root_len: int, + src_dir_bep: str, + out_root_dir: str) -> int: + """ + Create `.rst` files for the eligible `.h` found in `src_dir_bep` and + recursively for subdirectories below it. ("bep" = being processed.) + + Eligible + An `.h` file is eligible if Doxygen generated documentation for it. + The `EXCLUDE_PATTERNS` Doxygen configuration value can cause + Doxygen to skip certain files and directories, in which case, + the `.h` files skipped ARE NOT eligible. + + Whether a subdirectory is eligible to be included in an `index.rst` + file depends upon whether any eligible `.h` files were recursively + found within it. And that isn't known until this function finishes + (recursively) processing a directory and returns the number of + eligible `.h` files found. Thus, the steps taken within are: + + - Discover all eligible `.h` files directly contained in `src_dir_bep`. + - Recursively do the same for each subdirectory, adding the returned + count of eligible `.h` files to the sum (`elig_h_file_count`). + - If `elig_h_file_count > 0`: + - call _create_rst_files_for_dir() to generate appropriate + `.rst` files for this directory. + - Return `elig_h_file_count`. + + Once we have accumulated this information, then we can generate + all the `.rst` files for the current directory without any further + directory-tree walking. + + :param depth: Only used for testing/debugging + :param src_root_len: Length of source-root path + :param src_dir_bep: Source directory *being processed* + :param out_root_dir: Output root directory (used to build output paths) + :return: Number of `.h` files encountered (so caller knows + whether that directory recursively held any + eligible `.h` files, to know whether to include + "subdir/index" in caller's local `index.rst` file). + """ + elig_h_files = [] + sub_dirs = [] + elig_sub_dirs = [] + elig_h_file_count = 0 + + # For each "thing" found in `src_dir_bep`, build lists: + # `elig_sub_dirs` and `elig_h_files`. + # By design change, we are including files with 'private' + # in their names. Reason: advanced users who need to use + # the structs defined within will need the documentation + # in those API pages! + for dir_item in os.listdir(src_dir_bep): + path_bep = os.path.join(src_dir_bep, dir_item) + if os.path.isdir(path_bep): + sub_dirs.append(path_bep) # Add to sub-dir list. + else: + if dir_item.lower().endswith('.h'): + eligible = (dir_item in doxygen_xml.files) + if eligible: + elig_h_files.append(path_bep) # Add to .H file list. + elig_h_file_count += 1 + + # For each subdir... + for sub_dir in sub_dirs: + subdir_eligible_h_file_count = \ + _recursively_create_api_rst_files(depth + 1, + src_root_len, + sub_dir, + out_root_dir) + + if subdir_eligible_h_file_count > 0: + elig_sub_dirs.append(sub_dir) + elig_h_file_count += subdir_eligible_h_file_count + + if elig_h_file_count > 0: + # Create index.rst plus .RST files for any direct .H files in dir. + _create_rst_files_for_dir(src_root_len, + src_dir_bep, + elig_h_files, + elig_sub_dirs, + out_root_dir) + + return elig_h_file_count + + +def create_api_rst_files(src_root_dir: str, out_root_dir: str): + """ + Create `.rst` files for API pages based on the `.h` files found + in a tree-walk of `a_src_root` and the current contents of the + `doxygen_xml.files` dictionary (used to filter out `.h` files that + Doxygen generated no documentation for). Output the `.rst` files + into `out_root_dir` mirroring the `a_src_root` directory structure. + + :param src_root_dir: root source directory to walk + :param out_root_dir: output directory + :return: n/a + """ + src_root_len = len(src_root_dir) + 1 + _recursively_create_api_rst_files(0, src_root_len, src_root_dir, out_root_dir) + + +def build_api_docs(lvgl_src_dir, intermediate_dir, doxyfile_src_file, *doc_rel_paths): + """ + - Prep and run Doxygen, outputting XML. + - Load that XML in a form that can quickly tie C symbols to the + source files they came from. + - Generate API page `.rst` files for source files Doxygen generated + documentation for. + - Add hyperlinks to these API pages for `.rst` files in `*doc_rel_paths` + that are eligible. + + :param lvgl_src_dir: Path to LVGL src directory + :param intermediate_dir: Path to intermediate dir being built + :param doxyfile_src_file: Full path to src doxygen configuration file + :param doc_rel_paths: List of relative paths from `intermediate_dir` to + walk to find docs eligible for API hyperlinks. + """ + # --------------------------------------------------------------------- + # - Generate Doxyfile replacing tokens, + # - run Doxygen generating XML, and + # - load the generated XML from Doxygen output. + # --------------------------------------------------------------------- + doxygen_xml.EMIT_WARNINGS = EMIT_WARNINGS + + xml_parser = doxygen_xml.DoxygenXml(lvgl_src_dir, + intermediate_dir, + doxyfile_src_file, + silent_mode=False + ) + + # --------------------------------------------------------------------- + # Generate .RST files for API pages. + # --------------------------------------------------------------------- + announce(__file__, "Generating API documentation .RST files...") + api_out_root_dir = os.path.join(intermediate_dir, 'API') + create_api_rst_files(lvgl_src_dir, api_out_root_dir) + + # --------------------------------------------------------------------- + # For each directory entry in `doc_rel_paths` array... + # - add API hyperlinks to .RST files in the directories in passed array. + # --------------------------------------------------------------------- + announce(__file__, "Adding API-page hyperlinks to source docs...") + _add_hyperlinks_to_eligible_files(intermediate_dir, + True, + *doc_rel_paths) diff --git a/docs/build.py b/docs/build.py index 8850c5bfb7..9cd6c08e26 100755 --- a/docs/build.py +++ b/docs/build.py @@ -172,12 +172,10 @@ from datetime import datetime # LVGL Custom import example_list -import doc_builder +import api_doc_builder import config_builder -_ = os.path.abspath(os.path.dirname(__file__)) -docs_src_dir = os.path.join(_, 'src') -sys.path.insert(0, docs_src_dir) -from lvgl_version import lvgl_version # NoQA +from src.lvgl_version import lvgl_version +from announce import * # Not Currently Used # (Code is kept in case we want to re-implement it later.) @@ -188,7 +186,8 @@ from lvgl_version import lvgl_version # NoQA # ------------------------------------------------------------------------- # These are relative paths from the ./docs/ directory. cfg_project_dir = '..' -cfg_src_dir = 'src' +cfg_lvgl_src_dir = 'src' +cfg_doc_src_dir = 'src' cfg_examples_dir = 'examples' cfg_default_intermediate_dir = 'intermediate' cfg_default_output_dir = 'build' @@ -224,28 +223,29 @@ def print_usage_note(): def remove_dir(tgt_dir): """Remove directory `tgt_dir`.""" if os.path.isdir(tgt_dir): - print(f'Removing {tgt_dir}...') + announce(__file__, f'Removing {tgt_dir}...') shutil.rmtree(tgt_dir) else: - print(f'{tgt_dir} already removed...') + announce(__file__, f'{tgt_dir} already removed...') -def cmd(s, start_dir=None, exit_on_error=True): +def cmd(cmd_str, start_dir=None, exit_on_error=True): """Run external command and abort build on error.""" - if start_dir is None: - start_dir = os.getcwd() + saved_dir = None - saved_dir = os.getcwd() - os.chdir(start_dir) - print("") - print(s) - print("-------------------------------------") - result = os.system(s) - os.chdir(saved_dir) + if start_dir is not None: + saved_dir = os.getcwd() + os.chdir(start_dir) - if result != 0 and exit_on_error: - print("Exiting build due to previous error.") - sys.exit(result) + announce(__file__, f'Running [{cmd_str}] in [{os.getcwd()}]...') + return_code = os.system(cmd_str) + + if saved_dir is not None: + os.chdir(saved_dir) + + if return_code != 0 and exit_on_error: + announce(__file__, "Exiting build due to previous error.") + sys.exit(return_code) def intermediate_dir_contents_exists(dir): @@ -293,7 +293,7 @@ def run(args): def print_setting(setting_name, val): """Print one setting; used for debugging.""" - print(f'{setting_name:18} = [{val}]') + announce(__file__, f'{setting_name:18} = [{val}]') def print_settings(and_exit): """Print all settings and optionally exit; used for debugging.""" @@ -361,7 +361,7 @@ def run(args): # fully regenerated, even if not changed. # Note: Sphinx runs in ./docs/, but uses `intermediate_dir` for input. if fresh_sphinx_env: - print("Force-regenerating all files...") + announce(__file__, "Force-regenerating all files...") env_opt = '-E' else: env_opt = '' @@ -383,7 +383,7 @@ def run(args): base_dir = os.path.abspath(os.path.dirname(__file__)) project_dir = os.path.abspath(os.path.join(base_dir, cfg_project_dir)) examples_dir = os.path.join(project_dir, cfg_examples_dir) - lvgl_src_dir = os.path.join(project_dir, 'src') + lvgl_src_dir = os.path.join(project_dir, cfg_lvgl_src_dir) # Establish intermediate directory. The presence of environment variable # `LVGL_DOC_BUILD_INTERMEDIATE_DIR` overrides default in `cfg_default_intermediate_dir`. @@ -440,9 +440,9 @@ def run(args): # Change to script directory for consistent run-time environment. # --------------------------------------------------------------------- os.chdir(base_dir) - print(f'Intermediate dir: [{intermediate_dir}]') - print(f'Output dir : [{output_dir}]') - print(f'Running from : [{base_dir}]') + announce(__file__, f'Intermediate dir: [{intermediate_dir}]') + announce(__file__, f'Output dir : [{output_dir}]') + announce(__file__, f'Running from : [{base_dir}]') # --------------------------------------------------------------------- # Clean? If so, clean (like `make clean`), but do not exit. @@ -451,9 +451,9 @@ def run(args): or clean_all or (os.path.isdir(intermediate_dir) and build_intermediate) if some_cleaning_to_be_done: - print("****************") - print("Cleaning...") - print("****************") + announce(__file__, "****************") + announce(__file__, "Cleaning...") + announce(__file__, "****************") if clean_intermediate: remove_dir(intermediate_dir) @@ -483,7 +483,7 @@ def run(args): # - generated search window # - establishing canonical page for search engines # - `link_roles.py` to generate translation links - # - `doc_builder.py` to generate links to API pages + # - `doxygen_xml.py` to generate links to API pages # # LVGL_GITCOMMIT is used by: # - `conf.py` => html_context['github_version'] for @@ -527,8 +527,7 @@ def run(args): os.environ['LVGL_GITCOMMIT'] = branch # --------------------------------------------------------------------- - # Copy files to 'intermediate_dir' where they will be edited (translation - # link(s) and API links) before being used to generate new docs. + # Prep `intermediate_dir` to become the `sphinx-build` source dir. # --------------------------------------------------------------------- # dirsync `exclude_list` = list of regex patterns to exclude. intermediate_re = r'^' + cfg_default_intermediate_dir + r'.*' @@ -537,9 +536,9 @@ def run(args): if intermediate_dir_contents_exists(intermediate_dir): # We are just doing an update of the intermediate_dir contents. - print("****************") - print("Updating intermediate directory...") - print("****************") + announce(__file__, "****************") + announce(__file__, "Updating intermediate directory...") + announce(__file__, "****************") exclude_list.append(r'examples.*') options = { @@ -551,23 +550,24 @@ def run(args): } # action == 'sync' means copy files even when they do not already exist in tgt dir. # action == 'update' means DO NOT copy files when they do not already exist in tgt dir. - dirsync.sync(cfg_src_dir, intermediate_dir, 'sync', **options) + dirsync.sync(cfg_doc_src_dir, intermediate_dir, 'sync', **options) dirsync.sync(examples_dir, os.path.join(intermediate_dir, cfg_examples_dir), 'sync', **options) elif build_intermediate or build_html or build_latex: # We are having to create the intermediate_dir contents by copying. - print("****************") - print("Building intermediate directory...") - print("****************") + announce(__file__, "****************") + announce(__file__, "Building intermediate directory...") + announce(__file__, "****************") + t1 = datetime.now() copy_method = 1 # Both of these methods work. if copy_method == 0: # --------- Method 0: ignore_func = shutil.ignore_patterns('tmp*', 'output*') - print('Copying docs...') - shutil.copytree(cfg_src_dir, intermediate_dir, ignore=ignore_func, dirs_exist_ok=True) - print('Copying examples...') + announce(__file__, 'Copying docs...') + shutil.copytree(cfg_doc_src_dir, intermediate_dir, ignore=ignore_func, dirs_exist_ok=True) + announce(__file__, 'Copying examples...') shutil.copytree(examples_dir, os.path.join(intermediate_dir, cfg_examples_dir), dirs_exist_ok=True) else: # --------- Method 1: @@ -577,37 +577,26 @@ def run(args): } # action == 'sync' means copy files even when they do not already exist in tgt dir. # action == 'update' means DO NOT copy files when they do not already exist in tgt dir. - print('Copying docs...') - dirsync.sync(cfg_src_dir, intermediate_dir, 'sync', **options) - print('Copying examples...') + announce(__file__, 'Copying docs...') + dirsync.sync(cfg_doc_src_dir, intermediate_dir, 'sync', **options) + announce(__file__, 'Copying examples...') dirsync.sync(examples_dir, os.path.join(intermediate_dir, cfg_examples_dir), 'sync', **options) # ----------------------------------------------------------------- - # Build Example docs, Doxygen output, API docs, and API links. + # Build /lv_conf.h from lv_conf_template.h. # ----------------------------------------------------------------- - t1 = datetime.now() - - # Build /lv_conf.h from lv_conf_template.h for this build only. config_builder.run(lv_conf_file) + # ----------------------------------------------------------------- # Copy `lv_version.h` into intermediate directory. + # ----------------------------------------------------------------- shutil.copyfile(version_src_file, version_dst_file) - # Replace tokens in Doxyfile in 'intermediate_dir' with data from this run. - with open(doxyfile_src_file, 'rb') as f: - data = f.read().decode('utf-8') - - data = data.replace('<>', lv_conf_file) - data = data.replace('<>', f'"{lvgl_src_dir}"') - - with open(doxyfile_dst_file, 'wb') as f: - f.write(data.encode('utf-8')) - # ----------------------------------------------------------------- # Generate examples pages. Include sub-pages pages that get included # in individual documents where applicable. # ----------------------------------------------------------------- - print("Generating examples...") + announce(__file__, "Generating examples...") example_list.exec(intermediate_dir) # ----------------------------------------------------------------- @@ -617,68 +606,46 @@ def run(args): # ----------------------------------------------------------------- # Original code: # if True: - # print("Skipping adding translation links.") + # announce(__file__, "Skipping adding translation links.") # else: - # print("Adding translation links...") + # announce(__file__, "Adding translation links...") # add_translation.exec(intermediate_dir) - # --------------------------------------------------------------------- - # Generate API pages and links thereto. - # --------------------------------------------------------------------- if skip_api: - print("Skipping API generation as requested.") + announce(__file__, "Skipping API generation as requested.") else: - print("Running Doxygen...") - cmd('doxygen Doxyfile', intermediate_dir) + # ------------------------------------------------------------- + # Generate API pages and links thereto. + # ------------------------------------------------------------- + announce(__file__, "API page and link processing...") + api_doc_builder.EMIT_WARNINGS = False - print("API page and link processing...") - doc_builder.EMIT_WARNINGS = False - - # Create .RST files for API pages, plus - # add API hyperlinks to .RST files in the directories in passed array. - doc_builder.run( - project_dir, - intermediate_dir, - os.path.join(intermediate_dir, 'intro'), - os.path.join(intermediate_dir, 'details'), - os.path.join(intermediate_dir, 'details', 'common-widget-features'), - os.path.join(intermediate_dir, 'details', 'common-widget-features', 'layouts'), - os.path.join(intermediate_dir, 'details', 'common-widget-features', 'styles'), - os.path.join(intermediate_dir, 'details', 'debugging'), - os.path.join(intermediate_dir, 'details', 'integration'), - os.path.join(intermediate_dir, 'details', 'integration', 'adding-lvgl-to-your-project'), - os.path.join(intermediate_dir, 'details', 'integration', 'bindings'), - os.path.join(intermediate_dir, 'details', 'integration', 'building'), - os.path.join(intermediate_dir, 'details', 'integration', 'chip'), - os.path.join(intermediate_dir, 'details', 'integration', 'driver'), - os.path.join(intermediate_dir, 'details', 'integration', 'driver', 'display'), - os.path.join(intermediate_dir, 'details', 'integration', 'driver', 'touchpad'), - os.path.join(intermediate_dir, 'details', 'integration', 'framework'), - os.path.join(intermediate_dir, 'details', 'integration', 'ide'), - os.path.join(intermediate_dir, 'details', 'integration', 'os'), - os.path.join(intermediate_dir, 'details', 'integration', 'os', 'yocto'), - os.path.join(intermediate_dir, 'details', 'integration', 'renderers'), - os.path.join(intermediate_dir, 'details', 'libs'), - os.path.join(intermediate_dir, 'details', 'main-modules'), - # Note: details/main-modules/display omitted intentionally, - # since API links for those .RST files have been added manually. - os.path.join(intermediate_dir, 'details', 'auxiliary-modules'), - os.path.join(intermediate_dir, 'details', 'widgets') - ) + # api_doc_builder.run() => doxy_xml_parser.DoxygenXml() now: + # - preps and runs Doxygen generating XML, + # - loads generated XML. + # Then api_doc_builder.run(): + # - creates .RST files for API pages, and + # - adds API hyperlinks to .RST files in the directories in passed array. + api_doc_builder.build_api_docs(lvgl_src_dir, + intermediate_dir, + doxyfile_src_file, + 'details', + 'intro' + ) t2 = datetime.now() - print('Example/API run time: ' + str(t2 - t1)) + announce(__file__, 'Example/API run time: ' + str(t2 - t1)) # --------------------------------------------------------------------- # Build PDF # --------------------------------------------------------------------- if not build_latex: - print("Skipping Latex build.") + announce(__file__, "Skipping Latex build.") else: t1 = datetime.now() - print("****************") - print("Building Latex output...") - print("****************") + announce(__file__, "****************") + announce(__file__, "Building Latex output...") + announce(__file__, "****************") # If PDF link is present in top index.rst, remove it so PDF # does not have a link to itself. @@ -701,9 +668,9 @@ def run(args): cmd(cmd_line) # Generate PDF. - print("****************") - print("Building PDF...") - print("****************") + announce(__file__, "****************") + announce(__file__, "Building PDF...") + announce(__file__, "****************") cmd_line = 'latexmk -pdf "LVGL.tex"' cmd(cmd_line, latex_output_dir, False) @@ -713,19 +680,19 @@ def run(args): shutil.move(pdf_src_file, pdf_dst_file) t2 = datetime.now() - print('PDF : ' + pdf_dst_file) - print('Latex gen time: ' + str(t2 - t1)) + announce(__file__, 'PDF : ' + pdf_dst_file) + announce(__file__, 'Latex gen time: ' + str(t2 - t1)) # --------------------------------------------------------------------- # Build HTML # --------------------------------------------------------------------- if not build_html: - print("Skipping HTML build.") + announce(__file__, "Skipping HTML build.") else: t1 = datetime.now() - print("****************") - print("Building HTML output...") - print("****************") + announce(__file__, "****************") + announce(__file__, "Building HTML output...") + announce(__file__, "****************") # If PDF is present in build directory, copy it to # intermediate directory for use by HTML build. @@ -772,14 +739,14 @@ def run(args): cmd_line = f'sphinx-build -M html "{src}" "{dst}" -D version="{ver}" {env_opt} -j {cpu}' cmd(cmd_line) t2 = datetime.now() - print('HTML gen time : ' + str(t2 - t1)) + announce(__file__, 'HTML gen time : ' + str(t2 - t1)) # --------------------------------------------------------------------- # Indicate results. # --------------------------------------------------------------------- t_end = datetime.now() - print('Total run time: ' + str(t_end - t0)) - print('Done.') + announce(__file__, 'Total run time: ' + str(t_end - t0)) + announce(__file__, 'Done.') if __name__ == '__main__': diff --git a/docs/doxygen_config.py b/docs/doxygen_config.py new file mode 100644 index 0000000000..1763083f14 --- /dev/null +++ b/docs/doxygen_config.py @@ -0,0 +1,429 @@ +"""doxygen_config.py +Python Interface to Doxygen Config Files (Doxyfiles) + +Author : "Victor Wheeler" +Copyright: "Copyright (C) 2025 WGA Crystal Research, Inc." +License : "MIT" +Version : "1.0" + +This work was inspired by the `doxygen-python-interface` project at +https://github.com/TraceSoftwareInternational/doxygen-python-interface. +On 27-Feb-2025 I was engaged in a production project wherein I wanted +to find a Python module that I could re-use to reliably work with +Doxygen configuration files (Doxyfiles). The best one I found was +`doxygen-python-interface`. Unfortunately, the ``configParser`` from +that project could not be used because it both had important bugs and +design flaws in it (conflicts with legal Doxygen config syntax), and +it appears to have been abandoned after 26-Apr-2018, preventing these +things from being remedied. + +So a brand-new module has been created herewith based on sound O-O design +principles and a design that actually works in alignment with Doxygen +configuration syntax. + +Usage: + + import doxygen_config + ... + # 1. Load configuration from Doxyfile. + cfg = doxygen_config.DoxygenConfig() + cfg.load(doxyfile_src_file) + + # 2. Get a list of Doxygen option names. + opt_list = cfg.options() + ok_to_proceed = cfg.is_valid_option('PREDEFINED') \ + and cfg.is_valid_option('INPUT') + + # 3. Update it. + if ok_to_proceed: + temp = cfg.value('PREDEFINED') + temp = temp.replace('<>', config_file) + cfg.set('PREDEFINED', temp) + + temp = cfg.value('INPUT') + temp = temp.replace('<>', f'"{pjt_src_dir}"') + cfg.set('INPUT', temp) + + # 4. Save it. + # The original comments and order of config options are preserved. + # The ``bare`` argument discards comments from the output. + cfg.save(cfg_dict, doxyfile_dst_file, bare=True) + +Design Differences from `doxygen-python-interface`: + + - The DoxygenConfig class represents the actual Doxygen configuration, + in alignment with O-O theory --- it is not just a place to store a + set of functions that never needed to be a class. + + - If the user does a default ``save()`` (not requesting a "bare" + version of the Doxygen configuration), the saved Doxyfile + should be a binary match to the original Doxyfile loaded. + + Exceptions: + + 1. Any trailing whitespace in original Doxyfile after the ``=`` + on empty options is not preserved. + + 2. Multi-line lists that had unaligned backslashes after them like this: + + EXCLUDE_PATTERNS = */libs/barcode/code* \ + */libs/freetype/ft* \ + */libs/gif/gif* \ + */libs/lodepng/lode* \ + */libs/qrcode/qr* \ + */libs/thorvg/* \ + */libs/tiny_ttf/stb* \ + */libs/tjpgd/tjp* \ + */others/vg_lite_tvg/vg* + + will be saved like this: + + EXCLUDE_PATTERNS = */libs/barcode/code* \ + */libs/freetype/ft* \ + */libs/gif/gif* \ + */libs/lodepng/lode* \ + */libs/qrcode/qr* \ + */libs/thorvg/* \ + */libs/tiny_ttf/stb* \ + */libs/tjpgd/tjp* \ + */others/vg_lite_tvg/vg* + + ``doxygen-python-interface`` did not save the comments so an + "edit in place" of a Doxyfile could be catastrophic if the + comments were needed in the source Doxyfile as they often are + in production scenarios. + + - The ``save()`` method has an optional ``bare`` argument (default False) + that can be used to save a "bare" version of the Doxyfile options, + discarding the comments from the currently-loaded Doxyfile. + + - Input values are preserved exactly as they were found. The + `doxygen-python-interface`'s ``configParser`` class removed + quotation marks from incoming values and added quotation marks + to values containing spaces before storing them again. While + this "sounds nice", it was incompatible with Doxygen for every + type of item that could have a "list" as a value, such as the + PREDEFINED and ABBREVIATE_BRIEF options. + + Examples: + + PREDEFINED = USE_LIST USE_TABLE USE_CHART + + PREDEFINED = DOXYGEN CONFIG_PATH="/path with spaces/to/config.h" + + PREDEFINED = DOXYGEN \ + CONFIG_PATH="/path with spaces/to/config.h" + + These are all valid values for the PREDEFINED option and + MUST NOT have quotes around any of them! Can you imagine the havoc + that would result if a Python module meant to handle Doxygen Doxyfiles + altered Doxygen configuration items like this? + + PREDEFINED = "USE_LIST USE_TABLE USE_CHART" + + Thus, it is up to the user to know when values he is changing + have space(s) AND ALSO need quotes and take appropriate measures + by adding quotes when needed and not otherwise. + + - The storage of the list of Doxygen options is encapsulated + in the instance of the DoxygenConfig class instead of being + returned as a dictionary from the ``load...()`` function. + Its values are readable and writeable via methods. The + end user is not able to add options that were not part + of the original input Doxyfile, nor remove options that were + part of the original input Doxyfile. This gives some level of + control on retaining valid Doxygen options. + + It is an error to attempt to set a value with an option name + that does not exist in the configuration. A NameError exception + is raised if it is attempted. Attempting to read the value of + an option name that does not exist returns the value ``None``. + + While Doxygen options change from time to time, it is up to the + end user to use ``doxygen -u Doxyfile`` to keep his input + Doxyfile(s) up to date. + +Storage: + + The actual configuration values are represented in an internal + dictionary not intended to be accessed directly by the typical end + user. The keys are the Doxygen option names and the values are: + + - str : single values with possibly embedded spaces + - list: multi-line values with possibly embedded spaces + + Quotation marks are neither removed nor added, so it is up to the + user to set values compatible with Doxygen configuration syntax. + This also makes it okay for multi-line values to have more than one + value per line: if it is okay by Doxygen, then it is okay by + the DoxygenConfig class. + + If the user sets an option value passing a list, those values + will be represented as a multi-line value in the saved Doxyfile. + +The Philosophy of Removing Quotation Marks Is Not Workable for Doxygen: + + When one asks, "Is it appropriate to remove the quotation marks?" + What if a value looked like this (2 quoted items in one line), + removing quotation marks would be an error: + + "abc def" "ghi jkl" + + The ABBREVIATE_BRIEF list could indeed appear like this. + + If it were argued that all multi-value items should be formatted as + multi-line lists, then quotation marks theory works, as the + ABBREVIATE_BRIEF option does not require quotation marks around + every value. + + However, since Doxygen does not require this, there is still a + strong argument for not tampering with quotation marks at all + when importing values. The strongest reasons are: + + - Doxygen can and does accept values like this where the value of + an option can be a list. Doxygen sees this as 2 separate values: + + "abc def" "ghi jkl" + + - If the end user is going to set values with spaces in them, + it could be made the user's responsibility to know when + there are spaces and thus include quotes when needed. + + In the end, the "do not tamper with quotation marks" argument wins + for sake of reliability. So the policy is: quotation marks are + neither removed nor added. It is up to the user to know when they + are needed and add them himself. +""" +import logging +import os +import re + + +__author__ = "Victor Wheeler" +__copyright__ = "Copyright (C) 2025 WGA Crystal Research, Inc." +__license__ = "MIT" +__version__ = "1.0" + + +class ParseException(Exception): + """Exception thrown upon unexpected parsing errors.""" + pass + + +class DoxygenConfig: + """Doxygen Configurations (from/to Doxyfiles)""" + + def __init__(self): + """Prepare instantiated DoxygenConfig for use.""" + # Regexes used during Doxyfile parsing + self._re_single_line_option = re.compile(r'^\s*(\w+)\s*=\s*([^\\]*)\s*$') + self._re_top_of_multiline_option = re.compile(r'^\s*(\w+)\s*=\s*(|.*\S)\s*\\$') + # Doxygen cfg items by option name + self._cfg_items_dict = {} + # Comments by name of option below it. + # Comments at end of file have key 'self._end_key'. + self._cfg_comments_dict = {} + # Key used for comments found after last option in Doxyfile + self._end_key = 'END' + # Configuration to match Doxygen -g output (template Doxyfile) + self._char_count_before_equals = 23 + + def load(self, doxyfile: str): + """Load options and comments from `doxyfile` + + :param doxyfile: Path to doxyfile + + :raise FileNotFoundError: When doxyfile not found + :raise ParseException: When there is a parsing error + """ + + if not os.path.exists(doxyfile): + logging.error(f'Doxyfile not found {doxyfile}.') + raise FileNotFoundError(doxyfile) + + self._cfg_items_dict.clear() + self._cfg_comments_dict.clear() + + # Default encoding: UTF-8. + with open(doxyfile, 'r') as file: + in_multiline_opt = False + multiline_opt_name_bep = None # "bep" = "being processed" + accumulated_other_lines = [] + + for line in file.readlines(): + line = line.strip() + + if in_multiline_opt: + # There are 2 ways this list can end: + # 1. the normal way when last item has no trailing `\`, or + # 2. the last item has a trailing `\` and there is a blank- + # or comment-line after it, which should NOT be added + # to the list, but instead signal end-of-list. + if not line.endswith('\\'): + in_multiline_opt = False + + val = line.rstrip('\\').strip() + + if self._bool_comment_or_blank_line(val): + accumulated_other_lines.append(line) + in_multiline_opt = False + else: + self._cfg_items_dict[multiline_opt_name_bep].append(val) + + elif self._bool_comment_or_blank_line(line): + accumulated_other_lines.append(line) + + elif self._bool_top_of_multiline_option(line): + multiline_opt_name_bep, val = self._parse_multiline_option(line) + self._cfg_items_dict[multiline_opt_name_bep] = [val] + self._cfg_comments_dict[multiline_opt_name_bep] = accumulated_other_lines + accumulated_other_lines = [] + in_multiline_opt = True + + elif self._bool_single_line_option(line): + option_name, val = self._parse_single_line_option(line) + self._cfg_items_dict[option_name] = val + self._cfg_comments_dict[option_name] = accumulated_other_lines + accumulated_other_lines = [] + + # Any comments or blank lines found after last Doxygen option + # are represented in _cfg_comments_dict with key `self._end_key`. + if accumulated_other_lines: + self._cfg_comments_dict[self._end_key] = accumulated_other_lines + accumulated_other_lines.clear() + + def save(self, doxyfile: str, bare=False): + """Save configuration to `doxyfile`. + + :param doxyfile: Output path where Doxygen configuration will be + written. Overwrites file if it exists. + :param bare: Do not preserve comments from loaded file. + """ + + lines = [] + + for option_name, val in self._cfg_items_dict.items(): + if not bare: + lines.extend(self._cfg_comments_dict[option_name]) + + if type(val) is list: + # We will be aligning the backslashes after the + # items in the list, so we need to know the longest. + # First value in list: + multi_line_indent = ' ' * (self._char_count_before_equals + 2) + longest_len = len(max(val, key=len)) + val_w_len = val[0].ljust(longest_len) + lines.append(f'{option_name:<23}= {val_w_len} \\') + + # Next n-2 values in list: + if len(val) > 2: + for temp in val[1:-1]: + val_w_len = temp.ljust(longest_len) + lines.append(f'{multi_line_indent}{val_w_len} \\') + + # Last value in list: + lines.append(f'{multi_line_indent}{val[-1]}') + elif type(val) is str: + val_w_len = option_name.ljust(self._char_count_before_equals) + if len(val) == 0: + lines.append(f'{val_w_len}=') + else: + lines.append(f'{val_w_len}= {val}') + + if self._end_key in self._cfg_comments_dict: + if not bare: + lines.extend(self._cfg_comments_dict[self._end_key]) + + # Ensure there is exactly 1 newline at end of file. + lines.append('') + + with open(doxyfile, 'w') as file: + file.write('\n'.join(lines)) + + logging.debug(f'Saved configuration to [{doxyfile}].') + + def option_names(self): + """List of contained Doxygen option names""" + return self._cfg_items_dict.keys() + + def is_valid_option(self, option_name: str) -> bool: + """Is `option_name` a valid option name?""" + return option_name in self._cfg_items_dict + + def set(self, option_name: str, val: str or list): + """Set value of specified option + + :param option_name: Name of Doxygen option whose value to fetch + :param val: Value to set + - str = single-line value; + - list = multi-line value. + + :raises NameError: When ``name`` is not found. + """ + if option_name in self._cfg_items_dict: + self._cfg_items_dict[option_name] = val + if type(val) is list: + logging.debug(f'Item [{option_name}] set to list.') + else: + logging.debug(f'Item [{option_name}] set to [{val}].') + else: + logging.error(f'Doxyfile option {option_name} not found.') + raise NameError(f'Doxygen option {option_name} not found.') + + def value(self, option_name: str) -> str or list: + """Value of specified option + + :param option_name: Name of Doxygen option whose value to fetch + + :returns string: single-line value + :returns list: multi-line value + :returns None: When ``option_name`` is not found. + """ + if option_name in self._cfg_items_dict: + result = self._cfg_items_dict[option_name] + logging.debug(f'Item [{option_name}] fetched.') + else: + result = None + logging.debug(f'Item [{option_name}] not found.') + + return result + + def _parse_multiline_option(self, line) -> (str, str): + """Extract option name and first line of value of multi-line option. + + :param line: line to parse + :return: name and first line of multi-line option + :raise ParseException: When process fail to extract data + """ + + matches = self._re_top_of_multiline_option.search(line) + if matches is None or len(matches.groups()) != 2: + logging.error(f'Error extracting first value in multi-line option from [{line}].') + raise ParseException(f'Error extracting first value in multi-line option from [{line}].') + + return matches.group(1), matches.group(2) + + def _parse_single_line_option(self, line) -> (str, str): + """Extract option name and value of single line option. + + :param line: line to parse + :return: option name and value + :raise ParseException: When process fail to extract data + """ + + matches = self._re_single_line_option.search(line) + + if matches is None or len(matches.groups()) != 2: + logging.error(f'Error extracting option name and value from [{line}].') + raise ParseException(f'Error extracting option name and value from [{line}].') + + return matches.group(1), matches.group(2) + + def _bool_single_line_option(self, line: str) -> bool: + return self._re_single_line_option.match(line) is not None + + def _bool_comment_or_blank_line(self, line: str) -> bool: # NoQA + return line.startswith("#") or (len(line) == 0) + + def _bool_top_of_multiline_option(self, line) -> bool: + return self._re_top_of_multiline_option.match(line) is not None diff --git a/docs/doc_builder.py b/docs/doxygen_xml.py similarity index 56% rename from docs/doc_builder.py rename to docs/doxygen_xml.py index 644d214504..01cbb00125 100644 --- a/docs/doc_builder.py +++ b/docs/doxygen_xml.py @@ -1,23 +1,227 @@ -""" -This file opens the XML `index.xml` file generated by Doxygen -which has the following structure: +"""doxygen_xml.py + +Python Interface to Doxygen XML Output + +DoxygenXml is the primary class defined herein. +It uses all the other classes, and is used by: + +- ./docs/build.py via ./docs/api_doc_builder.py for the LVGL doc build to: + - edit Doxyfile to replace 2 string tokens + - run Doxygen + - load Doxygen XML output and use it to: + - generate API `.rst` pages, + - populate a set of `doxygen_xml.<...>` dictionaries, and + - use those dictionaries to determine in what LVGL documents + hyperlinks to API pages might be useful and appends them if they + are not already present. + +- ./scripts/gen_json/gen_json.py => pycparser_monkeypatch.py to + - edit Doxyfile to replace 2 string tokens + - silently run Doxygen + - load Doxygen XML output and use it to: + - populate a set of `doxygen_xml.<...>` dictionaries, and + - monkey-patch pycparser AST (Abstract Syntax Tree) output to + extract API documentation into its JSON output. + +"Loading Doxygen XML output" consists of + +- opening and parsing the XML `index.xml` file generated by Doxygen which + is a summary of Doxygen-generated documentation linked to additional `.xml` + filenames that have more details. Note that `index.xml` has a complete list + of documented symbol-names from the C code. `./docs/api_doc_builder.py` + searches those names using search term(s) to determine in what LVGL + documents hyperlinks to API pages might be useful, and appends them + to those documents if they are not already present. + + `index.xml` structure is described in the `doxygen/doxygen` repository + on GitHub under `./templates/xml/index.xsd`, and the structure of the + other files is described in `./templates/xml/compound.xsd`. - + Unlimited-length list of nodes. + ... + ... + ... ... -Each element has a 'kind' attribute that is one of the following: +In `index.xml`, every element: + + - always has attributes "refid" and "kind", + - always has exactly 1 element, and + - is followed by an unbounded number of elements. + +Each element's 'kind' attribute will be one of the following: + + class struct union interface + protocol category exception file + namespace group page example + dir type concept module + +which is a group that applies to many languages. Since Doxygen +is parsing a C-language project, we will only find these in the +`index.xml` file: + + - struct + - union + - file + - page (unused) + - dir (unused) + +In `'index.xml`, every element: + + - always has attributes "refid" and "kind", + - always has exactly 1 element. + +The contents of the elements differ based on the "kind" of + element they are contained by. + +kind="struct" + + - sub-elements only have `kind="variable"` attributes. + +kind="union" + + - sub-elements only have `kind="variable"` attributes. + +kind="file" + + - sub-elements can have `kind` attributes: + - define + - enum + - enumvalue has refid that begins with containing enum's refid + - variable + - function + - typedef + + - + Has child element with macro name. refid contains something + that looks like this: + + "lv__obj_8h_1a8ec0fe9743c5c015162d64df591cd719" + /____________________________________________/ + /_________/ | + | +-- ``id`` attribute of applicable item + | + +-- stem of `.xml` filename containing details + +lv__obj_8h.xml has this format: + + + lv_obj.h + Section dealing with #include's (list, dependency graph, etc.) + + + + + + + + + ...List of elements. + ...kind can be: + - define + - enum + - var + - func + + ... + ... + ... + ...List of all defines in file. + + + ... + + + + LV_STATE_DEFAULT + = 0x0000 + + + + + + ...List of all the enumeration's enumvalue's. + + ...List of all enums in the file. + + + + ...List of all variables in file. + + + + ...List of all functions in file. + + @file @brief description + + @file detailed description + + Only included if XML_PROGRAMLISTING = YES. + ... + ... + ... + + + + +kind="page" + + - Has no child members, but does have a child element + (e.g. "deprecated" and "todo"). + +kind="dir" + + - do not have sub-elements + - contents of element is a full path to a directory + - If sorted alphabetically, they form a sequence that can be + used to create something: + E: + E:/Dev + E:/Dev/lvgl + E:/Dev/lvgl/lvgl + E:/Dev/lvgl/lvgl/src + etc. + - refid="..." contains the stem of another `.xml` file that + contains more information on the directory, including any + documentation on it found by Doxygen. That XML file's structure is: + + .xml: + ------------ + + full_path_of_dir_wo_trailing_slash + subdirectory_path + subdirectory_path + subdirectory_path + ... (list elements --- one for each subdirectory in directory) + ... + filename + filename + filename + ... (list elements --- one for each file in directory) + ... + + Brief description of directory (Doxygen documentation) + + + Detailed description of directory (Doxygen documentation) + + + + + +list - define - - dir + - dir (unused) - enum - enumvalue - - example + - example (unused) - file - function - - page + - page (unused) - struct - typedef - union @@ -26,105 +230,96 @@ Each element has a 'kind' attribute that is one of the following: Most elements have child elements with their own contents depending on the 'kind' of element they are in. -This file defines classes for each of these except for - - - dir - - page - - example - -The remaining 'kind' values are: - - - define - - enum - - enumvalue - - file - - function - - struct - - typedef - - union - - variable +This file defines classes, whose class names are the upper-case names above +except for the unused ones. These classes are used to build a data structure +that matches that of the `index.xml` file. The list of classes is: - Class | Adds Self to Dictionary in __init__() - --------- | -------------------------------------- - DEFINE => `defines` - ENUM => `enums` - VARIABLE => `variables` - NAMESPACE => `namespaces` - STRUCT => `structures` - UNION appears to have a different purpose - TYPEDEF => `typedefs` - FUNCTION => `functions` - GROUP => `groups` - FILE => `files` - CLASS => `classes` + +-----------+----------------------------------------------------+ + | Class | Adds New Instance to this Dictionary in __init__() | + +===========+====================================================+ + | DEFINE | defines | + +-----------+----------------------------------------------------+ + | ENUM | enums (have child ENUMVALUE objects) | + +-----------+----------------------------------------------------+ + | VARIABLE | variables | + +-----------+----------------------------------------------------+ + | NAMESPACE | namespaces | + +-----------+----------------------------------------------------+ + | STRUCT | structures (have child STRUCT_FIELD objects) | + +-----------+----------------------------------------------------+ + | UNION | appears to be dysfunctional based on lacking input | + +-----------+----------------------------------------------------+ + | TYPEDEF | typedefs | + +-----------+----------------------------------------------------+ + | FUNCTION | functions (have child FUNC_ARG objects) | + +-----------+----------------------------------------------------+ + | GROUP | groups | + +-----------+----------------------------------------------------+ + | FILE | files | + +-----------+----------------------------------------------------+ + | CLASS | classes | + +-----------+----------------------------------------------------+ Additional classes: - - NAMESPACE(object): - - FUNC_ARG(object): (becomes members of FUNCTION objects) - - STRUCT_FIELD(object): (becomes members of STRUCT objects) - - GROUP(object): - - CLASS(object): - - XMLSearch(object): + - FUNC_ARG(object): (becomes child members of FUNCTION objects) + - STRUCT_FIELD(object): (becomes child members of STRUCT objects) + - ENUMVALUE(object): (becomes child members of ENUM objects) + - XMLSearch(object): (used only by ./scripts/gen_json/gen_json.py) Each of the above Dictionary variables has entries with - - keys = actual name of the code elements Doxygen found in the .H files. - - values = XML node generated by `xml.etree::ElementTree` + - keys = documented symbol name from C code that Doxygen found + - values = instances of classes named above Samples: - 'defines': {'ZERO_MEM_SENTINEL': , - 'LV_GLOBAL_DEFAULT': , - 'LV_ASSERT_OBJ': , - 'LV_TRACE_OBJ_CREATE': , + 'defines': {'ZERO_MEM_SENTINEL': , + 'LV_GLOBAL_DEFAULT': , + 'LV_ASSERT_OBJ': , + 'LV_TRACE_OBJ_CREATE': ,...} - 'enums': {'lv_key_t': , - 'lv_group_refocus_policy_t': , - 'lv_obj_flag_t': , - 'lv_obj_class_editable_t': , + 'enums': {'lv_key_t': , + 'lv_group_refocus_policy_t': , + 'lv_obj_flag_t': , + 'lv_obj_class_editable_t': ,...} - 'variables': {'lv_global': , - 'lv_obj_class': , - 'lv_font_montserrat_8': , - 'lv_font_montserrat_10': , + 'variables': {'lv_global': , + 'lv_obj_class': , + 'lv_font_montserrat_8': , + 'lv_font_montserrat_10': ,...} 'namespaces': {}, - 'structures': {'_lv_anim_t::_lv_anim_path_para_t': , - '_lv_anim_t': , - '_lv_animimg_t': , - '_lv_arc_t': , + 'structures': {'_lv_anim_t::_lv_anim_path_para_t': , + '_lv_anim_t': , + '_lv_animimg_t': , + '_lv_arc_t': ,...} 'unions': {}, - 'typedefs': {'lv_global_t': , - 'lv_group_focus_cb_t': , - 'lv_group_edge_cb_t': , + 'typedefs': {'lv_global_t': , + 'lv_group_focus_cb_t': , + 'lv_group_edge_cb_t': ,...} - 'functions': {'lv_group_create': , - 'lv_group_delete': , - 'lv_group_set_default': , + 'functions': {'lv_group_create': , + 'lv_group_delete': , + 'lv_group_set_default': ,...} Additional dictionaries: - 'files': {'lv_global.h': , - 'lv_group.h': , - 'lv_group_private.h': , - - 'html_files': {'lvgl': 'lvgl.html', - 'lv_api_map_v8': 'lv_api_map_v8.html', - 'lv_api_map_v9_0': 'lv_api_map_v9_0.html', - 'lv_api_map_v9_1': 'lv_api_map_v9_1.html', + 'files': {'lv_global.h': , + 'lv_group.h': , + 'lv_group_private.h': ,...} """ import os import sys -from xml.etree import ElementTree as ET - -base_path = '' -xml_path = '' +import subprocess +from xml.etree import ElementTree +import doxygen_config +from announce import * EMIT_WARNINGS = True DOXYGEN_OUTPUT = True @@ -143,6 +338,79 @@ MISSING_TYPEDEF = 'MissingTypedefDoc' MISSING_VARIABLE = 'MissingVariableDoc' MISSING_MACRO = 'MissingMacroDoc' +# Dictionaries built from Doxygen XML output via `xml.etree.ElementTree` +defines = {} # dictionary of doxygen_xml.DEFINE objects +enums = {} # dictionary of doxygen_xml.ENUM objects +variables = {} # dictionary of doxygen_xml.VARIABLE objects +namespaces = {} # dictionary of doxygen_xml.NAMESPACE objects +structures = {} # dictionary of doxygen_xml.STRUCT objects +typedefs = {} # dictionary of doxygen_xml.TYPEDEF objects +functions = {} # dictionary of doxygen_xml.FUNCTION objects +groups = {} # dictionary of doxygen_xml.GROUP objects +files = {} # dictionary of doxygen_xml.FILE objects +classes = {} # dictionary of doxygen_xml.CLASS objects +unions = {} # appears to be unused at this time (unions => structures dict). + +# Module-Global Variables +xml_path = '' + + +def run_ext_cmd(cmd_str: str, start_dir: str = None, quiet: bool = False, exit_on_error: bool = True): + """ + Run external command `cmd_str` (possibly silently) in directory + `start_dir` (if provided), and optionally abort execution on error. + + :param cmd_str: String to pass to OS. + :param start_dir: Directory to start in, or `None` to run in "cwd". + :param quiet: Should STDOUT and STDERR be suppressed? + :param exit_on_error: Should this function abort execution on + a non-zero exit status? + :return: n/a + """ + saved_dir = None + + if start_dir is not None: + saved_dir = os.getcwd() + os.chdir(start_dir) + + if quiet: + # This method of running Doxygen is used because we do not + # want anything going to STDOUT. Running it via `os.system()` + # would send its output to STDOUT. + p = subprocess.Popen( + cmd_str, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True + ) + + out, err = p.communicate() + + if p.returncode: + if out: + # Note the `.decode("utf-8")` is required because + # `sys.stdout.write()` requires a string, and `out` is + # a byte array; generates an exception if passed alone. + sys.stdout.write(out.decode("utf-8")) + sys.stdout.flush() + if err: + sys.stderr.write(err.decode("utf-8")) + sys.stdout.flush() + + if exit_on_error: + sys.exit(p.returncode) + else: + announce_start(__file__, f'Running [{cmd_str}] in [{os.getcwd()}]...') + return_code = os.system(cmd_str) + announce_finish() + + if return_code != 0 and exit_on_error: + print(f'Exiting due to error [{return_code}] running [{cmd_str}].') + sys.exit(return_code) + + if saved_dir is not None: + os.chdir(saved_dir) + def warn(warning_type, *args): if EMIT_WARNINGS: @@ -191,19 +459,19 @@ def build_docstring(element): def read_as_xml(d): try: - return ET.fromstring(d) + return ElementTree.fromstring(d) except: # NOQA return None -def load_xml(fle): +def load_xml_etree(fle): fle = os.path.join(xml_path, fle + '.xml') with open(fle, 'rb') as f: d = f.read().decode('utf-8') # This code is to correct a bug in Doxygen. That bug incorrectly parses - # a typedef and it causes an error to occur building the docs. The Error + # a typedef, and it causes an error to occur building the docs. The Error # doesn't stop the documentation from being generated, I just don't want # to see the ugly red output. # @@ -217,34 +485,60 @@ def load_xml(fle): # with open(fle, 'wb') as f: # f.write(d.encode('utf-8')) - return ET.fromstring(d) + return ElementTree.fromstring(d) -structures = {} -functions = {} -enums = {} -typedefs = {} -variables = {} -unions = {} -namespaces = {} -files = {} +def get_type(node): + def gt(n): + for c in n: + if c.tag == 'ref': + t = c.text.strip() + break + else: + t = node.text.strip() + + return t.replace('*', '').replace('(', '').replace(')', '').strip() + + for child in node: + if child.tag == 'type': + return gt(child) -# things to remove from description -# +def build_define(element): + define = None + + if element.text: + define = element.text.strip() + + for item in element: + ds = build_define(item) + if ds: + if define: + define += ' ' + ds + else: + define = ds.strip() + + if element.tail: + if define: + define += ' ' + element.tail.strip() + else: + define = element.tail.strip() + + return define class STRUCT_FIELD(object): - def __init__(self, name, type, description, file_name, line_no): + def __init__(self, name, _type, description, file_name, line_no): self.name = name - self.type = type + self.type = _type self.description = description self.file_name = file_name self.line_no = line_no class STRUCT(object): + """ elements in Doxygen `index.xml`""" _missing = MISSING_STRUCT _missing_field = MISSING_STRUCT_FIELD @@ -258,23 +552,51 @@ class STRUCT(object): ''' def __init__(self, parent, refid, name, **_): - if name in structures: - self.__dict__.update(structures[name].__dict__) - else: - structures[name] = self - self.parent = parent - self.refid = refid - self.name = name - self.types = set() - self._deps = None - self.header_file = '' - self.description = None - self.fields = [] - self.file_name = None - self.line_no = None + global structures + global unions - if parent and refid: - root = load_xml(refid) + if type(self) is UNION: + # UNION inherits from STRUCT. + if name in unions: + self.__dict__.update(unions[name].__dict__) + else: + unions[name] = self + self.parent = parent + self.refid = refid + self.name = name + self.types = set() + self._deps = None + self.header_file = '' + self.description = None + self.fields = [] + self.file_name = None + self.line_no = None + else: + # STRUCT type + if name in structures: + self.__dict__.update(structures[name].__dict__) + else: + structures[name] = self + self.parent = parent + self.refid = refid + self.name = name + self.types = set() + self._deps = None + self.header_file = '' + self.description = None + self.fields = [] + self.file_name = None + self.line_no = None + + # Prior to 9-Mar-2025, the code below was never executing since this + # __init__() was never called with a `parent` value other than `None`. + # Reason: `kind="struct"` only occurs in `index.xml` as a top-level + # entry, and not as a child element of `kind="file"` as do + # elements with kind = define, var, enum and func. + # Original code: + # if parent and refid: + if refid: + root = load_xml_etree(refid) for compounddef in root: if compounddef.attrib['id'] != self.refid: @@ -300,6 +622,7 @@ class STRUCT(object): file_name = None line_no = None + # For each struct member... for element in memberdef: if element.tag == 'location': file_name = element.attrib['file'] @@ -319,20 +642,21 @@ class STRUCT(object): self.types.add(t) - if not self.description: - warn(self._missing, self.name) - warn(None, 'FILE:', self.file_name) - warn(None, 'LINE:', self.line_no) - warn(None) - - for field in self.fields: - if not field.description: - warn(self._missing_field, self.name) - warn(None, 'FIELD:', field.name) - warn(None, 'FILE:', field.file_name) - warn(None, 'LINE:', field.line_no) + if EMIT_WARNINGS: + if not self.description: + warn(self._missing, self.name) + warn(None, 'FILE:', self.file_name) + warn(None, 'LINE:', self.line_no) warn(None) + for field in self.fields: + if not field.description: + warn(self._missing_field, self.name) + warn(None, 'FIELD:', field.name) + warn(None, 'FILE:', field.file_name) + warn(None, 'LINE:', field.line_no) + warn(None) + def get_field(self, name): for field in self.fields: if field.name == name: @@ -372,6 +696,7 @@ class STRUCT(object): class UNION(STRUCT): + """ elements in Doxygen `index.xml`""" _missing = MISSING_UNION _missing_field = MISSING_UNION_FIELD @@ -381,29 +706,17 @@ class UNION(STRUCT): ''' -def get_type(node): - def gt(n): - for c in n: - if c.tag == 'ref': - t = c.text.strip() - break - else: - t = node.text.strip() - - return t.replace('*', '').replace('(', '').replace(')', '').strip() - - for child in node: - if child.tag == 'type': - return gt(child) - - class VARIABLE(object): + """ + elements in Doxygen `index.xml`""" template = '''\ .. doxygenvariable:: {name} :project: lvgl ''' def __init__(self, parent, refid, name, **_): + global variables + if name in variables: self.__dict__.update(variables[name].__dict__) else: @@ -417,7 +730,7 @@ class VARIABLE(object): self.line_no = None if parent is not None: - root = load_xml(parent.refid) + root = load_xml_etree(parent.refid) for compounddef in root: if compounddef.attrib['id'] != parent.refid: @@ -454,6 +767,7 @@ class VARIABLE(object): class NAMESPACE(object): + """ elements in Doxygen `index.xml`""" template = '''\ .. doxygennamespace:: {name} :project: lvgl @@ -464,6 +778,8 @@ class NAMESPACE(object): ''' def __init__(self, parent, refid, name, **_): + global namespaces + if name in namespaces: self.__dict__.update(namespaces[name].__dict__) else: @@ -520,24 +836,16 @@ class NAMESPACE(object): return self.template.format(name=self.name) -class FUNC_ARG(object): - - def __init__(self, name, type): - self.name = name - self.type = type - self.description = None - - -groups = {} - - class GROUP(object): + """ elements in Doxygen `index.xml`""" template = '''\ .. doxygengroup:: {name} :project: lvgl ''' def __init__(self, parent, refid, name, **_): + global groups + if name in groups: self.__dict__.update(functions[name].__dict__) else: @@ -551,14 +859,26 @@ class GROUP(object): return self.template.format(name=self.name) +class FUNC_ARG(object): + """ elements in Doxygen `index.xml`""" + + def __init__(self, name, _type): + self.name = name + self.type = _type + self.description = None + class FUNCTION(object): + """ + elements in Doxygen `index.xml`""" template = '''\ .. doxygenfunction:: {name} :project: lvgl ''' def __init__(self, parent, refid, name, **_): + global functions + if name in functions: self.__dict__.update(functions[name].__dict__) else: @@ -577,7 +897,7 @@ class FUNCTION(object): self.void_return = False if parent is not None: - root = load_xml(parent.refid) + root = load_xml_etree(parent.refid) for compounddef in root: if compounddef.attrib['id'] != parent.refid: @@ -732,8 +1052,14 @@ class FUNCTION(object): class FILE(object): + """ elements in Doxygen `index.xml`""" def __init__(self, _, refid, name, node, **__): + global files + + if name.endswith('lv_types.h'): + return + if name in files: self.__dict__.update(files[name].__dict__) return @@ -743,6 +1069,7 @@ class FILE(object): self.refid = refid self.name = name self.header_file = os.path.splitext(name)[0] + self.types_contained = set() enums_ = [] @@ -750,7 +1077,9 @@ class FILE(object): if member.tag != 'member': continue - cls = globals()[member.attrib['kind'].upper()] + cls_name = member.attrib['kind'].upper() + self.types_contained.add(cls_name) + cls = globals()[cls_name] if cls == ENUM: if member[0].text: member.attrib['name'] = member[0].text.strip() @@ -768,14 +1097,48 @@ class FILE(object): cls(self, **member.attrib) +class ENUMVALUE(object): + """ + + elements in Doxygen `index.xml`""" + template = '''\ +.. doxygenenumvalue:: {name} + :project: lvgl +''' + + def __init__(self, parent, refid, name, **_): + self.parent = parent + self.refid = refid + self.name = name + self.description = None + self.file_name = None + self.line_no = None + + if parent is not None: + if parent.file_name is not None: + self.file_name = parent.file_name + elif hasattr(parent, 'parent') and parent.parent is not None: + if parent.parent.name is not None: + self.file_name = parent.parent.name + + def __str__(self): + return self.template.format(name=self.name) + + class ENUM(object): + """ + elements in Doxygen `index.xml`""" template = '''\ .. doxygenenum:: {name} :project: lvgl ''' def __init__(self, parent, refid, name, **_): + global enums + if name in enums: + # This happens when `name` is `None`, for example. + # This is true for unnamed enumerations. self.__dict__.update(enums[name].__dict__) else: @@ -790,7 +1153,7 @@ class ENUM(object): self.line_no = None if parent is not None: - root = load_xml(parent.refid) + root = load_xml_etree(parent.refid) for compounddef in root: if compounddef.attrib['id'] != parent.refid: @@ -901,39 +1264,17 @@ class ENUM(object): return '\n'.join(template) -defines = {} - - -def build_define(element): - define = None - - if element.text: - define = element.text.strip() - - for item in element: - ds = build_define(item) - if ds: - if define: - define += ' ' + ds - else: - define = ds.strip() - - if element.tail: - if define: - define += ' ' + element.tail.strip() - else: - define = element.tail.strip() - - return define - - class DEFINE(object): + """ + elements in Doxygen `index.xml`""" template = '''\ .. doxygendefine:: {name} :project: lvgl ''' def __init__(self, parent, refid, name, **_): + global defines + if name in defines: self.__dict__.update(defines[name].__dict__) else: @@ -949,7 +1290,8 @@ class DEFINE(object): self.initializer = None if parent is not None: - root = load_xml(parent.refid) + root = load_xml_etree(parent.refid) + memberdef = [] for compounddef in root: if compounddef.attrib['id'] != parent.refid: @@ -965,8 +1307,8 @@ class DEFINE(object): for memberdef in child: if memberdef.attrib['id'] == refid: break - else: - continue + else: + continue break else: @@ -1010,24 +1352,6 @@ class DEFINE(object): return self.template.format(name=self.name) -class ENUMVALUE(object): - template = '''\ -.. doxygenenumvalue:: {name} - :project: lvgl -''' - - def __init__(self, parent, refid, name, **_): - self.parent = parent - self.refid = refid - self.name = name - self.description = None - self.file_name = None - self.line_no = None - - def __str__(self): - return self.template.format(name=self.name) - - class TYPEDEF(object): template = '''\ .. doxygentypedef:: {name} @@ -1035,6 +1359,8 @@ class TYPEDEF(object): ''' def __init__(self, parent, refid, name, **_): + global typedefs + if name in typedefs: self.__dict__.update(typedefs[name].__dict__) else: @@ -1050,7 +1376,7 @@ class TYPEDEF(object): self.line_no = None if parent is not None: - root = load_xml(parent.refid) + root = load_xml_etree(parent.refid) for compounddef in root: if compounddef.attrib['id'] != parent.refid: @@ -1131,12 +1457,13 @@ class TYPEDEF(object): return self.template.format(name=self.name) -classes = {} - - class CLASS(object): + """ + elements in Doxygen `index.xml`""" def __init__(self, _, refid, name, node, **__): + global classes + if name in classes: self.__dict__.update(classes[name].__dict__) return @@ -1165,484 +1492,191 @@ class CLASS(object): cls(self, **member.attrib) -lvgl_src_path = '' -api_path = '' -html_files = {} +class DoxygenXml(object): + """Opens, parses and loads a Doxygen-generated `index.xml` file + and makes it available as a set of dictionary attributes of this + module, documented at the top of this file and named below in + `global` statements. + """ + def __init__(self, + lvgl_src_dir: str, + intermediate_dir: str, + doxyfile_src_file: str, + silent_mode=False, + doxy_tagfile=''): + """ + - Prepare and run Doxygen, generating XML output. + - Load that XML output, and use it to populate + :param lvgl_src_dir: + :param intermediate_dir: + :param doxyfile_src_file: + :param silent_mode: + """ + # Dictionaries to Be Populated: + global defines + global enums + global variables + global namespaces + global structures + global typedefs + global functions + global groups + global files + global classes + global unions -def iter_src(n, p): - if p: - out_path = os.path.join(api_path, p) - else: - out_path = api_path - - index_file = None - - if p: - src_path = os.path.join(lvgl_src_path, p) - else: - src_path = lvgl_src_path - - folders = [] - - for file in os.listdir(src_path): - if 'private' in file: - continue - - if os.path.isdir(os.path.join(src_path, file)): - folders.append((file, os.path.join(p, file))) - continue - - if not file.endswith('.h'): - continue - - if not os.path.exists(out_path): - os.makedirs(out_path) - - if index_file is None: - index_file = open(os.path.join(out_path, 'index.rst'), 'w') - if n: - index_file.write('=' * len(n)) - index_file.write('\n' + n + '\n') - index_file.write('=' * len(n)) - index_file.write('\n\n\n') - - index_file.write('.. toctree::\n :maxdepth: 2\n\n') - - name = os.path.splitext(file)[0] - index_file.write(' ' + name + '\n') - - rst_file = os.path.join(out_path, name + '.rst') - html_file = os.path.join(p, name + '.html') - html_files[name] = html_file - - with open(rst_file, 'w') as f: - f.write('.. _{0}_h:'.format(name)) - f.write('\n\n') - f.write('=' * len(file)) - f.write('\n') - f.write(file) - f.write('\n') - f.write('=' * len(file)) - f.write('\n\n\n') - - f.write('.. doxygenfile:: ' + file) - f.write('\n') - f.write(' :project: lvgl') - f.write('\n\n') - - for name, folder in folders: - if iter_src(name, folder): - if index_file is None: - index_file = open(os.path.join(out_path, 'index.rst'), 'w') - - if n: - index_file.write('=' * len(n)) - index_file.write('\n' + n + '\n') - index_file.write('=' * len(n)) - index_file.write('\n\n\n') - - index_file.write('.. toctree::\n :maxdepth: 2\n\n') - - index_file.write(' ' + os.path.split(folder)[-1] + '/index\n') - - if index_file is not None: - index_file.write('\n') - index_file.close() - return True - - return False - - -def clean_name(nme): - # Handle error: - # AttributeError: 'NoneType' object has no attribute 'startswith' - if nme is None: - return nme - - if nme.startswith('_lv_'): - nme = nme[4:] - elif nme.startswith('lv_'): - nme = nme[3:] - - if nme.endswith('_t'): - nme = nme[:-2] - - return nme - - -# Definitions: -# - "section" => The name "abc_def" has 2 sections. -# - N = number of sections in `item_name`. -# After removing leading '_lv_', 'lv_' and trailing '_t' from `obj_name`, -# do the remaining first N "sections" of `obj_name` match `item_name` -# (case sensitive)? -def is_name_match(item_name, obj_name): - # Handle error: - # AttributeError: 'NoneType' object has no attribute 'split' - if obj_name is None: - return False - - u_num = item_name.count('_') + 1 - - obj_name = obj_name.split('_') - - # Reject (False) if `obj_name` doesn't have as many sections as `item_name`. - if len(obj_name) < u_num: - return False - - obj_name = '_'.join(obj_name[:u_num]) - - return item_name == obj_name - - -def get_includes(name1, name2, obj, includes): - name2 = clean_name(name2) - - if not is_name_match(name1, name2): - return - - if obj.parent is not None and hasattr(obj.parent, 'header_file'): - header_file = obj.parent.header_file - elif hasattr(obj, 'header_file'): - header_file = obj.header_file - else: - return - - if not header_file: - return - - if header_file not in html_files: - return - - includes.add((header_file, html_files[header_file])) - - -class XMLSearch(object): - - def __init__(self, temp_directory): global xml_path - import subprocess - import re - import sys - bp = os.path.abspath(os.path.dirname(__file__)) + announce_set_silent_mode(silent_mode) + base_dir = os.path.abspath(os.path.dirname(__file__)) + doxyfile_filename = str(os.path.split(doxyfile_src_file)[1]) + doxyfile_dst_file = str(os.path.join(intermediate_dir, doxyfile_filename)) + lv_conf_file = os.path.join(intermediate_dir, 'lv_conf.h') + xml_path = os.path.join(intermediate_dir, 'xml') - lvgl_path = os.path.join(temp_directory, 'lvgl') - src_path = os.path.join(lvgl_path, 'src') - - doxy_path = os.path.join(bp, 'Doxyfile') - - with open(doxy_path, 'rb') as f: - data = f.read().decode('utf-8') - - data = data.replace( - '#*#*LV_CONF_PATH*#*#', - os.path.join(temp_directory, 'lv_conf.h') - ) - data = data.replace('*#*#SRC#*#*', '"{0}"'.format(src_path)) - - with open(os.path.join(temp_directory, 'Doxyfile'), 'wb') as f: - f.write(data.encode('utf-8')) + # In case DoxygenXml() is ever instantiated twice in 1 session, + # clear these dictionaries before they are (re-)populated below. + defines.clear() + enums.clear() + variables.clear() + namespaces.clear() + structures.clear() + typedefs.clear() + functions.clear() + groups.clear() + files.clear() + classes.clear() + unions.clear() # ----------------------------------------------------------------- - # Populate LVGL_URLPATH and LVGL_GITCOMMIT environment variables: - # - LVGL_URLPATH <= 'master' or '8.4' '9.2' etc. - # - LVGL_GITCOMMIT <= commit hash of HEAD. - # The previous version of this was populating LVGL_URLPATH with - # the multi-line list of all existing branches in the repository, - # which was not what was intended. + # Prep and run Doxygen # ----------------------------------------------------------------- - status, branch = subprocess.getstatusoutput("git branch --show-current") - _, gitcommit = subprocess.getstatusoutput("git rev-parse HEAD") + # Generate Doxyfile into `intermediate_dir` replacing certain + # config options for this run. + # 1. Load from Doxyfile + cfg = doxygen_config.DoxygenConfig() + cfg.load(doxyfile_src_file) + # 2. Update cfg. + cfg.set('OUTPUT_DIRECTORY', '.') + cfg.set('XML_OUTPUT', "xml") + cfg.set('HTML_OUTPUT', 'doxygen_html') + cfg.set('INPUT', lvgl_src_dir) + cfg.set('PREDEFINED', f'DOXYGEN LV_CONF_PATH="{lv_conf_file}"') + cfg.set('QUIET', 'YES') + cfg.set('GENERATE_HTML', 'NO') + cfg.set('GENERATE_DOCSET', 'NO') + cfg.set('GENERATE_HTMLHELP', 'NO') + cfg.set('GENERATE_CHI', 'NO') + cfg.set('GENERATE_QHP', 'NO') + cfg.set('GENERATE_ECLIPSEHELP', 'NO') + cfg.set('GENERATE_LATEX', 'NO') + cfg.set('GENERATE_RTF', 'NO') + cfg.set('GENERATE_MAN', 'NO') + cfg.set('GENERATE_XML', 'YES') + cfg.set('GENERATE_DOCBOOK', 'NO') + cfg.set('GENERATE_PERLMOD', 'NO') - # If above failed (i.e. `branch` not valid), default to 'master'. - if status != 0: - branch = 'master' - elif branch == 'master': - # Expected in most cases. Nothing to change. - pass - else: - # `branch` is valid. Capture release version if in a 'release/' branch. - if branch.startswith('release/'): - branch = branch[8:] - else: - # Default to 'master'. - branch = 'master' + # Include TAGFILES if requested. + if doxy_tagfile: + cfg.set('GENERATE_TAGFILE', doxy_tagfile) - os.environ['LVGL_URLPATH'] = branch - os.environ['LVGL_GITCOMMIT'] = gitcommit + # 3. Store it for use by Doxygen in intermediate directory. + cfg.save(doxyfile_dst_file) - # --------------------------------------------------------------------- - # Provide a way to run an external command and abort build on error. + # Run Doxygen in intermediate directory. + run_ext_cmd('doxygen Doxyfile', intermediate_dir, quiet=silent_mode) + + # ----------------------------------------------------------------- + # Load root of Doxygen output (index.xml) as an `xml.etree.ElementTree`. + # ----------------------------------------------------------------- + index_xml_etree = load_xml_etree('index') + + # Populate these dictionaries. + # Keys : C-code-element names (str) found by Doxygen. + # Values: The XML-node created by `xml.etree::ElementTree` in `load_xml()` above. # - # This is necessary because when tempdir created by tempfile.mkdtemp()` - # is on a different drive, the "cd tmpdir && doxygen Doxyfile" syntax - # fails because of the different semantics of the `cd` command on - # Windows: it doesn't change the default DRIVE if `cd` is executed - # from a different drive. The result, when this is the case, is that - # Doxygen runs in the current working directory instead of in the - # temporary directory as was intended. - # --------------------------------------------------------------------- - def cmd(cmd_str, start_dir=None): - if start_dir is None: - start_dir = os.getcwd() + # defines, enums, variables, + # namespaces, structures, typedefs, + # functions, unions, groups, + # files, classes. + announce_start(__file__, "Building source-code symbol dictionaries...") + module_namespace = globals() - saved_dir = os.getcwd() - os.chdir(start_dir) - - # This method of running Doxygen is used because if it - # succeeds, we do not want anything going to STDOUT. - # Running it via `os.system()` would send its output - # to STDOUT. - p = subprocess.Popen( - cmd_str, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True - ) - - out, err = p.communicate() - if p.returncode: - if out: - # Note the `.decode("utf-8")` is required here - # because `sys.stdout.write()` requires a string, - # and `out` by itself is a byte array -- it causes - # it to generate an exception and abort the script. - sys.stdout.write(out.decode("utf-8")) - sys.stdout.flush() - if err: - sys.stderr.write(err.decode("utf-8")) - sys.stdout.flush() - - sys.exit(p.returncode) - - # If execution arrived here, Doxygen exited with code 0. - os.chdir(saved_dir) - - # ----------------------------------------------------------------- - # Run Doxygen in temporary directory. - # ----------------------------------------------------------------- - cmd('doxygen Doxyfile', temp_directory) - - xml_path = os.path.join(temp_directory, 'xml') - - self.index = load_xml('index') - - for compound in self.index: + for compound in index_xml_etree: + # Here we will encounter these "kind" in the index.xml + # elements: dir, file, page, struct, union. compound.attrib['name'] = compound[0].text.strip() - if compound.attrib['kind'] in ('example', 'page', 'dir'): - continue - globals()[compound.attrib['kind'].upper()]( - None, - node=compound, - **compound.attrib - ) + # Filter out dir, page, example. + if compound.attrib['kind'] not in ('example', 'page', 'dir'): + class_name = compound.attrib['kind'].upper() + class_obj = module_namespace[class_name] + _ = class_obj(None, node=compound, **compound.attrib) + + announce_finish() + + # Additional data: the above instantiates the class, but doesn't + # store the resulting object anywhere, since each class' __init__() + # function adds the new object to the appropriate dictionary, + # and uses the arguments it gets to "build out" additional + # structure and/or populate additional dictionaries based on + # content. Each class __init__() function specifies args it + # in its parameter list. They match needed key values in the + # `**compound.attrib` dictionary. The remaining (unused) + # keys in that dictionary are accepted in the `**_` parameter + # at the end of the parameter list. + # + # FILE class populates `files` dict plus: + # - `defines` dictionary + # - `enums` dictionary with children: + # - ENUMVALUEs + # - `variables` dictionary + # - `typedefs` dictionary + # - `functions` dictionary with children: + # - FUNC_ARGs + # STRUCT class populates the `structures` dictionary with children: + # - STRUCT_FIELDs + # + # and possibly `namespaces`, `groups` and `classes` if + # they are present in `index.xml`. + def get_macros(self): + global defines return list(defines.values()) def get_enum_item(self, e_name): + global enums for enum, obj in enums.items(): for enum_item in obj.members: if enum_item.name == e_name: return enum_item def get_enum(self, e_name): + global enums return enums.get(e_name, None) def get_function(self, f_name): + global functions return functions.get(f_name, None) def get_variable(self, v_name): + global variables return variables.get(v_name, None) def get_union(self, u_name): + global unions return unions.get(u_name, None) def get_structure(self, s_name): + global structures return structures.get(s_name, None) def get_typedef(self, t_name): + global typedefs return typedefs.get(t_name, None) def get_macro(self, m_name): + global defines return defines.get(m_name, None) - - -def announce(*args): - args = ' '.join(repr(arg) for arg in args) - print(f'{os.path.basename(__file__)}: ', args) - - -def run(project_path, temp_directory, *doc_paths): - """ - This function does 2 things: - 1. Generates .RST files for the LVGL header files that will have API - pages generated for them. It places these in /API/... - following the /src/ directory structure. - 2. Add Sphinx hyperlinks to the end of source .RST files found - in the `doc_paths` array directories, whose file-name stems - match code-element names found by Doxygen. - - :param project_path: platform-appropriate path to LVGL root directory - :param temp_directory: platform-appropriate path to temp dir being operated on - :param doc_paths: list of platform-appropriate paths to find source .RST files. - :return: n/a - """ - global base_path - global xml_path - global lvgl_src_path - global api_path - - base_path = temp_directory - xml_path = os.path.join(base_path, 'xml') - api_path = os.path.join(base_path, 'API') - lvgl_src_path = os.path.join(project_path, 'src') - - announce("Generating API documentation .RST files...") - - if not os.path.exists(api_path): - os.makedirs(api_path) - - # Generate .RST files for API pages. - iter_src('API', '') - # Load index.xml -- core of what was generated by Doxygen. - index = load_xml('index') - - # Populate these dictionaries. - # Keys : C-code-element names (str) found by Doxygen. - # Values: The XML-node created by `xml.etree::ElementTree` in `load_xml()` above. - # - # - defines, - # - enums, - # - variables, - # - namespaces, - # - structures, - # - unions, - # - typedefs, - # - functions. - announce("Building source-code symbol tables...") - - for compound in index: - compound.attrib['name'] = compound[0].text.strip() - if compound.attrib['kind'] in ('example', 'page', 'dir'): - continue - - # This below highly-compressed command effectively does this: - # - # namespace_dict = globals() - # compound_elem_kind_upper = compound.attrib['kind'].upper() - # e.g. 'FUNCTION' - # class_obj = namespace_dict['FUNCTION'] - # # In each case of `class_obj`, the __init__ args are: - # # (self, parent, refid, name, **_) - # # So we get... - # attrib_keyword_args = **compound.attrib - # # Passing (**compound.attrib) as an argument creates and - # # passes a set of keyword arguments produced from the - # # dictionary `compound.attrib`. - # new_obj = class_obj(None, node=compound, attrib_keyword_args) - # - # Note carefully that `new_obj` gets thrown away, but the new object created - # doesn't go away because during its execution of __init__(), the new object - # adds itself to the global dictionary matching its "kind": - # - # Class Dictionary New Object Adds Itself To - # ------------ | ------------------------------------ - # - DEFINE => `defines` - # - ENUM => `enums` - # - VARIABLE => `variables` - # - NAMESPACE => `namespaces` - # - STRUCT => `structures` - # - UNION appears to have a different purpose - # - TYPEDEF => `typedefs` - # - FUNCTION => `functions` - # - GROUP => `groups` - # - FILE => `files` - # - CLASS => `classes` - # - # Populating these dictionaries takes quite a while: - # ~18-seconds on a medium-speed system. - globals()[compound.attrib['kind'].upper()]( - None, - node=compound, - **compound.attrib - ) - - # For each directory entry in `doc_paths` array... - announce("Adding API-page hyperlinks to source docs...") - - for folder in doc_paths: - # Fetch a list of '.rst' files excluding 'index.rst'. - rst_files = list( - (os.path.splitext(item)[0], os.path.join(folder, item)) - for item in os.listdir(folder) - if item.endswith('.rst') and 'index.rst' not in item - ) - - # For each .RST file in that directory... - for stem, path in rst_files: - # Start with an empty set. - html_includes = set() - - # Build `html_includes` set as a list of tuples containing - # (name, html_file). Example: "draw.rst" has `stem` == 'draw', - # and generates a list of tuples from .H files where matching - # C-code-element names were found. Example: - # {('lv_draw_line', 'draw\\lv_draw_line.html'), - # ('lv_draw_sdl', 'draw\\sdl\\lv_draw_sdl.html'), - # ('lv_draw_sw_blend_to_i1', 'draw\\sw\\blend\\lv_draw_sw_blend_to_i1.html'), - # etc.} - for container in ( - defines, - enums, - variables, - namespaces, - structures, - unions, - typedefs, - functions - ): - for n, o in container.items(): - get_includes(stem, n, o, html_includes) - - if html_includes: - # Convert `html_includes` set to a list of strings containing the - # Sphinx hyperlink syntax "link references". Example from above: - # [':ref:`lv_draw_line_h`\n', - # ':ref:`lv_draw_sdl_h`\n', - # ':ref:`lv_draw_sw_blend_to_i1_h`\n', - # etc.] - html_includes = list( - ':ref:`{0}_h`\n'.format(inc) - for inc, _ in html_includes - ) - - # Convert that list to a single string of Sphinx hyperlink - # references with blank lines between them. - # :ref:`lv_draw_line_h` - # - # :ref:`lv_draw_sdl_h` - # - # :ref:`lv_draw_sw_blend_to_i1_h` - # - # etc. - output = ('\n'.join(html_includes)) + '\n' - - # Append that string to the source .RST file being processed. - with open(path, 'rb') as f: - try: - data = f.read().decode('utf-8') - except UnicodeDecodeError: - print(path) - raise - - data = data.split('.. Autogenerated', 1)[0] - - data += '.. Autogenerated\n\n' - data += output - - with open(path, 'wb') as f: - f.write(data.encode('utf-8')) diff --git a/docs/make.bat b/docs/make.bat index f53c1849a3..90b964f631 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -15,7 +15,7 @@ if "%LVGL_DOC_BUILD_INTERMEDIATE_DIR%" == "" ( set SOURCEDIR=%LVGL_DOC_BUILD_INTERMEDIATE_DIR% ) if "%SPHINXOPTS%" == "" ( - rem python get_lvgl_version.py >_version_temp.txt + rem python ./src/lvgl_version.py >_version_temp.txt rem set /p VER=<_version_temp.txt rem del _version_temp.txt for /F %%v in ('python lvgl_version.py') do set VER=%%v diff --git a/docs/src/details/index.rst b/docs/src/details/index.rst index 027fecfcd8..90e833d6e6 100644 --- a/docs/src/details/index.rst +++ b/docs/src/details/index.rst @@ -18,4 +18,3 @@ Details ../contributing/index ../CHANGELOG ../API/index - ../ROADMAP diff --git a/scripts/gen_json/gen_json.py b/scripts/gen_json/gen_json.py index bd19533b48..d51c1a0d10 100644 --- a/scripts/gen_json/gen_json.py +++ b/scripts/gen_json/gen_json.py @@ -9,47 +9,49 @@ import subprocess base_path = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, base_path) -project_path = os.path.abspath(os.path.join(base_path, '..', '..')) -docs_path = os.path.join(project_path, 'docs') +project_dir = os.path.abspath(os.path.join(base_path, '..', '..')) +docs_path = os.path.join(project_dir, 'docs') sys.path.insert(0, docs_path) import create_fake_lib_c # NOQA import pycparser_monkeypatch # NOQA import pycparser # NOQA +doxyfile_filename = 'Doxyfile' DEVELOP = False -temp_directory = tempfile.mkdtemp(suffix='.lvgl_json') +intermediate_dir = tempfile.mkdtemp(suffix='.lvgl_json') -def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_private, no_docstrings, *compiler_args): +def run(output_path, lv_conf_file, output_to_stdout, target_header, filter_private, no_docstrings, *compiler_args): pycparser_monkeypatch.FILTER_PRIVATE = filter_private - lvgl_path = project_path - lvgl_src_path = os.path.join(lvgl_path, 'src') - temp_lvgl = os.path.join(temp_directory, 'lvgl') + lvgl_dir = project_dir + lvgl_src_dir = os.path.join(lvgl_dir, 'src') + int_lvgl_dir = os.path.join(intermediate_dir, 'lvgl') + lv_conf_dest_file = os.path.join(intermediate_dir, 'lv_conf.h') target_header_base_name = ( os.path.splitext(os.path.split(target_header)[-1])[0] ) try: - os.mkdir(temp_lvgl) - shutil.copytree(lvgl_src_path, os.path.join(temp_lvgl, 'src')) - shutil.copyfile(os.path.join(lvgl_path, 'lvgl.h'), os.path.join(temp_lvgl, 'lvgl.h')) + os.mkdir(int_lvgl_dir) + shutil.copytree(lvgl_src_dir, os.path.join(int_lvgl_dir, 'src')) + shutil.copyfile(os.path.join(lvgl_dir, 'lvgl.h'), os.path.join(int_lvgl_dir, 'lvgl.h')) - pp_file = os.path.join(temp_directory, target_header_base_name + '.pp') + pp_file = os.path.join(intermediate_dir, target_header_base_name + '.pp') - if lvgl_config_path is None: - lvgl_config_path = os.path.join(lvgl_path, 'lv_conf_template.h') + if lv_conf_file is None: + lv_conf_templ_file = os.path.join(lvgl_dir, 'lv_conf_template.h') - with open(lvgl_config_path, 'rb') as f: - data = f.read().decode('utf-8').split('\n') + with open(lv_conf_templ_file, 'rb') as f: + lines = f.read().decode('utf-8').split('\n') - for i, line in enumerate(data): + for i, line in enumerate(lines): if line.startswith('#if 0'): - data[i] = '#if 1' + lines[i] = '#if 1' else: for item in ( 'LV_USE_LOG', @@ -75,17 +77,15 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p 'LV_USE_FREETYPE' ): if line.startswith(f'#define {item} '): - data[i] = f'#define {item} 1' + lines[i] = f'#define {item} 1' break - with open(os.path.join(temp_directory, 'lv_conf.h'), 'wb') as f: - f.write('\n'.join(data).encode('utf-8')) + with open(lv_conf_dest_file, 'wb') as f: + f.write('\n'.join(lines).encode('utf-8')) else: - src = lvgl_config_path - dst = os.path.join(temp_directory, 'lv_conf.h') - shutil.copyfile(src, dst) + shutil.copyfile(lv_conf_file, lv_conf_dest_file) - include_dirs = [temp_directory, project_path] + include_dirs = [intermediate_dir, project_dir] if sys.platform.startswith('win'): import get_sdl2 @@ -103,7 +103,7 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p env = pyMSVC.setup_environment() # NOQA cpp_cmd = ['cl', '/std:c11', '/nologo', '/P'] output_pp = f'/Fi"{pp_file}"' - sdl2_include, _ = get_sdl2.get_sdl2(temp_directory) + sdl2_include, _ = get_sdl2.get_sdl2(intermediate_dir) include_dirs.append(sdl2_include) include_path_env_key = 'INCLUDE' @@ -120,7 +120,7 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p ] output_pp = f' >> "{pp_file}"' - fake_libc_path = create_fake_lib_c.run(temp_directory) + fake_libc_path = create_fake_lib_c.run(intermediate_dir) if include_path_env_key not in os.environ: os.environ[include_path_env_key] = '' @@ -178,12 +178,14 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p cparser = pycparser.CParser() ast = cparser.parse(pp_data, target_header) + doxyfile_src_file = os.path.join(docs_path, doxyfile_filename) - ast.setup_docs(no_docstrings, temp_directory) + ast.setup_docs(no_docstrings, lvgl_src_dir, + intermediate_dir, doxyfile_src_file, output_to_stdout) if not output_to_stdout and output_path is None: if not DEVELOP: - shutil.rmtree(temp_directory) + shutil.rmtree(intermediate_dir) return ast @@ -260,9 +262,9 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p error = 0 if DEVELOP: - print('temporary file path:', temp_directory) + print('temporary file path:', intermediate_dir) else: - shutil.rmtree(temp_directory) + shutil.rmtree(intermediate_dir) sys.exit(error) @@ -311,7 +313,7 @@ if __name__ == '__main__': "using this feature." ), action="store", - default=os.path.join(temp_directory, "lvgl", "lvgl.h") + default=os.path.join(intermediate_dir, "lvgl", "lvgl.h") ) parser.add_argument( '--filter-private', diff --git a/scripts/gen_json/pycparser_monkeypatch.py b/scripts/gen_json/pycparser_monkeypatch.py index 860d3daf0f..98adf30339 100644 --- a/scripts/gen_json/pycparser_monkeypatch.py +++ b/scripts/gen_json/pycparser_monkeypatch.py @@ -571,7 +571,7 @@ class FileAST(c_ast.FileAST): super().__init__(*args, **kwargs) self._parent = None - def setup_docs(self, no_docstrings, temp_directory): # NOQA + def setup_docs(self, no_docstrings, lvgl_src_dir, intermediate_dir, doxyfile_src_file, silent=False): # NOQA global get_enum_item_docs global get_enum_docs global get_func_docs @@ -601,22 +601,39 @@ class FileAST(c_ast.FileAST): get_macros = dummy_list else: - import doc_builder # NOQA + import doxygen_xml # NoQA - doc_builder.EMIT_WARNINGS = False - # doc_builder.DOXYGEN_OUTPUT = False + doxygen_xml.EMIT_WARNINGS = False + # doxygen_xml.DOXYGEN_OUTPUT = False - docs = doc_builder.XMLSearch(temp_directory) + # Instantiating a doxygen_xml.DoxygenXml object: + # - runs Doxygen in `temp_directory` + # - loads XML into `doxygen_xml.index` as a `xml.etree.ElementTree` + # - builds these dictionaries as direct children of `doxygen_xml`: + # = doxygen_xml.defines dictionary of doxygen_xml.DEFINE objects + # = doxygen_xml.enums dictionary of doxygen_xml.ENUM objects + # = doxygen_xml.variables dictionary of doxygen_xml.VARIABLE objects + # = doxygen_xml.namespaces dictionary of doxygen_xml.NAMESPACE objects + # = doxygen_xml.structures dictionary of doxygen_xml.STRUCT objects + # = doxygen_xml.typedefs dictionary of doxygen_xml.TYPEDEF objects + # = doxygen_xml.functions dictionary of doxygen_xml.FUNCTION objects + # = doxygen_xml.groups dictionary of doxygen_xml.GROUP objects + # = doxygen_xml.files dictionary of doxygen_xml.FILE objects + # = doxygen_xml.classes dictionary of doxygen_xml.CLASS objects + doxygen_xml = doxygen_xml.DoxygenXml(lvgl_src_dir, + intermediate_dir, + doxyfile_src_file, + silent_mode=True) - get_enum_item_docs = docs.get_enum_item - get_enum_docs = docs.get_enum - get_func_docs = docs.get_function - get_var_docs = docs.get_variable - get_union_docs = docs.get_union - get_struct_docs = docs.get_structure - get_typedef_docs = docs.get_typedef - get_macro_docs = docs.get_macro - get_macros = docs.get_macros + get_enum_item_docs = doxygen_xml.get_enum_item + get_enum_docs = doxygen_xml.get_enum + get_func_docs = doxygen_xml.get_function + get_var_docs = doxygen_xml.get_variable + get_union_docs = doxygen_xml.get_union + get_struct_docs = doxygen_xml.get_structure + get_typedef_docs = doxygen_xml.get_typedef + get_macro_docs = doxygen_xml.get_macro + get_macros = doxygen_xml.get_macros @property def name(self):