docs(doc_builder.py): overhaul, remove bugs, clean up, modularize (#7913)

scripts/gen_json/gen_json.py

@@ -9,47 +9,49 @@ import subprocess
 base_path = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, base_path)
 
-project_path = os.path.abspath(os.path.join(base_path, '..', '..'))
-docs_path = os.path.join(project_path, 'docs')
+project_dir = os.path.abspath(os.path.join(base_path, '..', '..'))
+docs_path = os.path.join(project_dir, 'docs')
 sys.path.insert(0, docs_path)
 
 import create_fake_lib_c  # NOQA
 import pycparser_monkeypatch  # NOQA
 import pycparser  # NOQA
 
+doxyfile_filename = 'Doxyfile'
 DEVELOP = False
 
 
-temp_directory = tempfile.mkdtemp(suffix='.lvgl_json')
+intermediate_dir = tempfile.mkdtemp(suffix='.lvgl_json')
 
 
-def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_private, no_docstrings, *compiler_args):
+def run(output_path, lv_conf_file, output_to_stdout, target_header, filter_private, no_docstrings, *compiler_args):
 
     pycparser_monkeypatch.FILTER_PRIVATE = filter_private
 
-    lvgl_path = project_path
-    lvgl_src_path = os.path.join(lvgl_path, 'src')
-    temp_lvgl = os.path.join(temp_directory, 'lvgl')
+    lvgl_dir = project_dir
+    lvgl_src_dir = os.path.join(lvgl_dir, 'src')
+    int_lvgl_dir = os.path.join(intermediate_dir, 'lvgl')
+    lv_conf_dest_file = os.path.join(intermediate_dir, 'lv_conf.h')
     target_header_base_name = (
         os.path.splitext(os.path.split(target_header)[-1])[0]
     )
 
     try:
-        os.mkdir(temp_lvgl)
-        shutil.copytree(lvgl_src_path, os.path.join(temp_lvgl, 'src'))
-        shutil.copyfile(os.path.join(lvgl_path, 'lvgl.h'), os.path.join(temp_lvgl, 'lvgl.h'))
+        os.mkdir(int_lvgl_dir)
+        shutil.copytree(lvgl_src_dir, os.path.join(int_lvgl_dir, 'src'))
+        shutil.copyfile(os.path.join(lvgl_dir, 'lvgl.h'), os.path.join(int_lvgl_dir, 'lvgl.h'))
 
-        pp_file = os.path.join(temp_directory, target_header_base_name + '.pp')
+        pp_file = os.path.join(intermediate_dir, target_header_base_name + '.pp')
 
-        if lvgl_config_path is None:
-            lvgl_config_path = os.path.join(lvgl_path, 'lv_conf_template.h')
+        if lv_conf_file is None:
+            lv_conf_templ_file = os.path.join(lvgl_dir, 'lv_conf_template.h')
 
-            with open(lvgl_config_path, 'rb') as f:
-                data = f.read().decode('utf-8').split('\n')
+            with open(lv_conf_templ_file, 'rb') as f:
+                lines = f.read().decode('utf-8').split('\n')
 
-            for i, line in enumerate(data):
+            for i, line in enumerate(lines):
                 if line.startswith('#if 0'):
-                    data[i] = '#if 1'
+                    lines[i] = '#if 1'
                 else:
                     for item in (
                         'LV_USE_LOG',
@@ -75,17 +77,15 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p
                         'LV_USE_FREETYPE'
                     ):
                         if line.startswith(f'#define {item} '):
-                            data[i] = f'#define {item} 1'
+                            lines[i] = f'#define {item} 1'
                             break
 
-            with open(os.path.join(temp_directory, 'lv_conf.h'), 'wb') as f:
-                f.write('\n'.join(data).encode('utf-8'))
+            with open(lv_conf_dest_file, 'wb') as f:
+                f.write('\n'.join(lines).encode('utf-8'))
         else:
-            src = lvgl_config_path
-            dst = os.path.join(temp_directory, 'lv_conf.h')
-            shutil.copyfile(src, dst)
+            shutil.copyfile(lv_conf_file, lv_conf_dest_file)
 
-        include_dirs = [temp_directory, project_path]
+        include_dirs = [intermediate_dir, project_dir]
 
         if sys.platform.startswith('win'):
             import get_sdl2
@@ -103,7 +103,7 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p
             env = pyMSVC.setup_environment()  # NOQA
             cpp_cmd = ['cl', '/std:c11', '/nologo', '/P']
             output_pp = f'/Fi"{pp_file}"'
-            sdl2_include, _ = get_sdl2.get_sdl2(temp_directory)
+            sdl2_include, _ = get_sdl2.get_sdl2(intermediate_dir)
             include_dirs.append(sdl2_include)
             include_path_env_key = 'INCLUDE'
 
@@ -120,7 +120,7 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p
             ]
             output_pp = f' >> "{pp_file}"'
 
-        fake_libc_path = create_fake_lib_c.run(temp_directory)
+        fake_libc_path = create_fake_lib_c.run(intermediate_dir)
 
         if include_path_env_key not in os.environ:
             os.environ[include_path_env_key] = ''
@@ -178,12 +178,14 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p
 
         cparser = pycparser.CParser()
         ast = cparser.parse(pp_data, target_header)
+        doxyfile_src_file = os.path.join(docs_path, doxyfile_filename)
 
-        ast.setup_docs(no_docstrings, temp_directory)
+        ast.setup_docs(no_docstrings, lvgl_src_dir,
+                       intermediate_dir, doxyfile_src_file, output_to_stdout)
 
         if not output_to_stdout and output_path is None:
             if not DEVELOP:
-                shutil.rmtree(temp_directory)
+                shutil.rmtree(intermediate_dir)
 
             return ast
 
@@ -260,9 +262,9 @@ def run(output_path, lvgl_config_path, output_to_stdout, target_header, filter_p
         error = 0
 
     if DEVELOP:
-        print('temporary file path:', temp_directory)
+        print('temporary file path:', intermediate_dir)
     else:
-        shutil.rmtree(temp_directory)
+        shutil.rmtree(intermediate_dir)
 
     sys.exit(error)
 
@@ -311,7 +313,7 @@ if __name__ == '__main__':
             "using this feature."
         ),
         action="store",
-        default=os.path.join(temp_directory, "lvgl", "lvgl.h")
+        default=os.path.join(intermediate_dir, "lvgl", "lvgl.h")
     )
     parser.add_argument(
         '--filter-private',

scripts/gen_json/pycparser_monkeypatch.py

@@ -571,7 +571,7 @@ class FileAST(c_ast.FileAST):
         super().__init__(*args, **kwargs)
         self._parent = None
 
-    def setup_docs(self, no_docstrings, temp_directory):  # NOQA
+    def setup_docs(self, no_docstrings, lvgl_src_dir, intermediate_dir, doxyfile_src_file, silent=False):  # NOQA
         global get_enum_item_docs
         global get_enum_docs
         global get_func_docs
@@ -601,22 +601,39 @@ class FileAST(c_ast.FileAST):
             get_macros = dummy_list
 
         else:
-            import doc_builder  # NOQA
+            import doxygen_xml  # NoQA
 
-            doc_builder.EMIT_WARNINGS = False
-            # doc_builder.DOXYGEN_OUTPUT = False
+            doxygen_xml.EMIT_WARNINGS = False
+            # doxygen_xml.DOXYGEN_OUTPUT = False
 
-            docs = doc_builder.XMLSearch(temp_directory)
+            # Instantiating a doxygen_xml.DoxygenXml object:
+            # - runs Doxygen in `temp_directory`
+            # - loads XML into `doxygen_xml.index` as a `xml.etree.ElementTree`
+            # - builds these dictionaries as direct children of `doxygen_xml`:
+            #   = doxygen_xml.defines     dictionary of doxygen_xml.DEFINE objects
+            #   = doxygen_xml.enums       dictionary of doxygen_xml.ENUM objects
+            #   = doxygen_xml.variables   dictionary of doxygen_xml.VARIABLE objects
+            #   = doxygen_xml.namespaces  dictionary of doxygen_xml.NAMESPACE objects
+            #   = doxygen_xml.structures  dictionary of doxygen_xml.STRUCT objects
+            #   = doxygen_xml.typedefs    dictionary of doxygen_xml.TYPEDEF objects
+            #   = doxygen_xml.functions   dictionary of doxygen_xml.FUNCTION objects
+            #   = doxygen_xml.groups      dictionary of doxygen_xml.GROUP objects
+            #   = doxygen_xml.files       dictionary of doxygen_xml.FILE objects
+            #   = doxygen_xml.classes     dictionary of doxygen_xml.CLASS objects
+            doxygen_xml = doxygen_xml.DoxygenXml(lvgl_src_dir,
+                                                 intermediate_dir,
+                                                 doxyfile_src_file,
+                                                 silent_mode=True)
 
-            get_enum_item_docs = docs.get_enum_item
-            get_enum_docs = docs.get_enum
-            get_func_docs = docs.get_function
-            get_var_docs = docs.get_variable
-            get_union_docs = docs.get_union
-            get_struct_docs = docs.get_structure
-            get_typedef_docs = docs.get_typedef
-            get_macro_docs = docs.get_macro
-            get_macros = docs.get_macros
+            get_enum_item_docs = doxygen_xml.get_enum_item
+            get_enum_docs = doxygen_xml.get_enum
+            get_func_docs = doxygen_xml.get_function
+            get_var_docs = doxygen_xml.get_variable
+            get_union_docs = doxygen_xml.get_union
+            get_struct_docs = doxygen_xml.get_structure
+            get_typedef_docs = doxygen_xml.get_typedef
+            get_macro_docs = doxygen_xml.get_macro
+            get_macros = doxygen_xml.get_macros
 
     @property
     def name(self):