Mirrors_Framework/awesome-python
1
0
Fork 0
mirror of https://github.com/vinta/awesome-python.git synced 2026-03-23 05:03:45 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(parser): remove resources parsing, preview, and content_html fields

Browse Source 
parse_readme now returns list[ParsedGroup] instead of a tuple. The
resources section (Newsletters, Podcasts), preview string, and
content_html rendering are no longer produced by the parser or consumed
by the build. Removes _render_section_html, _group_by_h2, and the
associated dead code and tests.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Vinta Chen
2026-03-23 01:43:19 +08:00
parent c5dd3060ef
commit 25a3f4d903
4 changed files with 37 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
website/build.py
View File

@@ -139,7 +139,7 @@ def build(repo_root: str) -> None:
subtitle = stripped subtitle = stripped
break break
parsed_groups, _ = parse_readme(readme_text) parsed_groups = parse_readme(readme_text)
categories = [cat for g in parsed_groups for cat in g["categories"]] categories = [cat for g in parsed_groups for cat in g["categories"]]
total_entries = sum(c["entry_count"] for c in categories) total_entries = sum(c["entry_count"] for c in categories)
@@ -172,7 +172,6 @@ def build(repo_root: str) -> None:
(site_dir / "index.html").write_text( (site_dir / "index.html").write_text(
tpl_index.render( tpl_index.render(
categories=categories, categories=categories,
groups=parsed_groups,
subtitle=subtitle, subtitle=subtitle,
entries=entries, entries=entries,
total_entries=total_entries, total_entries=total_entries,

132
website/readme_parser.py
View File

@@ -29,8 +29,6 @@ class ParsedSection(TypedDict):
description: str # plain text, links resolved to text description: str # plain text, links resolved to text
entries: list[ParsedEntry] entries: list[ParsedEntry]
entry_count: int entry_count: int
preview: str
content_html: str # rendered HTML, properly escaped
class ParsedGroup(TypedDict): class ParsedGroup(TypedDict):
@@ -258,69 +256,6 @@ def _parse_section_entries(content_nodes: list[SyntaxTreeNode]) -> list[ParsedEn
return entries return entries
# --- Content HTML rendering --------------------------------------------------
def _render_bullet_list_html(
bullet_list: SyntaxTreeNode,
*,
is_sub: bool = False,
) -> str:
"""Render a bullet_list node to HTML with entry/entry-sub/subcat classes."""
out: list[str] = []
for list_item in bullet_list.children:
if list_item.type != "list_item":
continue
inline = _find_inline(list_item)
if inline is None:
continue
first_link = _find_first_link(inline)
if first_link is None or not _is_leading_link(inline, first_link):
# Subcategory label (plain text or text-before-link)
label = str(escape(render_inline_text(inline.children)))
out.append(f'<div class="subcat">{label}</div>')
nested = _find_child(list_item, "bullet_list")
if nested:
out.append(_render_bullet_list_html(nested, is_sub=False))
continue
# Entry with a link
name = str(escape(render_inline_text(first_link.children)))
url = str(escape(first_link.attrGet("href") or ""))
if is_sub:
out.append(f'<div class="entry-sub"><a href="{url}">{name}</a></div>')
else:
desc = _extract_description_html(inline, first_link)
if desc:
out.append(
f'<div class="entry"><a href="{url}">{name}</a>'
f'<span class="sep">&mdash;</span>{desc}</div>'
)
else:
out.append(f'<div class="entry"><a href="{url}">{name}</a></div>')
# Nested items under an entry with a link are sub-entries
nested = _find_child(list_item, "bullet_list")
if nested:
out.append(_render_bullet_list_html(nested, is_sub=True))
return "\n".join(out)
def _render_section_html(content_nodes: list[SyntaxTreeNode]) -> str:
"""Render a section's content nodes to HTML."""
parts: list[str] = []
for node in content_nodes:
if node.type == "bullet_list":
parts.append(_render_bullet_list_html(node))
return "\n".join(parts)
# --- Section splitting ------------------------------------------------------- # --- Section splitting -------------------------------------------------------
@@ -330,45 +265,15 @@ def _build_section(name: str, body: list[SyntaxTreeNode]) -> ParsedSection:
content_nodes = body[1:] if desc else body content_nodes = body[1:] if desc else body
entries = _parse_section_entries(content_nodes) entries = _parse_section_entries(content_nodes)
entry_count = len(entries) + sum(len(e["also_see"]) for e in entries) entry_count = len(entries) + sum(len(e["also_see"]) for e in entries)
preview = ", ".join(e["name"] for e in entries[:4])
content_html = _render_section_html(content_nodes)
return ParsedSection( return ParsedSection(
name=name, name=name,
slug=slugify(name), slug=slugify(name),
description=desc, description=desc,
entries=entries, entries=entries,
entry_count=entry_count, entry_count=entry_count,
preview=preview,
content_html=content_html,
) )
def _group_by_h2(
nodes: list[SyntaxTreeNode],
) -> list[ParsedSection]:
"""Group AST nodes into sections by h2 headings."""
sections: list[ParsedSection] = []
current_name: str | None = None
current_body: list[SyntaxTreeNode] = []
def flush() -> None:
nonlocal current_name
if current_name is None:
return
sections.append(_build_section(current_name, current_body))
current_name = None
for node in nodes:
if node.type == "heading" and node.tag == "h2":
flush()
current_name = _heading_text(node)
current_body = []
elif current_name is not None:
current_body.append(node)
flush()
return sections
def _is_bold_marker(node: SyntaxTreeNode) -> str | None: def _is_bold_marker(node: SyntaxTreeNode) -> str | None:
"""Detect a bold-only paragraph used as a group marker. """Detect a bold-only paragraph used as a group marker.
@@ -445,43 +350,30 @@ def _parse_grouped_sections(
return groups return groups
def parse_readme(text: str) -> tuple[list[ParsedGroup], list[ParsedSection]]: def parse_readme(text: str) -> list[ParsedGroup]:
"""Parse README.md text into grouped categories and resources. """Parse README.md text into grouped categories.
Returns (groups, resources) where groups is a list of ParsedGroup dicts Returns a list of ParsedGroup dicts containing nested categories.
containing nested categories, and resources is a flat list of ParsedSection. Content between the thematic break (---) and # Resources or # Contributing
is parsed as categories grouped by bold markers (**Group Name**).
""" """
md = MarkdownIt("commonmark") md = MarkdownIt("commonmark")
tokens = md.parse(text) tokens = md.parse(text)
root = SyntaxTreeNode(tokens) root = SyntaxTreeNode(tokens)
children = root.children children = root.children
# Find thematic break (---), # Resources, and # Contributing in one pass # Find thematic break (---) and section boundaries in one pass
hr_idx = None hr_idx = None
resources_idx = None cat_end_idx = None
contributing_idx = None
for i, node in enumerate(children): for i, node in enumerate(children):
if hr_idx is None and node.type == "hr": if hr_idx is None and node.type == "hr":
hr_idx = i hr_idx = i
elif node.type == "heading" and node.tag == "h1": elif node.type == "heading" and node.tag == "h1":
text_content = _heading_text(node) text_content = _heading_text(node)
if text_content == "Resources": if cat_end_idx is None and text_content in ("Resources", "Contributing"):
resources_idx = i cat_end_idx = i
elif text_content == "Contributing":
contributing_idx = i
if hr_idx is None: if hr_idx is None:
return [], [] return []
# Slice into category and resource ranges cat_nodes = children[hr_idx + 1 : cat_end_idx or len(children)]
cat_end = resources_idx or contributing_idx or len(children) return _parse_grouped_sections(cat_nodes)
cat_nodes = children[hr_idx + 1 : cat_end]
res_nodes: list[SyntaxTreeNode] = []
if resources_idx is not None:
res_end = contributing_idx or len(children)
res_nodes = children[resources_idx + 1 : res_end]
groups = _parse_grouped_sections(cat_nodes)
resources = _group_by_h2(res_nodes)
return groups, resources

16
website/tests/test_build.py
View File

@@ -59,19 +59,13 @@ class TestBuild:
) )
(tpl_dir / "index.html").write_text( (tpl_dir / "index.html").write_text(
'{% extends "base.html" %}{% block content %}' '{% extends "base.html" %}{% block content %}'
"{% for group in groups %}" "{% for entry in entries %}"
'<section class="group">' '<div class="row">'
"<h2>{{ group.name }}</h2>" "<span>{{ entry.name }}</span>"
"{% for cat in group.categories %}" "<span>{{ entry.categories | join(', ') }}</span>"
'<div class="row" id="{{ cat.slug }}">' "<span>{{ entry.groups | join(', ') }}</span>"
"<span>{{ cat.name }}</span>"
"<span>{{ cat.preview }}</span>"
"<span>{{ cat.entry_count }}</span>"
'<div class="row-content" hidden>{{ cat.content_html | safe }}</div>'
"</div>" "</div>"
"{% endfor %}" "{% endfor %}"
"</section>"
"{% endfor %}"
"{% endblock %}", "{% endblock %}",
encoding="utf-8", encoding="utf-8",
) )

139
website/tests/test_readme_parser.py
View File

@@ -7,7 +7,6 @@ import pytest
from readme_parser import ( from readme_parser import (
_parse_section_entries, _parse_section_entries,
_render_section_html,
parse_readme, parse_readme,
render_inline_html, render_inline_html,
render_inline_text, render_inline_text,
@@ -159,50 +158,39 @@ GROUPED_README = textwrap.dedent("""\
class TestParseReadmeSections: class TestParseReadmeSections:
def test_ungrouped_categories_go_to_other(self): def test_ungrouped_categories_go_to_other(self):
groups, resources = parse_readme(MINIMAL_README) groups = parse_readme(MINIMAL_README)
assert len(groups) == 1 assert len(groups) == 1
assert groups[0]["name"] == "Other" assert groups[0]["name"] == "Other"
assert len(groups[0]["categories"]) == 2 assert len(groups[0]["categories"]) == 2
def test_ungrouped_category_names(self): def test_ungrouped_category_names(self):
groups, _ = parse_readme(MINIMAL_README) groups = parse_readme(MINIMAL_README)
cats = groups[0]["categories"] cats = groups[0]["categories"]
assert cats[0]["name"] == "Alpha" assert cats[0]["name"] == "Alpha"
assert cats[1]["name"] == "Beta" assert cats[1]["name"] == "Beta"
def test_resource_count(self):
_, resources = parse_readme(MINIMAL_README)
assert len(resources) == 2
def test_category_slugs(self): def test_category_slugs(self):
groups, _ = parse_readme(MINIMAL_README) groups = parse_readme(MINIMAL_README)
cats = groups[0]["categories"] cats = groups[0]["categories"]
assert cats[0]["slug"] == "alpha" assert cats[0]["slug"] == "alpha"
assert cats[1]["slug"] == "beta" assert cats[1]["slug"] == "beta"
def test_category_description(self): def test_category_description(self):
groups, _ = parse_readme(MINIMAL_README) groups = parse_readme(MINIMAL_README)
cats = groups[0]["categories"] cats = groups[0]["categories"]
assert cats[0]["description"] == "Libraries for alpha stuff." assert cats[0]["description"] == "Libraries for alpha stuff."
assert cats[1]["description"] == "Tools for beta." assert cats[1]["description"] == "Tools for beta."
def test_resource_names(self):
_, resources = parse_readme(MINIMAL_README)
assert resources[0]["name"] == "Newsletters"
assert resources[1]["name"] == "Podcasts"
def test_contributing_skipped(self): def test_contributing_skipped(self):
groups, resources = parse_readme(MINIMAL_README) groups = parse_readme(MINIMAL_README)
all_names = [] all_names = []
for g in groups: for g in groups:
all_names.extend(c["name"] for c in g["categories"]) all_names.extend(c["name"] for c in g["categories"])
all_names.extend(r["name"] for r in resources)
assert "Contributing" not in all_names assert "Contributing" not in all_names
def test_no_separator(self): def test_no_separator(self):
groups, resources = parse_readme("# Just a heading\n\nSome text.\n") groups = parse_readme("# Just a heading\n\nSome text.\n")
assert groups == [] assert groups == []
assert resources == []
def test_no_description(self): def test_no_description(self):
readme = textwrap.dedent("""\ readme = textwrap.dedent("""\
@@ -224,7 +212,7 @@ class TestParseReadmeSections:
Done. Done.
""") """)
groups, resources = parse_readme(readme) groups = parse_readme(readme)
cats = groups[0]["categories"] cats = groups[0]["categories"]
assert cats[0]["description"] == "" assert cats[0]["description"] == ""
assert cats[0]["entries"][0]["name"] == "item" assert cats[0]["entries"][0]["name"] == "item"
@@ -245,42 +233,37 @@ class TestParseReadmeSections:
Done. Done.
""") """)
groups, _ = parse_readme(readme) groups = parse_readme(readme)
cats = groups[0]["categories"] cats = groups[0]["categories"]
assert cats[0]["description"] == "Algorithms. Also see awesome-algos." assert cats[0]["description"] == "Algorithms. Also see awesome-algos."
class TestParseGroupedReadme: class TestParseGroupedReadme:
def test_group_count(self): def test_group_count(self):
groups, _ = parse_readme(GROUPED_README) groups = parse_readme(GROUPED_README)
assert len(groups) == 2 assert len(groups) == 2
def test_group_names(self): def test_group_names(self):
groups, _ = parse_readme(GROUPED_README) groups = parse_readme(GROUPED_README)
assert groups[0]["name"] == "Group One" assert groups[0]["name"] == "Group One"
assert groups[1]["name"] == "Group Two" assert groups[1]["name"] == "Group Two"
def test_group_slugs(self): def test_group_slugs(self):
groups, _ = parse_readme(GROUPED_README) groups = parse_readme(GROUPED_README)
assert groups[0]["slug"] == "group-one" assert groups[0]["slug"] == "group-one"
assert groups[1]["slug"] == "group-two" assert groups[1]["slug"] == "group-two"
def test_group_one_has_one_category(self): def test_group_one_has_one_category(self):
groups, _ = parse_readme(GROUPED_README) groups = parse_readme(GROUPED_README)
assert len(groups[0]["categories"]) == 1 assert len(groups[0]["categories"]) == 1
assert groups[0]["categories"][0]["name"] == "Alpha" assert groups[0]["categories"][0]["name"] == "Alpha"
def test_group_two_has_two_categories(self): def test_group_two_has_two_categories(self):
groups, _ = parse_readme(GROUPED_README) groups = parse_readme(GROUPED_README)
assert len(groups[1]["categories"]) == 2 assert len(groups[1]["categories"]) == 2
assert groups[1]["categories"][0]["name"] == "Beta" assert groups[1]["categories"][0]["name"] == "Beta"
assert groups[1]["categories"][1]["name"] == "Gamma" assert groups[1]["categories"][1]["name"] == "Gamma"
def test_resources_still_parsed(self):
_, resources = parse_readme(GROUPED_README)
assert len(resources) == 1
assert resources[0]["name"] == "Newsletters"
def test_empty_group_skipped(self): def test_empty_group_skipped(self):
readme = textwrap.dedent("""\ readme = textwrap.dedent("""\
# T # T
@@ -299,7 +282,7 @@ class TestParseGroupedReadme:
Done. Done.
""") """)
groups, _ = parse_readme(readme) groups = parse_readme(readme)
assert len(groups) == 1 assert len(groups) == 1
assert groups[0]["name"] == "HasCats" assert groups[0]["name"] == "HasCats"
@@ -319,7 +302,7 @@ class TestParseGroupedReadme:
Done. Done.
""") """)
groups, _ = parse_readme(readme) groups = parse_readme(readme)
# "Note:" has text after the strong node, so it's not a group marker # "Note:" has text after the strong node, so it's not a group marker
# Category goes into "Other" # Category goes into "Other"
assert len(groups) == 1 assert len(groups) == 1
@@ -345,7 +328,7 @@ class TestParseGroupedReadme:
Done. Done.
""") """)
groups, _ = parse_readme(readme) groups = parse_readme(readme)
assert len(groups) == 2 assert len(groups) == 2
assert groups[0]["name"] == "Other" assert groups[0]["name"] == "Other"
assert groups[0]["categories"][0]["name"] == "Orphan" assert groups[0]["categories"][0]["name"] == "Orphan"
@@ -438,33 +421,11 @@ class TestParseSectionEntries:
Done. Done.
""") """)
groups, _ = parse_readme(readme) groups = parse_readme(readme)
cats = groups[0]["categories"] cats = groups[0]["categories"]
# 2 main entries + 1 also_see = 3 # 2 main entries + 1 also_see = 3
assert cats[0]["entry_count"] == 3 assert cats[0]["entry_count"] == 3
def test_preview_first_four_names(self):
readme = textwrap.dedent("""\
# T
---
## Libs
- [alpha](https://x.com) - A.
- [beta](https://x.com) - B.
- [gamma](https://x.com) - C.
- [delta](https://x.com) - D.
- [epsilon](https://x.com) - E.
# Contributing
Done.
""")
groups, _ = parse_readme(readme)
cats = groups[0]["categories"]
assert cats[0]["preview"] == "alpha, beta, gamma, delta"
def test_description_html_escapes_xss(self): def test_description_html_escapes_xss(self):
nodes = _content_nodes('- [lib](https://x.com) - A <script>alert(1)</script> lib.\n') nodes = _content_nodes('- [lib](https://x.com) - A <script>alert(1)</script> lib.\n')
entries = _parse_section_entries(nodes) entries = _parse_section_entries(nodes)
@@ -472,58 +433,13 @@ class TestParseSectionEntries:
assert "&lt;script&gt;" in entries[0]["description"] assert "&lt;script&gt;" in entries[0]["description"]
class TestRenderSectionHtml:
def test_basic_entry(self):
nodes = _content_nodes("- [django](https://example.com) - A web framework.\n")
html = _render_section_html(nodes)
assert 'class="entry"' in html
assert 'href="https://example.com"' in html
assert "django" in html
assert "A web framework." in html
def test_subcategory_label(self):
nodes = _content_nodes(
"- Synchronous\n - [django](https://x.com) - Framework.\n"
)
html = _render_section_html(nodes)
assert 'class="subcat"' in html
assert "Synchronous" in html
assert 'class="entry"' in html
def test_sub_entry(self):
nodes = _content_nodes(
"- [django](https://x.com) - Framework.\n"
" - [awesome-django](https://y.com)\n"
)
html = _render_section_html(nodes)
assert 'class="entry-sub"' in html
assert "awesome-django" in html
def test_link_only_entry(self):
nodes = _content_nodes("- [tool](https://x.com)\n")
html = _render_section_html(nodes)
assert 'class="entry"' in html
assert 'href="https://x.com"' in html
assert "tool" in html
def test_xss_escaped_in_name(self):
nodes = _content_nodes('- [<img onerror=alert(1)>](https://x.com) - Bad.\n')
html = _render_section_html(nodes)
assert "onerror" not in html or "&" in html
def test_xss_escaped_in_subcat(self):
nodes = _content_nodes("- <script>alert(1)</script>\n")
html = _render_section_html(nodes)
assert "<script>" not in html
class TestParseRealReadme: class TestParseRealReadme:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def load_readme(self): def load_readme(self):
readme_path = os.path.join(os.path.dirname(__file__), "..", "..", "README.md") readme_path = os.path.join(os.path.dirname(__file__), "..", "..", "README.md")
with open(readme_path, encoding="utf-8") as f: with open(readme_path, encoding="utf-8") as f:
self.readme_text = f.read() self.readme_text = f.read()
self.groups, self.resources = parse_readme(self.readme_text) self.groups = parse_readme(self.readme_text)
self.cats = [c for g in self.groups for c in g["categories"]] self.cats = [c for g in self.groups for c in g["categories"]]
def test_at_least_11_groups(self): def test_at_least_11_groups(self):
@@ -535,13 +451,8 @@ class TestParseRealReadme:
def test_at_least_69_categories(self): def test_at_least_69_categories(self):
assert len(self.cats) >= 69 assert len(self.cats) >= 69
def test_resources_has_newsletters_and_podcasts(self):
names = [r["name"] for r in self.resources]
assert "Newsletters" in names
assert "Podcasts" in names
def test_contributing_not_in_results(self): def test_contributing_not_in_results(self):
all_names = [c["name"] for c in self.cats] + [r["name"] for r in self.resources] all_names = [c["name"] for c in self.cats]
assert "Contributing" not in all_names assert "Contributing" not in all_names
def test_first_category_is_ai_and_agents(self): def test_first_category_is_ai_and_agents(self):
@@ -560,18 +471,6 @@ class TestParseRealReadme:
for cat in self.cats: for cat in self.cats:
assert cat["entry_count"] > 0, f"{cat['name']} has 0 entries" assert cat["entry_count"] > 0, f"{cat['name']} has 0 entries"
def test_previews_nonempty(self):
for cat in self.cats:
assert cat["preview"], f"{cat['name']} has empty preview"
def test_content_html_nonempty(self):
for cat in self.cats:
assert cat["content_html"], f"{cat['name']} has empty content_html"
def test_algorithms_has_subcategories(self):
algos = next(c for c in self.cats if c["name"] == "Algorithms and Design Patterns")
assert 'class="subcat"' in algos["content_html"]
def test_async_has_also_see(self): def test_async_has_also_see(self):
async_cat = next(c for c in self.cats if c["name"] == "Asynchronous Programming") async_cat = next(c for c in self.cats if c["name"] == "Asynchronous Programming")
asyncio_entry = next(e for e in async_cat["entries"] if e["name"] == "asyncio") asyncio_entry = next(e for e in async_cat["entries"] if e["name"] == "asyncio")