mirror of
https://github.com/vinta/awesome-python.git
synced 2026-03-23 05:03:45 +08:00
refactor(parser): remove resources parsing, preview, and content_html fields
parse_readme now returns list[ParsedGroup] instead of a tuple. The resources section (Newsletters, Podcasts), preview string, and content_html rendering are no longer produced by the parser or consumed by the build. Removes _render_section_html, _group_by_h2, and the associated dead code and tests. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Vinta Chen
@@ -139,7 +139,7 @@ def build(repo_root: str) -> None:
|
||||
subtitle = stripped
|
||||
break
|
||||
|
||||
parsed_groups, _ = parse_readme(readme_text)
|
||||
parsed_groups = parse_readme(readme_text)
|
||||
|
||||
categories = [cat for g in parsed_groups for cat in g["categories"]]
|
||||
total_entries = sum(c["entry_count"] for c in categories)
|
||||
@@ -172,7 +172,6 @@ def build(repo_root: str) -> None:
|
||||
(site_dir / "index.html").write_text(
|
||||
tpl_index.render(
|
||||
categories=categories,
|
||||
groups=parsed_groups,
|
||||
subtitle=subtitle,
|
||||
entries=entries,
|
||||
total_entries=total_entries,
|
||||
|
||||
@@ -29,8 +29,6 @@ class ParsedSection(TypedDict):
|
||||
description: str # plain text, links resolved to text
|
||||
entries: list[ParsedEntry]
|
||||
entry_count: int
|
||||
preview: str
|
||||
content_html: str # rendered HTML, properly escaped
|
||||
|
||||
|
||||
class ParsedGroup(TypedDict):
|
||||
@@ -258,69 +256,6 @@ def _parse_section_entries(content_nodes: list[SyntaxTreeNode]) -> list[ParsedEn
|
||||
return entries
|
||||
|
||||
|
||||
# --- Content HTML rendering --------------------------------------------------
|
||||
|
||||
|
||||
def _render_bullet_list_html(
|
||||
bullet_list: SyntaxTreeNode,
|
||||
*,
|
||||
is_sub: bool = False,
|
||||
) -> str:
|
||||
"""Render a bullet_list node to HTML with entry/entry-sub/subcat classes."""
|
||||
out: list[str] = []
|
||||
|
||||
for list_item in bullet_list.children:
|
||||
if list_item.type != "list_item":
|
||||
continue
|
||||
|
||||
inline = _find_inline(list_item)
|
||||
if inline is None:
|
||||
continue
|
||||
|
||||
first_link = _find_first_link(inline)
|
||||
|
||||
if first_link is None or not _is_leading_link(inline, first_link):
|
||||
# Subcategory label (plain text or text-before-link)
|
||||
label = str(escape(render_inline_text(inline.children)))
|
||||
out.append(f'<div class="subcat">{label}</div>')
|
||||
nested = _find_child(list_item, "bullet_list")
|
||||
if nested:
|
||||
out.append(_render_bullet_list_html(nested, is_sub=False))
|
||||
continue
|
||||
|
||||
# Entry with a link
|
||||
name = str(escape(render_inline_text(first_link.children)))
|
||||
url = str(escape(first_link.attrGet("href") or ""))
|
||||
|
||||
if is_sub:
|
||||
out.append(f'<div class="entry-sub"><a href="{url}">{name}</a></div>')
|
||||
else:
|
||||
desc = _extract_description_html(inline, first_link)
|
||||
if desc:
|
||||
out.append(
|
||||
f'<div class="entry"><a href="{url}">{name}</a>'
|
||||
f'<span class="sep">—</span>{desc}</div>'
|
||||
)
|
||||
else:
|
||||
out.append(f'<div class="entry"><a href="{url}">{name}</a></div>')
|
||||
|
||||
# Nested items under an entry with a link are sub-entries
|
||||
nested = _find_child(list_item, "bullet_list")
|
||||
if nested:
|
||||
out.append(_render_bullet_list_html(nested, is_sub=True))
|
||||
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _render_section_html(content_nodes: list[SyntaxTreeNode]) -> str:
|
||||
"""Render a section's content nodes to HTML."""
|
||||
parts: list[str] = []
|
||||
for node in content_nodes:
|
||||
if node.type == "bullet_list":
|
||||
parts.append(_render_bullet_list_html(node))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
# --- Section splitting -------------------------------------------------------
|
||||
|
||||
|
||||
@@ -330,45 +265,15 @@ def _build_section(name: str, body: list[SyntaxTreeNode]) -> ParsedSection:
|
||||
content_nodes = body[1:] if desc else body
|
||||
entries = _parse_section_entries(content_nodes)
|
||||
entry_count = len(entries) + sum(len(e["also_see"]) for e in entries)
|
||||
preview = ", ".join(e["name"] for e in entries[:4])
|
||||
content_html = _render_section_html(content_nodes)
|
||||
return ParsedSection(
|
||||
name=name,
|
||||
slug=slugify(name),
|
||||
description=desc,
|
||||
entries=entries,
|
||||
entry_count=entry_count,
|
||||
preview=preview,
|
||||
content_html=content_html,
|
||||
)
|
||||
|
||||
|
||||
def _group_by_h2(
|
||||
nodes: list[SyntaxTreeNode],
|
||||
) -> list[ParsedSection]:
|
||||
"""Group AST nodes into sections by h2 headings."""
|
||||
sections: list[ParsedSection] = []
|
||||
current_name: str | None = None
|
||||
current_body: list[SyntaxTreeNode] = []
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current_name
|
||||
if current_name is None:
|
||||
return
|
||||
sections.append(_build_section(current_name, current_body))
|
||||
current_name = None
|
||||
|
||||
for node in nodes:
|
||||
if node.type == "heading" and node.tag == "h2":
|
||||
flush()
|
||||
current_name = _heading_text(node)
|
||||
current_body = []
|
||||
elif current_name is not None:
|
||||
current_body.append(node)
|
||||
|
||||
flush()
|
||||
return sections
|
||||
|
||||
|
||||
def _is_bold_marker(node: SyntaxTreeNode) -> str | None:
|
||||
"""Detect a bold-only paragraph used as a group marker.
|
||||
@@ -445,43 +350,30 @@ def _parse_grouped_sections(
|
||||
return groups
|
||||
|
||||
|
||||
def parse_readme(text: str) -> tuple[list[ParsedGroup], list[ParsedSection]]:
|
||||
"""Parse README.md text into grouped categories and resources.
|
||||
def parse_readme(text: str) -> list[ParsedGroup]:
|
||||
"""Parse README.md text into grouped categories.
|
||||
|
||||
Returns (groups, resources) where groups is a list of ParsedGroup dicts
|
||||
containing nested categories, and resources is a flat list of ParsedSection.
|
||||
Returns a list of ParsedGroup dicts containing nested categories.
|
||||
Content between the thematic break (---) and # Resources or # Contributing
|
||||
is parsed as categories grouped by bold markers (**Group Name**).
|
||||
"""
|
||||
md = MarkdownIt("commonmark")
|
||||
tokens = md.parse(text)
|
||||
root = SyntaxTreeNode(tokens)
|
||||
children = root.children
|
||||
|
||||
# Find thematic break (---), # Resources, and # Contributing in one pass
|
||||
# Find thematic break (---) and section boundaries in one pass
|
||||
hr_idx = None
|
||||
resources_idx = None
|
||||
contributing_idx = None
|
||||
cat_end_idx = None
|
||||
for i, node in enumerate(children):
|
||||
if hr_idx is None and node.type == "hr":
|
||||
hr_idx = i
|
||||
elif node.type == "heading" and node.tag == "h1":
|
||||
text_content = _heading_text(node)
|
||||
if text_content == "Resources":
|
||||
resources_idx = i
|
||||
elif text_content == "Contributing":
|
||||
contributing_idx = i
|
||||
if cat_end_idx is None and text_content in ("Resources", "Contributing"):
|
||||
cat_end_idx = i
|
||||
if hr_idx is None:
|
||||
return [], []
|
||||
return []
|
||||
|
||||
# Slice into category and resource ranges
|
||||
cat_end = resources_idx or contributing_idx or len(children)
|
||||
cat_nodes = children[hr_idx + 1 : cat_end]
|
||||
|
||||
res_nodes: list[SyntaxTreeNode] = []
|
||||
if resources_idx is not None:
|
||||
res_end = contributing_idx or len(children)
|
||||
res_nodes = children[resources_idx + 1 : res_end]
|
||||
|
||||
groups = _parse_grouped_sections(cat_nodes)
|
||||
resources = _group_by_h2(res_nodes)
|
||||
|
||||
return groups, resources
|
||||
cat_nodes = children[hr_idx + 1 : cat_end_idx or len(children)]
|
||||
return _parse_grouped_sections(cat_nodes)
|
||||
|
||||
@@ -59,19 +59,13 @@ class TestBuild:
|
||||
)
|
||||
(tpl_dir / "index.html").write_text(
|
||||
'{% extends "base.html" %}{% block content %}'
|
||||
"{% for group in groups %}"
|
||||
'<section class="group">'
|
||||
"<h2>{{ group.name }}</h2>"
|
||||
"{% for cat in group.categories %}"
|
||||
'<div class="row" id="{{ cat.slug }}">'
|
||||
"<span>{{ cat.name }}</span>"
|
||||
"<span>{{ cat.preview }}</span>"
|
||||
"<span>{{ cat.entry_count }}</span>"
|
||||
'<div class="row-content" hidden>{{ cat.content_html | safe }}</div>'
|
||||
"{% for entry in entries %}"
|
||||
'<div class="row">'
|
||||
"<span>{{ entry.name }}</span>"
|
||||
"<span>{{ entry.categories | join(', ') }}</span>"
|
||||
"<span>{{ entry.groups | join(', ') }}</span>"
|
||||
"</div>"
|
||||
"{% endfor %}"
|
||||
"</section>"
|
||||
"{% endfor %}"
|
||||
"{% endblock %}",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@@ -7,7 +7,6 @@ import pytest
|
||||
|
||||
from readme_parser import (
|
||||
_parse_section_entries,
|
||||
_render_section_html,
|
||||
parse_readme,
|
||||
render_inline_html,
|
||||
render_inline_text,
|
||||
@@ -159,50 +158,39 @@ GROUPED_README = textwrap.dedent("""\
|
||||
|
||||
class TestParseReadmeSections:
|
||||
def test_ungrouped_categories_go_to_other(self):
|
||||
groups, resources = parse_readme(MINIMAL_README)
|
||||
groups = parse_readme(MINIMAL_README)
|
||||
assert len(groups) == 1
|
||||
assert groups[0]["name"] == "Other"
|
||||
assert len(groups[0]["categories"]) == 2
|
||||
|
||||
def test_ungrouped_category_names(self):
|
||||
groups, _ = parse_readme(MINIMAL_README)
|
||||
groups = parse_readme(MINIMAL_README)
|
||||
cats = groups[0]["categories"]
|
||||
assert cats[0]["name"] == "Alpha"
|
||||
assert cats[1]["name"] == "Beta"
|
||||
|
||||
def test_resource_count(self):
|
||||
_, resources = parse_readme(MINIMAL_README)
|
||||
assert len(resources) == 2
|
||||
|
||||
def test_category_slugs(self):
|
||||
groups, _ = parse_readme(MINIMAL_README)
|
||||
groups = parse_readme(MINIMAL_README)
|
||||
cats = groups[0]["categories"]
|
||||
assert cats[0]["slug"] == "alpha"
|
||||
assert cats[1]["slug"] == "beta"
|
||||
|
||||
def test_category_description(self):
|
||||
groups, _ = parse_readme(MINIMAL_README)
|
||||
groups = parse_readme(MINIMAL_README)
|
||||
cats = groups[0]["categories"]
|
||||
assert cats[0]["description"] == "Libraries for alpha stuff."
|
||||
assert cats[1]["description"] == "Tools for beta."
|
||||
|
||||
def test_resource_names(self):
|
||||
_, resources = parse_readme(MINIMAL_README)
|
||||
assert resources[0]["name"] == "Newsletters"
|
||||
assert resources[1]["name"] == "Podcasts"
|
||||
|
||||
def test_contributing_skipped(self):
|
||||
groups, resources = parse_readme(MINIMAL_README)
|
||||
groups = parse_readme(MINIMAL_README)
|
||||
all_names = []
|
||||
for g in groups:
|
||||
all_names.extend(c["name"] for c in g["categories"])
|
||||
all_names.extend(r["name"] for r in resources)
|
||||
assert "Contributing" not in all_names
|
||||
|
||||
def test_no_separator(self):
|
||||
groups, resources = parse_readme("# Just a heading\n\nSome text.\n")
|
||||
groups = parse_readme("# Just a heading\n\nSome text.\n")
|
||||
assert groups == []
|
||||
assert resources == []
|
||||
|
||||
def test_no_description(self):
|
||||
readme = textwrap.dedent("""\
|
||||
@@ -224,7 +212,7 @@ class TestParseReadmeSections:
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, resources = parse_readme(readme)
|
||||
groups = parse_readme(readme)
|
||||
cats = groups[0]["categories"]
|
||||
assert cats[0]["description"] == ""
|
||||
assert cats[0]["entries"][0]["name"] == "item"
|
||||
@@ -245,42 +233,37 @@ class TestParseReadmeSections:
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, _ = parse_readme(readme)
|
||||
groups = parse_readme(readme)
|
||||
cats = groups[0]["categories"]
|
||||
assert cats[0]["description"] == "Algorithms. Also see awesome-algos."
|
||||
|
||||
|
||||
class TestParseGroupedReadme:
|
||||
def test_group_count(self):
|
||||
groups, _ = parse_readme(GROUPED_README)
|
||||
groups = parse_readme(GROUPED_README)
|
||||
assert len(groups) == 2
|
||||
|
||||
def test_group_names(self):
|
||||
groups, _ = parse_readme(GROUPED_README)
|
||||
groups = parse_readme(GROUPED_README)
|
||||
assert groups[0]["name"] == "Group One"
|
||||
assert groups[1]["name"] == "Group Two"
|
||||
|
||||
def test_group_slugs(self):
|
||||
groups, _ = parse_readme(GROUPED_README)
|
||||
groups = parse_readme(GROUPED_README)
|
||||
assert groups[0]["slug"] == "group-one"
|
||||
assert groups[1]["slug"] == "group-two"
|
||||
|
||||
def test_group_one_has_one_category(self):
|
||||
groups, _ = parse_readme(GROUPED_README)
|
||||
groups = parse_readme(GROUPED_README)
|
||||
assert len(groups[0]["categories"]) == 1
|
||||
assert groups[0]["categories"][0]["name"] == "Alpha"
|
||||
|
||||
def test_group_two_has_two_categories(self):
|
||||
groups, _ = parse_readme(GROUPED_README)
|
||||
groups = parse_readme(GROUPED_README)
|
||||
assert len(groups[1]["categories"]) == 2
|
||||
assert groups[1]["categories"][0]["name"] == "Beta"
|
||||
assert groups[1]["categories"][1]["name"] == "Gamma"
|
||||
|
||||
def test_resources_still_parsed(self):
|
||||
_, resources = parse_readme(GROUPED_README)
|
||||
assert len(resources) == 1
|
||||
assert resources[0]["name"] == "Newsletters"
|
||||
|
||||
def test_empty_group_skipped(self):
|
||||
readme = textwrap.dedent("""\
|
||||
# T
|
||||
@@ -299,7 +282,7 @@ class TestParseGroupedReadme:
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, _ = parse_readme(readme)
|
||||
groups = parse_readme(readme)
|
||||
assert len(groups) == 1
|
||||
assert groups[0]["name"] == "HasCats"
|
||||
|
||||
@@ -319,7 +302,7 @@ class TestParseGroupedReadme:
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, _ = parse_readme(readme)
|
||||
groups = parse_readme(readme)
|
||||
# "Note:" has text after the strong node, so it's not a group marker
|
||||
# Category goes into "Other"
|
||||
assert len(groups) == 1
|
||||
@@ -345,7 +328,7 @@ class TestParseGroupedReadme:
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, _ = parse_readme(readme)
|
||||
groups = parse_readme(readme)
|
||||
assert len(groups) == 2
|
||||
assert groups[0]["name"] == "Other"
|
||||
assert groups[0]["categories"][0]["name"] == "Orphan"
|
||||
@@ -438,33 +421,11 @@ class TestParseSectionEntries:
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, _ = parse_readme(readme)
|
||||
groups = parse_readme(readme)
|
||||
cats = groups[0]["categories"]
|
||||
# 2 main entries + 1 also_see = 3
|
||||
assert cats[0]["entry_count"] == 3
|
||||
|
||||
def test_preview_first_four_names(self):
|
||||
readme = textwrap.dedent("""\
|
||||
# T
|
||||
|
||||
---
|
||||
|
||||
## Libs
|
||||
|
||||
- [alpha](https://x.com) - A.
|
||||
- [beta](https://x.com) - B.
|
||||
- [gamma](https://x.com) - C.
|
||||
- [delta](https://x.com) - D.
|
||||
- [epsilon](https://x.com) - E.
|
||||
|
||||
# Contributing
|
||||
|
||||
Done.
|
||||
""")
|
||||
groups, _ = parse_readme(readme)
|
||||
cats = groups[0]["categories"]
|
||||
assert cats[0]["preview"] == "alpha, beta, gamma, delta"
|
||||
|
||||
def test_description_html_escapes_xss(self):
|
||||
nodes = _content_nodes('- [lib](https://x.com) - A <script>alert(1)</script> lib.\n')
|
||||
entries = _parse_section_entries(nodes)
|
||||
@@ -472,58 +433,13 @@ class TestParseSectionEntries:
|
||||
assert "<script>" in entries[0]["description"]
|
||||
|
||||
|
||||
class TestRenderSectionHtml:
|
||||
def test_basic_entry(self):
|
||||
nodes = _content_nodes("- [django](https://example.com) - A web framework.\n")
|
||||
html = _render_section_html(nodes)
|
||||
assert 'class="entry"' in html
|
||||
assert 'href="https://example.com"' in html
|
||||
assert "django" in html
|
||||
assert "A web framework." in html
|
||||
|
||||
def test_subcategory_label(self):
|
||||
nodes = _content_nodes(
|
||||
"- Synchronous\n - [django](https://x.com) - Framework.\n"
|
||||
)
|
||||
html = _render_section_html(nodes)
|
||||
assert 'class="subcat"' in html
|
||||
assert "Synchronous" in html
|
||||
assert 'class="entry"' in html
|
||||
|
||||
def test_sub_entry(self):
|
||||
nodes = _content_nodes(
|
||||
"- [django](https://x.com) - Framework.\n"
|
||||
" - [awesome-django](https://y.com)\n"
|
||||
)
|
||||
html = _render_section_html(nodes)
|
||||
assert 'class="entry-sub"' in html
|
||||
assert "awesome-django" in html
|
||||
|
||||
def test_link_only_entry(self):
|
||||
nodes = _content_nodes("- [tool](https://x.com)\n")
|
||||
html = _render_section_html(nodes)
|
||||
assert 'class="entry"' in html
|
||||
assert 'href="https://x.com"' in html
|
||||
assert "tool" in html
|
||||
|
||||
def test_xss_escaped_in_name(self):
|
||||
nodes = _content_nodes('- [<img onerror=alert(1)>](https://x.com) - Bad.\n')
|
||||
html = _render_section_html(nodes)
|
||||
assert "onerror" not in html or "&" in html
|
||||
|
||||
def test_xss_escaped_in_subcat(self):
|
||||
nodes = _content_nodes("- <script>alert(1)</script>\n")
|
||||
html = _render_section_html(nodes)
|
||||
assert "<script>" not in html
|
||||
|
||||
|
||||
class TestParseRealReadme:
|
||||
@pytest.fixture(autouse=True)
|
||||
def load_readme(self):
|
||||
readme_path = os.path.join(os.path.dirname(__file__), "..", "..", "README.md")
|
||||
with open(readme_path, encoding="utf-8") as f:
|
||||
self.readme_text = f.read()
|
||||
self.groups, self.resources = parse_readme(self.readme_text)
|
||||
self.groups = parse_readme(self.readme_text)
|
||||
self.cats = [c for g in self.groups for c in g["categories"]]
|
||||
|
||||
def test_at_least_11_groups(self):
|
||||
@@ -535,13 +451,8 @@ class TestParseRealReadme:
|
||||
def test_at_least_69_categories(self):
|
||||
assert len(self.cats) >= 69
|
||||
|
||||
def test_resources_has_newsletters_and_podcasts(self):
|
||||
names = [r["name"] for r in self.resources]
|
||||
assert "Newsletters" in names
|
||||
assert "Podcasts" in names
|
||||
|
||||
def test_contributing_not_in_results(self):
|
||||
all_names = [c["name"] for c in self.cats] + [r["name"] for r in self.resources]
|
||||
all_names = [c["name"] for c in self.cats]
|
||||
assert "Contributing" not in all_names
|
||||
|
||||
def test_first_category_is_ai_and_agents(self):
|
||||
@@ -560,18 +471,6 @@ class TestParseRealReadme:
|
||||
for cat in self.cats:
|
||||
assert cat["entry_count"] > 0, f"{cat['name']} has 0 entries"
|
||||
|
||||
def test_previews_nonempty(self):
|
||||
for cat in self.cats:
|
||||
assert cat["preview"], f"{cat['name']} has empty preview"
|
||||
|
||||
def test_content_html_nonempty(self):
|
||||
for cat in self.cats:
|
||||
assert cat["content_html"], f"{cat['name']} has empty content_html"
|
||||
|
||||
def test_algorithms_has_subcategories(self):
|
||||
algos = next(c for c in self.cats if c["name"] == "Algorithms and Design Patterns")
|
||||
assert 'class="subcat"' in algos["content_html"]
|
||||
|
||||
def test_async_has_also_see(self):
|
||||
async_cat = next(c for c in self.cats if c["name"] == "Asynchronous Programming")
|
||||
asyncio_entry = next(e for e in async_cat["entries"] if e["name"] == "asyncio")
|
||||
|
||||
Reference in New Issue
Block a user