Merge branch 'sitemap-git-lastmod'

.github/workflows/deploy-website.yml

@@ -24,6 +24,9 @@ jobs:
       url: https://awesome-python.com/
     steps:
       - uses: actions/checkout@v7
+        with:
+          # build.py reads each page's last change from git history for the sitemap's lastmod
+          fetch-depth: 0
 
       - name: Install uv
         uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0

website/build.py

@@ -4,6 +4,7 @@
 import json
 import re
 import shutil
+import subprocess
 import xml.etree.ElementTree as ET
 from collections import Counter
 from collections.abc import Sequence
@@ -401,6 +402,20 @@ def synthetic_category(name: str, slug: str) -> SyntheticCategory:
     return {"name": name, "slug": slug, "description": "", "description_html": ""}
 
 
+def git_last_change_date(repo_root: Path, *log_args: str) -> str:
+    """Return the date of the last commit matching `git log` args, or "" without git history."""
+    result = subprocess.run(["git", "log", "-1", "--format=%cs", "--no-patch", *log_args], cwd=repo_root, capture_output=True, text=True)
+    return result.stdout.strip()
+
+
+def section_line_range(readme_text: str, name: str) -> str:
+    """Return a section's README lines as a `git log -L` range, from its heading to the line before the next heading."""
+    lines = readme_text.split("\n")
+    start = next(i for i, line in enumerate(lines) if re.fullmatch(rf"#+ {re.escape(name)}", line))
+    end = next((i for i in range(start + 1, len(lines)) if lines[i].startswith("#")), len(lines))
+    return f"{start + 1},{end}:README.md"
+
+
 def write_sitemap_xml(path: Path, urls: Sequence[tuple[str, str]]) -> None:
     ET.register_namespace("", SITEMAP_NS)
     urlset = ET.Element(f"{{{SITEMAP_NS}}}urlset")
@@ -881,8 +896,6 @@ def build(repo_root: Path) -> None:
     if static_src.exists():
         shutil.copytree(static_src, static_dst, dirs_exist_ok=True)
 
-    sponsorship_md = repo_root / "SPONSORSHIP.md"
-    sponsorship_md_mtime = datetime.fromtimestamp(sponsorship_md.stat().st_mtime, tz=UTC).date().isoformat()
     llms_template = (website / "templates" / "llms.txt").read_text(encoding="utf-8")
     llms_txt = build_llms_txt(
         llms_template,
@@ -895,15 +908,25 @@ def build(repo_root: Path) -> None:
         total_entries=total_entries,
     )
     (site_dir / "robots.txt").write_text(build_robots_txt(), encoding="utf-8")
-    sitemap_date = build_date.date().isoformat()
-    sitemap_urls = [(SITE_URL, sitemap_date)]
-    sitemap_urls.extend((category_public_url(c), sitemap_date) for c in categories)
-    sitemap_urls.extend((group_public_url(g["slug"]), sitemap_date) for g in parsed_groups)
+    # Daily star and download refreshes are not significant changes, so lastmod follows commits to the page's own content.
+    build_day = build_date.date().isoformat()
+    readme_date = git_last_change_date(repo_root, "--", "README.md") or build_day
+    section_dates = {
+        c["slug"]: max(
+            git_last_change_date(repo_root, "-L", section_line_range(readme_text, c["name"])),
+            git_last_change_date(repo_root, "--", f"website/data/category_intros/{c['slug']}.md"),
+        )
+        or build_day
+        for c in categories
+    }
+    sitemap_urls = [(SITE_URL, readme_date)]
+    sitemap_urls.extend((category_public_url(c), section_dates[c["slug"]]) for c in categories)
+    sitemap_urls.extend((group_public_url(g["slug"]), max(section_dates[c["slug"]] for c in g["categories"])) for g in parsed_groups)
     if builtin_entries:
-        sitemap_urls.append((BUILTIN_PUBLIC_URL, sitemap_date))
+        sitemap_urls.append((BUILTIN_PUBLIC_URL, readme_date))
     for cat_slug, sub_slug, _ in sorted(subcat_meta.values()):
-        sitemap_urls.append((subcategory_public_url(cat_slug, sub_slug), sitemap_date))
-    sitemap_urls.append((SPONSORSHIP_PUBLIC_URL, sponsorship_md_mtime))
+        sitemap_urls.append((subcategory_public_url(cat_slug, sub_slug), section_dates[cat_slug]))
+    sitemap_urls.append((SPONSORSHIP_PUBLIC_URL, git_last_change_date(repo_root, "--", "SPONSORSHIP.md") or build_day))
     write_sitemap_xml(site_dir / "sitemap.xml", sitemap_urls)
     (site_dir / "llms.txt").write_text(llms_txt, encoding="utf-8")
 

website/tests/test_build.py

@@ -2,7 +2,9 @@
 
 import json
 import os
+import re
 import shutil
+import subprocess
 import textwrap
 import xml.etree.ElementTree as ET
 from datetime import UTC, date, datetime
@@ -198,9 +200,6 @@ class TestBuild:
             Help!
         """)
         self._make_repo(tmp_path, readme)
-        sponsorship_mtime = datetime(2024, 1, 2, tzinfo=UTC).timestamp()
-        os.utime(tmp_path / "SPONSORSHIP.md", (sponsorship_mtime, sponsorship_mtime))
-        expected_sponsorship_lastmod = "2024-01-02"
         start_date = datetime.now(UTC).date()
         build(tmp_path)
         end_date = datetime.now(UTC).date()
@@ -214,7 +213,6 @@ class TestBuild:
         ns = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"}
         locs = [loc.text or "" for loc in root.findall("sitemap:url/sitemap:loc", ns)]
         lastmods = [lastmod.text or "" for lastmod in root.findall("sitemap:url/sitemap:lastmod", ns)]
-        lastmod_by_loc = dict(zip(locs, lastmods, strict=True))
 
         assert root.tag == "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset"
         assert locs == [
@@ -225,11 +223,63 @@ class TestBuild:
             "https://awesome-python.com/sponsorship/",
         ]
         assert len(lastmods) == len(locs)
-        assert lastmod_by_loc["https://awesome-python.com/sponsorship/"] == expected_sponsorship_lastmod
-        assert all(start_date <= date.fromisoformat(lastmod) <= end_date for loc, lastmod in lastmod_by_loc.items() if loc != "https://awesome-python.com/sponsorship/")
+        # Outside a git repository every page falls back to the build date
+        assert all(start_date <= date.fromisoformat(lastmod) <= end_date for lastmod in lastmods)
         assert all(loc.startswith("https://awesome-python.com/") for loc in locs)
         assert all("?" not in loc for loc in locs)
 
+    def test_sitemap_lastmod_follows_git_history_of_each_page(self, tmp_path):
+        readme = textwrap.dedent("""\
+            # Awesome Python
+
+            Intro.
+
+            ## Projects
+
+            **Tools**
+
+            ### Widgets
+
+            - Sync
+
+                - [w1](https://example.com/w1) - A widget.
+
+            ### Gadgets
+
+            - [g1](https://example.com/g1) - A gadget.
+
+            ## Contributing
+
+            Help!
+        """)
+        self._make_repo(tmp_path, readme)
+
+        def commit(day, message):
+            env = {**os.environ, "GIT_AUTHOR_DATE": f"{day}T12:00:00Z", "GIT_COMMITTER_DATE": f"{day}T12:00:00Z"}
+            subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True)
+            subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", message], cwd=tmp_path, env=env, check=True)
+
+        subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True)
+        commit("2024-01-01", "initial")
+        (tmp_path / "README.md").write_text(readme.replace("A gadget.", "A better gadget."), encoding="utf-8")
+        commit("2024-02-01", "edit gadgets")
+        intros_dir = tmp_path / "website" / "data" / "category_intros"
+        intros_dir.mkdir(parents=True)
+        (intros_dir / "widgets.md").write_text("Use w1.\n", encoding="utf-8")
+        commit("2024-03-01", "add widgets intro")
+        build(tmp_path)
+
+        sitemap = (tmp_path / "website" / "output" / "sitemap.xml").read_text(encoding="utf-8")
+        lastmod_by_loc = dict(re.findall(r"<loc>https://awesome-python\.com(\S*?)</loc>\s*<lastmod>(\S+)</lastmod>", sitemap))
+        assert lastmod_by_loc == {
+            "/": "2024-02-01",
+            "/categories/widgets/": "2024-03-01",
+            "/categories/gadgets/": "2024-02-01",
+            "/categories/tools/": "2024-03-01",
+            "/categories/widgets/sync/": "2024-03-01",
+            "/sponsorship/": "2024-01-01",
+        }
+
     def test_build_creates_category_pages_with_metadata_and_links(self, tmp_path):
         readme = textwrap.dedent("""\
             # Awesome Python