diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml
index 36fa5e7b..4f5f28c7 100644
--- a/.github/workflows/deploy-website.yml
+++ b/.github/workflows/deploy-website.yml
@@ -24,6 +24,9 @@ jobs:
url: https://awesome-python.com/
steps:
- uses: actions/checkout@v7
+ with:
+ # build.py reads each page's last change from git history for the sitemap's lastmod
+ fetch-depth: 0
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
diff --git a/website/build.py b/website/build.py
index 0652d0f4..28cd7107 100644
--- a/website/build.py
+++ b/website/build.py
@@ -4,6 +4,7 @@
import json
import re
import shutil
+import subprocess
import xml.etree.ElementTree as ET
from collections import Counter
from collections.abc import Sequence
@@ -401,6 +402,20 @@ def synthetic_category(name: str, slug: str) -> SyntheticCategory:
return {"name": name, "slug": slug, "description": "", "description_html": ""}
+def git_last_change_date(repo_root: Path, *log_args: str) -> str:
+ """Return the date of the last commit matching `git log` args, or "" without git history."""
+ result = subprocess.run(["git", "log", "-1", "--format=%cs", "--no-patch", *log_args], cwd=repo_root, capture_output=True, text=True)
+ return result.stdout.strip()
+
+
+def section_line_range(readme_text: str, name: str) -> str:
+ """Return a section's README lines as a `git log -L` range, from its heading to the line before the next heading."""
+ lines = readme_text.split("\n")
+ start = next(i for i, line in enumerate(lines) if re.fullmatch(rf"#+ {re.escape(name)}", line))
+ end = next((i for i in range(start + 1, len(lines)) if lines[i].startswith("#")), len(lines))
+ return f"{start + 1},{end}:README.md"
+
+
def write_sitemap_xml(path: Path, urls: Sequence[tuple[str, str]]) -> None:
ET.register_namespace("", SITEMAP_NS)
urlset = ET.Element(f"{{{SITEMAP_NS}}}urlset")
@@ -881,8 +896,6 @@ def build(repo_root: Path) -> None:
if static_src.exists():
shutil.copytree(static_src, static_dst, dirs_exist_ok=True)
- sponsorship_md = repo_root / "SPONSORSHIP.md"
- sponsorship_md_mtime = datetime.fromtimestamp(sponsorship_md.stat().st_mtime, tz=UTC).date().isoformat()
llms_template = (website / "templates" / "llms.txt").read_text(encoding="utf-8")
llms_txt = build_llms_txt(
llms_template,
@@ -895,15 +908,25 @@ def build(repo_root: Path) -> None:
total_entries=total_entries,
)
(site_dir / "robots.txt").write_text(build_robots_txt(), encoding="utf-8")
- sitemap_date = build_date.date().isoformat()
- sitemap_urls = [(SITE_URL, sitemap_date)]
- sitemap_urls.extend((category_public_url(c), sitemap_date) for c in categories)
- sitemap_urls.extend((group_public_url(g["slug"]), sitemap_date) for g in parsed_groups)
+ # Daily star and download refreshes are not significant changes, so lastmod follows commits to the page's own content.
+ build_day = build_date.date().isoformat()
+ readme_date = git_last_change_date(repo_root, "--", "README.md") or build_day
+ section_dates = {
+ c["slug"]: max(
+ git_last_change_date(repo_root, "-L", section_line_range(readme_text, c["name"])),
+ git_last_change_date(repo_root, "--", f"website/data/category_intros/{c['slug']}.md"),
+ )
+ or build_day
+ for c in categories
+ }
+ sitemap_urls = [(SITE_URL, readme_date)]
+ sitemap_urls.extend((category_public_url(c), section_dates[c["slug"]]) for c in categories)
+ sitemap_urls.extend((group_public_url(g["slug"]), max(section_dates[c["slug"]] for c in g["categories"])) for g in parsed_groups)
if builtin_entries:
- sitemap_urls.append((BUILTIN_PUBLIC_URL, sitemap_date))
+ sitemap_urls.append((BUILTIN_PUBLIC_URL, readme_date))
for cat_slug, sub_slug, _ in sorted(subcat_meta.values()):
- sitemap_urls.append((subcategory_public_url(cat_slug, sub_slug), sitemap_date))
- sitemap_urls.append((SPONSORSHIP_PUBLIC_URL, sponsorship_md_mtime))
+ sitemap_urls.append((subcategory_public_url(cat_slug, sub_slug), section_dates[cat_slug]))
+ sitemap_urls.append((SPONSORSHIP_PUBLIC_URL, git_last_change_date(repo_root, "--", "SPONSORSHIP.md") or build_day))
write_sitemap_xml(site_dir / "sitemap.xml", sitemap_urls)
(site_dir / "llms.txt").write_text(llms_txt, encoding="utf-8")
diff --git a/website/tests/test_build.py b/website/tests/test_build.py
index c38c3979..5e1df4eb 100644
--- a/website/tests/test_build.py
+++ b/website/tests/test_build.py
@@ -2,7 +2,9 @@
import json
import os
+import re
import shutil
+import subprocess
import textwrap
import xml.etree.ElementTree as ET
from datetime import UTC, date, datetime
@@ -198,9 +200,6 @@ class TestBuild:
Help!
""")
self._make_repo(tmp_path, readme)
- sponsorship_mtime = datetime(2024, 1, 2, tzinfo=UTC).timestamp()
- os.utime(tmp_path / "SPONSORSHIP.md", (sponsorship_mtime, sponsorship_mtime))
- expected_sponsorship_lastmod = "2024-01-02"
start_date = datetime.now(UTC).date()
build(tmp_path)
end_date = datetime.now(UTC).date()
@@ -214,7 +213,6 @@ class TestBuild:
ns = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"}
locs = [loc.text or "" for loc in root.findall("sitemap:url/sitemap:loc", ns)]
lastmods = [lastmod.text or "" for lastmod in root.findall("sitemap:url/sitemap:lastmod", ns)]
- lastmod_by_loc = dict(zip(locs, lastmods, strict=True))
assert root.tag == "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset"
assert locs == [
@@ -225,11 +223,63 @@ class TestBuild:
"https://awesome-python.com/sponsorship/",
]
assert len(lastmods) == len(locs)
- assert lastmod_by_loc["https://awesome-python.com/sponsorship/"] == expected_sponsorship_lastmod
- assert all(start_date <= date.fromisoformat(lastmod) <= end_date for loc, lastmod in lastmod_by_loc.items() if loc != "https://awesome-python.com/sponsorship/")
+ # Outside a git repository every page falls back to the build date
+ assert all(start_date <= date.fromisoformat(lastmod) <= end_date for lastmod in lastmods)
assert all(loc.startswith("https://awesome-python.com/") for loc in locs)
assert all("?" not in loc for loc in locs)
+ def test_sitemap_lastmod_follows_git_history_of_each_page(self, tmp_path):
+ readme = textwrap.dedent("""\
+ # Awesome Python
+
+ Intro.
+
+ ## Projects
+
+ **Tools**
+
+ ### Widgets
+
+ - Sync
+
+ - [w1](https://example.com/w1) - A widget.
+
+ ### Gadgets
+
+ - [g1](https://example.com/g1) - A gadget.
+
+ ## Contributing
+
+ Help!
+ """)
+ self._make_repo(tmp_path, readme)
+
+ def commit(day, message):
+ env = {**os.environ, "GIT_AUTHOR_DATE": f"{day}T12:00:00Z", "GIT_COMMITTER_DATE": f"{day}T12:00:00Z"}
+ subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True)
+ subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", message], cwd=tmp_path, env=env, check=True)
+
+ subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True)
+ commit("2024-01-01", "initial")
+ (tmp_path / "README.md").write_text(readme.replace("A gadget.", "A better gadget."), encoding="utf-8")
+ commit("2024-02-01", "edit gadgets")
+ intros_dir = tmp_path / "website" / "data" / "category_intros"
+ intros_dir.mkdir(parents=True)
+ (intros_dir / "widgets.md").write_text("Use w1.\n", encoding="utf-8")
+ commit("2024-03-01", "add widgets intro")
+ build(tmp_path)
+
+ sitemap = (tmp_path / "website" / "output" / "sitemap.xml").read_text(encoding="utf-8")
+ lastmod_by_loc = dict(re.findall(r"https://awesome-python\.com(\S*?)\s*(\S+)", sitemap))
+ assert lastmod_by_loc == {
+ "/": "2024-02-01",
+ "/categories/widgets/": "2024-03-01",
+ "/categories/gadgets/": "2024-02-01",
+ "/categories/tools/": "2024-03-01",
+ "/categories/widgets/sync/": "2024-03-01",
+ "/sponsorship/": "2024-01-01",
+ }
+
def test_build_creates_category_pages_with_metadata_and_links(self, tmp_path):
readme = textwrap.dedent("""\
# Awesome Python