Merge pull request #2972 from vinta/feature/fetch-stats-workflow

ci: consolidate star fetch into deploy workflow with Actions cache

.github/workflows/deploy-website.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - master
+  schedule:
+    - cron: "0 0 * * *"
 
 permissions:
   contents: read
@@ -26,10 +28,40 @@ jobs:
           enable-cache: true
 
       - name: Install dependencies
-        run: uv sync --no-dev
+        run: uv sync --group build
+
+      - name: Restore star data cache
+        id: cache-stars
+        uses: actions/cache/restore@v4
+        with:
+          path: website/data/github_stars.json
+          key: github-stars-${{ github.run_id }}
+          restore-keys: github-stars-
+
+      - name: Fetch GitHub stars
+        id: fetch-stars
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make fetch_github_stars
+
+      - name: Save star data cache
+        if: steps.fetch-stars.outcome == 'success'
+        uses: actions/cache/save@v4
+        with:
+          path: website/data/github_stars.json
+          key: github-stars-${{ github.run_id }}
+
+      - name: Verify star data exists
+        run: |
+          if [ ! -f website/data/github_stars.json ]; then
+            echo "::error::github_stars.json not found. No cache and fetch failed or was skipped."
+            exit 1
+          fi
+          echo "Star data found: $(wc -l < website/data/github_stars.json) lines"
 
       - name: Build site
-        run: uv run python website/build.py
+        run: make build
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v4

.gitignore

@@ -7,6 +7,7 @@
 
 # website
 website/output/
+website/data/
 
 # claude code
 .claude/skills/

Makefile

@@ -4,7 +4,7 @@ export
 install:
 	uv sync
 
-fetch_stats:
+fetch_github_stars:
 	uv run python website/fetch_github_stars.py
 
 test:

website/fetch_github_stars.py

@@ -12,7 +12,7 @@ import httpx
 
 from build import extract_github_repo, load_stars
 
-CACHE_MAX_AGE_DAYS = 7
+CACHE_MAX_AGE_HOURS = 12
 DATA_DIR = Path(__file__).parent / "data"
 CACHE_FILE = DATA_DIR / "github_stars.json"
 README_PATH = Path(__file__).parent.parent / "README.md"
@@ -120,8 +120,8 @@ def main() -> None:
         entry = cache.get(repo)
         if entry and "fetched_at" in entry:
             fetched = datetime.fromisoformat(entry["fetched_at"])
-            age_days = (now - fetched).days
-            if age_days < CACHE_MAX_AGE_DAYS:
+            age_hours = (now - fetched).total_seconds() / 3600
+            if age_hours < CACHE_MAX_AGE_HOURS:
                 continue
         to_fetch.append(repo)
 

website/tests/test_fetch_github_stars.py

@@ -137,3 +137,95 @@ class TestParseGraphqlResponse:
         assert len(result) == 2
         assert result["a/x"]["stars"] == 100
         assert result["b/y"]["stars"] == 200
+
+
+class TestMainSkipsFreshCache:
+    """Verify that main() skips fetching when all cache entries are fresh."""
+
+    def test_skips_fetch_when_cache_is_fresh(self, tmp_path, monkeypatch, capsys):
+        from datetime import datetime, timedelta, timezone
+
+        from fetch_github_stars import main
+
+        # Set up a minimal README with one repo
+        readme = tmp_path / "README.md"
+        readme.write_text("* [req](https://github.com/psf/requests) - HTTP.\n")
+        monkeypatch.setattr("fetch_github_stars.README_PATH", readme)
+
+        # Pre-populate cache with a fresh entry (1 hour ago)
+        data_dir = tmp_path / "data"
+        data_dir.mkdir()
+        cache_file = data_dir / "github_stars.json"
+        now = datetime.now(timezone.utc)
+        fresh_cache = {
+            "psf/requests": {
+                "stars": 52000,
+                "owner": "psf",
+                "last_commit_at": "2025-01-01T00:00:00+00:00",
+                "fetched_at": (now - timedelta(hours=1)).isoformat(),
+            }
+        }
+        cache_file.write_text(json.dumps(fresh_cache), encoding="utf-8")
+        monkeypatch.setattr("fetch_github_stars.CACHE_FILE", cache_file)
+        monkeypatch.setattr("fetch_github_stars.DATA_DIR", data_dir)
+        monkeypatch.setenv("GITHUB_TOKEN", "fake-token")
+
+        main()
+
+        output = capsys.readouterr().out
+        assert "0 repos to fetch" in output
+        assert "Cache is up to date" in output
+
+    def test_fetches_when_cache_is_stale(self, tmp_path, monkeypatch, capsys):
+        from datetime import datetime, timedelta, timezone
+        from unittest.mock import MagicMock
+
+        from fetch_github_stars import main
+
+        # Set up a minimal README with one repo
+        readme = tmp_path / "README.md"
+        readme.write_text("* [req](https://github.com/psf/requests) - HTTP.\n")
+        monkeypatch.setattr("fetch_github_stars.README_PATH", readme)
+
+        # Pre-populate cache with a stale entry (24 hours ago)
+        data_dir = tmp_path / "data"
+        data_dir.mkdir()
+        cache_file = data_dir / "github_stars.json"
+        now = datetime.now(timezone.utc)
+        stale_cache = {
+            "psf/requests": {
+                "stars": 52000,
+                "owner": "psf",
+                "last_commit_at": "2025-01-01T00:00:00+00:00",
+                "fetched_at": (now - timedelta(hours=24)).isoformat(),
+            }
+        }
+        cache_file.write_text(json.dumps(stale_cache), encoding="utf-8")
+        monkeypatch.setattr("fetch_github_stars.CACHE_FILE", cache_file)
+        monkeypatch.setattr("fetch_github_stars.DATA_DIR", data_dir)
+        monkeypatch.setenv("GITHUB_TOKEN", "fake-token")
+
+        # Mock httpx.Client to avoid real API calls
+        mock_response = MagicMock()
+        mock_response.json.return_value = {
+            "data": {
+                "repo_0": {
+                    "stargazerCount": 53000,
+                    "owner": {"login": "psf"},
+                    "defaultBranchRef": {"target": {"committedDate": "2025-06-01T00:00:00Z"}},
+                }
+            }
+        }
+        mock_response.raise_for_status = MagicMock()
+        mock_client = MagicMock()
+        mock_client.__enter__ = MagicMock(return_value=mock_client)
+        mock_client.__exit__ = MagicMock(return_value=False)
+        mock_client.post.return_value = mock_response
+        monkeypatch.setattr("fetch_github_stars.httpx.Client", lambda **kwargs: mock_client)
+
+        main()
+
+        output = capsys.readouterr().out
+        assert "1 repos to fetch" in output
+        assert "Done. Fetched 1 repos" in output
+        mock_client.post.assert_called_once()