implement autopublish for new sphinx docs

.github/actions/upload-release/action.yml

@@ -0,0 +1,128 @@
+name: 'Upload (Pre)release'
+description: |
+  Pushes the specified local directory to our release file server and registers
+  the new content on our release index server.
+  Whether the release will be public depends on the channel and its settings.
+
+inputs:
+  release_type:
+    description: 'Release type (firmware, gui, docs, internal-docs).' # TODO: Currently only firmware is supported.
+    required: true
+  src_dir:
+    description: 'The source directory on the local system.'
+    required: true
+  do_access_key:
+    description: 'DigitalOcean access key'
+    required: true
+  do_secret_key:
+    description: 'DigitalOcean secret key'
+    required: true
+  odrive_api_key:
+    description: 'Key to our release index server'
+    required: true
+  product:
+    description: 'ODrive product name (for firmware releases only).'
+    required: false
+  app:
+    description: 'Firmware app name (default, bootloader) (for firmware releases only).'
+    required: false
+  variant:
+    description: 'Variant (for docs releases only).'
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Prerequisites
+      shell: bash
+      run: pip install aiohttp cryptography
+
+    - name: Install odrivetool
+      shell: bash
+      run: pip install odrive --pre
+
+    - name: Load Content Key
+      id: load-content-key
+      shell: python
+      run: |
+        import asyncio
+        import sys
+        
+        import aiohttp
+
+        sys.path.insert(0, '${{ github.workspace }}/.github/actions/upload-release')
+        from odrive.api_client import ApiClient
+        from private_release_api import PrivateReleaseApi # well not so private anymore
+        from odrive.crypto import safe_b64encode
+
+        content_key = PrivateReleaseApi.get_content_key('${{ inputs.src_dir }}', "${{ github.sha }}")
+
+        async def main():
+          async with aiohttp.ClientSession() as session:
+            api_client = ApiClient(session, key='${{ inputs.odrive_api_key }}')
+            release_api = PrivateReleaseApi(api_client)
+            manifest = await release_api.get_manifest('${{ inputs.release_type }}', content_key)
+            needs_upload = manifest is None
+
+          print("::set-output name=content-key::" + safe_b64encode(content_key))
+          print("::set-output name=needs-upload::" + ('true' if needs_upload else 'false'))
+
+        asyncio.run(main())
+
+      # This is for debugging only
+    - name: Dump context
+      shell: bash
+      env:
+        CONTEXT: ${{ toJson(steps) }}
+      run: |
+        echo "$CONTEXT"
+
+    - name: Upload to DigitalOcean
+      if: steps.load-content-key.outputs.needs-upload == 'true'
+      uses: BetaHuhn/do-spaces-action@v2
+      with:
+        access_key: ${{ inputs.do_access_key }}
+        secret_key: ${{ inputs.do_secret_key }}
+        space_name: odrive-cdn
+        space_region: nyc3
+        source: ${{ inputs.src_dir }}
+        out_dir: releases/${{ inputs.release_type }}/${{ steps.load-content-key.outputs.content-key }}
+
+    - name: Register on release server
+      shell: python
+      run: |
+        import asyncio
+        import sys
+        
+        import aiohttp
+
+        sys.path.insert(0, '${{ github.workspace }}/.github/actions/upload-release')
+        from odrive.api_client import ApiClient
+        from private_release_api import PrivateReleaseApi
+        from odrive.crypto import safe_b64decode
+        
+        channel="0.5.4"
+
+        print("Channel:  ", channel)
+        print("Commit hash: ", '${{ github.sha }}')
+        print("Content key: ", '${{ steps.load-content-key.outputs.content-key }}')
+
+        async def main():
+          async with aiohttp.ClientSession() as session:
+            api_client = ApiClient(session, key='${{ inputs.odrive_api_key }}')
+            release_api = PrivateReleaseApi(api_client)
+
+            qualifiers = {}
+            if '${{ inputs.product }}':
+              qualifiers['product'] = '${{ inputs.product }}'
+            if '${{ inputs.app }}':
+              qualifiers['app'] = '${{ inputs.app }}'
+            if '${{ inputs.variant }}':
+              qualifiers['variant'] = '${{ inputs.variant }}'
+            
+            content_key = safe_b64decode('${{ steps.load-content-key.outputs.content-key }}')
+            await release_api.register_content('${{ inputs.release_type }}', '${{ github.sha }}', content_key, **qualifiers)
+            await release_api.register_commit('${{ inputs.release_type }}', channel, '${{ github.sha }}')
+            await release_api.refresh_routes('${{ inputs.release_type }}')
+
+        asyncio.run(main())

.github/actions/upload-release/private_release_api.py

@@ -0,0 +1,68 @@
+
+import hashlib
+import os
+
+from odrive.api_client import ApiClient
+from odrive.crypto import b64encode
+
+class PrivateReleaseApi():
+    BASE_URL = '/releases'
+
+    @staticmethod
+    def get_content_key(path: str, commit_hash: str):
+        commit_hash_bytes = bytes.fromhex(commit_hash)
+
+        def _get_file_names(path: str, prefix: list):
+            with os.scandir(os.path.join(path, *prefix)) as it:
+                for entry in it:
+                    if entry.is_file():
+                        yield os.path.join(*prefix, entry.name)
+                    else:
+                        yield from _get_file_names(path, prefix + [entry.name])
+        filenames = sorted(_get_file_names(path, []))
+
+        dir_hasher = hashlib.sha256()
+        
+        for filename in filenames:
+            with open(os.path.join(path, filename), 'rb') as fp:
+                content = fp.read()
+
+            # Calculate commit-invariant hash of the file content
+            # This means that if a new compile differs only by embedded commit hash,
+            # it is considered equal.
+            patched_content = content.replace(commit_hash_bytes, bytes(len(commit_hash_bytes)))
+            file_hasher = hashlib.sha256()
+            file_hasher.update(patched_content)
+
+            dir_hasher.update(filename.encode('utf-8'))
+            dir_hasher.update(file_hasher.digest())
+
+        return dir_hasher.digest()
+
+    def __init__(self, api_client: 'ApiClient'):
+        self._api_client = api_client
+
+    async def get_manifest(self, release_type: str, content_key: bytes):
+        outputs = await self._api_client.call('GET', PrivateReleaseApi.BASE_URL + '/' + release_type + '/manifest', inputs={
+            'content_key': b64encode(content_key),
+        })
+        return outputs
+
+    async def register_content(self, release_type: str, commit_hash: str, content_key: bytes, **qualifiers):
+        args = {
+            'commit_hash': commit_hash,
+            'content_key': b64encode(content_key),
+            **qualifiers
+        }
+        outputs = await self._api_client.call('PUT', PrivateReleaseApi.BASE_URL + '/' + release_type + '/content', inputs=args)
+        return outputs['created']
+
+    async def register_commit(self, release_type: str, channel: str, commit_hash: str):
+        outputs = await self._api_client.call('PUT', PrivateReleaseApi.BASE_URL + '/' + release_type + '/commit', inputs={
+            'commit_hash': commit_hash,
+            'channel': channel
+        })
+        return outputs['published']
+
+    async def refresh_routes(self, release_type: str):
+        await self._api_client.call('PUT', PrivateReleaseApi.BASE_URL + '/' + release_type + '/refresh-routes')

.github/workflows/documentation.yml

@@ -3,72 +3,37 @@ name: Build and publish HTML documentation website
 on:
   push:
     branches: [ master ]
+    paths: ['docs/**', '.github/**']
 
 jobs:
-  jekyll:
+  make-html:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-
-    # Use GitHub Actions' cache for ruby and python packages to shorten build times and decrease load on servers
-    - name: Cache gems
-      uses: actions/cache@v2
-      with:
-        path: docs/vendor/bundle
-        key: ${{ runner.os }}-gems-${{ hashFiles('docs/Gemfile.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-gems-
     
     - name: Cache pip
       uses: actions/cache@v2
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-PyYAML-Jinja2-jsonschema
+        key: ${{ runner.os }}-pip-sphinx-sphinx-tabs-sphinx-design-sphinx_copybutton-sphinx_panels-sphinx_rtd_theme
         restore-keys: |
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
 
     - name: Install Python dependencies
-      run: pip install PyYAML Jinja2 jsonschema
-
-    # Autogenerate the API reference .md files in the python in the python/python3 container
-    - name: Autogenerate the API reference .md files in the python container
-      run: |
-        mkdir -p docs/_api docs/_includes
-        python Firmware/interface_generator_stub.py --definitions Firmware/odrive-interface.yaml --template docs/_layouts/api_documentation_template.j2 --outputs docs/_api/#.md
-        python Firmware/interface_generator_stub.py --definitions Firmware/odrive-interface.yaml --template docs/_layouts/api_index_template.j2 --output docs/_includes/apiindex.html
+      run: pip install sphinx sphinx-tabs sphinx-design sphinx_copybutton sphinx_panels sphinx_rtd_theme
     
-    - name: Build the site in the jekyll/builder container
+    - name: Build HTML docs
       run: |
-        docker run \
-        -v ${{ github.workspace }}:/srv/jekyll -e PAGES_REPO_NWO=${GITHUB_REPOSITORY} \
-        ruby:2.7-buster /bin/sh -c "
-          chmod 777 /srv/jekyll/docs && \
-          cd /srv/jekyll/docs && \
-          bundle config path vendor/bundle && \
-          bundle install && \
-          bundle exec jekyll build --baseurl \"\"
-          cd ..
-          mv docs/_site _site
-          rm -rdf docs
-          mv _site docs
-          touch docs/.nojekyll
-        "
-        # Extra checks to reduce likelihood of defect build
-        test -f docs/CNAME
-        test -f docs/index.html
+        cd docs/reStructuredText
+        make html
 
-    - name: Push to documentation branch
-      run: |
-        git config user.name "${GITHUB_ACTOR}"
-        git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
-        git add -f docs
-        git commit -m "jekyll build from Action ${GITHUB_SHA}"
-        git push --force origin HEAD:${REMOTE_BRANCH}
-      env:
-        REMOTE_BRANCH: gh-pages
+    - name: Upload docs
+      uses: ./.github/actions/upload-release
+      with:
+        release_type: docs
+        src_dir: docs/reStructuredText/_build/html
+        do_access_key: ${{ secrets.DIGITALOCEAN_ACCESS_KEY }}
+        do_secret_key: ${{ secrets.DIGITALOCEAN_SECRET_KEY }}
+        odrive_api_key: ${{ secrets.ODRIVE_API_KEY }}
+        variant: public

docs/reStructuredText/getting-started.rst

@@ -98,7 +98,7 @@ Most instructions in this guide refer to a utility called `odrivetool`, so you s
                 
                 pip install --upgrade odrive
 
-    .. tab:: OSX
+    .. tab:: macOS
 
         We are going to run the following commands for installation in Terminal.