From 7e4dbb6c80d97fb714f55730fe7c0aa9f1dfd450 Mon Sep 17 00:00:00 2001 From: Samuel Sadok Date: Thu, 3 Feb 2022 18:48:25 +0100 Subject: [PATCH] implement autopublish for new sphinx docs --- .github/actions/upload-release/action.yml | 128 ++++++++++++++++++ .../upload-release/private_release_api.py | 68 ++++++++++ .github/workflows/documentation.yml | 67 +++------ docs/reStructuredText/getting-started.rst | 2 +- 4 files changed, 213 insertions(+), 52 deletions(-) create mode 100644 .github/actions/upload-release/action.yml create mode 100644 .github/actions/upload-release/private_release_api.py diff --git a/.github/actions/upload-release/action.yml b/.github/actions/upload-release/action.yml new file mode 100644 index 00000000..b37c68d9 --- /dev/null +++ b/.github/actions/upload-release/action.yml @@ -0,0 +1,128 @@ +name: 'Upload (Pre)release' +description: | + Pushes the specified local directory to our release file server and registers + the new content on our release index server. + Whether the release will be public depends on the channel and its settings. + +inputs: + release_type: + description: 'Release type (firmware, gui, docs, internal-docs).' # TODO: Currently only firmware is supported. + required: true + src_dir: + description: 'The source directory on the local system.' + required: true + do_access_key: + description: 'DigitalOcean access key' + required: true + do_secret_key: + description: 'DigitalOcean secret key' + required: true + odrive_api_key: + description: 'Key to our release index server' + required: true + product: + description: 'ODrive product name (for firmware releases only).' + required: false + app: + description: 'Firmware app name (default, bootloader) (for firmware releases only).' + required: false + variant: + description: 'Variant (for docs releases only).' + required: false + +runs: + using: "composite" + steps: + - name: Install Prerequisites + shell: bash + run: pip install aiohttp cryptography + + - name: Install odrivetool + shell: bash + run: pip install odrive --pre + + - name: Load Content Key + id: load-content-key + shell: python + run: | + import asyncio + import sys + + import aiohttp + + sys.path.insert(0, '${{ github.workspace }}/.github/actions/upload-release') + from odrive.api_client import ApiClient + from private_release_api import PrivateReleaseApi # well not so private anymore + from odrive.crypto import safe_b64encode + + content_key = PrivateReleaseApi.get_content_key('${{ inputs.src_dir }}', "${{ github.sha }}") + + async def main(): + async with aiohttp.ClientSession() as session: + api_client = ApiClient(session, key='${{ inputs.odrive_api_key }}') + release_api = PrivateReleaseApi(api_client) + manifest = await release_api.get_manifest('${{ inputs.release_type }}', content_key) + needs_upload = manifest is None + + print("::set-output name=content-key::" + safe_b64encode(content_key)) + print("::set-output name=needs-upload::" + ('true' if needs_upload else 'false')) + + asyncio.run(main()) + + # This is for debugging only + - name: Dump context + shell: bash + env: + CONTEXT: ${{ toJson(steps) }} + run: | + echo "$CONTEXT" + + - name: Upload to DigitalOcean + if: steps.load-content-key.outputs.needs-upload == 'true' + uses: BetaHuhn/do-spaces-action@v2 + with: + access_key: ${{ inputs.do_access_key }} + secret_key: ${{ inputs.do_secret_key }} + space_name: odrive-cdn + space_region: nyc3 + source: ${{ inputs.src_dir }} + out_dir: releases/${{ inputs.release_type }}/${{ steps.load-content-key.outputs.content-key }} + + - name: Register on release server + shell: python + run: | + import asyncio + import sys + + import aiohttp + + sys.path.insert(0, '${{ github.workspace }}/.github/actions/upload-release') + from odrive.api_client import ApiClient + from private_release_api import PrivateReleaseApi + from odrive.crypto import safe_b64decode + + channel="0.5.4" + + print("Channel: ", channel) + print("Commit hash: ", '${{ github.sha }}') + print("Content key: ", '${{ steps.load-content-key.outputs.content-key }}') + + async def main(): + async with aiohttp.ClientSession() as session: + api_client = ApiClient(session, key='${{ inputs.odrive_api_key }}') + release_api = PrivateReleaseApi(api_client) + + qualifiers = {} + if '${{ inputs.product }}': + qualifiers['product'] = '${{ inputs.product }}' + if '${{ inputs.app }}': + qualifiers['app'] = '${{ inputs.app }}' + if '${{ inputs.variant }}': + qualifiers['variant'] = '${{ inputs.variant }}' + + content_key = safe_b64decode('${{ steps.load-content-key.outputs.content-key }}') + await release_api.register_content('${{ inputs.release_type }}', '${{ github.sha }}', content_key, **qualifiers) + await release_api.register_commit('${{ inputs.release_type }}', channel, '${{ github.sha }}') + await release_api.refresh_routes('${{ inputs.release_type }}') + + asyncio.run(main()) diff --git a/.github/actions/upload-release/private_release_api.py b/.github/actions/upload-release/private_release_api.py new file mode 100644 index 00000000..c75f61ea --- /dev/null +++ b/.github/actions/upload-release/private_release_api.py @@ -0,0 +1,68 @@ + +import hashlib +import os + +from odrive.api_client import ApiClient +from odrive.crypto import b64encode + +class PrivateReleaseApi(): + BASE_URL = '/releases' + + @staticmethod + def get_content_key(path: str, commit_hash: str): + commit_hash_bytes = bytes.fromhex(commit_hash) + + def _get_file_names(path: str, prefix: list): + with os.scandir(os.path.join(path, *prefix)) as it: + for entry in it: + if entry.is_file(): + yield os.path.join(*prefix, entry.name) + else: + yield from _get_file_names(path, prefix + [entry.name]) + filenames = sorted(_get_file_names(path, [])) + + dir_hasher = hashlib.sha256() + + for filename in filenames: + with open(os.path.join(path, filename), 'rb') as fp: + content = fp.read() + + # Calculate commit-invariant hash of the file content + # This means that if a new compile differs only by embedded commit hash, + # it is considered equal. + patched_content = content.replace(commit_hash_bytes, bytes(len(commit_hash_bytes))) + file_hasher = hashlib.sha256() + file_hasher.update(patched_content) + + dir_hasher.update(filename.encode('utf-8')) + dir_hasher.update(file_hasher.digest()) + + return dir_hasher.digest() + + def __init__(self, api_client: 'ApiClient'): + self._api_client = api_client + + async def get_manifest(self, release_type: str, content_key: bytes): + outputs = await self._api_client.call('GET', PrivateReleaseApi.BASE_URL + '/' + release_type + '/manifest', inputs={ + 'content_key': b64encode(content_key), + }) + return outputs + + async def register_content(self, release_type: str, commit_hash: str, content_key: bytes, **qualifiers): + args = { + 'commit_hash': commit_hash, + 'content_key': b64encode(content_key), + **qualifiers + } + outputs = await self._api_client.call('PUT', PrivateReleaseApi.BASE_URL + '/' + release_type + '/content', inputs=args) + return outputs['created'] + + async def register_commit(self, release_type: str, channel: str, commit_hash: str): + outputs = await self._api_client.call('PUT', PrivateReleaseApi.BASE_URL + '/' + release_type + '/commit', inputs={ + 'commit_hash': commit_hash, + 'channel': channel + }) + return outputs['published'] + + async def refresh_routes(self, release_type: str): + await self._api_client.call('PUT', PrivateReleaseApi.BASE_URL + '/' + release_type + '/refresh-routes') diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 33ea3f64..3e979f99 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -3,72 +3,37 @@ name: Build and publish HTML documentation website on: push: branches: [ master ] + paths: ['docs/**', '.github/**'] jobs: - jekyll: + make-html: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - # Use GitHub Actions' cache for ruby and python packages to shorten build times and decrease load on servers - - name: Cache gems - uses: actions/cache@v2 - with: - path: docs/vendor/bundle - key: ${{ runner.os }}-gems-${{ hashFiles('docs/Gemfile.lock') }} - restore-keys: | - ${{ runner.os }}-gems- - name: Cache pip uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-PyYAML-Jinja2-jsonschema + key: ${{ runner.os }}-pip-sphinx-sphinx-tabs-sphinx-design-sphinx_copybutton-sphinx_panels-sphinx_rtd_theme restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - name: Install Python dependencies - run: pip install PyYAML Jinja2 jsonschema - - # Autogenerate the API reference .md files in the python in the python/python3 container - - name: Autogenerate the API reference .md files in the python container - run: | - mkdir -p docs/_api docs/_includes - python Firmware/interface_generator_stub.py --definitions Firmware/odrive-interface.yaml --template docs/_layouts/api_documentation_template.j2 --outputs docs/_api/#.md - python Firmware/interface_generator_stub.py --definitions Firmware/odrive-interface.yaml --template docs/_layouts/api_index_template.j2 --output docs/_includes/apiindex.html + run: pip install sphinx sphinx-tabs sphinx-design sphinx_copybutton sphinx_panels sphinx_rtd_theme - - name: Build the site in the jekyll/builder container + - name: Build HTML docs run: | - docker run \ - -v ${{ github.workspace }}:/srv/jekyll -e PAGES_REPO_NWO=${GITHUB_REPOSITORY} \ - ruby:2.7-buster /bin/sh -c " - chmod 777 /srv/jekyll/docs && \ - cd /srv/jekyll/docs && \ - bundle config path vendor/bundle && \ - bundle install && \ - bundle exec jekyll build --baseurl \"\" - cd .. - mv docs/_site _site - rm -rdf docs - mv _site docs - touch docs/.nojekyll - " - # Extra checks to reduce likelihood of defect build - test -f docs/CNAME - test -f docs/index.html + cd docs/reStructuredText + make html - - name: Push to documentation branch - run: | - git config user.name "${GITHUB_ACTOR}" - git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" - git add -f docs - git commit -m "jekyll build from Action ${GITHUB_SHA}" - git push --force origin HEAD:${REMOTE_BRANCH} - env: - REMOTE_BRANCH: gh-pages + - name: Upload docs + uses: ./.github/actions/upload-release + with: + release_type: docs + src_dir: docs/reStructuredText/_build/html + do_access_key: ${{ secrets.DIGITALOCEAN_ACCESS_KEY }} + do_secret_key: ${{ secrets.DIGITALOCEAN_SECRET_KEY }} + odrive_api_key: ${{ secrets.ODRIVE_API_KEY }} + variant: public diff --git a/docs/reStructuredText/getting-started.rst b/docs/reStructuredText/getting-started.rst index 094a7763..94012ef9 100644 --- a/docs/reStructuredText/getting-started.rst +++ b/docs/reStructuredText/getting-started.rst @@ -98,7 +98,7 @@ Most instructions in this guide refer to a utility called `odrivetool`, so you s pip install --upgrade odrive - .. tab:: OSX + .. tab:: macOS We are going to run the following commands for installation in Terminal.