Merge remote-tracking branch 'origin/devel' into sam_winusb

Firmware/CHANGELOG.md

@@ -11,7 +11,10 @@ Please add a note of your changes below this heading if you make a Pull Request.
  * System stats (e.g. stack usage) are exposed under `<odrv>.system_stats`
 
 ### Changed
-* The DFU script now verifies the flash after writing
+* DFU script updates
+  * Verify the flash after writing
+  * Automatically download firmware from GitHub releases if no file is provided
+  * Retain configuration during firmware updates
 * Refactor python tools
   * The scripts `explore_odrive.py`, `liveplotter.py`, `drv_status.py` and `rate_test.py` have been merged into one single `odrivetool` script. Running this script without any arguments provides the shell that `explore_odrive.py` used to provide.
   * The command line options of `odrivetool` have changed compared to the original `explore_odrive.py`. See `odrivetool --help` for more details.
@@ -19,6 +22,7 @@ Please add a note of your changes below this heading if you make a Pull Request.
   * No need to restart the `odrivetool` shell when devices get disconnected and reconnected
   * ODrive accesses from within python tools are now thread-safe. That means you can read from the same remote property from multiple threads concurrently.
   * The liveplotter (`odrivetool liveplotter`, formerly `liveplotter.py`) does no longer steal focus and closes as expected
+  * Add commands `odrivetool backup-config` and `odrivetool restore-config`
   * (experimental: start liveplotter from `odrivetool` shell by typing `start_liveplotter(lambda: odrv0.motor0.encoder.encoder_state)`)
 * `make write_otp` command to burn the board version onto the ODrive's one-time programmable memory. If you have an ODrive v3.4 or older, you can run this once for a better firmware update user experience in the future. Run the command without any options for more details. Once set, the board version is exposed through the `hw_version_[...]` properties.
 * bake Git-derived firmware version into firmware binary. The firmware version is exposed through the `fw_version_[...]` properties.

Firmware/MotorControl/main.cpp

@@ -33,6 +33,8 @@ void save_configuration(void) {
             &motor_configs,
             &axis_configs)) {
         //printf("saving configuration failed\r\n"); osDelay(5);
+    } else {
+        user_config_loaded_ = true;
     }
 }
 

tools/odrive/configuration.py

@@ -0,0 +1,82 @@
+
+import json
+import os
+import tempfile
+import odrive.remote_object
+from odrive.utils import OperationAbortedException
+
+def get_dict(obj, is_config_object):
+    result = {}
+    for (k,v) in obj._remote_attributes.items():
+        if isinstance(v, odrive.remote_object.RemoteProperty) and is_config_object:
+            result[k] = v.get_value()
+        elif isinstance(v, odrive.remote_object.RemoteObject):
+            sub_dict = get_dict(v, k == 'config')
+            if sub_dict != {}:
+                result[k] = sub_dict
+    return result
+
+def set_dict(obj, path, config_dict):
+    errors = []
+    for (k,v) in config_dict.items():
+        name = path + ("." if path != "" else "") + k
+        if not k in obj._remote_attributes:
+            errors.append("Could not restore {}: property not found on device".format(name))
+            continue
+        remote_attribute = obj._remote_attributes[k]
+        if isinstance(remote_attribute, odrive.remote_object.RemoteObject):
+            errors += set_dict(remote_attribute, name, v)
+        else:
+            try:
+                remote_attribute.set_value(v)
+            except Exception as ex:
+                errors.append("Could not restore {}: {}".format(name, str(ex)))
+    return errors
+
+def get_temp_config_filename(device):
+    serial_number = odrive.utils.get_serial_number_str(device)
+    safe_serial_number = ''.join(filter(str.isalnum, serial_number))
+    return os.path.join(tempfile.gettempdir(), 'odrive-config-{}.json'.format(safe_serial_number))
+
+def backup_config(device, filename, logger):
+    """
+    Exports the configuration of an ODrive to a JSON file.
+    If no file name is provided, the file is placed into a
+    temporary directory.
+    """
+
+    if filename is None:
+        filename = get_temp_config_filename(device)
+
+    logger.info("Saving configuration to {}...".format(filename))
+
+    if os.path.exists(filename):
+        if not odrive.utils.yes_no_prompt("The file {} already exists. Do you want to override it?".format(filename), True):
+            raise OperationAbortedException()
+
+    data = get_dict(device, False)
+    with open(filename, 'w') as file:
+        json.dump(data, file)
+    logger.info("Configuration saved.")
+
+def restore_config(device, filename, logger):
+    """
+    Restores the configuration stored in a file 
+    """
+
+    if filename is None:
+        filename = get_temp_config_filename(device)
+
+    with open(filename) as file:
+        data = json.load(file)
+
+    logger.info("Restoring configuration from {}...".format(filename))
+    errors = odrive.configuration.set_dict(device, "", data)
+
+    for error in errors:
+        logger.info(error)
+    if errors:
+        logger.warn("Some of the configuration could not be restored.")
+    
+    device.save_configuration()
+    logger.info("Configuration restored.")

tools/odrive/dfuse/DfuDevice.py

@@ -1,5 +1,8 @@
 import usb.util
 import time
+import fractions
+import array
+from odrive.dfuse.DfuState import DfuState
 
 DFU_REQUEST_SEND = 0x21
 DFU_REQUEST_RECEIVE = 0xa1
@@ -12,6 +15,9 @@ DFU_CLRSTATUS = 0x04
 DFU_GETSTATE  = 0x05
 DFU_ABORT     = 0x06
 
+SIZE_MULTIPLIERS = {' ': 1, 'K': 1024, 'M' : 1024*1024}
+MAX_TRANSFER_SIZE = 2048
+
 # Order is LSB first
 def address_to_4bytes(a):
     return [ a % 256, (a >> 8)%256, (a >> 16)%256, (a >> 24)%256 ]
@@ -24,6 +30,7 @@ class DfuDevice:
         self.intf = None
         #self.dev.reset()
         self.cfg.set()
+        self.sectors = list(self.get_device_sectors())
 
     def alternates(self):
         return [(usb.util.get_string(self.dev, intf.iInterface), intf) for intf in self.cfg]
@@ -98,3 +105,115 @@ class DfuDevice:
         
         return status
 
+    ## High level functions ##
+    # by ODrive Robotics
+
+    def get_device_sectors(self):
+        """
+        Returns a list of all sectors on the device.
+        Each sector is represented as a dictionary with the following keys:
+        - name: name of the associated memory region (e.g. "Internal Flash")
+        - alt: USB alternate setting associated with this memory region
+        - addr: Start address of the sector (e.g. 0x08004000 for the second flash sectors)
+        - baseaddr: Start address of the memory region associated with the sector
+                    (e.g. 0x08000000 for all flash sectors)
+        - len: Number of bytes in the sector
+        """
+        for name, alt in self.alternates():
+            # example for name:
+            # '@Internal Flash  /0x08000000/04*016Kg,01*064Kg,07*128Kg'
+            label, baseaddr, layout = name.split('/')
+            baseaddr = int(baseaddr, 0) # convert hex to decimal
+            addr = baseaddr
+
+            for sector in layout.split(','):
+                repeat, size = map(int, sector[:-2].split('*'))
+                size *= SIZE_MULTIPLIERS[sector[-2].upper()]
+                mode = sector[-1]
+
+                while repeat > 0:
+                    # TODO: verify if the section is writable
+                    yield {
+                        'name': label.strip().strip('@'),
+                        'alt': alt,
+                        'baseaddr': baseaddr,
+                        'addr': addr,
+                        'len': size,
+                        'mode': mode
+                    }
+
+                    addr += size
+                    repeat -= 1
+
+    def set_alternate_safe(self, alt):
+        self.set_alternate(alt)
+        if self.get_state() == DfuState.DFU_ERROR:
+            self.clear_status()
+            self.wait_while_state(DfuState.DFU_ERROR)
+
+    #def clear_error(self)
+    def set_address_safe(self, addr):
+        self.set_address(addr)
+        status = self.wait_while_state(DfuState.DFU_DOWNLOAD_BUSY)
+        if status[1] != DfuState.DFU_DOWNLOAD_IDLE:
+            raise RuntimeError("An error occured. Device Status: %r" % status)
+        # take device out of DFU_DOWNLOAD_SYNC and into DFU_IDLE
+        self.abort()
+        status = self.wait_while_state(DfuState.DFU_DOWNLOAD_SYNC)
+        if status[1] != DfuState.DFU_IDLE:
+            raise RuntimeError("An error occured. Device Status: %r" % status)
+        
+
+    def erase_sector(self, sector):
+        self.set_alternate_safe(sector['alt'])
+        self.erase(sector['addr'])
+        status = self.wait_while_state(DfuState.DFU_DOWNLOAD_BUSY, timeout=sector['len']/32)
+        if status[1] != DfuState.DFU_DOWNLOAD_IDLE:
+            raise RuntimeError("An error occured. Device Status: %r" % status)
+
+    def write_sector(self, sector, data):
+        self.set_alternate_safe(sector['alt'])
+        self.set_address_safe(sector['addr'])
+
+        transfer_size = fractions.gcd(sector['len'], MAX_TRANSFER_SIZE)
+        
+        blocks = [data[i:i + transfer_size] for i in range(0, len(data), transfer_size)]
+        for blocknum, block in enumerate(blocks):
+            #print('write to {:08X} ({} bytes)'.format(
+            #        sector['addr'] + blocknum * TRANSFER_SIZE, len(block)))
+            self.write(blocknum, block)
+            status = self.wait_while_state(DfuState.DFU_DOWNLOAD_BUSY)
+            if status[1] != DfuState.DFU_DOWNLOAD_IDLE:
+                raise RuntimeError("An error occured. Device Status: %r" % status)
+
+    def read_sector(self, sector):
+        """
+        Reads data from the specified sector
+        Returns: a byte array containing the data
+        """
+        self.set_alternate_safe(sector['alt'])
+        self.set_address_safe(sector['addr'])
+
+        transfer_size = fractions.gcd(sector['len'], MAX_TRANSFER_SIZE)
+        #blocknum_offset = int((sector['addr'] - sector['baseaddr']) / transfer_size)
+
+        
+        data = array.array(u'B')
+        for blocknum in range(int(sector['len'] / transfer_size)):
+            #print('read at {:08X}'.format(sector['addr'] + blocknum * TRANSFER_SIZE))
+            deviceBlock = self.read(blocknum, transfer_size)
+            data.extend(deviceBlock)
+        self.abort() # take device into DFU_IDLE
+        return data
+
+    def jump_to_application(self, address):
+        self.set_address_safe(address)
+        #self.set_address(address)
+        #status = self.wait_while_state(DfuState.DFU_DOWNLOAD_BUSY)
+        #if status[1] != DfuState.DFU_DOWNLOAD_IDLE:
+        #    raise RuntimeError("An error occured. Device Status: {}".format(status[1]))
+
+        self.leave()
+        status = self.wait_while_state(DfuState.DFU_MANIFEST_SYNC)
+        if status[1] != DfuState.DFU_MANIFEST:
+            raise RuntimeError("An error occured. Device Status: {}".format(status[1]))

tools/odrive/discovery.py

@@ -62,7 +62,7 @@ def find_all(path, serial_number,
                 return
             json_data = {"name": "odrive", "members": json_data}
             obj = odrive.remote_object.RemoteObject(json_data, None, channel, printer)
-            device_serial_number = format(obj.serial_number, 'x').upper() if hasattr(obj, 'serial_number') else "[unknown serial number]"
+            device_serial_number = odrive.utils.get_serial_number_str(obj)
             if serial_number != None and device_serial_number != serial_number:
                 printer("Ignoring device with serial number {}".format(device_serial_number))
                 return

tools/odrive/shell.py

@@ -52,7 +52,7 @@ def did_discover_device(odrive, logger, app_shutdown_token):
     # Publish new ODrive to interactive console
     interactive_variables[interactive_name] = odrive
     globals()[interactive_name] = odrive # Add to globals so tab complete works
-    logger.info("{} to ODrive {:012X} as {}".format(verb, serial_number, interactive_name))
+    logger.notify("{} to ODrive {:012X} as {}".format(verb, serial_number, interactive_name))
 
     # Subscribe to disappearance of the device
     odrive.__channel__._channel_broken.subscribe(lambda: did_lose_device(interactive_name, logger, app_shutdown_token))

tools/odrive/usbbulk_transport.py

@@ -163,15 +163,18 @@ def discover_channels(path, serial_number, callback, cancellation_token, channel
   known_devices = []
   def device_matcher(device):
     #print("  test {:04X}:{:04X}".format(device.idVendor, device.idProduct))
-    if (device.bus, device.address) in known_devices:
-      return False
-    if bus != None and device.bus != bus:
-      return False
-    if address != None and device.address != address:
-      return False
-    if serial_number != None and device.serial_number != serial_number:
-      return False
-    if (device.idVendor, device.idProduct) not in ODRIVE_VID_PID_PAIRS:
+    try:
+      if (device.bus, device.address) in known_devices:
+        return False
+      if bus != None and device.bus != bus:
+        return False
+      if address != None and device.address != address:
+        return False
+      if serial_number != None and device.serial_number != serial_number:
+        return False
+      if (device.idVendor, device.idProduct) not in ODRIVE_VID_PID_PAIRS:
+        return False
+    except:
       return False
     return True
 

tools/odrive/utils.py

@@ -25,6 +25,9 @@ data_rate = 100
 plot_rate = 10
 num_samples = 1000
 
+class OperationAbortedException(Exception):
+    pass
+
 def start_liveplotter(get_var_callback):
     """
     Starts a liveplotter.
@@ -157,6 +160,11 @@ def setup_udev_rules(logger):
     subprocess.run(["udevadm", "trigger"], check=True)
     logger.info('udev rules configured successfully')
 
+def get_serial_number_str(device):
+    if hasattr(device, 'serial_number'):
+        return format(device.serial_number, 'x').upper()
+    else:
+        return "[unknown serial number]"
 
 ## Exceptions ##
 
@@ -357,9 +365,30 @@ class Logger():
     def success(self, text):
         self.print_colored(self._prefix + text, Logger.COLOR_GREEN)
     def info(self, text):
+        self.print_colored(self._prefix + text, Logger.COLOR_DEFAULT)
+    def notify(self, text):
         self.print_colored(self._prefix + text, Logger.COLOR_CYAN)
     def warn(self, text):
         self.print_colored(self._prefix + text, Logger.COLOR_YELLOW)
     def error(self, text):
         # TODO: write to stderr
         self.print_colored(self._prefix + text, Logger.COLOR_RED)
+
+def yes_no_prompt(question, default=None):
+    if default is None:
+        question += " [y/n] "
+    elif default == True:
+        question += " [Y/n] "
+    elif default == False:
+        question += " [y/N] "
+
+    while True:
+        print(question, end='')
+
+        choice = input().lower()
+        if choice in {'yes', 'y'}:
+            return True
+        elif choice in {'no', 'n'}:
+            return False
+        elif choice == '' and default is not None:
+            return default

tools/odrive/version.py

@@ -4,6 +4,20 @@ import subprocess
 import os
 import sys
 
+def version_str_to_tuple(version_string):
+    """
+    Converts a version string to a tuple of the form
+    (major, minor, revision, prerelease)
+
+    Example: "fw-v0.3.6-23" => (0, 3, 6, True)
+    """
+    regex=r'.*v([0-9a-zA-Z]+).([0-9a-zA-Z]+).([0-9a-zA-Z]+)(.*)'
+    return (int(re.sub(regex, r"\1", version_string)),
+            int(re.sub(regex, r"\2", version_string)),
+            int(re.sub(regex, r"\3", version_string)),
+            (re.sub(regex, r"\4", version_string) != ""))
+
+
 def get_version_from_git():
     script_dir = os.path.dirname(os.path.realpath(__file__))
     try:
@@ -12,19 +26,15 @@ def get_version_from_git():
             cwd=script_dir)
         git_tag = git_tag.decode(sys.stdout.encoding).rstrip('\n')
 
-        regex=r'.*v([0-9a-zA-Z]).([0-9a-zA-Z]).([0-9a-zA-Z])(.*)'
-        package_version_major = int(re.sub(regex, r"\1", git_tag))
-        package_version_minor = int(re.sub(regex, r"\2", git_tag))
-        package_version_revision = int(re.sub(regex, r"\3", git_tag))
-        package_version_unreleased = (re.sub(regex, r"\4", git_tag) != "")
+        (major, minor, revision, is_prerelease) = version_str_to_tuple(git_tag)
 
-        if package_version_unreleased:
-            package_version_revision += 1
+        if is_prerelease:
+            revision += 1
+        return git_tag, major, minor, revision, is_prerelease
 
     except Exception as ex:
         print(ex)
         return "[unknown version]", 0, 0, 0, 1
-    return git_tag, package_version_major, package_version_minor, package_version_revision, package_version_unreleased
 
 def get_version_str(git_only=False):
     """

tools/odrivetool

@@ -7,7 +7,8 @@ from __future__ import print_function
 import sys
 import argparse
 import odrive.discovery
-from odrive.utils import Logger, Event
+from odrive.utils import Logger, Event, OperationAbortedException
+from odrive.configuration import *
 
 # Flush stdout by default
 # Source:
@@ -33,8 +34,26 @@ shell_parser.add_argument("--no-ipython", action="store_true",
                           "instead of the IPython shell, "
                           "even if IPython is installed.")
 
-dfu_parser = subparsers.add_parser('dfu', help="Upgrade the ODrive device firmware")
-dfu_parser.add_argument('file', metavar='HEX', help='The .hex file to be flashed. Make sure your firmware board version matches the actual board version.')
+dfu_parser = subparsers.add_parser('dfu', help="Upgrade the ODrive device firmware."
+                                               "If no serial number is specified, the first ODrive that is found is updated")
+dfu_parser.add_argument('file', metavar='HEX', nargs='?',
+                        help='The .hex file to be flashed. Make sure target board version '
+                        'of the firmware file matches the actual board version. '
+                        'You can download the latest release manually from '
+                        'https://github.com/madcowswe/ODrive/releases. '
+                        'If no file is provided, the script automatically downloads '
+                        'the latest firmware.')
+
+
+dfu_parser = subparsers.add_parser('backup-config', help="Saves the configuration of the ODrive to a JSON file")
+dfu_parser.add_argument('file', nargs='?',
+                        help="Path to the file where to store the data. "
+                        "If no path is provided, the configuration is stored in {}.".format(tempfile.gettempdir()))
+
+dfu_parser = subparsers.add_parser('restore-config', help="Restores the configuration of the ODrive from a JSON file")
+dfu_parser.add_argument('file', nargs='?',
+                        help="Path to the file that contains the configuration data. "
+                        "If no path is provided, the configuration is loaded from {}.".format(tempfile.gettempdir()))
 
 subparsers.add_parser('liveplotter', help="Upgrade the ODrive's Firmware")
 subparsers.add_parser('drv-status', help="Show status of the on-board DRV8301 chips (for debugging only)")
@@ -110,12 +129,14 @@ try:
     elif args.command == 'dfu':
         print_version()
         import odrive.dfu
-        odrive.dfu.launch_dfu(args, app_shutdown_token)
+        odrive.dfu.launch_dfu(args, logger, app_shutdown_token)
 
     elif args.command == 'liveplotter':
         from odrive.utils import start_liveplotter
         print("Waiting for ODrive...")
-        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number)
+        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number,
+                                              search_cancellation_token=app_shutdown_token,
+                                              channel_termination_token=app_shutdown_token)
 
         # If you want to plot different values, change them here.
         # You can plot any number of values concurrently.
@@ -125,22 +146,44 @@ try:
     elif args.command == 'drv-status':
         from odrive.utils import print_drv_regs
         print("Waiting for ODrive...")
-        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number)
+        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number,
+                                              search_cancellation_token=app_shutdown_token,
+                                              channel_termination_token=app_shutdown_token)
         print_drv_regs("Motor 0", my_odrive.axis0.motor)
         print_drv_regs("Motor 1", my_odrive.axis1.motor)
 
     elif args.command == 'rate-test':
         from odrive.utils import rate_test
         print("Waiting for ODrive...")
-        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number)
+        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number,
+                                              search_cancellation_token=app_shutdown_token,
+                                              channel_termination_token=app_shutdown_token)
         rate_test(my_odrive)
 
     elif args.command == 'udev-setup':
         from odrive.utils import setup_udev_rules
         setup_udev_rules(logger)
 
+    elif args.command == 'backup-config':
+        from odrive.configuration import backup_config
+        print("Waiting for ODrive...")
+        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number,
+                                              search_cancellation_token=app_shutdown_token,
+                                              channel_termination_token=app_shutdown_token)
+        backup_config(my_odrive, args.file, logger)
+
+    elif args.command == 'restore-config':
+        from odrive.configuration import restore_config
+        print("Waiting for ODrive...")
+        my_odrive = odrive.discovery.find_any(path=args.path, serial_number=args.serial_number,
+                                              search_cancellation_token=app_shutdown_token,
+                                              channel_termination_token=app_shutdown_token)
+        restore_config(my_odrive, args.file, logger)
+
     else:
         raise Exception("unknown command: " + args.command)
 
+except OperationAbortedException:
+    logger.info("Operation aborted.")
 finally:
     app_shutdown_token.set()

tools/run_tests.py

@@ -136,7 +136,7 @@ try:
         if isinstance(test, ODriveTest):
             def odrv_test_thread(odrv_name):
                 odrv_ctx = odrives_by_name[odrv_name]
-                logger.info('* running {} on {}...'.format(type(test).__name__, odrv_name))
+                logger.notify('* running {} on {}...'.format(type(test).__name__, odrv_name))
                 try:
                     test.check_preconditions(odrv_ctx,
                               logger.indent('  {}: '.format(odrv_name)))
@@ -165,7 +165,7 @@ try:
                 try:
                     if not app_shutdown_token.is_set():
                         # Run test on this axis
-                        logger.info('* running {} on {}...'.format(type(test).__name__, axis_name))
+                        logger.notify('* running {} on {}...'.format(type(test).__name__, axis_name))
                         try:
                             test.check_preconditions(axis_ctx,
                                         logger.indent('  {}: '.format(axis_name)))
@@ -197,7 +197,7 @@ try:
                 try:
                     if not app_shutdown_token.is_set():
                         # Run test on this axis
-                        logger.info('* running {} on {}...'.format(type(test).__name__, coupling_name))
+                        logger.notify('* running {} on {}...'.format(type(test).__name__, coupling_name))
                         try:
                             test.check_preconditions(coupled_axes[0], coupled_axes[1],
                                         logger.indent('  {}: '.format(coupling_name)))