Merge pull request #353 from madcowswe/fiber_cache_refactor

- Add JSON caching to Fibre. This drastically reduces the time odrivetool needs to connect to an ODrive (except for the first time or after firmware updates). The updated firmware is compatible with the old odrivetool and the updated odrivetool is compatible with the old firmware. - Fix IPython `RuntimeWarning` that would occur every time `odrivetool` was started.
2026-09-26 12:27:45 +08:00 · 2020-04-24 11:24:23 +02:00
parent 1b9f442114 78b48f1a9b
commit c8b1881c51
6 changed files with 106 additions and 38 deletions
@@ -33,6 +33,8 @@ Please add a note of your changes below this heading if you make a Pull Request.
 * Cleaned up VSCode C/C++ Configuration settings on Windows with recursive includePath
 * Now compiling with C++17
 * Fixed a firmware hang that could occur from unlikely but possible user input
+* Added JSON caching to Fibre. This drastically reduces the time odrivetool needs to connect to an ODrive (except for the first time or after firmware updates).
+* Fix IPython `RuntimeWarning` that would occur every time `odrivetool` was started.

 # Releases
 ## [0.4.11] - 2019-07-25
@@ -1098,6 +1098,7 @@ public:
 extern Endpoint** endpoint_list_;
 extern size_t n_endpoints_;
 extern uint16_t json_crc_;
+extern uint32_t json_version_id_; // exposed to hosts to facilitate cache lookup
 extern JSONDescriptorEndpoint json_file_endpoint_;
 extern EndpointProvider* application_endpoints_;

@@ -1124,10 +1125,16 @@ int fibre_publish(T& application_objects) {
    // Calculate the CRC16 of the JSON file.
    // The init value is the protocol version.
    CRC16Calculator crc16_calculator(PROTOCOL_VERSION);
+
    uint8_t offset[4] = { 0 };
    json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator);
    json_crc_ = crc16_calculator.get_crc16();

+    // Add entropy for fibre cache
+    json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator);
+    json_version_id_ = (uint32_t) crc16_calculator.get_crc16();
+    json_version_id_ += json_crc_ << 16;
+
    return 0;
 }

@@ -13,9 +13,10 @@
 /* Global constant data ------------------------------------------------------*/
 /* Global variables ----------------------------------------------------------*/

-Endpoint** endpoint_list_ = nullptr; // initialized by calling fibre_publish
-size_t n_endpoints_ = 0; // initialized by calling fibre_publish
-uint16_t json_crc_; // initialized by calling fibre_publish
+Endpoint** endpoint_list_ = nullptr;    // initialized by calling fibre_publish
+size_t n_endpoints_ = 0;                // initialized by calling fibre_publish
+uint16_t json_crc_;                     // initialized by calling fibre_publish
+uint32_t json_version_id_;              // initialized by calling fibre_publish
 JSONDescriptorEndpoint json_file_endpoint_ = JSONDescriptorEndpoint();
 EndpointProvider* application_endpoints_;

@@ -140,15 +141,21 @@ void JSONDescriptorEndpoint::handle(const uint8_t* input, size_t input_length, S
        return;
    uint32_t offset = 0;
    read_le<uint32_t>(&offset, input);
-    NullStreamSink output_with_offset = NullStreamSink(offset, *output);

-    size_t id = 0;
-    write_string("[", &output_with_offset);
-    json_file_endpoint_.write_json(id, &output_with_offset);
-    id += decltype(json_file_endpoint_)::endpoint_count;
-    write_string(",", &output_with_offset);
-    application_endpoints_->write_json(id, &output_with_offset);
-    write_string("]", &output_with_offset);
+    // If the offset is special value 0xFFFFFFFF, send back the JSON version ID instead
+    if (offset == 0xffffffff) {
+        default_readwrite_endpoint_handler(&json_version_id_, nullptr, 0, output);
+    } else {
+        NullStreamSink output_with_offset = NullStreamSink(offset, *output);
+
+        size_t id = 0;
+        write_string("[", &output_with_offset);
+        json_file_endpoint_.write_json(id, &output_with_offset);
+        id += decltype(json_file_endpoint_)::endpoint_count;
+        write_string(",", &output_with_offset);
+        application_endpoints_->write_json(id, &output_with_offset);
+        write_string("]", &output_with_offset);
+    }
 }

 int BidirectionalPacketBasedChannel::process_packet(const uint8_t* buffer, size_t length) {
@@ -7,11 +7,14 @@ import json
 import time
 import threading
 import traceback
+import struct
 import fibre.protocol
 import fibre.utils
 import fibre.remote_object
 from fibre.utils import Event, Logger
 from fibre.protocol import ChannelBrokenException, TimeoutError
+import appdirs
+import os

 # Load all installed transport layers

@@ -64,25 +67,57 @@ def find_all(path, serial_number,
        """
        try:
            logger.debug("Connecting to device on " + channel._name)
+
+            cache_dir = appdirs.user_cache_dir("odrivetool")
+            cache_path = None
+
+            # Fetch the json version tag to check cache (only supported on firmware v0.5 or later)
            try:
+                json_version_tag = channel.remote_endpoint_operation(0, struct.pack("<I", 0xffffffff), True, 4)
+                json_version_tag = struct.unpack("<I", json_version_tag)[0]
+
+                logger.debug("Device reported JSON version ID: {:08d}".format(json_version_tag))
+                cache_path = os.path.join(cache_dir, 'fibre_schema_cache_{:08d}'.format(json_version_tag))
+            except:
+                logger.debug("Failed to get JSON checksum")
+
+            # Check cache
+            json_data = None
+            try:
+                if not cache_path is None:
+                    with open(cache_path, 'rb') as fp:
+                        json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, fp.read())
+                        fp.seek(0)
+                        json_data = json.load(fp)
+            except:
+                logger.debug(f"Failed load JSON cache file {cache_path}")
+
+            # Fallback to loading JSON from device
+            if json_data is None:
+                # Downloading json data
+                logger.info("Downloading json data from ODrive... (this might take a while)")
                json_bytes = channel.remote_endpoint_read_buffer(0)
-            except (TimeoutError, ChannelBrokenException):
-                logger.debug("no response - probably incompatible")
-                return
-            json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes)
-            channel._interface_definition_crc = json_crc16
-            try:
-                json_string = json_bytes.decode("ascii")
-            except UnicodeDecodeError:
-                logger.debug("device responded on endpoint 0 with something that is not ASCII")
-                return
-            logger.debug("JSON: " + json_string.replace('{"name"', '\n{"name"'))
-            logger.debug("JSON checksum: 0x{:02X} 0x{:02X}".format(json_crc16 & 0xff, (json_crc16 >> 8) & 0xff))
-            try:
+                try:
+                    json_string = json_bytes.decode("ascii")
+                except UnicodeDecodeError:
+                    logger.debug("Device responded on endpoint 0 with something that is not ASCII")
+                    raise UnicodeDecodeError
+
+                json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes)
                json_data = json.loads(json_string)
-            except json.decoder.JSONDecodeError as error:
-                logger.debug("device responded on endpoint 0 with something that is not JSON: " + str(error))
-                return
+
+                # Save JSON to cache
+                if not cache_path is None:
+                    logger.debug(f"Creating new JSON cache file {cache_path}")
+                    os.makedirs(cache_dir, exist_ok=True)
+                    with open(cache_path, 'w+') as json_cache:
+                        json_cache.write(json_string)
+                    logger.debug(f"Saved JSON to cache file {cache_path}")
+
+            channel._interface_definition_crc = json_crc16
+
+            logger.debug("JSON: " + str(json_data).replace("{'name'", "\n{'name'"))
+
            json_data = {"name": "fibre_node", "members": json_data}
            obj = fibre.remote_object.RemoteObject(json_data, None, channel, logger)

@@ -93,7 +128,10 @@ def find_all(path, serial_number,
            if serial_number != None and device_serial_number != serial_number:
                logger.debug("Ignoring device with serial number {}".format(device_serial_number))
                return
+            
            did_discover_object_callback(obj)
+
+
        except Exception:
            logger.debug("Unexpected exception after discovering channel: " + traceback.format_exc())

@@ -80,11 +80,23 @@ def launch_shell(args,

    # If IPython is installed, embed IPython shell, otherwise embed regular shell
    if use_ipython:
-        help = lambda: print_help(args, len(discovered_devices) > 0) # Override help function # pylint: disable=W0612
-        locals()['__name__'] = globals()['__name__'] # to fix broken "%run -i script.py"
+        # Override help function # pylint: disable=W0612
+        help = lambda: print_help(args, len(discovered_devices) > 0) 
+        # to fix broken "%run -i script.py"
+        locals()['__name__'] = globals()['__name__'] 
        console = IPython.terminal.embed.InteractiveShellEmbed(banner1='')
-        console.runcode = console.run_code # hack to make IPython look like the regular console
+
+        # hack to make IPython look like the regular console
+        console.runcode = console.run_cell 
        interact = console
+
+        # Catch ChannelBrokenException (since disconnect is not always an error)
+        default_exception_hook = console._showtraceback
+        def filtered_exception_hook(ex_class, ex, trace):
+            if(ex_class.__module__+'.'+ex_class.__name__ != 'fibre.protocol.ChannelBrokenException'):
+                default_exception_hook(ex_class,ex,trace)
+            
+        console._showtraceback = filtered_exception_hook
    else:
        # Enable tab complete if possible
        try:
@@ -100,13 +112,13 @@ def launch_shell(args,
        console = code.InteractiveConsole(locals=interactive_variables)
        interact = lambda: console.interact(banner='')

-    # install hook to hide ChannelBrokenException
-    console.runcode('import sys')
-    console.runcode('superexcepthook = sys.excepthook')
-    console.runcode('def newexcepthook(ex_class,ex,trace):\n'
-                    '  if ex_class.__module__ + "." + ex_class.__name__ != "fibre.ChannelBrokenException":\n'
-                    '    superexcepthook(ex_class,ex,trace)')
-    console.runcode('sys.excepthook=newexcepthook')
+        # Catch ChannelBrokenException (since disconnect is not alway an error)
+        console.runcode("import sys")
+        console.runcode("default_exception_hook = sys.excepthook")
+        console.runcode("def filtered_exception_hook(ex_class, ex, trace):\n"
+                        "  if ex_class.__module__ + '.' + ex_class.__name__ != 'fibre.protocol.ChannelBrokenException':\n"
+                        "    default_exception_hook(ex_class,ex,trace)")
+        console.runcode("sys.excepthook=filtered_exception_hook")


    # Launch shell
@@ -76,7 +76,9 @@ setup(
  license='MIT',
  url = 'https://github.com/samuelsadok/fibre',
  keywords = ['communication', 'transport-layer', 'rpc'],
-  install_requires = [],
+  install_requires = [
+    'appdirs',  # Used to find caching directory
+  ],
  #package_data={'': ['version.txt']},
  classifiers = [],
 )