From 1b9f44211478eb2d47013cc9849674aa9ec5a47a Mon Sep 17 00:00:00 2001 From: Samuel Sadok Date: Thu, 23 Apr 2020 17:44:07 +0200 Subject: [PATCH 01/15] clarify ascii doc --- docs/ascii-protocol.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/ascii-protocol.md b/docs/ascii-protocol.md index 5e1dc236..b82863e0 100644 --- a/docs/ascii-protocol.md +++ b/docs/ascii-protocol.md @@ -5,11 +5,13 @@ * **Via USB:** * **Windows:** Use the Zadig utility to set the ODrive's driver to "usbser". Windows will then make the device available as COM port. You can use [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/) to manually send commands or open the COM port using your favorite programming language - * **Linux/macOS:** Run `/dev/tty*` to list all serial ports. The ODrive will show up as `/dev/ttyACM0` on Linux and `/dev/tty.usbmodem[...]` on macOS. Once you know the name, you can use `screen /dev/ttyACM0` (with the correct name) to send commands manually or open the device using your favorite programming language. Serial ports on Unix can be opened, written to and read from like a normal file. + * **Linux/macOS:** Run `/dev/tty*` to list all serial ports. The ODrive will show up as `/dev/ttyACM0` (or similar) on Linux and `/dev/tty.usbmodem[...]` on macOS. Once you know the name, you can use `screen /dev/ttyACM0` (with the correct name) to send commands manually or open the device using your favorite programming language. Serial ports on Unix can be opened, written to and read from like a normal file. * **Via UART:** Connect the ODrive's TX (GPIO1) to your host's RX. Connect your ODrive's RX (GPIO2) to your host's TX. The logic level of the ODrive is 3.3V. * **Arduino:** You can use the [ODrive Arduino library](https://github.com/madcowswe/ODrive/tree/master/Arduino/ODriveArduino) to talk to the ODrive. * **Windows/Linux/macOS:** You can use an FTDI USB-UART cable to connect to the ODrive. +The ODrive does not echo commands. That means that when you type commands into a program like `screen`, the characters you type won't show up in the console. + ## Command format The ASCII protocol is human-readable and line-oriented, with each line having the following format: @@ -18,7 +20,7 @@ The ASCII protocol is human-readable and line-oriented, with each line having th command *42 ; comment [new line character] ``` - * `*42` stands for a GCode compatible checksum and can be omitted. If and only if a checksum is provided, the device will also include a checksum in the response, if any. + * `*42` stands for a GCode compatible checksum and can be omitted. If and only if a checksum is provided, the device will also include a checksum in the response, if any. The checksum is calculated as the bitwise xor of all characters before the asterisk (`*`).
Example of a valid checksum: `r vbus_voltage *93`. * comments are supported for GCode compatibility * the command is interpreted once the new-line character is encountered From 0f3baf816f57f2ae5788ede686e62ebdc375522f Mon Sep 17 00:00:00 2001 From: Oskar Weigl Date: Mon, 1 Apr 2019 18:40:32 -0700 Subject: [PATCH 02/15] accept 0xffffffff offset for json crc --- Firmware/fibre/cpp/protocol.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Firmware/fibre/cpp/protocol.cpp b/Firmware/fibre/cpp/protocol.cpp index 23a2e0a5..f3ffecf3 100644 --- a/Firmware/fibre/cpp/protocol.cpp +++ b/Firmware/fibre/cpp/protocol.cpp @@ -140,15 +140,21 @@ void JSONDescriptorEndpoint::handle(const uint8_t* input, size_t input_length, S return; uint32_t offset = 0; read_le(&offset, input); - NullStreamSink output_with_offset = NullStreamSink(offset, *output); - size_t id = 0; - write_string("[", &output_with_offset); - json_file_endpoint_.write_json(id, &output_with_offset); - id += decltype(json_file_endpoint_)::endpoint_count; - write_string(",", &output_with_offset); - application_endpoints_->write_json(id, &output_with_offset); - write_string("]", &output_with_offset); + // If the offset is special value 0xFFFFFFFF, send back the JSON crc instead + if (offset == 0xffffffff) { + default_readwrite_endpoint_handler(&json_crc_, nullptr, 0, output); + } else { + NullStreamSink output_with_offset = NullStreamSink(offset, *output); + + size_t id = 0; + write_string("[", &output_with_offset); + json_file_endpoint_.write_json(id, &output_with_offset); + id += decltype(json_file_endpoint_)::endpoint_count; + write_string(",", &output_with_offset); + application_endpoints_->write_json(id, &output_with_offset); + write_string("]", &output_with_offset); + } } int BidirectionalPacketBasedChannel::process_packet(const uint8_t* buffer, size_t length) { From f6eacd7ae4a84ead5b4341a1c35f84ffb6cfe049 Mon Sep 17 00:00:00 2001 From: Oskar Weigl Date: Mon, 1 Apr 2019 19:13:15 -0700 Subject: [PATCH 03/15] implement fetching of json crc on discovery --- Firmware/fibre/python/fibre/discovery.py | 25 ++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 6039d7e0..58cda567 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -7,6 +7,7 @@ import json import time import threading import traceback +import struct import fibre.protocol import fibre.utils import fibre.remote_object @@ -64,12 +65,24 @@ def find_all(path, serial_number, """ try: logger.debug("Connecting to device on " + channel._name) - try: - json_bytes = channel.remote_endpoint_read_buffer(0) - except (TimeoutError, ChannelBrokenException): - logger.debug("no response - probably incompatible") - return - json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) + + # Fetching the json crc to check cache + json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) + + #TODO check cache using json_crc16 + cache_miss = True + + if (cache_miss): + # Download the JSON data + try: + json_bytes = channel.remote_endpoint_read_buffer(0) + except (TimeoutError, ChannelBrokenException): + logger.debug("no response - probably incompatible") + return + json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) + channel._interface_definition_crc = json_crc16 try: json_string = json_bytes.decode("ascii") From 1e9e6479418605fb1e0af7cd998981980b327598 Mon Sep 17 00:00:00 2001 From: Oskar Weigl Date: Mon, 1 Apr 2019 19:49:26 -0700 Subject: [PATCH 04/15] backwards compatible with old firmware --- Firmware/fibre/python/fibre/discovery.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 58cda567..1e6ee6f7 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -67,12 +67,17 @@ def find_all(path, serial_number, logger.debug("Connecting to device on " + channel._name) # Fetching the json crc to check cache - json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) + cache_miss = True + try: + json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) + except Exception as error: + logger.debug("Error fetching JSON CRC, falling back to downloading full JSON") + #logger.debug(traceback.format_exc()) #TODO check cache using json_crc16 - cache_miss = True + # Hence set cache_miss = False if (cache_miss): # Download the JSON data From f7eadfeb3bc867ebfa71641b1d6ad004f60de75a Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Mon, 1 Apr 2019 22:02:05 -0700 Subject: [PATCH 05/15] Update discovery.py fxd --- Firmware/fibre/python/fibre/discovery.py | 44 +++++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 1e6ee6f7..224d0dc6 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -13,6 +13,8 @@ import fibre.utils import fibre.remote_object from fibre.utils import Event, Logger from fibre.protocol import ChannelBrokenException, TimeoutError +import tempfile +import os # Load all installed transport layers @@ -42,6 +44,7 @@ try: except ImportError: pass + def noprint(text): pass @@ -55,7 +58,6 @@ def find_all(path, serial_number, the callback for each Fibre node that is found. This function is non-blocking. """ - def did_discover_channel(channel): """ Inits an object from a given channel and then calls did_discover_object_callback @@ -66,8 +68,12 @@ def find_all(path, serial_number, try: logger.debug("Connecting to device on " + channel._name) - # Fetching the json crc to check cache + temp_dir = tempfile.gettempdir() + cache_miss = True + json_crc16 = 0 + + # Fetching the json crc to check cache try: json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) + try: json_data = json.loads(json_string) except json.decoder.JSONDecodeError as error: logger.debug("device responded on endpoint 0 with something that is not JSON: " + str(error)) return + json_data = {"name": "fibre_node", "members": json_data} obj = fibre.remote_object.RemoteObject(json_data, None, channel, logger) @@ -111,7 +142,10 @@ def find_all(path, serial_number, if serial_number != None and device_serial_number != serial_number: logger.debug("Ignoring device with serial number {}".format(device_serial_number)) return + did_discover_object_callback(obj) + + except Exception: logger.debug("Unexpected exception after discovering channel: " + traceback.format_exc()) From 4fc9f7e3fee92309dbbed3cf2cbff8ce2cd00287 Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Wed, 3 Apr 2019 15:48:45 -0700 Subject: [PATCH 06/15] Update discovery.py fixd --- Firmware/fibre/python/fibre/discovery.py | 71 ++++++++++++------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 224d0dc6..3489ab3a 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -71,49 +71,50 @@ def find_all(path, serial_number, temp_dir = tempfile.gettempdir() cache_miss = True - json_crc16 = 0 # Fetching the json crc to check cache try: json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) + + cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) + + try: + json_cache = open(cache_path, 'r+') + logger.debug("Found cache file with crc") + cache_miss = False + + except: + logger.debug("Cache miss, there is no cache file") + cache_miss = True + + # Download the JSON data + if(cache_miss == True): + try: + logger.debug("Getting json schema from USB... this is slow...") + json_bytes = channel.remote_endpoint_read_buffer(0) + json_cache = open(cache_path, 'w+') + json_cache.write(json_bytes.decode("ascii")) + logger.debug("saved json_bytes to file") + + except (TimeoutError, ChannelBrokenException): + logger.debug("no response - probably incompatible") + return + else: + try: + json_bytes = json_cache.read().encode("ascii") + logger.debug("loaded json_bytes from " + cache_path) + + except (TimeoutError, ChannelBrokenException): + logger.debug("could not read cache file") + return + except Exception as error: logger.debug("Error fetching JSON CRC, falling back to downloading full JSON") - #logger.debug(traceback.format_exc()) - - cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) - - try: - json_cache = open(cache_path, 'r+') - logger.debug("Found cache file with crc") - cache_miss = False - except: - logger.debug("Cache miss, there is no cache file") - cache_miss = True - - if (cache_miss): - # Download the JSON data - try: - logger.debug("Getting json schema from USB... this is slow...") - json_bytes = channel.remote_endpoint_read_buffer(0) - json_cache = open(cache_path, 'w+') - json_cache.write(json_bytes.decode("ascii")) - logger.debug("saved json_bytes to file") - - except (TimeoutError, ChannelBrokenException): - logger.debug("no response - probably incompatible") - return - else: - try: - json_bytes = json_cache.read().encode("ascii") - logger.debug("loaded json_bytes from " + cache_path) - - except (TimeoutError, ChannelBrokenException): - logger.debug("could not read cache file") - return - - json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) + logger.debug("Getting json schema from USB... this is slow...") + json_bytes = channel.remote_endpoint_read_buffer(0) + json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) channel._interface_definition_crc = json_crc16 From 3b4d2a5d34ac2a31a3f8ed391a8f430eff2ac46e Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Wed, 3 Apr 2019 16:17:11 -0700 Subject: [PATCH 07/15] Update discovery.py store string directly --- Firmware/fibre/python/fibre/discovery.py | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 3489ab3a..007c93fc 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -92,37 +92,41 @@ def find_all(path, serial_number, # Download the JSON data if(cache_miss == True): try: - logger.debug("Getting json schema from USB... this is slow...") + logger.debug("Getting JSON schema from USB... this is slow...") json_bytes = channel.remote_endpoint_read_buffer(0) + try: + json_string = json_bytes.decode("ascii") + except UnicodeDecodeError: + logger.debug("Device responded on endpoint 0 with something that is not ASCII") + return json_cache = open(cache_path, 'w+') - json_cache.write(json_bytes.decode("ascii")) - logger.debug("saved json_bytes to file") + json_cache.write(json_string) + logger.debug("Saved JSON to cache file " + cache_path) except (TimeoutError, ChannelBrokenException): - logger.debug("no response - probably incompatible") + logger.debug("No response - probably incompatible") return else: try: - json_bytes = json_cache.read().encode("ascii") - logger.debug("loaded json_bytes from " + cache_path) + json_string = json_cache.read() + logger.debug("Loaded JSON from cache file " + cache_path) except (TimeoutError, ChannelBrokenException): - logger.debug("could not read cache file") + logger.debug("Could not read cache file " + cache_path) return except Exception as error: logger.debug("Error fetching JSON CRC, falling back to downloading full JSON") logger.debug("Getting json schema from USB... this is slow...") json_bytes = channel.remote_endpoint_read_buffer(0) + try: + json_string = json_bytes.decode("ascii") + except UnicodeDecodeError: + logger.debug("Device responded on endpoint 0 with something that is not ASCII") + return json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) channel._interface_definition_crc = json_crc16 - - try: - json_string = json_bytes.decode("ascii") - except UnicodeDecodeError: - logger.debug("device responded on endpoint 0 with something that is not ASCII") - return logger.debug("JSON: " + json_string.replace('{"name"', '\n{"name"')) logger.debug("JSON checksum: 0x{:02X} 0x{:02X}".format(json_crc16 & 0xff, (json_crc16 >> 8) & 0xff)) @@ -130,7 +134,7 @@ def find_all(path, serial_number, try: json_data = json.loads(json_string) except json.decoder.JSONDecodeError as error: - logger.debug("device responded on endpoint 0 with something that is not JSON: " + str(error)) + logger.debug("Device responded on endpoint 0 with something that is not JSON: " + str(error)) return json_data = {"name": "fibre_node", "members": json_data} From 1445d3531a4b8fa15907c025781abcdc4a9fe182 Mon Sep 17 00:00:00 2001 From: Ben Wang Date: Mon, 3 Jun 2019 12:13:27 -0400 Subject: [PATCH 08/15] refactor fiber cache --- Firmware/fibre/python/fibre/discovery.py | 105 ++++++++++++----------- Firmware/fibre/python/setup.py | 4 +- 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 007c93fc..ea323d95 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -13,7 +13,7 @@ import fibre.utils import fibre.remote_object from fibre.utils import Event, Logger from fibre.protocol import ChannelBrokenException, TimeoutError -import tempfile +import appdirs import os # Load all installed transport layers @@ -54,6 +54,32 @@ def find_all(path, serial_number, channel_termination_token, logger): """ + Load a json file from disk, return None if it does not exist or is invalid json + """ + def load_json_file(name): + try: + file = open(name, "r+") + except: + logger.debug(f"Failed to open json file {name}") + return None + + try: + file_str = file.read() + except: + logger.debug(f"Failed to read json file {name}") + return None + + try: + json_data = json.loads(file_str) + except: + logger.debug(f"Failed to deserialize json file {name}") + return None + + file.close() + + return json_data + + """ Starts scanning for Fibre nodes that match the specified path spec and calls the callback for each Fibre node that is found. This function is non-blocking. @@ -68,75 +94,52 @@ def find_all(path, serial_number, try: logger.debug("Connecting to device on " + channel._name) - temp_dir = tempfile.gettempdir() + temp_dir = appdirs.user_cache_dir("odrivetool") + try: + os.mkdir(temp_dir) + except FileExistsError: + pass - cache_miss = True + cache_path = "" # Fetching the json crc to check cache try: json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) + cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) + except: + logger.debug("Failed to get JSON checksum") - cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) - - try: - json_cache = open(cache_path, 'r+') - logger.debug("Found cache file with crc") - cache_miss = False - - except: - logger.debug("Cache miss, there is no cache file") - cache_miss = True - - # Download the JSON data - if(cache_miss == True): - try: - logger.debug("Getting JSON schema from USB... this is slow...") - json_bytes = channel.remote_endpoint_read_buffer(0) - try: - json_string = json_bytes.decode("ascii") - except UnicodeDecodeError: - logger.debug("Device responded on endpoint 0 with something that is not ASCII") - return - json_cache = open(cache_path, 'w+') - json_cache.write(json_string) - logger.debug("Saved JSON to cache file " + cache_path) - - except (TimeoutError, ChannelBrokenException): - logger.debug("No response - probably incompatible") - return - else: - try: - json_string = json_cache.read() - logger.debug("Loaded JSON from cache file " + cache_path) - - except (TimeoutError, ChannelBrokenException): - logger.debug("Could not read cache file " + cache_path) - return - - except Exception as error: - logger.debug("Error fetching JSON CRC, falling back to downloading full JSON") - logger.debug("Getting json schema from USB... this is slow...") + if cache_path == "" or load_json_file(cache_path) is None: + # Downloading json data + logger.info("Downloading json data from ODrive... (this might take a while)") json_bytes = channel.remote_endpoint_read_buffer(0) try: json_string = json_bytes.decode("ascii") except UnicodeDecodeError: logger.debug("Device responded on endpoint 0 with something that is not ASCII") - return + raise UnicodeDecodeError + json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) + cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) + + logger.debug(f"Opening json cache file {cache_path}") + with open(cache_path, 'w+') as json_cache: + json_cache.write(json_string) + logger.debug("Saved JSON to cache file " + cache_path) + json_data = load_json_file(cache_path) + else: + with open(cache_path, "rb") as f: + json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, f.read()) + json_data = load_json_file(cache_path) + channel._interface_definition_crc = json_crc16 - logger.debug("JSON: " + json_string.replace('{"name"', '\n{"name"')) + logger.debug("JSON: " + str(json_data).replace('{"name"', '\n{"name"')) logger.debug("JSON checksum: 0x{:02X} 0x{:02X}".format(json_crc16 & 0xff, (json_crc16 >> 8) & 0xff)) - try: - json_data = json.loads(json_string) - except json.decoder.JSONDecodeError as error: - logger.debug("Device responded on endpoint 0 with something that is not JSON: " + str(error)) - return - json_data = {"name": "fibre_node", "members": json_data} obj = fibre.remote_object.RemoteObject(json_data, None, channel, logger) diff --git a/Firmware/fibre/python/setup.py b/Firmware/fibre/python/setup.py index 85f266f3..9edccf6b 100644 --- a/Firmware/fibre/python/setup.py +++ b/Firmware/fibre/python/setup.py @@ -76,7 +76,9 @@ setup( license='MIT', url = 'https://github.com/samuelsadok/fibre', keywords = ['communication', 'transport-layer', 'rpc'], - install_requires = [], + install_requires = [ + 'appdirs', # Used to find caching directory + ], #package_data={'': ['version.txt']}, classifiers = [], ) From 2fd20eb931ba5748f0c1ff8ad8e5c4b42160ebea Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Wed, 19 Jun 2019 16:17:38 -0700 Subject: [PATCH 09/15] Update protocol.cpp --- Firmware/fibre/cpp/protocol.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Firmware/fibre/cpp/protocol.cpp b/Firmware/fibre/cpp/protocol.cpp index f3ffecf3..857bb6bc 100644 --- a/Firmware/fibre/cpp/protocol.cpp +++ b/Firmware/fibre/cpp/protocol.cpp @@ -1,4 +1,3 @@ - /* Includes ------------------------------------------------------------------*/ #include @@ -13,9 +12,10 @@ /* Global constant data ------------------------------------------------------*/ /* Global variables ----------------------------------------------------------*/ -Endpoint** endpoint_list_ = nullptr; // initialized by calling fibre_publish -size_t n_endpoints_ = 0; // initialized by calling fibre_publish -uint16_t json_crc_; // initialized by calling fibre_publish +Endpoint** endpoint_list_ = nullptr; // initialized by calling fibre_publish +size_t n_endpoints_ = 0; // initialized by calling fibre_publish +uint16_t json_crc_; // initialized by calling fibre_publish +uint32_t json_fibre_cache_entropy_; // initialized by calling fibre_publish JSONDescriptorEndpoint json_file_endpoint_ = JSONDescriptorEndpoint(); EndpointProvider* application_endpoints_; @@ -143,7 +143,8 @@ void JSONDescriptorEndpoint::handle(const uint8_t* input, size_t input_length, S // If the offset is special value 0xFFFFFFFF, send back the JSON crc instead if (offset == 0xffffffff) { - default_readwrite_endpoint_handler(&json_crc_, nullptr, 0, output); + //default_readwrite_endpoint_handler(&json_crc_, nullptr, 0, output); + default_readwrite_endpoint_handler(&json_fibre_cache_entropy_, nullptr, 0, output); } else { NullStreamSink output_with_offset = NullStreamSink(offset, *output); @@ -187,8 +188,8 @@ int BidirectionalPacketBasedChannel::process_packet(const uint8_t* buffer, size_ // Verify packet trailer. The expected trailer value depends on the selected endpoint. // For endpoint 0 this is just the protocol version, for all other endpoints it's a // CRC over the entire JSON descriptor tree (this may change in future versions). - uint16_t expected_trailer = endpoint_id ? json_crc_ : PROTOCOL_VERSION; - uint16_t actual_trailer = buffer[length - 2] | (buffer[length - 1] << 8); + uint32_t expected_trailer = endpoint_id ? json_crc_ : PROTOCOL_VERSION; + uint32_t actual_trailer = buffer[length - 2] | (buffer[length - 1] << 8); if (expected_trailer != actual_trailer) { LOG_FIBRE("trailer mismatch for endpoint %d: expected %04x, got %04x\r\n", endpoint_id, expected_trailer, actual_trailer); return -1; From c93400aacf583d1e9e4e008158b01759d8db5930 Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Wed, 19 Jun 2019 16:18:27 -0700 Subject: [PATCH 10/15] Update protocol.hpp --- Firmware/fibre/cpp/include/fibre/protocol.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Firmware/fibre/cpp/include/fibre/protocol.hpp b/Firmware/fibre/cpp/include/fibre/protocol.hpp index 016d7064..bfe774db 100644 --- a/Firmware/fibre/cpp/include/fibre/protocol.hpp +++ b/Firmware/fibre/cpp/include/fibre/protocol.hpp @@ -1,3 +1,5 @@ +#pragma GCC optimize ("O3") + /* see protocol.md for the protocol specification */ @@ -1098,6 +1100,7 @@ public: extern Endpoint** endpoint_list_; extern size_t n_endpoints_; extern uint16_t json_crc_; +extern uint32_t json_fibre_cache_entropy_; extern JSONDescriptorEndpoint json_file_endpoint_; extern EndpointProvider* application_endpoints_; @@ -1124,12 +1127,16 @@ int fibre_publish(T& application_objects) { // Calculate the CRC16 of the JSON file. // The init value is the protocol version. CRC16Calculator crc16_calculator(PROTOCOL_VERSION); + uint8_t offset[4] = { 0 }; json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator); json_crc_ = crc16_calculator.get_crc16(); + // Add entropy for fibre cache + json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator); + json_fibre_cache_entropy_ = (uint32_t) crc16_calculator.get_crc16(); + json_fibre_cache_entropy_ += json_crc_ << 16; + return 0; } - - #endif From 0e1ee833d2d133353f0d20c1429d0cc1e2c9d8e6 Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Wed, 19 Jun 2019 17:56:14 -0700 Subject: [PATCH 11/15] Update discovery.py added entropy --- Firmware/fibre/python/fibre/discovery.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index ea323d95..347fae85 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -104,10 +104,12 @@ def find_all(path, serial_number, # Fetching the json crc to check cache try: - json_crc16 = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) - cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) + json_fibre_cache_entropy = channel.remote_endpoint_operation(0, struct.pack("> 16 + + logger.debug("Device reported JSON entropy: {:08d}".format(json_fibre_cache_entropy)) + cache_path = temp_dir + '/fibre_schema_cache_' + str(json_fibre_cache_entropy) except: logger.debug("Failed to get JSON checksum") @@ -122,23 +124,25 @@ def find_all(path, serial_number, raise UnicodeDecodeError json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) - - cache_path = temp_dir + '/fibre_schema_cache_' + str(json_crc16) - - logger.debug(f"Opening json cache file {cache_path}") + json_fibre_cache_entropy = fibre.protocol.calc_crc16(json_crc16, json_bytes) | (json_crc16 << 16) + cache_path = temp_dir + '/fibre_schema_cache_' + str(json_fibre_cache_entropy) + logger.debug(f"Creating new JSON cache file {cache_path}") with open(cache_path, 'w+') as json_cache: json_cache.write(json_string) logger.debug("Saved JSON to cache file " + cache_path) json_data = load_json_file(cache_path) else: with open(cache_path, "rb") as f: + logger.debug(f"Loaded JSON from cache file {cache_path}") json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, f.read()) + f.seek(0) + json_fibre_cache_entropy = fibre.protocol.calc_crc16(json_crc16, f.read()) | (json_crc16 << 16) json_data = load_json_file(cache_path) channel._interface_definition_crc = json_crc16 - logger.debug("JSON: " + str(json_data).replace('{"name"', '\n{"name"')) - logger.debug("JSON checksum: 0x{:02X} 0x{:02X}".format(json_crc16 & 0xff, (json_crc16 >> 8) & 0xff)) + logger.debug("JSON: " + str(json_data).replace("{'name'", "\n{'name'")) + logger.debug("Local cache JSON entropy: {:08d}".format(json_fibre_cache_entropy)) json_data = {"name": "fibre_node", "members": json_data} obj = fibre.remote_object.RemoteObject(json_data, None, channel, logger) From 5e3d8693c9278c4a2dc7e758c20338d16aeacf7a Mon Sep 17 00:00:00 2001 From: Adam Munich Date: Wed, 19 Jun 2019 17:57:18 -0700 Subject: [PATCH 12/15] Update shell.py fix broken exceptionhandler --- Firmware/fibre/python/fibre/shell.py | 32 +++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/Firmware/fibre/python/fibre/shell.py b/Firmware/fibre/python/fibre/shell.py index d5e24d85..c5a7257a 100644 --- a/Firmware/fibre/python/fibre/shell.py +++ b/Firmware/fibre/python/fibre/shell.py @@ -80,11 +80,23 @@ def launch_shell(args, # If IPython is installed, embed IPython shell, otherwise embed regular shell if use_ipython: - help = lambda: print_help(args, len(discovered_devices) > 0) # Override help function # pylint: disable=W0612 - locals()['__name__'] = globals()['__name__'] # to fix broken "%run -i script.py" + # Override help function # pylint: disable=W0612 + help = lambda: print_help(args, len(discovered_devices) > 0) + # to fix broken "%run -i script.py" + locals()['__name__'] = globals()['__name__'] console = IPython.terminal.embed.InteractiveShellEmbed(banner1='') - console.runcode = console.run_code # hack to make IPython look like the regular console + + # hack to make IPython look like the regular console + console.runcode = console.run_cell interact = console + + # Catch ChannelBrokenException (since disconnect is not always an error) + default_exception_hook = console._showtraceback + def filtered_exception_hook(ex_class, ex, trace): + if(ex_class.__module__+'.'+ex_class.__name__ != 'fibre.protocol.ChannelBrokenException'): + default_exception_hook(ex_class,ex,trace) + + console._showtraceback = filtered_exception_hook else: # Enable tab complete if possible try: @@ -100,13 +112,13 @@ def launch_shell(args, console = code.InteractiveConsole(locals=interactive_variables) interact = lambda: console.interact(banner='') - # install hook to hide ChannelBrokenException - console.runcode('import sys') - console.runcode('superexcepthook = sys.excepthook') - console.runcode('def newexcepthook(ex_class,ex,trace):\n' - ' if ex_class.__module__ + "." + ex_class.__name__ != "fibre.ChannelBrokenException":\n' - ' superexcepthook(ex_class,ex,trace)') - console.runcode('sys.excepthook=newexcepthook') + # Catch ChannelBrokenException (since disconnect is not alway an error) + console.runcode("import sys") + console.runcode("default_exception_hook = sys.excepthook") + console.runcode("def filtered_exception_hook(ex_class, ex, trace):\n" + " if ex_class.__module__ + '.' + ex_class.__name__ != 'fibre.protocol.ChannelBrokenException':\n" + " default_exception_hook(ex_class,ex,trace)") + console.runcode("sys.excepthook=filtered_exception_hook") # Launch shell From ff544bfc1983ca9a3e842a373eab6d9df845741a Mon Sep 17 00:00:00 2001 From: Samuel Sadok Date: Fri, 24 Apr 2020 10:19:20 +0200 Subject: [PATCH 13/15] simplify JSON caching logic, make JSON cache ID opaque, remove spurious changes --- Firmware/fibre/cpp/include/fibre/protocol.hpp | 10 +-- Firmware/fibre/cpp/protocol.cpp | 12 +-- Firmware/fibre/python/fibre/discovery.py | 87 +++++++------------ 3 files changed, 42 insertions(+), 67 deletions(-) diff --git a/Firmware/fibre/cpp/include/fibre/protocol.hpp b/Firmware/fibre/cpp/include/fibre/protocol.hpp index bfe774db..883ba544 100644 --- a/Firmware/fibre/cpp/include/fibre/protocol.hpp +++ b/Firmware/fibre/cpp/include/fibre/protocol.hpp @@ -1,5 +1,3 @@ -#pragma GCC optimize ("O3") - /* see protocol.md for the protocol specification */ @@ -1100,7 +1098,7 @@ public: extern Endpoint** endpoint_list_; extern size_t n_endpoints_; extern uint16_t json_crc_; -extern uint32_t json_fibre_cache_entropy_; +extern uint32_t json_version_id_; // exposed to hosts to facilitate cache lookup extern JSONDescriptorEndpoint json_file_endpoint_; extern EndpointProvider* application_endpoints_; @@ -1134,9 +1132,11 @@ int fibre_publish(T& application_objects) { // Add entropy for fibre cache json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator); - json_fibre_cache_entropy_ = (uint32_t) crc16_calculator.get_crc16(); - json_fibre_cache_entropy_ += json_crc_ << 16; + json_version_id_ = (uint32_t) crc16_calculator.get_crc16(); + json_version_id_ += json_crc_ << 16; return 0; } + + #endif diff --git a/Firmware/fibre/cpp/protocol.cpp b/Firmware/fibre/cpp/protocol.cpp index 857bb6bc..d5af8a0b 100644 --- a/Firmware/fibre/cpp/protocol.cpp +++ b/Firmware/fibre/cpp/protocol.cpp @@ -1,3 +1,4 @@ + /* Includes ------------------------------------------------------------------*/ #include @@ -15,7 +16,7 @@ Endpoint** endpoint_list_ = nullptr; // initialized by calling fibre_publish size_t n_endpoints_ = 0; // initialized by calling fibre_publish uint16_t json_crc_; // initialized by calling fibre_publish -uint32_t json_fibre_cache_entropy_; // initialized by calling fibre_publish +uint32_t json_version_id_; // initialized by calling fibre_publish JSONDescriptorEndpoint json_file_endpoint_ = JSONDescriptorEndpoint(); EndpointProvider* application_endpoints_; @@ -141,10 +142,9 @@ void JSONDescriptorEndpoint::handle(const uint8_t* input, size_t input_length, S uint32_t offset = 0; read_le(&offset, input); - // If the offset is special value 0xFFFFFFFF, send back the JSON crc instead + // If the offset is special value 0xFFFFFFFF, send back the JSON version ID instead if (offset == 0xffffffff) { - //default_readwrite_endpoint_handler(&json_crc_, nullptr, 0, output); - default_readwrite_endpoint_handler(&json_fibre_cache_entropy_, nullptr, 0, output); + default_readwrite_endpoint_handler(&json_version_id_, nullptr, 0, output); } else { NullStreamSink output_with_offset = NullStreamSink(offset, *output); @@ -188,8 +188,8 @@ int BidirectionalPacketBasedChannel::process_packet(const uint8_t* buffer, size_ // Verify packet trailer. The expected trailer value depends on the selected endpoint. // For endpoint 0 this is just the protocol version, for all other endpoints it's a // CRC over the entire JSON descriptor tree (this may change in future versions). - uint32_t expected_trailer = endpoint_id ? json_crc_ : PROTOCOL_VERSION; - uint32_t actual_trailer = buffer[length - 2] | (buffer[length - 1] << 8); + uint16_t expected_trailer = endpoint_id ? json_crc_ : PROTOCOL_VERSION; + uint16_t actual_trailer = buffer[length - 2] | (buffer[length - 1] << 8); if (expected_trailer != actual_trailer) { LOG_FIBRE("trailer mismatch for endpoint %d: expected %04x, got %04x\r\n", endpoint_id, expected_trailer, actual_trailer); return -1; diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 347fae85..12836f11 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -54,36 +54,11 @@ def find_all(path, serial_number, channel_termination_token, logger): """ - Load a json file from disk, return None if it does not exist or is invalid json - """ - def load_json_file(name): - try: - file = open(name, "r+") - except: - logger.debug(f"Failed to open json file {name}") - return None - - try: - file_str = file.read() - except: - logger.debug(f"Failed to read json file {name}") - return None - - try: - json_data = json.loads(file_str) - except: - logger.debug(f"Failed to deserialize json file {name}") - return None - - file.close() - - return json_data - - """ Starts scanning for Fibre nodes that match the specified path spec and calls the callback for each Fibre node that is found. This function is non-blocking. """ + def did_discover_channel(channel): """ Inits an object from a given channel and then calls did_discover_object_callback @@ -94,26 +69,32 @@ def find_all(path, serial_number, try: logger.debug("Connecting to device on " + channel._name) - temp_dir = appdirs.user_cache_dir("odrivetool") + cache_dir = appdirs.user_cache_dir("odrivetool") + cache_path = None + + # Fetch the json version tag to check cache (only supported on firmware v0.5 or later) try: - os.mkdir(temp_dir) - except FileExistsError: - pass + json_version_tag = channel.remote_endpoint_operation(0, struct.pack("> 16 - - logger.debug("Device reported JSON entropy: {:08d}".format(json_fibre_cache_entropy)) - cache_path = temp_dir + '/fibre_schema_cache_' + str(json_fibre_cache_entropy) + logger.debug("Device reported JSON version ID: {:08d}".format(json_version_tag)) + cache_path = os.path.join(cache_dir, 'fibre_schema_cache_{:08d}'.format(json_version_tag)) except: logger.debug("Failed to get JSON checksum") - if cache_path == "" or load_json_file(cache_path) is None: + # Check cache + json_data = None + try: + if not cache_path is None: + with open(cache_path, 'rb') as fp: + json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, fp.read()) + fp.seek(0) + json_data = json.load(fp) + except: + logger.debug(f"Failed load JSON cache file {cache_path}") + + # Fallback to loading JSON from device + if json_data is None: # Downloading json data logger.info("Downloading json data from ODrive... (this might take a while)") json_bytes = channel.remote_endpoint_read_buffer(0) @@ -124,25 +105,19 @@ def find_all(path, serial_number, raise UnicodeDecodeError json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) - json_fibre_cache_entropy = fibre.protocol.calc_crc16(json_crc16, json_bytes) | (json_crc16 << 16) - cache_path = temp_dir + '/fibre_schema_cache_' + str(json_fibre_cache_entropy) - logger.debug(f"Creating new JSON cache file {cache_path}") - with open(cache_path, 'w+') as json_cache: - json_cache.write(json_string) - logger.debug("Saved JSON to cache file " + cache_path) - json_data = load_json_file(cache_path) - else: - with open(cache_path, "rb") as f: - logger.debug(f"Loaded JSON from cache file {cache_path}") - json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, f.read()) - f.seek(0) - json_fibre_cache_entropy = fibre.protocol.calc_crc16(json_crc16, f.read()) | (json_crc16 << 16) - json_data = load_json_file(cache_path) + json_data = json.loads(json_string) + + # Save JSON to cache + if not cache_path is None: + logger.debug(f"Creating new JSON cache file {cache_path}") + os.makedirs(cache_dir, exist_ok=True) + with open(cache_path, 'w+') as json_cache: + json_cache.write(json_string) + logger.debug(f"Saved JSON to cache file {cache_path}") channel._interface_definition_crc = json_crc16 logger.debug("JSON: " + str(json_data).replace("{'name'", "\n{'name'")) - logger.debug("Local cache JSON entropy: {:08d}".format(json_fibre_cache_entropy)) json_data = {"name": "fibre_node", "members": json_data} obj = fibre.remote_object.RemoteObject(json_data, None, channel, logger) From d3a864785d2e6c8ed6177566f7b69ef966ae4568 Mon Sep 17 00:00:00 2001 From: Samuel Sadok Date: Fri, 24 Apr 2020 11:13:11 +0200 Subject: [PATCH 14/15] remove spurious line change --- Firmware/fibre/python/fibre/discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 12836f11..4407537b 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -44,7 +44,6 @@ try: except ImportError: pass - def noprint(text): pass From 78b48f1a9b6cd8ce528e44b3f35f79540392c752 Mon Sep 17 00:00:00 2001 From: Samuel Sadok Date: Fri, 24 Apr 2020 11:21:49 +0200 Subject: [PATCH 15/15] amend changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13d5fa31..52cffece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ Please add a note of your changes below this heading if you make a Pull Request. * Cleaned up VSCode C/C++ Configuration settings on Windows with recursive includePath * Now compiling with C++17 * Fixed a firmware hang that could occur from unlikely but possible user input +* Added JSON caching to Fibre. This drastically reduces the time odrivetool needs to connect to an ODrive (except for the first time or after firmware updates). +* Fix IPython `RuntimeWarning` that would occur every time `odrivetool` was started. # Releases ## [0.4.11] - 2019-07-25