diff --git a/CHANGELOG.md b/CHANGELOG.md index 13d5fa31..52cffece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ Please add a note of your changes below this heading if you make a Pull Request. * Cleaned up VSCode C/C++ Configuration settings on Windows with recursive includePath * Now compiling with C++17 * Fixed a firmware hang that could occur from unlikely but possible user input +* Added JSON caching to Fibre. This drastically reduces the time odrivetool needs to connect to an ODrive (except for the first time or after firmware updates). +* Fix IPython `RuntimeWarning` that would occur every time `odrivetool` was started. # Releases ## [0.4.11] - 2019-07-25 diff --git a/Firmware/fibre/cpp/include/fibre/protocol.hpp b/Firmware/fibre/cpp/include/fibre/protocol.hpp index 016d7064..883ba544 100644 --- a/Firmware/fibre/cpp/include/fibre/protocol.hpp +++ b/Firmware/fibre/cpp/include/fibre/protocol.hpp @@ -1098,6 +1098,7 @@ public: extern Endpoint** endpoint_list_; extern size_t n_endpoints_; extern uint16_t json_crc_; +extern uint32_t json_version_id_; // exposed to hosts to facilitate cache lookup extern JSONDescriptorEndpoint json_file_endpoint_; extern EndpointProvider* application_endpoints_; @@ -1124,10 +1125,16 @@ int fibre_publish(T& application_objects) { // Calculate the CRC16 of the JSON file. // The init value is the protocol version. CRC16Calculator crc16_calculator(PROTOCOL_VERSION); + uint8_t offset[4] = { 0 }; json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator); json_crc_ = crc16_calculator.get_crc16(); + // Add entropy for fibre cache + json_file_endpoint_.handle(offset, sizeof(offset), &crc16_calculator); + json_version_id_ = (uint32_t) crc16_calculator.get_crc16(); + json_version_id_ += json_crc_ << 16; + return 0; } diff --git a/Firmware/fibre/cpp/protocol.cpp b/Firmware/fibre/cpp/protocol.cpp index 23a2e0a5..d5af8a0b 100644 --- a/Firmware/fibre/cpp/protocol.cpp +++ b/Firmware/fibre/cpp/protocol.cpp @@ -13,9 +13,10 @@ /* Global constant data ------------------------------------------------------*/ /* Global variables ----------------------------------------------------------*/ -Endpoint** endpoint_list_ = nullptr; // initialized by calling fibre_publish -size_t n_endpoints_ = 0; // initialized by calling fibre_publish -uint16_t json_crc_; // initialized by calling fibre_publish +Endpoint** endpoint_list_ = nullptr; // initialized by calling fibre_publish +size_t n_endpoints_ = 0; // initialized by calling fibre_publish +uint16_t json_crc_; // initialized by calling fibre_publish +uint32_t json_version_id_; // initialized by calling fibre_publish JSONDescriptorEndpoint json_file_endpoint_ = JSONDescriptorEndpoint(); EndpointProvider* application_endpoints_; @@ -140,15 +141,21 @@ void JSONDescriptorEndpoint::handle(const uint8_t* input, size_t input_length, S return; uint32_t offset = 0; read_le(&offset, input); - NullStreamSink output_with_offset = NullStreamSink(offset, *output); - size_t id = 0; - write_string("[", &output_with_offset); - json_file_endpoint_.write_json(id, &output_with_offset); - id += decltype(json_file_endpoint_)::endpoint_count; - write_string(",", &output_with_offset); - application_endpoints_->write_json(id, &output_with_offset); - write_string("]", &output_with_offset); + // If the offset is special value 0xFFFFFFFF, send back the JSON version ID instead + if (offset == 0xffffffff) { + default_readwrite_endpoint_handler(&json_version_id_, nullptr, 0, output); + } else { + NullStreamSink output_with_offset = NullStreamSink(offset, *output); + + size_t id = 0; + write_string("[", &output_with_offset); + json_file_endpoint_.write_json(id, &output_with_offset); + id += decltype(json_file_endpoint_)::endpoint_count; + write_string(",", &output_with_offset); + application_endpoints_->write_json(id, &output_with_offset); + write_string("]", &output_with_offset); + } } int BidirectionalPacketBasedChannel::process_packet(const uint8_t* buffer, size_t length) { diff --git a/Firmware/fibre/python/fibre/discovery.py b/Firmware/fibre/python/fibre/discovery.py index 6039d7e0..4407537b 100644 --- a/Firmware/fibre/python/fibre/discovery.py +++ b/Firmware/fibre/python/fibre/discovery.py @@ -7,11 +7,14 @@ import json import time import threading import traceback +import struct import fibre.protocol import fibre.utils import fibre.remote_object from fibre.utils import Event, Logger from fibre.protocol import ChannelBrokenException, TimeoutError +import appdirs +import os # Load all installed transport layers @@ -64,25 +67,57 @@ def find_all(path, serial_number, """ try: logger.debug("Connecting to device on " + channel._name) + + cache_dir = appdirs.user_cache_dir("odrivetool") + cache_path = None + + # Fetch the json version tag to check cache (only supported on firmware v0.5 or later) try: + json_version_tag = channel.remote_endpoint_operation(0, struct.pack("> 8) & 0xff)) - try: + try: + json_string = json_bytes.decode("ascii") + except UnicodeDecodeError: + logger.debug("Device responded on endpoint 0 with something that is not ASCII") + raise UnicodeDecodeError + + json_crc16 = fibre.protocol.calc_crc16(fibre.protocol.PROTOCOL_VERSION, json_bytes) json_data = json.loads(json_string) - except json.decoder.JSONDecodeError as error: - logger.debug("device responded on endpoint 0 with something that is not JSON: " + str(error)) - return + + # Save JSON to cache + if not cache_path is None: + logger.debug(f"Creating new JSON cache file {cache_path}") + os.makedirs(cache_dir, exist_ok=True) + with open(cache_path, 'w+') as json_cache: + json_cache.write(json_string) + logger.debug(f"Saved JSON to cache file {cache_path}") + + channel._interface_definition_crc = json_crc16 + + logger.debug("JSON: " + str(json_data).replace("{'name'", "\n{'name'")) + json_data = {"name": "fibre_node", "members": json_data} obj = fibre.remote_object.RemoteObject(json_data, None, channel, logger) @@ -93,7 +128,10 @@ def find_all(path, serial_number, if serial_number != None and device_serial_number != serial_number: logger.debug("Ignoring device with serial number {}".format(device_serial_number)) return + did_discover_object_callback(obj) + + except Exception: logger.debug("Unexpected exception after discovering channel: " + traceback.format_exc()) diff --git a/Firmware/fibre/python/fibre/shell.py b/Firmware/fibre/python/fibre/shell.py index d5e24d85..c5a7257a 100644 --- a/Firmware/fibre/python/fibre/shell.py +++ b/Firmware/fibre/python/fibre/shell.py @@ -80,11 +80,23 @@ def launch_shell(args, # If IPython is installed, embed IPython shell, otherwise embed regular shell if use_ipython: - help = lambda: print_help(args, len(discovered_devices) > 0) # Override help function # pylint: disable=W0612 - locals()['__name__'] = globals()['__name__'] # to fix broken "%run -i script.py" + # Override help function # pylint: disable=W0612 + help = lambda: print_help(args, len(discovered_devices) > 0) + # to fix broken "%run -i script.py" + locals()['__name__'] = globals()['__name__'] console = IPython.terminal.embed.InteractiveShellEmbed(banner1='') - console.runcode = console.run_code # hack to make IPython look like the regular console + + # hack to make IPython look like the regular console + console.runcode = console.run_cell interact = console + + # Catch ChannelBrokenException (since disconnect is not always an error) + default_exception_hook = console._showtraceback + def filtered_exception_hook(ex_class, ex, trace): + if(ex_class.__module__+'.'+ex_class.__name__ != 'fibre.protocol.ChannelBrokenException'): + default_exception_hook(ex_class,ex,trace) + + console._showtraceback = filtered_exception_hook else: # Enable tab complete if possible try: @@ -100,13 +112,13 @@ def launch_shell(args, console = code.InteractiveConsole(locals=interactive_variables) interact = lambda: console.interact(banner='') - # install hook to hide ChannelBrokenException - console.runcode('import sys') - console.runcode('superexcepthook = sys.excepthook') - console.runcode('def newexcepthook(ex_class,ex,trace):\n' - ' if ex_class.__module__ + "." + ex_class.__name__ != "fibre.ChannelBrokenException":\n' - ' superexcepthook(ex_class,ex,trace)') - console.runcode('sys.excepthook=newexcepthook') + # Catch ChannelBrokenException (since disconnect is not alway an error) + console.runcode("import sys") + console.runcode("default_exception_hook = sys.excepthook") + console.runcode("def filtered_exception_hook(ex_class, ex, trace):\n" + " if ex_class.__module__ + '.' + ex_class.__name__ != 'fibre.protocol.ChannelBrokenException':\n" + " default_exception_hook(ex_class,ex,trace)") + console.runcode("sys.excepthook=filtered_exception_hook") # Launch shell diff --git a/Firmware/fibre/python/setup.py b/Firmware/fibre/python/setup.py index 85f266f3..9edccf6b 100644 --- a/Firmware/fibre/python/setup.py +++ b/Firmware/fibre/python/setup.py @@ -76,7 +76,9 @@ setup( license='MIT', url = 'https://github.com/samuelsadok/fibre', keywords = ['communication', 'transport-layer', 'rpc'], - install_requires = [], + install_requires = [ + 'appdirs', # Used to find caching directory + ], #package_data={'': ['version.txt']}, classifiers = [], ) diff --git a/docs/ascii-protocol.md b/docs/ascii-protocol.md index 5e1dc236..b82863e0 100644 --- a/docs/ascii-protocol.md +++ b/docs/ascii-protocol.md @@ -5,11 +5,13 @@ * **Via USB:** * **Windows:** Use the Zadig utility to set the ODrive's driver to "usbser". Windows will then make the device available as COM port. You can use [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/) to manually send commands or open the COM port using your favorite programming language - * **Linux/macOS:** Run `/dev/tty*` to list all serial ports. The ODrive will show up as `/dev/ttyACM0` on Linux and `/dev/tty.usbmodem[...]` on macOS. Once you know the name, you can use `screen /dev/ttyACM0` (with the correct name) to send commands manually or open the device using your favorite programming language. Serial ports on Unix can be opened, written to and read from like a normal file. + * **Linux/macOS:** Run `/dev/tty*` to list all serial ports. The ODrive will show up as `/dev/ttyACM0` (or similar) on Linux and `/dev/tty.usbmodem[...]` on macOS. Once you know the name, you can use `screen /dev/ttyACM0` (with the correct name) to send commands manually or open the device using your favorite programming language. Serial ports on Unix can be opened, written to and read from like a normal file. * **Via UART:** Connect the ODrive's TX (GPIO1) to your host's RX. Connect your ODrive's RX (GPIO2) to your host's TX. The logic level of the ODrive is 3.3V. * **Arduino:** You can use the [ODrive Arduino library](https://github.com/madcowswe/ODrive/tree/master/Arduino/ODriveArduino) to talk to the ODrive. * **Windows/Linux/macOS:** You can use an FTDI USB-UART cable to connect to the ODrive. +The ODrive does not echo commands. That means that when you type commands into a program like `screen`, the characters you type won't show up in the console. + ## Command format The ASCII protocol is human-readable and line-oriented, with each line having the following format: @@ -18,7 +20,7 @@ The ASCII protocol is human-readable and line-oriented, with each line having th command *42 ; comment [new line character] ``` - * `*42` stands for a GCode compatible checksum and can be omitted. If and only if a checksum is provided, the device will also include a checksum in the response, if any. + * `*42` stands for a GCode compatible checksum and can be omitted. If and only if a checksum is provided, the device will also include a checksum in the response, if any. The checksum is calculated as the bitwise xor of all characters before the asterisk (`*`).
Example of a valid checksum: `r vbus_voltage *93`. * comments are supported for GCode compatibility * the command is interpreted once the new-line character is encountered