From ce6c122449c2aff762caa140bbc251aaca84b6dc Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:16:57 +1200 Subject: [PATCH] [core] Move JSON string escaping into helpers (#17879) --- esphome/components/ble_scanner/ble_scanner.h | 20 ++--- .../captive_portal/captive_portal.cpp | 3 +- .../components/captive_portal/json_escape.h | 85 ------------------- esphome/core/helpers.cpp | 66 ++++++++++++++ esphome/core/helpers.h | 16 ++++ tests/components/captive_portal/__init__.py | 23 ----- .../json_escape_test.cpp | 55 +++++++++--- 7 files changed, 130 insertions(+), 138 deletions(-) delete mode 100644 esphome/components/captive_portal/json_escape.h delete mode 100644 tests/components/captive_portal/__init__.py rename tests/components/{captive_portal => core}/json_escape_test.cpp (66%) diff --git a/esphome/components/ble_scanner/ble_scanner.h b/esphome/components/ble_scanner/ble_scanner.h index c70ee637ef6..106171d38f6 100644 --- a/esphome/components/ble_scanner/ble_scanner.h +++ b/esphome/components/ble_scanner/ble_scanner.h @@ -5,6 +5,8 @@ #include #include "esphome/core/component.h" +#include "esphome/core/helpers.h" +#include "esphome/core/string_ref.h" #include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h" #include "esphome/components/text_sensor/text_sensor.h" @@ -18,22 +20,10 @@ class BLEScanner final : public text_sensor::TextSensor, public: bool parse_device(const esp32_ble_tracker::ESPBTDevice &device) override { char addr_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; - // Escape special characters in the device name for valid JSON - const char *name = device.get_name().c_str(); + // Escape special characters in the device name for valid JSON. Control characters stay in the \u00XX form this + // sensor has always published. char escaped_name[128]; - size_t pos = 0; - for (; *name != '\0' && pos < sizeof(escaped_name) - 7; name++) { - uint8_t c = static_cast(*name); - if (c == '"' || c == '\\') { - escaped_name[pos++] = '\\'; - escaped_name[pos++] = c; - } else if (c < 0x20) { - pos += snprintf(escaped_name + pos, sizeof(escaped_name) - pos, "\\u%04x", c); - } else { - escaped_name[pos++] = c; - } - } - escaped_name[pos] = '\0'; + json_escape_into_buffer(escaped_name, StringRef(device.get_name()), /*short_control_escapes=*/false); char buf[256]; snprintf(buf, sizeof(buf), "{\"timestamp\":%" PRId64 ",\"address\":\"%s\",\"rssi\":%d,\"name\":\"%s\"}", diff --git a/esphome/components/captive_portal/captive_portal.cpp b/esphome/components/captive_portal/captive_portal.cpp index e6a63b8275d..80949030081 100644 --- a/esphome/components/captive_portal/captive_portal.cpp +++ b/esphome/components/captive_portal/captive_portal.cpp @@ -2,9 +2,10 @@ #ifdef USE_CAPTIVE_PORTAL #include "esphome/core/log.h" #include "esphome/core/application.h" +#include "esphome/core/helpers.h" +#include "esphome/core/string_ref.h" #include "esphome/components/wifi/wifi_component.h" #include "captive_index.h" -#include "json_escape.h" namespace esphome::captive_portal { diff --git a/esphome/components/captive_portal/json_escape.h b/esphome/components/captive_portal/json_escape.h deleted file mode 100644 index 0b3c71cd74f..00000000000 --- a/esphome/components/captive_portal/json_escape.h +++ /dev/null @@ -1,85 +0,0 @@ -#pragma once -#include -#include -#include - -#include "esphome/core/helpers.h" -#include "esphome/core/string_ref.h" - -namespace esphome::captive_portal { - -/// Largest number of output bytes a single input byte can expand to (a \u00XX sequence). -static constexpr size_t JSON_ESCAPE_MAX_EXPANSION = 6; - -/// Copy value into buf, escaping the characters that cannot appear raw inside a JSON string literal. -/// -/// Escapes " and \ along with the control characters below 0x20, using the short forms where JSON defines one and -/// \u00XX otherwise. Bytes >= 0x20 are copied verbatim, so text containing valid UTF-8 survives intact. The result is -/// always null terminated; anything that would not fit is dropped rather than written partially. Returns buf so the -/// call can be used directly as an argument. -/// -/// To size buf so that no input is ever dropped, allow JSON_ESCAPE_MAX_EXPANSION bytes per input byte plus one for -/// the null terminator. -inline const char *json_escape_into_buffer(std::span buf, StringRef value) { - if (buf.empty()) - return ""; - // Reserve one byte for the null terminator. - const size_t limit = buf.size() - 1; - size_t pos = 0; - for (char ch : value) { - auto c = static_cast(ch); - // Every short form is a backslash followed by a single character, so only that character is needed here. Keeping - // it a char rather than a string avoids putting the sequences in read only data, which is RAM on the ESP8266. - char escape = '\0'; - switch (c) { - case '"': - escape = '"'; - break; - case '\\': - escape = '\\'; - break; - case '\n': - escape = 'n'; - break; - case '\r': - escape = 'r'; - break; - case '\t': - escape = 't'; - break; - case '\b': - escape = 'b'; - break; - case '\f': - escape = 'f'; - break; - default: - break; - } - if (escape != '\0') { - if (pos + 2 > limit) - break; - buf[pos++] = '\\'; - buf[pos++] = escape; - } else if (c < 0x20) { - // Remaining control characters have no short form and must be written as \u00XX. The value is below 0x20, so - // the two high hex digits are always zero. - if (pos + JSON_ESCAPE_MAX_EXPANSION > limit) - break; - buf[pos++] = '\\'; - buf[pos++] = 'u'; - buf[pos++] = '0'; - buf[pos++] = '0'; - buf[pos++] = format_hex_char(static_cast(c >> 4)); - buf[pos++] = format_hex_char(static_cast(c & 0x0F)); - } else { - if (pos + 1 > limit) - break; - buf[pos++] = static_cast(c); - } - } - buf[pos] = '\0'; - return buf.data(); -} - -} // namespace esphome::captive_portal diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index a7b63643a40..c8cf85d7d68 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -335,6 +335,72 @@ char *format_hex_to(char *buffer, size_t buffer_size, const uint8_t *data, size_ return format_hex_internal(buffer, buffer_size, data, length, 0, 'a'); } +const char *json_escape_into_buffer(std::span buf, StringRef value, bool short_control_escapes) { + if (buf.empty()) + return ""; + // Reserve one byte for the null terminator. + const size_t limit = buf.size() - 1; + size_t pos = 0; + for (char ch : value) { + auto c = static_cast(ch); + // Every short form is a backslash followed by a single character, so only that character is needed here. Keeping + // it a char rather than a string avoids putting the sequences in read only data, which is RAM on the ESP8266. + char escape = '\0'; + switch (c) { + case '"': + escape = '"'; + break; + case '\\': + escape = '\\'; + break; + case '\n': + escape = 'n'; + break; + case '\r': + escape = 'r'; + break; + case '\t': + escape = 't'; + break; + case '\b': + escape = 'b'; + break; + case '\f': + escape = 'f'; + break; + default: + break; + } + // " and \ are always written as two characters, but the control characters fall through to \u00XX when the + // caller did not ask for the short forms. + if (!short_control_escapes && c < 0x20) + escape = '\0'; + if (escape != '\0') { + if (pos + 2 > limit) + break; + buf[pos++] = '\\'; + buf[pos++] = escape; + } else if (c < 0x20) { + // Remaining control characters have no short form and must be written as \u00XX. The value is below 0x20, so + // the two high hex digits are always zero. + if (pos + JSON_ESCAPE_MAX_EXPANSION > limit) + break; + buf[pos++] = '\\'; + buf[pos++] = 'u'; + buf[pos++] = '0'; + buf[pos++] = '0'; + buf[pos++] = format_hex_char(static_cast(c >> 4)); + buf[pos++] = format_hex_char(static_cast(c & 0x0F)); + } else { + if (pos + 1 > limit) + break; + buf[pos++] = static_cast(c); + } + } + buf[pos] = '\0'; + return buf.data(); +} + // format_hex (std::string returning overloads) moved to alloc_helpers.cpp char *format_hex_pretty_to(char *buffer, size_t buffer_size, const uint8_t *data, size_t length, char separator) { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index b897b927e24..7940df8780e 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -1268,6 +1268,22 @@ ESPHOME_ALWAYS_INLINE inline char format_hex_char(uint8_t v) { return format_hex /// Convert a nibble (0-15) to uppercase hex char (used for pretty printing) ESPHOME_ALWAYS_INLINE inline char format_hex_pretty_char(uint8_t v) { return format_hex_char(v, 'A'); } +/// Largest number of output bytes a single input byte can expand to when JSON escaped (a \u00XX sequence). +static constexpr size_t JSON_ESCAPE_MAX_EXPANSION = 6; + +/// Copy value into buf, escaping the characters that cannot appear raw inside a JSON string literal. +/// +/// Escapes " and \ along with the control characters below 0x20. Bytes >= 0x20 are copied verbatim, so text +/// containing valid UTF-8 survives intact. The result is always null terminated; anything that would not fit is +/// dropped rather than written partially. Returns buf so the call can be used directly as an argument. +/// +/// With short_control_escapes the five control characters JSON gives a short form get it (\n \r \t \b \f) and the +/// rest become \u00XX. Pass false to write every control character as \u00XX, which some consumers expect. +/// +/// To size buf so that no input is ever dropped, allow JSON_ESCAPE_MAX_EXPANSION bytes per input byte plus one for +/// the null terminator. +const char *json_escape_into_buffer(std::span buf, StringRef value, bool short_control_escapes = true); + /// Write int8 value to buffer without modulo operations. /// Buffer must have at least 4 bytes free. Returns pointer past last char written. inline char *int8_to_str(char *buf, int8_t val) { diff --git a/tests/components/captive_portal/__init__.py b/tests/components/captive_portal/__init__.py deleted file mode 100644 index b13c81912c8..00000000000 --- a/tests/components/captive_portal/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Test-manifest overrides for the captive_portal C++ unit tests. - -``json_escape`` lives in a standalone, dependency-free header -(``esphome/components/captive_portal/json_escape.h``). The rest of the -captive_portal component and its auto-loaded dependencies (``web_server_base``, -``ota.web_server``) do not build for the ``host`` platform that the C++ unit -test harness targets. Strip those away and replace the real schema -- which is -restricted to non-host platforms via ``cv.only_on`` and requires a -``web_server_base`` instance via ``use_id`` -- with an empty one so the host -test config validates. ``to_code`` stays suppressed (the default), so -``USE_CAPTIVE_PORTAL`` is never defined and ``captive_portal.cpp`` compiles to an -empty translation unit; only ``json_escape.h`` is exercised by the test. -""" - -import esphome.config_validation as cv -from tests.testing_helpers import ComponentManifestOverride - - -def override_manifest(manifest: ComponentManifestOverride) -> None: - manifest.auto_load = [] - manifest.dependencies = [] - manifest.config_schema = cv.Schema({}) - manifest.final_validate_schema = None diff --git a/tests/components/captive_portal/json_escape_test.cpp b/tests/components/core/json_escape_test.cpp similarity index 66% rename from tests/components/captive_portal/json_escape_test.cpp rename to tests/components/core/json_escape_test.cpp index 98b5ce4ff79..4db6dce2eae 100644 --- a/tests/components/captive_portal/json_escape_test.cpp +++ b/tests/components/core/json_escape_test.cpp @@ -2,9 +2,10 @@ #include -#include "esphome/components/captive_portal/json_escape.h" +#include "esphome/core/helpers.h" +#include "esphome/core/string_ref.h" -namespace esphome::captive_portal::testing { +namespace esphome::testing { namespace { @@ -17,30 +18,36 @@ std::string escape(const std::string &value) { return json_escape_into_buffer(buf, StringRef(value.c_str(), value.size())); } +// Same, but with the short control forms turned off. +std::string escape_long(const std::string &value) { + char buf[TEST_BUFFER_SIZE]; + return json_escape_into_buffer(buf, StringRef(value.c_str(), value.size()), false); +} + } // namespace // Plain ASCII with no special characters is passed through unchanged. -TEST(CaptivePortalJsonEscape, PlainStringUnchanged) { +TEST(JsonEscape, PlainStringUnchanged) { EXPECT_EQ(escape("MyNetwork"), "MyNetwork"); EXPECT_EQ(escape(""), ""); } // A double quote is escaped so it does not terminate the surrounding JSON string. -TEST(CaptivePortalJsonEscape, EscapesDoubleQuote) { +TEST(JsonEscape, EscapesDoubleQuote) { EXPECT_EQ(escape("a\"b"), "a\\\"b"); // A double quote followed by other characters stays inside the JSON string. EXPECT_EQ(escape("\">end"), "\\\">end"); } // A backslash is doubled so it does not start an escape sequence in the output. -TEST(CaptivePortalJsonEscape, EscapesBackslash) { +TEST(JsonEscape, EscapesBackslash) { EXPECT_EQ(escape("a\\b"), "a\\\\b"); // A trailing backslash must not escape the closing quote of the JSON string. EXPECT_EQ(escape("net\\"), "net\\\\"); } // The control characters with short JSON forms use those forms. -TEST(CaptivePortalJsonEscape, EscapesShortFormControls) { +TEST(JsonEscape, EscapesShortFormControls) { EXPECT_EQ(escape("\n"), "\\n"); EXPECT_EQ(escape("\r"), "\\r"); EXPECT_EQ(escape("\t"), "\\t"); @@ -49,7 +56,7 @@ TEST(CaptivePortalJsonEscape, EscapesShortFormControls) { } // Other control characters (< 0x20) without a short form become \u00XX with lowercase hex. -TEST(CaptivePortalJsonEscape, EscapesOtherControlsAsUnicode) { +TEST(JsonEscape, EscapesOtherControlsAsUnicode) { EXPECT_EQ(escape(std::string("\x00", 1)), "\\u0000"); EXPECT_EQ(escape("\x01"), "\\u0001"); EXPECT_EQ(escape("\x10"), "\\u0010"); @@ -58,8 +65,28 @@ TEST(CaptivePortalJsonEscape, EscapesOtherControlsAsUnicode) { EXPECT_EQ(escape("\x7f"), "\x7f"); } +// With the short forms turned off, every control character is written as \u00XX instead. +TEST(JsonEscape, LongControlEscapes) { + EXPECT_EQ(escape_long("\n"), "\\u000a"); + EXPECT_EQ(escape_long("\r"), "\\u000d"); + EXPECT_EQ(escape_long("\t"), "\\u0009"); + EXPECT_EQ(escape_long("\b"), "\\u0008"); + EXPECT_EQ(escape_long("\f"), "\\u000c"); + // Controls without a short form are unaffected by the flag. + EXPECT_EQ(escape_long("\x01"), "\\u0001"); +} + +// The flag only affects control characters. A quote or backslash is never written as \u00XX, because that form is +// no shorter and both modes have always emitted the two character escape. +TEST(JsonEscape, LongModeStillUsesTwoCharQuoteAndBackslash) { + EXPECT_EQ(escape_long("a\"b"), "a\\\"b"); + EXPECT_EQ(escape_long("a\\b"), "a\\\\b"); + // Ordinary text is untouched in either mode. + EXPECT_EQ(escape_long("MyDevice"), "MyDevice"); +} + // Bytes >= 0x20, including multi-byte UTF-8 sequences, are passed through verbatim. -TEST(CaptivePortalJsonEscape, PassesThroughUtf8) { +TEST(JsonEscape, PassesThroughUtf8) { // "café" in UTF-8 (é == 0xC3 0xA9). EXPECT_EQ(escape("caf\xc3\xa9"), "caf\xc3\xa9"); // Emoji (📶, 4-byte UTF-8) survives unchanged. @@ -67,10 +94,10 @@ TEST(CaptivePortalJsonEscape, PassesThroughUtf8) { } // A mix of special and normal characters is escaped in place without disturbing the rest. -TEST(CaptivePortalJsonEscape, MixedContent) { EXPECT_EQ(escape("a\"b\\c\nd"), "a\\\"b\\\\c\\nd"); } +TEST(JsonEscape, MixedContent) { EXPECT_EQ(escape("a\"b\\c\nd"), "a\\\"b\\\\c\\nd"); } // A buffer sized at JSON_ESCAPE_MAX_EXPANSION bytes per input byte holds the worst case exactly. -TEST(CaptivePortalJsonEscape, WorstCaseInputFitsExactly) { +TEST(JsonEscape, WorstCaseInputFitsExactly) { constexpr size_t input_len = 8; char buf[input_len * JSON_ESCAPE_MAX_EXPANSION + 1]; const std::string input(input_len, '\x01'); @@ -82,7 +109,7 @@ TEST(CaptivePortalJsonEscape, WorstCaseInputFitsExactly) { // An escape sequence that would not fit is dropped whole rather than written partially, and the result stays null // terminated. -TEST(CaptivePortalJsonEscape, DropsEscapeThatWouldNotFit) { +TEST(JsonEscape, DropsEscapeThatWouldNotFit) { // Room for one \u00XX sequence plus the null terminator, but two are requested. char buf[JSON_ESCAPE_MAX_EXPANSION + 1]; const std::string input(2, '\x01'); @@ -92,16 +119,16 @@ TEST(CaptivePortalJsonEscape, DropsEscapeThatWouldNotFit) { } // Plain characters are truncated at the buffer size, leaving room for the null terminator. -TEST(CaptivePortalJsonEscape, TruncatesPlainInput) { +TEST(JsonEscape, TruncatesPlainInput) { char buf[5]; const std::string input(20, 'a'); EXPECT_STREQ(json_escape_into_buffer(buf, StringRef(input.c_str(), input.size())), "aaaa"); } // A zero length buffer cannot even hold a null terminator, so an empty string is returned instead of writing. -TEST(CaptivePortalJsonEscape, EmptyBufferIsSafe) { +TEST(JsonEscape, EmptyBufferIsSafe) { const std::string input("test"); EXPECT_STREQ(json_escape_into_buffer(std::span(), StringRef(input.c_str(), input.size())), ""); } -} // namespace esphome::captive_portal::testing +} // namespace esphome::testing