[core] Move JSON string escaping into helpers (#17879)

esphome/components/ble_scanner/ble_scanner.h

@@ -5,6 +5,8 @@
 #include <ctime>
 
 #include "esphome/core/component.h"
+#include "esphome/core/helpers.h"
+#include "esphome/core/string_ref.h"
 #include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h"
 #include "esphome/components/text_sensor/text_sensor.h"
 
@@ -18,22 +20,10 @@ class BLEScanner final : public text_sensor::TextSensor,
  public:
   bool parse_device(const esp32_ble_tracker::ESPBTDevice &device) override {
     char addr_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE];
-    // Escape special characters in the device name for valid JSON
-    const char *name = device.get_name().c_str();
+    // Escape special characters in the device name for valid JSON. Control characters stay in the \u00XX form this
+    // sensor has always published.
     char escaped_name[128];
-    size_t pos = 0;
-    for (; *name != '\0' && pos < sizeof(escaped_name) - 7; name++) {
-      uint8_t c = static_cast<uint8_t>(*name);
-      if (c == '"' || c == '\\') {
-        escaped_name[pos++] = '\\';
-        escaped_name[pos++] = c;
-      } else if (c < 0x20) {
-        pos += snprintf(escaped_name + pos, sizeof(escaped_name) - pos, "\\u%04x", c);
-      } else {
-        escaped_name[pos++] = c;
-      }
-    }
-    escaped_name[pos] = '\0';
+    json_escape_into_buffer(escaped_name, StringRef(device.get_name()), /*short_control_escapes=*/false);
 
     char buf[256];
     snprintf(buf, sizeof(buf), "{\"timestamp\":%" PRId64 ",\"address\":\"%s\",\"rssi\":%d,\"name\":\"%s\"}",

esphome/components/captive_portal/captive_portal.cpp

@@ -2,9 +2,10 @@
 #ifdef USE_CAPTIVE_PORTAL
 #include "esphome/core/log.h"
 #include "esphome/core/application.h"
+#include "esphome/core/helpers.h"
+#include "esphome/core/string_ref.h"
 #include "esphome/components/wifi/wifi_component.h"
 #include "captive_index.h"
-#include "json_escape.h"
 
 namespace esphome::captive_portal {
 

esphome/components/captive_portal/json_escape.h

@@ -1,85 +0,0 @@
-#pragma once
-#include <cstddef>
-#include <cstdint>
-#include <span>
-
-#include "esphome/core/helpers.h"
-#include "esphome/core/string_ref.h"
-
-namespace esphome::captive_portal {
-
-/// Largest number of output bytes a single input byte can expand to (a \u00XX sequence).
-static constexpr size_t JSON_ESCAPE_MAX_EXPANSION = 6;
-
-/// Copy value into buf, escaping the characters that cannot appear raw inside a JSON string literal.
-///
-/// Escapes " and \ along with the control characters below 0x20, using the short forms where JSON defines one and
-/// \u00XX otherwise. Bytes >= 0x20 are copied verbatim, so text containing valid UTF-8 survives intact. The result is
-/// always null terminated; anything that would not fit is dropped rather than written partially. Returns buf so the
-/// call can be used directly as an argument.
-///
-/// To size buf so that no input is ever dropped, allow JSON_ESCAPE_MAX_EXPANSION bytes per input byte plus one for
-/// the null terminator.
-inline const char *json_escape_into_buffer(std::span<char> buf, StringRef value) {
-  if (buf.empty())
-    return "";
-  // Reserve one byte for the null terminator.
-  const size_t limit = buf.size() - 1;
-  size_t pos = 0;
-  for (char ch : value) {
-    auto c = static_cast<unsigned char>(ch);
-    // Every short form is a backslash followed by a single character, so only that character is needed here. Keeping
-    // it a char rather than a string avoids putting the sequences in read only data, which is RAM on the ESP8266.
-    char escape = '\0';
-    switch (c) {
-      case '"':
-        escape = '"';
-        break;
-      case '\\':
-        escape = '\\';
-        break;
-      case '\n':
-        escape = 'n';
-        break;
-      case '\r':
-        escape = 'r';
-        break;
-      case '\t':
-        escape = 't';
-        break;
-      case '\b':
-        escape = 'b';
-        break;
-      case '\f':
-        escape = 'f';
-        break;
-      default:
-        break;
-    }
-    if (escape != '\0') {
-      if (pos + 2 > limit)
-        break;
-      buf[pos++] = '\\';
-      buf[pos++] = escape;
-    } else if (c < 0x20) {
-      // Remaining control characters have no short form and must be written as \u00XX. The value is below 0x20, so
-      // the two high hex digits are always zero.
-      if (pos + JSON_ESCAPE_MAX_EXPANSION > limit)
-        break;
-      buf[pos++] = '\\';
-      buf[pos++] = 'u';
-      buf[pos++] = '0';
-      buf[pos++] = '0';
-      buf[pos++] = format_hex_char(static_cast<uint8_t>(c >> 4));
-      buf[pos++] = format_hex_char(static_cast<uint8_t>(c & 0x0F));
-    } else {
-      if (pos + 1 > limit)
-        break;
-      buf[pos++] = static_cast<char>(c);
-    }
-  }
-  buf[pos] = '\0';
-  return buf.data();
-}
-
-}  // namespace esphome::captive_portal

esphome/core/helpers.cpp

@@ -335,6 +335,72 @@ char *format_hex_to(char *buffer, size_t buffer_size, const uint8_t *data, size_
   return format_hex_internal(buffer, buffer_size, data, length, 0, 'a');
 }
 
+const char *json_escape_into_buffer(std::span<char> buf, StringRef value, bool short_control_escapes) {
+  if (buf.empty())
+    return "";
+  // Reserve one byte for the null terminator.
+  const size_t limit = buf.size() - 1;
+  size_t pos = 0;
+  for (char ch : value) {
+    auto c = static_cast<unsigned char>(ch);
+    // Every short form is a backslash followed by a single character, so only that character is needed here. Keeping
+    // it a char rather than a string avoids putting the sequences in read only data, which is RAM on the ESP8266.
+    char escape = '\0';
+    switch (c) {
+      case '"':
+        escape = '"';
+        break;
+      case '\\':
+        escape = '\\';
+        break;
+      case '\n':
+        escape = 'n';
+        break;
+      case '\r':
+        escape = 'r';
+        break;
+      case '\t':
+        escape = 't';
+        break;
+      case '\b':
+        escape = 'b';
+        break;
+      case '\f':
+        escape = 'f';
+        break;
+      default:
+        break;
+    }
+    // " and \ are always written as two characters, but the control characters fall through to \u00XX when the
+    // caller did not ask for the short forms.
+    if (!short_control_escapes && c < 0x20)
+      escape = '\0';
+    if (escape != '\0') {
+      if (pos + 2 > limit)
+        break;
+      buf[pos++] = '\\';
+      buf[pos++] = escape;
+    } else if (c < 0x20) {
+      // Remaining control characters have no short form and must be written as \u00XX. The value is below 0x20, so
+      // the two high hex digits are always zero.
+      if (pos + JSON_ESCAPE_MAX_EXPANSION > limit)
+        break;
+      buf[pos++] = '\\';
+      buf[pos++] = 'u';
+      buf[pos++] = '0';
+      buf[pos++] = '0';
+      buf[pos++] = format_hex_char(static_cast<uint8_t>(c >> 4));
+      buf[pos++] = format_hex_char(static_cast<uint8_t>(c & 0x0F));
+    } else {
+      if (pos + 1 > limit)
+        break;
+      buf[pos++] = static_cast<char>(c);
+    }
+  }
+  buf[pos] = '\0';
+  return buf.data();
+}
+
 // format_hex (std::string returning overloads) moved to alloc_helpers.cpp
 
 char *format_hex_pretty_to(char *buffer, size_t buffer_size, const uint8_t *data, size_t length, char separator) {

esphome/core/helpers.h

@@ -1268,6 +1268,22 @@ ESPHOME_ALWAYS_INLINE inline char format_hex_char(uint8_t v) { return format_hex
 /// Convert a nibble (0-15) to uppercase hex char (used for pretty printing)
 ESPHOME_ALWAYS_INLINE inline char format_hex_pretty_char(uint8_t v) { return format_hex_char(v, 'A'); }
 
+/// Largest number of output bytes a single input byte can expand to when JSON escaped (a \u00XX sequence).
+static constexpr size_t JSON_ESCAPE_MAX_EXPANSION = 6;
+
+/// Copy value into buf, escaping the characters that cannot appear raw inside a JSON string literal.
+///
+/// Escapes " and \ along with the control characters below 0x20. Bytes >= 0x20 are copied verbatim, so text
+/// containing valid UTF-8 survives intact. The result is always null terminated; anything that would not fit is
+/// dropped rather than written partially. Returns buf so the call can be used directly as an argument.
+///
+/// With short_control_escapes the five control characters JSON gives a short form get it (\n \r \t \b \f) and the
+/// rest become \u00XX. Pass false to write every control character as \u00XX, which some consumers expect.
+///
+/// To size buf so that no input is ever dropped, allow JSON_ESCAPE_MAX_EXPANSION bytes per input byte plus one for
+/// the null terminator.
+const char *json_escape_into_buffer(std::span<char> buf, StringRef value, bool short_control_escapes = true);
+
 /// Write int8 value to buffer without modulo operations.
 /// Buffer must have at least 4 bytes free. Returns pointer past last char written.
 inline char *int8_to_str(char *buf, int8_t val) {

tests/components/captive_portal/__init__.py

@@ -1,23 +0,0 @@
-"""Test-manifest overrides for the captive_portal C++ unit tests.
-
-``json_escape`` lives in a standalone, dependency-free header
-(``esphome/components/captive_portal/json_escape.h``). The rest of the
-captive_portal component and its auto-loaded dependencies (``web_server_base``,
-``ota.web_server``) do not build for the ``host`` platform that the C++ unit
-test harness targets. Strip those away and replace the real schema -- which is
-restricted to non-host platforms via ``cv.only_on`` and requires a
-``web_server_base`` instance via ``use_id`` -- with an empty one so the host
-test config validates. ``to_code`` stays suppressed (the default), so
-``USE_CAPTIVE_PORTAL`` is never defined and ``captive_portal.cpp`` compiles to an
-empty translation unit; only ``json_escape.h`` is exercised by the test.
-"""
-
-import esphome.config_validation as cv
-from tests.testing_helpers import ComponentManifestOverride
-
-
-def override_manifest(manifest: ComponentManifestOverride) -> None:
-    manifest.auto_load = []
-    manifest.dependencies = []
-    manifest.config_schema = cv.Schema({})
-    manifest.final_validate_schema = None

tests/components/core/json_escape_test.cpp

@@ -2,9 +2,10 @@
 
 #include <string>
 
-#include "esphome/components/captive_portal/json_escape.h"
+#include "esphome/core/helpers.h"
+#include "esphome/core/string_ref.h"
 
-namespace esphome::captive_portal::testing {
+namespace esphome::testing {
 
 namespace {
 
@@ -17,30 +18,36 @@ std::string escape(const std::string &value) {
   return json_escape_into_buffer(buf, StringRef(value.c_str(), value.size()));
 }
 
+// Same, but with the short control forms turned off.
+std::string escape_long(const std::string &value) {
+  char buf[TEST_BUFFER_SIZE];
+  return json_escape_into_buffer(buf, StringRef(value.c_str(), value.size()), false);
+}
+
 }  // namespace
 
 // Plain ASCII with no special characters is passed through unchanged.
-TEST(CaptivePortalJsonEscape, PlainStringUnchanged) {
+TEST(JsonEscape, PlainStringUnchanged) {
   EXPECT_EQ(escape("MyNetwork"), "MyNetwork");
   EXPECT_EQ(escape(""), "");
 }
 
 // A double quote is escaped so it does not terminate the surrounding JSON string.
-TEST(CaptivePortalJsonEscape, EscapesDoubleQuote) {
+TEST(JsonEscape, EscapesDoubleQuote) {
   EXPECT_EQ(escape("a\"b"), "a\\\"b");
   // A double quote followed by other characters stays inside the JSON string.
   EXPECT_EQ(escape("\">end"), "\\\">end");
 }
 
 // A backslash is doubled so it does not start an escape sequence in the output.
-TEST(CaptivePortalJsonEscape, EscapesBackslash) {
+TEST(JsonEscape, EscapesBackslash) {
   EXPECT_EQ(escape("a\\b"), "a\\\\b");
   // A trailing backslash must not escape the closing quote of the JSON string.
   EXPECT_EQ(escape("net\\"), "net\\\\");
 }
 
 // The control characters with short JSON forms use those forms.
-TEST(CaptivePortalJsonEscape, EscapesShortFormControls) {
+TEST(JsonEscape, EscapesShortFormControls) {
   EXPECT_EQ(escape("\n"), "\\n");
   EXPECT_EQ(escape("\r"), "\\r");
   EXPECT_EQ(escape("\t"), "\\t");
@@ -49,7 +56,7 @@ TEST(CaptivePortalJsonEscape, EscapesShortFormControls) {
 }
 
 // Other control characters (< 0x20) without a short form become \u00XX with lowercase hex.
-TEST(CaptivePortalJsonEscape, EscapesOtherControlsAsUnicode) {
+TEST(JsonEscape, EscapesOtherControlsAsUnicode) {
   EXPECT_EQ(escape(std::string("\x00", 1)), "\\u0000");
   EXPECT_EQ(escape("\x01"), "\\u0001");
   EXPECT_EQ(escape("\x10"), "\\u0010");
@@ -58,8 +65,28 @@ TEST(CaptivePortalJsonEscape, EscapesOtherControlsAsUnicode) {
   EXPECT_EQ(escape("\x7f"), "\x7f");
 }
 
+// With the short forms turned off, every control character is written as \u00XX instead.
+TEST(JsonEscape, LongControlEscapes) {
+  EXPECT_EQ(escape_long("\n"), "\\u000a");
+  EXPECT_EQ(escape_long("\r"), "\\u000d");
+  EXPECT_EQ(escape_long("\t"), "\\u0009");
+  EXPECT_EQ(escape_long("\b"), "\\u0008");
+  EXPECT_EQ(escape_long("\f"), "\\u000c");
+  // Controls without a short form are unaffected by the flag.
+  EXPECT_EQ(escape_long("\x01"), "\\u0001");
+}
+
+// The flag only affects control characters. A quote or backslash is never written as \u00XX, because that form is
+// no shorter and both modes have always emitted the two character escape.
+TEST(JsonEscape, LongModeStillUsesTwoCharQuoteAndBackslash) {
+  EXPECT_EQ(escape_long("a\"b"), "a\\\"b");
+  EXPECT_EQ(escape_long("a\\b"), "a\\\\b");
+  // Ordinary text is untouched in either mode.
+  EXPECT_EQ(escape_long("MyDevice"), "MyDevice");
+}
+
 // Bytes >= 0x20, including multi-byte UTF-8 sequences, are passed through verbatim.
-TEST(CaptivePortalJsonEscape, PassesThroughUtf8) {
+TEST(JsonEscape, PassesThroughUtf8) {
   // "café" in UTF-8 (é == 0xC3 0xA9).
   EXPECT_EQ(escape("caf\xc3\xa9"), "caf\xc3\xa9");
   // Emoji (📶, 4-byte UTF-8) survives unchanged.
@@ -67,10 +94,10 @@ TEST(CaptivePortalJsonEscape, PassesThroughUtf8) {
 }
 
 // A mix of special and normal characters is escaped in place without disturbing the rest.
-TEST(CaptivePortalJsonEscape, MixedContent) { EXPECT_EQ(escape("a\"b\\c\nd"), "a\\\"b\\\\c\\nd"); }
+TEST(JsonEscape, MixedContent) { EXPECT_EQ(escape("a\"b\\c\nd"), "a\\\"b\\\\c\\nd"); }
 
 // A buffer sized at JSON_ESCAPE_MAX_EXPANSION bytes per input byte holds the worst case exactly.
-TEST(CaptivePortalJsonEscape, WorstCaseInputFitsExactly) {
+TEST(JsonEscape, WorstCaseInputFitsExactly) {
   constexpr size_t input_len = 8;
   char buf[input_len * JSON_ESCAPE_MAX_EXPANSION + 1];
   const std::string input(input_len, '\x01');
@@ -82,7 +109,7 @@ TEST(CaptivePortalJsonEscape, WorstCaseInputFitsExactly) {
 
 // An escape sequence that would not fit is dropped whole rather than written partially, and the result stays null
 // terminated.
-TEST(CaptivePortalJsonEscape, DropsEscapeThatWouldNotFit) {
+TEST(JsonEscape, DropsEscapeThatWouldNotFit) {
   // Room for one \u00XX sequence plus the null terminator, but two are requested.
   char buf[JSON_ESCAPE_MAX_EXPANSION + 1];
   const std::string input(2, '\x01');
@@ -92,16 +119,16 @@ TEST(CaptivePortalJsonEscape, DropsEscapeThatWouldNotFit) {
 }
 
 // Plain characters are truncated at the buffer size, leaving room for the null terminator.
-TEST(CaptivePortalJsonEscape, TruncatesPlainInput) {
+TEST(JsonEscape, TruncatesPlainInput) {
   char buf[5];
   const std::string input(20, 'a');
   EXPECT_STREQ(json_escape_into_buffer(buf, StringRef(input.c_str(), input.size())), "aaaa");
 }
 
 // A zero length buffer cannot even hold a null terminator, so an empty string is returned instead of writing.
-TEST(CaptivePortalJsonEscape, EmptyBufferIsSafe) {
+TEST(JsonEscape, EmptyBufferIsSafe) {
   const std::string input("test");
   EXPECT_STREQ(json_escape_into_buffer(std::span<char>(), StringRef(input.c_str(), input.size())), "");
 }
 
-}  // namespace esphome::captive_portal::testing
+}  // namespace esphome::testing