[api] Single-pass protobuf encode for BLE proxy advertisements (#14575)

esphome/components/api/api_pb2.cpp

@@ -120,16 +120,16 @@ void DeviceInfoResponse::encode(ProtoWriteBuffer &buffer) const {
 #endif
 #ifdef USE_DEVICES
   for (const auto &it : this->devices) {
-    buffer.encode_message(20, it);
+    buffer.encode_sub_message(20, it);
   }
 #endif
 #ifdef USE_AREAS
   for (const auto &it : this->areas) {
-    buffer.encode_message(21, it);
+    buffer.encode_sub_message(21, it);
   }
 #endif
 #ifdef USE_AREAS
-  buffer.encode_message(22, this->area, false);
+  buffer.encode_optional_sub_message(22, this->area);
 #endif
 #ifdef USE_ZWAVE_PROXY
   buffer.encode_uint32(23, this->zwave_proxy_feature_flags);
@@ -920,13 +920,13 @@ uint32_t HomeassistantServiceMap::calculate_size() const {
 void HomeassistantActionRequest::encode(ProtoWriteBuffer &buffer) const {
   buffer.encode_string(1, this->service);
   for (auto &it : this->data) {
-    buffer.encode_message(2, it);
+    buffer.encode_sub_message(2, it);
   }
   for (auto &it : this->data_template) {
-    buffer.encode_message(3, it);
+    buffer.encode_sub_message(3, it);
   }
   for (auto &it : this->variables) {
-    buffer.encode_message(4, it);
+    buffer.encode_sub_message(4, it);
   }
   buffer.encode_bool(5, this->is_event);
 #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES
@@ -1126,7 +1126,7 @@ void ListEntitiesServicesResponse::encode(ProtoWriteBuffer &buffer) const {
   buffer.encode_string(1, this->name);
   buffer.encode_fixed32(2, this->key);
   for (auto &it : this->args) {
-    buffer.encode_message(3, it);
+    buffer.encode_sub_message(3, it);
   }
   buffer.encode_uint32(4, static_cast<uint32_t>(this->supports_response));
 }
@@ -2133,7 +2133,7 @@ void ListEntitiesMediaPlayerResponse::encode(ProtoWriteBuffer &buffer) const {
   buffer.encode_uint32(7, static_cast<uint32_t>(this->entity_category));
   buffer.encode_bool(8, this->supports_pause);
   for (auto &it : this->supported_formats) {
-    buffer.encode_message(9, it);
+    buffer.encode_sub_message(9, it);
   }
 #ifdef USE_DEVICES
   buffer.encode_uint32(10, this->device_id);
@@ -2264,7 +2264,7 @@ uint32_t BluetoothLERawAdvertisement::calculate_size() const {
 }
 void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer &buffer) const {
   for (uint16_t i = 0; i < this->advertisements_len; i++) {
-    buffer.encode_message(1, this->advertisements[i]);
+    buffer.encode_sub_message(1, this->advertisements[i]);
   }
 }
 uint32_t BluetoothLERawAdvertisementsResponse::calculate_size() const {
@@ -2343,7 +2343,7 @@ void BluetoothGATTCharacteristic::encode(ProtoWriteBuffer &buffer) const {
   buffer.encode_uint32(2, this->handle);
   buffer.encode_uint32(3, this->properties);
   for (auto &it : this->descriptors) {
-    buffer.encode_message(4, it);
+    buffer.encode_sub_message(4, it);
   }
   buffer.encode_uint32(5, this->short_uuid);
 }
@@ -2370,7 +2370,7 @@ void BluetoothGATTService::encode(ProtoWriteBuffer &buffer) const {
   }
   buffer.encode_uint32(2, this->handle);
   for (auto &it : this->characteristics) {
-    buffer.encode_message(3, it);
+    buffer.encode_sub_message(3, it);
   }
   buffer.encode_uint32(4, this->short_uuid);
 }
@@ -2392,7 +2392,7 @@ uint32_t BluetoothGATTService::calculate_size() const {
 void BluetoothGATTGetServicesResponse::encode(ProtoWriteBuffer &buffer) const {
   buffer.encode_uint64(1, this->address);
   for (auto &it : this->services) {
-    buffer.encode_message(2, it);
+    buffer.encode_sub_message(2, it);
   }
 }
 uint32_t BluetoothGATTGetServicesResponse::calculate_size() const {
@@ -2673,7 +2673,7 @@ void VoiceAssistantRequest::encode(ProtoWriteBuffer &buffer) const {
   buffer.encode_bool(1, this->start);
   buffer.encode_string(2, this->conversation_id);
   buffer.encode_uint32(3, this->flags);
-  buffer.encode_message(4, this->audio_settings, false);
+  buffer.encode_optional_sub_message(4, this->audio_settings);
   buffer.encode_string(5, this->wake_word_phrase);
 }
 uint32_t VoiceAssistantRequest::calculate_size() const {
@@ -2906,7 +2906,7 @@ bool VoiceAssistantConfigurationRequest::decode_length(uint32_t field_id, ProtoL
 }
 void VoiceAssistantConfigurationResponse::encode(ProtoWriteBuffer &buffer) const {
   for (auto &it : this->available_wake_words) {
-    buffer.encode_message(1, it);
+    buffer.encode_sub_message(1, it);
   }
   for (const auto &it : *this->active_wake_words) {
     buffer.encode_string(2, it, true);

esphome/components/api/proto.cpp

@@ -1,5 +1,6 @@
 #include "proto.h"
 #include <cinttypes>
+#include <cstring>
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
@@ -87,6 +88,76 @@ uint32_t ProtoDecodableMessage::count_repeated_field(const uint8_t *buffer, size
   return count;
 }
 
+// Single-pass encode for repeated submessage elements (non-template core).
+// Writes field tag, reserves 1 byte for length varint, encodes the submessage body,
+// then backpatches the actual length. For the common case (body < 128 bytes), this is
+// just a single byte write with no memmove — all current repeated submessage types
+// (BLE advertisements at ~47B, GATT descriptors at ~24B, service args, etc.) take
+// this fast path.
+//
+// The memmove fallback for body >= 128 bytes exists only for correctness (e.g., a GATT
+// characteristic with many descriptors). It is safe because calculate_size() already
+// reserved space for the full multi-byte varint — the shift fills that reserved space:
+//
+//   calculate_size() allocates per element: tag + varint_size(body) + body_size
+//
+//   After encode, before memmove (1 byte reserved, body written):
+//   [tag][__][body ..... body][??]
+//         ^                   ^-- unused byte (v2 space from calculate_size)
+//         len_pos
+//
+//   After memmove(body_start+1, body_start, body_size):
+//   [tag][__][__][body ..... body]
+//         ^       ^-- body shifted forward, fills v2 space exactly
+//         len_pos
+//
+//   After writing 2-byte varint at len_pos:
+//   [tag][v1][v2][body ..... body]
+//                                ^-- pos_ = element end, within buffer
+void ProtoWriteBuffer::encode_sub_message(uint32_t field_id, const void *value,
+                                          void (*encode_fn)(const void *, ProtoWriteBuffer &)) {
+  this->encode_field_raw(field_id, 2);
+  // Reserve 1 byte for length varint (optimistic: submessage < 128 bytes)
+  uint8_t *len_pos = this->pos_;
+  this->debug_check_bounds_(1);
+  this->pos_++;
+  uint8_t *body_start = this->pos_;
+  encode_fn(value, *this);
+  uint32_t body_size = static_cast<uint32_t>(this->pos_ - body_start);
+  if (body_size < 128) [[likely]] {
+    // Common case: 1-byte varint, just backpatch
+    *len_pos = static_cast<uint8_t>(body_size);
+    return;
+  }
+  // Compute extra bytes needed for varint beyond the 1 already reserved
+  uint8_t extra = ProtoSize::varint(body_size) - 1;
+  // Shift body forward to make room for the extra varint bytes
+  this->debug_check_bounds_(extra);
+  std::memmove(body_start + extra, body_start, body_size);
+  uint8_t *end = this->pos_ + extra;
+  // Write the full varint at len_pos
+  this->pos_ = len_pos;
+  this->encode_varint_raw(body_size);
+  this->pos_ = end;
+}
+
+// Non-template core for encode_optional_sub_message.
+void ProtoWriteBuffer::encode_optional_sub_message(uint32_t field_id, uint32_t nested_size, const void *value,
+                                                   void (*encode_fn)(const void *, ProtoWriteBuffer &)) {
+  if (nested_size == 0)
+    return;
+  this->encode_field_raw(field_id, 2);
+  this->encode_varint_raw(nested_size);
+#ifdef ESPHOME_DEBUG_API
+  uint8_t *start = this->pos_;
+  encode_fn(value, *this);
+  if (static_cast<uint32_t>(this->pos_ - start) != nested_size)
+    this->debug_check_encode_size_(field_id, nested_size, this->pos_ - start);
+#else
+  encode_fn(value, *this);
+#endif
+}
+
 #ifdef ESPHOME_DEBUG_API
 void ProtoWriteBuffer::debug_check_bounds_(size_t bytes, const char *caller) {
   if (this->pos_ + bytes > this->buffer_->data() + this->buffer_->size()) {

esphome/components/api/proto.h

@@ -185,7 +185,7 @@ class ProtoVarInt {
 #endif
 };
 
-// Forward declarations for decode_to_message, encode_message and encode_packed_sint32
+// Forward declarations for decode_to_message and related encoding helpers
 class ProtoDecodableMessage;
 class ProtoMessage;
 class ProtoSize;
@@ -363,12 +363,18 @@ class ProtoWriteBuffer {
   }
   /// Encode a packed repeated sint32 field (zero-copy from vector)
   void encode_packed_sint32(uint32_t field_id, const std::vector<int32_t> &values);
-  /// Encode a nested message field (force=true for repeated, false for singular)
-  /// Templated so concrete message type is preserved for direct encode/calculate_size calls.
-  template<typename T> void encode_message(uint32_t field_id, const T &value, bool force = true);
-  // Non-template core for encode_message — all buffer work happens here
-  void encode_message(uint32_t field_id, uint32_t msg_length_bytes, const void *value,
-                      void (*encode_fn)(const void *, ProtoWriteBuffer &), bool force);
+  /// Single-pass encode for repeated submessage elements.
+  /// Thin template wrapper; all buffer work is in the non-template core.
+  template<typename T> void encode_sub_message(uint32_t field_id, const T &value);
+  /// Encode an optional singular submessage field — skips if empty.
+  /// Thin template wrapper; all buffer work is in the non-template core.
+  template<typename T> void encode_optional_sub_message(uint32_t field_id, const T &value);
+
+  // Non-template core for encode_sub_message — backpatch approach.
+  void encode_sub_message(uint32_t field_id, const void *value, void (*encode_fn)(const void *, ProtoWriteBuffer &));
+  // Non-template core for encode_optional_sub_message.
+  void encode_optional_sub_message(uint32_t field_id, uint32_t nested_size, const void *value,
+                                   void (*encode_fn)(const void *, ProtoWriteBuffer &));
   std::vector<uint8_t> *get_buffer() const { return buffer_; }
 
  protected:
@@ -690,26 +696,14 @@ template<typename T> void proto_encode_msg(const void *msg, ProtoWriteBuffer &bu
   static_cast<const T *>(msg)->encode(buf);
 }
 
-// Implementation of encode_message - must be after ProtoMessage is defined
-template<typename T> inline void ProtoWriteBuffer::encode_message(uint32_t field_id, const T &value, bool force) {
-  this->encode_message(field_id, value.calculate_size(), &value, &proto_encode_msg<T>, force);
+// Thin template wrapper; delegates to non-template core in proto.cpp.
+template<typename T> inline void ProtoWriteBuffer::encode_sub_message(uint32_t field_id, const T &value) {
+  this->encode_sub_message(field_id, &value, &proto_encode_msg<T>);
 }
 
-// Non-template core for encode_message
-inline void ProtoWriteBuffer::encode_message(uint32_t field_id, uint32_t msg_length_bytes, const void *value,
-                                             void (*encode_fn)(const void *, ProtoWriteBuffer &), bool force) {
-  if (msg_length_bytes == 0 && !force)
-    return;
-  this->encode_field_raw(field_id, 2);
-  this->encode_varint_raw(msg_length_bytes);
-#ifdef ESPHOME_DEBUG_API
-  uint8_t *start = this->pos_;
-  encode_fn(value, *this);
-  if (static_cast<uint32_t>(this->pos_ - start) != msg_length_bytes)
-    this->debug_check_encode_size_(field_id, msg_length_bytes, this->pos_ - start);
-#else
-  encode_fn(value, *this);
-#endif
+// Thin template wrapper; delegates to non-template core.
+template<typename T> inline void ProtoWriteBuffer::encode_optional_sub_message(uint32_t field_id, const T &value) {
+  this->encode_optional_sub_message(field_id, value.calculate_size(), &value, &proto_encode_msg<T>);
 }
 
 // Implementation of decode_to_message - must be after ProtoDecodableMessage is defined

script/api_protobuf/api_protobuf.py

@@ -690,15 +690,12 @@ class MessageType(TypeInfo):
 
     @property
     def encode_func(self) -> str:
-        return "encode_message"
+        return "encode_optional_sub_message"
 
     @property
     def encode_content(self) -> str:
-        # Singular message fields pass force=false (skip empty messages)
-        # The default for encode_nested_message is force=true (for repeated fields)
-        return (
-            f"buffer.{self.encode_func}({self.number}, this->{self.field_name}, false);"
-        )
+        # Singular message fields skip encoding when empty
+        return f"buffer.{self.encode_func}({self.number}, this->{self.field_name});"
 
     @property
     def decode_length(self) -> str:
@@ -1322,9 +1319,9 @@ class FixedArrayRepeatedType(TypeInfo):
         """Helper to generate encode statement for a single element."""
         if isinstance(self._ti, EnumType):
             return f"buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>({element}), true);"
-        # MessageType.encode_message doesn't have a force parameter
+        # Repeated message elements use encode_sub_message (force=true is default)
         if isinstance(self._ti, MessageType):
-            return f"buffer.{self._ti.encode_func}({self.number}, {element});"
+            return f"buffer.encode_sub_message({self.number}, {element});"
         return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);"
 
     @property
@@ -1650,9 +1647,9 @@ class RepeatedTypeInfo(TypeInfo):
         """Helper to generate encode call for a single element."""
         if isinstance(self._ti, EnumType):
             return f"buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>({element}), true);"
-        # MessageType.encode_message doesn't have a force parameter
+        # Repeated message elements use encode_sub_message (force=true is default)
         if isinstance(self._ti, MessageType):
-            return f"buffer.{self._ti.encode_func}({self.number}, {element});"
+            return f"buffer.encode_sub_message({self.number}, {element});"
         return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);"
 
     @property