net/arp: support queue iob when arp_out failed

arp_out will replace the dev->d_iob to arp request if the iob destination
address is not exist in arp table, this mechanism cause this iob lost
result in first received ping no response or first synack retransmit.
to fix this bug, we queue the iob if arp_out failed, allocate a new
iob to send arp request, after the arp request completed, then we retry
send the queue iob packet.

Signed-off-by: wenquan1 <wenquan1@xiaomi.com>

include/nuttx/net/netdev.h

@@ -432,6 +432,10 @@ struct net_driver_s
   struct iob_queue_s d_fragout;
 #endif
 
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+  struct iob_queue_s d_arpout;
+#endif
+
   /* The d_buf array is used to hold incoming and outgoing packets. The
    * device driver should place incoming data into this buffer.  When sending
    * data, the device driver should read the link level headers and the

net/arp/Kconfig

@@ -60,12 +60,20 @@ config NET_ARP_GRATUITOUS
 
 config NET_ARP_SEND
 	bool "ARP send"
-	default y
+	default !NET_ARP_SEND_QUEUE
 	---help---
 		Enable logic to send ARP requests if the target IP address mapping
 		does not appear in the ARP table.
 
-if NET_ARP_SEND
+config NET_ARP_SEND_QUEUE
+	bool "Iob queue in arp entry support"
+	default n
+	depends on IOB_NCHAINS > 0
+	---help---
+		Enable queue iob to arp entry when arp_out failed to wait for arp
+		finished, then resend queue iobs.
+
+if NET_ARP_SEND || NET_ARP_SEND_QUEUE
 
 config ARP_SEND_MAXTRIES
 	int "ARP send retries"

net/arp/arp.h

@@ -166,6 +166,10 @@ struct arp_entry_s
   clock_t                  at_time;     /* Time of last usage */
   uint8_t                  at_flags;    /* Flags, examples: ATF_PERM */
   FAR struct net_driver_s *at_dev;      /* The device driver structure */
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+  struct iob_queue_s       at_queue;    /* Queue iobs to wait arp complete */
+  struct work_s            at_work;     /* Arp response timeout handle */
+#endif
 };
 
 /****************************************************************************
@@ -609,6 +613,32 @@ void arp_acd_setup(FAR struct net_driver_s *dev);
 
 #endif /* CONFIG_NET_ARP_ACD */
 
+/****************************************************************************
+ * Name: arp_queue_iob
+ *
+ * Description:
+ *   Queue an IOB which L2 layer is unfinished to the target arp entry's
+ *   deley queue which in progress waiting for an ARP response
+ *
+ * Input Parameters:
+ *   dev     - The device driver structure
+ *   ipaddr  - The IP address as an inaddr_t
+ *   iob     - The IOB to be queued
+ *
+ * Returned Value:
+ *   Zero (OK) if the ARP table entry was successfully modified.  A negated
+ *   errno value is returned on any error.
+ *
+ * Assumptions
+ *   The network is locked to assure exclusive access to the ARP table
+ *
+ ****************************************************************************/
+
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+int arp_queue_iob(FAR struct net_driver_s *dev, in_addr_t ipaddr,
+                  FAR struct iob_s *iob);
+#endif
+
 #else /* CONFIG_NET_ARP */
 
 /* If ARP is disabled, stub out all ARP interfaces */

net/arp/arp_out.c

@@ -275,7 +275,12 @@ void arp_out(FAR struct net_driver_s *dev)
            * to prevent arp flood.
            */
 
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+          arp_queue_iob(dev, ipaddr, dev->d_iob);
+          netdev_iob_clear(dev);
+#else
           dev->d_len = 0;
+#endif
           return;
         }
 
@@ -284,6 +289,10 @@ void arp_out(FAR struct net_driver_s *dev)
        */
 
       arp_update(dev, ipaddr, NULL, 0);
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+      arp_queue_iob(dev, ipaddr, dev->d_iob);
+      netdev_iob_clear(dev);
+#endif
 
       /* The destination address was not in our ARP table, so we overwrite
        * the IP packet with an ARP request.

net/arp/arp_table.c

@@ -269,6 +269,14 @@ static void arp_get_arpreq(FAR struct arpreq *output,
 }
 #endif
 
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+static void arp_unreach_work(FAR void *param)
+{
+  FAR struct arp_entry_s *tabptr = (FAR struct arp_entry_s *)param;
+  iob_free_queue(&tabptr->at_queue);
+}
+#endif
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -301,8 +309,10 @@ int arp_update(FAR struct net_driver_s *dev, in_addr_t ipaddr,
   FAR struct arp_entry_s *tabptr = &g_arptable[0];
 #ifdef CONFIG_NETLINK_ROUTE
   struct arpreq arp_notify;
-  bool found = false;
   bool new_entry;
+#endif
+#if defined(CONFIG_NETLINK_ROUTE) || defined(CONFIG_NET_ARP_SEND_QUEUE)
+  bool found = false;
 #endif
   int i;
 
@@ -324,7 +334,7 @@ int arp_update(FAR struct net_driver_s *dev, in_addr_t ipaddr,
           /* An old entry found, break. */
 
           tabptr = &g_arptable[i];
-#ifdef CONFIG_NETLINK_ROUTE
+#if defined(CONFIG_NETLINK_ROUTE) || defined(CONFIG_NET_ARP_SEND_QUEUE)
           found = true;
 #endif
           break;
@@ -337,6 +347,26 @@ int arp_update(FAR struct net_driver_s *dev, in_addr_t ipaddr,
         }
     }
 
+  if ((tabptr->at_flags & ATF_PERM) != 0 && (flags & ATF_PERM) == 0)
+    {
+      return -ENOSPC;
+    }
+
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+  if (!found && tabptr->at_ipaddr != 0)
+    {
+      /* arp entry will be replaced, clean delayed iobs if exist */
+
+      work_cancel_sync(LPWORK, &tabptr->at_work);
+      iob_free_queue(&tabptr->at_queue);
+    }
+  else if (found && ethaddr != NULL)
+    {
+      work_cancel_sync(LPWORK, &tabptr->at_work);
+      iob_concat_queue(&dev->d_arpout, &tabptr->at_queue);
+    }
+#endif
+
   if (ethaddr == NULL)
     {
       ethaddr = g_zero_ethaddr.ether_addr_octet;
@@ -361,14 +391,11 @@ int arp_update(FAR struct net_driver_s *dev, in_addr_t ipaddr,
    * information.
    */
 
-  if ((tabptr->at_flags & ATF_PERM) == 0 || (flags & ATF_PERM) != 0)
-    {
-      memcpy(tabptr->at_ethaddr.ether_addr_octet, ethaddr, ETHER_ADDR_LEN);
-      tabptr->at_ipaddr = ipaddr;
-      tabptr->at_time   = clock_systime_ticks();
-      tabptr->at_flags  = flags;
-      tabptr->at_dev    = dev;
-    }
+  memcpy(tabptr->at_ethaddr.ether_addr_octet, ethaddr, ETHER_ADDR_LEN);
+  tabptr->at_ipaddr = ipaddr;
+  tabptr->at_time   = clock_systime_ticks();
+  tabptr->at_flags  = flags;
+  tabptr->at_dev    = dev;
 
   /* Notify the new entry */
 
@@ -380,6 +407,28 @@ int arp_update(FAR struct net_driver_s *dev, in_addr_t ipaddr,
     }
 #endif
 
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+  if (!IOB_QEMPTY(&dev->d_arpout))
+    {
+      /* in Rx context, we need to backup the dev iob and related members,
+       * as some dirvers netdev_txnotify_dev() execute transmit in sync mode
+       * which will modify iob and other members.
+       * we need to restore the dev iob and related members after
+       * netdev_txnotify_dev() return because iob will be used in left
+       * bottom Rx process.
+       */
+
+      uint16_t len = dev->d_len;
+      FAR struct iob_s *iob = dev->d_iob;
+
+      dev->d_iob = NULL;
+      dev->d_buf = NULL;
+      netdev_txnotify_dev(dev);
+      netdev_iob_replace(dev, iob);
+      dev->d_len = len;
+    }
+#endif
+
   return OK;
 }
 
@@ -567,6 +616,11 @@ void arp_cleanup(FAR struct net_driver_s *dev)
     {
       if (dev == g_arptable[i].at_dev)
         {
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+          work_cancel_sync(LPWORK, &g_arptable[i].at_work);
+          iob_free_queue(&g_arptable[i].at_queue);
+#endif
+
           memset(&g_arptable[i], 0, sizeof(g_arptable[i]));
         }
     }
@@ -622,5 +676,55 @@ unsigned int arp_snapshot(FAR struct arpreq *snapshot,
 }
 #endif
 
+/****************************************************************************
+ * Name: arp_queue_iob
+ *
+ * Description:
+ *   Queue an IOB which L2 layer is unfinished to the target arp entry's
+ *   delay queue while the entry is in progress waiting for an ARP response
+ *
+ * Input Parameters:
+ *   dev     - The device driver structure
+ *   ipaddr  - The IP address as an inaddr_t
+ *   iob     - The IOB to be queued
+ *
+ * Returned Value:
+ *   Zero (OK) if the ARP table entry was successfully modified.  A negated
+ *   errno value is returned on any error.
+ *
+ * Assumptions
+ *   The network is locked to assure exclusive access to the ARP table
+ *
+ ****************************************************************************/
+
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+int arp_queue_iob(FAR struct net_driver_s *dev, in_addr_t ipaddr,
+                  FAR struct iob_s *iob)
+{
+  FAR struct arp_entry_s *tabptr;
+
+  /* the IPv4 address should in the ARP table and arp in progress. */
+
+  tabptr = arp_lookup(ipaddr, dev, false);
+  if (tabptr && memcmp(&tabptr->at_ethaddr, &g_zero_ethaddr,
+                       sizeof(tabptr->at_ethaddr)) == 0)
+    {
+      if (iob_tryadd_queue(iob, &tabptr->at_queue) == 0)
+        {
+          if (work_available(&tabptr->at_work))
+            {
+              work_queue(LPWORK, &tabptr->at_work, arp_unreach_work,
+                         tabptr, ARP_INPROGRESS_TICK);
+            }
+
+          return OK;
+        }
+
+      return -ENOMEM;
+    }
+
+  return -ENOENT;
+}
+#endif
 #endif /* CONFIG_NET_ARP */
 #endif /* CONFIG_NET */

net/devif/devif_poll.c

@@ -675,6 +675,84 @@ static inline int devif_poll_tcp_connections(FAR struct net_driver_s *dev,
 #  define devif_poll_tcp_connections(dev, callback) (0)
 #endif
 
+/****************************************************************************
+ * Name: devif_poll_queue
+ *
+ * Description:
+ *   Poll iob to send.
+ *
+ * Input Parameters:
+ *   iobq - the iob queue to poll.
+ *   dev - NIC Device instance.
+ *   callback - the actual sending API provided by each NIC driver.
+ *
+ * Returned Value:
+ *   Zero indicated the polling will continue, else stop the polling.
+ *
+ * Assumptions:
+ *   This function is called from the MAC device driver with the network
+ *   locked.
+ *
+ ****************************************************************************/
+
+#if defined(CONFIG_NET_ARP_SEND_QUEUE) || defined(CONFIG_NET_IPFRAG)
+static int devif_poll_queue(FAR struct iob_queue_s *iobq,
+                            FAR struct net_driver_s *dev,
+                            devif_poll_callback_t callback)
+{
+  FAR struct iob_s *iob;
+  bool reused = false;
+  int bstop = false;
+
+  while (!bstop)
+    {
+      /* Dequeue outgoing iob from iobq */
+
+      iob = iob_remove_queue(iobq);
+      if (iob == NULL)
+        {
+          break;
+        }
+
+      /* buffer could be reused for other protocols */
+
+      reused = true;
+
+      /* Replace original iob */
+
+      netdev_iob_replace(dev, iob);
+
+      /* build L2 headers */
+
+      devif_out(dev);
+
+      /* Call back into the driver */
+
+      if (dev->d_len > 0)
+        {
+          bstop = callback(dev);
+        }
+    }
+
+  /* Notify the device driver that iob is available. */
+
+  if (iob_peek_queue(iobq) != NULL)
+    {
+      netdev_txnotify_dev(dev);
+    }
+
+  /* Reuse iob buffer */
+
+  if (!bstop && reused)
+    {
+      iob_update_pktlen(dev->d_iob, 0, false);
+      netdev_iob_prepare(dev, true, 0);
+    }
+
+  return bstop;
+}
+#endif
+
 /****************************************************************************
  * Name: devif_poll_ipfrag
  *
@@ -698,56 +776,34 @@ static inline int devif_poll_tcp_connections(FAR struct net_driver_s *dev,
 static int devif_poll_ipfrag(FAR struct net_driver_s *dev,
                              devif_poll_callback_t callback)
 {
-  FAR struct iob_s *frag;
-  bool reused = false;
-  int bstop = false;
+  return devif_poll_queue(&dev->d_fragout, dev, callback);
+}
+#endif
 
-  while (!bstop)
-    {
-      /* Dequeue outgoing fragment from dev->d_fragout */
+/****************************************************************************
+ * Name: devif_poll_arp
+ *
+ * Description:
+ *   Poll all queue iobs with arp finished to send.
+ *
+ * Input Parameters:
+ *   dev - NIC Device instance.
+ *   callback - the actual sending API provided by each NIC driver.
+ *
+ * Returned Value:
+ *   Zero indicated the polling will continue, else stop the polling.
+ *
+ * Assumptions:
+ *   This function is called from the MAC device driver with the network
+ *   locked.
+ *
+ ****************************************************************************/
 
-      frag = iob_remove_queue(&dev->d_fragout);
-      if (frag == NULL)
-        {
-          break;
-        }
-
-      /* Frag buffer could be reused for other protocols */
-
-      reused = true;
-
-      /* Replace original iob */
-
-      netdev_iob_replace(dev, frag);
-
-      /* build L2 headers */
-
-      devif_out(dev);
-
-      /* Call back into the driver */
-
-      if (dev->d_len > 0)
-        {
-          bstop = callback(dev);
-        }
-    }
-
-  /* Notify the device driver that ip fragments is available. */
-
-  if (iob_peek_queue(&dev->d_fragout) != NULL)
-    {
-      netdev_txnotify_dev(dev);
-    }
-
-  /* Reuse iob buffer */
-
-  if (!bstop && reused)
-    {
-      iob_update_pktlen(dev->d_iob, 0, false);
-      netdev_iob_prepare(dev, true, 0);
-    }
-
-  return bstop;
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+static int devif_poll_arp(FAR struct net_driver_s *dev,
+                          devif_poll_callback_t callback)
+{
+  return devif_poll_queue(&dev->d_arpout, dev, callback);
 }
 #endif
 
@@ -789,6 +845,14 @@ static int devif_poll_connections(FAR struct net_driver_s *dev,
    * action.
    */
 
+#ifdef CONFIG_NET_ARP_SEND_QUEUE
+  bstop = devif_poll_arp(dev, callback);
+  if (bstop)
+    {
+      return bstop;
+    }
+#endif
+
 #ifdef CONFIG_NET_IPFRAG
   /* Traverse all of ip fragments for available packets to transfer */