Squashed commit of the following:

net/utils:  return from net_breaklock() was being clobbered.
    net/:  Replace all calls to iob_alloc() with calls to net_ioballoc() which will release the network lock, if necessary.
    net/utils, tcp, include/net:  Separate out the special IOB allocation logic and place it in its own function.  Prototype is available in a public header file where it can also be used by network drivers.
    net/utils: net_timedwait() now uses new net_breaklock() and net_restorelock().

include/nuttx/net/net.h

@@ -290,12 +290,15 @@ void net_initialize(void);
 /****************************************************************************
  * Critical section management.
  *
- * Semaphore based locking is used:
+ * Re-entrant mutex based locking of the network is supported:
  *
- *   net_lock()          - Takes the semaphore().  Implements a re-entrant mutex.
- *   net_unlock()        - Gives the semaphore().
- *   net_lockedwait()    - Like pthread_cond_wait(); releases the semaphore
- *                         momentarily to wait on another semaphore()
+ *   net_lock()        - Locks the network via a re-entrant mutex.
+ *   net_unlock()      - Unlocks the network.
+ *   net_lockedwait()  - Like pthread_cond_wait() except releases the
+ *                       network momentarily to wait on another semaphore.
+ *   net_ioballoc()    - Like iob_alloc() except releases the network
+ *                       momentarily to wait for an IOB to become
+ *                       available.
  *
  ****************************************************************************/
 
@@ -368,6 +371,27 @@ int net_timedwait(sem_t *sem, FAR const struct timespec *abstime);
 
 int net_lockedwait(sem_t *sem);
 
+/****************************************************************************
+ * Name: net_ioballoc
+ *
+ * Description:
+ *   Allocate an IOB.  If no IOBs are available, then atomically wait for
+ *   for the IOB while temporarily releasing the lock on the network.
+ *
+ * Input Parameters:
+ *   throttled - An indication of the IOB allocation is "throttled"
+ *
+ * Returned Value:
+ *   A pointer to the newly allocated IOB is returned on success.  NULL is
+ *   returned on any allocation failure.
+ *
+ ****************************************************************************/
+
+#ifdef CONFIG_MM_IOB
+struct iob_s;  /* Forward reference */
+FAR struct iob_s *net_ioballoc(bool throttled);
+#endif
+
 /****************************************************************************
  * Name: net_setipid
  *