Squashed commit of the following:

many locations:  Change occurences of open() followed by file_detach() to file_open().  Change most non-controversion calls to open() to nx_open().

    fs/inode/fs_fileopen.c:  Flesh out file_open() with some interim, placeholder logic.

    fs/inode/fs_fileopen.c:  Add a framework for a file_open() implementation (no real logic in place yet).

    fs/vfs/fs_open.c:  Add nx_open() which is the same as open() except that it does not create a cancellation point nor does it modify the errno variable.

fs/vfs/fs_open.c

@@ -46,9 +46,7 @@
 #include <sched.h>
 #include <errno.h>
 #include <assert.h>
-#ifdef CONFIG_FILE_MODE
 #include <stdarg.h>
-#endif
 
 #include <nuttx/cancelpt.h>
 #include <nuttx/fs/fs.h>
@@ -82,14 +80,22 @@ int inode_checkflags(FAR struct inode *inode, int oflags)
 }
 
 /****************************************************************************
- * Name: open
+ * Name: nx_vopen
  *
  * Description:
- *   Standard 'open' interface
+ *   nx_vopen() is identical to 'nx_open' except that it accepts a va_list
+ *   as an argument versus taking a variable length list of arguments.
+ *
+ *   nx_vopen() is an internal NuttX interface and should not be called from
+ *   applications.
+ *
+ * Returned Value:
+ *   Zero (OK) is returned on success; a negated errno value is returned on
+ *   any failure.
  *
  ****************************************************************************/
 
-int open(FAR const char *path, int oflags, ...)
+int nx_vopen(FAR const char *path, int oflags, va_list ap)
 {
   struct inode_search_s desc;
   FAR struct file *filep;
@@ -100,14 +106,9 @@ int open(FAR const char *path, int oflags, ...)
   int ret;
   int fd;
 
-  /* open() is a cancellation point */
-
-  (void)enter_cancellation_point();
-
   if (path == NULL)
     {
-      set_errno(EINVAL);
-      goto errout;
+      return -EINVAL;
     }
 
 #ifdef CONFIG_FILE_MODE
@@ -119,7 +120,6 @@ int open(FAR const char *path, int oflags, ...)
 
   if ((oflags & (O_WRONLY | O_CREAT)) != 0)
     {
-      va_list ap;
       va_start(ap, oflags);
       mode = va_arg(ap, mode_t);
       va_end(ap);
@@ -138,7 +138,6 @@ int open(FAR const char *path, int oflags, ...)
        * symbolic link."
        */
 
-      ret = -ret;
       goto errout_with_search;
     }
 
@@ -169,14 +168,12 @@ int open(FAR const char *path, int oflags, ...)
        fd = block_proxy(path, oflags);
        if (fd < 0)
          {
-           ret = -fd;
            goto errout_with_search;
          }
 
        /* Return the file descriptor */
 
        RELEASE_SEARCH(&desc);
-       leave_cancellation_point();
        return fd;
      }
    else
@@ -193,7 +190,7 @@ int open(FAR const char *path, int oflags, ...)
   if (!INODE_IS_DRIVER(inode) || !inode->u.i_ops)
 #endif
     {
-      ret = ENXIO;
+      ret = -ENXIO;
       goto errout_with_inode;
     }
 
@@ -202,7 +199,6 @@ int open(FAR const char *path, int oflags, ...)
   ret = inode_checkflags(inode, oflags);
   if (ret < 0)
     {
-      ret = -ret;
       goto errout_with_inode;
     }
 
@@ -211,7 +207,7 @@ int open(FAR const char *path, int oflags, ...)
   fd = files_allocate(inode, oflags, 0, 0);
   if (fd < 0)
     {
-      ret = EMFILE;
+      ret = -EMFILE;
       goto errout_with_inode;
     }
 
@@ -220,7 +216,6 @@ int open(FAR const char *path, int oflags, ...)
   ret = fs_getfilep(fd, &filep);
   if (ret < 0)
     {
-      ret = -ret;
       goto errout_with_inode;
     }
 
@@ -246,7 +241,6 @@ int open(FAR const char *path, int oflags, ...)
 
   if (ret < 0)
     {
-      ret = -ret;
       goto errout_with_fd;
     }
 
@@ -282,7 +276,6 @@ int open(FAR const char *path, int oflags, ...)
 #endif
 
   RELEASE_SEARCH(&desc);
-  leave_cancellation_point();
   return fd;
 
 errout_with_fd:
@@ -293,9 +286,74 @@ errout_with_inode:
 
 errout_with_search:
   RELEASE_SEARCH(&desc);
-  set_errno(ret);
-
-errout:
-  leave_cancellation_point();
-  return ERROR;
+  return ret;
+}
+
+/****************************************************************************
+ * Name: nx_open
+ *
+ * Description:
+ *   nx_open() is similar to the standard 'open' interface except that is is
+ *   not a cancellation point and it does not modify the errno variable.
+ *
+ *   nx_open() is an internal NuttX interface and should not be called from
+ *   applications.
+ *
+ * Returned Value:
+ *   Zero (OK) is returned on success; a negated errno value is returned on
+ *   any failure.
+ *
+ ****************************************************************************/
+
+int nx_open(FAR const char *path, int oflags, ...)
+{
+  va_list ap;
+  int fd;
+
+  /* Let nx_vopen() do all of the work */
+
+  va_start(ap, oflags);
+  fd = nx_vopen(path, oflags, ap);
+  va_end(ap);
+
+  return fd;
+}
+
+/****************************************************************************
+ * Name: open
+ *
+ * Description:
+ *   Standard 'open' interface
+ *
+ * Returned Value:
+ *   Zero (OK) is returned on success; -1 (ERROR) is returned on any failure
+ *   the the errno value set appropriately.
+ *
+ ****************************************************************************/
+
+int open(FAR const char *path, int oflags, ...)
+{
+  va_list ap;
+  int fd;
+
+  /* open() is a cancellation point */
+
+  (void)enter_cancellation_point();
+
+  /* Let nx_vopen() do most of the work */
+
+  va_start(ap, oflags);
+  fd = nx_vopen(path, oflags, ap);
+  va_end(ap);
+
+  /* Set the errno value if any errors were reported by nx_open() */
+
+  if (fd < 0)
+    {
+      set_errno(-fd);
+      fd = ERROR;
+    }
+
+  leave_cancellation_point();
+  return fd;
 }