sem_open: return error code, sem returned by parameter

pointer comparison is unsigned, when returning -errno will be converted
to a large positive number, can not enter the error handling branch,
therefore, the error code is returned directly and the sem is returned
through the parameters.

Signed-off-by: zhanghongyu <zhanghongyu@xiaomi.com>

fs/semaphore/sem_open.c

@@ -65,7 +65,8 @@
  *   calls to sem_unlink()).
  *
  * Input Parameters:
- *   name  - Semaphore name
+ *   sem    - Location to return the semaphore reference.
+ *   name   - Semaphore name.
  *   oflags - Semaphore creation options.  This may either or both of the
  *     following bit settings.
  *     oflags = 0:  Connect to the semaphore only if it already exists.
@@ -81,17 +82,16 @@
  *        SEM_VALUE_MAX.
  *
  * Returned Value:
- *   A pointer to sem_t or negated errno if unsuccessful.
+ *   0 (OK), or negated errno if unsuccessful.
  *
  * Assumptions:
  *
  ****************************************************************************/
 
-FAR sem_t *nxsem_open(FAR const char *name, int oflags, ...)
+int nxsem_open(FAR sem_t **sem, FAR const char *name, int oflags, ...)
 {
   FAR struct inode *inode;
   FAR struct nsem_inode_s *nsem;
-  FAR sem_t *sem;
   struct inode_search_s desc;
   char fullpath[MAX_SEMPATH];
   mode_t mode;
@@ -139,7 +139,7 @@ FAR sem_t *nxsem_open(FAR const char *name, int oflags, ...)
        * count on the inode.
        */
 
-      sem = &inode->u.i_nsem->ns_sem;
+      *sem = &inode->u.i_nsem->ns_sem;
     }
   else
     {
@@ -218,18 +218,18 @@ FAR sem_t *nxsem_open(FAR const char *name, int oflags, ...)
 
       /* Return a reference to the semaphore */
 
-      sem = &nsem->ns_sem;
+      *sem = &nsem->ns_sem;
     }
 
   RELEASE_SEARCH(&desc);
-  return sem;
+  return OK;
 
 errout_with_inode:
   inode_release(inode);
 
 errout_with_lock:
   RELEASE_SEARCH(&desc);
-  return (FAR sem_t *)(intptr_t)ret;
+  return ret;
 }
 
 #endif /* CONFIG_FS_NAMED_SEMAPHORES */

include/nuttx/semaphore.h

@@ -401,7 +401,8 @@ int nxsem_get_value(FAR sem_t *sem, FAR int *sval);
  *   calls to sem_unlink()).
  *
  * Input Parameters:
- *   name  - Semaphore name
+ *   sem    - Location to return the semaphore reference.
+ *   name   - Semaphore name.
  *   oflags - Semaphore creation options.  This may either or both of the
  *     following bit settings.
  *     oflags = 0:  Connect to the semaphore only if it already exists.
@@ -417,13 +418,13 @@ int nxsem_get_value(FAR sem_t *sem, FAR int *sval);
  *        SEM_VALUE_MAX.
  *
  * Returned Value:
- *   A pointer to sem_t or negated errno if unsuccessful.
+ *   0 (OK), or negated errno if unsuccessful.
  *
  * Assumptions:
  *
  ****************************************************************************/
 
-FAR sem_t *nxsem_open(FAR const char *name, int oflags, ...);
+int nxsem_open(FAR sem_t **sem, FAR const char *name, int oflags, ...);
 
 /****************************************************************************
  * Name:  nxsem_close

include/sys/syscall_lookup.h

@@ -89,7 +89,7 @@ SYSCALL_LOOKUP(nxsem_wait,                 1)
 /* Named semaphores */
 
 #ifdef CONFIG_FS_NAMED_SEMAPHORES
-  SYSCALL_LOOKUP(nxsem_open,               4)
+  SYSCALL_LOOKUP(nxsem_open,               5)
   SYSCALL_LOOKUP(nxsem_close,              1)
   SYSCALL_LOOKUP(nxsem_unlink,             1)
 #endif

libs/libc/semaphore/sem_open.c

@@ -82,6 +82,7 @@ FAR sem_t *sem_open(FAR const char *name, int oflags, ...)
   va_list ap;
   mode_t mode;
   unsigned int value;
+  int errcode;
 
   /* Make sure that a non-NULL name is supplied */
 
@@ -111,10 +112,10 @@ FAR sem_t *sem_open(FAR const char *name, int oflags, ...)
 
   /* Let nxsem_open() do the work */
 
-  sem = nxsem_open(name, oflags, mode, value);
+  errcode = nxsem_open(&sem, name, oflags, mode, value);
-  if (sem < 0)
+  if (errcode < 0)
     {
-      set_errno(-((intptr_t)sem));
+      set_errno(-errcode);
       return SEM_FAILED;
     }
 

syscall/syscall.csv

@@ -84,7 +84,7 @@
 "nxsem_clockwait","nuttx/semaphore.h","","int","FAR sem_t *","clockid_t","FAR const struct timespec *"
 "nxsem_close","nuttx/semaphore.h","defined(CONFIG_FS_NAMED_SEMAPHORES)","int","FAR sem_t *"
 "nxsem_destroy","nuttx/semaphore.h","","int","FAR sem_t *"
-"nxsem_open","nuttx/semaphore.h","defined(CONFIG_FS_NAMED_SEMAPHORES)","FAR sem_t *","FAR const char *","int","...","mode_t","unsigned int"
+"nxsem_open","nuttx/semaphore.h","defined(CONFIG_FS_NAMED_SEMAPHORES)","int","FAR sem_t **","FAR const char *","int","...","mode_t","unsigned int"
 "nxsem_post","nuttx/semaphore.h","","int","FAR sem_t *"
 "nxsem_set_protocol","nuttx/semaphore.h","defined(CONFIG_PRIORITY_INHERITANCE)","int","FAR sem_t *","int"
 "nxsem_timedwait","nuttx/semaphore.h","","int","FAR sem_t *","FAR const struct timespec *"