mq_open: add long file name check and parameter check

Signed-off-by: anjiahao <anjiahao@xiaomi.com>

fs/mqueue/mq_open.c

@@ -177,6 +177,12 @@ static int file_mq_vopen(FAR struct file *mq, FAR const char *mq_name,
       goto errout;
     }
 
+  if (strlen(mq_name) > NAME_MAX)
+    {
+      ret = -ENAMETOOLONG;
+      goto errout;
+    }
+
   /* Were we asked to create it? */
 
   if ((oflags & O_CREAT) != 0)
@@ -289,10 +295,9 @@ static int file_mq_vopen(FAR struct file *mq, FAR const char *mq_name,
        * be created with a reference count of zero.
        */
 
-      msgq = (FAR struct mqueue_inode_s *)nxmq_alloc_msgq(attr);
+      ret = nxmq_alloc_msgq(attr, &msgq);
-      if (!msgq)
+      if (ret < 0)
         {
-          ret = -ENOSPC;
           goto errout_with_inode;
         }
 

fs/mqueue/mqueue.h

@@ -39,6 +39,6 @@
 
 /* Sizes of things */
 
-#define MAX_MQUEUE_PATH 64
+#define MAX_MQUEUE_PATH (sizeof(CONFIG_FS_MQUEUE_MPATH) + NAME_MAX)
 
 #endif /* __FS_MQUEUE_MQUEUE_H */

include/nuttx/mqueue.h

@@ -385,14 +385,21 @@ void nxmq_free_msgq(FAR struct mqueue_inode_s *msgq);
  *   attr   - The mq_maxmsg attribute is used at the time that the message
  *            queue is created to determine the maximum number of
  *            messages that may be placed in the message queue.
+ *   pmsgq  - This parameter is a address of a pointer
  *
  * Returned Value:
- *   The allocated and initialized message queue structure or NULL in the
+ *   Zero (OK) is returned on success. Otherwise, a negated errno value is
- *   event of a failure.
+ *   returned to indicate the nature of the failure.
+ *
+ *   EINVAL    attr is NULL or either attr->mq_mqssize or attr->mq_maxmsg
+ *             have an invalid value
+ *   ENOSPC    There is insufficient space for the creation of the new
+ *             message queue
  *
  ****************************************************************************/
 
-FAR struct mqueue_inode_s *nxmq_alloc_msgq(FAR struct mq_attr *attr);
+int nxmq_alloc_msgq(FAR struct mq_attr *attr,
+                    FAR struct mqueue_inode_s **pmsgq);
 
 /****************************************************************************
  * Name: nxmq_pollnotify

sched/mqueue/mq_msgqalloc.c

@@ -49,14 +49,21 @@
  *   attr   - The mq_maxmsg attribute is used at the time that the message
  *            queue is created to determine the maximum number of
  *            messages that may be placed in the message queue.
+ *   pmsgq  - This parameter is a address of a pointer
  *
  * Returned Value:
- *   The allocated and initialized message queue structure or NULL in the
+ *   Zero (OK) is returned on success. Otherwise, a negated errno value is
- *   event of a failure.
+ *   returned to indicate the nature of the failure.
+ *
+ *   EINVAL    attr is NULL or either attr->mq_mqssize or attr->mq_maxmsg
+ *             have an invalid value
+ *   ENOSPC    There is insufficient space for the creation of the new
+ *             message queue
  *
  ****************************************************************************/
 
-FAR struct mqueue_inode_s *nxmq_alloc_msgq(FAR struct mq_attr *attr)
+int nxmq_alloc_msgq(FAR struct mq_attr *attr,
+                    FAR struct mqueue_inode_s **pmsgq)
 {
   FAR struct mqueue_inode_s *msgq;
 
@@ -64,10 +71,10 @@ FAR struct mqueue_inode_s *nxmq_alloc_msgq(FAR struct mq_attr *attr)
    * larger than the configured maximum message size.
    */
 
-  DEBUGASSERT(!attr || attr->mq_msgsize <= MQ_MAX_BYTES);
+  DEBUGASSERT((!attr || attr->mq_msgsize <= MQ_MAX_BYTES) && pmsgq);
-  if (attr && attr->mq_msgsize > MQ_MAX_BYTES)
+  if ((attr && attr->mq_msgsize > MQ_MAX_BYTES) || !pmsgq)
     {
-      return NULL;
+      return -EINVAL;
     }
 
   /* Allocate memory for the new message queue. */
@@ -93,6 +100,11 @@ FAR struct mqueue_inode_s *nxmq_alloc_msgq(FAR struct mq_attr *attr)
 
       msgq->ntpid = INVALID_PROCESS_ID;
     }
+  else
+    {
+      return -ENOSPC;
+    }
 
-  return msgq;
+  *pmsgq = msgq;
+  return OK;
 }