From 2c9093ad9d1697137794ed07969f65a98ccdfcbf Mon Sep 17 00:00:00 2001
From: anjiahao <anjiahao@xiaomi.com>
Date: Mon, 12 Jul 2021 22:15:25 +0800
Subject: [PATCH] mq_open: add long file name check and parameter check

N/A

Signed-off-by: anjiahao <anjiahao@xiaomi.com>
Change-Id: I2d2c78e4477b4a6af13e51b79f24e8fd1794d291
---
 fs/mqueue/mq_open.c         |  5 ++---
 include/nuttx/mqueue.h      |  3 ++-
 sched/mqueue/mq_msgqalloc.c | 44 +++++++++++++++++++++++--------------
 3 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/fs/mqueue/mq_open.c b/fs/mqueue/mq_open.c
index bb774b29405..5c288279305 100644
--- a/fs/mqueue/mq_open.c
+++ b/fs/mqueue/mq_open.c
@@ -295,10 +295,9 @@ static int file_mq_vopen(FAR struct file *mq, FAR const char *mq_name,
        * be created with a reference count of zero.
        */
 
-      msgq = (FAR struct mqueue_inode_s *)nxmq_alloc_msgq(attr);
-      if (!msgq)
+      ret = nxmq_alloc_msgq(attr, &msgq);
+      if (ret < 0)
         {
-          ret = -ENOSPC;
           goto errout_with_inode;
         }
 
diff --git a/include/nuttx/mqueue.h b/include/nuttx/mqueue.h
index 149453ac56a..b6d99543936 100644
--- a/include/nuttx/mqueue.h
+++ b/include/nuttx/mqueue.h
@@ -397,7 +397,8 @@ void nxmq_free_msgq(FAR struct mqueue_inode_s *msgq);
  *
  ****************************************************************************/
 
-FAR struct mqueue_inode_s *nxmq_alloc_msgq(FAR struct mq_attr *attr);
+int nxmq_alloc_msgq(FAR struct mq_attr *attr,
+                    FAR struct mqueue_inode_s **msgq);
 
 /****************************************************************************
  * Name: nxmq_pollnotify
diff --git a/sched/mqueue/mq_msgqalloc.c b/sched/mqueue/mq_msgqalloc.c
index 7209f433a9f..80a0f32596f 100644
--- a/sched/mqueue/mq_msgqalloc.c
+++ b/sched/mqueue/mq_msgqalloc.c
@@ -49,50 +49,60 @@
  *   attr   - The mq_maxmsg attribute is used at the time that the message
  *            queue is created to determine the maximum number of
  *            messages that may be placed in the message queue.
+ *   msgq   - This parameter is a address of a pointer
  *
  * Returned Value:
- *   The allocated and initialized message queue structure or NULL in the
- *   event of a failure.
+ *   IF the function runs successfully，will return Zero(OK),else will
+ *   return a error code.
+ *
+ *   EINVAL    attr is a null pointer and attr->mq_mqssize or attr->mq_maxmsg
+ *             is an invalid value.
+ *   ENOMEM    No memery to alloc.
  *
  ****************************************************************************/
 
-FAR struct mqueue_inode_s *nxmq_alloc_msgq(FAR struct mq_attr *attr)
+int nxmq_alloc_msgq(FAR struct mq_attr *attr,
+                    FAR struct mqueue_inode_s **msgq)
 {
-  FAR struct mqueue_inode_s *msgq;
-
   /* Check if the caller is attempting to allocate a message for messages
    * larger than the configured maximum message size.
    */
 
-  DEBUGASSERT(!attr || attr->mq_msgsize <= MQ_MAX_BYTES);
-  if (attr && attr->mq_msgsize > MQ_MAX_BYTES)
+  DEBUGASSERT((!attr || (attr->mq_msgsize <= MQ_MAX_BYTES &&
+              attr->mq_maxmsg <= MQ_MAX_MSGS)) && msgq);
+  if ((attr && (attr->mq_msgsize > MQ_MAX_BYTES ||
+      attr->mq_maxmsg > MQ_MAX_MSGS)) || !msgq)
     {
-      return NULL;
+      return -EINVAL;
     }
 
   /* Allocate memory for the new message queue. */
 
-  msgq = (FAR struct mqueue_inode_s *)
+  *msgq = (FAR struct mqueue_inode_s *)
     kmm_zalloc(sizeof(struct mqueue_inode_s));
 
-  if (msgq)
+  if (*msgq)
     {
       /* Initialize the new named message queue */
 
-      sq_init(&msgq->msglist);
+      sq_init(&(*msgq)->msglist);
       if (attr)
         {
-          msgq->maxmsgs    = (int16_t)attr->mq_maxmsg;
-          msgq->maxmsgsize = (int16_t)attr->mq_msgsize;
+          (*msgq)->maxmsgs    = (int16_t)attr->mq_maxmsg;
+          (*msgq)->maxmsgsize = (int16_t)attr->mq_msgsize;
         }
       else
         {
-          msgq->maxmsgs    = MQ_MAX_MSGS;
-          msgq->maxmsgsize = MQ_MAX_BYTES;
+          (*msgq)->maxmsgs    = MQ_MAX_MSGS;
+          (*msgq)->maxmsgsize = MQ_MAX_BYTES;
         }
 
-      msgq->ntpid = INVALID_PROCESS_ID;
+      (*msgq)->ntpid = INVALID_PROCESS_ID;
+    }
+  else
+    {
+      return -ENOMEM;
     }
 
-  return msgq;
+  return OK;
 }