VNC: Fix error in message header size calculation; Add logic to try copy with pixel format changes.

graphics/vnc/server/vnc_server.h

@@ -208,8 +208,8 @@ struct vnc_session_s
   /* Display geometry and color characteristics */
 
   uint8_t display;             /* Display number (for debug) */
-  uint8_t colorfmt;            /* Remote color format (See include/nuttx/fb.h) */
+  volatile uint8_t colorfmt;   /* Remote color format (See include/nuttx/fb.h) */
-  uint8_t bpp;                 /* Remote bits per pixel */
+  volatile uint8_t bpp;        /* Remote bits per pixel */
   FAR uint8_t *fb;             /* Allocated local frame buffer */
 
   /* Updater information */

graphics/vnc/server/vnc_updater.c

@@ -759,6 +759,7 @@ static FAR void *vnc_updater(FAR void *arg)
   unsigned int maxwidth;
   size_t size;
   ssize_t nsent;
+  uint8_t colorfmt;
 
   union
   {
@@ -795,7 +796,8 @@ static FAR void *vnc_updater(FAR void *arg)
 
       /* Set up the color conversion */
 
-      switch (session->colorfmt)
+      colorfmt = session->colorfmt;
+      switch (colorfmt)
         {
           case FB_FMT_RGB8_222:
             convert.bpp8 = vnc_convert_rgb8_222;
@@ -854,10 +856,13 @@ static FAR void *vnc_updater(FAR void *arg)
        * Loop until all sub-rectangles have been output.  Start with the
        * top row and transfer rectangles horizontally across each swath.
        * The height of the swath is destwidth (the last may be shorter).
+       *
+       * NOTE that the loop also terminates of the color format changes
+       * asynchronously.
        */
 
       for (y = srcrect->rect.pt1.y;
-           srcheight > 0;
+           srcheight > 0 && colorfmt == session->colorfmt;
            srcheight -= updheight, y += updheight)
         {
           /* updheight = Height to update on this pass through the loop.
@@ -876,10 +881,13 @@ static FAR void *vnc_updater(FAR void *arg)
            * horizontally with width of destwidth until all srcwidth
            * columns have been transferred (the last rectangle may be
            * narrower).
+           *
+           * NOTE that the loop also terminates of the color format
+           * changes asynchronously.
            */
 
           for (width = srcwidth, x = srcrect->rect.pt1.x;
-               width > 0;
+               width > 0 && colorfmt == session->colorfmt;
                width -= updwidth, x += updwidth)
             {
               /* updwidth = Width to update on this pass through the loop.
@@ -928,11 +936,22 @@ static FAR void *vnc_updater(FAR void *arg)
 
               DEBUGASSERT(size <= CONFIG_VNCSERVER_UPDATE_BUFSIZE);
 
-              /* Then send the update packet to the VNC client */
+              /* We are ready to send the update packet to the VNC client */
 
-              size += SIZEOF_RFB_FRAMEBUFFERUPDATE_S(0);
+              size += SIZEOF_RFB_FRAMEBUFFERUPDATE_S(SIZEOF_RFB_RECTANGE_S(0));
               src   = session->outbuf;
 
+              /* At the very last most, make certain that the color format
+               * has not changed asynchronously.
+               */
+
+              if (colorfmt == session->colorfmt)
+                {
+                  /* Okay send until all of the bytes are out.  This may
+                   * loop for the case where TCP write buffering is enabled
+                   * and there are a limited number of IOBs available.
+                   */
+
                   do
                     {
                       nsent = psock_send(&session->connect, src, size, 0);
@@ -950,6 +969,7 @@ static FAR void *vnc_updater(FAR void *arg)
                   while (size > 0);
                 }
             }
+        }
 
       vnc_free_update(session, srcrect);
     }

include/nuttx/video/rfb.h

@@ -547,8 +547,8 @@ struct rfb_rectangle_s
   uint8_t data[1];               /* Pixel data, actual size varies */
 };
 
-#define SIZEOF_RFB_RECTANGES(d) \
+#define SIZEOF_RFB_RECTANGE_S(d) \
-  (sizeof(struct rfb_framebufferupdate_s) + (d) - 1)
+  (sizeof(struct rfb_rectangle_s) + (d) - 1)
 
 struct rfb_framebufferupdate_s
 {