VNC: Add some very basic RRE encodings

graphics/vnc/server/Make.defs

@@ -36,7 +36,7 @@
 ifeq ($(CONFIG_VNCSERVER),y)
 
 CSRCS += vnc_server.c vnc_negotiate.c vnc_updater.c vnc_receiver.c
-CSRCS += vnc_raw.c vnc_color.c vnc_fbdev.c
+CSRCS += vnc_raw.c vnc_rre.c vnc_color.c vnc_fbdev.c
 
 ifeq ($(CONFIG_NX_KBD),y)
 CSRCS += vnc_keymap.c

graphics/vnc/server/vnc_color.c

@@ -40,6 +40,8 @@
 #include <nuttx/config.h>
 
 #include <stdint.h>
+#include <assert.h>
+#include <errno.h>
 
 #include "vnc_server.h"
 
@@ -195,3 +197,164 @@ uint32_t vnc_convert_rgb32_888(lfb_color_t rgb)
 #else
 #  error Unspecified/unsupported color format
 #endif
+
+/****************************************************************************
+ * Name: vnc_colors
+ *
+ * Description:
+ *  Test the update rectangle to see if it contains complex colors.  If it
+ *  contains only a few colors, then it may be a candidate for some type
+ *  run-length encoding.
+ *
+ * Input Parameters:
+ *   session   - An instance of the session structure.
+ *   rect      - The update region in the local frame buffer.
+ *   maxcolors - The maximum number of colors that should be returned.  This
+ *               currently cannot exceed eight.
+ *   colors    - The top 'maxcolors' most frequency colors are returned.
+ *
+ * Returned Value:
+ *   The number of valid colors in the colors[] array are returned, the
+ *   first entry being the most frequent.  A negated errno value is returned
+ *   if the colors cannot be determined.  This would be the case if the color
+ *   depth is > 8 and there are more than 'maxcolors' colors in the update
+ *   rectangle.
+ *
+ ****************************************************************************/
+
+int vnc_colors(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect,
+               unsigned int maxcolors, FAR lfb_color_t *colors)
+{
+#if RFB_PIXELDEPTH > 8
+  FAR const lfb_color_t *rowstart;
+  FAR const lfb_color_t *pixptr;
+  lfb_color_t pixel;
+  unsigned int counts[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+  nxgl_coord_t x;
+  nxgl_coord_t y;
+  int ncolors = 0;
+  int pixndx;
+  int maxndx;
+  int cmpndx;
+
+  DEBUGASSERT(session != NULL && rect != NULL && maxcolors <= 8 && colors != NULL);
+
+  /* Pointer to the first pixel in the first row in the local framebuffer */
+
+  rowstart = (FAR lfb_color_t *)
+    (session->fb + RFB_STRIDE * rect->pt1.y + RFB_BYTESPERPIXEL * rect->pt1.x);
+
+  /* Loop for each row in the rectangle */
+
+  for (y = rect->pt1.y; y <= rect->pt2.y; y++)
+    {
+      /* Loop for each column in the row */
+
+      pixptr = rowstart;
+      for (x = rect->pt1.x; x <= rect->pt2.x; x++)
+        {
+          /* Compare this pix to all of the others we have seen */
+
+          pixel = *pixptr++;
+          for (pixndx = 0; pixndx < ncolors; pixndx++)
+            {
+              if (colors[pixndx] == pixel)
+                {
+                  break;
+                }
+            }
+
+          /* Have we seen this color before? */
+
+          if (pixndx < ncolors)
+            {
+              /* Yes.. just increment the count of the number of times we
+               * have seen it.
+               */
+
+              counts[pixndx]++;
+            }
+
+           /* Do we have space for another color? */
+
+          else if (ncolors >= maxcolors)
+            {
+              /* No, then bail.  We don't have enough memory to deal with
+               * large number of colors.
+               */
+
+              return -E2BIG;
+            }
+
+          /* Add the new color to the list of colors that we have found */
+
+          else
+            {
+              colors[ncolors] = pixel;
+              counts[ncolors] = 1;
+              ncolors++;
+            }
+        }
+
+      /* Set the point to the start of the next row */
+
+      rowstart = (FAR lfb_color_t *)((uintptr_t)rowstart + RFB_STRIDE);
+    }
+
+  /* Now sort the colors by how often we saw them with the most frequent
+   * color in the first position.
+   */
+
+  /* Loop for colors N={0..(ncolors-1)} */
+
+  for (pixndx = 0; pixndx < ncolors - 1; pixndx++)
+    {
+      /* Compare color N with with colors M={(N_1)..ncolors} */
+
+      maxndx = pixndx;
+      for (cmpndx = maxndx + 1; cmpndx < ncolors; cmpndx++)
+        {
+          /* Have we seen color M more often that color N? */
+
+          if (counts[cmpndx] > counts[maxndx])
+            {
+              /* Yes.. then color M has been seen more frequently */
+
+              maxndx = cmpndx;
+            }
+        }
+
+      /* Do nothing if color N is the most often seen */
+
+      if (maxndx != pixndx)
+        {
+          /* Otherwise swap color N and color M */
+          /* Remember color N */
+
+          lfb_color_t tmpcolor = colors[pixndx];
+          int tmpcount         = counts[pixndx];
+
+          /* Set color N to color M */
+
+          colors[pixndx]       = colors[maxndx];
+          counts[pixndx]       = counts[maxndx];
+
+          /* Set color M to color N */
+
+          colors[maxndx]       = tmpcolor;
+          counts[maxndx]       = tmpcount;
+        }
+    }
+
+  /* And return the number of colors that we found */
+
+  return ncolors;
+
+#else
+  /* For small colors, we can keep a local array for all color formats and
+   * always return the exact result, no matter now many colors.  OR we could
+   * just remove this conditional compilation and live with 8 colors max.
+   */
+#  error No support for small colors
+#endif
+}

graphics/vnc/server/vnc_negotiate.c

@@ -98,6 +98,7 @@ int vnc_negotiate(FAR struct vnc_session_s *session)
   FAR struct rfb_serverinit_s *serverinit;
   FAR struct rfb_pixelfmt_s *pixelfmt;
   FAR struct rfb_setpixelformat_s *setformat;
+  FAR struct rfb_setencodings_s *encodings;
   ssize_t nsent;
   ssize_t nrecvd;
   size_t len;
@@ -396,14 +397,35 @@ int vnc_negotiate(FAR struct vnc_session_s *session)
       return ret;
     }
 
-  /* Receive supported encoding types from client, but ignore them.
-   * we will do only raw format.
-   */
+  /* Receive supported encoding types from client. */
 
   gvdbg("Receive encoding types\n");
 
-  (void)psock_recv(&session->connect, session->inbuf,
+  encodings = (FAR struct rfb_setencodings_s *)session->inbuf;
+
+  nrecvd = psock_recv(&session->connect, encodings,
                    CONFIG_VNCSERVER_INBUFFER_SIZE, 0);
+  if (nrecvd < 0)
+    {
+      errcode = get_errno();
+      gdbg("ERROR: Receive SetEncodings failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
+    }
+
+  if (nrecvd > 0 && encodings->msgtype == RFB_SETENCODINGS_MSG)
+    {
+      DEBUGASSERT(nrecvd >=  SIZEOF_RFB_SETENCODINGS_S(0));
+
+      /* Pick out any mutually supported encodings */
+
+      ret = vnc_client_encodings(session, encodings);
+      if (ret < 0)
+        {
+          gdbg("ERROR: vnc_set_encodings failed: %d\n", ret);
+          return ret;
+        }
+    }
 
   session->state = VNCSERVER_CONFIGURED;
   return OK;

graphics/vnc/server/vnc_raw.c

@@ -75,7 +75,7 @@ static size_t vnc_copy8(FAR struct vnc_session_s *session,
                          vnc_convert8_t convert)
 {
   FAR struct rfb_framebufferupdate_s *update;
-  FAR const uint16_t *srcleft;
+  FAR const lfb_color_t *srcleft;
   FAR const lfb_color_t *src;
   FAR uint8_t *dest;
   nxgl_coord_t x;
@@ -133,7 +133,7 @@ static size_t vnc_copy16(FAR struct vnc_session_s *session,
                          vnc_convert16_t convert)
 {
   FAR struct rfb_framebufferupdate_s *update;
-  FAR const uint16_t *srcleft;
+  FAR const lfb_color_t *srcleft;
   FAR const lfb_color_t *src;
   FAR uint16_t *dest;
   nxgl_coord_t x;
@@ -142,11 +142,11 @@ static size_t vnc_copy16(FAR struct vnc_session_s *session,
   /* Destination rectangle start address */
 
   update = (FAR struct rfb_framebufferupdate_s *)session->outbuf;
-  dest   = (FAR lfb_color_t *)update->rect[0].data;
+  dest   = (FAR uint16_t *)update->rect[0].data;
 
   /* Source rectangle start address (left/top)*/
 
-  srcleft = (FAR uint16_t *)(session->fb + RFB_STRIDE * row + RFB_BYTESPERPIXEL * col);
+  srcleft = (FAR lfb_color_t *)(session->fb + RFB_STRIDE * row + RFB_BYTESPERPIXEL * col);
 
   /* Transfer each row from the source buffer into the update buffer */
 
@@ -190,7 +190,7 @@ static size_t vnc_copy32(FAR struct vnc_session_s *session,
                          vnc_convert32_t convert)
 {
   FAR struct rfb_framebufferupdate_s *update;
-  FAR const uint16_t *srcleft;
+  FAR const lfb_color_t *srcleft;
   FAR const lfb_color_t *src;
   FAR uint32_t *dest;
   nxgl_coord_t x;
@@ -234,12 +234,12 @@ static size_t vnc_copy32(FAR struct vnc_session_s *session,
  *  must be supported by all VNC clients.
  *
  * Input Parameters:
- *   pixel - The src color in local framebuffer format.
+ *   session - An instance of the session structure.
  *   rect  - Describes the rectangle in the local framebuffer.
  *
  * Returned Value:
  *   Zero (OK) on success; A negated errno value is returned on failure that
- *   indicates the the natture of the failure.  A failure is only returned
+ *   indicates the the nature of the failure.  A failure is only returned
  *   in cases of a network failure and unexpected internal failures.
  *
  ****************************************************************************/

graphics/vnc/server/vnc_receiver.c

@@ -240,7 +240,13 @@ int vnc_receiver(FAR struct vnc_session_s *session)
                     }
                   else
                     {
-                      /* REVISIT: SetEncodings is currently ignored */
+                      /* Pick out any mutually supported encodings */
+
+                      ret = vnc_client_encodings(session, encodings);
+                      if (ret < 0)
+                        {
+                          gdbg("ERROR: vnc_set_encodings failed: %d\n", ret);
+                        }
                     }
                 }
             }
@@ -418,3 +424,53 @@ int vnc_receiver(FAR struct vnc_session_s *session)
 
   return -ENOSYS;
 }
+
+/****************************************************************************
+ * Name: vnc_client_encodings
+ *
+ * Description:
+ *   Pick out any mutually supported encodings from the Client-to-Server
+ *   SetEncodings message
+ *
+ * Input Parameters:
+ *   session   - An instance of the session structure.
+ *   encodings - The received SetEncodings message
+ *
+ * Returned Value:
+ *   At present, always returns OK
+ *
+ ****************************************************************************/
+
+int vnc_client_encodings(FAR struct vnc_session_s *session,
+                         FAR struct rfb_setencodings_s *encodings)
+{
+  uint32_t encoding;
+  unsigned int nencodings;
+  unsigned int i;
+
+  DEBUGASSERT(session != NULL && encodings != NULL);
+
+  /* Assume that there are no common encodings (other than RAW) */
+
+  session->rre = false;
+
+  /* Loop for each client supported encoding */
+
+  nencodings = rfb_getbe32(encodings->nencodings);
+  for (i = 0; i < nencodings; i++)
+    {
+      /* Get the next encoding */
+
+      encoding = rfb_getbe32(&encodings->encodings[i << 2]);
+
+      /* Only a limited support for of RRE is vailable now. */
+
+      if (encoding == RFB_ENCODING_RRE)
+        {
+          session->rre = true;
+          return OK;
+        }
+    }
+
+  return OK;
+}

graphics/vnc/server/vnc_rre.c

@@ -0,0 +1,192 @@
+/****************************************************************************
+ * graphics/vnc/vnc_rre.c
+ *
+ *   Copyright (C) 2016 Gregory Nutt. All rights reserved.
+ *   Author: Gregory Nutt <gnutt@nuttx.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ * 3. Neither the name NuttX nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdint.h>
+#include <assert.h>
+#include <errno.h>
+#include <debug.h>
+
+#include "vnc_server.h"
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+struct rre_data_s
+{
+#if RFB_BITSPERPIXEL == 16
+  struct rfb_rrehdr16_s hdr;    /*    Header with background pixel */
+  struct rfb_rrerect16_s srect; /*    Sub-rectangle */
+#elif RFB_BITSPERPIXEL == 32
+  struct rfb_rrehdr32_s hdr;    /*    Header with background pixel */
+  struct rfb_rrerect32_s srect; /*    Sub-rectangle */
+#endif
+};
+
+struct rre_rectangle_s
+{
+  uint8_t xpos[2];              /* U16 X position */
+  uint8_t ypos[2];              /* U16 Y position */
+  uint8_t width[2];             /* U16 Width */
+  uint8_t height[2];            /* U16 Height */
+  uint8_t encoding[4];          /* S32 Encoding type = RFB_ENCODING_RRE */
+  struct rre_data_s data;       /* Pixel data, actual size varies */
+};
+
+struct rre_framebufferupdate_s
+{
+  uint8_t msgtype;              /* U8  Message type = RFB_FBUPDATE_MSG*/
+  uint8_t padding;
+  uint8_t nrect[2];             /* U16 Number of rectangles  = 1*/
+  struct rre_rectangle_s rect;  /*     RRE encoded rectangle */
+};
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: vnc_rre
+ *
+ * Description:
+ *  This function does not really do RRE encoding.  It just checks if the
+ *  update region is one color then uses the RRE encoding format to send
+ *  the constant color rectangle.
+ *
+ * Input Parameters:
+ *   session - An instance of the session structure.
+ *   rect  - Describes the rectangle in the local framebuffer.
+ *
+ * Returned Value:
+ *   Zero is returned if RRE coding was not performed (but not error was)
+ *   encountered.  Otherwise, eith the size of the framebuffer update
+ *   message is returned on success or a negated errno value is returned on
+ *   failure that indicates the the nature of the failure.  A failure is
+ *   only returned in cases of a network failure and unexpected internal
+ *   failures.
+ *
+ ****************************************************************************/
+
+int vnc_rre(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect)
+{
+  FAR struct rre_framebufferupdate_s *rre;
+  FAR struct rre_rectangle_s *rrect;
+  FAR struct rre_data_s *rdata;
+  lfb_color_t bgcolor;
+  nxgl_coord_t width;
+  nxgl_coord_t height;
+  ssize_t nsent;
+  int ret;
+
+  /* Check if the client supports the RRE encoding */
+
+  if (session->rre)
+    {
+      /* Check if the update region contains only a single color */
+
+      ret = vnc_colors(session, rect, 1, &bgcolor);
+      if (ret == 1)
+        {
+          width         = rect->pt2.x - rect->pt1.x + 1;
+          height        = rect->pt2.y - rect->pt1.y + 1;
+
+          /* Format the FrameBuffer Update with a single RRE encoded
+           * rectangle.
+           */
+
+          rre           = (FAR struct rre_framebufferupdate_s *)session->outbuf;
+          rre->msgtype  = RFB_FBUPDATE_MSG;
+          rre->padding  = 0;
+          rfb_putbe16(rre->nrect,          1);
+
+          rrect         = (FAR struct rre_rectangle_s *)&rre->rect;
+          rfb_putbe16(rrect->xpos,         rect->pt1.x);
+          rfb_putbe16(rrect->ypos,         rect->pt1.y);
+          rfb_putbe16(rrect->width,        width);
+          rfb_putbe16(rrect->height,       height);
+          rfb_putbe32(rrect->encoding,     RFB_ENCODING_RRE);
+
+          rdata         = (FAR struct rre_data_s *)&rrect->data;
+          rfb_putbe32(rdata->hdr.nsubrects, 1);
+#if RFB_BITSPERPIXEL == 16
+          rfb_putbe16(rdata->hdr.pixel,    bgcolor);
+          rfb_putbe16(rdata->srect.pixel,  bgcolor);
+#elif RFB_BITSPERPIXEL == 32
+          rfb_putbe32(rdata->hdr.pixel,    bgcolor);
+          rfb_putbe32(rdata->srect.pixel,  bgcolor);
+#endif
+          rfb_putbe16(rdata->srect.xpos,   rect->pt1.x);
+          rfb_putbe16(rdata->srect.ypos,   rect->pt1.y);
+          rfb_putbe16(rdata->srect.width,  width);
+          rfb_putbe16(rdata->srect.height, height);
+
+          /* At the very last most, make certain that the supported encoding
+           * has not changed asynchronously.
+           */
+
+          if (session->rre)
+            {
+              /* Okay send until all of the bytes are out.  This may
+               * loop for the case where TCP write buffering is enabled
+               * and there are a limited number of IOBs available.
+               */
+
+              nsent = psock_send(&session->connect, rre,
+                                 sizeof(struct rre_framebufferupdate_s), 0);
+              if (nsent < 0)
+                {
+                  int errcode = get_errno();
+                  gdbg("ERROR: Send RRE FrameBufferUpdate failed: %d\n",
+                       errcode);
+                  DEBUGASSERT(errcode > 0);
+                  return -errcode;
+                }
+
+              DEBUGASSERT(nsent == sizeof(struct rre_framebufferupdate_s));
+              return sizeof(struct rre_framebufferupdate_s);
+            }
+
+          return -EINVAL;
+        }
+    }
+
+  return 0;
+}

graphics/vnc/server/vnc_server.h

@@ -210,6 +210,7 @@ struct vnc_session_s
   uint8_t display;             /* Display number (for debug) */
   volatile uint8_t colorfmt;   /* Remote color format (See include/nuttx/fb.h) */
   volatile uint8_t bpp;        /* Remote bits per pixel */
+  volatile bool rre;           /* Remote supports RRE encoding */
   FAR uint8_t *fb;             /* Allocated local frame buffer */
 
   /* Updater information */
@@ -335,6 +336,25 @@ int vnc_negotiate(FAR struct vnc_session_s *session);
 int vnc_client_pixelformat(FAR struct vnc_session_s *session,
                            FAR struct rfb_pixelfmt_s *pixelfmt);
 
+/****************************************************************************
+ * Name: vnc_client_encodings
+ *
+ * Description:
+ *   Pick out any mutually supported encodings from the Client-to-Server
+ *   SetEncodings message
+ *
+ * Input Parameters:
+ *   session   - An instance of the session structure.
+ *   encodings - The received SetEncodings message
+ *
+ * Returned Value:
+ *   At present, always returns OK
+ *
+ ****************************************************************************/
+
+int vnc_client_encodings(FAR struct vnc_session_s *session,
+                         FAR struct rfb_setencodings_s *encodings);
+
 /****************************************************************************
  * Name: vnc_start_updater
  *
@@ -404,6 +424,30 @@ int vnc_update_rectangle(FAR struct vnc_session_s *session,
 
 int vnc_receiver(FAR struct vnc_session_s *session);
 
+/****************************************************************************
+ * Name: vnc_rre
+ *
+ * Description:
+ *  This function does not really do RRE encoding.  It just checks if the
+ *  update region is one color then uses the RRE encoding format to send
+ *  the constant color rectangle.
+ *
+ * Input Parameters:
+ *   session - An instance of the session structure.
+ *   rect  - Describes the rectangle in the local framebuffer.
+ *
+ * Returned Value:
+ *   Zero is returned if RRE coding was not performed (but not error was)
+ *   encountered.  Otherwise, eith the size of the framebuffer update
+ *   message is returned on success or a negated errno value is returned on
+ *   failure that indicates the the nature of the failure.  A failure is
+ *   only returned in cases of a network failure and unexpected internal
+ *   failures.
+ *
+ ****************************************************************************/
+
+int vnc_rre(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect);
+
 /****************************************************************************
  * Name: vnc_raw
  *
@@ -412,12 +456,12 @@ int vnc_receiver(FAR struct vnc_session_s *session);
  *  must be supported by all VNC clients.
  *
  * Input Parameters:
- *   pixel - The src color in local framebuffer format.
+ *   session - An instance of the session structure.
  *   rect  - Describes the rectangle in the local framebuffer.
  *
  * Returned Value:
  *   Zero (OK) on success; A negated errno value is returned on failure that
- *   indicates the the natture of the failure.  A failure is only returned
+ *   indicates the the nature of the failure.  A failure is only returned
  *   in cases of a network failure and unexpected internal failures.
  *
  ****************************************************************************/
@@ -486,6 +530,33 @@ uint16_t vnc_convert_rgb16_555(lfb_color_t rgb);
 uint16_t vnc_convert_rgb16_565(lfb_color_t rgb);
 uint32_t vnc_convert_rgb32_888(lfb_color_t rgb);
 
+/****************************************************************************
+ * Name: vnc_colors
+ *
+ * Description:
+ *  Test the update rectangle to see if it contains complex colors.  If it
+ *  contains only a few colors, then it may be a candidate for some type
+ *  run-length encoding.
+ *
+ * Input Parameters:
+ *   session   - An instance of the session structure.
+ *   rect      - The update region in the local frame buffer.
+ *   maxcolors - The maximum number of colors that should be returned.  This
+ *               currently cannot exceed eight.
+ *   colors    - The top 'maxcolors' most frequency colors are returned.
+ *
+ * Returned Value:
+ *   The number of valid colors in the colors[] array are returned, the
+ *   first entry being the most frequent.  A negated errno value is returned
+ *   if the colors cannot be determined.  This would be the case if the color
+ *   depth is > 8 and there are more than 'maxcolors' colors in the update
+ *   rectangle.
+ *
+ ****************************************************************************/
+
+int vnc_colors(FAR struct vnc_session_s *session, FAR struct nxgl_rect_s *rect,
+               unsigned int maxcolors, FAR lfb_color_t *colors);
+
 #undef EXTERN
 #ifdef __cplusplus
 }

graphics/vnc/server/vnc_updater.c

@@ -325,9 +325,15 @@ static FAR void *vnc_updater(FAR void *arg)
       srcrect = vnc_remove_queue(session);
       DEBUGASSERT(srcrect != NULL);
 
-      /* Perform the framebuffer update using the RAW encoding */
+      /* Attempt to use RRE encoding */
 
-      ret = vnc_raw(session, &srcrect->rect);
+      ret = vnc_rre(session, &srcrect->rect);
+      if (ret == 0)
+        {
+          /* Perform the framebuffer update using the default RAW encoding */
+
+          ret = vnc_raw(session, &srcrect->rect);
+        }
 
       /* Release the update structure */
 
@@ -469,13 +475,27 @@ int vnc_update_rectangle(FAR struct vnc_session_s *session,
       update = vnc_alloc_update(session);
       DEBUGASSERT(update != NULL);
 
-      /* Copy the rectangle into the update structure */
+      /* Clip and copy the rectangle into the update structure */
 
-      memcpy(&update->rect, rect, sizeof(struct nxgl_rect_s));
+      update->rect.pt1.x = MAX(rect->pt1.x, 0);
+      update->rect.pt1.y = MAX(rect->pt1.y, 0);
+      update->rect.pt2.x = MIN(rect->pt2.x, (CONFIG_VNCSERVER_SCREENWIDTH - 1));
+      update->rect.pt2.y = MIN(rect->pt2.y, (CONFIG_VNCSERVER_SCREENHEIGHT - 1));
 
-      /* Add the upate to the end of the update queue. */
+      /* Make sure that the rectangle still has area after clipping */
 
-      vnc_add_queue(session, update);
+      if (nxgl_nullrect(rect))
+        {
+          /* No.. free the structure and ignore the update */
+
+          vnc_free_update(session, update);
+        }
+      else
+        {
+          /* Yes.. add the upate to the end of the update queue. */
+
+          vnc_add_queue(session, update);
+        }
     }
 
   /* Since we ignore bad rectangles and wait for updata structures, there is

include/nuttx/video/rfb.h

@@ -716,6 +716,15 @@ struct rfb_rrerect16_s
   uint8_t height[2];             /* U16 Height */
 };
 
+struct rfb_rrerect32_s
+{
+  uint8_t pixel[4];              /* U32 sub-rect pixel value */
+  uint8_t xpos[2];               /* U16 X position */
+  uint8_t ypos[2];               /* U16 Y position */
+  uint8_t width[2];              /* U16 Width */
+  uint8_t height[2];             /* U16 Height */
+};
+
 /* 6.6.4 Hextile encoding
  *
  * "Hextile is a variation on the RRE idea. Rectangles are split up into