VNC: Finish v3.3 negotiation

graphics/vnc/server/Kconfig

@@ -41,6 +41,26 @@ config VNCSERVER_STACKSIZE
 	int "VNC server stack size"
 	default 2048
 
+choice
+	prompt "VNC color format"
+	default VNCSERVER_COLORFMT_RGB16
+
+config VNCSERVER_COLORFMT_RGB16
+	bool "RGB16 5:6:5"
+
+config VNCSERVER_COLORFMT_RGB32
+	bool "RGB24 (32-bit) or RGB32 (w/tranparency)"
+
+endchoice # VNC color format
+
+config VNCSERVER_SCREENWIDTH
+	int "Framebuffer width (pixels)"
+	default 320
+
+config VNCSERVER_SCREENHEIGHT
+	int "Framebuffer height (rows)"
+	default 240
+
 config VNCSERVER_KBDENCODE
 	bool "Encode keyboard input"
 	default n

graphics/vnc/server/vnc_fbdev.c

@@ -50,15 +50,6 @@
 
 #include "vnc_server.h"
 
-/****************************************************************************
- * Pre-processor Definitions
- ****************************************************************************/
-
-/* Framebuffer characteristics in bytes */
-
-#define FB_WIDTH(s)  (((s)->screen.w * (s)->bpp + 7) / 8)
-#define FB_SIZE(s)   (FB_WIDTH(s) * (s)->screen.h)
-
 /****************************************************************************
  * Private Types
  ****************************************************************************/
@@ -207,8 +198,8 @@ static int up_getplaneinfo(FAR struct fb_vtable_s *vtable, int planeno,
       DEBUGASSERT(session->fb != NULL);
 
       pinfo->fbmem    = (FAR void *)&session->fb;
-      pinfo->fblen    = FB_SIZE(session);
+      pinfo->fblen    = (uint32_t)session->stride * CONFIG_VNCSERVER_SCREENWIDTH;
-      pinfo->stride   = FB_WIDTH(session);
+      pinfo->stride   = (fb_coord_t)session->stride;
       pinfo->bpp      = session->bpp;
 
       return OK;

graphics/vnc/server/vnc_negotiate.c

@@ -42,7 +42,11 @@
 #include <stdint.h>
 #include <string.h>
 #include <errno.h>
+#include <assert.h>
+#include <debug.h>
 
+#include <nuttx/kmalloc.h>
+#include <nuttx/video/fb.h>
 #include <nuttx/video/rfb.h>
 
 #include "vnc_server.h"
@@ -57,28 +61,6 @@ static const char g_vncproto[] = RFB_PROTOCOL_VERSION_3p3;
 static const char g_vncproto[] = RFB_PROTOCOL_VERSION_3p8;
 #endif
 
-/****************************************************************************
- * Private Functions
- ****************************************************************************/
-
-/****************************************************************************
- * Name: nvc_xyz
- *
- * Description:
- *  [Re-]initialize a VNC session
- *
- * Input Parameters:
- *   session - the VNC session to be initialized
- *
- * Returned Value:
- *   None
- *
- ****************************************************************************/
-
-static void nvc_xyz(FAR struct vnc_session_s *session)
-{
-}
-
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -103,59 +85,258 @@ static void nvc_xyz(FAR struct vnc_session_s *session)
 #ifdef CONFIG_VNCSERVER_PROTO3p3
 int vnc_negotiate(FAR struct vnc_session_s *session)
 {
+  FAR struct rfb_sectype_s *sectype;
+  FAR struct rfb_serverinit_s *serverinit;
+  FAR struct rfb_pixelfmt_s *pixelfmt;
+  FAR struct rfb_setpixelformat_s *setformat;
+  ssize_t nsent;
   ssize_t nrecvd;
+  size_t alloc;
   size_t len;
   int errcode;
 
-  /* Inform the client of the VNC protocol */
+  /* Inform the client of the VNC protocol version */
 
   len = strlen(g_vncproto);
-  nrecvd = psock_send(&session->connect, g_vncproto, len, 0);
+  nsent = psock_send(&session->connect, g_vncproto, len, 0);
-  if (nrecvd < 0)
+  if (nsent < 0)
     {
-      goto errout_with_errno;
+      errcode = get_errno();
+      gdbg("ERROR: Send ProtocolVersion failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
     }
 
+  DEBUGASSERT(nsent == len);
+
+  /* Receive the echo of the protocol string */
+
   nrecvd = psock_recv(&session->connect, session->iobuf, len, 0);
   if (nrecvd <= 0)
     {
-      goto errout_with_errno;
+      errcode = get_errno();
+      gdbg("ERROR: Receive protocol confirmation failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
     }
 
-  /* Tell the client that we won't use any stinkin' security */
+  DEBUGASSERT(nrecvd == len);
 
+  /* Tell the client that we won't use any stinkin' security.
+   *
+   * "Version 3.3 The server decides the security type and sends a single
+   *  word:"
+   */
+
+  sectype = (FAR struct rfb_sectype_s *)session->iobuf;
+  rfb_putbe32(sectype->type, RFB_SECTYPE_NONE);
+
+  nsent = psock_send(&session->connect, sectype,
+                     sizeof(struct rfb_sectype_s), 0);
+  if (nsent < 0)
+    {
+      errcode = get_errno();
+      gdbg("ERROR: Send Security failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
+    }
+
+  DEBUGASSERT(nsent == sizeof(struct rfb_sectype_s));
+
+  /* Receive the ClientInit message
+   *
+   * "Once the client and server are sure that they’re happy to talk to one
+   *  another using the agreed security type, the protocol passes to the
+   *  initialisation phase. The client sends a ClientInit message followed
+   *  by the server sending a ServerInit message."
+   *
+   * In this implementation, the sharing flag is ignored.
+   */
+
+  nrecvd = psock_recv(&session->connect, session->iobuf,
+                      sizeof(struct rfb_clientinit_s), 0);
+  if (nrecvd < 0)
+    {
+      errcode = get_errno();
+      gdbg("ERROR: Receive ClientInit failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
+    }
+
+  DEBUGASSERT(nrecvd == sizeof(struct rfb_clientinit_s));
+
+  /* Send the ClientInit message
+   *
+   * "After receiving the ClientInit message, the server sends a ServerInit
+   *  message. This tells the client the width and height of the server’s 
+   *  framebuffer, its pixel format and the name associated with the desktop:
+   */
+
+  serverinit          = (FAR struct rfb_serverinit_s *)session->iobuf;
+
+  rfb_putbe16(serverinit->width, CONFIG_VNCSERVER_SCREENWIDTH);
+  rfb_putbe16(serverinit->height, CONFIG_VNCSERVER_SCREENHEIGHT);
+  rfb_putbe32(serverinit->namelen, 0);
+
+  pixelfmt            = &serverinit->format;
+  pixelfmt->bpp       = RFB_BITSPERPIXEL;
+  pixelfmt->depth     = RFB_PIXELDEPTH;
+  pixelfmt->bigendian = 0;
+  pixelfmt->truecolor = RFB_TRUECOLOR;
+
+  rfb_putbe16(pixelfmt->rmax, RFB_RMAX);
+  rfb_putbe16(pixelfmt->gmax, RFB_GMAX);
+  rfb_putbe16(pixelfmt->bmax, RFB_BMAX);
+
+  pixelfmt->rshift    = RFB_RSHIFT;
+  pixelfmt->gshift    = RFB_GSHIFT;
+  pixelfmt->bshift    = RFB_BSHIFT;
+
+  nsent = psock_send(&session->connect, serverinit,
+                     SIZEOF_RFB_SERVERINIT_S(0), 0);
+  if (nsent < 0)
+    {
+      errcode = get_errno();
+      gdbg("ERROR: Send ServerInit failed: %d\n", errcode);
+      return -errcode;
+    }
+
+  DEBUGASSERT(nsent == SIZEOF_RFB_SERVERINIT_S(0));
+
+  /* We now expect to receive the SetPixelFormat message from the client.
+   * This may override some of our framebuffer settings.
+   */
+
+  setformat = (FAR struct rfb_setpixelformat_s *)session->iobuf;
+
+  nrecvd = psock_recv(&session->connect, setformat,
+                      sizeof(struct rfb_setpixelformat_s), 0);
+  if (nrecvd < 0)
+    {
+      errcode = get_errno();
+      gdbg("ERROR: Receive SetFormat failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
+    }
+  else if (nrecvd != sizeof(struct rfb_setpixelformat_s))
+    {
+      /* Must not be a SetPixelFormat message? */
+
+      gdbg("ERROR: SetFormat wrong size: %d\n", (int)nrecvd);
+      return -EPROTO;
+    }
+  else if (setformat->msgtype != RFB_SETPIXELFMT_MSG)
+    {
+      gdbg("ERROR: Not a SetFormat message: %d\n", (int)setformat->type);
+      return -EPROTO;
+    }
+
+  /* Check if the client request format is one that we can handle. */
+
+  pixelfmt = &setformat->format;
+
+  if (pixelfmt->truecolor == 0)
+    {
+      /* At present, we support only TrueColor formats */
+
+      gdbg("ERROR: No support for palette colors\n");
+      return -ENOSYS;
+    }
+
+  if (pixelfmt->bpp == 16 && pixelfmt->depth == 15)
+    {
+      session->colorfmt = FB_FMT_RGB16_555;
+      session->bpp      = 16;
+    }
+  else if (pixelfmt->bpp == 16 && pixelfmt->depth == 16)
+    {
+      session->colorfmt = FB_FMT_RGB16_565;
+      session->bpp      = 16;
+    }
+  else if (pixelfmt->bpp == 32 && pixelfmt->depth == 24)
+    {
+      session->colorfmt = FB_FMT_RGB32;
+      session->bpp      = 32;
+    }
+  else if (pixelfmt->bpp == 32 && pixelfmt->depth == 32)
+    {
+      session->colorfmt = FB_FMT_RGB32;
+      session->bpp      = 32;
+    }
+  else
+    {
+      /* We do not support any other conversions */
+
+      gdbg("ERROR: No support for this BPP=%d and depth=%d\n",
+            pixelfmt->bpp, pixelfmt->depth);
+      return -ENOSYS;
+    }
+
+  session->screen.w = CONFIG_VNCSERVER_SCREENWIDTH;
+  session->screen.h = CONFIG_VNCSERVER_SCREENHEIGHT;
+
+  /* Now allocate the framebuffer memory */
+
+  len               = (session->bpp + 7) >> 3;
+  session->stride   = len * CONFIG_VNCSERVER_SCREENWIDTH;
+  alloc             = (size_t)session->stride * CONFIG_VNCSERVER_SCREENHEIGHT;
+
+  session->fb       = (FAR uint8_t *)kmm_zalloc(alloc);
+  if (session->fb)
+    {
+      gdbg("ERROR: Failed to allocate framebuffer memory: %lu\n",
+           (unsigned long)alloc);
+      return -ENOMEM;
+    }
+
+  /* Receive supported encoding types from client, but ignore them.
+   * we will do only raw format.
+   */
+
+  (void)psock_recv(&session->connect, session->iobuf,
+                   CONFIG_VNCSERVER_IOBUFFER_SIZE, 0);
+
+  session->state = VNCSERVER_CONFIGURED;
   return OK;
-
-errout_with_errno:
-  errcode = get_errno();
-
-errout_with_errcode:
-  DEBUGASSERT(errcode > 0);
-  return -errcode;
 }
 #endif
 
 #ifdef CONFIG_VNCSERVER_PROTO3p8
 int vnc_negotiate(FAR struct vnc_session_s *session)
 {
-  ssize_t nbytes;
+  ssize_t nsent;
+  ssize_t nrecvd;
+  size_t len;
 
-  /* Inform the client of the VNC protocol */
+  /* Inform the client of the VNC protocol version */
 
   len = strlen(g_vncproto);
-  nrecvd = psock_send(&session->connect, g_vncproto, len, 0);
+  nsent = psock_send(&session->connect, g_vncproto, len, 0);
-  if (nrecvd < 0)
+  if (nsent < 0)
     {
-      goto errout_with_errno;
+      errcode = get_errno();
+      gdbg("ERROR: Send ProtocolVersion failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
     }
 
+  DEBUGASSERT(nsent == len);
+
+  /* Receive the echo of the protocol string */
+
   nrecvd = psock_recv(&session->connect, session->iobuf, len, 0);
   if (nrecvd <= 0)
     {
-      goto errout_with_errno;
+      errcode = get_errno();
+      gdbg("ERROR: Receive protocol confirmation failed: %d\n", errcode);
+      DEBUGASSERT(errcode > 0);
+      return -errcode;
     }
 
+  DEBUGASSERT(nrecvd == len);
+
   /* Offer the client a choice of security -- where None is the only option. */
+#warning Missing logic
 
   return OK;
 }

graphics/vnc/server/vnc_server.h

@@ -67,6 +67,38 @@
 #  define CONFIG_VNCSERVER_NDISPLAYS 1
 #endif
 
+#if defined(CONFIG_VNCSERVER_COLORFMT_RGB16)
+#  define RFB_BITSPERPIXEL 16
+#  define RFB_PIXELDEPTH   16
+#  define RFB_TRUECOLOR    1
+#  define RFB_RMAX         0x1f
+#  define RFB_GMAX         0x3f
+#  define RFB_BMAX         0x1f
+#  define RFB_RSHIFT       11
+#  define RFB_GSHIFT       5
+#  define RFB_BSHIFT       0
+#elif defined(CONFIG_VNCSERVER_COLORFMT_RGB32)
+#  define RFB_BITSPERPIXEL 32
+#  define RFB_PIXELDEPTH   24
+#  define RFB_TRUECOLOR    1
+#  define RFB_RMAX         0xff
+#  define RFB_GMAX         0xff
+#  define RFB_BMAX         0xff
+#  define RFB_RSHIFT       16
+#  define RFB_GSHIFT       8
+#  define RFB_BSHIFT       0
+#else
+#  error Unspecified color format
+#endif
+
+#ifndef CONFIG_VNCSERVER_SCREENWIDTH
+#  define CONFIG_VNCSERVER_SCREENWIDTH 320
+#endif
+
+#ifndef CONFIG_VNCSERVER_SCREENHEIGHT
+#  define CONFIG_VNCSERVER_SCREENHEIGHT 240
+#endif
+
 #ifndef CONFIG_VNCSERVER_PRIO
 #  define CONFIG_VNCSERVER_PRIO 100
 #endif
@@ -88,11 +120,11 @@
 /* Miscellaneous */
 
 #ifndef MIN
-#  define MIN(a,b) (((a) < (b)) ? (a) : (b))
+#  define MIN(a,b)          (((a) < (b)) ? (a) : (b))
 #endif
 
 #ifndef MAX
-#  define MAX(a,b) (((a) > (b)) ? (a) : (b))
+#  define MAX(a,b)          (((a) > (b)) ? (a) : (b))
 #endif
 
 /****************************************************************************
@@ -127,6 +159,7 @@ struct vnc_session_s
 
   uint8_t colorfmt;            /* See include/nuttx/fb.h */
   uint8_t bpp;                 /* Bits per pixel */
+  size_t stride;               /* Width of a row in bytes */
   struct nxgl_size_s screen;   /* Size of the screen in pixels x rows */
   FAR uint8_t *fb;             /* Allocated local frame buffer */