root/usr/src/cmd/bhyve/common/rfb_impl.h
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
 */

#ifndef _RFB_IMPL_H
#define _RFB_IMPL_H

#include <stdatomic.h>
#include <stdbool.h>
#include <zlib.h>
#include <sys/types.h>
#include <sys/list.h>

#include "mevent.h"

/*
 * The ProtocolVersion message consists of 12 bytes interpreted as a string of
 * ASCII characters in the format "RFB xxx.yyy\n" where xxx and yyy are the
 * major and minor version numbers, padded with zeros.
 */
#define RFB_VERSION                     "RFB 003.008\n"
#define RFB_VERSION_LEN                 (sizeof (RFB_VERSION) - 1)

_Static_assert(RFB_VERSION_LEN == 12, "RFB_VERSION length incorrect");

/* Keep synchronised with pci_fbuf.c */
#define RFB_MAX_WIDTH                   1920
#define RFB_MAX_HEIGHT                  1200

#define RFB_MAX_CLIENTS                 10

/* Framebuffer pixel format */
#define RFB_PIX_BPP                     32
#define RFB_PIX_DEPTH                   24
#define RFB_PIX_RSHIFT                  16
#define RFB_PIX_GSHIFT                  8
#define RFB_PIX_BSHIFT                  0
#define RFB_PIX_RMAX                    255
#define RFB_PIX_GMAX                    255
#define RFB_PIX_BMAX                    255

#define RFB_ZLIB_BUFSZ                  (RFB_MAX_WIDTH * RFB_MAX_HEIGHT * 4)

#define RFB_PIX_PER_CELL                32
#define RFB_PIXCELL_SHIFT               5
#define RFB_PIXCELL_MASK                0x1f
#define RFB_SENDALL_THRESH              25

#define RFB_SEL_DELAY_US                10000
#define RFB_SCREEN_REFRESH_DELAY        33300   /* 30 Hz */
#define RFB_SCREEN_POLL_DELAY           (RFB_SCREEN_REFRESH_DELAY / 2)

/* Client-to-server message types */
#define RFBP_CS_SET_PIXEL_FORMAT        0
#define RFBP_CS_SET_ENCODINGS           2
#define RFBP_CS_UPDATE_REQUEST          3
#define RFBP_CS_KEY_EVENT               4
#define RFBP_CS_POINTER_EVENT           5
#define RFBP_CS_CUT_TEXT                6
#define RFBP_CS_QEMU                    255
#define RFBP_CS_QEMU_KEVENT             0

/* Server-to-client message types */
#define RFBP_SC_UPDATE                  0
#define RFBP_SC_SET_COLOURMAP_ENTRIES   1
#define RFBP_SC_BELL                    2
#define RFBP_SC_CUT_TEXT                3

/* Encodings */
#define RFBP_ENCODING_RAW               0
#define RFBP_ENCODING_ZLIB              6
/* Pseudo-encodings */
#define RFBP_ENCODING_RESIZE            -223
#define RFBP_ENCODING_EXT_KEVENT        -258    /* QEMU ext. key event */
#define RFBP_ENCODING_DESKTOP_NAME      -307

/* Security types */
#define RFBP_SECURITY_INVALID           0
#define RFBP_SECURITY_NONE              1
#define RFBP_SECURITY_VNC_AUTH          2

#define RFBP_SECURITY_VNC_AUTH_LEN      16
#define RFBP_SECURITY_VNC_PASSWD_LEN    8

typedef enum rfb_loglevel {
        RFB_LOGDEBUG,
        RFB_LOGWARN,
        RFB_LOGERR
} rfb_loglevel_t;

typedef enum rfb_encodings {
        RFB_ENCODING_RAW                = (1ULL << 0),
        RFB_ENCODING_ZLIB               = (1ULL << 1),
        RFB_ENCODING_RESIZE             = (1ULL << 2),
        RFB_ENCODING_EXT_KEVENT         = (1ULL << 3),
        RFB_ENCODING_DESKTOP_NAME       = (1ULL << 4)
} rfb_encodings_t;

typedef enum rfb_cver {
        RFB_CVER_3_3,
        RFB_CVER_3_7,
        RFB_CVER_3_8
} rfb_cver_t;

typedef struct rfb_pixfmt {
        uint8_t                 rp_bpp;
        uint8_t                 rp_depth;
        uint8_t                 rp_bigendian;
        uint8_t                 rp_truecolour;
        uint16_t                rp_r_max;
        uint16_t                rp_g_max;
        uint16_t                rp_b_max;
        uint8_t                 rp_r_shift;
        uint8_t                 rp_g_shift;
        uint8_t                 rp_b_shift;
        uint8_t                 rp_pad[3];
} __packed rfb_pixfmt_t;

/* Server-to-client message formats */

typedef struct rfb_server_info {
        uint16_t                rsi_width;
        uint16_t                rsi_height;
        rfb_pixfmt_t            rsi_pixfmt;
        uint32_t                rsi_namelen;
} __packed rfb_server_info_t;

typedef struct rfb_server_update_msg {
        uint8_t                 rss_type;
        uint8_t                 rss_pad;
        uint16_t                rss_numrects;
} __packed rfb_server_update_msg_t;

typedef struct rfb_rect_hdr {
        uint16_t                rr_x;
        uint16_t                rr_y;
        uint16_t                rr_width;
        uint16_t                rr_height;
        uint32_t                rr_encoding;
} __packed rfb_rect_hdr_t;

/* Client-to-server message formats */

typedef struct rfb_cs_pixfmt_msg  {
        uint8_t                 rp_pad[3];
        rfb_pixfmt_t            rp_pixfmt;
} __packed rfb_cs_pixfmt_msg_t;

typedef struct rfb_cs_update_msg {
        uint8_t                 rum_incremental;
        uint16_t                rum_x;
        uint16_t                rum_y;
        uint16_t                rum_width;
        uint16_t                rum_height;
} __packed rfb_cs_update_msg_t;

typedef struct rfb_cs_encodings_msg {
        uint8_t                 re_pad;
        uint16_t                re_numencs;
} __packed rfb_cs_encodings_msg_t;

typedef struct rfb_cs_key_event_msg {
        uint8_t                 rke_down;
        uint16_t                rke_pad;
        uint32_t                rke_sym;
} __packed rfb_cs_key_event_msg_t;

typedef struct rfb_cs_pointer_event_msg {
        uint8_t                 rpe_button;
        uint16_t                rpe_x;
        uint16_t                rpe_y;
} __packed rfb_cs_pointer_event_msg_t;

typedef struct rfb_cs_cut_text_msg {
        uint8_t                 rct_padding[3];
        uint32_t                rct_length;
} __packed rfb_cs_cut_text_msg_t;

typedef struct rfb_cs_qemu_msg {
        uint8_t                 rq_subtype;
} __packed rfb_cs_qemu_msg_t;

typedef struct rfb_cs_qemu_extended_key_msg {
        uint16_t                rqek_down;
        uint32_t                rqek_sym;
        uint32_t                rqek_code;
} __packed rfb_cs_qemu_extended_key_msg_t;

/* Client/server data structures */

typedef struct rfb_server {
        list_node_t             rs_node;
        int                     rs_fd;
        const char              *rs_name;
        const char              *rs_password;

        struct mevent           *rs_connevent;

        uint_t                  rs_clientcount;
        pthread_mutex_t         rs_clientlock;
        list_t                  rs_clients;

        bool                    rs_exclusive;

        rfb_pixfmt_t            rs_pixfmt;
} rfb_server_t;

typedef struct rfb_client {
        list_node_t             rc_node;
        uint_t                  rc_instance;

        int                     rc_fd;
        rfb_server_t            *rc_s;
        rfb_server_info_t       rc_sinfo;
        pthread_t               rc_rx_tid;
        pthread_t               rc_tx_tid;

        int                     rc_width;
        int                     rc_height;
        size_t                  rc_cells;
        uint32_t                *rc_crc;
        uint32_t                *rc_crc_tmp;
        z_stream                rc_zstream;
        uint8_t                 *rc_zbuf;

        struct bhyvegc_image    rc_gci;

        rfb_cver_t              rc_cver;
        rfb_encodings_t         rc_encodings;

        atomic_bool             rc_closing;
        atomic_bool             rc_pending;
        atomic_bool             rc_input_detected;
        atomic_bool             rc_crc_reset;
        atomic_bool             rc_send_fullscreen;

        bool                    rc_custom_pixfmt;
        bool                    rc_keyevent_sent;
} rfb_client_t;

#endif  /* _RFB_IMPL_H */