tigervnc: switch to GnuTLS 3

Alexander Pyhalov

2016-12-12 99d2377971aa9ee1c991a53127699bf4de750731

tigervnc: switch to GnuTLS 3

 components/x11/tigervnc/Makefile

@@ -17,7 +17,7 @@

COMPONENT_NAME= tigervnc
COMPONENT_VERSION= 1.1.0
COMPONENT_REVISION= 1
COMPONENT_REVISION= 2
COMPONENT_SUMMARY= TigerVNC X11/VNC server
COMPONENT_SRC= $(COMPONENT_NAME)-$(COMPONENT_VERSION)
COMPONENT_ARCHIVE= $(COMPONENT_SRC).tar.gz
@@ -65,6 +65,15 @@

$(COMPONENT_DIR)/../xorg-server/xorg-server-$(XORG_SERVER_VERSION)/.prep:
    $(GMAKE) -C $(COMPONENT_DIR)/../xorg-server prep

# Force use of gnutls-3 pkgconfig during 2.x->3.x transition
GNUTLS_PKG_CONFIG_PATH_32 = /usr/lib/pkgconfig/gnutls-3
GNUTLS_PKG_CONFIG_PATH_64 = /usr/lib/$(MACH64)/pkgconfig/gnutls-3
PKG_CONFIG_PATH = $(GNUTLS_PKG_CONFIG_PATH_$(BITS)):$(PKG_CONFIG_PATH.$(BITS))

LD_OPTIONS.32 += -L/usr/lib/gnutls-3 -R/usr/lib/gnutls-3
LD_OPTIONS.64 += -L/usr/lib/$(MACH64)/gnutls-3 -R/usr/lib/$(MACH64)/gnutls-3
LD_OPTIONS += $(LD_OPTIONS.$(BITS))

COMPONENT_PREP_ACTION =        (cd $(@D) &&\
                    aclocal -I m4 && \
@@ -174,3 +183,21 @@
build: $(BUILD_32) $(BUILD_DIR)/.built_jar $(BUILD_DIR)/.built_Xvnc

install: $(INSTALL_32) $(BUILD_DIR)/.installed_jar $(BUILD_DIR)/.installed_Xvnc

REQUIRED_PACKAGES += image/library/libjpeg6
REQUIRED_PACKAGES += image/library/libjpeg6-ijg
REQUIRED_PACKAGES += library/gnutls-3
REQUIRED_PACKAGES += library/graphics/pixman
REQUIRED_PACKAGES += library/zlib
REQUIRED_PACKAGES += runtime/perl-522
REQUIRED_PACKAGES += system/library
REQUIRED_PACKAGES += system/library/g++-4-runtime
REQUIRED_PACKAGES += system/library/gcc-4-runtime
REQUIRED_PACKAGES += system/library/math
REQUIRED_PACKAGES += x11/library/libx11
REQUIRED_PACKAGES += x11/library/libxau
REQUIRED_PACKAGES += x11/library/libxdmcp
REQUIRED_PACKAGES += x11/library/libxext
REQUIRED_PACKAGES += x11/library/libxfont
REQUIRED_PACKAGES += x11/library/libxtst
REQUIRED_PACKAGES += x11/server/xserver-common

 components/x11/tigervnc/patches/13.gnutls-3.patch

New file
@@ -0,0 +1,847 @@
GnuTLS 3 support from upstream, extracted from these commits and backported
to TigerVNC 1.1 (including use of automake instead of Cmake):

From fe48cd4d2427c0262cd58b30c74331a9fce756c7 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Tue, 3 Jul 2012 14:43:38 +0000
Subject: [PATCH] Refactor the TLS code so that the push/pull functions are
 aware of their containing stream object. This is in
 preparation for supporting GnuTLS 3.x.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4921 3789f03b-4d11-0410-bbf8-ca57d06f2519
---
From 2137f4f78f2b80faf96707907cd08562d83f1216 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Tue, 3 Jul 2012 14:52:26 +0000
Subject: [PATCH] GnuTLS 3.x has removed gnutls_transport_set_global_errno()
 in favour of gnutls_transport_set_errno(). Make sure we
 call the right errno function depending on which GnuTLS
 we're using.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4922 3789f03b-4d11-0410-bbf8-ca57d06f2519
---
From 88c24edd8f7a793561104be50b6ecf2c85b42956 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Thu, 29 Jan 2015 13:12:22 +0100
Subject: [PATCH] Raise GnuTLS requirements to 3.x

This allows us to simplify things by getting rid of some old
compatibility code. People should really be using current versions
of GnuTLS anyway to stay secure.
---
From 27eb55e1975c4f558f6a53d573091e76064cc8e7 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Thu, 29 Jan 2015 13:31:06 +0100
Subject: [PATCH] Add parameter to override GnuTLS priority

---
From 660f1081d7cd1e4c763675ace8181ed34d377d34 Mon Sep 17 00:00:00 2001
From: Pierre Ossman <ossman@cendio.se>
Date: Wed, 2 Mar 2011 12:44:12 +0000
Subject: [PATCH] Make it easier to read string parameters by providing an
 implicit conversion to const char*.

git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4307 3789f03b-4d11-0410-bbf8-ca57d06f2519
---

diff --git a/common/os/Makefile.am b/common/os/Makefile.am
index ab4f2d7..d87a0da 100644
--- common/os/Makefile.am
+++ common/os/Makefile.am
@@ -1,8 +1,8 @@
 noinst_LTLIBRARIES = libos.la
 
-HDRS = net.h print.h os.h tls.h
+HDRS = net.h print.h os.h
 
-libos_la_SOURCES = $(HDRS) print.c net.c os.cxx tls.cxx
+libos_la_SOURCES = $(HDRS) print.c net.c os.cxx
 
 libos_la_CPPFLAGS = -I$(top_srcdir)/common
 
diff --git a/common/os/tls.cxx b/./prev/common/os/tls.cxx
deleted file mode 100644
index c092996..0000000
--- common/os/tls.cxx
+++ /dev/null
@@ -1,198 +0,0 @@
-/* Copyright (C) 2011 TightVNC Team.  All Rights Reserved.
- *
- * This is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this software; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
- * USA.
- */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <os/tls.h>
-
-#include <iomanip>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sstream>
-#include <sys/types.h>
-#include <time.h>
-
-using namespace std;
-
-#if defined(HAVE_GNUTLS) && !defined(WIN32)
-#include <gnutls/gnutls.h>
-#include <gnutls/x509.h>
-
-#ifndef HAVE_GNUTLS_X509_CRT_PRINT
-
-/* Ancient GNUTLS... */
-#if !defined(GNUTLS_VERSION_NUMBER) && !defined(LIBGNUTLS_VERSION_NUMBER)
-#define GNUTLS_DIG_SHA1 GNUTLS_DIG_SHA
-#endif
-
-#define UNKNOWN_SUBJECT(err) \
-    do { \
-        ss << "unknown subject (" << gnutls_strerror(err) << "), "; \
-    } while (0)
-
-#define UNKNOWN_ISSUER(err) \
-    do { \
-        ss << "unknown issuer (" << gnutls_strerror(err) << "), "; \
-    } while (0)
-
-
-static void
-hexprint(ostringstream &ss, const char *data, size_t len)
-{
-    size_t j;
-    char tmp[3];
-
-    if (len == 0)
-        ss << "00";
-    else {
-        for (j = 0; j < len; j++) {
-            snprintf(tmp, sizeof(tmp), "%.2x", (unsigned char) data[j]);
-            ss << tmp;
-        }
-    }
-}
-
-/* Implementation based on gnutls_x509_crt_print from GNUTLS */
-int
-gnutls_x509_crt_print(gnutls_x509_crt_t cert,
-              gnutls_certificate_print_formats_t format,
-              gnutls_datum_t * out)
-{
-    ostringstream ss;
-	
-    int err;
-
-    char *dn;
-    size_t dn_size = 0;
-
-    /* Subject */
-    err = gnutls_x509_crt_get_dn(cert, NULL, &dn_size);
-    if (err != GNUTLS_E_SHORT_MEMORY_BUFFER)
-        UNKNOWN_SUBJECT(err);
-    else {
-        dn = (char *)malloc(dn_size);
-        if (dn == NULL) {
-            UNKNOWN_SUBJECT(GNUTLS_E_MEMORY_ERROR);
-        } else {
-            err = gnutls_x509_crt_get_dn(cert, dn, &dn_size);
-            if (err < 0) {
-                UNKNOWN_SUBJECT(err);
-            } else
-                ss << "subject `" << dn << "', ";
-            free(dn);
-        }
-    }
-
-    /* Issuer */
-    dn = NULL;
-    dn_size = 0;
-    err = gnutls_x509_crt_get_issuer_dn(cert, NULL, &dn_size);
-    if (err != GNUTLS_E_SHORT_MEMORY_BUFFER)
-        UNKNOWN_ISSUER(err);
-    else {
-        dn = (char *)malloc(dn_size);
-        if (dn == NULL) {
-            UNKNOWN_ISSUER(GNUTLS_E_MEMORY_ERROR);
-        } else {
-            err = gnutls_x509_crt_get_issuer_dn(cert, dn, &dn_size);
-            if (err < 0)
-                UNKNOWN_ISSUER(err);
-            else
-                ss << "issuer `" << dn << "', ";
-            free(dn);
-        }
-    }
-
-    /* Key algorithm and size */
-    unsigned int bits;
-    const char *name;
-    name = gnutls_pk_algorithm_get_name( (gnutls_pk_algorithm_t)
-        gnutls_x509_crt_get_pk_algorithm(cert, &bits));
-    if (name == NULL)
-        name = "Unknown";
-    ss << name << " key " << bits << " bits, ";
-
-    /* Signature algorithm */
-    err = gnutls_x509_crt_get_signature_algorithm(cert);
-    if (err < 0) {
-        ss << "unknown signature algorithm (" << gnutls_strerror(err)
-           << "), ";
-    } else {
-        const char *name;
-        name = gnutls_sign_algorithm_get_name((gnutls_sign_algorithm_t)err);
-        if (name == NULL)
-            name = "Unknown";
-
-        ss << "signed using " << name;
-        if (err == GNUTLS_SIGN_RSA_MD5 || err == GNUTLS_SIGN_RSA_MD2)
-            ss << " (broken!)";
-        ss << ", ";
-    }
-
-    /* Validity */
-    time_t tim;
-    char s[42];
-    size_t max = sizeof(s);
-    struct tm t;
-
-    tim = gnutls_x509_crt_get_activation_time(cert);
-    if (gmtime_r(&tim, &t) == NULL)
-        ss << "unknown activation (" << (unsigned long) tim << ")";
-    else if (strftime(s, max, "%Y-%m-%d %H:%M:%S UTC", &t) == 0)
-        ss << "failed activation (" << (unsigned long) tim << ")";
-    else
-        ss << "activated `" << s << "'";
-    ss << ", ";
-
-    tim = gnutls_x509_crt_get_expiration_time(cert);
-    if (gmtime_r(&tim, &t) == NULL)
-        ss << "unknown expiry (" << (unsigned long) tim << ")";
-    else if (strftime(s, max, "%Y-%m-%d %H:%M:%S UTC", &t) == 0)
-        ss << "failed expiry (" << (unsigned long) tim << ")";
-    else
-        ss << "expires `" << s << "'";
-    ss << ", ";
-
-    /* Fingerprint */
-    char buffer[20];
-    size_t size = sizeof(buffer);
-
-    err = gnutls_x509_crt_get_fingerprint(cert, GNUTLS_DIG_SHA1, buffer, &size);
-    if (err < 0)
-        ss << "unknown fingerprint (" << gnutls_strerror(err) << ")";
-    else {
-        ss << "SHA-1 fingerprint `";
-        hexprint(ss, buffer, size);
-        ss << "'";
-    }
-
-    out->data = (unsigned char *) strdup(ss.str().c_str());
-    if (out->data == NULL)
-        return GNUTLS_E_MEMORY_ERROR;
-    out->size = strlen((char *)out->data);
-
-    return 0;
-}
-
-#endif /* HAVE_GNUTLS_X509_CRT_PRINT */
-
-#endif /* HAVE_GNUTLS */
-
diff --git a/common/os/tls.h b/./prev/common/os/tls.h
deleted file mode 100644
index 8980197..0000000
--- common/os/tls.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/* Copyright (C) 2011 TightVNC Team.  All Rights Reserved.
- *
- * This is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- * 
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * 
- * You should have received a copy of the GNU General Public License
- * along with this software; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
- * USA.
- */
-
-#ifndef OS_TLS_H
-#define OS_TLS_H
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-/*
- * Windows builds are build against fairly new GNUTLS, ignore compatibility
- * code.
- */
-#if defined(HAVE_GNUTLS) && !defined(WIN32)
-#include <gnutls/gnutls.h>
-
-#ifndef HAVE_GNUTLS_DATUM_T
-typedef gnutls_datum gnutls_datum_t;
-#endif
-#ifndef HAVE_GNUTLS_CRT_T
-typedef gnutls_x509_crt gnutls_x509_crt_t;
-#endif
-#ifndef HAVE_GNUTLS_PK_ALGORITHM_T
-typedef gnutls_pk_algorithm gnutls_pk_algorithm_t;
-#endif
-#ifndef HAVE_GNUTLS_SIGN_ALGORITHM_T
-typedef gnutls_sign_algorithm gnutls_sign_algorithm_t;
-#endif
-
-#ifndef HAVE_GNUTLS_X509_CRT_PRINT
-
-typedef enum {
-    GNUTLS_CRT_PRINT_ONELINE = 1
-} gnutls_certificate_print_formats_t;
-
-/*
- * Prints certificate in human-readable form.
- */
-int
-gnutls_x509_crt_print(gnutls_x509_crt_t cert,
-              gnutls_certificate_print_formats_t format,
-              gnutls_datum_t * out);
-#endif /* HAVE_GNUTLS_X509_CRT_PRINT */
-#endif /* HAVE_GNUTLS */
-
-#endif /* OS_TLS_H */
-
diff --git a/common/rdr/TLSInStream.cxx b/common/rdr/TLSInStream.cxx
index ddc9991..ef030c1 100644
--- common/rdr/TLSInStream.cxx
+++ common/rdr/TLSInStream.cxx
@@ -27,23 +27,19 @@
 #include <rdr/TLSInStream.h>
 #include <errno.h>
 
-#ifdef HAVE_OLD_GNUTLS
-#define gnutls_transport_set_global_errno(A) do { errno = (A); } while(0)
-#endif
-
 #ifdef HAVE_GNUTLS 
 using namespace rdr;
 
 enum { DEFAULT_BUF_SIZE = 16384 };
 
-ssize_t rdr::gnutls_InStream_pull(gnutls_transport_ptr str, void* data,
-                  size_t size)
+ssize_t TLSInStream::pull(gnutls_transport_ptr_t str, void* data, size_t size)
 {
-  InStream* in= (InStream*) str;
+  TLSInStream* self= (TLSInStream*) str;
+  InStream *in = self->in;
 
   try {
     if (!in->check(1, 1, false)) {
-      gnutls_transport_set_global_errno(EAGAIN);
+      gnutls_transport_set_errno(self->session, EAGAIN);
       return -1;
     }
 
@@ -53,21 +49,29 @@ ssize_t rdr::gnutls_InStream_pull(gnutls_transport_ptr str, void* data,
     in->readBytes(data, size);
 
   } catch (Exception& e) {
-    gnutls_transport_set_global_errno(EINVAL);
+    gnutls_transport_set_errno(self->session, EINVAL);
     return -1;
   }
 
   return size;
 }
 
-TLSInStream::TLSInStream(InStream* _in, gnutls_session _session)
+TLSInStream::TLSInStream(InStream* _in, gnutls_session_t _session)
   : session(_session), in(_in), bufSize(DEFAULT_BUF_SIZE), offset(0)
 {
+  gnutls_transport_ptr_t recv, send;
+
   ptr = end = start = new U8[bufSize];
+
+  gnutls_transport_set_pull_function(session, pull);
+  gnutls_transport_get_ptr2(session, &recv, &send);
+  gnutls_transport_set_ptr2(session, this, send);
 }
 
 TLSInStream::~TLSInStream()
 {
+  gnutls_transport_set_pull_function(session, NULL);
+
   delete[] start;
 }
 
diff --git a/common/rdr/TLSInStream.h b/common/rdr/TLSInStream.h
index 7fad6fa..b16d9f5 100644
--- common/rdr/TLSInStream.h
+++ common/rdr/TLSInStream.h
@@ -33,7 +33,7 @@ namespace rdr {
 
   class TLSInStream : public InStream {
   public:
-    TLSInStream(InStream* in, gnutls_session session);
+    TLSInStream(InStream* in, gnutls_session_t session);
     virtual ~TLSInStream();
 
     int pos();
@@ -41,16 +41,14 @@ namespace rdr {
   private:
     int overrun(int itemSize, int nItems, bool wait);
     int readTLS(U8* buf, int len, bool wait);
+    static ssize_t pull(gnutls_transport_ptr_t str, void* data, size_t size);
 
-    gnutls_session session;
+    gnutls_session_t session;
     InStream* in;
     int bufSize;
     int offset;
     U8* start;
   };
-
-  ssize_t gnutls_InStream_pull(gnutls_transport_ptr,void*, size_t);
-
 };
 
 #endif
diff --git a/common/rdr/TLSOutStream.cxx b/common/rdr/TLSOutStream.cxx
index d577ccc..44d2d9f 100644
--- common/rdr/TLSOutStream.cxx
+++ common/rdr/TLSOutStream.cxx
@@ -27,36 +27,39 @@
 #include <rdr/TLSOutStream.h>
 #include <errno.h>
 
-#ifdef HAVE_OLD_GNUTLS
-#define gnutls_transport_set_global_errno(A) do { errno = (A); } while(0)
-#endif
-
 #ifdef HAVE_GNUTLS
 using namespace rdr;
 
 enum { DEFAULT_BUF_SIZE = 16384 };
 
-ssize_t rdr::gnutls_OutStream_push(gnutls_transport_ptr str, const void* data,
+ssize_t TLSOutStream::push(gnutls_transport_ptr_t str, const void* data,
                    size_t size)
 {
-  OutStream* out = (OutStream*) str;
+  TLSOutStream* self= (TLSOutStream*) str;
+  OutStream *out = self->out;
 
   try {
     out->writeBytes(data, size);
     out->flush();
   } catch (Exception& e) {
-    gnutls_transport_set_global_errno(EINVAL);
+    gnutls_transport_set_errno(self->session, EINVAL);
     return -1;
   }
 
   return size;
 }
 
-TLSOutStream::TLSOutStream(OutStream* _out, gnutls_session _session)
+TLSOutStream::TLSOutStream(OutStream* _out, gnutls_session_t _session)
   : session(_session), out(_out), bufSize(DEFAULT_BUF_SIZE), offset(0)
 {
+  gnutls_transport_ptr_t recv, send;
+
   ptr = start = new U8[bufSize];
   end = start + bufSize;
+
+  gnutls_transport_set_push_function(session, push);
+  gnutls_transport_get_ptr2(session, &recv, &send);
+  gnutls_transport_set_ptr2(session, recv, this);
 }
 
 TLSOutStream::~TLSOutStream()
@@ -67,6 +70,8 @@ TLSOutStream::~TLSOutStream()
   } catch (Exception&) {
   }
 #endif
+  gnutls_transport_set_push_function(session, NULL);
+
   delete [] start;
 }
 
diff --git a/common/rdr/TLSOutStream.h b/common/rdr/TLSOutStream.h
index 5eb512e..81dd237 100644
--- common/rdr/TLSOutStream.h
+++ common/rdr/TLSOutStream.h
@@ -32,7 +32,7 @@ namespace rdr {
 
   class TLSOutStream : public OutStream {
   public:
-    TLSOutStream(OutStream* out, gnutls_session session);
+    TLSOutStream(OutStream* out, gnutls_session_t session);
     virtual ~TLSOutStream();
 
     void flush();
@@ -43,15 +43,14 @@ namespace rdr {
 
   private:
     int writeTLS(const U8* data, int length);
+    static ssize_t push(gnutls_transport_ptr_t str, const void* data, size_t size);
 
-    gnutls_session session;
+    gnutls_session_t session;
     OutStream* out;
     int bufSize;
     U8* start;
     int offset;
   };
-
-  ssize_t gnutls_OutStream_push(gnutls_transport_ptr, const void*, size_t);
 };
 
 #endif
diff --git a/common/rfb/Configuration.cxx b/common/rfb/Configuration.cxx
index e9eee1a..8137501 100644
--- common/rfb/Configuration.cxx
+++ common/rfb/Configuration.cxx
@@ -444,6 +444,10 @@ char* StringParameter::getValueStr() const {
   return strDup(value);
 }
 
+StringParameter::operator const char *() const {
+  return value;
+}
+
 // -=- BinaryParameter
 
 BinaryParameter::BinaryParameter(const char* name_, const char* desc_,
diff --git a/common/rfb/Configuration.h b/common/rfb/Configuration.h
index 276651d..35e31fc 100644
--- common/rfb/Configuration.h
+++ common/rfb/Configuration.h
@@ -241,6 +241,7 @@ namespace rfb {
     virtual char* getDefaultStr() const;
     virtual char* getValueStr() const;
     void setDefaultStr(const char* v);
+    operator const char*() const;
 
     // getData() returns a copy of the data - it must be delete[]d by the
     // caller.
diff --git a/common/rfb/CSecurityTLS.cxx b/common/rfb/CSecurityTLS.cxx
index 3421de5..a60dbdd 100644
--- common/rfb/CSecurityTLS.cxx
+++ common/rfb/CSecurityTLS.cxx
@@ -43,7 +43,6 @@
 #include <rdr/TLSOutStream.h>
 #include <os/os.h>
 #include <os/print.h>
-#include <os/tls.h>
 
 #include <gnutls/x509.h>
 
@@ -188,20 +187,20 @@ bool CSecurityTLS::processMsg(CConnection* cc)
       throw AuthFailureException("gnutls_set_default_priority failed");
 
     setParam();
-    
-    gnutls_transport_set_pull_function(session, rdr::gnutls_InStream_pull);
-    gnutls_transport_set_push_function(session, rdr::gnutls_OutStream_push);
-    gnutls_transport_set_ptr2(session,
-                  (gnutls_transport_ptr) is,
-                  (gnutls_transport_ptr) os);
   }
 
+  rdr::TLSInStream *tlsis = new rdr::TLSInStream(is, session);
+  rdr::TLSOutStream *tlsos = new rdr::TLSOutStream(os, session);
+
   int err;
   err = gnutls_handshake(session);
-  if (err != GNUTLS_E_SUCCESS && !gnutls_error_is_fatal(err))
-    return false;
-
   if (err != GNUTLS_E_SUCCESS) {
+    delete tlsis;
+    delete tlsos;
+
+    if (!gnutls_error_is_fatal(err))
+      return false;
+
     vlog.error("TLS Handshake failed: %s\n", gnutls_strerror (err));
     shutdown(false);
     throw AuthFailureException("TLS Handshake failed");
@@ -209,22 +208,39 @@ bool CSecurityTLS::processMsg(CConnection* cc)
 
   checkSession();
 
-  cc->setStreams(fis = new rdr::TLSInStream(is, session),
-         fos = new rdr::TLSOutStream(os, session));
+  cc->setStreams(fis = tlsis, fos = tlsos);
 
   return true;
 }
 
 void CSecurityTLS::setParam()
 {
-  static const int kx_anon_priority[] = { GNUTLS_KX_ANON_DH, 0 };
-  static const int kx_priority[] = { GNUTLS_KX_DHE_DSS, GNUTLS_KX_RSA,
-                     GNUTLS_KX_DHE_RSA, GNUTLS_KX_SRP, 0 };
+  static const char kx_anon_priority[] = ":+ANON-ECDH:+ANON-DH";
 
-  if (anon) {
-    if (gnutls_kx_set_priority(session, kx_anon_priority) != GNUTLS_E_SUCCESS)
-      throw AuthFailureException("gnutls_kx_set_priority failed");
+  int ret;
+  char *prio;
+  const char *err;
+
+  prio = (char*)malloc(strlen(Security::GnuTLSPriority) +
+                       strlen(kx_anon_priority) + 1);
+  if (prio == NULL)
+    throw AuthFailureException("Not enough memory for GnuTLS priority string");
 
+  strcpy(prio, Security::GnuTLSPriority);
+  if (anon)
+    strcat(prio, kx_anon_priority);
+
+  ret = gnutls_priority_set_direct(session, prio, &err);
+
+  free(prio);
+
+  if (ret != GNUTLS_E_SUCCESS) {
+    if (ret == GNUTLS_E_INVALID_REQUEST)
+      vlog.error("GnuTLS priority syntax error at: %s", err);
+    throw AuthFailureException("gnutls_set_priority_direct failed");
+  }
+
+  if (anon) {
     if (gnutls_anon_allocate_client_credentials(&anon_cred) != GNUTLS_E_SUCCESS)
       throw AuthFailureException("gnutls_anon_allocate_client_credentials failed");
 
@@ -233,9 +249,6 @@ void CSecurityTLS::setParam()
 
     vlog.debug("Anonymous session has been set");
   } else {
-    if (gnutls_kx_set_priority(session, kx_priority) != GNUTLS_E_SUCCESS)
-      throw AuthFailureException("gnutls_kx_set_priority failed");
-
     if (gnutls_certificate_allocate_credentials(&cert_cred) != GNUTLS_E_SUCCESS)
       throw AuthFailureException("gnutls_certificate_allocate_credentials failed");
 
@@ -274,10 +287,10 @@ void CSecurityTLS::checkSession()
                   GNUTLS_CERT_SIGNER_NOT_FOUND |
                   GNUTLS_CERT_SIGNER_NOT_CA;
   unsigned int status;
-  const gnutls_datum *cert_list;
+  const gnutls_datum_t *cert_list;
   unsigned int cert_list_size = 0;
   int err;
-  gnutls_datum info;
+  gnutls_datum_t info;
 
   if (anon)
     return;
@@ -313,7 +326,7 @@ void CSecurityTLS::checkSession()
     throw AuthFailureException("empty certificate chain");
 
   /* Process only server's certificate, not issuer's certificate */
-  gnutls_x509_crt crt;
+  gnutls_x509_crt_t crt;
   gnutls_x509_crt_init(&crt);
 
   if (gnutls_x509_crt_import(crt, &cert_list[0], GNUTLS_X509_FMT_DER) < 0)
diff --git a/common/rfb/CSecurityTLS.h b/common/rfb/CSecurityTLS.h
index c19307e..50aaf30 100644
--- common/rfb/CSecurityTLS.h
+++ common/rfb/CSecurityTLS.h
@@ -64,9 +64,9 @@ namespace rfb {
   private:
     static void initGlobal();
 
-    gnutls_session session;
-    gnutls_anon_client_credentials anon_cred;
-    gnutls_certificate_credentials cert_cred;
+    gnutls_session_t session;
+    gnutls_anon_client_credentials_t anon_cred;
+    gnutls_certificate_credentials_t cert_cred;
     bool anon;
 
     char *cafile, *crlfile;
diff --git a/common/rfb/SSecurityTLS.cxx b/common/rfb/SSecurityTLS.cxx
index 2ea84e0..c2b1ded 100644
--- common/rfb/SSecurityTLS.cxx
+++ common/rfb/SSecurityTLS.cxx
@@ -27,6 +27,8 @@
 #error "This source should not be compiled without HAVE_GNUTLS defined"
 #endif
 
+#include <stdlib.h>
+
 #include <rfb/SSecurityTLS.h>
 #include <rfb/SConnection.h>
 #include <rfb/LogWriter.h>
@@ -148,17 +150,19 @@ bool SSecurityTLS::processMsg(SConnection *sc)
       throw;
     }
 
-    gnutls_transport_set_pull_function(session,rdr::gnutls_InStream_pull);
-    gnutls_transport_set_push_function(session,rdr::gnutls_OutStream_push);
-    gnutls_transport_set_ptr2(session,
-                  (gnutls_transport_ptr)is,
-                  (gnutls_transport_ptr)os);
     os->writeU8(1);
     os->flush();
   }
 
+  rdr::TLSInStream *tlsis = new rdr::TLSInStream(is, session);
+  rdr::TLSOutStream *tlsos = new rdr::TLSOutStream(os, session);
+
   int err;
-  if ((err = gnutls_handshake(session)) != GNUTLS_E_SUCCESS) {
+  err = gnutls_handshake(session);
+  if (err != GNUTLS_E_SUCCESS) {
+    delete tlsis;
+    delete tlsos;
+
     if (!gnutls_error_is_fatal(err)) {
       vlog.debug("Deferring completion of TLS handshake: %s", gnutls_strerror(err));
       return false;
@@ -170,21 +174,37 @@ bool SSecurityTLS::processMsg(SConnection *sc)
 
   vlog.debug("Handshake completed");
 
-  sc->setStreams(fis=new rdr::TLSInStream(is,session),
-         fos=new rdr::TLSOutStream(os,session));
+  sc->setStreams(fis = tlsis, fos = tlsos);
 
   return true;
 }
 
-void SSecurityTLS::setParams(gnutls_session session)
+void SSecurityTLS::setParams(gnutls_session_t session)
 {
-  static const int kx_anon_priority[] = { GNUTLS_KX_ANON_DH, 0 };
-  static const int kx_priority[] = { GNUTLS_KX_DHE_DSS, GNUTLS_KX_RSA,
-                     GNUTLS_KX_DHE_RSA, GNUTLS_KX_SRP, 0 };
+  static const char kx_anon_priority[] = ":+ANON-ECDH:+ANON-DH";
+
+  int ret;
+  char *prio;
+  const char *err;
+
+  prio = (char*)malloc(strlen(Security::GnuTLSPriority) +
+                       strlen(kx_anon_priority) + 1);
+  if (prio == NULL)
+    throw AuthFailureException("Not enough memory for GnuTLS priority string");
 
-  if (gnutls_kx_set_priority(session, anon ? kx_anon_priority : kx_priority)
-      != GNUTLS_E_SUCCESS)
-    throw AuthFailureException("gnutls_kx_set_priority failed");
+  strcpy(prio, Security::GnuTLSPriority);
+  if (anon)
+    strcat(prio, kx_anon_priority);
+
+  ret = gnutls_priority_set_direct(session, prio, &err);
+
+  free(prio);
+
+  if (ret != GNUTLS_E_SUCCESS) {
+    if (ret == GNUTLS_E_INVALID_REQUEST)
+      vlog.error("GnuTLS priority syntax error at: %s", err);
+    throw AuthFailureException("gnutls_set_priority_direct failed");
+  }
 
   if (gnutls_dh_params_init(&dh_params) != GNUTLS_E_SUCCESS)
     throw AuthFailureException("gnutls_dh_params_init failed");
diff --git a/common/rfb/SSecurityTLS.h b/common/rfb/SSecurityTLS.h
index 4eebc7e..a793205 100644
--- common/rfb/SSecurityTLS.h
+++ common/rfb/SSecurityTLS.h
@@ -51,15 +51,15 @@ namespace rfb {
 
   protected:
     void shutdown();
-    void setParams(gnutls_session session);
+    void setParams(gnutls_session_t session);
 
   private:
     static void initGlobal();
 
-    gnutls_session session;
-    gnutls_dh_params dh_params;
-    gnutls_anon_server_credentials anon_cred;
-    gnutls_certificate_credentials cert_cred;
+    gnutls_session_t session;
+    gnutls_dh_params_t dh_params;
+    gnutls_anon_server_credentials_t anon_cred;
+    gnutls_certificate_credentials_t cert_cred;
     char *keyfile, *certfile;
 
     int type;
diff --git a/common/rfb/Security.cxx b/common/rfb/Security.cxx
index e51c891..342aa8f 100644
--- common/rfb/Security.cxx
+++ common/rfb/Security.cxx
@@ -52,6 +52,12 @@ using namespace std;
 
 static LogWriter vlog("Security");
 
+#ifdef HAVE_GNUTLS
+StringParameter Security::GnuTLSPriority("GnuTLSPriority",
+  "GnuTLS priority string that controls the TLS session’s handshake algorithms",
+  "NORMAL");
+#endif
+
 Security::Security(StringParameter &secTypes)
 {
   char *secTypesStr;
diff --git a/common/rfb/Security.h b/common/rfb/Security.h
index 832735a..e69adf6 100644
--- common/rfb/Security.h
+++ common/rfb/Security.h
@@ -93,6 +93,10 @@ namespace rfb {
     /* Output char* is stored in static array */
     char *ToString(void);
 
+#ifdef HAVE_GNUTLS
+    static StringParameter GnuTLSPriority;
+#endif
+
   private:
     std::list<rdr::U32> enabledSecTypes;
   };
diff --git a/configure.ac b/configure.ac
index b0b6501..cf1ad6e 100644
--- configure.ac
+++ configure.ac
@@ -100,6 +100,11 @@ if test "x$enable_gnutls" = xyes; then
                [GNUTLS_LIBS=${GNUTLS_LDFLAGS}
             AC_DEFINE(HAVE_GNUTLS, 1, [Is gnutls present? ])
             AC_MSG_RESULT(yes)], AC_MSG_RESULT(no))
+    AC_MSG_CHECKING([for gnutls_transport_set_errno() function])
+    AC_LINK_IFELSE(AC_LANG_CALL([], gnutls_transport_set_errno),
+               AC_MSG_RESULT(yes),
+               [AC_DEFINE(HAVE_GNUTLS_SET_ERRNO, 1, [Does gnutls have the gnutls_transport_set_errno() function? ])
+            AC_MSG_RESULT(no)])
     AC_MSG_CHECKING([for gnutls_transport_set_global_errno() function])
     AC_LINK_IFELSE(AC_LANG_CALL([], gnutls_transport_set_global_errno),
                AC_MSG_RESULT(yes),

 components/x11/tigervnc/patches/13.gnutls-disable-sslv3.patch

File was deleted

 components/x11/tigervnc/patches/14.gnutls-drop-ecdh.patch

New file
@@ -0,0 +1,30 @@
Need to drop ECDH from algorithm list when building on Solaris,
since our GnuTLS build disables it in
userland/components/gnutls-3/patches/02_remove_elliptical.patch

diff --git a/common/rfb/CSecurityTLS.cxx b/common/rfb/CSecurityTLS.cxx
index a60dbdd..032cbdd 100644
--- common/rfb/CSecurityTLS.cxx
+++ common/rfb/CSecurityTLS.cxx
@@ -215,7 +215,7 @@ bool CSecurityTLS::processMsg(CConnection* cc)
 
 void CSecurityTLS::setParam()
 {
-  static const char kx_anon_priority[] = ":+ANON-ECDH:+ANON-DH";
+  static const char kx_anon_priority[] = ":+ANON-DH";
 
   int ret;
   char *prio;
diff --git a/common/rfb/SSecurityTLS.cxx b/common/rfb/SSecurityTLS.cxx
index c2b1ded..59fa561 100644
--- common/rfb/SSecurityTLS.cxx
+++ common/rfb/SSecurityTLS.cxx
@@ -181,7 +181,7 @@ bool SSecurityTLS::processMsg(SConnection *sc)
 
 void SSecurityTLS::setParams(gnutls_session_t session)
 {
-  static const char kx_anon_priority[] = ":+ANON-ECDH:+ANON-DH";
+  static const char kx_anon_priority[] = ":+ANON-DH";
 
   int ret;
   char *prio;