[qtnetwork] Add OpenSSL 1.1.x support. JB#50900

Created patch for Qt 5.6 with a patch for Qt 4.8 as inspiration by reusing the existing macro system.
Switch to -openssl-linked like Fedora Linux or Arch Linux.
Runtime linking is broken. Support for OpenSSL 1.0.x is still in place.

Signed-off-by: Björn Bidar <bjorn.bidar@jolla.com>

rpm/qtbase.spec

@@ -475,6 +475,7 @@ MAKEFLAGS=%{?_smp_mflags} \
     -no-rpath \
     -optimized-qmake \
     -dbus-linked \
+    -openssl-linked \
     -no-strip \
     -no-separate-debug-info \
     -verbose \
@@ -496,7 +497,8 @@ MAKEFLAGS=%{?_smp_mflags} \
     -gbm \
     -qreal float \
     -journald \
-    -no-use-gold-linker
+    -no-use-gold-linker \
+    -openssl-linked
 fi # config.status check
 #%if ! 0%{?qt5_release_build}
 #    -developer-build \

src/network/ssl/qsslcertificate_openssl.cpp

@@ -58,9 +58,11 @@ bool QSslCertificate::operator==(const QSslCertificate &other) const
 uint qHash(const QSslCertificate &key, uint seed) Q_DECL_NOTHROW
 {
     if (X509 * const x509 = key.d->x509) {
-        (void)q_X509_cmp(x509, x509); // populate x509->sha1_hash
-                                      // (if someone knows a better way...)
-        return qHashBits(x509->sha1_hash, SHA_DIGEST_LENGTH, seed);
+        const EVP_MD *evp_md_sha1 = q_EVP_sha1();
+        unsigned int len = 0;
+        unsigned char evp_md[EVP_MAX_MD_SIZE];
+        q_X509_digest(x509, evp_md_sha1, evp_md, &len);
+        return qHashBits(evp_md, len, seed);
     } else {
         return seed;
     }
@@ -84,7 +86,7 @@ QByteArray QSslCertificate::version() const
     QMutexLocker lock(QMutexPool::globalInstanceGet(d.data()));
     if (d->versionString.isEmpty() && d->x509)
         d->versionString =
-            QByteArray::number(qlonglong(q_ASN1_INTEGER_get(d->x509->cert_info->version)) + 1);
+            QByteArray::number(qlonglong(X509_get_version(d->x509)) + 1);
 
     return d->versionString;
 }
@@ -93,7 +95,7 @@ QByteArray QSslCertificate::serialNumber() const
 {
     QMutexLocker lock(QMutexPool::globalInstanceGet(d.data()));
     if (d->serialNumberString.isEmpty() && d->x509) {
-        ASN1_INTEGER *serialNumber = d->x509->cert_info->serialNumber;
+        ASN1_INTEGER *serialNumber = X509_get_serialNumber(d->x509);
         QByteArray hexString;
         hexString.reserve(serialNumber->length * 3);
         for (int a = 0; a < serialNumber->length; ++a) {
@@ -229,25 +231,22 @@ QSslKey QSslCertificate::publicKey() const
     QSslKey key;
 
     key.d->type = QSsl::PublicKey;
-    X509_PUBKEY *xkey = d->x509->cert_info->key;
+    X509_PUBKEY *xkey = X509_get_X509_PUBKEY(d->x509);
     EVP_PKEY *pkey = q_X509_PUBKEY_get(xkey);
     Q_ASSERT(pkey);
 
-    if (q_EVP_PKEY_type(pkey->type) == EVP_PKEY_RSA) {
-        key.d->rsa = q_EVP_PKEY_get1_RSA(pkey);
+    if ((key.d->rsa = q_EVP_PKEY_get1_RSA(pkey))) {
         key.d->algorithm = QSsl::Rsa;
         key.d->isNull = false;
-    } else if (q_EVP_PKEY_type(pkey->type) == EVP_PKEY_DSA) {
-        key.d->dsa = q_EVP_PKEY_get1_DSA(pkey);
+    } else if ((key.d->dsa = q_EVP_PKEY_get1_DSA(pkey))) {
         key.d->algorithm = QSsl::Dsa;
         key.d->isNull = false;
 #ifndef OPENSSL_NO_EC
-    } else if (q_EVP_PKEY_type(pkey->type) == EVP_PKEY_EC) {
-        key.d->ec = q_EVP_PKEY_get1_EC_KEY(pkey);
+    } else if ((key.d->ec = q_EVP_PKEY_get1_EC_KEY(pkey))) {
         key.d->algorithm = QSsl::Ec;
         key.d->isNull = false;
 #endif
-    } else if (q_EVP_PKEY_type(pkey->type) == EVP_PKEY_DH) {
+    } else if (EVP_PKEY_get1_DH(pkey)) {
         // DH unsupported
     } else {
         // error?
@@ -601,7 +600,11 @@ static QMap<QByteArray, QString> _q_mapFromX509Name(X509_NAME *name)
         unsigned char *data = 0;
         int size = q_ASN1_STRING_to_UTF8(&data, q_X509_NAME_ENTRY_get_data(e));
         info.insertMulti(name, QString::fromUtf8((char*)data, size));
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	q_CRYPTO_free(data, OPENSSL_FILE, OPENSSL_LINE);
+#else
         q_CRYPTO_free(data);
+#endif
     }
 
     return info;

src/network/ssl/qsslcontext_openssl.cpp

@@ -104,7 +104,7 @@ QSslContext* QSslContext::fromConfiguration(QSslSocket::SslMode mode, const QSsl
 init_context:
     switch (sslContext->sslConfiguration.protocol()) {
     case QSsl::SslV2:
-#ifndef OPENSSL_NO_SSL2
+#if !defined(OPENSSL_NO_SSL2) && OPENSSL_VERSION_NUMBER < 0x10100000L
         sslContext->ctx = q_SSL_CTX_new(client ? q_SSLv2_client_method() : q_SSLv2_server_method());
 #else
         // SSL 2 not supported by the system, but chosen deliberately -> error
@@ -185,7 +185,7 @@ QSslContext* QSslContext::fromConfiguration(QSslSocket::SslMode mode, const QSsl
     }
 
     // Enable bug workarounds.
-    long options = QSslSocketBackendPrivate::setupOpenSslOptions(configuration.protocol(), configuration.d->sslOptions);
+    unsigned long options = QSslSocketBackendPrivate::setupOpenSslOptions(configuration.protocol(), configuration.d->sslOptions);
     q_SSL_CTX_set_options(sslContext->ctx, options);
 
 #if OPENSSL_VERSION_NUMBER >= 0x10000000L
@@ -324,7 +324,7 @@ QSslContext* QSslContext::fromConfiguration(QSslSocket::SslMode mode, const QSsl
     q_DH_free(dh);
 
 #ifndef OPENSSL_NO_EC
-#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L && OPENSSL_VERSION_NUMBER < 0x10100000L
     if (q_SSLeay() >= 0x10002000L) {
         q_SSL_CTX_ctrl(sslContext->ctx, SSL_CTRL_SET_ECDH_AUTO, 1, NULL);
     } else
@@ -470,7 +470,7 @@ bool QSslContext::cacheSession(SSL* ssl)
             unsigned char *data = reinterpret_cast<unsigned char *>(m_sessionASN1.data());
             if (!q_i2d_SSL_SESSION(session, &data))
                 qCWarning(lcSsl, "could not store persistent version of SSL session");
-            m_sessionTicketLifeTimeHint = session->tlsext_tick_lifetime_hint;
+            m_sessionTicketLifeTimeHint = q_SSL_SESSION_get_ticket_lifetime_hint(session);
         }
     }
 

src/network/ssl/qsslcontext_openssl_p.h

@@ -99,7 +99,7 @@ class QSslContext
     EVP_PKEY *pkey;
     SSL_SESSION *session;
     QByteArray m_sessionASN1;
-    int m_sessionTicketLifeTimeHint;
+    unsigned long m_sessionTicketLifeTimeHint;
     QSslError::SslError errorCode;
     QString errorStr;
     QSslConfiguration sslConfiguration;

src/network/ssl/qsslkey_openssl.cpp

@@ -78,28 +78,26 @@ void QSslKeyPrivate::clear(bool deep)
 
 bool QSslKeyPrivate::fromEVP_PKEY(EVP_PKEY *pkey)
 {
-    if (pkey->type == EVP_PKEY_RSA) {
+    if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA) {
         isNull = false;
         algorithm = QSsl::Rsa;
         type = QSsl::PrivateKey;
 
-        rsa = q_RSA_new();
-        memcpy(rsa, q_EVP_PKEY_get1_RSA(pkey), sizeof(RSA));
+        rsa = q_EVP_PKEY_get1_RSA(pkey);
 
         return true;
     }
-    else if (pkey->type == EVP_PKEY_DSA) {
+    else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_DSA) {
         isNull = false;
         algorithm = QSsl::Dsa;
         type = QSsl::PrivateKey;
 
-        dsa = q_DSA_new();
-        memcpy(dsa, q_EVP_PKEY_get1_DSA(pkey), sizeof(DSA));
+        dsa = q_EVP_PKEY_get1_DSA(pkey);
 
         return true;
     }
 #ifndef OPENSSL_NO_EC
-    else if (pkey->type == EVP_PKEY_EC) {
+    else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
         isNull = false;
         algorithm = QSsl::Ec;
         type = QSsl::PrivateKey;
@@ -172,8 +170,8 @@ int QSslKeyPrivate::length() const
         return -1;
 
     switch (algorithm) {
-        case QSsl::Rsa: return q_BN_num_bits(rsa->n);
-        case QSsl::Dsa: return q_BN_num_bits(dsa->p);
+        case QSsl::Rsa: return q_RSA_bits(rsa);
+        case QSsl::Dsa: return q_DSA_bits(dsa);
 #ifndef OPENSSL_NO_EC
         case QSsl::Ec: return q_EC_GROUP_get_degree(q_EC_KEY_get0_group(ec));
 #endif
@@ -267,7 +265,7 @@ Qt::HANDLE QSslKeyPrivate::handle() const
 
 static QByteArray doCrypt(QSslKeyPrivate::Cipher cipher, const QByteArray &data, const QByteArray &key, const QByteArray &iv, int enc)
 {
-    EVP_CIPHER_CTX ctx;
+    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
     const EVP_CIPHER* type = 0;
     int i = 0, len = 0;
 
@@ -285,21 +283,22 @@ static QByteArray doCrypt(QSslKeyPrivate::Cipher cipher, const QByteArray &data,
 
     QByteArray output;
     output.resize(data.size() + EVP_MAX_BLOCK_LENGTH);
-    q_EVP_CIPHER_CTX_init(&ctx);
-    q_EVP_CipherInit(&ctx, type, NULL, NULL, enc);
-    q_EVP_CIPHER_CTX_set_key_length(&ctx, key.size());
+    q_EVP_CIPHER_CTX_init(ctx);
+    q_EVP_CipherInit(ctx, type, NULL, NULL, enc);
+    q_EVP_CIPHER_CTX_set_key_length(ctx, key.size());
     if (cipher == QSslKeyPrivate::Rc2Cbc)
-        q_EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_SET_RC2_KEY_BITS, 8 * key.size(), NULL);
-    q_EVP_CipherInit(&ctx, NULL,
+        q_EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_SET_RC2_KEY_BITS, 8 * key.size(), NULL);
+    q_EVP_CipherInit(ctx, NULL,
         reinterpret_cast<const unsigned char *>(key.constData()),
         reinterpret_cast<const unsigned char *>(iv.constData()), enc);
-    q_EVP_CipherUpdate(&ctx,
+    q_EVP_CipherUpdate(ctx,
         reinterpret_cast<unsigned char *>(output.data()), &len,
         reinterpret_cast<const unsigned char *>(data.constData()), data.size());
-    q_EVP_CipherFinal(&ctx,
+    q_EVP_CipherFinal(ctx,
         reinterpret_cast<unsigned char *>(output.data()) + len, &i);
     len += i;
-    q_EVP_CIPHER_CTX_cleanup(&ctx);
+    q_EVP_CIPHER_CTX_cleanup(ctx);
+    EVP_CIPHER_CTX_free(ctx);
 
     return output.left(len);
 }

src/network/ssl/qsslsocket_openssl.cpp

@@ -1567,7 +1567,7 @@ void QSslSocketBackendPrivate::continueHandshake()
     if (readBufferMaxSize)
         plainSocket->setReadBufferSize(readBufferMaxSize);
 
-    if (q_SSL_ctrl((ssl), SSL_CTRL_GET_SESSION_REUSED, 0, NULL))
+    if (SSL_session_reused(ssl))
         configuration.peerSessionShared = true;
 
 #ifdef QT_DECRYPT_SSL_TRAFFIC