QSslServer only fires `newConnection` on second incoming request, then fails waitForEncrypted

Question

I generate a self-signed certificate for testing purposes with localhost in subjectAltName

openssl req -x509 -newkey ec:<(openssl ecparam -name secp384r1) -sha256 -nodes \
  -keyout "$KEY_FILE" -out "$CERTIFICATE_FILE" -days "$DAYS" \
  -subj "/C=GB/ST=England/L=London/O=$COMPANY_NAME/CN=$WEBSITE" \
  -addext "subjectAltName=DNS:$WEBSITE,DNS:localhost"

Full C++ qt6 minimal server application which calls setLocalCertificate on a self-signed certificate and captures the newConnection signal, and traces incomingConnection:

#include <QCoreApplication>
#include <QFile>
#include <QSslCertificate>
#include <QSslKey>
#include <QSslServer>

class Server : public QSslServer
{
public:
    Server(
        const QSslCertificate & certificate,
        const QSslKey& private_key)
    {
        QSslConfiguration config;
        {
            config.setLocalCertificate(certificate);
            config.setPrivateKey(private_key);
            config.setProtocol(QSsl::TlsV1_2OrLater);
        }

        setSslConfiguration(config);

        connect(this, &QSslServer::newConnection, [this]()
        {
            this->new_connection(); // not a slot
        });

        auto digest = certificate.digest(QCryptographicHash::Sha256);
        qDebug() << "Server starting using " << digest.toHex(':') << ", " << (private_key.isNull() ? "NULL KEY" : "KEY OK");
    }

private:
    void incomingConnection(qintptr fd) override
    {
        qDebug() << "incomingConnection(" << fd << ")";
        QSslServer::incomingConnection(fd);
    }
    
    void new_connection()
    {
        while (auto tcp_socket = nextPendingConnection())
        {
            // In full implementation, this will be in another thread,
            // which will use synchronous calls on the socket.
            auto ssl_socket = dynamic_cast<QSslSocket*>(tcp_socket);
            qDebug() << "nextPendingConnection() " << ssl_socket->peerAddress();
            if (!ssl_socket->waitForEncrypted(5000))
                qDebug() << "Handshake failed: "  << ssl_socket->errorString();
            else
            {
                auto data = ssl_socket->readAll();
                qDebug() << "Read " << data.size();
            }
        }
    }
};


int main (int argc, char ** argv)
{
    QCoreApplication app(argc, argv);

    QFile crt_file(argv[1]);
    crt_file.open(QIODevice::ReadOnly);
    QSslCertificate certificate(crt_file.readAll());

    QFile private_key_file(argv[2]);
    private_key_file.open(QIODevice::ReadOnly);
    auto private_key_contents = private_key_file.readAll();
    QSslKey private_key(private_key_contents, QSsl::Ec);

    Server server(certificate, private_key);
    server.listen(QHostAddress::Any, 2000);

    qDebug() << "Listening.";

    return app.exec();
}

When I initially run it:

Server starting using  "87:etc..." ,  KEY OK
Listening.

If I test this with openssl s_client -connect localhost:2000 -servername localhost then the server ouptputs

incomingConnection( 7 )

and nothing else, the client outputs

CONNECTED(00000003)
depth=0 C = GB, ST = England, L = London, O = Test, CN = test.com
verify error:num=18:self-signed certificate
verify return:1
depth=0 C = GB, ST = England, L = London, O = Test, CN = test.com
verify return:1
---
Certificate chain
 0 s:C = GB, ST = England, L = London, O = Test, CN = test.com
   i:C = GB, ST = England, L = London, O = Test, CN = test.com
   a:PKEY: id-ecPublicKey, 384 (bit); sigalg: ecdsa-with-SHA256
   v:NotBefore: Sep 26 15:43:08 2025 GMT; NotAfter: Sep  2 15:43:08 2125 GMT
---
Server certificate
-----BEGIN CERTIFICATE-----
MIICYjCCAeigAwIBAgIUBX/Jd7LiL0wpkpHVaJmLx15Rp5owCgYIKoZIzj0EAwIw
VjELMAkGA1UEBhMCR0IxEDAOBgNVBAgMB0VuZ2xhbmQxDzANBgNVBAcMBkxvbmRv
bjEPMA0GA1UECgwGQWx0RXJ0MRMwEQYDVQQDDAphbHRlcnQuY29tMCAXDTI1MDky
NjE1NDMwOFoYDzIxMjUwOTAyMTU0MzA4WjBWMQswCQYDVQQGEwJHQjEQMA4GA1UE
CAwHRW5nbGFuZDEPMA0GA1UEBwwGTG9uZG9uMQ8wDQYDVQQKDAZBbHRFcnQxEzAR
BgNVBAMMCmFsdGVydC5jb20wdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAR0uQUUbb+T
Ak1iYeuVpFxFjcVAHLDNLf49a57YPuvcbwZYWWlqNfktCt7biIcv/63Z5XBwT+JQ
Cv5QFw5wcLFgPPRAt/FuTNU3Zb7XvQzMIH7yLJTMEPg1qbTE4HKVT+2jdTBzMB0G
A1UdDgQWBBQU3v/oII+S2Ri7HImjvQ+SCOXZ1DAfBgNVHSMEGDAWgBQU3v/oII+S
2Ri7HImjvQ+SCOXZ1DAPBgNVHRMBAf8EBTADAQH/MCAGA1UdEQQZMBeCCmFsdGVy
dC5jb22CCWxvY2FsaG9zdDAKBggqhkjOPQQDAgNoADBlAjB2iKlqe338FGxsZtpC
ywNnNqAgLR9x1AVvAbkL67FFH5GOaHKgB00re1L//4lqiAgCMQCf/NjP4ebJ+ZrF
GmkjHBIlHkHiHtl0oYQNB68gkc6AI0Dok50HCoUw+/MF3E0mdrI=
-----END CERTIFICATE-----
subject=C = GB, ST = England, L = London, O = Test, CN = test.com
issuer=C = GB, ST = England, L = London, O = Test, CN = test.com
---
No client certificate CA names sent
Requested Signature Algorithms: ECDSA+SHA256:ECDSA+SHA384:ECDSA+SHA512:Ed25519:Ed448:RSA-PSS+SHA256:RSA-PSS+SHA384:RSA-PSS+SHA512:RSA-PSS+SHA256:RSA-PSS+SHA384:RSA-PSS+SHA512:RSA+SHA256:RSA+SHA384:RSA+SHA512:ECDSA+SHA224:RSA+SHA224
Shared Requested Signature Algorithms: ECDSA+SHA256:ECDSA+SHA384:ECDSA+SHA512:Ed25519:Ed448:RSA-PSS+SHA256:RSA-PSS+SHA384:RSA-PSS+SHA512:RSA-PSS+SHA256:RSA-PSS+SHA384:RSA-PSS+SHA512:RSA+SHA256:RSA+SHA384:RSA+SHA512
Peer signing digest: SHA384
Peer signature type: ECDSA
Server Temp Key: X25519, 253 bits
---
SSL handshake has read 1084 bytes and written 421 bytes
Verification error: self-signed certificate
---
New, TLSv1.3, Cipher is TLS_AES_256_GCM_SHA384
Server public key is 384 bit
Secure Renegotiation IS NOT supported
Compression: NONE
Expansion: NONE
No ALPN negotiated
Early data was not sent
Verify return code: 18 (self-signed certificate)
---
---
Post-Handshake New Session Ticket arrived:
SSL-Session:
    Protocol  : TLSv1.3
    Cipher    : TLS_AES_256_GCM_SHA384
    Session-ID: 1B284C118337B0692DFACB79FC46AB6494F7CB34518696E32862BD3C3FFA08BD
    Session-ID-ctx: 
    Resumption PSK: D6AB8A82D73E1BF7074ECBF2F4D63E9F87226D4BF1F79B421DE0A257B808B7A0F7A63B5A7064D49B247825E50DFCF3B2
    PSK identity: None
    PSK identity hint: None
    SRP username: None
    TLS session ticket lifetime hint: 7200 (seconds)
    TLS session ticket:
    0000 - 2a f0 0e e7 ef a9 9c d3-ad bb 6d 03 db 1b f2 70   *.........m....p
    0010 - 5f 78 87 8a 09 a4 16 07-fd e0 e1 2a 81 2a 9f e9   _x.........*.*..
    0020 - 03 52 5c cb 80 24 66 b2-74 94 3a 0d 18 7c 6f b6   .R\..$f.t.:..|o.
    0030 - 59 80 8d 75 ce 02 fa 36-78 44 34 b8 f2 92 d4 88   Y..u...6xD4.....
    0040 - 2c 20 11 b3 42 ad fa 37-3c 3d 60 f6 67 27 2e 94   , ..B..7<=`.g'..
    0050 - d7 17 ae 3c 8d 1d b3 42-03 21 ad 80 35 f9 8b 97   ...<...B.!..5...
    0060 - 05 7b f1 60 ef 36 65 99-c0 2c 81 7c 7b 08 e6 d7   .{.`.6e..,.|{...
    0070 - 15 7b d9 89 53 1d 66 56-55 2c e4 e6 5f fb c1 0f   .{..S.fVU,.._...
    0080 - eb 1e ff fe cc 17 8c c0-b5 91 8c 2c 9b 37 4f 5b   ...........,.7O[
    0090 - 59 c4 3f f0 f0 dc 8c 94-cd 0e 9d be 32 c9 c8 aa   Y.?.........2...
    00a0 - 05 ca 7d d2 5c 71 4a fa-d7 e4 06 98 39 f6 05 63   ..}.\qJ.....9..c
    00b0 - 94 ea c4 35 af 69 9b 97-d0 73 fb 26 c8 0d 78 50   ...5.i...s.&..xP
    00c0 - 6a bb 38 1e 9f b3 92 9d-d4 80 65 84 83 75 20 39   j.8.......e..u 9

    Start Time: 1759134516
    Timeout   : 7200 (sec)
    Verify return code: 18 (self-signed certificate)
    Extended master secret: no
    Max Early Data: 0
---
read R BLOCK
---
Post-Handshake New Session Ticket arrived:
SSL-Session:
    Protocol  : TLSv1.3
    Cipher    : TLS_AES_256_GCM_SHA384
    Session-ID: 8068F0D12FF8EE09F7E370D3F87C917DC91BEE31A5B1AD0BA486E075F18C1404
    Session-ID-ctx: 
    Resumption PSK: FF47D687D8CE26AE8D5805144C957AAF58E36A5652FF56F5A1ED112EF3468666BAFDD7E597AF8ABA0E4932B473AC8E60
    PSK identity: None
    PSK identity hint: None
    SRP username: None
    TLS session ticket lifetime hint: 7200 (seconds)
    TLS session ticket:
    0000 - 2a f0 0e e7 ef a9 9c d3-ad bb 6d 03 db 1b f2 70   *.........m....p
    0010 - c5 95 84 be 95 d6 6f 0f-cf d1 03 ba bd 87 05 95   ......o.........
    0020 - 8f 3f 70 42 5a 67 b4 e2-bc 7e 0e a4 d7 0f 18 00   .?pBZg...~......
    0030 - b8 8b 6a ed c0 f1 f1 62-d6 bf f7 5b 20 5f 73 d8   ..j....b...[ _s.
    0040 - 97 5e 3a 97 ae d1 bc e5-d2 24 07 51 48 3f e1 e6   .^:......$.QH?..
    0050 - 6e 56 61 a5 40 3b 1c c0-19 9c 33 e0 fa e5 16 cd   nVa.@;....3.....
    0060 - e9 b3 78 23 4f 94 75 49-aa 62 7e be 93 cc 3a 31   ..x#O.uI.b~...:1
    0070 - 4d de 74 11 a7 bb dc c6-5c 56 fb dc f3 44 8d 85   M.t.....\V...D..
    0080 - 06 83 92 7f 85 d5 40 5f-f2 60 ad 8c 7c 56 2a 41   ......@_.`..|V*A
    0090 - 5b 76 9d 3f b2 42 66 b8-bb 23 ab a9 4b b2 46 a2   [v.?.Bf..#..K.F.
    00a0 - ab 66 23 ce 10 65 f6 c7-62 04 d4 e2 af c6 98 d9   .f#..e..b.......
    00b0 - 73 b7 92 91 fb 6e dd eb-27 39 86 31 57 3e b0 66   s....n..'9.1W>.f
    00c0 - 05 b4 95 0b d9 2b b4 2e-66 f9 0a 74 9f aa db fe   .....+..f..t....

    Start Time: 1759134516
    Timeout   : 7200 (sec)
    Verify return code: 18 (self-signed certificate)
    Extended master secret: no
    Max Early Data: 0
---
read R BLOCK

then hangs.

If I run the same client command a second time, the server outputs

incomingConnection( 7 )
nextPendingConnection()  QHostAddress("")
Handshake failed:  "The remote host closed the connection"

Girum Senay · Accepted Answer · 2025-09-29 08:46:00Z

from what i see your TLS handshake never completes in Qt, even though openssl s_client shows that the TLS negotiation is done. On the server side you only see incomingConnection(), and your waitForEncrypted() call fails or hangs.

Why

With QSslServer/QSslSocket, the handshake is not started automatically when you accept a connection.

nextPendingConnection() gives you a QSslSocket, but at that point it’s still a plain TCP socket.
Unless you explicitly call startServerEncryption(), Qt never begins the TLS handshake.
Calling waitForEncrypted() right away won’t work because no handshake is running yet.

That’s why the client appears to succeed (OpenSSL finishes the handshake on its side), but the server never transitions the socket to the encrypted state.

To fix start the handshake with startServerEncryption() and handle the process asynchronously via signals, instead of blocking with waitForEncrypted().

Here’s a minimal working example based on your code:

#include <QCoreApplication>
#include <QFile>
#include <QSslCertificate>
#include <QSslConfiguration>
#include <QSslKey>
#include <QSslServer>
#include <QSslSocket>
#include <QDebug>

class Server : public QSslServer
{
public:
    Server(const QSslCertificate &cert, const QSslKey &key)
    {
        QSslConfiguration config;
        config.setLocalCertificate(cert);
        config.setPrivateKey(key);
        config.setProtocol(QSsl::TlsV1_2OrLater);

        setSslConfiguration(config);

        connect(this, &QSslServer::newConnection, this, &Server::new_connection);

        qDebug() << "Server starting with cert digest"
                 << cert.digest(QCryptographicHash::Sha256).toHex(':');
    }

private:
    void incomingConnection(qintptr fd) override
    {
        qDebug() << "incomingConnection(" << fd << ")";
        QSslServer::incomingConnection(fd);
    }

    void new_connection()
    {
        while (auto tcp_socket = nextPendingConnection())
        {
            auto ssl_socket = qobject_cast<QSslSocket*>(tcp_socket);
            if (!ssl_socket) {
                qWarning() << "Not an SSL socket!";
                continue;
            }

            connect(ssl_socket, &QSslSocket::encrypted, [ssl_socket]() {
                qDebug() << "Handshake succeeded with" << ssl_socket->peerAddress();
                ssl_socket->write("Hello, TLS client!\n");
                ssl_socket->flush();
            });

            connect(ssl_socket,
                    qOverload<const QList<QSslError>&>(&QSslSocket::sslErrors),
                    [ssl_socket](const QList<QSslError> &errors) {
                        qDebug() << "SSL errors:" << errors;
                        // For testing only — don't ignore errors in production
                        ssl_socket->ignoreSslErrors();
                    });

            connect(ssl_socket, &QSslSocket::disconnected, [ssl_socket]() {
                qDebug() << "Client disconnected";
                ssl_socket->deleteLater();
            });

            // *** Important: kick off the handshake ***
            ssl_socket->startServerEncryption();
        }
    }
};

int main(int argc, char **argv)
{
    QCoreApplication app(argc, argv);

    QFile crt_file(argv[1]);
    crt_file.open(QIODevice::ReadOnly);
    QSslCertificate certificate(crt_file.readAll());

    QFile key_file(argv[2]);
    key_file.open(QIODevice::ReadOnly);
    QSslKey key(key_file.readAll(), QSsl::Ec);

    Server server(certificate, key);
    if (!server.listen(QHostAddress::Any, 2000))
        qFatal("Failed to listen: %s", qPrintable(server.errorString()));

    qDebug() << "Listening on port 2000";
    return app.exec();
}

to test it run the server:

./server cert.pem key.pem

Then connect:

openssl s_client -connect localhost:2000 -servername localhost

Server output:

Server starting with cert digest "87:..."
Listening on port 2000
incomingConnection( 7 )
Handshake succeeded with QHostAddress("127.0.0.1")

Client output:

...
Verify return code: 18 (self-signed certificate)
---
Hello, TLS client!

note that:

ignoreSslErrors() is only for local testing with self-signed certs. In production, validate properly.
Don’t block with waitForEncrypted(). Use the encrypted signal.
For TLSv1.3 this async model is especially important, because the handshake involves post-handshake messages.

Reminder: Answers generated by AI tools are not allowed due to Stack Overflow's artificial intelligence policy.
Thank you, but when I run your code I still get incomingConnection with no new_connection the first time a request is sent. Also, when I send a second request to your code, I get cannot start handshake on non-plain connection

Collectives™ on Stack Overflow

QSslServer only fires `newConnection` on second incoming request, then fails waitForEncrypted

1 Answer 1

Why

2 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Why

2 Comments

Your Answer

Sign up or log in

Post as a guest

Related