Implement SSL server with OpenSSL backend / still flaky

ixwebsocket/IXSocket.cpp

@@ -49,13 +49,13 @@ namespace ix
                                 std::shared_ptr<SelectInterrupt> selectInterrupt)
     {
         //
-        // We used to use ::select to poll but on Android 9 we get large fds out of ::connect
+        // We used to use ::select to poll but on Android 9 we get large fds out of
-        // which crash in FD_SET as they are larger than FD_SETSIZE.
+        // ::connect which crash in FD_SET as they are larger than FD_SETSIZE. Switching
-        // Switching to ::poll does fix that.
+        // to ::poll does fix that.
         //
-        // However poll isn't as portable as select and has bugs on Windows, so we should write a
+        // However poll isn't as portable as select and has bugs on Windows, so we
-        // shim to fallback to select on those platforms.
+        // should write a shim to fallback to select on those platforms. See
-        // See https://github.com/mpv-player/mpv/pull/5203/files for such a select wrapper.
+        // https://github.com/mpv-player/mpv/pull/5203/files for such a select wrapper.
         //
         nfds_t nfds = 1;
         struct pollfd fds[2];
@@ -164,6 +164,16 @@ namespace ix
         return _selectInterrupt->notify(wakeUpCode);
     }
 
+    bool Socket::accept(std::string& errMsg)
+    {
+        if (_sockfd == -1)
+        {
+            errMsg = "Socket is uninitialized";
+            return false;
+        }
+        return true;
+    }
+
     bool Socket::connect(const std::string& host,
                          int port,
                          std::string& errMsg,

ixwebsocket/IXSocket.h

@@ -64,6 +64,8 @@ namespace ix
         PollResultType isReadyToRead(int timeoutMs);
 
         // Virtual methods
+        virtual bool accept(std::string& errMsg);
+
         virtual bool connect(const std::string& url,
                              int port,
                              std::string& errMsg,

ixwebsocket/IXSocketConnect.cpp

@@ -23,8 +23,9 @@ namespace ix
 {
     //
     // This function can be cancelled every 50 ms
-    // This is important so that we don't block the main UI thread when shutting down a connection
+    // This is important so that we don't block the main UI thread when shutting down a
-    // which is already trying to reconnect, and can be blocked waiting for ::connect to respond.
+    // connection which is already trying to reconnect, and can be blocked waiting for
+    // ::connect to respond.
     //
     int SocketConnect::connectToAddress(const struct addrinfo* address,
                                         std::string& errMsg,

ixwebsocket/IXSocketOpenSSL.cpp

@@ -22,8 +22,8 @@ namespace ix
         "ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 "
         "ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA "
         "ECDHE-RSA-AES256-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 "
-        "DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA "
+        "DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA "
-        "DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256";
+        "DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256";
 
     std::atomic<bool> SocketOpenSSL::_openSSLInitializationSuccessful(false);
     std::once_flag SocketOpenSSL::_openSSLInitFlag;
@@ -122,6 +122,9 @@ namespace ix
         SSL_CTX* ctx = SSL_CTX_new(_ssl_method);
         if (ctx)
         {
+            SSL_CTX_set_mode(ctx,
+                             SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+
             SSL_CTX_set_options(
                 ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_CIPHER_SERVER_PREFERENCE);
         }
@@ -240,6 +243,41 @@ namespace ix
         }
     }
 
+    bool SocketOpenSSL::openSSLServerHandshake(std::string& errMsg)
+    {
+        while (true)
+        {
+            if (_ssl_connection == nullptr || _ssl_context == nullptr)
+            {
+                return false;
+            }
+
+            ERR_clear_error();
+            int accept_result = SSL_accept(_ssl_connection);
+            if (accept_result == 1)
+            {
+                return true;
+            }
+            int reason = SSL_get_error(_ssl_connection, accept_result);
+
+            bool rc = false;
+            if (reason == SSL_ERROR_WANT_READ || reason == SSL_ERROR_WANT_WRITE)
+            {
+                rc = true;
+            }
+            else
+            {
+                errMsg = getSSLError(accept_result);
+                rc = false;
+            }
+
+            if (!rc)
+            {
+                return false;
+            }
+        }
+    }
+
     bool SocketOpenSSL::handleTLSOptions(std::string& errMsg)
     {
         ERR_clear_error();
@@ -325,6 +363,154 @@ namespace ix
         return true;
     }
 
+    bool SocketOpenSSL::accept(std::string& errMsg)
+    {
+        bool handshakeSuccessful = false;
+        {
+            std::lock_guard<std::mutex> lock(_mutex);
+
+            if (!_openSSLInitializationSuccessful)
+            {
+                errMsg = "OPENSSL_init_ssl failure";
+                return false;
+            }
+
+            if (_sockfd == -1)
+            {
+                return false;
+            }
+
+            {
+                const SSL_METHOD* method = SSLv23_server_method();
+                if (method == nullptr)
+                {
+                    errMsg = "SSLv23_server_method failure";
+                    _ssl_context = nullptr;
+                }
+                else
+                {
+                    _ssl_method = method;
+
+                    _ssl_context = SSL_CTX_new(_ssl_method);
+                    if (_ssl_context)
+                    {
+                        SSL_CTX_set_mode(_ssl_context, SSL_MODE_ENABLE_PARTIAL_WRITE);
+                        SSL_CTX_set_mode(_ssl_context, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+                        SSL_CTX_set_options(_ssl_context,
+                                            SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
+                    }
+                }
+            }
+
+            if (_ssl_context == nullptr)
+            {
+                return false;
+            }
+
+            ERR_clear_error();
+            if (_tlsOptions.hasCertAndKey())
+            {
+                if (SSL_CTX_use_certificate_chain_file(_ssl_context,
+                                                       _tlsOptions.certFile.c_str()) != 1)
+                {
+                    auto sslErr = ERR_get_error();
+                    errMsg = "OpenSSL failed - SSL_CTX_use_certificate_chain_file(\"" +
+                             _tlsOptions.certFile + "\") failed: ";
+                    errMsg += ERR_error_string(sslErr, nullptr);
+                }
+                else if (SSL_CTX_use_PrivateKey_file(
+                             _ssl_context, _tlsOptions.keyFile.c_str(), SSL_FILETYPE_PEM) != 1)
+                {
+                    auto sslErr = ERR_get_error();
+                    errMsg = "OpenSSL failed - SSL_CTX_use_PrivateKey_file(\"" +
+                             _tlsOptions.keyFile + "\") failed: ";
+                    errMsg += ERR_error_string(sslErr, nullptr);
+                }
+            }
+
+
+            ERR_clear_error();
+            if (!_tlsOptions.isPeerVerifyDisabled())
+            {
+                if (_tlsOptions.isUsingSystemDefaults())
+                {
+                    if (SSL_CTX_set_default_verify_paths(_ssl_context) == 0)
+                    {
+                        auto sslErr = ERR_get_error();
+                        errMsg = "OpenSSL failed - SSL_CTX_default_verify_paths loading failed: ";
+                        errMsg += ERR_error_string(sslErr, nullptr);
+                    }
+                }
+                else
+                {
+                    const char* root_ca_file = _tlsOptions.caFile.c_str();
+                    STACK_OF(X509_NAME) * rootCAs;
+                    rootCAs = SSL_load_client_CA_file(root_ca_file);
+                    if (rootCAs == NULL)
+                    {
+                        auto sslErr = ERR_get_error();
+                        errMsg = "OpenSSL failed - SSL_load_client_CA_file('" + _tlsOptions.caFile +
+                                 "') failed: ";
+                        errMsg += ERR_error_string(sslErr, nullptr);
+                    }
+                    else
+                    {
+                        SSL_CTX_set_client_CA_list(_ssl_context, rootCAs);
+                        if (SSL_CTX_load_verify_locations(_ssl_context, root_ca_file, nullptr) != 1)
+                        {
+                            auto sslErr = ERR_get_error();
+                            errMsg = "OpenSSL failed - SSL_CTX_load_verify_locations(\"" +
+                                     _tlsOptions.caFile + "\") failed: ";
+                            errMsg += ERR_error_string(sslErr, nullptr);
+                        }
+                    }
+                }
+
+                SSL_CTX_set_verify(
+                    _ssl_context, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
+                SSL_CTX_set_verify_depth(_ssl_context, 4);
+            }
+            else
+            {
+                SSL_CTX_set_verify(_ssl_context, SSL_VERIFY_NONE, nullptr);
+            }
+            if (_tlsOptions.isUsingDefaultCiphers())
+            {
+                if (SSL_CTX_set_cipher_list(_ssl_context, kDefaultCiphers.c_str()) != 1)
+                {
+                    return false;
+                }
+            }
+            else if (SSL_CTX_set_cipher_list(_ssl_context, _tlsOptions.ciphers.c_str()) != 1)
+            {
+                return false;
+            }
+
+            _ssl_connection = SSL_new(_ssl_context);
+            if (_ssl_connection == nullptr)
+            {
+                errMsg = "OpenSSL failed to connect";
+                SSL_CTX_free(_ssl_context);
+                _ssl_context = nullptr;
+                return false;
+            }
+
+            SSL_set_ecdh_auto(_ssl_connection, 1);
+
+            SSL_set_fd(_ssl_connection, _sockfd);
+
+            handshakeSuccessful = openSSLServerHandshake(errMsg);
+        }
+
+        if (!handshakeSuccessful)
+        {
+            close();
+            return false;
+        }
+
+        return true;
+    }
+
     bool SocketOpenSSL::connect(const std::string& host,
                                 int port,
                                 std::string& errMsg,

ixwebsocket/IXSocketOpenSSL.h

@@ -24,6 +24,8 @@ namespace ix
         SocketOpenSSL(const SocketTLSOptions& tlsOptions, int fd = -1);
         ~SocketOpenSSL();
 
+        virtual bool accept(std::string& errMsg) final;
+
         virtual bool connect(const std::string& host,
                              int port,
                              std::string& errMsg,
@@ -42,6 +44,7 @@ namespace ix
         bool openSSLCheckServerCert(SSL* ssl, const std::string& hostname, std::string& errMsg);
         bool checkHost(const std::string& host, const char* pattern);
         bool handleTLSOptions(std::string& errMsg);
+        bool openSSLServerHandshake(std::string& errMsg);
 
         SSL* _ssl_connection;
         SSL_CTX* _ssl_context;

ixwebsocket/IXSocketServer.cpp

@@ -283,6 +283,13 @@ namespace ix
             // Set the socket to non blocking mode + other tweaks
             SocketConnect::configure(clientFd);
 
+            if (!socket->accept(errorMsg))
+            {
+                logError("SocketServer::run() tls accept failed: " + errorMsg);
+                Socket::closeSocket(clientFd);
+                continue;
+            }
+
             // Launch the handleConnection work asynchronously in its own thread.
             std::lock_guard<std::mutex> lock(_connectionsThreadsMutex);
             _connectionsThreads.push_back(std::make_pair(

makefile

@@ -29,6 +29,9 @@ tag:
 xcode:
 	cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 -GXcode && open ixwebsocket.xcodeproj
 
+xcode_openssl:
+	cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 -GXcode && open ixwebsocket.xcodeproj
+
 .PHONY: docker
 
 NAME   := bsergean/ws

ws/generate_certs.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -eo pipefail
+
+
+generate_key() {
+    local path=${1}
+    local base=${2}
+    local type=${3:-'rsa'} # "ec" or "rsa"
+    
+    mkdir -p ${path}
+    if [[ "${type}" == "rsa" ]]; then
+        openssl genrsa -out "${path}/${base}-key.pem"
+    elif [[ "${type}" == "ec" ]]; then
+        openssl ecparam -genkey -param_enc named_curve -name prime256v1 -out "${path}/${base}-key.pem"
+    else
+        echo "Error: usage: type (param \$2) should be 'rsa' or 'ec'" >&2 && exit 1
+    fi
+    echo "generated ${path}/${base}-key.pem"
+}
+
+generate_ca() {
+    local path="${1}"
+    local base="${2:-'root-ca'}"
+    local type="${3:-'rsa'}" # "ec" or "rsa"
+    local org="${4:-'/O=machinezone/O=IXWebSocket'}"
+    
+    mkdir -p ${path}
+
+    generate_key "${path}" "${base}" "${type}"
+
+    openssl req -new -x509 -sha256 -days 3650 \
+        -reqexts v3_req -extensions v3_ca \
+        -subj "${org}/CN=${base}" \
+        -key "${path}/${base}-key.pem" \
+        -out "${path}/${base}-crt.pem"
+
+    echo "generated ${path}/${base}-crt.pem"
+}
+
+generate_cert() {
+    local path="$1"
+    local base="$2"
+    local cabase="$3"
+    local type="${4:-'rsa'}" # "ec" or "rsa"
+    local org="${5:-'/O=machinezone/O=IXWebSocket'}"
+    local san="${6:-'DNS:localhost,DNS:127.0.0.1'}"
+
+    mkdir -p ${path}
+
+    generate_key "${path}" "${base}" "${type}"
+
+    openssl req -new -sha256 \
+        -key "${path}/${base}-key.pem" \
+        -subj "${org}/CN=${base}" \
+        -out "${path}/${base}.csr"
+    
+
+    if [ "${base}" == "${cabase}" ]; then
+        # self-signed
+        openssl x509 -req -in "${path}/${base}.csr" \
+            -signkey "${path}/${base}-key.pem" -days 365 -sha256 \
+            -extfile <(printf "subjectAltName=${san}") \
+            -outform PEM -out "${path}/${base}-crt.pem" 
+    else
+        openssl x509 -req -in ${path}/${base}.csr \
+            -CA "${path}/${cabase}-crt.pem" \
+            -CAkey "${path}/${cabase}-key.pem" \
+            -CAcreateserial -days 365 -sha256 \
+            -extfile <(printf "subjectAltName=${san}") \
+            -outform PEM -out "${path}/${base}-crt.pem"
+    fi
+
+    rm -f ${path}/${base}.csr
+    echo "generated ${path}/${base}-crt.pem"
+}
+
+# main
+
+outdir=${1:-'./.certs'}
+type=${2:-'rsa'}
+org=${3:-'/O=machinezone/O=IXWebSocket'}
+
+if ! which openssl &>/dev/null; then
+    
+    if ! grep -qa -E 'docker|lxc' /proc/1/cgroup; then
+        # launch a container with openssl and run this script there
+        docker run --rm -i -v $(pwd):/work alpine sh -c "apk add bash openssl && /work/generate_certs.sh /work/${outdir} && chown -R $(id -u):$(id -u) /work/${outdir}"
+    else
+        echo "Please install openssl in this container to generate test certs, or launch outside of docker" >&2 && exit 1
+    fi
+else
+    
+    generate_ca   "${outdir}" "trusted-ca" "${type}" "${org}"
+    
+    generate_cert "${outdir}" "trusted-server" "trusted-ca" "${type}" "${org}"
+    generate_cert "${outdir}" "trusted-client" "trusted-ca" "${type}" "${org}"
+
+    generate_ca   "${outdir}" "untrusted-ca" "${type}" "${org}"
+
+    generate_cert "${outdir}" "untrusted-client"  "untrusted-ca"        "${type}" "${org}"
+    generate_cert "${outdir}" "selfsigned-client" "selfsigned-client" "${type}" "${org}"
+
+fi

ws/ws.cpp

@@ -28,6 +28,10 @@ int main(int argc, char** argv)
     ix::IXCoreLogger::LogFunc logFunc = [](const char* msg) { spdlog::info(msg); };
     ix::IXCoreLogger::setLogFunction(logFunc);
 
+#ifndef _WIN32
+    signal(SIGPIPE, SIG_IGN);
+#endif
+
     // Display command.
     if (getenv("DEBUG"))
     {
@@ -80,6 +84,7 @@ int main(int argc, char** argv)
     bool binaryMode = false;
     bool redirect = false;
     bool version = false;
+    bool verifyNone = false;
     int port = 8008;
     int redisPort = 6379;
     int statsdPort = 8125;
@@ -91,7 +96,7 @@ int main(int argc, char** argv)
     int jobs = 4;
     uint32_t maxWaitBetweenReconnectionRetries;
 
-    auto addTLSOptions = [&tlsOptions](CLI::App* app) {
+    auto addTLSOptions = [&tlsOptions, &verifyNone](CLI::App* app) {
         app->add_option(
                "--cert-file", tlsOptions.certFile, "Path to the (PEM format) TLS cert file")
             ->check(CLI::ExistingPath);
@@ -102,9 +107,8 @@ int main(int argc, char** argv)
         app->add_option("--ciphers",
                         tlsOptions.ciphers,
                         "A (comma/space/colon) separated list of ciphers to use for TLS");
-        app->add_flag("--tls",
+        app->add_flag("--tls", tlsOptions.tls, "Enable TLS (server only)");
-                      tlsOptions.tls,
+        app->add_flag("--verify_none", verifyNone, "Disable peer cert verification");
-                      "Enable TLS");
     };
 
     app.add_flag("--version", version, "Connection url");
@@ -294,6 +298,11 @@ int main(int argc, char** argv)
         f.close();
     }
 
+    if (verifyNone)
+    {
+        tlsOptions.caFile = "NONE";
+    }
+
     int ret = 1;
     if (app.got_subcommand("transfer"))
     {