(http client) Add support for multipart HTTP POST upload + (ixsentry) Add support for uploading a minidump to sentry

DOCKER_VERSION

@@ -1 +1 @@
-7.3.4
+7.4.0

README.md

@@ -6,7 +6,7 @@ IXWebSocket is a C++ library for WebSocket client and server development. It has
 
 It is been used on big mobile video game titles sending and receiving tons of messages since 2017 (iOS and Android). It was tested on macOS, iOS, Linux, Android, Windows and FreeBSD. Two important design goals are simplicity and correctness.
 
-```
+```cpp
 # Required on Windows
 ix::initNetSystem();
 

docs/CHANGELOG.md

@@ -1,6 +1,15 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [7.4.0] - 2019-11-25
+
+- (http client) Add support for multipart HTTP POST upload
+- (ixsentry) Add support for uploading a minidump to sentry
+
+## [7.3.5] - 2019-11-20
+
+- On Darwin SSL, add ability to skip peer verification.
+
 ## [7.3.4] - 2019-11-20
 
 - 32-bits compile fix, courtesy of @fcojavmc

ixsentry/CMakeLists.txt

@@ -19,7 +19,7 @@ add_library(ixsentry STATIC
 set(IXSENTRY_INCLUDE_DIRS
     .
     ..
-    ../third_party
+    ../ixcore
-    ../third_party/spdlog/include)
+    ../third_party)
 
 target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} )

ixsentry/ixsentry/IXSentryClient.cpp

@@ -8,8 +8,9 @@
 
 #include <chrono>
 #include <iostream>
+#include <fstream>
 #include <ixwebsocket/IXWebSocketHttpHeaders.h>
-#include <spdlog/spdlog.h>
+#include <ixcore/utils/IXCoreLogger.h>
 
 
 namespace ix
@@ -18,6 +19,7 @@ namespace ix
         : _dsn(dsn)
         , _validDsn(false)
         , _luaFrameRegex("\t([^/]+):([0-9]+): in function ['<]([^/]+)['>]")
+        , _httpClient(std::make_shared<HttpClient>(true))
     {
         const std::regex dsnRegex("(http[s]?)://([^:]+):([^@]+)@([^/]+)/([0-9]+)");
         std::smatch group;
@@ -169,39 +171,64 @@ namespace ix
 
     std::pair<HttpResponsePtr, std::string> SentryClient::send(const Json::Value& msg, bool verbose)
     {
-        auto args = _httpClient.createRequest();
+        auto args = _httpClient->createRequest();
         args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader();
         args->connectTimeout = 60;
         args->transferTimeout = 5 * 60;
         args->followRedirects = true;
         args->verbose = verbose;
-        args->logger = [](const std::string& msg) { spdlog::info("request logger: {}", msg); };
+        args->logger = [](const std::string& msg) { ix::IXCoreLogger::Log(msg.c_str()); };
 
         std::string body = computePayload(msg);
-        HttpResponsePtr response = _httpClient.post(_url, body, args);
+        HttpResponsePtr response = _httpClient->post(_url, body, args);
-
-        if (verbose)
-        {
-            for (auto it : response->headers)
-            {
-                spdlog::info("{}: {}", it.first, it.second);
-            }
-
-            spdlog::info("Upload size: {}", response->uploadSize);
-            spdlog::info("Download size: {}", response->downloadSize);
-
-            spdlog::info("Status: {}", response->statusCode);
-            if (response->errorCode != HttpErrorCode::Ok)
-            {
-                spdlog::info("error message: {}", response->errorMsg);
-            }
-
-            if (response->headers["Content-Type"] != "application/octet-stream")
-            {
-                spdlog::info("payload: {}", response->payload);
-            }
-        }
 
         return std::make_pair(response, body);
     }
+
+    // https://sentry.io/api/12345/minidump?sentry_key=abcdefgh");
+    std::string SentryClient::computeUrl(const std::string& project, const std::string& key)
+    {
+        std::stringstream ss;
+        ss << "https://sentry.io/api/"
+           << project
+           << "/minidump?sentry_key="
+           << key;
+
+        return ss.str();
+    }
+
+    //
+    // curl -v -X POST -F upload_file_minidump=@ws/crash.dmp 'https://sentry.io/api/123456/minidump?sentry_key=12344567890'
+    //
+    void SentryClient::uploadMinidump(
+        const std::string& sentryMetadata,
+        const std::string& minidumpBytes,
+        const std::string& project,
+        const std::string& key,
+        bool verbose,
+        const OnResponseCallback& onResponseCallback)
+    {
+        std::string multipartBoundary = _httpClient->generateMultipartBoundary();
+
+        auto args = _httpClient->createRequest();
+        args->verb = HttpClient::kPost;
+        args->connectTimeout = 60;
+        args->transferTimeout = 5 * 60;
+        args->followRedirects = true;
+        args->verbose = verbose;
+        args->multipartBoundary = multipartBoundary;
+        args->logger = [](const std::string& msg) { ix::IXCoreLogger::Log(msg.c_str()); };
+
+        HttpFormDataParameters httpFormDataParameters;
+        httpFormDataParameters["upload_file_minidump"] = minidumpBytes;
+
+        HttpParameters httpParameters;
+        httpParameters["sentry"] = sentryMetadata;
+
+        args->url = computeUrl(project, key);
+        args->body = _httpClient->serializeHttpFormDataParameters(multipartBoundary, httpFormDataParameters, httpParameters);
+
+
+        _httpClient->performRequest(args, onResponseCallback);
+    }
 } // namespace ix

ixsentry/ixsentry/IXSentryClient.h

@@ -10,6 +10,7 @@
 #include <ixwebsocket/IXHttpClient.h>
 #include <jsoncpp/json/json.h>
 #include <regex>
+#include <memory>
 
 namespace ix
 {
@@ -23,12 +24,24 @@ namespace ix
 
         Json::Value parseLuaStackTrace(const std::string& stack);
 
+        void uploadMinidump(
+            const std::string& sentryMetadata,
+            const std::string& minidumpBytes,
+            const std::string& project,
+            const std::string& key,
+            bool verbose,
+            const OnResponseCallback& onResponseCallback);
+
     private:
         int64_t getTimestamp();
         std::string computeAuthHeader();
         std::string getIso8601();
         std::string computePayload(const Json::Value& msg);
 
+        std::string computeUrl(const std::string& project, const std::string& key);
+
+        void displayReponse(HttpResponsePtr response);
+
         std::string _dsn;
         bool _validDsn;
         std::string _url;
@@ -41,7 +54,7 @@ namespace ix
 
         std::regex _luaFrameRegex;
 
-        HttpClient _httpClient;
+        std::shared_ptr<HttpClient> _httpClient;
     };
 
 } // namespace ix

ixwebsocket/IXHttp.h

@@ -66,6 +66,7 @@ namespace ix
 
     using HttpResponsePtr = std::shared_ptr<HttpResponse>;
     using HttpParameters = std::map<std::string, std::string>;
+    using HttpFormDataParameters = std::map<std::string, std::string>;
     using Logger = std::function<void(const std::string&)>;
     using OnResponseCallback = std::function<void(const HttpResponsePtr&)>;
 
@@ -75,6 +76,7 @@ namespace ix
         std::string verb;
         WebSocketHttpHeaders extraHeaders;
         std::string body;
+        std::string multipartBoundary;
         int connectTimeout;
         int transferTimeout;
         bool followRedirects;

ixwebsocket/IXHttpClient.cpp

@@ -13,6 +13,7 @@
 #include <assert.h>
 #include <cstring>
 #include <iomanip>
+#include <random>
 #include <sstream>
 #include <vector>
 #include <zlib.h>
@@ -197,10 +198,18 @@ namespace ix
 
             // Set default Content-Type if unspecified
             if (args->extraHeaders.find("Content-Type") == args->extraHeaders.end())
+            {
+                if (args->multipartBoundary.empty())
                 {
                     ss << "Content-Type: application/x-www-form-urlencoded"
                        << "\r\n";
                 }
+                else
+                {
+                    ss << "Content-Type: multipart/form-data; boundary=" << args->multipartBoundary
+                       << "\r\n";
+                }
+            }
             ss << "\r\n";
             ss << body;
         }
@@ -597,6 +606,53 @@ namespace ix
         return ss.str();
     }
 
+    std::string HttpClient::serializeHttpFormDataParameters(
+        const std::string& multipartBoundary,
+        const HttpFormDataParameters& httpFormDataParameters,
+        const HttpParameters& httpParameters)
+    {
+        //
+        // --AaB03x
+        // Content-Disposition: form-data; name="submit-name"
+
+        // Larry
+        // --AaB03x
+        // Content-Disposition: form-data; name="foo.txt"; filename="file1.txt"
+        // Content-Type: text/plain
+
+        // ... contents of file1.txt ...
+        // --AaB03x--
+        //
+        std::stringstream ss;
+
+        for (auto&& it : httpFormDataParameters)
+        {
+            ss << "--" << multipartBoundary << "\r\n"
+               << "Content-Disposition:"
+               << " form-data; name=\"" << it.first << "\";"
+               << " filename=\"" << it.first << "\""
+               << "\r\n"
+               << "Content-Type: application/octet-stream"
+               << "\r\n"
+               << "\r\n"
+               << it.second << "\r\n";
+        }
+
+        for (auto&& it : httpParameters)
+        {
+            ss << "--" << multipartBoundary << "\r\n"
+               << "Content-Disposition:"
+               << " form-data; name=\"" << it.first << "\";"
+               << "\r\n"
+               << "\r\n"
+               << it.second << "\r\n";
+        }
+
+        ss << "--" << multipartBoundary << "\r\n";
+
+        return ss.str();
+    }
+
     bool HttpClient::gzipInflate(const std::string& in, std::string& out)
     {
         z_stream inflateState;
@@ -649,4 +705,16 @@ namespace ix
             args->logger(msg);
         }
     }
+
+    std::string HttpClient::generateMultipartBoundary()
+    {
+        std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
+
+        static std::random_device rd;
+        static std::mt19937 generator(rd());
+
+        std::shuffle(str.begin(), str.end(), generator);
+
+        return str;
+    }
 } // namespace ix

ixwebsocket/IXHttpClient.h

@@ -64,6 +64,13 @@ namespace ix
 
         std::string serializeHttpParameters(const HttpParameters& httpParameters);
 
+        std::string serializeHttpFormDataParameters(
+            const std::string& multipartBoundary,
+            const HttpFormDataParameters& httpFormDataParameters,
+            const HttpParameters& httpParameters = HttpParameters());
+
+        std::string generateMultipartBoundary();
+
         std::string urlEncode(const std::string& value);
 
         const static std::string kPost;

ixwebsocket/IXSocketAppleSSL.cpp

@@ -168,10 +168,32 @@ namespace ix
             SSLSetProtocolVersionMin(_sslContext, kTLSProtocol12);
             SSLSetPeerDomainName(_sslContext, host.c_str(), host.size());
 
+            if (_tlsOptions.isPeerVerifyDisabled())
+            {
+                Boolean option(1);
+                SSLSetSessionOption(_sslContext, kSSLSessionOptionBreakOnServerAuth, option);
+
                 do
                 {
                     status = SSLHandshake(_sslContext);
                 } while (errSSLWouldBlock == status || errSSLServerAuthCompleted == status);
+
+                if (status == errSSLServerAuthCompleted)
+                {
+                    // proceed with the handshake
+                    do
+                    {
+                        status = SSLHandshake(_sslContext);
+                    } while (errSSLWouldBlock == status || errSSLServerAuthCompleted == status);
+                }
+            }
+            else
+            {
+                do
+                {
+                    status = SSLHandshake(_sslContext);
+                } while (errSSLWouldBlock == status || errSSLServerAuthCompleted == status);
+            }
         }
 
         if (noErr != status)

ixwebsocket/IXWebSocketVersion.h

@@ -6,4 +6,4 @@
 
 #pragma once
 
-#define IX_WEBSOCKET_VERSION "7.3.4"
+#define IX_WEBSOCKET_VERSION "7.4.0"

ws/CMakeLists.txt

@@ -56,6 +56,7 @@ add_executable(ws
   ws_httpd.cpp
   ws_autobahn.cpp
   ws_proxy_server.cpp
+  ws_sentry_minidump_upload.cpp
   ws.cpp)
 
 target_link_libraries(ws ixsnake)

ws/ws.cpp

@@ -73,6 +73,10 @@ int main(int argc, char** argv)
     std::string appsConfigPath("appsConfig.json");
     std::string subprotocol;
     std::string remoteHost;
+    std::string minidump;
+    std::string metadata;
+    std::string project;
+    std::string key;
     ix::SocketTLSOptions tlsOptions;
     std::string ciphers;
     std::string redirectUrl;
@@ -311,6 +315,13 @@ int main(int argc, char** argv)
     proxyServerApp->add_option("--remote_host", remoteHost, "Remote Hostname");
     proxyServerApp->add_flag("-v", verbose, "Verbose");
 
+    CLI::App* minidumpApp = app.add_subcommand("upload_minidump", "Upload a minidump to sentry");
+    minidumpApp->add_option("--minidump", minidump, "Minidump path")->check(CLI::ExistingPath);
+    minidumpApp->add_option("--metadata", metadata, "Hostname")->check(CLI::ExistingPath);
+    minidumpApp->add_option("--project", project, "Sentry Project")->required();
+    minidumpApp->add_option("--key", key, "Sentry Key")->required();
+    minidumpApp->add_flag("-v", verbose, "Verbose");
+
     CLI11_PARSE(app, argc, argv);
 
     // pid file handling
@@ -453,6 +464,10 @@ int main(int argc, char** argv)
     {
         ret = ix::ws_proxy_server_main(port, hostname, tlsOptions, remoteHost, verbose);
     }
+    else if (app.got_subcommand("upload_minidump"))
+    {
+        ret = ix::ws_sentry_minidump_upload(metadata, minidump, project, key, verbose);
+    }
     else if (version)
     {
         std::cout << "ws " << ix::userAgent() << std::endl;

ws/ws.h

@@ -148,4 +148,10 @@ namespace ix
                              const ix::SocketTLSOptions& tlsOptions,
                              const std::string& remoteHost,
                              bool verbose);
+
+    int ws_sentry_minidump_upload(const std::string& metadataPath,
+                                  const std::string& minidump,
+                                  const std::string& project,
+                                  const std::string& key,
+                                  bool verbose);
 } // namespace ix

ws/ws_cobra_to_sentry.cpp

@@ -81,6 +81,29 @@ namespace ix
 
                 auto ret = sentryClient.send(msg, verbose);
                 HttpResponsePtr response = ret.first;
+
+                if (verbose)
+                {
+                    for (auto it : response->headers)
+                    {
+                        spdlog::info("{}: {}", it.first, it.second);
+                    }
+
+                    spdlog::info("Upload size: {}", response->uploadSize);
+                    spdlog::info("Download size: {}", response->downloadSize);
+
+                    spdlog::info("Status: {}", response->statusCode);
+                    if (response->errorCode != HttpErrorCode::Ok)
+                    {
+                        spdlog::info("error message: {}", response->errorMsg);
+                    }
+
+                    if (response->headers["Content-Type"] != "application/octet-stream")
+                    {
+                        spdlog::info("payload: {}", response->payload);
+                    }
+                }
+
                 if (response->statusCode != 200)
                 {
                     spdlog::error("Error sending data to sentry: {}", response->statusCode);

ws/ws_sentry_minidump_upload.cpp

@@ -0,0 +1,105 @@
+/*
+ *  ws_sentry_minidump_upload.cpp
+ *  Author: Benjamin Sergeant
+ *  Copyright (c) 2019 Machine Zone, Inc. All rights reserved.
+ */
+
+#include <fstream>
+#include <ixsentry/IXSentryClient.h>
+#include <jsoncpp/json/json.h>
+#include <spdlog/spdlog.h>
+#include <sstream>
+
+
+namespace
+{
+    // Assume the file exists
+    std::string readBytes(const std::string& path)
+    {
+        std::vector<uint8_t> memblock;
+        std::ifstream file(path);
+
+        file.seekg(0, file.end);
+        std::streamoff size = file.tellg();
+        file.seekg(0, file.beg);
+
+        memblock.resize(size);
+
+        file.read((char*) &memblock.front(), static_cast<std::streamsize>(size));
+
+        std::string bytes(memblock.begin(), memblock.end());
+        return bytes;
+    }
+} // namespace
+
+namespace ix
+{
+    int ws_sentry_minidump_upload(const std::string& metadataPath,
+                                  const std::string& minidump,
+                                  const std::string& project,
+                                  const std::string& key,
+                                  bool verbose)
+    {
+        SentryClient sentryClient((std::string()));
+
+        // Read minidump file from disk
+        std::string minidumpBytes = readBytes(minidump);
+
+        // Read json data
+        std::string sentryMetadata = readBytes(metadataPath);
+
+
+        std::atomic<bool> done(false);
+
+        sentryClient.uploadMinidump(
+            sentryMetadata,
+            minidumpBytes,
+            project,
+            key,
+            verbose,
+            [verbose, &done](const HttpResponsePtr& response) {
+                if (verbose)
+                {
+                    for (auto it : response->headers)
+                    {
+                        spdlog::info("{}: {}", it.first, it.second);
+                    }
+
+                    spdlog::info("Upload size: {}", response->uploadSize);
+                    spdlog::info("Download size: {}", response->downloadSize);
+
+                    spdlog::info("Status: {}", response->statusCode);
+                    if (response->errorCode != HttpErrorCode::Ok)
+                    {
+                        spdlog::info("error message: {}", response->errorMsg);
+                    }
+
+                    if (response->headers["Content-Type"] != "application/octet-stream")
+                    {
+                        spdlog::info("payload: {}", response->payload);
+                    }
+                }
+
+                if (response->statusCode != 200)
+                {
+                    spdlog::error("Error sending data to sentry: {}", response->statusCode);
+                    spdlog::error("Status: {}", response->statusCode);
+                    spdlog::error("Response: {}", response->payload);
+                }
+                else
+                {
+                    spdlog::info("Event sent to sentry");
+                }
+
+                done = true;
+            });
+
+        while (!done)
+        {
+            std::chrono::duration<double, std::milli> duration(10);
+            std::this_thread::sleep_for(duration);
+        }
+
+        return 0;
+    }
+} // namespace ix