(http client) Add support for multipart HTTP POST upload + (ixsentry) Add support for uploading a minidump to sentry

DOCKER_VERSION

@@ -1 +1 @@
-7.3.5
+7.4.0

docs/CHANGELOG.md

@@ -1,6 +1,11 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [7.4.0] - 2019-11-25
+
+- (http client) Add support for multipart HTTP POST upload
+- (ixsentry) Add support for uploading a minidump to sentry
+
 ## [7.3.5] - 2019-11-20
 
 - On Darwin SSL, add ability to skip peer verification.

ixsentry/CMakeLists.txt

@@ -19,7 +19,7 @@ add_library(ixsentry STATIC
 set(IXSENTRY_INCLUDE_DIRS
     .
     ..
-    ../third_party
+    ../ixcore
-    ../third_party/spdlog/include)
+    ../third_party)
 
 target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} )

ixsentry/ixsentry/IXSentryClient.cpp

@@ -8,8 +8,9 @@
 
 #include <chrono>
 #include <iostream>
+#include <fstream>
 #include <ixwebsocket/IXWebSocketHttpHeaders.h>
-#include <spdlog/spdlog.h>
+#include <ixcore/utils/IXCoreLogger.h>
 
 
 namespace ix
@@ -18,6 +19,7 @@ namespace ix
         : _dsn(dsn)
         , _validDsn(false)
         , _luaFrameRegex("\t([^/]+):([0-9]+): in function ['<]([^/]+)['>]")
+        , _httpClient(std::make_shared<HttpClient>(true))
     {
         const std::regex dsnRegex("(http[s]?)://([^:]+):([^@]+)@([^/]+)/([0-9]+)");
         std::smatch group;
@@ -169,39 +171,64 @@ namespace ix
 
     std::pair<HttpResponsePtr, std::string> SentryClient::send(const Json::Value& msg, bool verbose)
     {
-        auto args = _httpClient.createRequest();
+        auto args = _httpClient->createRequest();
         args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader();
         args->connectTimeout = 60;
         args->transferTimeout = 5 * 60;
         args->followRedirects = true;
         args->verbose = verbose;
-        args->logger = [](const std::string& msg) { spdlog::info("request logger: {}", msg); };
+        args->logger = [](const std::string& msg) { ix::IXCoreLogger::Log(msg.c_str()); };
 
         std::string body = computePayload(msg);
-        HttpResponsePtr response = _httpClient.post(_url, body, args);
+        HttpResponsePtr response = _httpClient->post(_url, body, args);
-
-        if (verbose)
-        {
-            for (auto it : response->headers)
-            {
-                spdlog::info("{}: {}", it.first, it.second);
-            }
-
-            spdlog::info("Upload size: {}", response->uploadSize);
-            spdlog::info("Download size: {}", response->downloadSize);
-
-            spdlog::info("Status: {}", response->statusCode);
-            if (response->errorCode != HttpErrorCode::Ok)
-            {
-                spdlog::info("error message: {}", response->errorMsg);
-            }
-
-            if (response->headers["Content-Type"] != "application/octet-stream")
-            {
-                spdlog::info("payload: {}", response->payload);
-            }
-        }
 
         return std::make_pair(response, body);
     }
+
+    // https://sentry.io/api/12345/minidump?sentry_key=abcdefgh");
+    std::string SentryClient::computeUrl(const std::string& project, const std::string& key)
+    {
+        std::stringstream ss;
+        ss << "https://sentry.io/api/"
+           << project
+           << "/minidump?sentry_key="
+           << key;
+
+        return ss.str();
+    }
+
+    //
+    // curl -v -X POST -F upload_file_minidump=@ws/crash.dmp 'https://sentry.io/api/123456/minidump?sentry_key=12344567890'
+    //
+    void SentryClient::uploadMinidump(
+        const std::string& sentryMetadata,
+        const std::string& minidumpBytes,
+        const std::string& project,
+        const std::string& key,
+        bool verbose,
+        const OnResponseCallback& onResponseCallback)
+    {
+        std::string multipartBoundary = _httpClient->generateMultipartBoundary();
+
+        auto args = _httpClient->createRequest();
+        args->verb = HttpClient::kPost;
+        args->connectTimeout = 60;
+        args->transferTimeout = 5 * 60;
+        args->followRedirects = true;
+        args->verbose = verbose;
+        args->multipartBoundary = multipartBoundary;
+        args->logger = [](const std::string& msg) { ix::IXCoreLogger::Log(msg.c_str()); };
+
+        HttpFormDataParameters httpFormDataParameters;
+        httpFormDataParameters["upload_file_minidump"] = minidumpBytes;
+
+        HttpParameters httpParameters;
+        httpParameters["sentry"] = sentryMetadata;
+
+        args->url = computeUrl(project, key);
+        args->body = _httpClient->serializeHttpFormDataParameters(multipartBoundary, httpFormDataParameters, httpParameters);
+
+
+        _httpClient->performRequest(args, onResponseCallback);
+    }
 } // namespace ix

ixsentry/ixsentry/IXSentryClient.h

@@ -10,6 +10,7 @@
 #include <ixwebsocket/IXHttpClient.h>
 #include <jsoncpp/json/json.h>
 #include <regex>
+#include <memory>
 
 namespace ix
 {
@@ -23,12 +24,24 @@ namespace ix
 
         Json::Value parseLuaStackTrace(const std::string& stack);
 
+        void uploadMinidump(
+            const std::string& sentryMetadata,
+            const std::string& minidumpBytes,
+            const std::string& project,
+            const std::string& key,
+            bool verbose,
+            const OnResponseCallback& onResponseCallback);
+
     private:
         int64_t getTimestamp();
         std::string computeAuthHeader();
         std::string getIso8601();
         std::string computePayload(const Json::Value& msg);
 
+        std::string computeUrl(const std::string& project, const std::string& key);
+
+        void displayReponse(HttpResponsePtr response);
+
         std::string _dsn;
         bool _validDsn;
         std::string _url;
@@ -41,7 +54,7 @@ namespace ix
 
         std::regex _luaFrameRegex;
 
-        HttpClient _httpClient;
+        std::shared_ptr<HttpClient> _httpClient;
     };
 
 } // namespace ix

ixwebsocket/IXHttp.h

@@ -66,6 +66,7 @@ namespace ix
 
     using HttpResponsePtr = std::shared_ptr<HttpResponse>;
     using HttpParameters = std::map<std::string, std::string>;
+    using HttpFormDataParameters = std::map<std::string, std::string>;
     using Logger = std::function<void(const std::string&)>;
     using OnResponseCallback = std::function<void(const HttpResponsePtr&)>;
 
@@ -75,6 +76,7 @@ namespace ix
         std::string verb;
         WebSocketHttpHeaders extraHeaders;
         std::string body;
+        std::string multipartBoundary;
         int connectTimeout;
         int transferTimeout;
         bool followRedirects;

ixwebsocket/IXHttpClient.cpp

@@ -13,6 +13,7 @@
 #include <assert.h>
 #include <cstring>
 #include <iomanip>
+#include <random>
 #include <sstream>
 #include <vector>
 #include <zlib.h>
@@ -197,10 +198,18 @@ namespace ix
 
             // Set default Content-Type if unspecified
             if (args->extraHeaders.find("Content-Type") == args->extraHeaders.end())
+            {
+                if (args->multipartBoundary.empty())
                 {
                     ss << "Content-Type: application/x-www-form-urlencoded"
                        << "\r\n";
                 }
+                else
+                {
+                    ss << "Content-Type: multipart/form-data; boundary=" << args->multipartBoundary
+                       << "\r\n";
+                }
+            }
             ss << "\r\n";
             ss << body;
         }
@@ -597,6 +606,53 @@ namespace ix
         return ss.str();
     }
 
+    std::string HttpClient::serializeHttpFormDataParameters(
+        const std::string& multipartBoundary,
+        const HttpFormDataParameters& httpFormDataParameters,
+        const HttpParameters& httpParameters)
+    {
+        //
+        // --AaB03x
+        // Content-Disposition: form-data; name="submit-name"
+
+        // Larry
+        // --AaB03x
+        // Content-Disposition: form-data; name="foo.txt"; filename="file1.txt"
+        // Content-Type: text/plain
+
+        // ... contents of file1.txt ...
+        // --AaB03x--
+        //
+        std::stringstream ss;
+
+        for (auto&& it : httpFormDataParameters)
+        {
+            ss << "--" << multipartBoundary << "\r\n"
+               << "Content-Disposition:"
+               << " form-data; name=\"" << it.first << "\";"
+               << " filename=\"" << it.first << "\""
+               << "\r\n"
+               << "Content-Type: application/octet-stream"
+               << "\r\n"
+               << "\r\n"
+               << it.second << "\r\n";
+        }
+
+        for (auto&& it : httpParameters)
+        {
+            ss << "--" << multipartBoundary << "\r\n"
+               << "Content-Disposition:"
+               << " form-data; name=\"" << it.first << "\";"
+               << "\r\n"
+               << "\r\n"
+               << it.second << "\r\n";
+        }
+
+        ss << "--" << multipartBoundary << "\r\n";
+
+        return ss.str();
+    }
+
     bool HttpClient::gzipInflate(const std::string& in, std::string& out)
     {
         z_stream inflateState;
@@ -649,4 +705,16 @@ namespace ix
             args->logger(msg);
         }
     }
+
+    std::string HttpClient::generateMultipartBoundary()
+    {
+        std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
+
+        static std::random_device rd;
+        static std::mt19937 generator(rd());
+
+        std::shuffle(str.begin(), str.end(), generator);
+
+        return str;
+    }
 } // namespace ix

ixwebsocket/IXHttpClient.h

@@ -64,6 +64,13 @@ namespace ix
 
         std::string serializeHttpParameters(const HttpParameters& httpParameters);
 
+        std::string serializeHttpFormDataParameters(
+            const std::string& multipartBoundary,
+            const HttpFormDataParameters& httpFormDataParameters,
+            const HttpParameters& httpParameters = HttpParameters());
+
+        std::string generateMultipartBoundary();
+
         std::string urlEncode(const std::string& value);
 
         const static std::string kPost;

ixwebsocket/IXWebSocketVersion.h

@@ -6,4 +6,4 @@
 
 #pragma once
 
-#define IX_WEBSOCKET_VERSION "7.3.5"
+#define IX_WEBSOCKET_VERSION "7.4.0"

ws/CMakeLists.txt

@@ -56,6 +56,7 @@ add_executable(ws
   ws_httpd.cpp
   ws_autobahn.cpp
   ws_proxy_server.cpp
+  ws_sentry_minidump_upload.cpp
   ws.cpp)
 
 target_link_libraries(ws ixsnake)

ws/ws.cpp

@@ -73,6 +73,10 @@ int main(int argc, char** argv)
     std::string appsConfigPath("appsConfig.json");
     std::string subprotocol;
     std::string remoteHost;
+    std::string minidump;
+    std::string metadata;
+    std::string project;
+    std::string key;
     ix::SocketTLSOptions tlsOptions;
     std::string ciphers;
     std::string redirectUrl;
@@ -311,6 +315,13 @@ int main(int argc, char** argv)
     proxyServerApp->add_option("--remote_host", remoteHost, "Remote Hostname");
     proxyServerApp->add_flag("-v", verbose, "Verbose");
 
+    CLI::App* minidumpApp = app.add_subcommand("upload_minidump", "Upload a minidump to sentry");
+    minidumpApp->add_option("--minidump", minidump, "Minidump path")->check(CLI::ExistingPath);
+    minidumpApp->add_option("--metadata", metadata, "Hostname")->check(CLI::ExistingPath);
+    minidumpApp->add_option("--project", project, "Sentry Project")->required();
+    minidumpApp->add_option("--key", key, "Sentry Key")->required();
+    minidumpApp->add_flag("-v", verbose, "Verbose");
+
     CLI11_PARSE(app, argc, argv);
 
     // pid file handling
@@ -453,6 +464,10 @@ int main(int argc, char** argv)
     {
         ret = ix::ws_proxy_server_main(port, hostname, tlsOptions, remoteHost, verbose);
     }
+    else if (app.got_subcommand("upload_minidump"))
+    {
+        ret = ix::ws_sentry_minidump_upload(metadata, minidump, project, key, verbose);
+    }
     else if (version)
     {
         std::cout << "ws " << ix::userAgent() << std::endl;

ws/ws.h

@@ -148,4 +148,10 @@ namespace ix
                              const ix::SocketTLSOptions& tlsOptions,
                              const std::string& remoteHost,
                              bool verbose);
+
+    int ws_sentry_minidump_upload(const std::string& metadataPath,
+                                  const std::string& minidump,
+                                  const std::string& project,
+                                  const std::string& key,
+                                  bool verbose);
 } // namespace ix

ws/ws_cobra_to_sentry.cpp

@@ -81,6 +81,29 @@ namespace ix
 
                 auto ret = sentryClient.send(msg, verbose);
                 HttpResponsePtr response = ret.first;
+
+                if (verbose)
+                {
+                    for (auto it : response->headers)
+                    {
+                        spdlog::info("{}: {}", it.first, it.second);
+                    }
+
+                    spdlog::info("Upload size: {}", response->uploadSize);
+                    spdlog::info("Download size: {}", response->downloadSize);
+
+                    spdlog::info("Status: {}", response->statusCode);
+                    if (response->errorCode != HttpErrorCode::Ok)
+                    {
+                        spdlog::info("error message: {}", response->errorMsg);
+                    }
+
+                    if (response->headers["Content-Type"] != "application/octet-stream")
+                    {
+                        spdlog::info("payload: {}", response->payload);
+                    }
+                }
+
                 if (response->statusCode != 200)
                 {
                     spdlog::error("Error sending data to sentry: {}", response->statusCode);

ws/ws_sentry_minidump_upload.cpp

@@ -0,0 +1,105 @@
+/*
+ *  ws_sentry_minidump_upload.cpp
+ *  Author: Benjamin Sergeant
+ *  Copyright (c) 2019 Machine Zone, Inc. All rights reserved.
+ */
+
+#include <fstream>
+#include <ixsentry/IXSentryClient.h>
+#include <jsoncpp/json/json.h>
+#include <spdlog/spdlog.h>
+#include <sstream>
+
+
+namespace
+{
+    // Assume the file exists
+    std::string readBytes(const std::string& path)
+    {
+        std::vector<uint8_t> memblock;
+        std::ifstream file(path);
+
+        file.seekg(0, file.end);
+        std::streamoff size = file.tellg();
+        file.seekg(0, file.beg);
+
+        memblock.resize(size);
+
+        file.read((char*) &memblock.front(), static_cast<std::streamsize>(size));
+
+        std::string bytes(memblock.begin(), memblock.end());
+        return bytes;
+    }
+} // namespace
+
+namespace ix
+{
+    int ws_sentry_minidump_upload(const std::string& metadataPath,
+                                  const std::string& minidump,
+                                  const std::string& project,
+                                  const std::string& key,
+                                  bool verbose)
+    {
+        SentryClient sentryClient((std::string()));
+
+        // Read minidump file from disk
+        std::string minidumpBytes = readBytes(minidump);
+
+        // Read json data
+        std::string sentryMetadata = readBytes(metadataPath);
+
+
+        std::atomic<bool> done(false);
+
+        sentryClient.uploadMinidump(
+            sentryMetadata,
+            minidumpBytes,
+            project,
+            key,
+            verbose,
+            [verbose, &done](const HttpResponsePtr& response) {
+                if (verbose)
+                {
+                    for (auto it : response->headers)
+                    {
+                        spdlog::info("{}: {}", it.first, it.second);
+                    }
+
+                    spdlog::info("Upload size: {}", response->uploadSize);
+                    spdlog::info("Download size: {}", response->downloadSize);
+
+                    spdlog::info("Status: {}", response->statusCode);
+                    if (response->errorCode != HttpErrorCode::Ok)
+                    {
+                        spdlog::info("error message: {}", response->errorMsg);
+                    }
+
+                    if (response->headers["Content-Type"] != "application/octet-stream")
+                    {
+                        spdlog::info("payload: {}", response->payload);
+                    }
+                }
+
+                if (response->statusCode != 200)
+                {
+                    spdlog::error("Error sending data to sentry: {}", response->statusCode);
+                    spdlog::error("Status: {}", response->statusCode);
+                    spdlog::error("Response: {}", response->payload);
+                }
+                else
+                {
+                    spdlog::info("Event sent to sentry");
+                }
+
+                done = true;
+            });
+
+        while (!done)
+        {
+            std::chrono::duration<double, std::milli> duration(10);
+            std::this_thread::sleep_for(duration);
+        }
+
+        return 0;
+    }
+} // namespace ix