From a0d5f374029ab14a35cf8b0cc1914337e0f3db91 Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Mon, 25 Nov 2019 21:08:43 -0800 Subject: [PATCH] (http client) Add support for multipart HTTP POST upload + (ixsentry) Add support for uploading a minidump to sentry --- DOCKER_VERSION | 2 +- docs/CHANGELOG.md | 5 ++ ixsentry/CMakeLists.txt | 4 +- ixsentry/ixsentry/IXSentryClient.cpp | 79 +++++++++++++------- ixsentry/ixsentry/IXSentryClient.h | 15 +++- ixwebsocket/IXHttp.h | 2 + ixwebsocket/IXHttpClient.cpp | 72 +++++++++++++++++- ixwebsocket/IXHttpClient.h | 7 ++ ixwebsocket/IXWebSocketVersion.h | 2 +- ws/CMakeLists.txt | 1 + ws/ws.cpp | 15 ++++ ws/ws.h | 6 ++ ws/ws_cobra_to_sentry.cpp | 23 ++++++ ws/ws_sentry_minidump_upload.cpp | 105 +++++++++++++++++++++++++++ 14 files changed, 305 insertions(+), 33 deletions(-) create mode 100644 ws/ws_sentry_minidump_upload.cpp diff --git a/DOCKER_VERSION b/DOCKER_VERSION index d9edd15e..ba7f754d 100644 --- a/DOCKER_VERSION +++ b/DOCKER_VERSION @@ -1 +1 @@ -7.3.5 +7.4.0 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index db732569..6fc5f896 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog All notable changes to this project will be documented in this file. +## [7.4.0] - 2019-11-25 + +- (http client) Add support for multipart HTTP POST upload +- (ixsentry) Add support for uploading a minidump to sentry + ## [7.3.5] - 2019-11-20 - On Darwin SSL, add ability to skip peer verification. diff --git a/ixsentry/CMakeLists.txt b/ixsentry/CMakeLists.txt index 438733d8..d513fb0a 100644 --- a/ixsentry/CMakeLists.txt +++ b/ixsentry/CMakeLists.txt @@ -19,7 +19,7 @@ add_library(ixsentry STATIC set(IXSENTRY_INCLUDE_DIRS . .. - ../third_party - ../third_party/spdlog/include) + ../ixcore + ../third_party) target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} ) diff --git a/ixsentry/ixsentry/IXSentryClient.cpp b/ixsentry/ixsentry/IXSentryClient.cpp index 1949f749..c14041a4 100644 --- a/ixsentry/ixsentry/IXSentryClient.cpp +++ b/ixsentry/ixsentry/IXSentryClient.cpp @@ -8,8 +8,9 @@ #include #include +#include #include -#include +#include namespace ix @@ -18,6 +19,7 @@ namespace ix : _dsn(dsn) , _validDsn(false) , _luaFrameRegex("\t([^/]+):([0-9]+): in function ['<]([^/]+)['>]") + , _httpClient(std::make_shared(true)) { const std::regex dsnRegex("(http[s]?)://([^:]+):([^@]+)@([^/]+)/([0-9]+)"); std::smatch group; @@ -169,39 +171,64 @@ namespace ix std::pair SentryClient::send(const Json::Value& msg, bool verbose) { - auto args = _httpClient.createRequest(); + auto args = _httpClient->createRequest(); args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader(); args->connectTimeout = 60; args->transferTimeout = 5 * 60; args->followRedirects = true; args->verbose = verbose; - args->logger = [](const std::string& msg) { spdlog::info("request logger: {}", msg); }; + args->logger = [](const std::string& msg) { ix::IXCoreLogger::Log(msg.c_str()); }; std::string body = computePayload(msg); - HttpResponsePtr response = _httpClient.post(_url, body, args); - - if (verbose) - { - for (auto it : response->headers) - { - spdlog::info("{}: {}", it.first, it.second); - } - - spdlog::info("Upload size: {}", response->uploadSize); - spdlog::info("Download size: {}", response->downloadSize); - - spdlog::info("Status: {}", response->statusCode); - if (response->errorCode != HttpErrorCode::Ok) - { - spdlog::info("error message: {}", response->errorMsg); - } - - if (response->headers["Content-Type"] != "application/octet-stream") - { - spdlog::info("payload: {}", response->payload); - } - } + HttpResponsePtr response = _httpClient->post(_url, body, args); return std::make_pair(response, body); } + + // https://sentry.io/api/12345/minidump?sentry_key=abcdefgh"); + std::string SentryClient::computeUrl(const std::string& project, const std::string& key) + { + std::stringstream ss; + ss << "https://sentry.io/api/" + << project + << "/minidump?sentry_key=" + << key; + + return ss.str(); + } + + // + // curl -v -X POST -F upload_file_minidump=@ws/crash.dmp 'https://sentry.io/api/123456/minidump?sentry_key=12344567890' + // + void SentryClient::uploadMinidump( + const std::string& sentryMetadata, + const std::string& minidumpBytes, + const std::string& project, + const std::string& key, + bool verbose, + const OnResponseCallback& onResponseCallback) + { + std::string multipartBoundary = _httpClient->generateMultipartBoundary(); + + auto args = _httpClient->createRequest(); + args->verb = HttpClient::kPost; + args->connectTimeout = 60; + args->transferTimeout = 5 * 60; + args->followRedirects = true; + args->verbose = verbose; + args->multipartBoundary = multipartBoundary; + args->logger = [](const std::string& msg) { ix::IXCoreLogger::Log(msg.c_str()); }; + + HttpFormDataParameters httpFormDataParameters; + httpFormDataParameters["upload_file_minidump"] = minidumpBytes; + + HttpParameters httpParameters; + httpParameters["sentry"] = sentryMetadata; + + args->url = computeUrl(project, key); + args->body = _httpClient->serializeHttpFormDataParameters(multipartBoundary, httpFormDataParameters, httpParameters); + + + _httpClient->performRequest(args, onResponseCallback); + } } // namespace ix diff --git a/ixsentry/ixsentry/IXSentryClient.h b/ixsentry/ixsentry/IXSentryClient.h index 30b0b48c..a0a25a80 100644 --- a/ixsentry/ixsentry/IXSentryClient.h +++ b/ixsentry/ixsentry/IXSentryClient.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace ix { @@ -23,12 +24,24 @@ namespace ix Json::Value parseLuaStackTrace(const std::string& stack); + void uploadMinidump( + const std::string& sentryMetadata, + const std::string& minidumpBytes, + const std::string& project, + const std::string& key, + bool verbose, + const OnResponseCallback& onResponseCallback); + private: int64_t getTimestamp(); std::string computeAuthHeader(); std::string getIso8601(); std::string computePayload(const Json::Value& msg); + std::string computeUrl(const std::string& project, const std::string& key); + + void displayReponse(HttpResponsePtr response); + std::string _dsn; bool _validDsn; std::string _url; @@ -41,7 +54,7 @@ namespace ix std::regex _luaFrameRegex; - HttpClient _httpClient; + std::shared_ptr _httpClient; }; } // namespace ix diff --git a/ixwebsocket/IXHttp.h b/ixwebsocket/IXHttp.h index ec7e4f5e..4e5ccdaa 100644 --- a/ixwebsocket/IXHttp.h +++ b/ixwebsocket/IXHttp.h @@ -66,6 +66,7 @@ namespace ix using HttpResponsePtr = std::shared_ptr; using HttpParameters = std::map; + using HttpFormDataParameters = std::map; using Logger = std::function; using OnResponseCallback = std::function; @@ -75,6 +76,7 @@ namespace ix std::string verb; WebSocketHttpHeaders extraHeaders; std::string body; + std::string multipartBoundary; int connectTimeout; int transferTimeout; bool followRedirects; diff --git a/ixwebsocket/IXHttpClient.cpp b/ixwebsocket/IXHttpClient.cpp index 282e5f80..aa526720 100644 --- a/ixwebsocket/IXHttpClient.cpp +++ b/ixwebsocket/IXHttpClient.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -198,8 +199,16 @@ namespace ix // Set default Content-Type if unspecified if (args->extraHeaders.find("Content-Type") == args->extraHeaders.end()) { - ss << "Content-Type: application/x-www-form-urlencoded" - << "\r\n"; + if (args->multipartBoundary.empty()) + { + ss << "Content-Type: application/x-www-form-urlencoded" + << "\r\n"; + } + else + { + ss << "Content-Type: multipart/form-data; boundary=" << args->multipartBoundary + << "\r\n"; + } } ss << "\r\n"; ss << body; @@ -597,6 +606,53 @@ namespace ix return ss.str(); } + std::string HttpClient::serializeHttpFormDataParameters( + const std::string& multipartBoundary, + const HttpFormDataParameters& httpFormDataParameters, + const HttpParameters& httpParameters) + { + // + // --AaB03x + // Content-Disposition: form-data; name="submit-name" + + // Larry + // --AaB03x + // Content-Disposition: form-data; name="foo.txt"; filename="file1.txt" + // Content-Type: text/plain + + // ... contents of file1.txt ... + // --AaB03x-- + // + std::stringstream ss; + + for (auto&& it : httpFormDataParameters) + { + ss << "--" << multipartBoundary << "\r\n" + << "Content-Disposition:" + << " form-data; name=\"" << it.first << "\";" + << " filename=\"" << it.first << "\"" + << "\r\n" + << "Content-Type: application/octet-stream" + << "\r\n" + << "\r\n" + << it.second << "\r\n"; + } + + for (auto&& it : httpParameters) + { + ss << "--" << multipartBoundary << "\r\n" + << "Content-Disposition:" + << " form-data; name=\"" << it.first << "\";" + << "\r\n" + << "\r\n" + << it.second << "\r\n"; + } + + ss << "--" << multipartBoundary << "\r\n"; + + return ss.str(); + } + bool HttpClient::gzipInflate(const std::string& in, std::string& out) { z_stream inflateState; @@ -649,4 +705,16 @@ namespace ix args->logger(msg); } } + + std::string HttpClient::generateMultipartBoundary() + { + std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + + static std::random_device rd; + static std::mt19937 generator(rd()); + + std::shuffle(str.begin(), str.end(), generator); + + return str; + } } // namespace ix diff --git a/ixwebsocket/IXHttpClient.h b/ixwebsocket/IXHttpClient.h index 7c6177a7..58144687 100644 --- a/ixwebsocket/IXHttpClient.h +++ b/ixwebsocket/IXHttpClient.h @@ -64,6 +64,13 @@ namespace ix std::string serializeHttpParameters(const HttpParameters& httpParameters); + std::string serializeHttpFormDataParameters( + const std::string& multipartBoundary, + const HttpFormDataParameters& httpFormDataParameters, + const HttpParameters& httpParameters = HttpParameters()); + + std::string generateMultipartBoundary(); + std::string urlEncode(const std::string& value); const static std::string kPost; diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index ba91a595..de9080f5 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "7.3.5" +#define IX_WEBSOCKET_VERSION "7.4.0" diff --git a/ws/CMakeLists.txt b/ws/CMakeLists.txt index 29a98283..313a5115 100644 --- a/ws/CMakeLists.txt +++ b/ws/CMakeLists.txt @@ -56,6 +56,7 @@ add_executable(ws ws_httpd.cpp ws_autobahn.cpp ws_proxy_server.cpp + ws_sentry_minidump_upload.cpp ws.cpp) target_link_libraries(ws ixsnake) diff --git a/ws/ws.cpp b/ws/ws.cpp index 1cb39a1b..fac3dcd9 100644 --- a/ws/ws.cpp +++ b/ws/ws.cpp @@ -73,6 +73,10 @@ int main(int argc, char** argv) std::string appsConfigPath("appsConfig.json"); std::string subprotocol; std::string remoteHost; + std::string minidump; + std::string metadata; + std::string project; + std::string key; ix::SocketTLSOptions tlsOptions; std::string ciphers; std::string redirectUrl; @@ -311,6 +315,13 @@ int main(int argc, char** argv) proxyServerApp->add_option("--remote_host", remoteHost, "Remote Hostname"); proxyServerApp->add_flag("-v", verbose, "Verbose"); + CLI::App* minidumpApp = app.add_subcommand("upload_minidump", "Upload a minidump to sentry"); + minidumpApp->add_option("--minidump", minidump, "Minidump path")->check(CLI::ExistingPath); + minidumpApp->add_option("--metadata", metadata, "Hostname")->check(CLI::ExistingPath); + minidumpApp->add_option("--project", project, "Sentry Project")->required(); + minidumpApp->add_option("--key", key, "Sentry Key")->required(); + minidumpApp->add_flag("-v", verbose, "Verbose"); + CLI11_PARSE(app, argc, argv); // pid file handling @@ -453,6 +464,10 @@ int main(int argc, char** argv) { ret = ix::ws_proxy_server_main(port, hostname, tlsOptions, remoteHost, verbose); } + else if (app.got_subcommand("upload_minidump")) + { + ret = ix::ws_sentry_minidump_upload(metadata, minidump, project, key, verbose); + } else if (version) { std::cout << "ws " << ix::userAgent() << std::endl; diff --git a/ws/ws.h b/ws/ws.h index 42dae715..861c181f 100644 --- a/ws/ws.h +++ b/ws/ws.h @@ -148,4 +148,10 @@ namespace ix const ix::SocketTLSOptions& tlsOptions, const std::string& remoteHost, bool verbose); + + int ws_sentry_minidump_upload(const std::string& metadataPath, + const std::string& minidump, + const std::string& project, + const std::string& key, + bool verbose); } // namespace ix diff --git a/ws/ws_cobra_to_sentry.cpp b/ws/ws_cobra_to_sentry.cpp index 3759a0b9..bf54cb18 100644 --- a/ws/ws_cobra_to_sentry.cpp +++ b/ws/ws_cobra_to_sentry.cpp @@ -81,6 +81,29 @@ namespace ix auto ret = sentryClient.send(msg, verbose); HttpResponsePtr response = ret.first; + + if (verbose) + { + for (auto it : response->headers) + { + spdlog::info("{}: {}", it.first, it.second); + } + + spdlog::info("Upload size: {}", response->uploadSize); + spdlog::info("Download size: {}", response->downloadSize); + + spdlog::info("Status: {}", response->statusCode); + if (response->errorCode != HttpErrorCode::Ok) + { + spdlog::info("error message: {}", response->errorMsg); + } + + if (response->headers["Content-Type"] != "application/octet-stream") + { + spdlog::info("payload: {}", response->payload); + } + } + if (response->statusCode != 200) { spdlog::error("Error sending data to sentry: {}", response->statusCode); diff --git a/ws/ws_sentry_minidump_upload.cpp b/ws/ws_sentry_minidump_upload.cpp new file mode 100644 index 00000000..51cbc273 --- /dev/null +++ b/ws/ws_sentry_minidump_upload.cpp @@ -0,0 +1,105 @@ +/* + * ws_sentry_minidump_upload.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include + + +namespace +{ + // Assume the file exists + std::string readBytes(const std::string& path) + { + std::vector memblock; + std::ifstream file(path); + + file.seekg(0, file.end); + std::streamoff size = file.tellg(); + file.seekg(0, file.beg); + + memblock.resize(size); + + file.read((char*) &memblock.front(), static_cast(size)); + + std::string bytes(memblock.begin(), memblock.end()); + return bytes; + } +} // namespace + +namespace ix +{ + int ws_sentry_minidump_upload(const std::string& metadataPath, + const std::string& minidump, + const std::string& project, + const std::string& key, + bool verbose) + { + SentryClient sentryClient((std::string())); + + // Read minidump file from disk + std::string minidumpBytes = readBytes(minidump); + + // Read json data + std::string sentryMetadata = readBytes(metadataPath); + + + std::atomic done(false); + + sentryClient.uploadMinidump( + sentryMetadata, + minidumpBytes, + project, + key, + verbose, + [verbose, &done](const HttpResponsePtr& response) { + if (verbose) + { + for (auto it : response->headers) + { + spdlog::info("{}: {}", it.first, it.second); + } + + spdlog::info("Upload size: {}", response->uploadSize); + spdlog::info("Download size: {}", response->downloadSize); + + spdlog::info("Status: {}", response->statusCode); + if (response->errorCode != HttpErrorCode::Ok) + { + spdlog::info("error message: {}", response->errorMsg); + } + + if (response->headers["Content-Type"] != "application/octet-stream") + { + spdlog::info("payload: {}", response->payload); + } + } + + if (response->statusCode != 200) + { + spdlog::error("Error sending data to sentry: {}", response->statusCode); + spdlog::error("Status: {}", response->statusCode); + spdlog::error("Response: {}", response->payload); + } + else + { + spdlog::info("Event sent to sentry"); + } + + done = true; + }); + + while (!done) + { + std::chrono::duration duration(10); + std::this_thread::sleep_for(duration); + } + + return 0; + } +} // namespace ix