From 67cb48537ac60d856bf4c18f745c5350e1a0ba49 Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Fri, 9 Oct 2020 17:51:56 -0700 Subject: [PATCH] (http client + server + ws) Add support for compressing http client requests with gzip. --compress_request argument is used in ws to enable this. The Content-Encoding is set to gzip, and decoded on the server side if present. --- docs/CHANGELOG.md | 4 ++ ixwebsocket/IXHttp.cpp | 18 +++++++++ ixwebsocket/IXHttp.h | 1 + ixwebsocket/IXHttpClient.cpp | 65 +++++++++++++++++--------------- ixwebsocket/IXHttpClient.h | 8 ++++ ixwebsocket/IXWebSocketVersion.h | 2 +- ws/ws.cpp | 5 +++ 7 files changed, 71 insertions(+), 32 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 25a7a85b..8b51e0dc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ All changes to this project will be documented in this file. +## [10.5.1] - 2020-10-09 + +(http client + server + ws) Add support for compressing http client requests with gzip. --compress_request argument is used in ws to enable this. The Content-Encoding is set to gzip, and decoded on the server side if present. + ## [10.5.0] - 2020-09-30 (http client + server + ws) Add support for uploading files with ws -F foo=@filename, new -D http server option to debug incoming client requests, internal api changed for http POST, PUT and PATCH to supply an HttpFormDataParameters diff --git a/ixwebsocket/IXHttp.cpp b/ixwebsocket/IXHttp.cpp index bccb6b4d..c6b43521 100644 --- a/ixwebsocket/IXHttp.cpp +++ b/ixwebsocket/IXHttp.cpp @@ -7,6 +7,7 @@ #include "IXHttp.h" #include "IXCancellationRequest.h" +#include "IXGzipCodec.h" #include "IXSocket.h" #include #include @@ -157,6 +158,23 @@ namespace ix body = res.second; } + // If the content was compressed with gzip, decode it + if (headers["Content-Encoding"] == "gzip") + { +#ifdef IXWEBSOCKET_USE_ZLIB + std::string decompressedPayload; + if (!gzipDecompress(body, decompressedPayload)) + { + return std::make_tuple( + false, std::string("Error during gzip decompression of the body"), httpRequest); + } + body = decompressedPayload; +#else + std::string errorMsg("ixwebsocket was not compiled with gzip support on"); + return std::make_tuple(false, errorMsg, httpRequest); +#endif + } + httpRequest = std::make_shared(uri, method, httpVersion, body, headers); return std::make_tuple(true, "", httpRequest); } diff --git a/ixwebsocket/IXHttp.h b/ixwebsocket/IXHttp.h index a0ad5c59..bfdaefcc 100644 --- a/ixwebsocket/IXHttp.h +++ b/ixwebsocket/IXHttp.h @@ -84,6 +84,7 @@ namespace ix int maxRedirects = 5; bool verbose = false; bool compress = true; + bool compressRequest = false; Logger logger; OnProgressCallback onProgressCallback; }; diff --git a/ixwebsocket/IXHttpClient.cpp b/ixwebsocket/IXHttpClient.cpp index c643656e..bac4f653 100644 --- a/ixwebsocket/IXHttpClient.cpp +++ b/ixwebsocket/IXHttpClient.cpp @@ -203,6 +203,13 @@ namespace ix if (verb == kPost || verb == kPut || verb == kPatch || _forceBody) { + // Set request compression header + if (args->compressRequest) + { + ss << "Content-Encoding: gzip" + << "\r\n"; + } + ss << "Content-Length: " << body.size() << "\r\n"; // Set default Content-Type if unspecified @@ -553,23 +560,41 @@ namespace ix return request(url, kDel, std::string(), args); } - HttpResponsePtr HttpClient::post(const std::string& url, - const HttpParameters& httpParameters, - const HttpFormDataParameters& httpFormDataParameters, - HttpRequestArgsPtr args) + HttpResponsePtr HttpClient::request(const std::string& url, + const std::string& verb, + const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, + HttpRequestArgsPtr args) { + std::string body; + if (httpFormDataParameters.empty()) { - return request(url, kPost, serializeHttpParameters(httpParameters), args); + body = serializeHttpParameters(httpParameters); } else { std::string multipartBoundary = generateMultipartBoundary(); args->multipartBoundary = multipartBoundary; - std::string body = serializeHttpFormDataParameters( + body = serializeHttpFormDataParameters( multipartBoundary, httpFormDataParameters, httpParameters); - return request(url, kPost, body, args); } + + if (args->compressRequest) + { + body = gzipCompress(body); + } + + return request(url, verb, body, args); + } + + + HttpResponsePtr HttpClient::post(const std::string& url, + const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, + HttpRequestArgsPtr args) + { + return request(url, kPost, httpParameters, httpFormDataParameters, args); } HttpResponsePtr HttpClient::post(const std::string& url, @@ -584,18 +609,7 @@ namespace ix const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - if (httpFormDataParameters.empty()) - { - return request(url, kPut, serializeHttpParameters(httpParameters), args); - } - else - { - std::string multipartBoundary = generateMultipartBoundary(); - args->multipartBoundary = multipartBoundary; - std::string body = serializeHttpFormDataParameters( - multipartBoundary, httpFormDataParameters, httpParameters); - return request(url, kPut, body, args); - } + return request(url, kPut, httpParameters, httpFormDataParameters, args); } HttpResponsePtr HttpClient::put(const std::string& url, @@ -610,18 +624,7 @@ namespace ix const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - if (httpFormDataParameters.empty()) - { - return request(url, kPatch, serializeHttpParameters(httpParameters), args); - } - else - { - std::string multipartBoundary = generateMultipartBoundary(); - args->multipartBoundary = multipartBoundary; - std::string body = serializeHttpFormDataParameters( - multipartBoundary, httpFormDataParameters, httpParameters); - return request(url, kPatch, body, args); - } + return request(url, kPatch, httpParameters, httpFormDataParameters, args); } HttpResponsePtr HttpClient::patch(const std::string& url, diff --git a/ixwebsocket/IXHttpClient.h b/ixwebsocket/IXHttpClient.h index d14c4881..c192c4bb 100644 --- a/ixwebsocket/IXHttpClient.h +++ b/ixwebsocket/IXHttpClient.h @@ -61,7 +61,15 @@ namespace ix const std::string& body, HttpRequestArgsPtr args, int redirects = 0); + + HttpResponsePtr request(const std::string& url, + const std::string& verb, + const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, + HttpRequestArgsPtr args); + void setForceBody(bool value); + // Async API HttpRequestArgsPtr createRequest(const std::string& url = std::string(), const std::string& verb = HttpClient::kGet); diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index 49aa4946..85a76afb 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "10.5.0" +#define IX_WEBSOCKET_VERSION "10.5.1" diff --git a/ws/ws.cpp b/ws/ws.cpp index b728c7ef..f7d8b816 100644 --- a/ws/ws.cpp +++ b/ws/ws.cpp @@ -1540,6 +1540,7 @@ namespace ix bool save, const std::string& output, bool compress, + bool compressRequest, const ix::SocketTLSOptions& tlsOptions) { HttpClient httpClient; @@ -1553,6 +1554,7 @@ namespace ix args->maxRedirects = maxRedirects; args->verbose = verbose; args->compress = compress; + args->compressRequest = compressRequest; args->logger = [](const std::string& msg) { spdlog::info(msg); }; args->onProgressCallback = [verbose](int current, int total) -> bool { if (verbose) @@ -3022,6 +3024,7 @@ int main(int argc, char** argv) bool quiet = false; bool fluentd = false; bool compress = false; + bool compressRequest = false; bool stress = false; bool disableAutomaticReconnection = false; bool disablePerMessageDeflate = false; @@ -3198,6 +3201,7 @@ int main(int argc, char** argv) httpClientApp->add_flag("-v", verbose, "Verbose"); httpClientApp->add_flag("-O", save, "Save output to disk"); httpClientApp->add_flag("--compressed", compress, "Enable gzip compression"); + httpClientApp->add_flag("--compress_request", compressRequest, "Compress request with gzip"); httpClientApp->add_option("--connect-timeout", connectTimeOut, "Connection timeout"); httpClientApp->add_option("--transfer-timeout", transferTimeout, "Transfer timeout"); addTLSOptions(httpClientApp); @@ -3523,6 +3527,7 @@ int main(int argc, char** argv) save, output, compress, + compressRequest, tlsOptions); } else if (app.got_subcommand("redis_cli"))