From fa0408e70be9acec606c011d533182b35d5a35ea Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Thu, 8 Oct 2020 12:43:18 -0700 Subject: [PATCH] (http client + server + ws) Add support for uploading files with ws -F foo=@filename, new -D http server option to debug incoming client requests, internal api changed for http POST, PUT and PATCH to supply an HttpFormDataParameters --- docs/CHANGELOG.md | 4 + docs/usage.md | 9 ++- ixwebsocket/IXHttpClient.cpp | 42 +++++++++- ixwebsocket/IXHttpClient.h | 3 + ixwebsocket/IXHttpServer.cpp | 36 +++++++++ ixwebsocket/IXHttpServer.h | 2 + ixwebsocket/IXWebSocketVersion.h | 2 +- ws/ws.cpp | 128 ++++++++++++++++++++++--------- 8 files changed, 183 insertions(+), 43 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f497787d..25a7a85b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ All changes to this project will be documented in this file. +## [10.5.0] - 2020-09-30 + +(http client + server + ws) Add support for uploading files with ws -F foo=@filename, new -D http server option to debug incoming client requests, internal api changed for http POST, PUT and PATCH to supply an HttpFormDataParameters + ## [10.4.9] - 2020-09-30 (http server + utility code) Add support for doing gzip compression with libdeflate library, if available diff --git a/docs/usage.md b/docs/usage.md index e4b0faaf..9c2f38b2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -458,11 +458,18 @@ out = httpClient.get(url, args); // POST request with parameters HttpParameters httpParameters; httpParameters["foo"] = "bar"; -out = httpClient.post(url, httpParameters, args); + +// HTTP form data can be passed in as well, for multi-part upload of files +HttpFormDataParameters httpFormDataParameters; +httpParameters["baz"] = "booz"; + +out = httpClient.post(url, httpParameters, httpFormDataParameters, args); // POST request with a body out = httpClient.post(url, std::string("foo=bar"), args); +// PUT and PATCH are available too. + // // Result // diff --git a/ixwebsocket/IXHttpClient.cpp b/ixwebsocket/IXHttpClient.cpp index f852e0f4..c643656e 100644 --- a/ixwebsocket/IXHttpClient.cpp +++ b/ixwebsocket/IXHttpClient.cpp @@ -555,9 +555,21 @@ namespace ix HttpResponsePtr HttpClient::post(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - return request(url, kPost, serializeHttpParameters(httpParameters), args); + if (httpFormDataParameters.empty()) + { + return request(url, kPost, serializeHttpParameters(httpParameters), args); + } + else + { + std::string multipartBoundary = generateMultipartBoundary(); + args->multipartBoundary = multipartBoundary; + std::string body = serializeHttpFormDataParameters( + multipartBoundary, httpFormDataParameters, httpParameters); + return request(url, kPost, body, args); + } } HttpResponsePtr HttpClient::post(const std::string& url, @@ -569,9 +581,21 @@ namespace ix HttpResponsePtr HttpClient::put(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - return request(url, kPut, serializeHttpParameters(httpParameters), args); + if (httpFormDataParameters.empty()) + { + return request(url, kPut, serializeHttpParameters(httpParameters), args); + } + else + { + std::string multipartBoundary = generateMultipartBoundary(); + args->multipartBoundary = multipartBoundary; + std::string body = serializeHttpFormDataParameters( + multipartBoundary, httpFormDataParameters, httpParameters); + return request(url, kPut, body, args); + } } HttpResponsePtr HttpClient::put(const std::string& url, @@ -583,9 +607,21 @@ namespace ix HttpResponsePtr HttpClient::patch(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - return request(url, kPatch, serializeHttpParameters(httpParameters), args); + if (httpFormDataParameters.empty()) + { + return request(url, kPatch, serializeHttpParameters(httpParameters), args); + } + else + { + std::string multipartBoundary = generateMultipartBoundary(); + args->multipartBoundary = multipartBoundary; + std::string body = serializeHttpFormDataParameters( + multipartBoundary, httpFormDataParameters, httpParameters); + return request(url, kPatch, body, args); + } } HttpResponsePtr HttpClient::patch(const std::string& url, diff --git a/ixwebsocket/IXHttpClient.h b/ixwebsocket/IXHttpClient.h index afa18f85..d14c4881 100644 --- a/ixwebsocket/IXHttpClient.h +++ b/ixwebsocket/IXHttpClient.h @@ -34,6 +34,7 @@ namespace ix HttpResponsePtr post(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args); HttpResponsePtr post(const std::string& url, const std::string& body, @@ -41,6 +42,7 @@ namespace ix HttpResponsePtr put(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args); HttpResponsePtr put(const std::string& url, const std::string& body, @@ -48,6 +50,7 @@ namespace ix HttpResponsePtr patch(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args); HttpResponsePtr patch(const std::string& url, const std::string& body, diff --git a/ixwebsocket/IXHttpServer.cpp b/ixwebsocket/IXHttpServer.cpp index 8f830497..070d6fee 100644 --- a/ixwebsocket/IXHttpServer.cpp +++ b/ixwebsocket/IXHttpServer.cpp @@ -190,4 +190,40 @@ namespace ix 301, "OK", HttpErrorCode::Ok, headers, std::string()); }); } + + // + // Display the client parameter and body on the console + // + void HttpServer::makeDebugServer() + { + setOnConnectionCallback( + [this](HttpRequestPtr request, + std::shared_ptr connectionState) -> HttpResponsePtr { + WebSocketHttpHeaders headers; + headers["Server"] = userAgent(); + + // Log request + std::stringstream ss; + ss << connectionState->getRemoteIp() << ":" << connectionState->getRemotePort() + << " " << request->method << " " << request->headers["User-Agent"] << " " + << request->uri; + logInfo(ss.str()); + + logInfo("== Headers == "); + for (auto&& it : request->headers) + { + std::ostringstream oss; + oss << it.first << ": " << it.second; + logInfo(oss.str()); + } + logInfo(""); + + logInfo("== Body == "); + logInfo(request->body); + logInfo(""); + + return std::make_shared( + 200, "OK", HttpErrorCode::Ok, headers, std::string("OK")); + }); + } } // namespace ix diff --git a/ixwebsocket/IXHttpServer.h b/ixwebsocket/IXHttpServer.h index 08e9a5d0..31cd77fb 100644 --- a/ixwebsocket/IXHttpServer.h +++ b/ixwebsocket/IXHttpServer.h @@ -38,6 +38,8 @@ namespace ix void makeRedirectServer(const std::string& redirectUrl); + void makeDebugServer(); + private: // Member variables OnConnectionCallback _onConnectionCallback; diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index 40f70906..49aa4946 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "10.4.9" +#define IX_WEBSOCKET_VERSION "10.5.0" diff --git a/ws/ws.cpp b/ws/ws.cpp index 76f9343a..b728c7ef 100644 --- a/ws/ws.cpp +++ b/ws/ws.cpp @@ -1271,43 +1271,42 @@ namespace ix // Setup a callback to be fired // when a message or an event (open, close, ping, pong, error) is received - webSocket.setOnMessageCallback( - [&webSocket, &receivedCountPerSecs, &target, &stop, &condition, &bench]( - const ix::WebSocketMessagePtr& msg) { - if (msg->type == ix::WebSocketMessageType::Message) - { - receivedCountPerSecs++; + webSocket.setOnMessageCallback([&receivedCountPerSecs, &target, &stop, &condition, &bench]( + const ix::WebSocketMessagePtr& msg) { + if (msg->type == ix::WebSocketMessageType::Message) + { + receivedCountPerSecs++; - target -= 1; - if (target == 0) - { - stop = true; - condition.notify_one(); + target -= 1; + if (target == 0) + { + stop = true; + condition.notify_one(); - bench.report(); - } + bench.report(); } - else if (msg->type == ix::WebSocketMessageType::Open) - { - bench.reset(); + } + else if (msg->type == ix::WebSocketMessageType::Open) + { + bench.reset(); - spdlog::info("ws_autoroute: connected"); - spdlog::info("Uri: {}", msg->openInfo.uri); - spdlog::info("Headers:"); - for (auto it : msg->openInfo.headers) - { - spdlog::info("{}: {}", it.first, it.second); - } - } - else if (msg->type == ix::WebSocketMessageType::Pong) + spdlog::info("ws_autoroute: connected"); + spdlog::info("Uri: {}", msg->openInfo.uri); + spdlog::info("Headers:"); + for (auto it : msg->openInfo.headers) { - spdlog::info("Received pong {}", msg->str); + spdlog::info("{}: {}", it.first, it.second); } - else if (msg->type == ix::WebSocketMessageType::Close) - { - spdlog::info("ws_autoroute: connection closed"); - } - }); + } + else if (msg->type == ix::WebSocketMessageType::Pong) + { + spdlog::info("Received pong {}", msg->str); + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + spdlog::info("ws_autoroute: connection closed"); + } + }); auto timer = [&receivedCountPerSecs, &stop] { setThreadName("Timer"); @@ -1462,7 +1461,7 @@ namespace ix // Useful endpoint to test HTTP post // https://postman-echo.com/post // - HttpParameters parsePostParameters(const std::string& data) + HttpParameters parseHttpParameters(const std::string& data) { HttpParameters httpParameters; @@ -1487,9 +1486,51 @@ namespace ix return httpParameters; } + HttpFormDataParameters parseHttpFormDataParameters(const std::string& data) + { + HttpFormDataParameters httpFormDataParameters; + + // Split by \n + std::string token; + std::stringstream tokenStream(data); + + while (std::getline(tokenStream, token)) + { + std::size_t pos = token.rfind('='); + + // Bail out if last '.' is found + if (pos == std::string::npos) continue; + + auto key = token.substr(0, pos); + auto val = token.substr(pos + 1); + + spdlog::info("{}: {}", key, val); + + if (val[0] == '@') + { + std::string filename = token.substr(pos + 2); + + auto res = readAsString(filename); + bool found = res.first; + if (!found) + { + spdlog::error("Cannot read content of {}", filename); + continue; + } + + val = res.second; + } + + httpFormDataParameters[key] = val; + } + + return httpFormDataParameters; + } + int ws_http_client_main(const std::string& url, const std::string& headersData, const std::string& data, + const std::string& formData, bool headersOnly, int connectTimeout, int transferTimeout, @@ -1521,20 +1562,21 @@ namespace ix return true; }; - HttpParameters httpParameters = parsePostParameters(data); + HttpParameters httpParameters = parseHttpParameters(data); + HttpFormDataParameters httpFormDataParameters = parseHttpFormDataParameters(formData); HttpResponsePtr response; if (headersOnly) { response = httpClient.head(url, args); } - else if (data.empty()) + else if (data.empty() && formData.empty()) { response = httpClient.get(url, args); } else { - response = httpClient.post(url, httpParameters, args); + response = httpClient.post(url, httpParameters, httpFormDataParameters, args); } spdlog::info(""); @@ -1591,6 +1633,7 @@ namespace ix const std::string& hostname, bool redirect, const std::string& redirectUrl, + bool debug, const ix::SocketTLSOptions& tlsOptions) { spdlog::info("Listening on {}:{}", hostname, port); @@ -1603,6 +1646,11 @@ namespace ix server.makeRedirectServer(redirectUrl); } + if (debug) + { + server.makeDebugServer(); + } + auto res = server.listen(); if (!res.first) { @@ -2930,6 +2978,7 @@ int main(int argc, char** argv) std::string path; std::string user; std::string data; + std::string formData; std::string headers; std::string output; std::string hostname("127.0.0.1"); @@ -2983,6 +3032,7 @@ int main(int argc, char** argv) bool version = false; bool verifyNone = false; bool disablePong = false; + bool debug = false; int port = 8008; int redisPort = 6379; int statsdPort = 8125; @@ -3139,7 +3189,7 @@ int main(int argc, char** argv) httpClientApp->fallthrough(); httpClientApp->add_option("url", url, "Connection url")->required(); httpClientApp->add_option("-d", data, "Form data")->join(); - httpClientApp->add_option("-F", data, "Form data")->join(); + httpClientApp->add_option("-F", formData, "Form data")->join(); httpClientApp->add_option("-H", headers, "Header")->join(); httpClientApp->add_option("--output", output, "Output file"); httpClientApp->add_flag("-I", headersOnly, "Send a HEAD request"); @@ -3147,7 +3197,7 @@ int main(int argc, char** argv) httpClientApp->add_option("--max-redirects", maxRedirects, "Max Redirects"); httpClientApp->add_flag("-v", verbose, "Verbose"); httpClientApp->add_flag("-O", save, "Save output to disk"); - httpClientApp->add_flag("--compress", compress, "Enable gzip compression"); + httpClientApp->add_flag("--compressed", compress, "Enable gzip compression"); httpClientApp->add_option("--connect-timeout", connectTimeOut, "Connection timeout"); httpClientApp->add_option("--transfer-timeout", transferTimeout, "Transfer timeout"); addTLSOptions(httpClientApp); @@ -3281,6 +3331,7 @@ int main(int argc, char** argv) httpServerApp->add_option("--host", hostname, "Hostname"); httpServerApp->add_flag("-L", redirect, "Redirect all request to redirect_url"); httpServerApp->add_option("--redirect_url", redirectUrl, "Url to redirect to"); + httpServerApp->add_flag("-D", debug, "Debug server"); addTLSOptions(httpServerApp); CLI::App* autobahnApp = app.add_subcommand("autobahn", "Test client Autobahn compliance"); @@ -3462,6 +3513,7 @@ int main(int argc, char** argv) ret = ix::ws_http_client_main(url, headers, data, + formData, headersOnly, connectTimeOut, transferTimeout, @@ -3580,7 +3632,7 @@ int main(int argc, char** argv) } else if (app.got_subcommand("httpd")) { - ret = ix::ws_httpd_main(port, hostname, redirect, redirectUrl, tlsOptions); + ret = ix::ws_httpd_main(port, hostname, redirect, redirectUrl, debug, tlsOptions); } else if (app.got_subcommand("autobahn")) {