From 6f188a5131a4170e31bdb1732cbfabceadc10261 Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Mon, 28 Sep 2020 10:19:27 -0700 Subject: [PATCH] (ws) add gzip and gunzip ws sub commands --- CMakeLists.txt | 2 + docs/CHANGELOG.md | 4 + ixcrypto/ixcrypto/IXHash.cpp | 12 +++ ixcrypto/ixcrypto/IXHash.h | 2 + ixwebsocket/IXGzipCodec.cpp | 104 +++++++++++++++++++++++ ixwebsocket/IXGzipCodec.h | 15 ++++ ixwebsocket/IXHttpClient.cpp | 67 ++++----------- ixwebsocket/IXHttpClient.h | 4 - ixwebsocket/IXHttpServer.cpp | 50 +----------- ixwebsocket/IXWebSocketVersion.h | 2 +- ws/ws.cpp | 136 +++++++++++++++++++++++++++---- 11 files changed, 275 insertions(+), 123 deletions(-) create mode 100644 ixwebsocket/IXGzipCodec.cpp create mode 100644 ixwebsocket/IXGzipCodec.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b7b6f84..6564c77e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ set( IXWEBSOCKET_SOURCES ixwebsocket/IXDNSLookup.cpp ixwebsocket/IXExponentialBackoff.cpp ixwebsocket/IXGetFreePort.cpp + ixwebsocket/IXGzipCodec.cpp ixwebsocket/IXHttp.cpp ixwebsocket/IXHttpClient.cpp ixwebsocket/IXHttpServer.cpp @@ -66,6 +67,7 @@ set( IXWEBSOCKET_HEADERS ixwebsocket/IXDNSLookup.h ixwebsocket/IXExponentialBackoff.h ixwebsocket/IXGetFreePort.h + ixwebsocket/IXGzipCodec.h ixwebsocket/IXHttp.h ixwebsocket/IXHttpClient.h ixwebsocket/IXHttpServer.h diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 91547f5c..359d09d9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ All changes to this project will be documented in this file. +## [10.4.7] - 2020-09-28 + +(ws) add gzip and gunzip ws sub commands + ## [10.4.6] - 2020-09-26 (cmake) use FetchContent cmake module to retrieve jsoncpp third party dependency diff --git a/ixcrypto/ixcrypto/IXHash.cpp b/ixcrypto/ixcrypto/IXHash.cpp index a62e0871..99bfb264 100644 --- a/ixcrypto/ixcrypto/IXHash.cpp +++ b/ixcrypto/ixcrypto/IXHash.cpp @@ -19,4 +19,16 @@ namespace ix return hashAddress; } + + uint64_t djb2HashStr(const std::string& data) + { + uint64_t hashAddress = 5381; + + for (size_t i = 0; i < data.size(); ++i) + { + hashAddress = ((hashAddress << 5) + hashAddress) + data[i]; + } + + return hashAddress; + } } // namespace ix diff --git a/ixcrypto/ixcrypto/IXHash.h b/ixcrypto/ixcrypto/IXHash.h index 98d660e1..85ed97f7 100644 --- a/ixcrypto/ixcrypto/IXHash.h +++ b/ixcrypto/ixcrypto/IXHash.h @@ -8,8 +8,10 @@ #include #include +#include namespace ix { uint64_t djb2Hash(const std::vector& data); + uint64_t djb2HashStr(const std::string& data); } diff --git a/ixwebsocket/IXGzipCodec.cpp b/ixwebsocket/IXGzipCodec.cpp new file mode 100644 index 00000000..00825aab --- /dev/null +++ b/ixwebsocket/IXGzipCodec.cpp @@ -0,0 +1,104 @@ +/* + * IXGzipCodec.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + */ + +#include "IXGzipCodec.h" + +#include + +#ifdef IXWEBSOCKET_USE_ZLIB +#include +#endif + +namespace ix +{ +#ifdef IXWEBSOCKET_USE_ZLIB + std::string gzipCompress(const std::string& str) + { + z_stream zs; // z_stream is zlib's control structure + memset(&zs, 0, sizeof(zs)); + + // deflateInit2 configure the file format: request gzip instead of deflate + const int windowBits = 15; + const int GZIP_ENCODING = 16; + + deflateInit2(&zs, + Z_DEFAULT_COMPRESSION, + Z_DEFLATED, + windowBits | GZIP_ENCODING, + 8, + Z_DEFAULT_STRATEGY); + + zs.next_in = (Bytef*) str.data(); + zs.avail_in = (uInt) str.size(); // set the z_stream's input + + int ret; + char outbuffer[32768]; + std::string outstring; + + // retrieve the compressed bytes blockwise + do + { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = deflate(&zs, Z_FINISH); + + if (outstring.size() < zs.total_out) + { + // append the block to the output string + outstring.append(outbuffer, zs.total_out - outstring.size()); + } + } while (ret == Z_OK); + + deflateEnd(&zs); + + return outstring; + } + + bool gzipDecompress(const std::string& in, std::string& out) + { + z_stream inflateState; + std::memset(&inflateState, 0, sizeof(inflateState)); + + inflateState.zalloc = Z_NULL; + inflateState.zfree = Z_NULL; + inflateState.opaque = Z_NULL; + inflateState.avail_in = 0; + inflateState.next_in = Z_NULL; + + if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK) + { + return false; + } + + inflateState.avail_in = (uInt) in.size(); + inflateState.next_in = (unsigned char*) (const_cast(in.data())); + + const int kBufferSize = 1 << 14; + std::array compressBuffer; + + do + { + inflateState.avail_out = (uInt) kBufferSize; + inflateState.next_out = &compressBuffer.front(); + + int ret = inflate(&inflateState, Z_SYNC_FLUSH); + + if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) + { + inflateEnd(&inflateState); + return false; + } + + out.append(reinterpret_cast(&compressBuffer.front()), + kBufferSize - inflateState.avail_out); + } while (inflateState.avail_out == 0); + + inflateEnd(&inflateState); + return true; + } +#endif +} // namespace ix diff --git a/ixwebsocket/IXGzipCodec.h b/ixwebsocket/IXGzipCodec.h new file mode 100644 index 00000000..8a5fc113 --- /dev/null +++ b/ixwebsocket/IXGzipCodec.h @@ -0,0 +1,15 @@ +/* + * IXGzipCodec.h + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + */ + +#pragma once + +#include + +namespace ix +{ + std::string gzipCompress(const std::string& str); + bool gzipDecompress(const std::string& in, std::string& out); +} // namespace ix diff --git a/ixwebsocket/IXHttpClient.cpp b/ixwebsocket/IXHttpClient.cpp index 2673bf7b..f852e0f4 100644 --- a/ixwebsocket/IXHttpClient.cpp +++ b/ixwebsocket/IXHttpClient.cpp @@ -6,11 +6,11 @@ #include "IXHttpClient.h" +#include "IXGzipCodec.h" #include "IXSocketFactory.h" #include "IXUrlParser.h" #include "IXUserAgent.h" #include "IXWebSocketHttpHeaders.h" -#include #include #include #include @@ -18,10 +18,6 @@ #include #include -#ifdef IXWEBSOCKET_USE_ZLIB -#include -#endif - namespace ix { const std::string HttpClient::kPost = "POST"; @@ -501,12 +497,12 @@ namespace ix downloadSize = payload.size(); -#ifdef IXWEBSOCKET_USE_ZLIB // If the content was compressed with gzip, decode it if (headers["Content-Encoding"] == "gzip") { +#ifdef IXWEBSOCKET_USE_ZLIB std::string decompressedPayload; - if (!gzipInflate(payload, decompressedPayload)) + if (!gzipDecompress(payload, decompressedPayload)) { std::string errorMsg("Error decompressing payload"); return std::make_shared(code, @@ -519,8 +515,18 @@ namespace ix downloadSize); } payload = decompressedPayload; - } +#else + std::string errorMsg("ixwebsocket was not compiled with gzip support on"); + return std::make_shared(code, + description, + HttpErrorCode::Gzip, + headers, + payload, + errorMsg, + uploadSize, + downloadSize); #endif + } return std::make_shared(code, description, @@ -680,51 +686,6 @@ namespace ix return ss.str(); } -#ifdef IXWEBSOCKET_USE_ZLIB - bool HttpClient::gzipInflate(const std::string& in, std::string& out) - { - z_stream inflateState; - std::memset(&inflateState, 0, sizeof(inflateState)); - - inflateState.zalloc = Z_NULL; - inflateState.zfree = Z_NULL; - inflateState.opaque = Z_NULL; - inflateState.avail_in = 0; - inflateState.next_in = Z_NULL; - - if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK) - { - return false; - } - - inflateState.avail_in = (uInt) in.size(); - inflateState.next_in = (unsigned char*) (const_cast(in.data())); - - const int kBufferSize = 1 << 14; - std::array compressBuffer; - - do - { - inflateState.avail_out = (uInt) kBufferSize; - inflateState.next_out = &compressBuffer.front(); - - int ret = inflate(&inflateState, Z_SYNC_FLUSH); - - if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) - { - inflateEnd(&inflateState); - return false; - } - - out.append(reinterpret_cast(&compressBuffer.front()), - kBufferSize - inflateState.avail_out); - } while (inflateState.avail_out == 0); - - inflateEnd(&inflateState); - return true; - } -#endif - void HttpClient::log(const std::string& msg, HttpRequestArgsPtr args) { if (args->logger) diff --git a/ixwebsocket/IXHttpClient.h b/ixwebsocket/IXHttpClient.h index 9e585e7d..afa18f85 100644 --- a/ixwebsocket/IXHttpClient.h +++ b/ixwebsocket/IXHttpClient.h @@ -90,10 +90,6 @@ namespace ix private: void log(const std::string& msg, HttpRequestArgsPtr args); -#ifdef IXWEBSOCKET_USE_ZLIB - bool gzipInflate(const std::string& in, std::string& out); -#endif - // Async API background thread runner void run(); // Async API diff --git a/ixwebsocket/IXHttpServer.cpp b/ixwebsocket/IXHttpServer.cpp index ecc9038a..8f830497 100644 --- a/ixwebsocket/IXHttpServer.cpp +++ b/ixwebsocket/IXHttpServer.cpp @@ -6,6 +6,7 @@ #include "IXHttpServer.h" +#include "IXGzipCodec.h" #include "IXNetSystem.h" #include "IXSocketConnect.h" #include "IXUserAgent.h" @@ -14,10 +15,6 @@ #include #include -#ifdef IXWEBSOCKET_USE_ZLIB -#include -#endif - namespace { std::pair> load(const std::string& path) @@ -43,51 +40,6 @@ namespace auto vec = res.second; return std::make_pair(res.first, std::string(vec.begin(), vec.end())); } - -#ifdef IXWEBSOCKET_USE_ZLIB - std::string gzipCompress(const std::string& str) - { - z_stream zs; // z_stream is zlib's control structure - memset(&zs, 0, sizeof(zs)); - - // deflateInit2 configure the file format: request gzip instead of deflate - const int windowBits = 15; - const int GZIP_ENCODING = 16; - - deflateInit2(&zs, - Z_DEFAULT_COMPRESSION, - Z_DEFLATED, - windowBits | GZIP_ENCODING, - 8, - Z_DEFAULT_STRATEGY); - - zs.next_in = (Bytef*) str.data(); - zs.avail_in = (uInt) str.size(); // set the z_stream's input - - int ret; - char outbuffer[32768]; - std::string outstring; - - // retrieve the compressed bytes blockwise - do - { - zs.next_out = reinterpret_cast(outbuffer); - zs.avail_out = sizeof(outbuffer); - - ret = deflate(&zs, Z_FINISH); - - if (outstring.size() < zs.total_out) - { - // append the block to the output string - outstring.append(outbuffer, zs.total_out - outstring.size()); - } - } while (ret == Z_OK); - - deflateEnd(&zs); - - return outstring; - } -#endif } // namespace namespace ix diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index 7f2cb9cf..f3c3f898 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "10.4.6" +#define IX_WEBSOCKET_VERSION "10.4.7" diff --git a/ws/ws.cpp b/ws/ws.cpp index 07b08f14..1f9f59bd 100644 --- a/ws/ws.cpp +++ b/ws/ws.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -127,6 +128,38 @@ namespace std::ifstream infile(fileName); return infile.good(); } + + std::string extractFilename(const std::string& path) + { + std::string::size_type idx; + + idx = path.rfind('/'); + if (idx != std::string::npos) + { + std::string filename = path.substr(idx + 1); + return filename; + } + else + { + return path; + } + } + + std::string removeExtension(const std::string& path) + { + std::string::size_type idx; + + idx = path.rfind('.'); + if (idx != std::string::npos) + { + std::string filename = path.substr(idx + 1); + return filename; + } + else + { + return path; + } + } } // namespace namespace ix @@ -1103,6 +1136,76 @@ namespace ix return 0; } + int ws_gzip(const std::string& filename) + { + auto res = readAsString(filename); + bool found = res.first; + if (!found) + { + spdlog::error("Cannot read content of {}", filename); + return 1; + } + + spdlog::info("gzip input: {} cksum {}", filename, ix::djb2HashStr(res.second)); + + std::string compressedBytes; + + { + Bench bench("compressing file"); + compressedBytes = gzipCompress(res.second); + } + + std::string outputFilename(filename); + outputFilename += ".gz"; + + std::ofstream f; + f.open(outputFilename); + f << compressedBytes; + f.close(); + + spdlog::info("gzip output: {} cksum {}", outputFilename, ix::djb2HashStr(compressedBytes)); + + return 0; + } + + int ws_gunzip(const std::string& filename) + { + spdlog::info("filename to gunzip: {}", filename); + + auto res = readAsString(filename); + bool found = res.first; + if (!found) + { + spdlog::error("Cannot read content of {}", filename); + return 1; + } + + spdlog::info("gunzip input: {} cksum {}", filename, ix::djb2HashStr(res.second)); + + std::string decompressedBytes; + + { + Bench bench("decompressing file"); + if (!gzipDecompress(res.second, decompressedBytes)) + { + spdlog::error("Cannot decompress content of {}", filename); + return 1; + } + } + + std::string outputFilename(removeExtension(filename)); + + std::ofstream f; + f.open(outputFilename); + f << decompressedBytes; + f.close(); + + spdlog::info( + "gunzip output: {} cksum {}", outputFilename, ix::djb2HashStr(decompressedBytes)); + + return 0; + } + int ws_autoroute(const std::string& url, bool disablePerMessageDeflate, const ix::SocketTLSOptions& tlsOptions, @@ -1305,22 +1408,6 @@ namespace ix return 0; } - std::string extractFilename(const std::string& path) - { - std::string::size_type idx; - - idx = path.rfind('/'); - if (idx != std::string::npos) - { - std::string filename = path.substr(idx + 1); - return filename; - } - else - { - return path; - } - } - WebSocketHttpHeaders parseHeaders(const std::string& data) { WebSocketHttpHeaders headers; @@ -2848,6 +2935,7 @@ int main(int argc, char** argv) std::string publisherRolename; std::string publisherRolesecret; std::string sendMsg("hello world"); + std::string filename; ix::SocketTLSOptions tlsOptions; ix::CobraConfig cobraConfig; ix::CobraBotConfig cobraBotConfig; @@ -3206,6 +3294,14 @@ int main(int argc, char** argv) dnsLookupApp->fallthrough(); dnsLookupApp->add_option("host", hostname, "Hostname")->required(); + CLI::App* gzipApp = app.add_subcommand("gzip", "Gzip compressor"); + gzipApp->fallthrough(); + gzipApp->add_option("filename", filename, "Filename")->required(); + + CLI::App* gunzipApp = app.add_subcommand("gunzip", "Gzip decompressor"); + gunzipApp->fallthrough(); + gunzipApp->add_option("filename", filename, "Filename")->required(); + CLI11_PARSE(app, argc, argv); // pid file handling @@ -3502,6 +3598,14 @@ int main(int argc, char** argv) { ret = ix::ws_dns_lookup(hostname); } + else if (app.got_subcommand("gzip")) + { + ret = ix::ws_gzip(filename); + } + else if (app.got_subcommand("gunzip")) + { + ret = ix::ws_gunzip(filename); + } else if (version) { std::cout << "ws " << ix::userAgent() << std::endl;