From dc8408040156d80afbbd08a894de1bf63fc8bfda Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Wed, 30 Sep 2020 14:25:41 -0700 Subject: [PATCH] Add support for gzip compression through libdeflate --- CMake/FindDeflate.cmake | 19 ++++++++++++++ CMakeLists.txt | 8 ++++++ docs/CHANGELOG.md | 4 +++ ixwebsocket/IXBench.cpp | 21 +++++++++++++--- ixwebsocket/IXBench.h | 4 ++- ixwebsocket/IXGzipCodec.cpp | 43 ++++++++++++++++++++++++++++++++ ixwebsocket/IXWebSocketVersion.h | 2 +- ws/ws.cpp | 43 ++++++++++++++++++++++++++------ 8 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 CMake/FindDeflate.cmake diff --git a/CMake/FindDeflate.cmake b/CMake/FindDeflate.cmake new file mode 100644 index 00000000..99aec8c8 --- /dev/null +++ b/CMake/FindDeflate.cmake @@ -0,0 +1,19 @@ +# Find package structure taken from libcurl + +include(FindPackageHandleStandardArgs) + +find_path(DEFLATE_INCLUDE_DIRS libdeflate.h) +find_library(DEFLATE_LIBRARY deflate) + +find_package_handle_standard_args(DEFLATE + FOUND_VAR + DEFLATE_FOUND + REQUIRED_VARS + DEFLATE_LIBRARY + DEFLATE_INCLUDE_DIRS + FAIL_MESSAGE + "Could NOT find deflate" +) + +set(DEFLATE_INCLUDE_DIRS ${DEFLATE_INCLUDE_DIRS}) +set(DEFLATE_LIBRARIES ${DEFLATE_LIBRARY}) diff --git a/CMakeLists.txt b/CMakeLists.txt index 762742f7..ce3b4a25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -202,6 +202,14 @@ if (USE_ZLIB) target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_ZLIB) endif() +# brew install libdeflate +find_package(DEFLATE) +if (DEFLATE_FOUND) + include_directories(${DEFLATE_INCLUDE_DIRS}) + target_link_libraries(ixwebsocket ${DEFLATE_LIBRARIES}) + target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_DEFLATE) +endif() + if (WIN32) target_link_libraries(ixwebsocket wsock32 ws2_32 shlwapi) add_definitions(-D_CRT_SECURE_NO_WARNINGS) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 83916329..f497787d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ All changes to this project will be documented in this file. +## [10.4.9] - 2020-09-30 + +(http server + utility code) Add support for doing gzip compression with libdeflate library, if available + ## [10.4.8] - 2020-09-30 (cmake) Stop using FetchContent cmake module to retrieve jsoncpp third party dependency diff --git a/ixwebsocket/IXBench.cpp b/ixwebsocket/IXBench.cpp index d29f54f4..75497f85 100644 --- a/ixwebsocket/IXBench.cpp +++ b/ixwebsocket/IXBench.cpp @@ -33,16 +33,29 @@ namespace ix void Bench::report() { auto now = std::chrono::high_resolution_clock::now(); - auto milliseconds = std::chrono::duration_cast(now - _start); + auto microseconds = std::chrono::duration_cast(now - _start); - _ms = milliseconds.count(); - std::cerr << _description << " completed in " << _ms << "ms" << std::endl; + _duration = microseconds.count(); + std::cerr << _description << " completed in " << _duration << " us" << std::endl; + setReported(); + } + + void Bench::record() + { + auto now = std::chrono::high_resolution_clock::now(); + auto microseconds = std::chrono::duration_cast(now - _start); + + _duration = microseconds.count(); + } + + void Bench::setReported() + { _reported = true; } uint64_t Bench::getDuration() const { - return _ms; + return _duration; } } // namespace ix diff --git a/ixwebsocket/IXBench.h b/ixwebsocket/IXBench.h index 3c46e192..c4f904b7 100644 --- a/ixwebsocket/IXBench.h +++ b/ixwebsocket/IXBench.h @@ -18,13 +18,15 @@ namespace ix ~Bench(); void reset(); + void record(); void report(); + void setReported(); uint64_t getDuration() const; private: std::string _description; std::chrono::time_point _start; - uint64_t _ms; + uint64_t _duration; bool _reported; }; } // namespace ix diff --git a/ixwebsocket/IXGzipCodec.cpp b/ixwebsocket/IXGzipCodec.cpp index b85fbada..cb7f9766 100644 --- a/ixwebsocket/IXGzipCodec.cpp +++ b/ixwebsocket/IXGzipCodec.cpp @@ -6,6 +6,7 @@ #include "IXGzipCodec.h" +#include "IXBench.h" #include #include @@ -13,11 +14,52 @@ #include #endif +#ifdef IXWEBSOCKET_USE_DEFLATE +#include +#endif + namespace ix { #ifdef IXWEBSOCKET_USE_ZLIB std::string gzipCompress(const std::string& str) { +#ifdef IXWEBSOCKET_USE_DEFLATE + int compressionLevel = 6; + struct libdeflate_compressor* compressor; + + compressor = libdeflate_alloc_compressor(compressionLevel); + + const void* uncompressed_data = str.data(); + size_t uncompressed_size = str.size(); + void* compressed_data; + size_t actual_compressed_size; + size_t max_compressed_size; + + max_compressed_size = libdeflate_gzip_compress_bound(compressor, uncompressed_size); + compressed_data = malloc(max_compressed_size); + + if (compressed_data == NULL) + { + return std::string(); + } + + actual_compressed_size = libdeflate_gzip_compress( + compressor, uncompressed_data, uncompressed_size, compressed_data, max_compressed_size); + + libdeflate_free_compressor(compressor); + + if (actual_compressed_size == 0) + { + free(compressed_data); + return std::string(); + } + + std::string out; + out.assign(reinterpret_cast(compressed_data), actual_compressed_size); + free(compressed_data); + + return out; +#else z_stream zs; // z_stream is zlib's control structure memset(&zs, 0, sizeof(zs)); @@ -57,6 +99,7 @@ namespace ix deflateEnd(&zs); return outstring; +#endif } bool gzipDecompress(const std::string& in, std::string& out) diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index 76d19df6..40f70906 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "10.4.8" +#define IX_WEBSOCKET_VERSION "10.4.9" diff --git a/ws/ws.cpp b/ws/ws.cpp index 1f9f59bd..76f9343a 100644 --- a/ws/ws.cpp +++ b/ws/ws.cpp @@ -1136,7 +1136,7 @@ namespace ix return 0; } - int ws_gzip(const std::string& filename) + int ws_gzip(const std::string& filename, int runCount) { auto res = readAsString(filename); bool found = res.first; @@ -1146,13 +1146,30 @@ namespace ix return 1; } - spdlog::info("gzip input: {} cksum {}", filename, ix::djb2HashStr(res.second)); + spdlog::info("gzip input: {} size {} cksum {}", + filename, + res.second.size(), + ix::djb2HashStr(res.second)); std::string compressedBytes; + spdlog::info("compressing {} times", runCount); + std::vector durations; { Bench bench("compressing file"); - compressedBytes = gzipCompress(res.second); + bench.setReported(); + + for (int i = 0; i < runCount; ++i) + { + bench.reset(); + compressedBytes = gzipCompress(res.second); + bench.record(); + durations.push_back(bench.getDuration()); + } + + size_t medianIdx = durations.size() / 2; + uint64_t medianRuntime = durations[medianIdx]; + spdlog::info("median runtime to compress file: {}", medianRuntime); } std::string outputFilename(filename); @@ -1163,7 +1180,10 @@ namespace ix f << compressedBytes; f.close(); - spdlog::info("gzip output: {} cksum {}", outputFilename, ix::djb2HashStr(compressedBytes)); + spdlog::info("gzip output: {} size {} cksum {}", + outputFilename, + compressedBytes.size(), + ix::djb2HashStr(compressedBytes)); return 0; } @@ -1180,7 +1200,10 @@ namespace ix return 1; } - spdlog::info("gunzip input: {} cksum {}", filename, ix::djb2HashStr(res.second)); + spdlog::info("gunzip input: {} size {} cksum {}", + filename, + res.second.size(), + ix::djb2HashStr(res.second)); std::string decompressedBytes; @@ -1200,8 +1223,10 @@ namespace ix f << decompressedBytes; f.close(); - spdlog::info( - "gunzip output: {} cksum {}", outputFilename, ix::djb2HashStr(decompressedBytes)); + spdlog::info("gunzip output: {} size {} cksum {}", + outputFilename, + decompressedBytes.size(), + ix::djb2HashStr(decompressedBytes)); return 0; } @@ -2969,6 +2994,7 @@ int main(int argc, char** argv) int msgCount = 1000 * 1000; uint32_t maxWaitBetweenReconnectionRetries; int pingIntervalSecs = 30; + int runCount = 1; auto addGenericOptions = [&pidfile](CLI::App* app) { app->add_option("--pidfile", pidfile, "Pid file"); @@ -3297,6 +3323,7 @@ int main(int argc, char** argv) CLI::App* gzipApp = app.add_subcommand("gzip", "Gzip compressor"); gzipApp->fallthrough(); gzipApp->add_option("filename", filename, "Filename")->required(); + gzipApp->add_option("--run_count", runCount, "Number of time to run the compression"); CLI::App* gunzipApp = app.add_subcommand("gunzip", "Gzip decompressor"); gunzipApp->fallthrough(); @@ -3600,7 +3627,7 @@ int main(int argc, char** argv) } else if (app.got_subcommand("gzip")) { - ret = ix::ws_gzip(filename); + ret = ix::ws_gzip(filename, runCount); } else if (app.got_subcommand("gunzip")) {