Add support for gzip compression through libdeflate

CMake/FindDeflate.cmake

@@ -0,0 +1,19 @@
+# Find package structure taken from libcurl
+
+include(FindPackageHandleStandardArgs)
+
+find_path(DEFLATE_INCLUDE_DIRS libdeflate.h)
+find_library(DEFLATE_LIBRARY deflate)
+
+find_package_handle_standard_args(DEFLATE
+    FOUND_VAR
+      DEFLATE_FOUND
+    REQUIRED_VARS
+      DEFLATE_LIBRARY
+      DEFLATE_INCLUDE_DIRS
+    FAIL_MESSAGE
+      "Could NOT find deflate"
+)
+
+set(DEFLATE_INCLUDE_DIRS ${DEFLATE_INCLUDE_DIRS})
+set(DEFLATE_LIBRARIES ${DEFLATE_LIBRARY})

CMakeLists.txt

@@ -202,6 +202,14 @@ if (USE_ZLIB)
   target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_ZLIB)
 endif()
 
+# brew install libdeflate
+find_package(DEFLATE)
+if (DEFLATE_FOUND)
+  include_directories(${DEFLATE_INCLUDE_DIRS})
+  target_link_libraries(ixwebsocket ${DEFLATE_LIBRARIES})
+  target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_DEFLATE)
+endif()
+
 if (WIN32)
   target_link_libraries(ixwebsocket wsock32 ws2_32 shlwapi)
   add_definitions(-D_CRT_SECURE_NO_WARNINGS)

docs/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All changes to this project will be documented in this file.
 
+## [10.4.9] - 2020-09-30
+
+(http server + utility code) Add support for doing gzip compression with libdeflate library, if available
+
 ## [10.4.8] - 2020-09-30
 
 (cmake) Stop using FetchContent cmake module to retrieve jsoncpp third party dependency

ixwebsocket/IXBench.cpp

@@ -33,16 +33,29 @@ namespace ix
     void Bench::report()
     {
         auto now = std::chrono::high_resolution_clock::now();
-        auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(now - _start);
+        auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(now - _start);
 
-        _ms = milliseconds.count();
+        _duration = microseconds.count();
-        std::cerr << _description << " completed in " << _ms << "ms" << std::endl;
+        std::cerr << _description << " completed in " << _duration << " us" << std::endl;
 
+        setReported();
+    }
+
+    void Bench::record()
+    {
+        auto now = std::chrono::high_resolution_clock::now();
+        auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(now - _start);
+
+        _duration = microseconds.count();
+    }
+
+    void Bench::setReported()
+    {
         _reported = true;
     }
 
     uint64_t Bench::getDuration() const
     {
-        return _ms;
+        return _duration;
     }
 } // namespace ix

ixwebsocket/IXBench.h

@@ -18,13 +18,15 @@ namespace ix
         ~Bench();
 
         void reset();
+        void record();
         void report();
+        void setReported();
         uint64_t getDuration() const;
 
     private:
         std::string _description;
         std::chrono::time_point<std::chrono::high_resolution_clock> _start;
-        uint64_t _ms;
+        uint64_t _duration;
         bool _reported;
     };
 } // namespace ix

ixwebsocket/IXGzipCodec.cpp

@@ -6,6 +6,7 @@
 
 #include "IXGzipCodec.h"
 
+#include "IXBench.h"
 #include <array>
 #include <string.h>
 
@@ -13,11 +14,52 @@
 #include <zlib.h>
 #endif
 
+#ifdef IXWEBSOCKET_USE_DEFLATE
+#include <libdeflate.h>
+#endif
+
 namespace ix
 {
 #ifdef IXWEBSOCKET_USE_ZLIB
     std::string gzipCompress(const std::string& str)
     {
+#ifdef IXWEBSOCKET_USE_DEFLATE
+        int compressionLevel = 6;
+        struct libdeflate_compressor* compressor;
+
+        compressor = libdeflate_alloc_compressor(compressionLevel);
+
+        const void* uncompressed_data = str.data();
+        size_t uncompressed_size = str.size();
+        void* compressed_data;
+        size_t actual_compressed_size;
+        size_t max_compressed_size;
+
+        max_compressed_size = libdeflate_gzip_compress_bound(compressor, uncompressed_size);
+        compressed_data = malloc(max_compressed_size);
+
+        if (compressed_data == NULL)
+        {
+            return std::string();
+        }
+
+        actual_compressed_size = libdeflate_gzip_compress(
+            compressor, uncompressed_data, uncompressed_size, compressed_data, max_compressed_size);
+
+        libdeflate_free_compressor(compressor);
+
+        if (actual_compressed_size == 0)
+        {
+            free(compressed_data);
+            return std::string();
+        }
+
+        std::string out;
+        out.assign(reinterpret_cast<char*>(compressed_data), actual_compressed_size);
+        free(compressed_data);
+
+        return out;
+#else
         z_stream zs; // z_stream is zlib's control structure
         memset(&zs, 0, sizeof(zs));
 
@@ -57,6 +99,7 @@ namespace ix
         deflateEnd(&zs);
 
         return outstring;
+#endif
     }
 
     bool gzipDecompress(const std::string& in, std::string& out)

ixwebsocket/IXWebSocketVersion.h

@@ -6,4 +6,4 @@
 
 #pragma once
 
-#define IX_WEBSOCKET_VERSION "10.4.8"
+#define IX_WEBSOCKET_VERSION "10.4.9"

ws/ws.cpp

@@ -1136,7 +1136,7 @@ namespace ix
         return 0;
     }
 
-    int ws_gzip(const std::string& filename)
+    int ws_gzip(const std::string& filename, int runCount)
     {
         auto res = readAsString(filename);
         bool found = res.first;
@@ -1146,13 +1146,30 @@ namespace ix
             return 1;
         }
 
-        spdlog::info("gzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
+        spdlog::info("gzip input: {} size {} cksum {}",
+                     filename,
+                     res.second.size(),
+                     ix::djb2HashStr(res.second));
 
         std::string compressedBytes;
 
+        spdlog::info("compressing {} times", runCount);
+        std::vector<uint64_t> durations;
         {
             Bench bench("compressing file");
-            compressedBytes = gzipCompress(res.second);
+            bench.setReported();
+
+            for (int i = 0; i < runCount; ++i)
+            {
+                bench.reset();
+                compressedBytes = gzipCompress(res.second);
+                bench.record();
+                durations.push_back(bench.getDuration());
+            }
+
+            size_t medianIdx = durations.size() / 2;
+            uint64_t medianRuntime = durations[medianIdx];
+            spdlog::info("median runtime to compress file: {}", medianRuntime);
         }
 
         std::string outputFilename(filename);
@@ -1163,7 +1180,10 @@ namespace ix
         f << compressedBytes;
         f.close();
 
-        spdlog::info("gzip output: {} cksum {}", outputFilename, ix::djb2HashStr(compressedBytes));
+        spdlog::info("gzip output: {} size {} cksum {}",
+                     outputFilename,
+                     compressedBytes.size(),
+                     ix::djb2HashStr(compressedBytes));
 
         return 0;
     }
@@ -1180,7 +1200,10 @@ namespace ix
             return 1;
         }
 
-        spdlog::info("gunzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
+        spdlog::info("gunzip input: {} size {} cksum {}",
+                     filename,
+                     res.second.size(),
+                     ix::djb2HashStr(res.second));
 
         std::string decompressedBytes;
 
@@ -1200,8 +1223,10 @@ namespace ix
         f << decompressedBytes;
         f.close();
 
-        spdlog::info(
+        spdlog::info("gunzip output: {} size {} cksum {}",
-            "gunzip output: {} cksum {}", outputFilename, ix::djb2HashStr(decompressedBytes));
+                     outputFilename,
+                     decompressedBytes.size(),
+                     ix::djb2HashStr(decompressedBytes));
 
         return 0;
     }
@@ -2969,6 +2994,7 @@ int main(int argc, char** argv)
     int msgCount = 1000 * 1000;
     uint32_t maxWaitBetweenReconnectionRetries;
     int pingIntervalSecs = 30;
+    int runCount = 1;
 
     auto addGenericOptions = [&pidfile](CLI::App* app) {
         app->add_option("--pidfile", pidfile, "Pid file");
@@ -3297,6 +3323,7 @@ int main(int argc, char** argv)
     CLI::App* gzipApp = app.add_subcommand("gzip", "Gzip compressor");
     gzipApp->fallthrough();
     gzipApp->add_option("filename", filename, "Filename")->required();
+    gzipApp->add_option("--run_count", runCount, "Number of time to run the compression");
 
     CLI::App* gunzipApp = app.add_subcommand("gunzip", "Gzip decompressor");
     gunzipApp->fallthrough();
@@ -3600,7 +3627,7 @@ int main(int argc, char** argv)
     }
     else if (app.got_subcommand("gzip"))
     {
-        ret = ix::ws_gzip(filename);
+        ret = ix::ws_gzip(filename, runCount);
     }
     else if (app.got_subcommand("gunzip"))
     {