(ws) add gzip and gunzip ws sub commands

CMakeLists.txt

@@ -31,6 +31,7 @@ set( IXWEBSOCKET_SOURCES
     ixwebsocket/IXDNSLookup.cpp
     ixwebsocket/IXExponentialBackoff.cpp
     ixwebsocket/IXGetFreePort.cpp
+    ixwebsocket/IXGzipCodec.cpp
     ixwebsocket/IXHttp.cpp
     ixwebsocket/IXHttpClient.cpp
     ixwebsocket/IXHttpServer.cpp
@@ -66,6 +67,7 @@ set( IXWEBSOCKET_HEADERS
     ixwebsocket/IXDNSLookup.h
     ixwebsocket/IXExponentialBackoff.h
     ixwebsocket/IXGetFreePort.h
+    ixwebsocket/IXGzipCodec.h
     ixwebsocket/IXHttp.h
     ixwebsocket/IXHttpClient.h
     ixwebsocket/IXHttpServer.h

docs/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All changes to this project will be documented in this file.
 
+## [10.4.7] - 2020-09-28
+
+(ws) add gzip and gunzip ws sub commands
+
 ## [10.4.6] - 2020-09-26
 
 (cmake) use FetchContent cmake module to retrieve jsoncpp third party dependency

ixcrypto/ixcrypto/IXHash.cpp

@@ -19,4 +19,16 @@ namespace ix
 
         return hashAddress;
     }
+
+    uint64_t djb2HashStr(const std::string& data)
+    {
+        uint64_t hashAddress = 5381;
+
+        for (size_t i = 0; i < data.size(); ++i)
+        {
+            hashAddress = ((hashAddress << 5) + hashAddress) + data[i];
+        }
+
+        return hashAddress;
+    }
 } // namespace ix

ixcrypto/ixcrypto/IXHash.h

@@ -8,8 +8,10 @@
 
 #include <cstdint>
 #include <vector>
+#include <string>
 
 namespace ix
 {
     uint64_t djb2Hash(const std::vector<uint8_t>& data);
+    uint64_t djb2HashStr(const std::string& data);
 }

ixwebsocket/IXGzipCodec.cpp

@@ -0,0 +1,104 @@
+/*
+ *  IXGzipCodec.cpp
+ *  Author: Benjamin Sergeant
+ *  Copyright (c) 2020 Machine Zone, Inc. All rights reserved.
+ */
+
+#include "IXGzipCodec.h"
+
+#include <array>
+
+#ifdef IXWEBSOCKET_USE_ZLIB
+#include <zlib.h>
+#endif
+
+namespace ix
+{
+#ifdef IXWEBSOCKET_USE_ZLIB
+    std::string gzipCompress(const std::string& str)
+    {
+        z_stream zs; // z_stream is zlib's control structure
+        memset(&zs, 0, sizeof(zs));
+
+        // deflateInit2 configure the file format: request gzip instead of deflate
+        const int windowBits = 15;
+        const int GZIP_ENCODING = 16;
+
+        deflateInit2(&zs,
+                     Z_DEFAULT_COMPRESSION,
+                     Z_DEFLATED,
+                     windowBits | GZIP_ENCODING,
+                     8,
+                     Z_DEFAULT_STRATEGY);
+
+        zs.next_in = (Bytef*) str.data();
+        zs.avail_in = (uInt) str.size(); // set the z_stream's input
+
+        int ret;
+        char outbuffer[32768];
+        std::string outstring;
+
+        // retrieve the compressed bytes blockwise
+        do
+        {
+            zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
+            zs.avail_out = sizeof(outbuffer);
+
+            ret = deflate(&zs, Z_FINISH);
+
+            if (outstring.size() < zs.total_out)
+            {
+                // append the block to the output string
+                outstring.append(outbuffer, zs.total_out - outstring.size());
+            }
+        } while (ret == Z_OK);
+
+        deflateEnd(&zs);
+
+        return outstring;
+    }
+
+    bool gzipDecompress(const std::string& in, std::string& out)
+    {
+        z_stream inflateState;
+        std::memset(&inflateState, 0, sizeof(inflateState));
+
+        inflateState.zalloc = Z_NULL;
+        inflateState.zfree = Z_NULL;
+        inflateState.opaque = Z_NULL;
+        inflateState.avail_in = 0;
+        inflateState.next_in = Z_NULL;
+
+        if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK)
+        {
+            return false;
+        }
+
+        inflateState.avail_in = (uInt) in.size();
+        inflateState.next_in = (unsigned char*) (const_cast<char*>(in.data()));
+
+        const int kBufferSize = 1 << 14;
+        std::array<unsigned char, kBufferSize> compressBuffer;
+
+        do
+        {
+            inflateState.avail_out = (uInt) kBufferSize;
+            inflateState.next_out = &compressBuffer.front();
+
+            int ret = inflate(&inflateState, Z_SYNC_FLUSH);
+
+            if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
+            {
+                inflateEnd(&inflateState);
+                return false;
+            }
+
+            out.append(reinterpret_cast<char*>(&compressBuffer.front()),
+                       kBufferSize - inflateState.avail_out);
+        } while (inflateState.avail_out == 0);
+
+        inflateEnd(&inflateState);
+        return true;
+    }
+#endif
+} // namespace ix

ixwebsocket/IXGzipCodec.h

@@ -0,0 +1,15 @@
+/*
+ *  IXGzipCodec.h
+ *  Author: Benjamin Sergeant
+ *  Copyright (c) 2020 Machine Zone, Inc. All rights reserved.
+ */
+
+#pragma once
+
+#include <string>
+
+namespace ix
+{
+    std::string gzipCompress(const std::string& str);
+    bool gzipDecompress(const std::string& in, std::string& out);
+} // namespace ix

ixwebsocket/IXHttpClient.cpp

@@ -6,11 +6,11 @@
 
 #include "IXHttpClient.h"
 
+#include "IXGzipCodec.h"
 #include "IXSocketFactory.h"
 #include "IXUrlParser.h"
 #include "IXUserAgent.h"
 #include "IXWebSocketHttpHeaders.h"
-#include <array>
 #include <assert.h>
 #include <cstring>
 #include <iomanip>
@@ -18,10 +18,6 @@
 #include <sstream>
 #include <vector>
 
-#ifdef IXWEBSOCKET_USE_ZLIB
-#include <zlib.h>
-#endif
-
 namespace ix
 {
     const std::string HttpClient::kPost = "POST";
@@ -501,12 +497,12 @@ namespace ix
 
         downloadSize = payload.size();
 
-#ifdef IXWEBSOCKET_USE_ZLIB
         // If the content was compressed with gzip, decode it
         if (headers["Content-Encoding"] == "gzip")
         {
+#ifdef IXWEBSOCKET_USE_ZLIB
             std::string decompressedPayload;
-            if (!gzipInflate(payload, decompressedPayload))
+            if (!gzipDecompress(payload, decompressedPayload))
             {
                 std::string errorMsg("Error decompressing payload");
                 return std::make_shared<HttpResponse>(code,
@@ -519,8 +515,18 @@ namespace ix
                                                       downloadSize);
             }
             payload = decompressedPayload;
-        }
+#else
+            std::string errorMsg("ixwebsocket was not compiled with gzip support on");
+            return std::make_shared<HttpResponse>(code,
+                                                  description,
+                                                  HttpErrorCode::Gzip,
+                                                  headers,
+                                                  payload,
+                                                  errorMsg,
+                                                  uploadSize,
+                                                  downloadSize);
 #endif
+        }
 
         return std::make_shared<HttpResponse>(code,
                                               description,
@@ -680,51 +686,6 @@ namespace ix
         return ss.str();
     }
 
-#ifdef IXWEBSOCKET_USE_ZLIB
-    bool HttpClient::gzipInflate(const std::string& in, std::string& out)
-    {
-        z_stream inflateState;
-        std::memset(&inflateState, 0, sizeof(inflateState));
-
-        inflateState.zalloc = Z_NULL;
-        inflateState.zfree = Z_NULL;
-        inflateState.opaque = Z_NULL;
-        inflateState.avail_in = 0;
-        inflateState.next_in = Z_NULL;
-
-        if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK)
-        {
-            return false;
-        }
-
-        inflateState.avail_in = (uInt) in.size();
-        inflateState.next_in = (unsigned char*) (const_cast<char*>(in.data()));
-
-        const int kBufferSize = 1 << 14;
-        std::array<unsigned char, kBufferSize> compressBuffer;
-
-        do
-        {
-            inflateState.avail_out = (uInt) kBufferSize;
-            inflateState.next_out = &compressBuffer.front();
-
-            int ret = inflate(&inflateState, Z_SYNC_FLUSH);
-
-            if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
-            {
-                inflateEnd(&inflateState);
-                return false;
-            }
-
-            out.append(reinterpret_cast<char*>(&compressBuffer.front()),
-                       kBufferSize - inflateState.avail_out);
-        } while (inflateState.avail_out == 0);
-
-        inflateEnd(&inflateState);
-        return true;
-    }
-#endif
-
     void HttpClient::log(const std::string& msg, HttpRequestArgsPtr args)
     {
         if (args->logger)

ixwebsocket/IXHttpClient.h

@@ -90,10 +90,6 @@ namespace ix
     private:
         void log(const std::string& msg, HttpRequestArgsPtr args);
 
-#ifdef IXWEBSOCKET_USE_ZLIB
-        bool gzipInflate(const std::string& in, std::string& out);
-#endif
-
         // Async API background thread runner
         void run();
         // Async API

ixwebsocket/IXHttpServer.cpp

@@ -6,6 +6,7 @@
 
 #include "IXHttpServer.h"
 
+#include "IXGzipCodec.h"
 #include "IXNetSystem.h"
 #include "IXSocketConnect.h"
 #include "IXUserAgent.h"
@@ -14,10 +15,6 @@
 #include <sstream>
 #include <vector>
 
-#ifdef IXWEBSOCKET_USE_ZLIB
-#include <zlib.h>
-#endif
-
 namespace
 {
     std::pair<bool, std::vector<uint8_t>> load(const std::string& path)
@@ -43,51 +40,6 @@ namespace
         auto vec = res.second;
         return std::make_pair(res.first, std::string(vec.begin(), vec.end()));
     }
-
-#ifdef IXWEBSOCKET_USE_ZLIB
-    std::string gzipCompress(const std::string& str)
-    {
-        z_stream zs; // z_stream is zlib's control structure
-        memset(&zs, 0, sizeof(zs));
-
-        // deflateInit2 configure the file format: request gzip instead of deflate
-        const int windowBits = 15;
-        const int GZIP_ENCODING = 16;
-
-        deflateInit2(&zs,
-                     Z_DEFAULT_COMPRESSION,
-                     Z_DEFLATED,
-                     windowBits | GZIP_ENCODING,
-                     8,
-                     Z_DEFAULT_STRATEGY);
-
-        zs.next_in = (Bytef*) str.data();
-        zs.avail_in = (uInt) str.size(); // set the z_stream's input
-
-        int ret;
-        char outbuffer[32768];
-        std::string outstring;
-
-        // retrieve the compressed bytes blockwise
-        do
-        {
-            zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
-            zs.avail_out = sizeof(outbuffer);
-
-            ret = deflate(&zs, Z_FINISH);
-
-            if (outstring.size() < zs.total_out)
-            {
-                // append the block to the output string
-                outstring.append(outbuffer, zs.total_out - outstring.size());
-            }
-        } while (ret == Z_OK);
-
-        deflateEnd(&zs);
-
-        return outstring;
-    }
-#endif
 } // namespace
 
 namespace ix

ixwebsocket/IXWebSocketVersion.h

@@ -6,4 +6,4 @@
 
 #pragma once
 
-#define IX_WEBSOCKET_VERSION "10.4.6"
+#define IX_WEBSOCKET_VERSION "10.4.7"

ws/ws.cpp

@@ -33,6 +33,7 @@
 #include <ixsentry/IXSentryClient.h>
 #include <ixsnake/IXSnakeServer.h>
 #include <ixwebsocket/IXDNSLookup.h>
+#include <ixwebsocket/IXGzipCodec.h>
 #include <ixwebsocket/IXHttpClient.h>
 #include <ixwebsocket/IXHttpServer.h>
 #include <ixwebsocket/IXNetSystem.h>
@@ -127,6 +128,38 @@ namespace
         std::ifstream infile(fileName);
         return infile.good();
     }
+
+    std::string extractFilename(const std::string& path)
+    {
+        std::string::size_type idx;
+
+        idx = path.rfind('/');
+        if (idx != std::string::npos)
+        {
+            std::string filename = path.substr(idx + 1);
+            return filename;
+        }
+        else
+        {
+            return path;
+        }
+    }
+
+    std::string removeExtension(const std::string& path)
+    {
+        std::string::size_type idx;
+
+        idx = path.rfind('.');
+        if (idx != std::string::npos)
+        {
+            std::string filename = path.substr(idx + 1);
+            return filename;
+        }
+        else
+        {
+            return path;
+        }
+    }
 } // namespace
 
 namespace ix
@@ -1103,6 +1136,76 @@ namespace ix
         return 0;
     }
 
+    int ws_gzip(const std::string& filename)
+    {
+        auto res = readAsString(filename);
+        bool found = res.first;
+        if (!found)
+        {
+            spdlog::error("Cannot read content of {}", filename);
+            return 1;
+        }
+
+        spdlog::info("gzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
+
+        std::string compressedBytes;
+
+        {
+            Bench bench("compressing file");
+            compressedBytes = gzipCompress(res.second);
+        }
+
+        std::string outputFilename(filename);
+        outputFilename += ".gz";
+
+        std::ofstream f;
+        f.open(outputFilename);
+        f << compressedBytes;
+        f.close();
+
+        spdlog::info("gzip output: {} cksum {}", outputFilename, ix::djb2HashStr(compressedBytes));
+
+        return 0;
+    }
+
+    int ws_gunzip(const std::string& filename)
+    {
+        spdlog::info("filename to gunzip: {}", filename);
+
+        auto res = readAsString(filename);
+        bool found = res.first;
+        if (!found)
+        {
+            spdlog::error("Cannot read content of {}", filename);
+            return 1;
+        }
+
+        spdlog::info("gunzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
+
+        std::string decompressedBytes;
+
+        {
+            Bench bench("decompressing file");
+            if (!gzipDecompress(res.second, decompressedBytes))
+            {
+                spdlog::error("Cannot decompress content of {}", filename);
+                return 1;
+            }
+        }
+
+        std::string outputFilename(removeExtension(filename));
+
+        std::ofstream f;
+        f.open(outputFilename);
+        f << decompressedBytes;
+        f.close();
+
+        spdlog::info(
+            "gunzip output: {} cksum {}", outputFilename, ix::djb2HashStr(decompressedBytes));
+
+        return 0;
+    }
+
     int ws_autoroute(const std::string& url,
                      bool disablePerMessageDeflate,
                      const ix::SocketTLSOptions& tlsOptions,
@@ -1305,22 +1408,6 @@ namespace ix
         return 0;
     }
 
-    std::string extractFilename(const std::string& path)
-    {
-        std::string::size_type idx;
-
-        idx = path.rfind('/');
-        if (idx != std::string::npos)
-        {
-            std::string filename = path.substr(idx + 1);
-            return filename;
-        }
-        else
-        {
-            return path;
-        }
-    }
-
     WebSocketHttpHeaders parseHeaders(const std::string& data)
     {
         WebSocketHttpHeaders headers;
@@ -2848,6 +2935,7 @@ int main(int argc, char** argv)
     std::string publisherRolename;
     std::string publisherRolesecret;
     std::string sendMsg("hello world");
+    std::string filename;
     ix::SocketTLSOptions tlsOptions;
     ix::CobraConfig cobraConfig;
     ix::CobraBotConfig cobraBotConfig;
@@ -3206,6 +3294,14 @@ int main(int argc, char** argv)
     dnsLookupApp->fallthrough();
     dnsLookupApp->add_option("host", hostname, "Hostname")->required();
 
+    CLI::App* gzipApp = app.add_subcommand("gzip", "Gzip compressor");
+    gzipApp->fallthrough();
+    gzipApp->add_option("filename", filename, "Filename")->required();
+
+    CLI::App* gunzipApp = app.add_subcommand("gunzip", "Gzip decompressor");
+    gunzipApp->fallthrough();
+    gunzipApp->add_option("filename", filename, "Filename")->required();
+
     CLI11_PARSE(app, argc, argv);
 
     // pid file handling
@@ -3502,6 +3598,14 @@ int main(int argc, char** argv)
     {
         ret = ix::ws_dns_lookup(hostname);
     }
+    else if (app.got_subcommand("gzip"))
+    {
+        ret = ix::ws_gzip(filename);
+    }
+    else if (app.got_subcommand("gunzip"))
+    {
+        ret = ix::ws_gunzip(filename);
+    }
     else if (version)
     {
         std::cout << "ws " << ix::userAgent() << std::endl;