(http code) With zlib disabled, some code should not be reached

* (http client) #ifdefs so we dont try to compress http requests with zlib

* (http client) Remove some #ifdefs for including zlib and removing fields

CMake/FindDeflate.cmake

@@ -0,0 +1,19 @@
+# Find package structure taken from libcurl
+
+include(FindPackageHandleStandardArgs)
+
+find_path(DEFLATE_INCLUDE_DIRS libdeflate.h)
+find_library(DEFLATE_LIBRARY deflate)
+
+find_package_handle_standard_args(DEFLATE
+    FOUND_VAR
+      DEFLATE_FOUND
+    REQUIRED_VARS
+      DEFLATE_LIBRARY
+      DEFLATE_INCLUDE_DIRS
+    FAIL_MESSAGE
+      "Could NOT find deflate"
+)
+
+set(DEFLATE_INCLUDE_DIRS ${DEFLATE_INCLUDE_DIRS})
+set(DEFLATE_LIBRARIES ${DEFLATE_LIBRARY})

CMakeLists.txt

@@ -202,6 +202,14 @@ if (USE_ZLIB)
   target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_ZLIB)
 endif()
 
+# brew install libdeflate
+find_package(DEFLATE)
+if (DEFLATE_FOUND)
+  include_directories(${DEFLATE_INCLUDE_DIRS})
+  target_link_libraries(ixwebsocket ${DEFLATE_LIBRARIES})
+  target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_DEFLATE)
+endif()
+
 if (WIN32)
   target_link_libraries(ixwebsocket wsock32 ws2_32 shlwapi)
   add_definitions(-D_CRT_SECURE_NO_WARNINGS)
@@ -258,11 +266,7 @@ if (USE_WS OR USE_TEST)
       GIT_REPOSITORY "https://github.com/gabime/spdlog"
       GIT_TAG "v1.8.0") 
 
-  FetchContent_Declare(jsoncpp
-      GIT_REPOSITORY "https://github.com/open-source-parsers/jsoncpp"
-      GIT_TAG "1.9.4") 
-
-  FetchContent_MakeAvailable(spdlog jsoncpp)
+  FetchContent_MakeAvailable(spdlog)
 
   if (USE_WS)
       add_subdirectory(ws)

docs/CHANGELOG.md

@@ -2,6 +2,30 @@
 
 All changes to this project will be documented in this file.
 
+## [10.5.3] - 2020-10-19
+
+(http code) With zlib disabled, some code should not be reached
+
+## [10.5.2] - 2020-10-12
+
+(ws curl) Add support for --data-binary option, to set the request body. When present the request will be sent with the POST verb
+
+## [10.5.1] - 2020-10-09
+
+(http client + server + ws) Add support for compressing http client requests with gzip. --compress_request argument is used in ws to enable this. The Content-Encoding is set to gzip, and decoded on the server side if present.
+
+## [10.5.0] - 2020-09-30
+
+(http client + server + ws) Add support for uploading files with ws -F foo=@filename, new -D http server option to debug incoming client requests, internal api changed for http POST, PUT and PATCH to supply an HttpFormDataParameters
+
+## [10.4.9] - 2020-09-30
+
+(http server + utility code) Add support for doing gzip compression with libdeflate library, if available
+
+## [10.4.8] - 2020-09-30
+
+(cmake) Stop using FetchContent cmake module to retrieve jsoncpp third party dependency
+
 ## [10.4.7] - 2020-09-28
 
 (ws) add gzip and gunzip ws sub commands

docs/usage.md

@@ -458,11 +458,18 @@ out = httpClient.get(url, args);
 // POST request with parameters
 HttpParameters httpParameters;
 httpParameters["foo"] = "bar";
-out = httpClient.post(url, httpParameters, args);
+
+// HTTP form data can be passed in as well, for multi-part upload of files
+HttpFormDataParameters httpFormDataParameters;
+httpParameters["baz"] = "booz";
+
+out = httpClient.post(url, httpParameters, httpFormDataParameters, args);
 
 // POST request with a body
 out = httpClient.post(url, std::string("foo=bar"), args);
 
+// PUT and PATCH are available too.
+
 //
 // Result
 //

ixbots/CMakeLists.txt

@@ -31,6 +31,11 @@ add_library(ixbots STATIC
     ${IXBOTS_HEADERS}
 )
 
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp)
+endif()
+
 if (USE_PYTHON)
   target_compile_definitions(ixbots PUBLIC IXBOTS_USE_PYTHON)
   find_package(Python COMPONENTS Development)
@@ -43,11 +48,12 @@ set(IXBOTS_INCLUDE_DIRS
     ../ixwebsocket
     ../ixcobra
     ../ixredis
-    ../ixsentry)
+    ../ixsentry
+    ${JSONCPP_INCLUDE_DIRS}
+    ${SPDLOG_INCLUDE_DIRS})
 
 if (USE_PYTHON)
   set(IXBOTS_INCLUDE_DIRS ${IXBOTS_INCLUDE_DIRS} ${Python_INCLUDE_DIRS})
 endif()
 
 target_include_directories( ixbots PUBLIC ${IXBOTS_INCLUDE_DIRS} )
-target_link_libraries( ixbots jsoncpp_static )

ixcobra/CMakeLists.txt

@@ -22,11 +22,16 @@ add_library(ixcobra STATIC
     ${IXCOBRA_HEADERS}
 )
 
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp)
+endif()
+
 set(IXCOBRA_INCLUDE_DIRS
     .
     ..
     ../ixcore
-    ../ixcrypto)
+    ../ixcrypto
+    ${JSONCPP_INCLUDE_DIRS})
 
 target_include_directories( ixcobra PUBLIC ${IXCOBRA_INCLUDE_DIRS} )
-target_link_libraries( ixcobra jsoncpp_static )

ixsentry/CMakeLists.txt

@@ -16,10 +16,15 @@ add_library(ixsentry STATIC
   ${IXSENTRY_HEADERS}
 )
 
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp)
+endif()
+
 set(IXSENTRY_INCLUDE_DIRS
     .
     ..
-    ../ixcore)
+    ../ixcore
+    ${JSONCPP_INCLUDE_DIRS})
 
 target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} )
-target_link_libraries( ixsentry jsoncpp_static )

ixwebsocket/IXBench.cpp

@@ -33,16 +33,29 @@ namespace ix
     void Bench::report()
     {
         auto now = std::chrono::high_resolution_clock::now();
-        auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(now - _start);
+        auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(now - _start);
 
-        _ms = milliseconds.count();
-        std::cerr << _description << " completed in " << _ms << "ms" << std::endl;
+        _duration = microseconds.count();
+        std::cerr << _description << " completed in " << _duration << " us" << std::endl;
 
+        setReported();
+    }
+
+    void Bench::record()
+    {
+        auto now = std::chrono::high_resolution_clock::now();
+        auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(now - _start);
+
+        _duration = microseconds.count();
+    }
+
+    void Bench::setReported()
+    {
         _reported = true;
     }
 
     uint64_t Bench::getDuration() const
     {
-        return _ms;
+        return _duration;
     }
 } // namespace ix

ixwebsocket/IXBench.h

@@ -18,13 +18,15 @@ namespace ix
         ~Bench();
 
         void reset();
+        void record();
         void report();
+        void setReported();
         uint64_t getDuration() const;
 
     private:
         std::string _description;
         std::chrono::time_point<std::chrono::high_resolution_clock> _start;
-        uint64_t _ms;
+        uint64_t _duration;
         bool _reported;
     };
 } // namespace ix

ixwebsocket/IXExponentialBackoff.cpp

@@ -1,5 +1,5 @@
 /*
- *  IXExponentialBackoff.h
+ *  IXExponentialBackoff.cpp
  *  Author: Benjamin Sergeant
  *  Copyright (c) 2017-2019 Machine Zone, Inc. All rights reserved.
  */

ixwebsocket/IXGzipCodec.cpp

@@ -6,6 +6,7 @@
 
 #include "IXGzipCodec.h"
 
+#include "IXBench.h"
 #include <array>
 #include <string.h>
 
@@ -13,11 +14,52 @@
 #include <zlib.h>
 #endif
 
+#ifdef IXWEBSOCKET_USE_DEFLATE
+#include <libdeflate.h>
+#endif
+
 namespace ix
 {
 #ifdef IXWEBSOCKET_USE_ZLIB
     std::string gzipCompress(const std::string& str)
     {
+#ifdef IXWEBSOCKET_USE_DEFLATE
+        int compressionLevel = 6;
+        struct libdeflate_compressor* compressor;
+
+        compressor = libdeflate_alloc_compressor(compressionLevel);
+
+        const void* uncompressed_data = str.data();
+        size_t uncompressed_size = str.size();
+        void* compressed_data;
+        size_t actual_compressed_size;
+        size_t max_compressed_size;
+
+        max_compressed_size = libdeflate_gzip_compress_bound(compressor, uncompressed_size);
+        compressed_data = malloc(max_compressed_size);
+
+        if (compressed_data == NULL)
+        {
+            return std::string();
+        }
+
+        actual_compressed_size = libdeflate_gzip_compress(
+            compressor, uncompressed_data, uncompressed_size, compressed_data, max_compressed_size);
+
+        libdeflate_free_compressor(compressor);
+
+        if (actual_compressed_size == 0)
+        {
+            free(compressed_data);
+            return std::string();
+        }
+
+        std::string out;
+        out.assign(reinterpret_cast<char*>(compressed_data), actual_compressed_size);
+        free(compressed_data);
+
+        return out;
+#else
         z_stream zs; // z_stream is zlib's control structure
         memset(&zs, 0, sizeof(zs));
 
@@ -57,6 +99,7 @@ namespace ix
         deflateEnd(&zs);
 
         return outstring;
+#endif
     }
 
     bool gzipDecompress(const std::string& in, std::string& out)

ixwebsocket/IXHttp.cpp

@@ -7,6 +7,7 @@
 #include "IXHttp.h"
 
 #include "IXCancellationRequest.h"
+#include "IXGzipCodec.h"
 #include "IXSocket.h"
 #include <sstream>
 #include <vector>
@@ -157,6 +158,23 @@ namespace ix
             body = res.second;
         }
 
+        // If the content was compressed with gzip, decode it
+        if (headers["Content-Encoding"] == "gzip")
+        {
+#ifdef IXWEBSOCKET_USE_ZLIB
+            std::string decompressedPayload;
+            if (!gzipDecompress(body, decompressedPayload))
+            {
+                return std::make_tuple(
+                    false, std::string("Error during gzip decompression of the body"), httpRequest);
+            }
+            body = decompressedPayload;
+#else
+            std::string errorMsg("ixwebsocket was not compiled with gzip support on");
+            return std::make_tuple(false, errorMsg, httpRequest);
+#endif
+        }
+
         httpRequest = std::make_shared<HttpRequest>(uri, method, httpVersion, body, headers);
         return std::make_tuple(true, "", httpRequest);
     }

ixwebsocket/IXHttp.h

@@ -84,6 +84,7 @@ namespace ix
         int maxRedirects = 5;
         bool verbose = false;
         bool compress = true;
+        bool compressRequest = false;
         Logger logger;
         OnProgressCallback onProgressCallback;
     };

ixwebsocket/IXHttpClient.cpp

@@ -203,6 +203,15 @@ namespace ix
 
         if (verb == kPost || verb == kPut || verb == kPatch || _forceBody)
         {
+            // Set request compression header
+#ifdef IXWEBSOCKET_USE_ZLIB
+            if (args->compressRequest)
+            {
+                ss << "Content-Encoding: gzip"
+                   << "\r\n";
+            }
+#endif
+
             ss << "Content-Length: " << body.size() << "\r\n";
 
             // Set default Content-Type if unspecified
@@ -553,11 +562,42 @@ namespace ix
         return request(url, kDel, std::string(), args);
     }
 
+    HttpResponsePtr HttpClient::request(const std::string& url,
+                                        const std::string& verb,
+                                        const HttpParameters& httpParameters,
+                                        const HttpFormDataParameters& httpFormDataParameters,
+                                        HttpRequestArgsPtr args)
+    {
+        std::string body;
+
+        if (httpFormDataParameters.empty())
+        {
+            body = serializeHttpParameters(httpParameters);
+        }
+        else
+        {
+            std::string multipartBoundary = generateMultipartBoundary();
+            args->multipartBoundary = multipartBoundary;
+            body = serializeHttpFormDataParameters(
+                multipartBoundary, httpFormDataParameters, httpParameters);
+        }
+
+#ifdef IXWEBSOCKET_USE_ZLIB
+        if (args->compressRequest)
+        {
+            body = gzipCompress(body);
+        }
+#endif
+
+        return request(url, verb, body, args);
+    }
+
     HttpResponsePtr HttpClient::post(const std::string& url,
                                      const HttpParameters& httpParameters,
+                                     const HttpFormDataParameters& httpFormDataParameters,
                                      HttpRequestArgsPtr args)
     {
-        return request(url, kPost, serializeHttpParameters(httpParameters), args);
+        return request(url, kPost, httpParameters, httpFormDataParameters, args);
     }
 
     HttpResponsePtr HttpClient::post(const std::string& url,
@@ -569,9 +609,10 @@ namespace ix
 
     HttpResponsePtr HttpClient::put(const std::string& url,
                                     const HttpParameters& httpParameters,
+                                    const HttpFormDataParameters& httpFormDataParameters,
                                     HttpRequestArgsPtr args)
     {
-        return request(url, kPut, serializeHttpParameters(httpParameters), args);
+        return request(url, kPut, httpParameters, httpFormDataParameters, args);
     }
 
     HttpResponsePtr HttpClient::put(const std::string& url,
@@ -583,9 +624,10 @@ namespace ix
 
     HttpResponsePtr HttpClient::patch(const std::string& url,
                                       const HttpParameters& httpParameters,
+                                      const HttpFormDataParameters& httpFormDataParameters,
                                       HttpRequestArgsPtr args)
     {
-        return request(url, kPatch, serializeHttpParameters(httpParameters), args);
+        return request(url, kPatch, httpParameters, httpFormDataParameters, args);
     }
 
     HttpResponsePtr HttpClient::patch(const std::string& url,

ixwebsocket/IXHttpClient.h

@@ -34,6 +34,7 @@ namespace ix
 
         HttpResponsePtr post(const std::string& url,
                              const HttpParameters& httpParameters,
+                             const HttpFormDataParameters& httpFormDataParameters,
                              HttpRequestArgsPtr args);
         HttpResponsePtr post(const std::string& url,
                              const std::string& body,
@@ -41,6 +42,7 @@ namespace ix
 
         HttpResponsePtr put(const std::string& url,
                             const HttpParameters& httpParameters,
+                            const HttpFormDataParameters& httpFormDataParameters,
                             HttpRequestArgsPtr args);
         HttpResponsePtr put(const std::string& url,
                             const std::string& body,
@@ -48,6 +50,7 @@ namespace ix
 
         HttpResponsePtr patch(const std::string& url,
                               const HttpParameters& httpParameters,
+                              const HttpFormDataParameters& httpFormDataParameters,
                               HttpRequestArgsPtr args);
         HttpResponsePtr patch(const std::string& url,
                               const std::string& body,
@@ -58,7 +61,15 @@ namespace ix
                                 const std::string& body,
                                 HttpRequestArgsPtr args,
                                 int redirects = 0);
+
+        HttpResponsePtr request(const std::string& url,
+                                const std::string& verb,
+                                const HttpParameters& httpParameters,
+                                const HttpFormDataParameters& httpFormDataParameters,
+                                HttpRequestArgsPtr args);
+
         void setForceBody(bool value);
+
         // Async API
         HttpRequestArgsPtr createRequest(const std::string& url = std::string(),
                                          const std::string& verb = HttpClient::kGet);

ixwebsocket/IXHttpServer.cpp

@@ -190,4 +190,40 @@ namespace ix
                     301, "OK", HttpErrorCode::Ok, headers, std::string());
             });
     }
+
+    //
+    // Display the client parameter and body on the console
+    //
+    void HttpServer::makeDebugServer()
+    {
+        setOnConnectionCallback(
+            [this](HttpRequestPtr request,
+                   std::shared_ptr<ConnectionState> connectionState) -> HttpResponsePtr {
+                WebSocketHttpHeaders headers;
+                headers["Server"] = userAgent();
+
+                // Log request
+                std::stringstream ss;
+                ss << connectionState->getRemoteIp() << ":" << connectionState->getRemotePort()
+                   << " " << request->method << " " << request->headers["User-Agent"] << " "
+                   << request->uri;
+                logInfo(ss.str());
+
+                logInfo("== Headers == ");
+                for (auto&& it : request->headers)
+                {
+                    std::ostringstream oss;
+                    oss << it.first << ": " << it.second;
+                    logInfo(oss.str());
+                }
+                logInfo("");
+
+                logInfo("== Body == ");
+                logInfo(request->body);
+                logInfo("");
+
+                return std::make_shared<HttpResponse>(
+                    200, "OK", HttpErrorCode::Ok, headers, std::string("OK"));
+            });
+    }
 } // namespace ix

ixwebsocket/IXHttpServer.h

@@ -38,6 +38,8 @@ namespace ix
 
         void makeRedirectServer(const std::string& redirectUrl);
 
+        void makeDebugServer();
+
     private:
         // Member variables
         OnConnectionCallback _onConnectionCallback;

ixwebsocket/IXWebSocketVersion.h

@@ -6,4 +6,4 @@
 
 #pragma once
 
-#define IX_WEBSOCKET_VERSION "10.4.7"
+#define IX_WEBSOCKET_VERSION "10.5.3"

test/CMakeLists.txt

@@ -16,8 +16,18 @@ include_directories(
   ../ws
 )
 
+add_definitions(-DSPDLOG_COMPILED_LIB=1)
+
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  include_directories(../third_party/jsoncpp)
+  set(JSONCPP_SOURCES ../third_party/jsoncpp/jsoncpp.cpp)
+endif()
+
 # Shared sources
 set (SOURCES
+  ${JSONCPP_SOURCES}
+
   test_runner.cpp
   IXTest.cpp
   ../third_party/msgpack11/msgpack11.cpp
@@ -98,7 +108,6 @@ target_link_libraries(ixwebsocket_unittest ixcrypto)
 target_link_libraries(ixwebsocket_unittest ixcore)
 
 target_link_libraries(ixwebsocket_unittest spdlog)
-target_link_libraries(ixwebsocket_unittest jsoncpp_static)
 if (USE_PYTHON)
   target_link_libraries(ixwebsocket_unittest ${Python_LIBRARIES})
 endif()

third_party/jsoncpp/json/json-forwards.h

@@ -0,0 +1,316 @@
+/// Json-cpp amalgamated forward header (http://jsoncpp.sourceforge.net/).
+/// It is intended to be used with #include "json/json-forwards.h"
+/// This header provides forward declaration for all JsonCpp types.
+
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: LICENSE
+// //////////////////////////////////////////////////////////////////////
+
+/*
+The JsonCpp library's source code, including accompanying documentation,
+tests and demonstration applications, are licensed under the following
+conditions...
+
+Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all
+jurisdictions which recognize such a disclaimer. In such jurisdictions,
+this software is released into the Public Domain.
+
+In jurisdictions which do not recognize Public Domain property (e.g. Germany as of
+2010), this software is Copyright (c) 2007-2010 by Baptiste Lepilleur and
+The JsonCpp Authors, and is released under the terms of the MIT License (see below).
+
+In jurisdictions which recognize Public Domain property, the user of this
+software may choose to accept it either as 1) Public Domain, 2) under the
+conditions of the MIT License (see below), or 3) under the terms of dual
+Public Domain/MIT License conditions described here, as they choose.
+
+The MIT License is about as close to Public Domain as a license can get, and is
+described in clear, concise terms at:
+
+   http://en.wikipedia.org/wiki/MIT_License
+
+The full text of the MIT License follows:
+
+========================================================================
+Copyright (c) 2007-2010 Baptiste Lepilleur and The JsonCpp Authors
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy,
+modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+========================================================================
+(END LICENSE TEXT)
+
+The MIT license is compatible with both the GPL and commercial
+software, affording one all of the rights of Public Domain with the
+minor nuisance of being required to keep the above copyright notice
+and license text in the source code. Note also that by accepting the
+Public Domain "license" you can re-license your copy using whatever
+license you like.
+
+*/
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: LICENSE
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+#ifndef JSON_FORWARD_AMALGAMATED_H_INCLUDED
+# define JSON_FORWARD_AMALGAMATED_H_INCLUDED
+/// If defined, indicates that the source file is amalgamated
+/// to prevent private header inclusion.
+#define JSON_IS_AMALGAMATION
+
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: include/json/config.h
+// //////////////////////////////////////////////////////////////////////
+
+// Copyright 2007-2010 Baptiste Lepilleur and The JsonCpp Authors
+// Distributed under MIT license, or public domain if desired and
+// recognized in your jurisdiction.
+// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
+
+#ifndef JSON_CONFIG_H_INCLUDED
+#define JSON_CONFIG_H_INCLUDED
+#include <cstddef>
+#include <cstdint>
+#include <istream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+// If non-zero, the library uses exceptions to report bad input instead of C
+// assertion macros. The default is to use exceptions.
+#ifndef JSON_USE_EXCEPTION
+#define JSON_USE_EXCEPTION 1
+#endif
+
+// Temporary, tracked for removal with issue #982.
+#ifndef JSON_USE_NULLREF
+#define JSON_USE_NULLREF 1
+#endif
+
+/// If defined, indicates that the source file is amalgamated
+/// to prevent private header inclusion.
+/// Remarks: it is automatically defined in the generated amalgamated header.
+// #define JSON_IS_AMALGAMATION
+
+// Export macros for DLL visibility
+#if defined(JSON_DLL_BUILD)
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#define JSON_API __declspec(dllexport)
+#define JSONCPP_DISABLE_DLL_INTERFACE_WARNING
+#elif defined(__GNUC__) || defined(__clang__)
+#define JSON_API __attribute__((visibility("default")))
+#endif // if defined(_MSC_VER)
+
+#elif defined(JSON_DLL)
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#define JSON_API __declspec(dllimport)
+#define JSONCPP_DISABLE_DLL_INTERFACE_WARNING
+#endif // if defined(_MSC_VER)
+#endif // ifdef JSON_DLL_BUILD
+
+#if !defined(JSON_API)
+#define JSON_API
+#endif
+
+#if defined(_MSC_VER) && _MSC_VER < 1800
+#error                                                                         \
+    "ERROR:  Visual Studio 12 (2013) with _MSC_VER=1800 is the oldest supported compiler with sufficient C++11 capabilities"
+#endif
+
+#if defined(_MSC_VER) && _MSC_VER < 1900
+// As recommended at
+// https://stackoverflow.com/questions/2915672/snprintf-and-visual-studio-2010
+extern JSON_API int msvc_pre1900_c99_snprintf(char* outBuf, size_t size,
+                                              const char* format, ...);
+#define jsoncpp_snprintf msvc_pre1900_c99_snprintf
+#else
+#define jsoncpp_snprintf std::snprintf
+#endif
+
+// If JSON_NO_INT64 is defined, then Json only support C++ "int" type for
+// integer
+// Storages, and 64 bits integer support is disabled.
+// #define JSON_NO_INT64 1
+
+// JSONCPP_OVERRIDE is maintained for backwards compatibility of external tools.
+// C++11 should be used directly in JSONCPP.
+#define JSONCPP_OVERRIDE override
+
+#if __cplusplus >= 201103L
+#define JSONCPP_NOEXCEPT noexcept
+#define JSONCPP_OP_EXPLICIT explicit
+#elif defined(_MSC_VER) && _MSC_VER < 1900
+#define JSONCPP_NOEXCEPT throw()
+#define JSONCPP_OP_EXPLICIT explicit
+#elif defined(_MSC_VER) && _MSC_VER >= 1900
+#define JSONCPP_NOEXCEPT noexcept
+#define JSONCPP_OP_EXPLICIT explicit
+#else
+#define JSONCPP_NOEXCEPT throw()
+#define JSONCPP_OP_EXPLICIT
+#endif
+
+#ifdef __clang__
+#if __has_extension(attribute_deprecated_with_message)
+#define JSONCPP_DEPRECATED(message) __attribute__((deprecated(message)))
+#endif
+#elif defined(__GNUC__) // not clang (gcc comes later since clang emulates gcc)
+#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
+#define JSONCPP_DEPRECATED(message) __attribute__((deprecated(message)))
+#elif (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))
+#define JSONCPP_DEPRECATED(message) __attribute__((__deprecated__))
+#endif                  // GNUC version
+#elif defined(_MSC_VER) // MSVC (after clang because clang on Windows emulates
+                        // MSVC)
+#define JSONCPP_DEPRECATED(message) __declspec(deprecated(message))
+#endif // __clang__ || __GNUC__ || _MSC_VER
+
+#if !defined(JSONCPP_DEPRECATED)
+#define JSONCPP_DEPRECATED(message)
+#endif // if !defined(JSONCPP_DEPRECATED)
+
+#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
+#define JSON_USE_INT64_DOUBLE_CONVERSION 1
+#endif
+
+#if !defined(JSON_IS_AMALGAMATION)
+
+#include "allocator.h"
+#include "version.h"
+
+#endif // if !defined(JSON_IS_AMALGAMATION)
+
+namespace Json {
+using Int = int;
+using UInt = unsigned int;
+#if defined(JSON_NO_INT64)
+using LargestInt = int;
+using LargestUInt = unsigned int;
+#undef JSON_HAS_INT64
+#else                 // if defined(JSON_NO_INT64)
+// For Microsoft Visual use specific types as long long is not supported
+#if defined(_MSC_VER) // Microsoft Visual Studio
+using Int64 = __int64;
+using UInt64 = unsigned __int64;
+#else                 // if defined(_MSC_VER) // Other platforms, use long long
+using Int64 = int64_t;
+using UInt64 = uint64_t;
+#endif                // if defined(_MSC_VER)
+using LargestInt = Int64;
+using LargestUInt = UInt64;
+#define JSON_HAS_INT64
+#endif // if defined(JSON_NO_INT64)
+
+template <typename T>
+using Allocator =
+    typename std::conditional<JSONCPP_USING_SECURE_MEMORY, SecureAllocator<T>,
+                              std::allocator<T>>::type;
+using String = std::basic_string<char, std::char_traits<char>, Allocator<char>>;
+using IStringStream =
+    std::basic_istringstream<String::value_type, String::traits_type,
+                             String::allocator_type>;
+using OStringStream =
+    std::basic_ostringstream<String::value_type, String::traits_type,
+                             String::allocator_type>;
+using IStream = std::istream;
+using OStream = std::ostream;
+} // namespace Json
+
+// Legacy names (formerly macros).
+using JSONCPP_STRING = Json::String;
+using JSONCPP_ISTRINGSTREAM = Json::IStringStream;
+using JSONCPP_OSTRINGSTREAM = Json::OStringStream;
+using JSONCPP_ISTREAM = Json::IStream;
+using JSONCPP_OSTREAM = Json::OStream;
+
+#endif // JSON_CONFIG_H_INCLUDED
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: include/json/config.h
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: include/json/forwards.h
+// //////////////////////////////////////////////////////////////////////
+
+// Copyright 2007-2010 Baptiste Lepilleur and The JsonCpp Authors
+// Distributed under MIT license, or public domain if desired and
+// recognized in your jurisdiction.
+// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
+
+#ifndef JSON_FORWARDS_H_INCLUDED
+#define JSON_FORWARDS_H_INCLUDED
+
+#if !defined(JSON_IS_AMALGAMATION)
+#include "config.h"
+#endif // if !defined(JSON_IS_AMALGAMATION)
+
+namespace Json {
+
+// writer.h
+class StreamWriter;
+class StreamWriterBuilder;
+class Writer;
+class FastWriter;
+class StyledWriter;
+class StyledStreamWriter;
+
+// reader.h
+class Reader;
+class CharReader;
+class CharReaderBuilder;
+
+// json_features.h
+class Features;
+
+// value.h
+using ArrayIndex = unsigned int;
+class StaticString;
+class Path;
+class PathArgument;
+class Value;
+class ValueIteratorBase;
+class ValueIterator;
+class ValueConstIterator;
+
+} // namespace Json
+
+#endif // JSON_FORWARDS_H_INCLUDED
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: include/json/forwards.h
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+#endif //ifndef JSON_FORWARD_AMALGAMATED_H_INCLUDED

ws/CMakeLists.txt

@@ -22,6 +22,12 @@ include_directories(ws .)
 include_directories(ws ..)
 include_directories(ws ../third_party/cpp-linenoise)
 
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  include_directories(../third_party/jsoncpp)
+  set(JSONCPP_SOURCES ../third_party/jsoncpp/jsoncpp.cpp)
+endif()
+
 if (USE_PYTHON)
   find_package(Python COMPONENTS Development)
   if (NOT Python_FOUND)
@@ -50,9 +56,13 @@ target_link_libraries(ws ixcrypto)
 target_link_libraries(ws ixcore)
 
 target_link_libraries(ws spdlog)
-target_link_libraries(ws jsoncpp_static)
 if (USE_PYTHON)
   target_link_libraries(ws ${Python_LIBRARIES})
 endif()
 
+if (JSONCPP_FOUND)
+  target_include_directories(ws PUBLIC ${JSONCPP_INCLUDE_DIRS})
+  target_link_libraries(ws ${JSONCPP_LIBRARIES})
+endif()
+
 install(TARGETS ws RUNTIME DESTINATION bin)

ws/ws.cpp

@@ -1136,7 +1136,7 @@ namespace ix
         return 0;
     }
 
-    int ws_gzip(const std::string& filename)
+    int ws_gzip(const std::string& filename, int runCount)
     {
         auto res = readAsString(filename);
         bool found = res.first;
@@ -1146,13 +1146,30 @@ namespace ix
             return 1;
         }
 
-        spdlog::info("gzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
+        spdlog::info("gzip input: {} size {} cksum {}",
+                     filename,
+                     res.second.size(),
+                     ix::djb2HashStr(res.second));
 
         std::string compressedBytes;
 
+        spdlog::info("compressing {} times", runCount);
+        std::vector<uint64_t> durations;
         {
             Bench bench("compressing file");
-            compressedBytes = gzipCompress(res.second);
+            bench.setReported();
+
+            for (int i = 0; i < runCount; ++i)
+            {
+                bench.reset();
+                compressedBytes = gzipCompress(res.second);
+                bench.record();
+                durations.push_back(bench.getDuration());
+            }
+
+            size_t medianIdx = durations.size() / 2;
+            uint64_t medianRuntime = durations[medianIdx];
+            spdlog::info("median runtime to compress file: {}", medianRuntime);
         }
 
         std::string outputFilename(filename);
@@ -1163,7 +1180,10 @@ namespace ix
         f << compressedBytes;
         f.close();
 
-        spdlog::info("gzip output: {} cksum {}", outputFilename, ix::djb2HashStr(compressedBytes));
+        spdlog::info("gzip output: {} size {} cksum {}",
+                     outputFilename,
+                     compressedBytes.size(),
+                     ix::djb2HashStr(compressedBytes));
 
         return 0;
     }
@@ -1180,7 +1200,10 @@ namespace ix
             return 1;
         }
 
-        spdlog::info("gunzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
+        spdlog::info("gunzip input: {} size {} cksum {}",
+                     filename,
+                     res.second.size(),
+                     ix::djb2HashStr(res.second));
 
         std::string decompressedBytes;
 
@@ -1200,8 +1223,10 @@ namespace ix
         f << decompressedBytes;
         f.close();
 
-        spdlog::info(
-            "gunzip output: {} cksum {}", outputFilename, ix::djb2HashStr(decompressedBytes));
+        spdlog::info("gunzip output: {} size {} cksum {}",
+                     outputFilename,
+                     decompressedBytes.size(),
+                     ix::djb2HashStr(decompressedBytes));
 
         return 0;
     }
@@ -1246,43 +1271,42 @@ namespace ix
 
         // Setup a callback to be fired
         // when a message or an event (open, close, ping, pong, error) is received
-        webSocket.setOnMessageCallback(
-            [&webSocket, &receivedCountPerSecs, &target, &stop, &condition, &bench](
-                const ix::WebSocketMessagePtr& msg) {
-                if (msg->type == ix::WebSocketMessageType::Message)
-                {
-                    receivedCountPerSecs++;
+        webSocket.setOnMessageCallback([&receivedCountPerSecs, &target, &stop, &condition, &bench](
+                                           const ix::WebSocketMessagePtr& msg) {
+            if (msg->type == ix::WebSocketMessageType::Message)
+            {
+                receivedCountPerSecs++;
 
-                    target -= 1;
-                    if (target == 0)
-                    {
-                        stop = true;
-                        condition.notify_one();
+                target -= 1;
+                if (target == 0)
+                {
+                    stop = true;
+                    condition.notify_one();
 
-                        bench.report();
-                    }
+                    bench.report();
                 }
-                else if (msg->type == ix::WebSocketMessageType::Open)
-                {
-                    bench.reset();
+            }
+            else if (msg->type == ix::WebSocketMessageType::Open)
+            {
+                bench.reset();
 
-                    spdlog::info("ws_autoroute: connected");
-                    spdlog::info("Uri: {}", msg->openInfo.uri);
-                    spdlog::info("Headers:");
-                    for (auto it : msg->openInfo.headers)
-                    {
-                        spdlog::info("{}: {}", it.first, it.second);
-                    }
-                }
-                else if (msg->type == ix::WebSocketMessageType::Pong)
+                spdlog::info("ws_autoroute: connected");
+                spdlog::info("Uri: {}", msg->openInfo.uri);
+                spdlog::info("Headers:");
+                for (auto it : msg->openInfo.headers)
                 {
-                    spdlog::info("Received pong {}", msg->str);
+                    spdlog::info("{}: {}", it.first, it.second);
                 }
-                else if (msg->type == ix::WebSocketMessageType::Close)
-                {
-                    spdlog::info("ws_autoroute: connection closed");
-                }
-            });
+            }
+            else if (msg->type == ix::WebSocketMessageType::Pong)
+            {
+                spdlog::info("Received pong {}", msg->str);
+            }
+            else if (msg->type == ix::WebSocketMessageType::Close)
+            {
+                spdlog::info("ws_autoroute: connection closed");
+            }
+        });
 
         auto timer = [&receivedCountPerSecs, &stop] {
             setThreadName("Timer");
@@ -1437,7 +1461,7 @@ namespace ix
     // Useful endpoint to test HTTP post
     // https://postman-echo.com/post
     //
-    HttpParameters parsePostParameters(const std::string& data)
+    HttpParameters parseHttpParameters(const std::string& data)
     {
         HttpParameters httpParameters;
 
@@ -1462,9 +1486,52 @@ namespace ix
         return httpParameters;
     }
 
+    HttpFormDataParameters parseHttpFormDataParameters(const std::string& data)
+    {
+        HttpFormDataParameters httpFormDataParameters;
+
+        // Split by \n
+        std::string token;
+        std::stringstream tokenStream(data);
+
+        while (std::getline(tokenStream, token))
+        {
+            std::size_t pos = token.rfind('=');
+
+            // Bail out if last '.' is found
+            if (pos == std::string::npos) continue;
+
+            auto key = token.substr(0, pos);
+            auto val = token.substr(pos + 1);
+
+            spdlog::info("{}: {}", key, val);
+
+            if (val[0] == '@')
+            {
+                std::string filename = token.substr(pos + 2);
+
+                auto res = readAsString(filename);
+                bool found = res.first;
+                if (!found)
+                {
+                    spdlog::error("Cannot read content of {}", filename);
+                    continue;
+                }
+
+                val = res.second;
+            }
+
+            httpFormDataParameters[key] = val;
+        }
+
+        return httpFormDataParameters;
+    }
+
     int ws_http_client_main(const std::string& url,
                             const std::string& headersData,
                             const std::string& data,
+                            const std::string& formData,
+                            const std::string& dataBinary,
                             bool headersOnly,
                             int connectTimeout,
                             int transferTimeout,
@@ -1474,6 +1541,7 @@ namespace ix
                             bool save,
                             const std::string& output,
                             bool compress,
+                            bool compressRequest,
                             const ix::SocketTLSOptions& tlsOptions)
     {
         HttpClient httpClient;
@@ -1487,6 +1555,7 @@ namespace ix
         args->maxRedirects = maxRedirects;
         args->verbose = verbose;
         args->compress = compress;
+        args->compressRequest = compressRequest;
         args->logger = [](const std::string& msg) { spdlog::info(msg); };
         args->onProgressCallback = [verbose](int current, int total) -> bool {
             if (verbose)
@@ -1496,20 +1565,30 @@ namespace ix
             return true;
         };
 
-        HttpParameters httpParameters = parsePostParameters(data);
+        HttpParameters httpParameters = parseHttpParameters(data);
+        HttpFormDataParameters httpFormDataParameters = parseHttpFormDataParameters(formData);
 
         HttpResponsePtr response;
         if (headersOnly)
         {
             response = httpClient.head(url, args);
         }
-        else if (data.empty())
+        else if (data.empty() && formData.empty() && dataBinary.empty())
         {
             response = httpClient.get(url, args);
         }
+        else if (!dataBinary.empty())
+        {
+            std::string body = dataBinary;
+            if (compressRequest)
+            {
+                body = gzipCompress(dataBinary);
+            }
+            response = httpClient.request(url, "POST", body, args, 0);
+        }
         else
         {
-            response = httpClient.post(url, httpParameters, args);
+            response = httpClient.post(url, httpParameters, httpFormDataParameters, args);
         }
 
         spdlog::info("");
@@ -1566,6 +1645,7 @@ namespace ix
                       const std::string& hostname,
                       bool redirect,
                       const std::string& redirectUrl,
+                      bool debug,
                       const ix::SocketTLSOptions& tlsOptions)
     {
         spdlog::info("Listening on {}:{}", hostname, port);
@@ -1578,6 +1658,11 @@ namespace ix
             server.makeRedirectServer(redirectUrl);
         }
 
+        if (debug)
+        {
+            server.makeDebugServer();
+        }
+
         auto res = server.listen();
         if (!res.first)
         {
@@ -2905,6 +2990,8 @@ int main(int argc, char** argv)
     std::string path;
     std::string user;
     std::string data;
+    std::string formData;
+    std::string binaryData;
     std::string headers;
     std::string output;
     std::string hostname("127.0.0.1");
@@ -2948,6 +3035,7 @@ int main(int argc, char** argv)
     bool quiet = false;
     bool fluentd = false;
     bool compress = false;
+    bool compressRequest = false;
     bool stress = false;
     bool disableAutomaticReconnection = false;
     bool disablePerMessageDeflate = false;
@@ -2958,6 +3046,7 @@ int main(int argc, char** argv)
     bool version = false;
     bool verifyNone = false;
     bool disablePong = false;
+    bool debug = false;
     int port = 8008;
     int redisPort = 6379;
     int statsdPort = 8125;
@@ -2969,6 +3058,7 @@ int main(int argc, char** argv)
     int msgCount = 1000 * 1000;
     uint32_t maxWaitBetweenReconnectionRetries;
     int pingIntervalSecs = 30;
+    int runCount = 1;
 
     auto addGenericOptions = [&pidfile](CLI::App* app) {
         app->add_option("--pidfile", pidfile, "Pid file");
@@ -3113,7 +3203,8 @@ int main(int argc, char** argv)
     httpClientApp->fallthrough();
     httpClientApp->add_option("url", url, "Connection url")->required();
     httpClientApp->add_option("-d", data, "Form data")->join();
-    httpClientApp->add_option("-F", data, "Form data")->join();
+    httpClientApp->add_option("-F", formData, "Form data")->join();
+    httpClientApp->add_option("--data-binary", binaryData, "Body binary data")->join();
     httpClientApp->add_option("-H", headers, "Header")->join();
     httpClientApp->add_option("--output", output, "Output file");
     httpClientApp->add_flag("-I", headersOnly, "Send a HEAD request");
@@ -3121,7 +3212,8 @@ int main(int argc, char** argv)
     httpClientApp->add_option("--max-redirects", maxRedirects, "Max Redirects");
     httpClientApp->add_flag("-v", verbose, "Verbose");
     httpClientApp->add_flag("-O", save, "Save output to disk");
-    httpClientApp->add_flag("--compress", compress, "Enable gzip compression");
+    httpClientApp->add_flag("--compressed", compress, "Enable gzip compression");
+    httpClientApp->add_flag("--compress_request", compressRequest, "Compress request with gzip");
     httpClientApp->add_option("--connect-timeout", connectTimeOut, "Connection timeout");
     httpClientApp->add_option("--transfer-timeout", transferTimeout, "Transfer timeout");
     addTLSOptions(httpClientApp);
@@ -3255,6 +3347,7 @@ int main(int argc, char** argv)
     httpServerApp->add_option("--host", hostname, "Hostname");
     httpServerApp->add_flag("-L", redirect, "Redirect all request to redirect_url");
     httpServerApp->add_option("--redirect_url", redirectUrl, "Url to redirect to");
+    httpServerApp->add_flag("-D", debug, "Debug server");
     addTLSOptions(httpServerApp);
 
     CLI::App* autobahnApp = app.add_subcommand("autobahn", "Test client Autobahn compliance");
@@ -3297,6 +3390,7 @@ int main(int argc, char** argv)
     CLI::App* gzipApp = app.add_subcommand("gzip", "Gzip compressor");
     gzipApp->fallthrough();
     gzipApp->add_option("filename", filename, "Filename")->required();
+    gzipApp->add_option("--run_count", runCount, "Number of time to run the compression");
 
     CLI::App* gunzipApp = app.add_subcommand("gunzip", "Gzip decompressor");
     gunzipApp->fallthrough();
@@ -3435,6 +3529,8 @@ int main(int argc, char** argv)
         ret = ix::ws_http_client_main(url,
                                       headers,
                                       data,
+                                      formData,
+                                      binaryData,
                                       headersOnly,
                                       connectTimeOut,
                                       transferTimeout,
@@ -3444,6 +3540,7 @@ int main(int argc, char** argv)
                                       save,
                                       output,
                                       compress,
+                                      compressRequest,
                                       tlsOptions);
     }
     else if (app.got_subcommand("redis_cli"))
@@ -3553,7 +3650,7 @@ int main(int argc, char** argv)
     }
     else if (app.got_subcommand("httpd"))
     {
-        ret = ix::ws_httpd_main(port, hostname, redirect, redirectUrl, tlsOptions);
+        ret = ix::ws_httpd_main(port, hostname, redirect, redirectUrl, debug, tlsOptions);
     }
     else if (app.got_subcommand("autobahn"))
     {
@@ -3600,7 +3697,7 @@ int main(int argc, char** argv)
     }
     else if (app.got_subcommand("gzip"))
     {
-        ret = ix::ws_gzip(filename);
+        ret = ix::ws_gzip(filename, runCount);
     }
     else if (app.got_subcommand("gunzip"))
     {