fix Dockerfile link

* unittest working / uses shared_ptr for a bunch of things 🗿

* fix command line tools

* fix ws + add doc

* add more logging

README.md

@@ -129,33 +129,33 @@ Here is what the HTTP client API looks like. Note that HTTP client support is ve
 // Preparation
 //
 HttpClient httpClient;
-HttpRequestArgs args;
+HttpRequestArgsPtr args = httpClient.createRequest();
 
 // Custom headers can be set
 WebSocketHttpHeaders headers;
 headers["Foo"] = "bar";
-args.extraHeaders = headers;
+args->extraHeaders = headers;
 
 // Timeout options
-args.connectTimeout = connectTimeout;
-args.transferTimeout = transferTimeout;
+args->connectTimeout = connectTimeout;
+args->transferTimeout = transferTimeout;
 
 // Redirect options
-args.followRedirects = followRedirects;
-args.maxRedirects = maxRedirects;
+args->followRedirects = followRedirects;
+args->maxRedirects = maxRedirects;
 
 // Misc
-args.compress = compress; // Enable gzip compression
-args.verbose = verbose;
-args.logger = [](const std::string& msg)
+args->compress = compress; // Enable gzip compression
+args->verbose = verbose;
+args->logger = [](const std::string& msg)
 {
     std::cout << msg;
 };
 
 //
-// Request
+// Synchronous Request
 //
-HttpResponse out;
+HttpResponsePtr out;
 std::string url = "https://www.google.com";
 
 // HEAD request
@@ -175,13 +175,30 @@ out = httpClient.post(url, std::string("foo=bar"), args);
 //
 // Result
 //
-auto statusCode = std::get<0>(out);
-auto errorCode = std::get<1>(out);
-auto responseHeaders = std::get<2>(out);
-auto payload = std::get<3>(out);
-auto errorMsg = std::get<4>(out);
-auto uploadSize = std::get<5>(out);
-auto downloadSize = std::get<6>(out);
+auto errorCode = response->errorCode; // Can be HttpErrorCode::Ok, HttpErrorCode::UrlMalformed, etc...
+auto errorCode = response->errorCode; // 200, 404, etc...
+auto responseHeaders = response->headers; // All the headers in a special case-insensitive unordered_map of (string, string)
+auto payload = response->payload; // All the bytes from the response as an std::string
+auto errorMsg = response->errorMsg; // Descriptive error message in case of failure
+auto uploadSize = response->uploadSize; // Byte count of uploaded data
+auto downloadSize = response->downloadSize; // Byte count of downloaded data
+
+//
+// Asynchronous Request
+//
+bool async = true;
+HttpClient httpClient(async);
+auto args = httpClient.createRequest(url, HttpClient::kGet);
+
+// Push the request to a queue,
+bool ok = httpClient.performRequest(args, [](const HttpResponsePtr& response)
+    {
+        // This callback execute in a background thread. Make sure you uses appropriate protection such as mutex
+        auto statusCode = response->statusCode; // acess results
+    }
+);
+
+// ok will be false if your httpClient is not async
 ```
 
 ## Build

docker/Dockerfile.ubuntu_bionic

@@ -0,0 +1,23 @@
+# Build time
+FROM ubuntu:bionic as build
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update 
+RUN apt-get -y install wget 
+RUN mkdir -p /tmp/cmake
+WORKDIR /tmp/cmake
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz 
+RUN tar zxf cmake-3.14.0-Linux-x86_64.tar.gz
+
+RUN apt-get -y install g++ 
+RUN apt-get -y install libssl-dev
+RUN apt-get -y install libz-dev
+RUN apt-get -y install make
+RUN apt-get -y install python
+
+COPY . .
+
+ARG CMAKE_BIN_PATH=/tmp/cmake/cmake-3.14.0-Linux-x86_64/bin
+ENV PATH="${CMAKE_BIN_PATH}:${PATH}"
+
+RUN ["make", "ws"]

ixwebsocket/IXHttpClient.cpp

@@ -21,24 +21,98 @@ namespace ix
     const std::string HttpClient::kPost = "POST";
     const std::string HttpClient::kGet = "GET";
     const std::string HttpClient::kHead = "HEAD";
+    const std::string HttpClient::kDel = "DEL";
+    const std::string HttpClient::kPut = "PUT";
 
-    HttpClient::HttpClient()
+    HttpClient::HttpClient(bool async) : _async(async), _stop(false)
     {
+        if (!_async) return;
 
+        _thread = std::thread(&HttpClient::run, this);
     }
 
     HttpClient::~HttpClient()
     {
+        if (!_thread.joinable()) return;
 
+        _stop = true;
+        _condition.notify_one();
+        _thread.join();
     }
 
-    HttpResponse HttpClient::request(
+    HttpRequestArgsPtr HttpClient::createRequest(const std::string& url,
+                                                 const std::string& verb)
+    {
+        auto request = std::make_shared<HttpRequestArgs>();
+        request->url = url;
+        request->verb = verb;
+        return request;
+    }
+
+    bool HttpClient::performRequest(HttpRequestArgsPtr args,
+                                    const OnResponseCallback& onResponseCallback)
+    {
+        if (!_async) return false;
+
+        // Enqueue the task
+        {
+            // acquire lock
+            std::unique_lock<std::mutex> lock(_queueMutex);
+
+            // add the task
+            _queue.push(std::make_pair(args, onResponseCallback));
+        } // release lock
+
+        // wake up one thread
+        _condition.notify_one();
+
+        return true;
+    }
+
+    void HttpClient::run()
+    {
+        while (true)
+        {
+            HttpRequestArgsPtr args;
+            OnResponseCallback onResponseCallback;
+
+            {
+                std::unique_lock<std::mutex> lock(_queueMutex);
+
+                while (!_stop && _queue.empty())
+                {
+                    _condition.wait(lock);
+                }
+
+                if (_stop) return;
+
+                auto p = _queue.front();
+                _queue.pop();
+
+                args = p.first;
+                onResponseCallback = p.second;
+            }
+
+            if (_stop) return;
+
+            HttpResponsePtr response = request(args->url, args->verb, args->body, args);
+            onResponseCallback(response);
+
+            if (_stop) return;
+        }
+    }
+
+    HttpResponsePtr HttpClient::request(
         const std::string& url,
         const std::string& verb,
         const std::string& body,
-        const HttpRequestArgs& args,
+        HttpRequestArgsPtr args,
         int redirects)
     {
+        // We only have one socket connection, so we cannot 
+        // make multiple requests concurrently.
+        std::lock_guard<std::mutex> lock(_mutex);
+
         uint64_t uploadSize = 0;
         uint64_t downloadSize = 0;
         int code = 0;
@@ -52,7 +126,7 @@ namespace ix
         {
             std::stringstream ss;
             ss << "Cannot parse url: " << url;
-            return std::make_tuple(code, HttpErrorCode::UrlMalformed,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::UrlMalformed,
                                 headers, payload, ss.str(),
                                 uploadSize, downloadSize);
         }
@@ -63,7 +137,7 @@ namespace ix
 
         if (!_socket)
         {
-            return std::make_tuple(code, HttpErrorCode::CannotCreateSocket,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::CannotCreateSocket,
                                 headers, payload, errorMsg,
                                 uploadSize, downloadSize);
         }
@@ -72,26 +146,36 @@ namespace ix
         std::stringstream ss;
         ss << verb << " " << path << " HTTP/1.1\r\n";
         ss << "Host: " << host << "\r\n";
-        ss << "User-Agent: ixwebsocket/1.0.0" << "\r\n";
-        ss << "Accept: */*" << "\r\n";
 
-        if (args.compress)
+        if (args->compress)
         {
             ss << "Accept-Encoding: gzip" << "\r\n";
         }
 
         // Append extra headers
-        for (auto&& it : args.extraHeaders)
+        for (auto&& it : args->extraHeaders)
         {
             ss << it.first << ": " << it.second << "\r\n";
         }
 
-        if (verb == kPost)
+        // Set a default Accept header if none is present
+        if (headers.find("Accept") == headers.end())
+        {
+            ss << "Accept: */*" << "\r\n";
+        }
+
+        // Set a default User agent if none is present
+        if (headers.find("User-Agent") == headers.end())
+        {
+            ss << "User-Agent: ixwebsocket" << "\r\n";
+        }
+
+        if (verb == kPost || verb == kPut)
         {
             ss << "Content-Length: " << body.size() << "\r\n";
 
             // Set default Content-Type if unspecified
-            if (args.extraHeaders.find("Content-Type") == args.extraHeaders.end())
+            if (args->extraHeaders.find("Content-Type") == args->extraHeaders.end())
             {
                 ss << "Content-Type: application/x-www-form-urlencoded" << "\r\n";
             }
@@ -109,23 +193,23 @@ namespace ix
 
         // Make a cancellation object dealing with connection timeout
         auto isCancellationRequested =
-            makeCancellationRequestWithTimeout(args.connectTimeout, requestInitCancellation);
+            makeCancellationRequestWithTimeout(args->connectTimeout, requestInitCancellation);
 
         bool success = _socket->connect(host, port, errMsg, isCancellationRequested);
         if (!success)
         {
             std::stringstream ss;
             ss << "Cannot connect to url: " << url << " / error : " << errMsg;
-            return std::make_tuple(code, HttpErrorCode::CannotConnect,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::CannotConnect,
                                                   headers, payload, ss.str(),
                                                   uploadSize, downloadSize);
         }
 
         // Make a new cancellation object dealing with transfer timeout
         isCancellationRequested =
-            makeCancellationRequestWithTimeout(args.transferTimeout, requestInitCancellation);
+            makeCancellationRequestWithTimeout(args->transferTimeout, requestInitCancellation);
 
-        if (args.verbose)
+        if (args->verbose)
         {
             std::stringstream ss;
             ss << "Sending " << verb << " request "
@@ -142,7 +226,7 @@ namespace ix
         if (!_socket->writeBytes(req, isCancellationRequested))
         {
             std::string errorMsg("Cannot send request");
-            return std::make_tuple(code, HttpErrorCode::SendError,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::SendError,
                                                   headers, payload, errorMsg,
                                                   uploadSize, downloadSize);
         }
@@ -156,12 +240,12 @@ namespace ix
         if (!lineValid)
         {
             std::string errorMsg("Cannot retrieve status line");
-            return std::make_tuple(code, HttpErrorCode::CannotReadStatusLine,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::CannotReadStatusLine,
                                                   headers, payload, errorMsg,
                                                   uploadSize, downloadSize);
         }
 
-        if (args.verbose)
+        if (args->verbose)
         {
             std::stringstream ss;
             ss << "Status line " << line;
@@ -171,7 +255,7 @@ namespace ix
         if (sscanf(line.c_str(), "HTTP/1.1 %d", &code) != 1)
         {
             std::string errorMsg("Cannot parse response code from status line");
-            return std::make_tuple(code, HttpErrorCode::MissingStatus,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::MissingStatus,
                                                   headers, payload, errorMsg,
                                                   uploadSize, downloadSize);
         }
@@ -183,27 +267,27 @@ namespace ix
         if (!headersValid)
         {
             std::string errorMsg("Cannot parse http headers");
-            return std::make_tuple(code, HttpErrorCode::HeaderParsingError,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::HeaderParsingError,
                                                   headers, payload, errorMsg,
                                                   uploadSize, downloadSize);
         }
 
         // Redirect ?
-        if ((code >= 301 && code <= 308) && args.followRedirects)
+        if ((code >= 301 && code <= 308) && args->followRedirects)
         {
             if (headers.find("Location") == headers.end())
             {
                 std::string errorMsg("Missing location header for redirect");
-                return std::make_tuple(code, HttpErrorCode::MissingLocation,
+                return std::make_shared<HttpResponse>(code, HttpErrorCode::MissingLocation,
                                                       headers, payload, errorMsg,
                                                       uploadSize, downloadSize);
             }
 
-            if (redirects >= args.maxRedirects)
+            if (redirects >= args->maxRedirects)
             {
                 std::stringstream ss;
                 ss << "Too many redirects: " << redirects;
-                return std::make_tuple(code, HttpErrorCode::TooManyRedirects,
+                return std::make_shared<HttpResponse>(code, HttpErrorCode::TooManyRedirects,
                                                       headers, payload, ss.str(),
                                                       uploadSize, downloadSize);
             }
@@ -215,7 +299,7 @@ namespace ix
 
         if (verb == "HEAD")
         {
-            return std::make_tuple(code, HttpErrorCode::Ok,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::Ok,
                                                   headers, payload, std::string(),
                                                   uploadSize, downloadSize);
         }
@@ -231,12 +315,12 @@ namespace ix
             payload.reserve(contentLength);
 
             auto chunkResult = _socket->readBytes(contentLength,
-                                                  args.onProgressCallback,
+                                                  args->onProgressCallback,
                                                   isCancellationRequested);
             if (!chunkResult.first)
             {
                 errorMsg = "Cannot read chunk";
-                return std::make_tuple(code, HttpErrorCode::ChunkReadError,
+                return std::make_shared<HttpResponse>(code, HttpErrorCode::ChunkReadError,
                                                       headers, payload, errorMsg,
                                                       uploadSize, downloadSize);
             }
@@ -254,7 +338,7 @@ namespace ix
 
                 if (!lineResult.first)
                 {
-                    return std::make_tuple(code, HttpErrorCode::ChunkReadError,
+                    return std::make_shared<HttpResponse>(code, HttpErrorCode::ChunkReadError,
                                                           headers, payload, errorMsg,
                                                           uploadSize, downloadSize);
                 }
@@ -264,7 +348,7 @@ namespace ix
                 ss << std::hex << line;
                 ss >> chunkSize;
 
-                if (args.verbose)
+                if (args->verbose)
                 {
                     std::stringstream oss;
                     oss << "Reading " << chunkSize << " bytes"
@@ -276,12 +360,12 @@ namespace ix
 
                 // Read a chunk
                 auto chunkResult = _socket->readBytes((size_t) chunkSize,
-                                                      args.onProgressCallback,
+                                                      args->onProgressCallback,
                                                       isCancellationRequested);
                 if (!chunkResult.first)
                 {
                     errorMsg = "Cannot read chunk";
-                    return std::make_tuple(code, HttpErrorCode::ChunkReadError,
+                    return std::make_shared<HttpResponse>(code, HttpErrorCode::ChunkReadError,
                                                           headers, payload, errorMsg,
                                                           uploadSize, downloadSize);
                 }
@@ -292,7 +376,7 @@ namespace ix
 
                 if (!lineResult.first)
                 {
-                    return std::make_tuple(code, HttpErrorCode::ChunkReadError,
+                    return std::make_shared<HttpResponse>(code, HttpErrorCode::ChunkReadError,
                                                           headers, payload, errorMsg,
                                                           uploadSize, downloadSize);
                 }
@@ -307,7 +391,7 @@ namespace ix
         else
         {
             std::string errorMsg("Cannot read http body");
-            return std::make_tuple(code, HttpErrorCode::CannotReadBody,
+            return std::make_shared<HttpResponse>(code, HttpErrorCode::CannotReadBody,
                                                   headers, payload, errorMsg,
                                                   uploadSize, downloadSize);
         }
@@ -321,44 +405,64 @@ namespace ix
             if (!gzipInflate(payload, decompressedPayload))
             {
                 std::string errorMsg("Error decompressing payload");
-                return std::make_tuple(code, HttpErrorCode::Gzip,
+                return std::make_shared<HttpResponse>(code, HttpErrorCode::Gzip,
                                                       headers, payload, errorMsg,
                                                       uploadSize, downloadSize);
             }
             payload = decompressedPayload;
         }
 
-        return std::make_tuple(code, HttpErrorCode::Ok,
+        return std::make_shared<HttpResponse>(code, HttpErrorCode::Ok,
                                               headers, payload, std::string(),
                                               uploadSize, downloadSize);
     }
 
-    HttpResponse HttpClient::get(const std::string& url,
-                                 const HttpRequestArgs& args)
+    HttpResponsePtr HttpClient::get(const std::string& url,
+                                    HttpRequestArgsPtr args)
     {
         return request(url, kGet, std::string(), args);
     }
 
-    HttpResponse HttpClient::head(const std::string& url,
-                                  const HttpRequestArgs& args)
+    HttpResponsePtr HttpClient::head(const std::string& url,
+                                     HttpRequestArgsPtr args)
     {
         return request(url, kHead, std::string(), args);
     }
 
-    HttpResponse HttpClient::post(const std::string& url,
+    HttpResponsePtr HttpClient::del(const std::string& url,
+                                    HttpRequestArgsPtr args)
+    {
+        return request(url, kDel, std::string(), args);
+    }
+
+    HttpResponsePtr HttpClient::post(const std::string& url,
                                      const HttpParameters& httpParameters,
-                                  const HttpRequestArgs& args)
+                                     HttpRequestArgsPtr args)
     {
         return request(url, kPost, serializeHttpParameters(httpParameters), args);
     }
 
-    HttpResponse HttpClient::post(const std::string& url,
+    HttpResponsePtr HttpClient::post(const std::string& url,
                                      const std::string& body,
-                                  const HttpRequestArgs& args)
+                                     HttpRequestArgsPtr args)
     {
         return request(url, kPost, body, args);
     }
 
+    HttpResponsePtr HttpClient::put(const std::string& url,
+                                    const HttpParameters& httpParameters,
+                                    HttpRequestArgsPtr args)
+    {
+        return request(url, kPut, serializeHttpParameters(httpParameters), args);
+    }
+
+    HttpResponsePtr HttpClient::put(const std::string& url,
+                                    const std::string& body,
+                                    const HttpRequestArgsPtr args)
+    {
+        return request(url, kPut, body, args);
+    }
+
     std::string HttpClient::urlEncode(const std::string& value)
     {
         std::ostringstream escaped;
@@ -456,11 +560,11 @@ namespace ix
     }
 
     void HttpClient::log(const std::string& msg,
-                         const HttpRequestArgs& args)
+                         HttpRequestArgsPtr args)
     {
-        if (args.logger)
+        if (args->logger)
         {
-            args.logger(msg);
+            args->logger(msg);
         }
     }
 }

ixwebsocket/IXHttpClient.h

@@ -10,11 +10,13 @@
 #include "IXWebSocketHttpHeaders.h"
 #include <algorithm>
 #include <atomic>
+#include <condition_variable>
 #include <functional>
 #include <map>
 #include <memory>
 #include <mutex>
-#include <tuple>
+#include <queue>
+#include <thread>
 
 namespace ix
 {
@@ -34,23 +36,48 @@ namespace ix
         MissingLocation = 11,
         TooManyRedirects = 12,
         ChunkReadError = 13,
-        CannotReadBody = 14
+        CannotReadBody = 14,
+        Invalid = 100
     };
 
-    using HttpResponse = std::tuple<int,           // status
-                                    HttpErrorCode, // error code
-                                    WebSocketHttpHeaders,
-                                    std::string, // payload
-                                    std::string, // error msg
-                                    uint64_t,    // upload size
-                                    uint64_t>;   // download size
+    struct HttpResponse
+    {
+        int statusCode;
+        HttpErrorCode errorCode;
+        WebSocketHttpHeaders headers;
+        std::string payload;
+        std::string errorMsg;
+        uint64_t uploadSize;
+        uint64_t downloadSize;
 
+        HttpResponse(int s = 0,
+                     const HttpErrorCode& c = HttpErrorCode::Ok,
+                     const WebSocketHttpHeaders& h = WebSocketHttpHeaders(),
+                     const std::string& p = std::string(),
+                     const std::string& e = std::string(),
+                     uint64_t u = 0,
+                     uint64_t d = 0)
+            : statusCode(s)
+            , errorCode(c)
+            , headers(h)
+            , payload(p)
+            , errorMsg(e)
+            , uploadSize(u)
+            , downloadSize(d)
+        {
+            ;
+        }
+    };
+
+    using HttpResponsePtr = std::shared_ptr<HttpResponse>;
     using HttpParameters = std::map<std::string, std::string>;
     using Logger = std::function<void(const std::string&)>;
+    using OnResponseCallback = std::function<void(const HttpResponsePtr&)>;
 
     struct HttpRequestArgs
     {
         std::string url;
+        std::string verb;
         WebSocketHttpHeaders extraHeaders;
         std::string body;
         int connectTimeout;
@@ -63,41 +90,72 @@ namespace ix
         OnProgressCallback onProgressCallback;
     };
 
+    using HttpRequestArgsPtr = std::shared_ptr<HttpRequestArgs>;
+
     class HttpClient
     {
     public:
-        HttpClient();
+        HttpClient(bool async = false);
         ~HttpClient();
 
-        HttpResponse get(const std::string& url, const HttpRequestArgs& args);
-        HttpResponse head(const std::string& url, const HttpRequestArgs& args);
+        HttpResponsePtr get(const std::string& url, HttpRequestArgsPtr args);
+        HttpResponsePtr head(const std::string& url, HttpRequestArgsPtr args);
+        HttpResponsePtr del(const std::string& url, HttpRequestArgsPtr args);
 
-        HttpResponse post(const std::string& url,
+        HttpResponsePtr post(const std::string& url,
                              const HttpParameters& httpParameters,
-                          const HttpRequestArgs& args);
-        HttpResponse post(const std::string& url,
+                             HttpRequestArgsPtr args);
+        HttpResponsePtr post(const std::string& url,
                              const std::string& body,
-                          const HttpRequestArgs& args);
+                             HttpRequestArgsPtr args);
 
-    private:
-        HttpResponse request(const std::string& url,
+        HttpResponsePtr put(const std::string& url,
+                            const HttpParameters& httpParameters,
+                            HttpRequestArgsPtr args);
+        HttpResponsePtr put(const std::string& url,
+                            const std::string& body,
+                            HttpRequestArgsPtr args);
+
+        HttpResponsePtr request(const std::string& url,
                                 const std::string& verb,
                                 const std::string& body,
-                             const HttpRequestArgs& args,
+                                HttpRequestArgsPtr args,
                                 int redirects = 0);
 
+        // Async API
+        HttpRequestArgsPtr createRequest(const std::string& url = std::string(),
+                                         const std::string& verb = HttpClient::kGet);
+
+        bool performRequest(HttpRequestArgsPtr request,
+                            const OnResponseCallback& onResponseCallback);
+
         std::string serializeHttpParameters(const HttpParameters& httpParameters);
 
         std::string urlEncode(const std::string& value);
 
-        void log(const std::string& msg, const HttpRequestArgs& args);
-
-        bool gzipInflate(const std::string& in, std::string& out);
-
-        std::shared_ptr<Socket> _socket;
-
         const static std::string kPost;
         const static std::string kGet;
         const static std::string kHead;
+        const static std::string kDel;
+        const static std::string kPut;
+
+    private:
+        void log(const std::string& msg, HttpRequestArgsPtr args);
+
+        bool gzipInflate(const std::string& in, std::string& out);
+
+        // Async API background thread runner
+        void run();
+
+        // Async API
+        bool _async;
+        std::queue<std::pair<HttpRequestArgsPtr, OnResponseCallback>> _queue;
+        mutable std::mutex _queueMutex;
+        std::condition_variable _condition;
+        std::atomic<bool> _stop;
+        std::thread _thread;
+
+        std::shared_ptr<Socket> _socket;
+        std::mutex _mutex; // to protect accessing the _socket (only one socket per client)
     };
 } // namespace ix

makefile

@@ -44,7 +44,7 @@ trail:
 	sh third_party/remote_trailing_whitespaces.sh
 
 format:
-	sh third_party/indent.sh
+	find ixwebsocket ws -name '*.cpp' -o -name '*.h' -exec clang-format -i {} \;
 
 # That target is used to start a node server, but isn't required as we have 
 # a builtin C++ server started in the unittest now

test/CMakeLists.txt

@@ -37,6 +37,7 @@ set (SOURCES
   IXWebSocketTestConnectionDisconnection.cpp
   IXUrlParserTest.cpp
   IXWebSocketServerTest.cpp
+  IXHttpClientTest.cpp
 )
 
 # Some unittest don't work on windows yet

test/IXHttpClientTest.cpp

@@ -0,0 +1,233 @@
+/*
+ *  IXSocketTest.cpp
+ *  Author: Benjamin Sergeant
+ *  Copyright (c) 2019 Machine Zone. All rights reserved.
+ */
+
+#include <iostream>
+#include <ixwebsocket/IXHttpClient.h>
+
+#include "catch.hpp"
+
+using namespace ix;
+
+TEST_CASE("http client", "[http]")
+{
+    SECTION("Connect to a remote HTTP server")
+    {
+        HttpClient httpClient;
+        WebSocketHttpHeaders headers;
+
+        std::string url("http://httpbin.org/");
+        auto args = httpClient.createRequest(url);
+
+        args->extraHeaders = headers;
+        args->connectTimeout = 60;
+        args->transferTimeout = 60;
+        args->followRedirects = true;
+        args->maxRedirects = 10;
+        args->verbose = true;
+        args->compress = true;
+        args->logger = [](const std::string& msg)
+        {
+            std::cout << msg;
+        };
+        args->onProgressCallback = [](int current, int total) -> bool
+        {
+            std::cerr << "\r" << "Downloaded "
+                      << current << " bytes out of " << total;
+            return true;
+        };
+
+        auto response = httpClient.get(url, args);
+
+        for (auto it : response->headers)
+        {
+            std::cerr << it.first << ": " << it.second << std::endl;
+        }
+
+        std::cerr << "Upload size: " << response->uploadSize << std::endl;
+        std::cerr << "Download size: " << response->downloadSize << std::endl;
+        std::cerr << "Status: " << response->statusCode << std::endl;
+        std::cerr << "Error message: " << response->errorMsg << std::endl;
+
+        REQUIRE(response->errorCode == HttpErrorCode::Ok);
+        REQUIRE(response->statusCode == 200);
+    }
+
+    SECTION("Connect to a remote HTTPS server")
+    {
+        HttpClient httpClient;
+        WebSocketHttpHeaders headers;
+
+        std::string url("https://httpbin.org/");
+        auto args = httpClient.createRequest(url);
+
+        args->extraHeaders = headers;
+        args->connectTimeout = 60;
+        args->transferTimeout = 60;
+        args->followRedirects = true;
+        args->maxRedirects = 10;
+        args->verbose = true;
+        args->compress = true;
+        args->logger = [](const std::string& msg)
+        {
+            std::cout << msg;
+        };
+        args->onProgressCallback = [](int current, int total) -> bool
+        {
+            std::cerr << "\r" << "Downloaded "
+                      << current << " bytes out of " << total;
+            return true;
+        };
+
+        auto response = httpClient.get(url, args);
+
+        for (auto it : response->headers)
+        {
+            std::cerr << it.first << ": " << it.second << std::endl;
+        }
+
+        std::cerr << "Upload size: " << response->uploadSize << std::endl;
+        std::cerr << "Download size: " << response->downloadSize << std::endl;
+        std::cerr << "Status: " << response->statusCode << std::endl;
+        std::cerr << "Error message: " << response->errorMsg << std::endl;
+
+        REQUIRE(response->errorCode == HttpErrorCode::Ok);
+        REQUIRE(response->statusCode == 200);
+    }
+
+    SECTION("Async API, one call")
+    {
+        bool async = true;
+        HttpClient httpClient(async);
+        WebSocketHttpHeaders headers;
+
+        std::string url("https://httpbin.org/");
+        auto args = httpClient.createRequest(url);
+
+        args->extraHeaders = headers;
+        args->connectTimeout = 60;
+        args->transferTimeout = 60;
+        args->followRedirects = true;
+        args->maxRedirects = 10;
+        args->verbose = true;
+        args->compress = true;
+        args->logger = [](const std::string& msg)
+        {
+            std::cout << msg;
+        };
+        args->onProgressCallback = [](int current, int total) -> bool
+        {
+            std::cerr << "\r" << "Downloaded "
+                      << current << " bytes out of " << total;
+            return true;
+        };
+
+        std::atomic<bool> requestCompleted(false);
+        std::atomic<int> statusCode(0);
+
+        httpClient.performRequest(args, [&requestCompleted, &statusCode]
+            (const HttpResponsePtr& response)
+            {
+                std::cerr << "Upload size: " << response->uploadSize << std::endl;
+                std::cerr << "Download size: " << response->downloadSize << std::endl;
+                std::cerr << "Status: " << response->statusCode << std::endl;
+                std::cerr << "Error message: " << response->errorMsg << std::endl;
+
+                // In case of failure, print response->errorMsg
+                statusCode = response->statusCode;
+                requestCompleted = true;
+            }
+        );
+
+        int wait = 0;
+        while (wait < 5000)
+        {
+            if (requestCompleted) break;
+
+            std::chrono::duration<double, std::milli> duration(10);
+            std::this_thread::sleep_for(duration);
+            wait += 10;
+        }
+
+        std::cerr << "Done" << std::endl;
+        REQUIRE(statusCode == 200);
+    }
+
+    SECTION("Async API, multiple calls")
+    {
+        bool async = true;
+        HttpClient httpClient(async);
+        WebSocketHttpHeaders headers;
+
+        std::string url("http://httpbin.org/");
+        auto args = httpClient.createRequest(url);
+
+        args->extraHeaders = headers;
+        args->connectTimeout = 60;
+        args->transferTimeout = 60;
+        args->followRedirects = true;
+        args->maxRedirects = 10;
+        args->verbose = true;
+        args->compress = true;
+        args->logger = [](const std::string& msg)
+        {
+            std::cout << msg;
+        };
+        args->onProgressCallback = [](int current, int total) -> bool
+        {
+            std::cerr << "\r" << "Downloaded "
+                      << current << " bytes out of " << total;
+            return true;
+        };
+
+        std::atomic<bool> requestCompleted(false);
+        std::atomic<int> statusCode0(0);
+        std::atomic<int> statusCode1(0);
+        std::atomic<int> statusCode2(0);
+
+        for (int i = 0; i < 3; ++i)
+        {
+            httpClient.performRequest(args, [i, &requestCompleted, &statusCode0, &statusCode1, &statusCode2]
+                (const HttpResponsePtr& response)
+                {
+                    std::cerr << "Upload size: " << response->uploadSize << std::endl;
+                    std::cerr << "Download size: " << response->downloadSize << std::endl;
+                    std::cerr << "Status: " << response->statusCode << std::endl;
+                    std::cerr << "Error message: " << response->errorMsg << std::endl;
+
+                    // In case of failure, print response->errorMsg
+                    if (i == 0)
+                    {
+                        statusCode0 = response->statusCode;
+                    }
+                    else if (i == 1)
+                    {
+                        statusCode1 = response->statusCode;
+                    }
+                    else if (i == 2)
+                    {
+                        statusCode2 = response->statusCode;
+                        requestCompleted = true;
+                    }
+                }
+            );
+        }
+
+        int wait = 0;
+        while (wait < 10000)
+        {
+            if (requestCompleted) break;
+
+            std::chrono::duration<double, std::milli> duration(10);
+            std::this_thread::sleep_for(duration);
+            wait += 10;
+        }
+
+        std::cerr << "Done" << std::endl;
+        REQUIRE(statusCode0 == 200);
+        REQUIRE(statusCode1 == 200);
+        REQUIRE(statusCode2 == 200);
+    }
+}

ws/IXSentryClient.cpp

@@ -8,6 +8,7 @@
 
 #include <chrono>
 #include <iostream>
+#include <spdlog/spdlog.h>
 
 #include <ixwebsocket/IXWebSocketHttpHeaders.h>
 
@@ -88,7 +89,7 @@ namespace ix
                 ss >> lineno;
 
                 Json::Value frame;
-                frame["lineno"] = lineno;
+                frame["lineno"] = Json::UInt64(lineno);
                 frame["filename"] = fileName;
                 frame["function"] = function;
 
@@ -114,6 +115,7 @@ namespace ix
     std::string SentryClient::computePayload(const Json::Value& msg)
     {
         Json::Value payload;
+
         payload["platform"] = "python";
         payload["sdk"]["name"] = "ws";
         payload["sdk"]["version"] = "1.0.0";
@@ -132,59 +134,77 @@ namespace ix
 
         Json::Value extra;
         extra["cobra_event"] = msg;
+        extra["cobra_event"] = msg;
 
-        exception["extra"] = extra;
+        //
+        // "tags": [
+        //   [
+        //     "a",
+        //     "b"
+        //   ],
+        //  ]
+        // 
+        Json::Value tags;
+
+        Json::Value gameTag;
+        gameTag.append("game");
+        gameTag.append(msg["device"]["game"]);
+        tags.append(gameTag);
+
+        Json::Value userIdTag;
+        userIdTag.append("userid");
+        userIdTag.append(msg["device"]["user_id"]);
+        tags.append(userIdTag);
+
+        Json::Value environmentTag;
+        environmentTag.append("environment");
+        environmentTag.append(msg["device"]["environment"]);
+        tags.append(environmentTag);
+
+        payload["tags"] = tags;
 
         return _jsonWriter.write(payload);
     }
 
-    bool SentryClient::send(const Json::Value& msg,
+    std::pair<HttpResponsePtr, std::string> SentryClient::send(const Json::Value& msg,
                                                                bool verbose)
     {
-        HttpRequestArgs args;
-        args.extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader();
-        args.connectTimeout = 60;
-        args.transferTimeout = 5 * 60;
-        args.followRedirects = true;
-        args.verbose = verbose;
-        args.logger = [](const std::string& msg)
+        auto args = _httpClient.createRequest();
+        args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader();
+        args->connectTimeout = 60;
+        args->transferTimeout = 5 * 60;
+        args->followRedirects = true;
+        args->verbose = verbose;
+        args->logger = [](const std::string& msg)
         {
-            std::cout << msg;
+            spdlog::info("request logger: {}", msg);
         };
 
         std::string body = computePayload(msg);
-        HttpResponse out = _httpClient.post(_url, body, args);
-
-        auto statusCode = std::get<0>(out);
-        auto errorCode = std::get<1>(out);
-        auto responseHeaders = std::get<2>(out);
-        auto payload = std::get<3>(out);
-        auto errorMsg = std::get<4>(out);
-        auto uploadSize = std::get<5>(out);
-        auto downloadSize = std::get<6>(out);
+        HttpResponsePtr response = _httpClient.post(_url, body, args);
 
         if (verbose)
         {
-            for (auto it : responseHeaders)
+            for (auto it : response->headers)
             {
-                std::cerr << it.first << ": " << it.second << std::endl;
+                spdlog::info("{}: {}", it.first, it.second);
             }
 
-            std::cerr << "Upload size: " << uploadSize << std::endl;
-            std::cerr << "Download size: " << downloadSize << std::endl;
+            spdlog::info("Upload size: {}", response->uploadSize);
+            spdlog::info("Download size: {}", response->downloadSize);
 
-            std::cerr << "Status: " << statusCode << std::endl;
-            if (errorCode != HttpErrorCode::Ok)
+            spdlog::info("Status: {}", response->statusCode);
+            if (response->errorCode != HttpErrorCode::Ok)
             {
-                std::cerr << "error message: " << errorMsg << std::endl;
+                spdlog::info("error message: {}", response->errorMsg);
             }
 
-            if (responseHeaders["Content-Type"] != "application/octet-stream")
+            if (response->headers["Content-Type"] != "application/octet-stream")
             {
-                std::cerr << "payload: " << payload << std::endl;
+                spdlog::info("payload: {}", response->payload);
             }
         }
 
-        return statusCode == 200;
+        return std::make_pair(response, body);
     }
 } // namespace ix

ws/IXSentryClient.h

@@ -9,6 +9,7 @@
 #include <ixwebsocket/IXHttpClient.h>
 #include <jsoncpp/json/json.h>
 #include <regex>
+#include <algorithm>
 
 namespace ix
 {
@@ -18,7 +19,7 @@ namespace ix
         SentryClient(const std::string& dsn);
         ~SentryClient() = default;
 
-        bool send(const Json::Value& msg, bool verbose);
+        std::pair<HttpResponsePtr, std::string> send(const Json::Value& msg, bool verbose);
 
     private:
         int64_t getTimestamp();

ws/ixcobra/IXCobraMetricsPublisher.cpp

@@ -184,7 +184,7 @@ namespace ix
         msg["data"] = data;
         msg["session"] = _session;
         msg["version"] = kVersion;
-        msg["timestamp"] = getMillisecondsSinceEpoch();
+        msg["timestamp"] = Json::UInt64(getMillisecondsSinceEpoch());
 
         {
             std::lock_guard<std::mutex> lock(_device_mutex);

ws/ixcobra/IXCobraMetricsThreadedPublisher.cpp

@@ -99,7 +99,7 @@ namespace ix
     void CobraMetricsThreadedPublisher::pushMessage(MessageKind messageKind,
                                                     const Json::Value& msg)
     {
-        // Now actually enqueue the task
+        // Enqueue the task
         {
             // acquire lock
             std::unique_lock<std::mutex> lock(_queue_mutex);

ws/ixcobra/IXCobraMetricsThreadedPublisher.h

@@ -25,7 +25,6 @@ namespace ix
         ~CobraMetricsThreadedPublisher();
 
         /// Configuration / set keys, etc...
-        /// All input data but the channel name is encrypted with rc4
         void configure(const std::string& appkey,
                        const std::string& endpoint,
                        const std::string& channel,

ws/ixcrypto/IXHMac.cpp

@@ -10,7 +10,6 @@
 #if defined(IXWEBSOCKET_USE_MBED_TLS)
 # include <mbedtls/md.h>
 #elif defined(__APPLE__)
-#  include <ixwebsocket/IXSocketMbedTLS.h>
 # include <CommonCrypto/CommonHMAC.h>
 #else
 # include <openssl/hmac.h>

ws/ws_cobra_to_sentry.cpp

@@ -14,6 +14,7 @@
 #include <mutex>
 #include <condition_variable>
 #include <ixcobra/IXCobraConnection.h>
+#include <spdlog/spdlog.h>
 
 #include "IXSentryClient.h"
 
@@ -50,6 +51,8 @@ namespace ix
                              &queue, verbose, &errorSending, &sentCount,
                              &stop, &dsn]
         {
+            SentryClient sentryClient(dsn);
+
             while (true)
             {
                 Json::Value msg;
@@ -62,10 +65,13 @@ namespace ix
                     queue.pop();
                 }
 
-                SentryClient sc(dsn);
-
-                if (!sc.send(msg, verbose))
+                auto ret = sentryClient.send(msg, verbose);
+                HttpResponsePtr response = ret.first;
+                if (response->statusCode != 200)
                 {
+                    spdlog::error("Error sending data to sentry: {}", response->statusCode);
+                    spdlog::error("Body: {}", ret.second);
+                    spdlog::error("Response: {}", response->payload);
                     errorSending = true;
                 }
                 else
@@ -99,16 +105,16 @@ namespace ix
             {
                 if (eventType == ix::CobraConnection_EventType_Open)
                 {
-                    std::cerr << "Subscriber: connected" << std::endl;
+                    spdlog::info("Subscriber connected");
 
                     for (auto it : headers)
                     {
-                        std::cerr << it.first << ": " << it.second << std::endl;
+                        spdlog::info("{}: {}", it.first, it.second);
                     }
                 }
                 if (eventType == ix::CobraConnection_EventType_Closed)
                 {
-                    std::cerr << "Subscriber: closed" << std::endl;
+                    spdlog::info("Subscriber closed");
                 }
                 else if (eventType == ix::CobraConnection_EventType_Authenticated)
                 {
@@ -122,7 +128,7 @@ namespace ix
                                    {
                                        if (verbose)
                                        {
-                                           std::cerr << jsonWriter.write(msg) << std::endl;
+                                           spdlog::info(jsonWriter.write(msg));
                                        }
 
                                        // If we cannot send to sentry fast enough, drop the message
@@ -132,8 +138,7 @@ namespace ix
                                            receivedCount != 0 &&
                                            (sentCount * scaleFactor < receivedCount))
                                        {
-                                           std::cerr << "message dropped: sending is backlogged !"
-                                                     << std::endl;
+                                           spdlog::warn("message dropped: sending is backlogged !");
 
                                            condition.notify_one();
                                            progressCondition.notify_one();
@@ -153,15 +158,15 @@ namespace ix
                 }
                 else if (eventType == ix::CobraConnection_EventType_Subscribed)
                 {
-                    std::cerr << "Subscriber: subscribed to channel " << subscriptionId << std::endl;
+                    spdlog::info("Subscriber: subscribed to channel {}", subscriptionId);
                 }
                 else if (eventType == ix::CobraConnection_EventType_UnSubscribed)
                 {
-                    std::cerr << "Subscriber: unsubscribed from channel " << subscriptionId << std::endl;
+                    spdlog::info("Subscriber: unsubscribed from channel {}", subscriptionId);
                 }
                 else if (eventType == ix::CobraConnection_EventType_Error)
                 {
-                    std::cerr << "Subscriber: error" << errMsg << std::endl;
+                    spdlog::error("Subscriber: error {}", errMsg);
                 }
             }
         );
@@ -172,18 +177,21 @@ namespace ix
             std::unique_lock<std::mutex> lock(progressConditionVariableMutex);
             progressCondition.wait(lock);
 
-            std::cout << "messages"
-                      << " received " << receivedCount
-                      << " sent " << sentCount
-                      << std::endl;
+            spdlog::info("messages received {} sent {}", receivedCount, sentCount);
 
             if (strict && errorSending) break;
         }
 
         conn.disconnect();
 
-        // FIXME: join all the bg threads and stop them.
+        // join all the bg threads and stop them.
+        stop = true;
+        for (int i = 0; i < jobs; i++)
+        {
+            spdlog::error("joining thread {}", i);
+            pool[i].join();
+        }
 
-        return 0;
+        return (strict && errorSending) ? 1 : 0;
     }
 }

ws/ws_http_client.cpp

@@ -95,19 +95,20 @@ namespace ix
                             const std::string& output,
                             bool compress)
     {
-        HttpRequestArgs args;
-        args.extraHeaders = parseHeaders(headersData);
-        args.connectTimeout = connectTimeout;
-        args.transferTimeout = transferTimeout;
-        args.followRedirects = followRedirects;
-        args.maxRedirects = maxRedirects;
-        args.verbose = verbose;
-        args.compress = compress;
-        args.logger = [](const std::string& msg)
+        HttpClient httpClient;
+        auto args = httpClient.createRequest();
+        args->extraHeaders = parseHeaders(headersData);
+        args->connectTimeout = connectTimeout;
+        args->transferTimeout = transferTimeout;
+        args->followRedirects = followRedirects;
+        args->maxRedirects = maxRedirects;
+        args->verbose = verbose;
+        args->compress = compress;
+        args->logger = [](const std::string& msg)
         {
             std::cout << msg;
         };
-        args.onProgressCallback = [](int current, int total) -> bool
+        args->onProgressCallback = [](int current, int total) -> bool
         {
             std::cerr << "\r" << "Downloaded "
                       << current << " bytes out of " << total;
@@ -116,46 +117,37 @@ namespace ix
 
         HttpParameters httpParameters = parsePostParameters(data);
 
-        HttpClient httpClient;
-        HttpResponse out;
+        HttpResponsePtr response;
         if (headersOnly)
         {
-            out = httpClient.head(url, args);
+            response = httpClient.head(url, args);
         }
         else if (data.empty())
         {
-            out = httpClient.get(url, args);
+            response = httpClient.get(url, args);
         }
         else
         {
-            out = httpClient.post(url, httpParameters, args);
+            response = httpClient.post(url, httpParameters, args);
         }
 
         std::cerr << std::endl;
 
-        auto statusCode = std::get<0>(out);
-        auto errorCode = std::get<1>(out);
-        auto responseHeaders = std::get<2>(out);
-        auto payload = std::get<3>(out);
-        auto errorMsg = std::get<4>(out);
-        auto uploadSize = std::get<5>(out);
-        auto downloadSize = std::get<6>(out);
-
-        for (auto it : responseHeaders)
+        for (auto it : response->headers)
         {
             std::cerr << it.first << ": " << it.second << std::endl;
         }
 
-        std::cerr << "Upload size: " << uploadSize << std::endl;
-        std::cerr << "Download size: " << downloadSize << std::endl;
+        std::cerr << "Upload size: " << response->uploadSize << std::endl;
+        std::cerr << "Download size: " << response->downloadSize << std::endl;
 
-        std::cerr << "Status: " << statusCode << std::endl;
-        if (errorCode != HttpErrorCode::Ok)
+        std::cerr << "Status: " << response->statusCode << std::endl;
+        if (response->errorCode != HttpErrorCode::Ok)
         {
-            std::cerr << "error message: " << errorMsg << std::endl;
+            std::cerr << "error message: " << response->errorMsg << std::endl;
         }
 
-        if (!headersOnly && errorCode == HttpErrorCode::Ok)
+        if (!headersOnly && response->errorCode == HttpErrorCode::Ok)
         {
             if (save || !output.empty())
             {
@@ -168,14 +160,14 @@ namespace ix
 
                 std::cout << "Writing to disk: " << filename << std::endl;
                 std::ofstream out(filename);
-                out.write((char*)&payload.front(), payload.size());
+                out.write((char*)&response->payload.front(), response->payload.size());
                 out.close();
             }
             else
             {
-                if (responseHeaders["Content-Type"] != "application/octet-stream")
+                if (response->headers["Content-Type"] != "application/octet-stream")
                 {
-                    std::cout << "payload: " << payload << std::endl;
+                    std::cout << "payload: " << response->payload << std::endl;
                 }
                 else
                 {