bump version

CobraConnection: sets a counter as a field for each event published.

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+
+-   repo: https://github.com/pocc/pre-commit-hooks
+    rev: ''
+    hooks:
+    -   id: clang-format

.travis.yml

@@ -15,15 +15,17 @@ matrix:
         - python test/run.py
         - make ws
 
-    # # Linux
+    # Linux
-    # - os: linux
+    - os: linux
-    #   dist: xenial
+      dist: bionic
-    #   script: 
+      before_install:
-    #     - python test/run.py
+        - sudo apt-get install -y libmbedtls-dev
-    #     - make ws
+      script: 
-    #   env:
+        - python test/run.py
-    #     - CC=gcc
+        - make ws
-    #     - CXX=g++
+      env:
+        - CC=gcc
+        - CXX=g++
 
     # Clang + Linux disabled for now
     # - os: linux

CHANGELOG.md

@@ -1,6 +1,27 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [5.0.7] - 2019-08-23
+- WebSocket: add new option to pass in extra HTTP headers when connecting.
+- `ws connect` add new option (-H, works like [curl](https://stackoverflow.com/questions/356705/how-to-send-a-header-using-a-http-request-through-a-curl-call)) to pass in extra HTTP headers when connecting 
+
+If you run against `ws echo_server` you will see the headers being received printed in the terminal.
+```
+ws connect -H "foo: bar" -H "baz: buz" ws://127.0.0.1:8008
+```
+
+- CobraConnection: sets a unique id field for all messages sent to [cobra](https://github.com/machinezone/cobra).
+- CobraConnection: sets a counter as a field for each event published.
+
+## [5.0.6] - 2019-08-22
+- Windows: silly compile error (poll should be in the global namespace)
+
+## [5.0.5] - 2019-08-22
+- Windows: use select instead of WSAPoll, through a poll wrapper
+
+## [5.0.4] - 2019-08-20
+- Windows build fixes (there was a problem with the use of ::poll that has a different name on Windows (WSAPoll))
+
 ## [5.0.3] - 2019-08-14
 - CobraMetricThreadedPublisher _enable flag is an atomic, and CobraMetricsPublisher is enabled by default
 

DOCKER_VERSION

@@ -1 +1 @@
-5.0.3
+5.0.7

README.md

@@ -16,9 +16,32 @@
 
 The [*ws*](https://github.com/machinezone/IXWebSocket/tree/master/ws) folder countains many interactive programs for chat, [file transfers](https://github.com/machinezone/IXWebSocket/blob/master/ws/ws_send.cpp), [curl like](https://github.com/machinezone/IXWebSocket/blob/master/ws/ws_http_client.cpp) http clients, demonstrating client and server usage.
 
+### Windows note
+
+To use the network system on Windows, you need to initialize it once with *WSAStartup()* and clean it up with *WSACleanup()*. We have helpers for that which you can use, see below. This init would typically take place in your main function.
+
+```
+#include <ixwebsocket/IXNetSystem.h>
+
+int main()
+{
+    ix::initNetSystem();
+
+    ...
+
+    ix::uninitNetSystem();
+    return 0;
+}
+```
+
 ### WebSocket client API
 
 ```
+#include <ixwebsocket/IXWebSocket.h>
+
+...
+
+# Our websocket object
 ix::WebSocket webSocket;
 
 std::string url("ws://localhost:8080/");
@@ -59,6 +82,10 @@ webSocket.stop()
 ### WebSocket server API
 
 ```
+#include <ixwebsocket/IXWebSocketServer.h>
+
+...
+
 // Run a server on localhost at a given port.
 // Bound host name, max connections and listen backlog can also be passed in as parameters.
 ix::WebSocketServer server(port);
@@ -120,6 +147,10 @@ server.wait();
 ### HTTP client API
 
 ```
+#include <ixwebsocket/IXHttpClient.h>
+
+...
+
 //
 // Preparation
 //
@@ -199,6 +230,8 @@ bool ok = httpClient.performRequest(args, [](const HttpResponsePtr& response)
 ### HTTP server API
 
 ```
+#include <ixwebsocket/IXHttpServer.h>
+
 ix::HttpServer server(port, hostname);
 
 auto res = server.listen();
@@ -477,3 +510,13 @@ idle connection.
 ```
 webSocket.setHeartBeatPeriod(45);
 ```
+
+### Supply extra HTTP headers.
+
+You can set extra HTTP headers to be sent during the WebSocket handshake.
+
+```
+WebSocketHttpHeaders headers;
+headers["foo"] = "bar";
+webSocket.setExtraHeaders(headers);
+```

ixwebsocket/IXNetSystem.cpp

@@ -15,9 +15,8 @@ namespace ix
         WSADATA wsaData;
         int err;
 
-        /* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */
+        // Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h
         wVersionRequested = MAKEWORD(2, 2);
-
         err = WSAStartup(wVersionRequested, &wsaData);
 
         return err == 0;
@@ -30,10 +29,83 @@ namespace ix
     {
 #ifdef _WIN32
         int err = WSACleanup();
-
         return err == 0;
 #else
         return true;
 #endif
     }
 }
+
+// This function should be in the global namespace
+#ifdef _WIN32
+    //
+    // That function could 'return WSAPoll(pfd, nfds, timeout);'
+    // but WSAPoll is said to have weird behaviors on the internet
+    // (the curl folks have had problems with it).
+    //
+    // So we make it a select wrapper
+    //
+    int poll(struct pollfd *fds, nfds_t nfds, int timeout)
+    {
+        int maxfd = 0;
+        fd_set readfds, writefds, errorfds;
+        FD_ZERO(&readfds);
+        FD_ZERO(&writefds);
+        FD_ZERO(&errorfds);
+
+        for (nfds_t i = 0; i < nfds; ++i)
+        {
+            struct pollfd *fd = &fds[i];
+
+            if (fd->fd > maxfd)
+            {
+                maxfd = fd->fd;
+            }
+            if ((fd->events & POLLIN))
+            {
+                FD_SET(fd->fd, &readfds);
+            }
+            if ((fd->events & POLLOUT))
+            {
+                FD_SET(fd->fd, &writefds);
+            }
+            if ((fd->events & POLLERR))
+            {
+                FD_SET(fd->fd, &errorfds);
+            }
+        }
+
+        struct timeval tv;
+        tv.tv_sec = timeout / 1000;
+        tv.tv_usec = (timeout % 1000) * 1000;
+
+        int ret = select(maxfd + 1, &readfds, &writefds, &errorfds,
+                         timeout != -1 ? &tv : NULL);
+
+        if (ret < 0) 
+        {
+            return ret;
+        }
+
+        for (nfds_t i = 0; i < nfds; ++i)
+        {
+            struct pollfd *fd = &fds[i];
+            fd->revents = 0;
+
+            if (FD_ISSET(fd->fd, &readfds))
+            {
+                fd->revents |= POLLIN;
+            }
+            if (FD_ISSET(fd->fd, &writefds))
+            {
+                fd->revents |= POLLOUT;
+            }
+            if (FD_ISSET(fd->fd, &errorfds))
+            {
+                fd->revents |= POLLERR;
+            }
+        }
+
+        return ret;
+    }
+#endif

ixwebsocket/IXNetSystem.h

@@ -12,11 +12,18 @@
 #include <basetsd.h>
 #include <io.h>
 #include <ws2def.h>
+
+// Define our own poll on Windows
+typedef unsigned long int nfds_t;
+
+int poll(struct pollfd* fds, nfds_t nfds, int timeout);
+
 #else
 #include <arpa/inet.h>
 #include <errno.h>
 #include <netdb.h>
 #include <netinet/tcp.h>
+#include <poll.h>
 #include <sys/select.h>
 #include <sys/socket.h>
 #include <sys/stat.h>

ixwebsocket/IXSocket.cpp

@@ -17,7 +17,6 @@
 #include <stdint.h>
 #include <fcntl.h>
 #include <sys/types.h>
-#include <poll.h>
 
 #include <algorithm>
 
@@ -59,7 +58,7 @@ namespace ix
         // shim to fallback to select on those platforms.
         // See https://github.com/mpv-player/mpv/pull/5203/files for such a select wrapper.
         //
-        int nfds = 1;
+        nfds_t nfds = 1;
         struct pollfd fds[2];
 
         fds[0].fd = sockfd;

ixwebsocket/IXWebSocket.cpp

@@ -70,6 +70,11 @@ namespace ix
         std::lock_guard<std::mutex> lock(_configMutex);
         _url = url;
     }
+    void WebSocket::setExtraHeaders(const WebSocketHttpHeaders& headers)
+    {
+        std::lock_guard<std::mutex> lock(_configMutex);
+        _extraHeaders = headers;
+    }
 
     const std::string& WebSocket::getUrl() const
     {
@@ -176,7 +181,7 @@ namespace ix
                           _pingTimeoutSecs);
         }
 
-        WebSocketInitResult status = _ws.connectToUrl(_url, timeoutSecs);
+        WebSocketInitResult status = _ws.connectToUrl(_url, _extraHeaders, timeoutSecs);
         if (!status.success)
         {
             return status;

ixwebsocket/IXWebSocket.h

@@ -44,6 +44,9 @@ namespace ix
         ~WebSocket();
 
         void setUrl(const std::string& url);
+
+        // send extra headers in client handshake request
+        void setExtraHeaders(const WebSocketHttpHeaders& headers);
         void setPerMessageDeflateOptions(
             const WebSocketPerMessageDeflateOptions& perMessageDeflateOptions);
         void setHeartBeatPeriod(int heartBeatPeriodSecs);
@@ -111,6 +114,8 @@ namespace ix
         WebSocketTransport _ws;
 
         std::string _url;
+        WebSocketHttpHeaders _extraHeaders;
+
         WebSocketPerMessageDeflateOptions _perMessageDeflateOptions;
         mutable std::mutex _configMutex; // protect all config variables access
 

ixwebsocket/IXWebSocketHandshake.cpp

@@ -88,6 +88,7 @@ namespace ix
     }
 
     WebSocketInitResult WebSocketHandshake::clientHandshake(const std::string& url,
+                                                            const WebSocketHttpHeaders& extraHeaders,
                                                             const std::string& host,
                                                             const std::string& path,
                                                             int port,
@@ -127,6 +128,9 @@ namespace ix
         ss << "Sec-WebSocket-Version: 13\r\n";
         ss << "Sec-WebSocket-Key: " << secWebSocketKey << "\r\n";
 
+        for (auto& it : extraHeaders) {
+            ss << it.first << ":" << it.second << "\r\n";
+        }
         if (_enablePerMessageDeflate)
         {
             ss << _perMessageDeflateOptions.generateHeader();

ixwebsocket/IXWebSocketHandshake.h

@@ -16,6 +16,7 @@
 #include <memory>
 #include <string>
 #include <tuple>
+#include <unordered_map>
 
 namespace ix
 {
@@ -50,11 +51,13 @@ namespace ix
                            WebSocketPerMessageDeflateOptions& perMessageDeflateOptions,
                            std::atomic<bool>& enablePerMessageDeflate);
 
-        WebSocketInitResult clientHandshake(const std::string& url,
+        WebSocketInitResult clientHandshake(
-                                            const std::string& host,
+            const std::string& url,
-                                            const std::string& path,
+            const WebSocketHttpHeaders& extraHeaders,
-                                            int port,
+            const std::string& host,
-                                            int timeoutSecs);
+            const std::string& path,
+            int port,
+            int timeoutSecs);
 
         WebSocketInitResult serverHandshake(int fd, int timeoutSecs);
 

ixwebsocket/IXWebSocketTransport.cpp

@@ -127,8 +127,10 @@ namespace ix
     }
 
     // Client
-    WebSocketInitResult WebSocketTransport::connectToUrl(const std::string& url,
+    WebSocketInitResult WebSocketTransport::connectToUrl(
-                                                         int timeoutSecs)
+        const std::string& url,
+        const WebSocketHttpHeaders& headers,
+        int timeoutSecs)
     {
         std::lock_guard<std::mutex> lock(_socketMutex);
 
@@ -156,8 +158,8 @@ namespace ix
                                               _perMessageDeflateOptions,
                                               _enablePerMessageDeflate);
 
-        auto result = webSocketHandshake.clientHandshake(url, host, path, port,
+        auto result = webSocketHandshake.clientHandshake(url, headers, host, path,
-                                                         timeoutSecs);
+                                                         port, timeoutSecs);
         if (result.success)
         {
             setReadyState(ReadyState::OPEN);

ixwebsocket/IXWebSocketTransport.h

@@ -24,6 +24,7 @@
 #include <memory>
 #include <mutex>
 #include <string>
+
 #include <vector>
 
 namespace ix
@@ -75,8 +76,10 @@ namespace ix
                        int pingIntervalSecs,
                        int pingTimeoutSecs);
 
-        WebSocketInitResult connectToUrl(const std::string& url, // Client
+        WebSocketInitResult connectToUrl( // Client
-                                         int timeoutSecs);
+            const std::string& url,
+            const WebSocketHttpHeaders& headers,
+            int timeoutSecs);
         WebSocketInitResult connectToSocket(int fd, // Server
                                             int timeoutSecs);
 

makefile

@@ -17,6 +17,9 @@ ws:
 uninstall:
 	xargs rm -fv < build/install_manifest.txt
 
+tag:
+	git tag v"`cat DOCKER_VERSION`"
+
 .PHONY: docker
 
 NAME   := bsergean/ws

ws/ixcobra/IXCobraConnection.cpp

@@ -24,7 +24,8 @@ namespace ix
         _webSocket(new WebSocket()),
         _publishMode(CobraConnection_PublishMode_Immediate),
         _authenticated(false),
-        _eventCallback(nullptr)
+        _eventCallback(nullptr),
+        _id(0)
     {
         _pdu["action"] = "rtm/publish";
 
@@ -244,6 +245,7 @@ namespace ix
         Json::Value pdu;
         pdu["action"] = "auth/handshake";
         pdu["body"] = body;
+        pdu["id"] = _id++;
 
         std::string serializedJson = serializeJson(pdu);
         CobraConnection::invokeTrafficTrackerCallback(serializedJson.size(), false);
@@ -306,6 +308,7 @@ namespace ix
         Json::Value pdu;
         pdu["action"] = "auth/authenticate";
         pdu["body"] = body;
+        pdu["id"] = _id++;
 
         std::string serializedJson = serializeJson(pdu);
         CobraConnection::invokeTrafficTrackerCallback(serializedJson.size(), false);
@@ -402,6 +405,7 @@ namespace ix
         _body["channels"] = channels;
         _body["message"] = msg;
         _pdu["body"] = _body;
+        _pdu["id"] = _id++;
 
         std::string serializedJson = serializeJson(_pdu);
 
@@ -444,6 +448,7 @@ namespace ix
         Json::Value pdu;
         pdu["action"] = "rtm/subscribe";
         pdu["body"] = body;
+        pdu["id"] = _id++;
 
         _webSocket->send(pdu.toStyledString());
 
@@ -469,6 +474,7 @@ namespace ix
         Json::Value pdu;
         pdu["action"] = "rtm/unsubscribe";
         pdu["body"] = body;
+        pdu["id"] = _id++;
 
         _webSocket->send(pdu.toStyledString());
     }

ws/ixcobra/IXCobraConnection.h

@@ -168,6 +168,9 @@ namespace ix
 
         // Cap the queue size (100 elems so far -> ~100k)
         static constexpr size_t kQueueMaxSize = 256;
+
+        // Each pdu sent should have an incremental unique id
+        std::atomic<uint64_t> _id;
     };
 
 } // namespace ix

ws/ixcobra/IXCobraMetricsPublisher.cpp

@@ -191,6 +191,19 @@ namespace ix
             msg["device"] = _device;
         }
 
+        {
+            //
+            // Bump a counter for each id
+            // This is used to make sure that we are not
+            // dropping messages, by checking that all the ids is the list of
+            // all natural numbers until the last value sent (0, 1, 2, ..., N)
+            //
+            std::lock_guard<std::mutex> lock(_device_mutex);
+            auto it = _counters.emplace(id, 0);
+            msg["per_id_counter"] = it.first->second;
+            it.first->second += 1;
+        }
+
         // Now actually enqueue the task
         _cobra_metrics_theaded_publisher.push(msg);
     }

ws/ixcobra/IXCobraMetricsPublisher.h

@@ -7,10 +7,10 @@
 #pragma once
 
 #include "IXCobraMetricsThreadedPublisher.h"
+#include <atomic>
 #include <chrono>
 #include <jsoncpp/json/json.h>
 #include <string>
-#include <atomic>
 #include <unordered_map>
 
 namespace ix
@@ -151,6 +151,10 @@ namespace ix
             _last_update;
         mutable std::mutex _last_update_mutex; // protect access to _last_update
 
+        /// Bump a counter for each metric type
+        std::unordered_map<std::string, int> _counters;
+        mutable std::mutex _counters_mutex; // protect access to _counters
+
         // const strings for internal ids
         static const std::string kSetRateControlId;
         static const std::string kSetBlacklistId;

ws/ws.cpp

@@ -109,6 +109,7 @@ int main(int argc, char** argv)
 
     CLI::App* connectApp = app.add_subcommand("connect", "Connect to a remote server");
     connectApp->add_option("url", url, "Connection url")->required();
+    connectApp->add_option("-H", headers, "Header")->join();
     connectApp->add_flag("-d", disableAutomaticReconnection, "Disable Automatic Reconnection");
     connectApp->add_flag("-x", disablePerMessageDeflate, "Disable per message deflate");
     connectApp->add_flag("-b", binaryMode, "Send in binary mode");
@@ -252,7 +253,7 @@ int main(int argc, char** argv)
     }
     else if (app.got_subcommand("connect"))
     {
-        ret = ix::ws_connect_main(url, disableAutomaticReconnection,
+        ret = ix::ws_connect_main(url, headers, disableAutomaticReconnection,
                                   disablePerMessageDeflate, binaryMode);
     }
     else if (app.got_subcommand("chat"))

ws/ws.h

@@ -31,6 +31,7 @@ namespace ix
     int ws_chat_main(const std::string& url, const std::string& user);
 
     int ws_connect_main(const std::string& url,
+                        const std::string& headers,
                         bool disableAutomaticReconnection,
                         bool disablePerMessageDeflate,
                         bool binaryMode);

ws/ws_connect.cpp

@@ -18,6 +18,7 @@ namespace ix
     {
         public:
             WebSocketConnect(const std::string& _url,
+                             const std::string& headers,
                              bool disableAutomaticReconnection,
                              bool disablePerMessageDeflate,
                              bool binaryMode);
@@ -30,14 +31,17 @@ namespace ix
 
         private:
             std::string _url;
+            WebSocketHttpHeaders _headers;
             ix::WebSocket _webSocket;
             bool _disablePerMessageDeflate;
             bool _binaryMode;
 
             void log(const std::string& msg);
+            WebSocketHttpHeaders parseHeaders(const std::string& data);
     };
 
     WebSocketConnect::WebSocketConnect(const std::string& url,
+                                       const std::string& headers,
                                        bool disableAutomaticReconnection,
                                        bool disablePerMessageDeflate,
                                        bool binaryMode) :
@@ -49,6 +53,8 @@ namespace ix
         {
             _webSocket.disableAutomaticReconnection();
         }
+
+        _headers = parseHeaders(headers);
     }
 
     void WebSocketConnect::log(const std::string& msg)
@@ -56,6 +62,31 @@ namespace ix
         std::cout << msg << std::endl;
     }
 
+    WebSocketHttpHeaders WebSocketConnect::parseHeaders(const std::string& data)
+    {
+        WebSocketHttpHeaders headers;
+
+        // Split by \n
+        std::string token;
+        std::stringstream tokenStream(data);
+
+        while (std::getline(tokenStream, token))
+        {
+            std::size_t pos = token.rfind(':');
+
+            // Bail out if last '.' is found
+            if (pos == std::string::npos) continue;
+
+            auto key = token.substr(0, pos);
+            auto val = token.substr(pos+1);
+
+            std::cerr << key << ": " << val << std::endl;
+            headers[key] = val;
+        }
+
+        return headers;
+    }
+
     void WebSocketConnect::stop()
     {
         _webSocket.stop();
@@ -64,6 +95,7 @@ namespace ix
     void WebSocketConnect::start()
     {
         _webSocket.setUrl(_url);
+        _webSocket.setExtraHeaders(_headers);
 
         if (_disablePerMessageDeflate)
         {
@@ -151,12 +183,14 @@ namespace ix
     }
 
     int ws_connect_main(const std::string& url,
+                        const std::string& headers,
                         bool disableAutomaticReconnection,
                         bool disablePerMessageDeflate,
                         bool binaryMode)
     {
         std::cout << "Type Ctrl-D to exit prompt..." << std::endl;
         WebSocketConnect webSocketChat(url,
+                                       headers,
                                        disableAutomaticReconnection,
                                        disablePerMessageDeflate,
                                        binaryMode);

ws/ws_http_client.cpp

@@ -44,7 +44,7 @@ namespace ix
             if (pos == std::string::npos) continue;
 
             auto key = token.substr(0, pos);
-            auto val = token.substr(pos+2);
+            auto val = token.substr(pos+1);
 
             std::cerr << key << ": " << val << std::endl;
             headers[key] = val;