From b89621fa7833b067ef082014c7c77cf479f09352 Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Fri, 25 Dec 2020 15:32:15 -0800 Subject: [PATCH] remove ixbots / ixsnake / ixcobra / ixredis (which should go in their own standalone project --- docs/cobra.md | 81 -- docs/ws.md | 142 +--- ixbots/CMakeLists.txt | 59 -- ixbots/ixbots/IXCobraBot.cpp | 326 -------- ixbots/ixbots/IXCobraBot.h | 36 - ixbots/ixbots/IXCobraBotConfig.h | 32 - ixbots/ixbots/IXCobraMetricsToRedisBot.cpp | 149 ---- ixbots/ixbots/IXCobraMetricsToRedisBot.h | 20 - ixbots/ixbots/IXCobraToCobraBot.cpp | 45 -- ixbots/ixbots/IXCobraToCobraBot.h | 20 - ixbots/ixbots/IXCobraToPythonBot.cpp | 329 -------- ixbots/ixbots/IXCobraToPythonBot.h | 19 - ixbots/ixbots/IXCobraToSentryBot.cpp | 76 -- ixbots/ixbots/IXCobraToSentryBot.h | 18 - ixbots/ixbots/IXCobraToStatsdBot.cpp | 143 ---- ixbots/ixbots/IXCobraToStatsdBot.h | 22 - ixbots/ixbots/IXCobraToStdoutBot.cpp | 88 --- ixbots/ixbots/IXCobraToStdoutBot.h | 18 - ixbots/ixbots/IXStatsdClient.cpp | 161 ---- ixbots/ixbots/IXStatsdClient.h | 59 -- ixcobra/CMakeLists.txt | 37 - ixcobra/ixcobra/IXCobraConfig.h | 37 - ixcobra/ixcobra/IXCobraConnection.cpp | 713 ------------------ ixcobra/ixcobra/IXCobraConnection.h | 224 ------ ixcobra/ixcobra/IXCobraEvent.h | 44 -- ixcobra/ixcobra/IXCobraEventType.h | 26 - ixcobra/ixcobra/IXCobraMetricsPublisher.cpp | 232 ------ ixcobra/ixcobra/IXCobraMetricsPublisher.h | 175 ----- .../IXCobraMetricsThreadedPublisher.cpp | 240 ------ .../ixcobra/IXCobraMetricsThreadedPublisher.h | 101 --- ixcobra/ixcobra/README.md | 1 - ixredis/CMakeLists.txt | 27 - ixredis/ixredis/IXRedisClient.cpp | 457 ----------- ixredis/ixredis/IXRedisClient.h | 77 -- ixredis/ixredis/IXRedisServer.cpp | 287 ------- ixredis/ixredis/IXRedisServer.h | 65 -- ixsentry/CMakeLists.txt | 44 -- ixsentry/ixsentry/IXSentryClient.cpp | 316 -------- ixsentry/ixsentry/IXSentryClient.h | 74 -- ixsnake/CMakeLists.txt | 34 - ixsnake/ixsnake/IXAppConfig.cpp | 54 -- ixsnake/ixsnake/IXAppConfig.h | 48 -- ixsnake/ixsnake/IXSnakeConnectionState.h | 86 --- ixsnake/ixsnake/IXSnakeProtocol.cpp | 320 -------- ixsnake/ixsnake/IXSnakeProtocol.h | 26 - ixsnake/ixsnake/IXSnakeServer.cpp | 147 ---- ixsnake/ixsnake/IXSnakeServer.h | 31 - ixsnake/ixsnake/IXStreamSql.cpp | 63 -- ixsnake/ixsnake/IXStreamSql.h | 29 - ixsnake/ixsnake/appsConfig.json | 14 - 50 files changed, 3 insertions(+), 5869 deletions(-) delete mode 100644 docs/cobra.md delete mode 100644 ixbots/CMakeLists.txt delete mode 100644 ixbots/ixbots/IXCobraBot.cpp delete mode 100644 ixbots/ixbots/IXCobraBot.h delete mode 100644 ixbots/ixbots/IXCobraBotConfig.h delete mode 100644 ixbots/ixbots/IXCobraMetricsToRedisBot.cpp delete mode 100644 ixbots/ixbots/IXCobraMetricsToRedisBot.h delete mode 100644 ixbots/ixbots/IXCobraToCobraBot.cpp delete mode 100644 ixbots/ixbots/IXCobraToCobraBot.h delete mode 100644 ixbots/ixbots/IXCobraToPythonBot.cpp delete mode 100644 ixbots/ixbots/IXCobraToPythonBot.h delete mode 100644 ixbots/ixbots/IXCobraToSentryBot.cpp delete mode 100644 ixbots/ixbots/IXCobraToSentryBot.h delete mode 100644 ixbots/ixbots/IXCobraToStatsdBot.cpp delete mode 100644 ixbots/ixbots/IXCobraToStatsdBot.h delete mode 100644 ixbots/ixbots/IXCobraToStdoutBot.cpp delete mode 100644 ixbots/ixbots/IXCobraToStdoutBot.h delete mode 100644 ixbots/ixbots/IXStatsdClient.cpp delete mode 100644 ixbots/ixbots/IXStatsdClient.h delete mode 100644 ixcobra/CMakeLists.txt delete mode 100644 ixcobra/ixcobra/IXCobraConfig.h delete mode 100644 ixcobra/ixcobra/IXCobraConnection.cpp delete mode 100644 ixcobra/ixcobra/IXCobraConnection.h delete mode 100644 ixcobra/ixcobra/IXCobraEvent.h delete mode 100644 ixcobra/ixcobra/IXCobraEventType.h delete mode 100644 ixcobra/ixcobra/IXCobraMetricsPublisher.cpp delete mode 100644 ixcobra/ixcobra/IXCobraMetricsPublisher.h delete mode 100644 ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp delete mode 100644 ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h delete mode 100644 ixcobra/ixcobra/README.md delete mode 100644 ixredis/CMakeLists.txt delete mode 100644 ixredis/ixredis/IXRedisClient.cpp delete mode 100644 ixredis/ixredis/IXRedisClient.h delete mode 100644 ixredis/ixredis/IXRedisServer.cpp delete mode 100644 ixredis/ixredis/IXRedisServer.h delete mode 100644 ixsentry/CMakeLists.txt delete mode 100644 ixsentry/ixsentry/IXSentryClient.cpp delete mode 100644 ixsentry/ixsentry/IXSentryClient.h delete mode 100644 ixsnake/CMakeLists.txt delete mode 100644 ixsnake/ixsnake/IXAppConfig.cpp delete mode 100644 ixsnake/ixsnake/IXAppConfig.h delete mode 100644 ixsnake/ixsnake/IXSnakeConnectionState.h delete mode 100644 ixsnake/ixsnake/IXSnakeProtocol.cpp delete mode 100644 ixsnake/ixsnake/IXSnakeProtocol.h delete mode 100644 ixsnake/ixsnake/IXSnakeServer.cpp delete mode 100644 ixsnake/ixsnake/IXSnakeServer.h delete mode 100644 ixsnake/ixsnake/IXStreamSql.cpp delete mode 100644 ixsnake/ixsnake/IXStreamSql.h delete mode 100644 ixsnake/ixsnake/appsConfig.json diff --git a/docs/cobra.md b/docs/cobra.md deleted file mode 100644 index fd195774..00000000 --- a/docs/cobra.md +++ /dev/null @@ -1,81 +0,0 @@ -## General - -[cobra](https://github.com/machinezone/cobra) is a real time messaging server. The `ws` utility can run a cobra server (named snake), and has client to publish and subscribe to a cobra server. - -Bring up 3 terminals and run a server, a publisher and a subscriber in each one. As you publish data you should see it being received by the subscriber. You can run `redis-cli MONITOR` too to see how redis is being used. - -### Server - -You will need to have a redis server running locally. To run the server: - -```bash -$ cd /ixsnake/ixsnake -$ ws snake -{ - "apps": { - "FC2F10139A2BAc53BB72D9db967b024f": { - "roles": { - "_sub": { - "secret": "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba" - }, - "_pub": { - "secret": "1c04DB8fFe76A4EeFE3E318C72d771db" - } - } - } - } -} - -redis host: 127.0.0.1 -redis password: -redis port: 6379 -``` - -### Publisher - -```bash -$ cd /ws -$ ws cobra_publish --appkey FC2F10139A2BAc53BB72D9db967b024f --endpoint ws://127.0.0.1:8008 --rolename _pub --rolesecret 1c04DB8fFe76A4EeFE3E318C72d771db test_channel cobraMetricsSample.json -[2019-11-27 09:06:12.980] [info] Publisher connected -[2019-11-27 09:06:12.980] [info] Connection: Upgrade -[2019-11-27 09:06:12.980] [info] Sec-WebSocket-Accept: zTtQKMKbvwjdivURplYXwCVUCWM= -[2019-11-27 09:06:12.980] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -[2019-11-27 09:06:12.980] [info] Server: ixwebsocket/7.4.0 macos ssl/DarwinSSL zlib 1.2.11 -[2019-11-27 09:06:12.980] [info] Upgrade: websocket -[2019-11-27 09:06:12.982] [info] Publisher authenticated -[2019-11-27 09:06:12.982] [info] Published msg 3 -[2019-11-27 09:06:12.982] [info] Published message id 3 acked -``` - -### Subscriber - -```bash -$ ws cobra_subscribe --appkey FC2F10139A2BAc53BB72D9db967b024f --endpoint ws://127.0.0.1:8008 --rolename _pub --rolesecret 1c04DB8fFe76A4EeFE3E318C72d771db test_channel -#messages 0 msg/s 0 -[2019-11-27 09:07:39.341] [info] Subscriber connected -[2019-11-27 09:07:39.341] [info] Connection: Upgrade -[2019-11-27 09:07:39.341] [info] Sec-WebSocket-Accept: 9vkQWofz49qMCUlTSptCCwHWm+Q= -[2019-11-27 09:07:39.341] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -[2019-11-27 09:07:39.341] [info] Server: ixwebsocket/7.4.0 macos ssl/DarwinSSL zlib 1.2.11 -[2019-11-27 09:07:39.341] [info] Upgrade: websocket -[2019-11-27 09:07:39.342] [info] Subscriber authenticated -[2019-11-27 09:07:39.345] [info] Subscriber: subscribed to channel test_channel -#messages 0 msg/s 0 -#messages 0 msg/s 0 -#messages 0 msg/s 0 -{"baz":123,"foo":"bar"} - -#messages 1 msg/s 1 -#messages 1 msg/s 0 -#messages 1 msg/s 0 -{"baz":123,"foo":"bar"} - -{"baz":123,"foo":"bar"} - -#messages 3 msg/s 2 -#messages 3 msg/s 0 -{"baz":123,"foo":"bar"} - -#messages 4 msg/s 1 -^C -``` diff --git a/docs/ws.md b/docs/ws.md index 75d11115..348e2baf 100644 --- a/docs/ws.md +++ b/docs/ws.md @@ -19,13 +19,6 @@ Subcommands: broadcast_server Broadcasting server ping Ping pong curl HTTP Client - redis_publish Redis publisher - redis_subscribe Redis subscriber - cobra_subscribe Cobra subscriber - cobra_publish Cobra publisher - cobra_to_statsd Cobra to statsd - cobra_to_sentry Cobra to sentry - snake Snake server httpd HTTP server ``` @@ -265,13 +258,9 @@ You can also use a more complex setup if you want to redirect to different webso A JSON config file is used to express that mapping ; here connecting to echo.jeanserge.com will proxy the client to ws://localhost:8008 on the local machine (which actually runs ws echo_server), while connecting to bavarde.jeanserge.com will proxy the client to ws://localhost:5678 where a cobra python server is running. As a side note you will need a wildcard SSL certificate if you want to have SSL enabled on that machine. -```json -{ - "remote_urls": { - "echo.jeanserge.com": "ws://localhost:8008", - "bavarde.jeanserge.com": "ws://localhost:5678" - } -} +``` +echo.jeanserge.com=ws://localhost:8008 +bavarde.jeanserge.com=ws://localhost:5678 ``` The --config_path option is required to instruct ws proxy_server to read that file. @@ -317,128 +306,3 @@ Options: --connect-timeout INT Connection timeout --transfer-timeout INT Transfer timeout ``` - -## Cobra client and server - -[cobra](https://github.com/machinezone/cobra) is a real time messenging server. ws has several sub-command to interact with cobra. There is also a minimal cobra compatible server named snake available. - -Below are examples on running a snake server and clients with TLS enabled (the server only works with the OpenSSL and the Mbed TLS backend for now). - -First, generate certificates. - -``` -$ cd /path/to/IXWebSocket -$ cd ixsnake/ixsnake -$ bash ../../ws/generate_certs.sh -Generating RSA private key, 2048 bit long modulus -.....+++ -.................+++ -e is 65537 (0x10001) -generated ./.certs/trusted-ca-key.pem -generated ./.certs/trusted-ca-crt.pem -Generating RSA private key, 2048 bit long modulus -..+++ -.......................................+++ -e is 65537 (0x10001) -generated ./.certs/trusted-server-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=trusted-server -Getting CA Private Key -generated ./.certs/trusted-server-crt.pem -Generating RSA private key, 2048 bit long modulus -...................................+++ -..................................................+++ -e is 65537 (0x10001) -generated ./.certs/trusted-client-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=trusted-client -Getting CA Private Key -generated ./.certs/trusted-client-crt.pem -Generating RSA private key, 2048 bit long modulus -..............+++ -.......................................+++ -e is 65537 (0x10001) -generated ./.certs/untrusted-ca-key.pem -generated ./.certs/untrusted-ca-crt.pem -Generating RSA private key, 2048 bit long modulus -..........+++ -................................................+++ -e is 65537 (0x10001) -generated ./.certs/untrusted-client-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=untrusted-client -Getting CA Private Key -generated ./.certs/untrusted-client-crt.pem -Generating RSA private key, 2048 bit long modulus -.....................................................................................+++ -...........+++ -e is 65537 (0x10001) -generated ./.certs/selfsigned-client-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=selfsigned-client -Getting Private key -generated ./.certs/selfsigned-client-crt.pem -``` - -Now run the snake server. - -``` -$ export certs=.certs -$ ws snake --tls --port 8765 --cert-file ${certs}/trusted-server-crt.pem --key-file ${certs}/trusted-server-key.pem --ca-file ${certs}/trusted-ca-crt.pem -{ - "apps": { - "FC2F10139A2BAc53BB72D9db967b024f": { - "roles": { - "_sub": { - "secret": "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba" - }, - "_pub": { - "secret": "1c04DB8fFe76A4EeFE3E318C72d771db" - } - } - } - } -} - -redis host: 127.0.0.1 -redis password: -redis port: 6379 -``` - -As a new connection comes in, such output should be printed - -``` -[2019-12-19 20:27:19.724] [info] New connection -id: 0 -Uri: /v2?appkey=_health -Headers: -Connection: Upgrade -Host: 127.0.0.1:8765 -Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -Sec-WebSocket-Key: d747B0fE61Db73f7Eh47c0== -Sec-WebSocket-Protocol: json -Sec-WebSocket-Version: 13 -Upgrade: websocket -User-Agent: ixwebsocket/7.5.8 macos ssl/OpenSSL OpenSSL 1.0.2q 20 Nov 2018 zlib 1.2.11 -``` - -To connect and publish a message, do: - -``` -$ export certs=.certs -$ cd /path/to/ws/folder -$ ls cobraMetricsSample.json -cobraMetricsSample.json -$ ws cobra_publish --endpoint wss://127.0.0.1:8765 --appkey FC2F10139A2BAc53BB72D9db967b024f --rolename _pub --rolesecret 1c04DB8fFe76A4EeFE3E318C72d771db --channel foo --cert-file ${certs}/trusted-client-crt.pem --key-file ${certs}/trusted-client-key.pem --ca-file ${certs}/trusted-ca-crt.pem cobraMetricsSample.json -[2019-12-19 20:46:42.656] [info] Publisher connected -[2019-12-19 20:46:42.657] [info] Connection: Upgrade -[2019-12-19 20:46:42.657] [info] Sec-WebSocket-Accept: rs99IFThoBrhSg+k8G4ixH9yaq4= -[2019-12-19 20:46:42.657] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -[2019-12-19 20:46:42.657] [info] Server: ixwebsocket/7.5.8 macos ssl/OpenSSL OpenSSL 1.0.2q 20 Nov 2018 zlib 1.2.11 -[2019-12-19 20:46:42.657] [info] Upgrade: websocket -[2019-12-19 20:46:42.658] [info] Publisher authenticated -[2019-12-19 20:46:42.658] [info] Published msg 3 -[2019-12-19 20:46:42.659] [info] Published message id 3 acked -``` - -To use OpenSSL on macOS, compile with `make ws_openssl`. First you will have to install OpenSSL libraries, which can be done with Homebrew. Use `make ws_mbedtls` accordingly to use MbedTLS. diff --git a/ixbots/CMakeLists.txt b/ixbots/CMakeLists.txt deleted file mode 100644 index 423b88b7..00000000 --- a/ixbots/CMakeLists.txt +++ /dev/null @@ -1,59 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXBOTS_SOURCES - ixbots/IXCobraBot.cpp - ixbots/IXCobraToCobraBot.cpp - ixbots/IXCobraToSentryBot.cpp - ixbots/IXCobraToStatsdBot.cpp - ixbots/IXCobraToStdoutBot.cpp - ixbots/IXCobraMetricsToRedisBot.cpp - ixbots/IXCobraToPythonBot.cpp - ixbots/IXStatsdClient.cpp -) - -set (IXBOTS_HEADERS - ixbots/IXCobraBot.h - ixbots/IXCobraBotConfig.h - ixbots/IXCobraToCobraBot.h - ixbots/IXCobraToSentryBot.h - ixbots/IXCobraToStatsdBot.h - ixbots/IXCobraToStdoutBot.h - ixbots/IXCobraMetricsToRedisBot.h - ixbots/IXCobraToPythonBot.h - ixbots/IXStatsdClient.h -) - -add_library(ixbots STATIC - ${IXBOTS_SOURCES} - ${IXBOTS_HEADERS} -) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) -endif() - -if (USE_PYTHON) - target_compile_definitions(ixbots PUBLIC IXBOTS_USE_PYTHON) - find_package(Python COMPONENTS Development) -endif() - -set(IXBOTS_INCLUDE_DIRS - . - .. - ../ixcore - ../ixwebsocket - ../ixcobra - ../ixredis - ../ixsentry - ${JSONCPP_INCLUDE_DIRS} - ${SPDLOG_INCLUDE_DIRS}) - -if (USE_PYTHON) - set(IXBOTS_INCLUDE_DIRS ${IXBOTS_INCLUDE_DIRS} ${Python_INCLUDE_DIRS}) -endif() - -target_include_directories( ixbots PUBLIC ${IXBOTS_INCLUDE_DIRS} ) diff --git a/ixbots/ixbots/IXCobraBot.cpp b/ixbots/ixbots/IXCobraBot.cpp deleted file mode 100644 index f75dd4b9..00000000 --- a/ixbots/ixbots/IXCobraBot.cpp +++ /dev/null @@ -1,326 +0,0 @@ -/* - * IXCobraBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraBot.h" - -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace ix -{ - int64_t CobraBot::run(const CobraBotConfig& botConfig) - { - auto config = botConfig.cobraConfig; - auto channel = botConfig.channel; - auto filter = botConfig.filter; - auto position = botConfig.position; - auto enableHeartbeat = botConfig.enableHeartbeat; - auto heartBeatTimeout = botConfig.heartBeatTimeout; - auto runtime = botConfig.runtime; - auto maxEventsPerMinute = botConfig.maxEventsPerMinute; - auto limitReceivedEvents = botConfig.limitReceivedEvents; - auto batchSize = botConfig.batchSize; - - config.headers["X-Cobra-Channel"] = channel; - - ix::CobraConnection conn; - conn.configure(config); - conn.connect(); - - std::atomic sentCount(0); - std::atomic receivedCount(0); - uint64_t sentCountTotal(0); - uint64_t receivedCountTotal(0); - uint64_t sentCountPerSecs(0); - uint64_t receivedCountPerSecs(0); - std::atomic receivedCountPerMinutes(0); - std::atomic stop(false); - std::atomic throttled(false); - std::atomic fatalCobraError(false); - std::atomic stalledConnection(false); - int minuteCounter = 0; - - auto timer = [&sentCount, - &receivedCount, - &sentCountTotal, - &receivedCountTotal, - &sentCountPerSecs, - &receivedCountPerSecs, - &receivedCountPerMinutes, - &minuteCounter, - &conn, - &stop] { - setThreadName("Bot progress"); - while (!stop) - { - // - // We cannot write to sentCount and receivedCount - // as those are used externally, so we need to introduce - // our own counters - // - std::stringstream ss; - ss << "messages received " - << receivedCountPerSecs - << " " - << receivedCountTotal - << " sent " - << sentCountPerSecs - << " " - << sentCountTotal; - - if (conn.isAuthenticated()) - { - CoreLogger::info(ss.str()); - } - - receivedCountPerSecs = receivedCount - receivedCountTotal; - sentCountPerSecs = sentCount - sentCountTotal; - - receivedCountTotal += receivedCountPerSecs; - sentCountTotal += sentCountPerSecs; - - auto duration = std::chrono::seconds(1); - std::this_thread::sleep_for(duration); - - if (minuteCounter++ == 60) - { - receivedCountPerMinutes = 0; - minuteCounter = 0; - } - } - - CoreLogger::info("timer thread done"); - }; - - std::thread t1(timer); - - auto heartbeat = [&sentCount, - &receivedCount, - &stop, - &enableHeartbeat, - &heartBeatTimeout, - &stalledConnection] - { - setThreadName("Bot heartbeat"); - std::string state("na"); - - if (!enableHeartbeat) return; - - while (!stop) - { - std::stringstream ss; - ss << "messages received " << receivedCount; - ss << "messages sent " << sentCount; - - std::string currentState = ss.str(); - - if (currentState == state) - { - ss.str(""); - ss << "no messages received or sent for " - << heartBeatTimeout << " seconds, reconnecting"; - - CoreLogger::warn(ss.str()); - stalledConnection = true; - } - state = currentState; - - auto duration = std::chrono::seconds(heartBeatTimeout); - std::this_thread::sleep_for(duration); - } - - CoreLogger::info("heartbeat thread done"); - }; - - std::thread t2(heartbeat); - - std::string subscriptionPosition(position); - - conn.setEventCallback([this, - &conn, - &channel, - &filter, - &subscriptionPosition, - &throttled, - &receivedCount, - &receivedCountPerMinutes, - maxEventsPerMinute, - limitReceivedEvents, - batchSize, - &fatalCobraError, - &sentCount](const CobraEventPtr& event) { - if (event->type == ix::CobraEventType::Open) - { - CoreLogger::info("Subscriber connected"); - - for (auto&& it : event->headers) - { - CoreLogger::info(it.first + ": " + it.second); - } - } - else if (event->type == ix::CobraEventType::Closed) - { - CoreLogger::info("Subscriber closed: " + event->errMsg); - } - else if (event->type == ix::CobraEventType::Handshake) - { - CoreLogger::info("Subscriber: Cobra handshake connection id: " + event->connectionId); - } - else if (event->type == ix::CobraEventType::Authenticated) - { - CoreLogger::info("Subscriber authenticated"); - CoreLogger::info("Subscribing to " + channel); - CoreLogger::info("Subscribing at position " + subscriptionPosition); - CoreLogger::info("Subscribing with filter " + filter); - conn.subscribe(channel, filter, subscriptionPosition, batchSize, - [&sentCount, &receivedCountPerMinutes, - maxEventsPerMinute, limitReceivedEvents, - &throttled, &receivedCount, - &subscriptionPosition, &fatalCobraError, - this](const Json::Value& msg, const std::string& position) { - subscriptionPosition = position; - ++receivedCount; - - ++receivedCountPerMinutes; - if (limitReceivedEvents) - { - if (receivedCountPerMinutes > maxEventsPerMinute) - { - return; - } - } - - // If we cannot send to sentry fast enough, drop the message - if (throttled) - { - return; - } - - _onBotMessageCallback( - msg, position, throttled, - fatalCobraError, sentCount); - }); - } - else if (event->type == ix::CobraEventType::Subscribed) - { - CoreLogger::info("Subscriber: subscribed to channel " + event->subscriptionId); - } - else if (event->type == ix::CobraEventType::UnSubscribed) - { - CoreLogger::info("Subscriber: unsubscribed from channel " + event->subscriptionId); - } - else if (event->type == ix::CobraEventType::Error) - { - CoreLogger::error("Subscriber: error " + event->errMsg); - } - else if (event->type == ix::CobraEventType::Published) - { - CoreLogger::error("Published message hacked: " + std::to_string(event->msgId)); - } - else if (event->type == ix::CobraEventType::Pong) - { - CoreLogger::info("Received websocket pong: " + event->errMsg); - } - else if (event->type == ix::CobraEventType::HandshakeError) - { - CoreLogger::error("Subscriber: Handshake error: " + event->errMsg); - fatalCobraError = true; - } - else if (event->type == ix::CobraEventType::AuthenticationError) - { - CoreLogger::error("Subscriber: Authentication error: " + event->errMsg); - fatalCobraError = true; - } - else if (event->type == ix::CobraEventType::SubscriptionError) - { - CoreLogger::error("Subscriber: Subscription error: " + event->errMsg); - fatalCobraError = true; - } - }); - - // Run forever - if (runtime == -1) - { - while (true) - { - auto duration = std::chrono::seconds(1); - std::this_thread::sleep_for(duration); - - if (fatalCobraError) break; - - if (stalledConnection) - { - conn.disconnect(); - conn.connect(); - stalledConnection = false; - } - } - } - // Run for a duration, used by unittesting now - else - { - for (int i = 0; i < runtime; ++i) - { - auto duration = std::chrono::seconds(1); - std::this_thread::sleep_for(duration); - - if (fatalCobraError) break; - - if (stalledConnection) - { - conn.disconnect(); - conn.connect(); - stalledConnection = false; - } - } - } - - // - // Cleanup. - // join all the bg threads and stop them. - // - conn.disconnect(); - stop = true; - - // progress thread - t1.join(); - - // heartbeat thread - if (t2.joinable()) t2.join(); - - return fatalCobraError ? -1 : (int64_t) sentCount; - } - - void CobraBot::setOnBotMessageCallback(const OnBotMessageCallback& callback) - { - _onBotMessageCallback = callback; - } - - std::string CobraBot::getDeviceIdentifier(const Json::Value& msg) - { - std::string deviceId("na"); - - auto osName = msg["device"]["os_name"]; - if (osName == "Android") - { - deviceId = msg["device"]["model"].asString(); - } - else if (osName == "iOS") - { - deviceId = msg["device"]["hardware_model"].asString(); - } - - return deviceId; - } - -} // namespace ix diff --git a/ixbots/ixbots/IXCobraBot.h b/ixbots/ixbots/IXCobraBot.h deleted file mode 100644 index 199da104..00000000 --- a/ixbots/ixbots/IXCobraBot.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * IXCobraBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - using OnBotMessageCallback = std::function&, - std::atomic&, - std::atomic&)>; - - class CobraBot - { - public: - CobraBot() = default; - - int64_t run(const CobraBotConfig& botConfig); - void setOnBotMessageCallback(const OnBotMessageCallback& callback); - - std::string getDeviceIdentifier(const Json::Value& msg); - - private: - OnBotMessageCallback _onBotMessageCallback; - }; -} // namespace ix diff --git a/ixbots/ixbots/IXCobraBotConfig.h b/ixbots/ixbots/IXCobraBotConfig.h deleted file mode 100644 index 4783206c..00000000 --- a/ixbots/ixbots/IXCobraBotConfig.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * IXCobraBotConfig.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include - -#ifdef max -#undef max -#endif - -namespace ix -{ - struct CobraBotConfig - { - CobraConfig cobraConfig; - std::string channel; - std::string filter; - std::string position = std::string("$"); - bool enableHeartbeat = true; - int heartBeatTimeout = 60; - int runtime = -1; - int maxEventsPerMinute = std::numeric_limits::max(); - bool limitReceivedEvents = false; - int batchSize = 1; - }; -} // namespace ix diff --git a/ixbots/ixbots/IXCobraMetricsToRedisBot.cpp b/ixbots/ixbots/IXCobraMetricsToRedisBot.cpp deleted file mode 100644 index 6f97fe56..00000000 --- a/ixbots/ixbots/IXCobraMetricsToRedisBot.cpp +++ /dev/null @@ -1,149 +0,0 @@ -/* - * IXCobraMetricsToRedisBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraMetricsToRedisBot.h" - -#include "IXCobraBot.h" -#include "IXStatsdClient.h" -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace -{ - std::string removeSpaces(const std::string& str) - { - std::string out(str); - out.erase( - std::remove_if(out.begin(), out.end(), [](unsigned char x) { return std::isspace(x); }), - out.end()); - - return out; - } -} - -namespace ix -{ - bool processPerfMetricsEventSlowFrames(const Json::Value& msg, - RedisClient& redisClient, - const std::string& deviceId) - { - auto frameRateHistogramCounts = msg["data"]["FrameRateHistogramCounts"]; - - int slowFrames = 0; - slowFrames += frameRateHistogramCounts[4].asInt(); - slowFrames += frameRateHistogramCounts[5].asInt(); - slowFrames += frameRateHistogramCounts[6].asInt(); - slowFrames += frameRateHistogramCounts[7].asInt(); - - // - // XADD without a device id - // - std::stringstream ss; - ss << msg["id"].asString() << "_slow_frames" << "." - << msg["device"]["game"].asString() << "." - << msg["device"]["os_name"].asString() << "." - << removeSpaces(msg["data"]["Tag"].asString()); - - int maxLen; - maxLen = 100000; - std::string id = ss.str(); - std::string errMsg; - if (redisClient.xadd(id, std::to_string(slowFrames), maxLen, errMsg).empty()) - { - CoreLogger::info(std::string("redis XADD error: ") + errMsg); - } - - // - // XADD with a device id - // - ss.str(""); // reset the stringstream - ss << msg["id"].asString() << "_slow_frames_by_device" << "." - << deviceId << "." - << msg["device"]["game"].asString() << "." - << msg["device"]["os_name"].asString() << "." - << removeSpaces(msg["data"]["Tag"].asString()); - - id = ss.str(); - maxLen = 1000; - if (redisClient.xadd(id, std::to_string(slowFrames), maxLen, errMsg).empty()) - { - CoreLogger::info(std::string("redis XADD error: ") + errMsg); - } - - // - // Add device to the device zset, and increment the score - // so that we know which devices are used more than others - // ZINCRBY myzset 1 one - // - ss.str(""); // reset the stringstream - ss << msg["id"].asString() << "_slow_frames_devices" << "." - << msg["device"]["game"].asString(); - - id = ss.str(); - std::vector args = { - "ZINCRBY", id, "1", deviceId - }; - auto response = redisClient.send(args, errMsg); - if (response.first == RespType::Error) - { - CoreLogger::info(std::string("redis ZINCRBY error: ") + errMsg); - } - - return true; - } - - int64_t cobra_metrics_to_redis_bot(const ix::CobraBotConfig& config, - RedisClient& redisClient, - bool verbose) - { - CobraBot bot; - - bot.setOnBotMessageCallback( - [&redisClient, &verbose, &bot] - (const Json::Value& msg, - const std::string& /*position*/, - std::atomic& /*throttled*/, - std::atomic& /*fatalCobraError*/, - std::atomic& sentCount) -> void { - if (msg["device"].isNull()) - { - CoreLogger::info("no device entry, skipping event"); - return; - } - - if (msg["id"].isNull()) - { - CoreLogger::info("no id entry, skipping event"); - return; - } - - // - // Display full message with - if (verbose) - { - CoreLogger::info(msg.toStyledString()); - } - - bool success = false; - if (msg["id"].asString() == "engine_performance_metrics_id") - { - auto deviceId = bot.getDeviceIdentifier(msg); - success = processPerfMetricsEventSlowFrames(msg, redisClient, deviceId); - } - - if (success) sentCount++; - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraMetricsToRedisBot.h b/ixbots/ixbots/IXCobraMetricsToRedisBot.h deleted file mode 100644 index 5fbe3905..00000000 --- a/ixbots/ixbots/IXCobraMetricsToRedisBot.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * IXCobraMetricsToRedisBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_metrics_to_redis_bot(const ix::CobraBotConfig& config, - RedisClient& redisClient, - bool verbose); -} // namespace ix - diff --git a/ixbots/ixbots/IXCobraToCobraBot.cpp b/ixbots/ixbots/IXCobraToCobraBot.cpp deleted file mode 100644 index 45848a7a..00000000 --- a/ixbots/ixbots/IXCobraToCobraBot.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * IXCobraToCobraBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToCobraBot.h" - -#include "IXCobraBot.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_cobra_bot(const ix::CobraBotConfig& cobraBotConfig, - const std::string& republishChannel, - const std::string& publisherRolename, - const std::string& publisherRolesecret) - { - CobraBot bot; - - CobraMetricsPublisher cobraMetricsPublisher; - CobraConfig cobraPublisherConfig = cobraBotConfig.cobraConfig; - cobraPublisherConfig.rolename = publisherRolename; - cobraPublisherConfig.rolesecret = publisherRolesecret; - cobraPublisherConfig.headers["X-Cobra-Republish-Channel"] = republishChannel; - - cobraMetricsPublisher.configure(cobraPublisherConfig, republishChannel); - - bot.setOnBotMessageCallback( - [&republishChannel, &cobraMetricsPublisher](const Json::Value& msg, - const std::string& /*position*/, - std::atomic& /*throttled*/, - std::atomic& /*fatalCobraError*/, - std::atomic& sentCount) -> void { - Json::Value msgWithNoId(msg); - msgWithNoId.removeMember("id"); - - cobraMetricsPublisher.push(republishChannel, msg); - sentCount++; - }); - - return bot.run(cobraBotConfig); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToCobraBot.h b/ixbots/ixbots/IXCobraToCobraBot.h deleted file mode 100644 index 7d32dbc4..00000000 --- a/ixbots/ixbots/IXCobraToCobraBot.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * IXCobraToCobraBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_cobra_bot(const ix::CobraBotConfig& config, - const std::string& republishChannel, - const std::string& publisherRolename, - const std::string& publisherRolesecret); -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToPythonBot.cpp b/ixbots/ixbots/IXCobraToPythonBot.cpp deleted file mode 100644 index 7f8fc5d3..00000000 --- a/ixbots/ixbots/IXCobraToPythonBot.cpp +++ /dev/null @@ -1,329 +0,0 @@ -/* - * IXCobraToPythonBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToPythonBot.h" - -#include "IXCobraBot.h" -#include "IXStatsdClient.h" -#include -#include -#include -#include -#include -#include -#include -#include - -// -// I cannot get Windows to easily build on CI (github action) so support -// is disabled for now. It should be a simple fix -// (linking error about missing debug build) -// - -#ifdef IXBOTS_USE_PYTHON -#define PY_SSIZE_T_CLEAN -#include -#endif - -#ifdef IXBOTS_USE_PYTHON -namespace -{ - // - // This function is unused at this point. It produce a correct output, - // but triggers memory leaks when called repeateadly, as I cannot figure out how to - // make the reference counting Python functions to work properly (Py_DECREF and friends) - // - PyObject* jsonToPythonObject(const Json::Value& val) - { - switch(val.type()) - { - case Json::nullValue: - { - return Py_None; - } - - case Json::intValue: - { - return PyLong_FromLong(val.asInt64()); - } - - case Json::uintValue: - { - return PyLong_FromLong(val.asUInt64()); - } - - case Json::realValue: - { - return PyFloat_FromDouble(val.asDouble()); - } - - case Json::stringValue: - { - return PyUnicode_FromString(val.asCString()); - } - - case Json::booleanValue: - { - return val.asBool() ? Py_True : Py_False; - } - - case Json::arrayValue: - { - PyObject* list = PyList_New(val.size()); - Py_ssize_t i = 0; - for (auto&& it = val.begin(); it != val.end(); ++it) - { - PyList_SetItem(list, i++, jsonToPythonObject(*it)); - } - return list; - } - - case Json::objectValue: - { - PyObject* dict = PyDict_New(); - for (auto&& it = val.begin(); it != val.end(); ++it) - { - PyObject* key = jsonToPythonObject(it.key()); - PyObject* value = jsonToPythonObject(*it); - - PyDict_SetItem(dict, key, value); - } - return dict; - } - } - } -} -#endif - -namespace ix -{ - int64_t cobra_to_python_bot(const ix::CobraBotConfig& config, - StatsdClient& statsdClient, - const std::string& moduleName) - { -#ifndef IXBOTS_USE_PYTHON - CoreLogger::error("Command is disabled. " - "Needs to be configured with USE_PYTHON=1"); - return -1; -#else - CobraBot bot; - Py_InitializeEx(0); // 0 arg so that we do not install signal handlers - // which prevent us from using Ctrl-C - - PyObject* pyModuleName = PyUnicode_DecodeFSDefault(moduleName.c_str()); - - if (pyModuleName == nullptr) - { - CoreLogger::error("Python error: Cannot decode file system path"); - PyErr_Print(); - return false; - } - - // Import module - PyObject* pyModule = PyImport_Import(pyModuleName); - Py_DECREF(pyModuleName); - if (pyModule == nullptr) - { - CoreLogger::error("Python error: Cannot import module."); - CoreLogger::error("Module name cannot countain dash characters."); - CoreLogger::error("Is PYTHONPATH set correctly ?"); - PyErr_Print(); - return false; - } - - // module main funtion name is named 'run' - const std::string entryPoint("run"); - PyObject* pyFunc = PyObject_GetAttrString(pyModule, entryPoint.c_str()); - - if (!pyFunc) - { - CoreLogger::error("run symbol is missing from module."); - PyErr_Print(); - return false; - } - - if (!PyCallable_Check(pyFunc)) - { - CoreLogger::error("run symbol is not a function."); - PyErr_Print(); - return false; - } - - bot.setOnBotMessageCallback( - [&statsdClient, pyFunc] - (const Json::Value& msg, - const std::string& /*position*/, - std::atomic& /*throttled*/, - std::atomic& fatalCobraError, - std::atomic& sentCount) -> void { - // - // Invoke python script here. First build function parameters, a tuple - // - const int kVersion = 1; // We can bump this and let the interface evolve - - PyObject *pyArgs = PyTuple_New(2); - PyTuple_SetItem(pyArgs, 0, PyLong_FromLong(kVersion)); // First argument - - // - // It would be better to create a Python object (a dictionary) - // from the json msg, but it is simpler to serialize it to a string - // and decode it on the Python side of the fence - // - PyObject* pySerializedJson = PyUnicode_FromString(msg.toStyledString().c_str()); - PyTuple_SetItem(pyArgs, 1, pySerializedJson); // Second argument - - // Invoke the python routine - PyObject* pyList = PyObject_CallObject(pyFunc, pyArgs); - - // Error calling the function - if (pyList == nullptr) - { - fatalCobraError = true; - CoreLogger::error("run() function call failed. Input msg: "); - auto serializedMsg = msg.toStyledString(); - CoreLogger::error(serializedMsg); - PyErr_Print(); - CoreLogger::error("================"); - return; - } - - // Invalid return type - if (!PyList_Check(pyList)) - { - fatalCobraError = true; - CoreLogger::error("run() return type should be a list"); - return; - } - - // The result is a list of dict containing sufficient info - // to send messages to statsd - auto listSize = PyList_Size(pyList); - - for (Py_ssize_t i = 0 ; i < listSize; ++i) - { - PyObject* dict = PyList_GetItem(pyList, i); - - // Make sure this is a dict - if (!PyDict_Check(dict)) - { - fatalCobraError = true; - CoreLogger::error("list element is not a dict"); - continue; - } - - // - // Retrieve object kind - // - PyObject* pyKind = PyDict_GetItemString(dict, "kind"); - if (!PyUnicode_Check(pyKind)) - { - fatalCobraError = true; - CoreLogger::error("kind entry is not a string"); - continue; - } - std::string kind(PyUnicode_AsUTF8(pyKind)); - - bool counter = false; - bool gauge = false; - bool timing = false; - - if (kind == "counter") - { - counter = true; - } - else if (kind == "gauge") - { - gauge = true; - } - else if (kind == "timing") - { - timing = true; - } - else - { - fatalCobraError = true; - CoreLogger::error(std::string("invalid kind entry: ") + kind + - ". Supported ones are counter, gauge, timing"); - continue; - } - - // - // Retrieve object key - // - PyObject* pyKey = PyDict_GetItemString(dict, "key"); - if (!PyUnicode_Check(pyKey)) - { - fatalCobraError = true; - CoreLogger::error("key entry is not a string"); - continue; - } - std::string key(PyUnicode_AsUTF8(pyKey)); - - // - // Retrieve object value and send data to statsd - // - PyObject* pyValue = PyDict_GetItemString(dict, "value"); - - // Send data to statsd - if (PyFloat_Check(pyValue)) - { - double value = PyFloat_AsDouble(pyValue); - - if (counter) - { - statsdClient.count(key, value); - } - else if (gauge) - { - statsdClient.gauge(key, value); - } - else if (timing) - { - statsdClient.timing(key, value); - } - } - else if (PyLong_Check(pyValue)) - { - long value = PyLong_AsLong(pyValue); - - if (counter) - { - statsdClient.count(key, value); - } - else if (gauge) - { - statsdClient.gauge(key, value); - } - else if (timing) - { - statsdClient.timing(key, value); - } - } - else - { - fatalCobraError = true; - CoreLogger::error("value entry is neither an int or a float"); - continue; - } - - sentCount++; // should we update this for each statsd object sent ? - } - - Py_DECREF(pyArgs); - Py_DECREF(pyList); - }); - - bool status = bot.run(config); - - // Cleanup - we should do something similar in all exit case ... - Py_DECREF(pyFunc); - Py_DECREF(pyModule); - Py_FinalizeEx(); - - return status; -#endif - } -} diff --git a/ixbots/ixbots/IXCobraToPythonBot.h b/ixbots/ixbots/IXCobraToPythonBot.h deleted file mode 100644 index ad96c110..00000000 --- a/ixbots/ixbots/IXCobraToPythonBot.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * IXCobraMetricsToStatsdBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_python_bot(const ix::CobraBotConfig& config, - StatsdClient& statsdClient, - const std::string& moduleName); -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToSentryBot.cpp b/ixbots/ixbots/IXCobraToSentryBot.cpp deleted file mode 100644 index ebb8277d..00000000 --- a/ixbots/ixbots/IXCobraToSentryBot.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * IXCobraToSentryBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToSentryBot.h" - -#include "IXCobraBot.h" -#include -#include - -#include -#include -#include - -namespace ix -{ - int64_t cobra_to_sentry_bot(const CobraBotConfig& config, - SentryClient& sentryClient, - bool verbose) - { - CobraBot bot; - bot.setOnBotMessageCallback([&sentryClient, &verbose](const Json::Value& msg, - const std::string& /*position*/, - std::atomic& throttled, - std::atomic& /*fatalCobraError*/, - std::atomic& sentCount) -> void { - sentryClient.send(msg, verbose, - [&sentCount, &throttled](const HttpResponsePtr& response) { - if (!response) - { - CoreLogger::warn("Null HTTP Response"); - return; - } - - if (response->statusCode == 200) - { - sentCount++; - } - else - { - CoreLogger::error("Error sending data to sentry: " + std::to_string(response->statusCode)); - CoreLogger::error("Response: " + response->body); - - // Error 429 Too Many Requests - if (response->statusCode == 429) - { - auto retryAfter = response->headers["Retry-After"]; - std::stringstream ss; - ss << retryAfter; - int seconds; - ss >> seconds; - - if (!ss.eof() || ss.fail()) - { - seconds = 30; - CoreLogger::warn("Error parsing Retry-After header. " - "Using " + retryAfter + " for the sleep duration"); - } - - CoreLogger::warn("Error 429 - Too Many Requests. ws will sleep " - "and retry after " + retryAfter + " seconds"); - - throttled = true; - auto duration = std::chrono::seconds(seconds); - std::this_thread::sleep_for(duration); - throttled = false; - } - } - }); - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToSentryBot.h b/ixbots/ixbots/IXCobraToSentryBot.h deleted file mode 100644 index 7d9178b9..00000000 --- a/ixbots/ixbots/IXCobraToSentryBot.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * IXCobraToSentryBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_sentry_bot(const CobraBotConfig& config, - SentryClient& sentryClient, - bool verbose); -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStatsdBot.cpp b/ixbots/ixbots/IXCobraToStatsdBot.cpp deleted file mode 100644 index 1f717162..00000000 --- a/ixbots/ixbots/IXCobraToStatsdBot.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * IXCobraToStatsdBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToStatsdBot.h" - -#include "IXCobraBot.h" -#include "IXStatsdClient.h" -#include -#include -#include -#include -#include - -namespace ix -{ - // fields are command line argument that can be specified multiple times - std::vector parseFields(const std::string& fields) - { - std::vector tokens; - - // Split by \n - std::string token; - std::stringstream tokenStream(fields); - - while (std::getline(tokenStream, token)) - { - tokens.push_back(token); - } - - return tokens; - } - - // - // Extract an attribute from a Json Value. - // extractAttr("foo.bar", {"foo": {"bar": "baz"}}) => baz - // - Json::Value extractAttr(const std::string& attr, const Json::Value& jsonValue) - { - // Split by . - std::string token; - std::stringstream tokenStream(attr); - - Json::Value val(jsonValue); - - while (std::getline(tokenStream, token, '.')) - { - val = val[token]; - } - - return val; - } - - int64_t cobra_to_statsd_bot(const ix::CobraBotConfig& config, - StatsdClient& statsdClient, - const std::string& fields, - const std::string& gauge, - const std::string& timer, - bool verbose) - { - auto tokens = parseFields(fields); - - CobraBot bot; - bot.setOnBotMessageCallback( - [&statsdClient, &tokens, &gauge, &timer, &verbose](const Json::Value& msg, - const std::string& /*position*/, - std::atomic& /*throttled*/, - std::atomic& fatalCobraError, - std::atomic& sentCount) -> void { - std::string id; - size_t idx = 0; - for (auto&& attr : tokens) - { - auto val = extractAttr(attr, msg); - id += val.asString(); - - // We add a dot separator unless we are processing the last token - if (idx++ != tokens.size() - 1) - { - id += "."; - } - } - - if (gauge.empty() && timer.empty()) - { - statsdClient.count(id, 1); - } - else - { - std::string attrName = (!gauge.empty()) ? gauge : timer; - auto val = extractAttr(attrName, msg); - size_t x; - - if (val.isInt()) - { - x = (size_t) val.asInt(); - } - else if (val.isInt64()) - { - x = (size_t) val.asInt64(); - } - else if (val.isUInt()) - { - x = (size_t) val.asUInt(); - } - else if (val.isUInt64()) - { - x = (size_t) val.asUInt64(); - } - else if (val.isDouble()) - { - x = (size_t) val.asUInt64(); - } - else - { - CoreLogger::error("Gauge " + gauge + " is not a numeric type"); - fatalCobraError = true; - return; - } - - if (verbose) - { - CoreLogger::info(id + " - " + attrName + " -> " + std::to_string(x)); - } - - if (!gauge.empty()) - { - statsdClient.gauge(id, x); - } - else - { - statsdClient.timing(id, x); - } - } - - sentCount++; - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStatsdBot.h b/ixbots/ixbots/IXCobraToStatsdBot.h deleted file mode 100644 index 50025957..00000000 --- a/ixbots/ixbots/IXCobraToStatsdBot.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * IXCobraToStatsdBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_statsd_bot(const ix::CobraBotConfig& config, - StatsdClient& statsdClient, - const std::string& fields, - const std::string& gauge, - const std::string& timer, - bool verbose); -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStdoutBot.cpp b/ixbots/ixbots/IXCobraToStdoutBot.cpp deleted file mode 100644 index df266f7b..00000000 --- a/ixbots/ixbots/IXCobraToStdoutBot.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * IXCobraToStdoutBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToStdoutBot.h" - -#include "IXCobraBot.h" -#include -#include -#include - -namespace ix -{ - using StreamWriterPtr = std::unique_ptr; - - StreamWriterPtr makeStreamWriter() - { - Json::StreamWriterBuilder builder; - builder["commentStyle"] = "None"; - builder["indentation"] = ""; // will make the JSON object compact - std::unique_ptr jsonWriter(builder.newStreamWriter()); - return jsonWriter; - } - - std::string timeSinceEpoch() - { - std::chrono::system_clock::time_point tp = std::chrono::system_clock::now(); - std::chrono::system_clock::duration dtn = tp.time_since_epoch(); - - std::stringstream ss; - ss << dtn.count() * std::chrono::system_clock::period::num / - std::chrono::system_clock::period::den; - return ss.str(); - } - - void writeToStdout(bool fluentd, - const StreamWriterPtr& jsonWriter, - const Json::Value& msg, - const std::string& position) - { - Json::Value enveloppe; - if (fluentd) - { - enveloppe["producer"] = "cobra"; - enveloppe["consumer"] = "fluentd"; - - Json::Value nestedMessage(msg); - nestedMessage["position"] = position; - nestedMessage["created_at"] = timeSinceEpoch(); - enveloppe["message"] = nestedMessage; - - jsonWriter->write(enveloppe, &std::cout); - std::cout << std::endl; // add lf and flush - } - else - { - enveloppe = msg; - std::cout << position << " "; - jsonWriter->write(enveloppe, &std::cout); - std::cout << std::endl; - } - } - - int64_t cobra_to_stdout_bot(const ix::CobraBotConfig& config, - bool fluentd, - bool quiet) - { - CobraBot bot; - auto jsonWriter = makeStreamWriter(); - - bot.setOnBotMessageCallback( - [&fluentd, &quiet, &jsonWriter](const Json::Value& msg, - const std::string& position, - std::atomic& /*throttled*/, - std::atomic& /*fatalCobraError*/, - std::atomic& sentCount) -> void { - if (!quiet) - { - writeToStdout(fluentd, jsonWriter, msg, position); - } - sentCount++; - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStdoutBot.h b/ixbots/ixbots/IXCobraToStdoutBot.h deleted file mode 100644 index a99d83b0..00000000 --- a/ixbots/ixbots/IXCobraToStdoutBot.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * IXCobraToStdoutBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_stdout_bot(const ix::CobraBotConfig& config, - bool fluentd, - bool quiet); -} // namespace ix diff --git a/ixbots/ixbots/IXStatsdClient.cpp b/ixbots/ixbots/IXStatsdClient.cpp deleted file mode 100644 index 85a7fc61..00000000 --- a/ixbots/ixbots/IXStatsdClient.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2014, Rex - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * * Neither the name of the {organization} nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * IXStatsdClient.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -// Adapted from statsd-client-cpp -// test with netcat as a server: `nc -ul 8125` - -#include "IXStatsdClient.h" - -#include -#include -#include -#include -#include -#include - -namespace ix -{ - StatsdClient::StatsdClient(const std::string& host, - int port, - const std::string& prefix, - bool verbose) - : _host(host) - , _port(port) - , _prefix(prefix) - , _stop(false) - , _verbose(verbose) - { - _thread = std::thread([this] { - setThreadName("Statsd"); - - while (!_stop) - { - flushQueue(); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }); - } - - StatsdClient::~StatsdClient() - { - _stop = true; - if (_thread.joinable()) _thread.join(); - - _socket.close(); - } - - bool StatsdClient::init(std::string& errMsg) - { - return _socket.init(_host, _port, errMsg); - } - - /* will change the original string */ - void StatsdClient::cleanup(std::string& key) - { - size_t pos = key.find_first_of(":|@"); - while (pos != std::string::npos) - { - key[pos] = '_'; - pos = key.find_first_of(":|@"); - } - } - - int StatsdClient::dec(const std::string& key) - { - return count(key, -1); - } - - int StatsdClient::inc(const std::string& key) - { - return count(key, 1); - } - - int StatsdClient::count(const std::string& key, size_t value) - { - return send(key, value, "c"); - } - - int StatsdClient::gauge(const std::string& key, size_t value) - { - return send(key, value, "g"); - } - - int StatsdClient::timing(const std::string& key, size_t ms) - { - return send(key, ms, "ms"); - } - - int StatsdClient::send(std::string key, size_t value, const std::string& type) - { - cleanup(key); - - std::stringstream ss; - ss << _prefix << "." << key << ":" << value << "|" << type; - - if (_verbose) - { - CoreLogger::info(ss.str()); - } - - enqueue(ss.str() + "\n"); - return 0; - } - - void StatsdClient::enqueue(const std::string& message) - { - std::lock_guard lock(_mutex); - _queue.push_back(message); - } - - void StatsdClient::flushQueue() - { - std::lock_guard lock(_mutex); - - while (!_queue.empty()) - { - auto message = _queue.front(); - auto ret = _socket.sendto(message); - if (ret == -1) - { - CoreLogger::error(std::string("statsd error: ") + strerror(UdpSocket::getErrno())); - } - - // we always dequeue regardless of the ability to send the message - // so that we keep our queue size under control - _queue.pop_front(); - } - } -} // end namespace ix diff --git a/ixbots/ixbots/IXStatsdClient.h b/ixbots/ixbots/IXStatsdClient.h deleted file mode 100644 index 9f0f77c9..00000000 --- a/ixbots/ixbots/IXStatsdClient.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * IXStatsdClient.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace ix -{ - class StatsdClient - { - public: - StatsdClient(const std::string& host = "127.0.0.1", - int port = 8125, - const std::string& prefix = "", - bool verbose = false); - ~StatsdClient(); - - bool init(std::string& errMsg); - int inc(const std::string& key); - int dec(const std::string& key); - int count(const std::string& key, size_t value); - int gauge(const std::string& key, size_t value); - int timing(const std::string& key, size_t ms); - - private: - void enqueue(const std::string& message); - - /* (Low Level Api) manually send a message - * type = "c", "g" or "ms" - */ - int send(std::string key, size_t value, const std::string& type); - - void cleanup(std::string& key); - void flushQueue(); - - UdpSocket _socket; - - std::string _host; - int _port; - std::string _prefix; - - std::atomic _stop; - std::thread _thread; - std::mutex _mutex; // for the queue - - std::deque _queue; - bool _verbose; - }; - -} // end namespace ix diff --git a/ixcobra/CMakeLists.txt b/ixcobra/CMakeLists.txt deleted file mode 100644 index 8a03d1dc..00000000 --- a/ixcobra/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXCOBRA_SOURCES - ixcobra/IXCobraConnection.cpp - ixcobra/IXCobraMetricsThreadedPublisher.cpp - ixcobra/IXCobraMetricsPublisher.cpp -) - -set (IXCOBRA_HEADERS - ixcobra/IXCobraConnection.h - ixcobra/IXCobraMetricsThreadedPublisher.h - ixcobra/IXCobraMetricsPublisher.h - ixcobra/IXCobraConfig.h - ixcobra/IXCobraEventType.h -) - -add_library(ixcobra STATIC - ${IXCOBRA_SOURCES} - ${IXCOBRA_HEADERS} -) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) -endif() - -set(IXCOBRA_INCLUDE_DIRS - . - .. - ../ixcore - ../ixcrypto - ${JSONCPP_INCLUDE_DIRS}) - -target_include_directories( ixcobra PUBLIC ${IXCOBRA_INCLUDE_DIRS} ) diff --git a/ixcobra/ixcobra/IXCobraConfig.h b/ixcobra/ixcobra/IXCobraConfig.h deleted file mode 100644 index ccf95653..00000000 --- a/ixcobra/ixcobra/IXCobraConfig.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * IXCobraConfig.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include - -namespace ix -{ - struct CobraConfig - { - std::string appkey; - std::string endpoint; - std::string rolename; - std::string rolesecret; - WebSocketPerMessageDeflateOptions webSocketPerMessageDeflateOptions; - SocketTLSOptions socketTLSOptions; - WebSocketHttpHeaders headers; - - CobraConfig(const std::string& a = std::string(), - const std::string& e = std::string(), - const std::string& r = std::string(), - const std::string& s = std::string()) - : appkey(a) - , endpoint(e) - , rolename(r) - , rolesecret(s) - { - ; - } - }; -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraConnection.cpp b/ixcobra/ixcobra/IXCobraConnection.cpp deleted file mode 100644 index 2b1aed6a..00000000 --- a/ixcobra/ixcobra/IXCobraConnection.cpp +++ /dev/null @@ -1,713 +0,0 @@ -/* - * IXCobraConnection.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017-2018 Machine Zone. All rights reserved. - */ - -#include "IXCobraConnection.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace ix -{ - TrafficTrackerCallback CobraConnection::_trafficTrackerCallback = nullptr; - PublishTrackerCallback CobraConnection::_publishTrackerCallback = nullptr; - constexpr size_t CobraConnection::kQueueMaxSize; - constexpr CobraConnection::MsgId CobraConnection::kInvalidMsgId; - constexpr int CobraConnection::kPingIntervalSecs; - - CobraConnection::CobraConnection() - : _webSocket(new WebSocket()) - , _publishMode(CobraConnection_PublishMode_Immediate) - , _authenticated(false) - , _eventCallback(nullptr) - , _id(1) - { - _pdu["action"] = "rtm/publish"; - - _webSocket->addSubProtocol("json"); - initWebSocketOnMessageCallback(); - } - - CobraConnection::~CobraConnection() - { - disconnect(); - setEventCallback(nullptr); - } - - void CobraConnection::setTrafficTrackerCallback(const TrafficTrackerCallback& callback) - { - _trafficTrackerCallback = callback; - } - - void CobraConnection::resetTrafficTrackerCallback() - { - setTrafficTrackerCallback(nullptr); - } - - void CobraConnection::invokeTrafficTrackerCallback(size_t size, bool incoming) - { - if (_trafficTrackerCallback) - { - _trafficTrackerCallback(size, incoming); - } - } - - void CobraConnection::setPublishTrackerCallback(const PublishTrackerCallback& callback) - { - _publishTrackerCallback = callback; - } - - void CobraConnection::resetPublishTrackerCallback() - { - setPublishTrackerCallback(nullptr); - } - - void CobraConnection::invokePublishTrackerCallback(bool sent, bool acked) - { - if (_publishTrackerCallback) - { - _publishTrackerCallback(sent, acked); - } - } - - void CobraConnection::setEventCallback(const EventCallback& eventCallback) - { - std::lock_guard lock(_eventCallbackMutex); - _eventCallback = eventCallback; - } - - void CobraConnection::invokeEventCallback(ix::CobraEventType eventType, - const std::string& errorMsg, - const WebSocketHttpHeaders& headers, - const std::string& subscriptionId, - CobraConnection::MsgId msgId, - const std::string& connectionId) - { - std::lock_guard lock(_eventCallbackMutex); - if (_eventCallback) - { - _eventCallback( - ix::make_unique(eventType, errorMsg, headers, subscriptionId, msgId, connectionId)); - } - } - - void CobraConnection::invokeErrorCallback(const std::string& errorMsg, - const std::string& serializedPdu) - { - std::stringstream ss; - ss << errorMsg << " : received pdu => " << serializedPdu; - invokeEventCallback(ix::CobraEventType::Error, ss.str()); - } - - void CobraConnection::disconnect() - { - auto subscriptionIds = getSubscriptionsIds(); - for (auto&& subscriptionId : subscriptionIds) - { - unsubscribe(subscriptionId); - } - - _authenticated = false; - _webSocket->stop(); - } - - void CobraConnection::initWebSocketOnMessageCallback() - { - _webSocket->setOnMessageCallback([this](const ix::WebSocketMessagePtr& msg) { - CobraConnection::invokeTrafficTrackerCallback(msg->wireSize, true); - - std::stringstream ss; - if (msg->type == ix::WebSocketMessageType::Open) - { - invokeEventCallback(ix::CobraEventType::Open, std::string(), msg->openInfo.headers); - sendHandshakeMessage(); - } - else if (msg->type == ix::WebSocketMessageType::Close) - { - _authenticated = false; - - std::stringstream ss; - ss << "Close code " << msg->closeInfo.code; - ss << " reason " << msg->closeInfo.reason; - invokeEventCallback(ix::CobraEventType::Closed, ss.str()); - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - Json::Value data; - Json::Reader reader; - if (!reader.parse(msg->str, data)) - { - invokeErrorCallback("Invalid json", msg->str); - return; - } - - if (!data.isMember("action")) - { - invokeErrorCallback("Missing action", msg->str); - return; - } - - auto action = data["action"].asString(); - - if (action == "auth/handshake/ok") - { - if (!handleHandshakeResponse(data)) - { - invokeErrorCallback("Error extracting nonce from handshake response", - msg->str); - } - } - else if (action == "auth/handshake/error") - { - invokeEventCallback(ix::CobraEventType::HandshakeError, msg->str); - } - else if (action == "auth/authenticate/ok") - { - _authenticated = true; - invokeEventCallback(ix::CobraEventType::Authenticated); - flushQueue(); - } - else if (action == "auth/authenticate/error") - { - invokeEventCallback(ix::CobraEventType::AuthenticationError, msg->str); - } - else if (action == "rtm/subscription/data") - { - handleSubscriptionData(data); - } - else if (action == "rtm/subscribe/ok") - { - if (!handleSubscriptionResponse(data)) - { - invokeErrorCallback("Error processing subscribe response", msg->str); - } - } - else if (action == "rtm/subscribe/error") - { - invokeEventCallback(ix::CobraEventType::SubscriptionError, msg->str); - } - else if (action == "rtm/unsubscribe/ok") - { - if (!handleUnsubscriptionResponse(data)) - { - invokeErrorCallback("Error processing unsubscribe response", msg->str); - } - } - else if (action == "rtm/unsubscribe/error") - { - invokeErrorCallback("Unsubscription error", msg->str); - } - else if (action == "rtm/publish/ok") - { - if (!handlePublishResponse(data)) - { - invokeErrorCallback("Error processing publish response", msg->str); - } - } - else if (action == "rtm/publish/error") - { - invokeErrorCallback("Publish error", msg->str); - } - else - { - invokeErrorCallback("Un-handled message type", msg->str); - } - } - else if (msg->type == ix::WebSocketMessageType::Error) - { - std::stringstream ss; - ss << "Connection error: " << msg->errorInfo.reason << std::endl; - ss << "#retries: " << msg->errorInfo.retries << std::endl; - ss << "Wait time(ms): " << msg->errorInfo.wait_time << std::endl; - ss << "HTTP Status: " << msg->errorInfo.http_status << std::endl; - invokeErrorCallback(ss.str(), std::string()); - } - else if (msg->type == ix::WebSocketMessageType::Pong) - { - invokeEventCallback(ix::CobraEventType::Pong, msg->str); - } - }); - } - - void CobraConnection::setPublishMode(CobraConnectionPublishMode publishMode) - { - _publishMode = publishMode; - } - - CobraConnectionPublishMode CobraConnection::getPublishMode() - { - return _publishMode; - } - - void CobraConnection::configure( - const std::string& appkey, - const std::string& endpoint, - const std::string& rolename, - const std::string& rolesecret, - const WebSocketPerMessageDeflateOptions& webSocketPerMessageDeflateOptions, - const SocketTLSOptions& socketTLSOptions, - const WebSocketHttpHeaders& headers) - { - _roleName = rolename; - _roleSecret = rolesecret; - - std::stringstream ss; - ss << endpoint; - ss << "/v2?appkey="; - ss << appkey; - - std::string url = ss.str(); - _webSocket->setUrl(url); - _webSocket->setPerMessageDeflateOptions(webSocketPerMessageDeflateOptions); - _webSocket->setTLSOptions(socketTLSOptions); - _webSocket->setExtraHeaders(headers); - - // Send a websocket ping every N seconds (N = 30) now - // This should keep the connection open and prevent some load balancers such as - // the Amazon one from shutting it down - _webSocket->setPingInterval(kPingIntervalSecs); - } - - void CobraConnection::configure(const ix::CobraConfig& config) - { - configure(config.appkey, - config.endpoint, - config.rolename, - config.rolesecret, - config.webSocketPerMessageDeflateOptions, - config.socketTLSOptions, - config.headers); - } - - // - // Handshake message schema. - // - // handshake = { - // "action": "auth/handshake", - // "body": { - // "data": { - // "role": role - // }, - // "method": "role_secret" - // }, - // } - // - // - bool CobraConnection::sendHandshakeMessage() - { - Json::Value data; - data["role"] = _roleName; - - Json::Value body; - body["data"] = data; - body["method"] = "role_secret"; - - Json::Value pdu; - pdu["action"] = "auth/handshake"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - std::string serializedJson = serializeJson(pdu); - CobraConnection::invokeTrafficTrackerCallback(serializedJson.size(), false); - - return _webSocket->send(serializedJson).success; - } - - // - // Extract the nonce from the handshake response - // use it to compute a hash during authentication - // - // { - // "action": "auth/handshake/ok", - // "body": { - // "data": { - // "nonce": "MTI0Njg4NTAyMjYxMzgxMzgzMg==", - // "version": "0.0.24" - // } - // } - // } - // - bool CobraConnection::handleHandshakeResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - if (!body.isMember("data")) return false; - Json::Value data = body["data"]; - - if (!data.isMember("nonce")) return false; - Json::Value nonce = data["nonce"]; - - if (!nonce.isString()) return false; - - if (!data.isMember("connection_id")) return false; - Json::Value connectionId = data["connection_id"]; - - if (!connectionId.isString()) return false; - - invokeEventCallback(ix::CobraEventType::Handshake, - std::string(), - WebSocketHttpHeaders(), - std::string(), - 0, - connectionId.asString()); - - return sendAuthMessage(nonce.asString()); - } - - // - // Authenticate message schema. - // - // challenge = { - // "action": "auth/authenticate", - // "body": { - // "method": "role_secret", - // "credentials": { - // "hash": computeHash(secret, nonce) - // } - // }, - // } - // - bool CobraConnection::sendAuthMessage(const std::string& nonce) - { - Json::Value credentials; - credentials["hash"] = hmac(nonce, _roleSecret); - - Json::Value body; - body["credentials"] = credentials; - body["method"] = "role_secret"; - - Json::Value pdu; - pdu["action"] = "auth/authenticate"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - std::string serializedJson = serializeJson(pdu); - CobraConnection::invokeTrafficTrackerCallback(serializedJson.size(), false); - - return _webSocket->send(serializedJson).success; - } - - bool CobraConnection::handleSubscriptionResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - if (!body.isMember("subscription_id")) return false; - Json::Value subscriptionId = body["subscription_id"]; - - if (!subscriptionId.isString()) return false; - - invokeEventCallback(ix::CobraEventType::Subscribed, - std::string(), - WebSocketHttpHeaders(), - subscriptionId.asString()); - return true; - } - - bool CobraConnection::handleUnsubscriptionResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - if (!body.isMember("subscription_id")) return false; - Json::Value subscriptionId = body["subscription_id"]; - - if (!subscriptionId.isString()) return false; - - invokeEventCallback(ix::CobraEventType::UnSubscribed, - std::string(), - WebSocketHttpHeaders(), - subscriptionId.asString()); - return true; - } - - bool CobraConnection::handleSubscriptionData(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - // Identify subscription_id, so that we can find - // which callback to execute - if (!body.isMember("subscription_id")) return false; - Json::Value subscriptionId = body["subscription_id"]; - - std::lock_guard lock(_cbsMutex); - auto cb = _cbs.find(subscriptionId.asString()); - if (cb == _cbs.end()) return false; // cannot find callback - - // Extract messages now - if (!body.isMember("messages")) return false; - Json::Value messages = body["messages"]; - - if (!body.isMember("position")) return false; - std::string position = body["position"].asString(); - - for (auto&& msg : messages) - { - cb->second(msg, position); - } - - return true; - } - - bool CobraConnection::handlePublishResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("id")) return false; - Json::Value id = pdu["id"]; - - if (!id.isUInt64()) return false; - - uint64_t msgId = id.asUInt64(); - - invokeEventCallback(ix::CobraEventType::Published, - std::string(), - WebSocketHttpHeaders(), - std::string(), - msgId); - - invokePublishTrackerCallback(false, true); - - return true; - } - - bool CobraConnection::connect() - { - _webSocket->start(); - return true; - } - - bool CobraConnection::isConnected() const - { - return _webSocket->getReadyState() == ix::ReadyState::Open; - } - - bool CobraConnection::isAuthenticated() const - { - return isConnected() && _authenticated; - } - - std::string CobraConnection::serializeJson(const Json::Value& value) - { - std::lock_guard lock(_jsonWriterMutex); - return _jsonWriter.write(value); - } - - std::pair CobraConnection::prePublish( - const Json::Value& channels, const Json::Value& msg, bool addToQueue) - { - std::lock_guard lock(_prePublishMutex); - - invokePublishTrackerCallback(true, false); - - CobraConnection::MsgId msgId = _id; - - _body["channels"] = channels; - _body["message"] = msg; - _pdu["body"] = _body; - _pdu["id"] = Json::UInt64(_id++); - - std::string serializedJson = serializeJson(_pdu); - - if (addToQueue) - { - enqueue(serializedJson); - } - - return std::make_pair(msgId, serializedJson); - } - - bool CobraConnection::publishNext() - { - std::lock_guard lock(_queueMutex); - - if (_messageQueue.empty()) return true; - - auto&& msg = _messageQueue.back(); - if (!_authenticated || !publishMessage(msg)) - { - return false; - } - _messageQueue.pop_back(); - return true; - } - - // - // publish is not thread safe as we are trying to reuse some Json objects. - // - CobraConnection::MsgId CobraConnection::publish(const Json::Value& channels, - const Json::Value& msg) - { - auto p = prePublish(channels, msg, false); - auto msgId = p.first; - auto serializedJson = p.second; - - // - // 1. When we use batch mode, we just enqueue and will do the flush explicitely - // 2. When we aren't authenticated yet to the cobra server, we need to enqueue - // and retry later - // 3. If the network connection was droped (WebSocket::send will return false), - // it means the message won't be sent so we need to enqueue as well. - // - // The order of the conditionals is important. - // - if (_publishMode == CobraConnection_PublishMode_Batch || !_authenticated || - !publishMessage(serializedJson)) - { - enqueue(serializedJson); - } - - return msgId; - } - - void CobraConnection::subscribe(const std::string& channel, - const std::string& filter, - const std::string& position, - int batchSize, - SubscriptionCallback cb) - { - // Create and send a subscribe pdu - Json::Value body; - body["channel"] = channel; - body["batch_size"] = batchSize; - - if (!filter.empty()) - { - body["filter"] = filter; - } - - if (!position.empty()) - { - body["position"] = position; - } - - Json::Value pdu; - pdu["action"] = "rtm/subscribe"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - _webSocket->send(pdu.toStyledString()); - - // Set the callback - std::lock_guard lock(_cbsMutex); - _cbs[channel] = cb; - } - - void CobraConnection::unsubscribe(const std::string& channel) - { - { - std::lock_guard lock(_cbsMutex); - auto cb = _cbs.find(channel); - if (cb == _cbs.end()) return; - - _cbs.erase(cb); - } - - // Create and send an unsubscribe pdu - Json::Value body; - body["subscription_id"] = channel; - - Json::Value pdu; - pdu["action"] = "rtm/unsubscribe"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - _webSocket->send(pdu.toStyledString()); - } - - std::vector CobraConnection::getSubscriptionsIds() - { - std::vector subscriptionIds; - std::lock_guard lock(_cbsMutex); - - for (auto&& it : _cbs) - { - subscriptionIds.push_back(it.first); - } - return subscriptionIds; - } - - // - // Enqueue strategy drops old messages when we are at full capacity - // - // If we want to keep only 3 items max in the queue: - // - // enqueue(A) -> [A] - // enqueue(B) -> [B, A] - // enqueue(C) -> [C, B, A] - // enqueue(D) -> [D, C, B] -- now we drop A, the oldest message, - // -- and keep the 'fresh ones' - // - void CobraConnection::enqueue(const std::string& msg) - { - std::lock_guard lock(_queueMutex); - - if (_messageQueue.size() == CobraConnection::kQueueMaxSize) - { - _messageQueue.pop_back(); - } - _messageQueue.push_front(msg); - } - - // - // We process messages back (oldest) to front (newest) to respect ordering - // when sending them. If we fail to send something, we put it back in the queue - // at the end we picked it up originally (at the end). - // - bool CobraConnection::flushQueue() - { - while (!isQueueEmpty()) - { - bool ok = publishNext(); - if (!ok) return false; - } - - return true; - } - - bool CobraConnection::isQueueEmpty() - { - std::lock_guard lock(_queueMutex); - return _messageQueue.empty(); - } - - bool CobraConnection::publishMessage(const std::string& serializedJson) - { - auto webSocketSendInfo = _webSocket->send(serializedJson); - CobraConnection::invokeTrafficTrackerCallback(webSocketSendInfo.wireSize, false); - return webSocketSendInfo.success; - } - - void CobraConnection::suspend() - { - disconnect(); - } - - void CobraConnection::resume() - { - connect(); - } - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraConnection.h b/ixcobra/ixcobra/IXCobraConnection.h deleted file mode 100644 index 74a7fca6..00000000 --- a/ixcobra/ixcobra/IXCobraConnection.h +++ /dev/null @@ -1,224 +0,0 @@ -/* - * IXCobraConnection.h - * Author: Benjamin Sergeant - * Copyright (c) 2017-2018 Machine Zone. All rights reserved. - */ - -#pragma once - -#include "IXCobraConfig.h" -#include "IXCobraEvent.h" -#include "IXCobraEventType.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef max -#undef max -#endif - -namespace ix -{ - class WebSocket; - struct SocketTLSOptions; - - enum CobraConnectionPublishMode - { - CobraConnection_PublishMode_Immediate = 0, - CobraConnection_PublishMode_Batch = 1 - }; - - using SubscriptionCallback = std::function; - using EventCallback = std::function; - - using TrafficTrackerCallback = std::function; - using PublishTrackerCallback = std::function; - - class CobraConnection - { - public: - using MsgId = uint64_t; - - CobraConnection(); - ~CobraConnection(); - - /// Configuration / set keys, etc... - /// All input data but the channel name is encrypted with rc4 - void configure(const std::string& appkey, - const std::string& endpoint, - const std::string& rolename, - const std::string& rolesecret, - const WebSocketPerMessageDeflateOptions& webSocketPerMessageDeflateOptions, - const SocketTLSOptions& socketTLSOptions, - const WebSocketHttpHeaders& headers); - - void configure(const ix::CobraConfig& config); - - /// Set the traffic tracker callback - static void setTrafficTrackerCallback(const TrafficTrackerCallback& callback); - - /// Reset the traffic tracker callback to an no-op one. - static void resetTrafficTrackerCallback(); - - /// Set the publish tracker callback - static void setPublishTrackerCallback(const PublishTrackerCallback& callback); - - /// Reset the publish tracker callback to an no-op one. - static void resetPublishTrackerCallback(); - - /// Set the closed callback - void setEventCallback(const EventCallback& eventCallback); - - /// Start the worker thread, used for background publishing - void start(); - - /// Publish a message to a channel - /// - /// No-op if the connection is not established - MsgId publish(const Json::Value& channels, const Json::Value& msg); - - // Subscribe to a channel, and execute a callback when an incoming - // message arrives. - void subscribe(const std::string& channel, - const std::string& filter = std::string(), - const std::string& position = std::string(), - int batchSize = 1, - SubscriptionCallback cb = nullptr); - - /// Unsubscribe from a channel - void unsubscribe(const std::string& channel); - - /// Close the connection - void disconnect(); - - /// Connect to Cobra and authenticate the connection - bool connect(); - - /// Returns true only if we're connected - bool isConnected() const; - - /// Returns true only if we're authenticated - bool isAuthenticated() const; - - /// Flush the publish queue - bool flushQueue(); - - /// Set the publish mode - void setPublishMode(CobraConnectionPublishMode publishMode); - - /// Query the publish mode - CobraConnectionPublishMode getPublishMode(); - - /// Lifecycle management. Free resources when backgrounding - void suspend(); - void resume(); - - /// Prepare a message for transmission - /// (update the pdu, compute a msgId, serialize json to a string) - std::pair prePublish(const Json::Value& channels, - const Json::Value& msg, - bool addToQueue); - - /// Attempt to send next message from the internal queue - bool publishNext(); - - // An invalid message id, signifying an error. - static constexpr MsgId kInvalidMsgId = 0; - - private: - bool sendHandshakeMessage(); - bool handleHandshakeResponse(const Json::Value& data); - bool sendAuthMessage(const std::string& nonce); - bool handleSubscriptionData(const Json::Value& pdu); - bool handleSubscriptionResponse(const Json::Value& pdu); - bool handleUnsubscriptionResponse(const Json::Value& pdu); - bool handlePublishResponse(const Json::Value& pdu); - - void initWebSocketOnMessageCallback(); - - bool publishMessage(const std::string& serializedJson); - void enqueue(const std::string& msg); - std::string serializeJson(const Json::Value& pdu); - - /// Invoke the traffic tracker callback - static void invokeTrafficTrackerCallback(size_t size, bool incoming); - - /// Invoke the publish tracker callback - static void invokePublishTrackerCallback(bool sent, bool acked); - - /// Invoke event callbacks - void invokeEventCallback(CobraEventType eventType, - const std::string& errorMsg = std::string(), - const WebSocketHttpHeaders& headers = WebSocketHttpHeaders(), - const std::string& subscriptionId = std::string(), - uint64_t msgId = std::numeric_limits::max(), - const std::string& connectionId = std::string()); - - void invokeErrorCallback(const std::string& errorMsg, const std::string& serializedPdu); - - /// Tells whether the internal queue is empty or not - bool isQueueEmpty(); - - /// Retrieve all subscriptions ids - std::vector getSubscriptionsIds(); - - /// - /// Member variables - /// - std::unique_ptr _webSocket; - - /// Configuration data - std::string _roleName; - std::string _roleSecret; - std::atomic _publishMode; - - // Can be set on control+background thread, protecting with an atomic - std::atomic _authenticated; - - // Keep some objects around - Json::Value _body; - Json::Value _pdu; - Json::FastWriter _jsonWriter; - mutable std::mutex _jsonWriterMutex; - std::mutex _prePublishMutex; - - /// Traffic tracker callback - static TrafficTrackerCallback _trafficTrackerCallback; - - /// Publish tracker callback - static PublishTrackerCallback _publishTrackerCallback; - - /// Cobra events callbacks - EventCallback _eventCallback; - mutable std::mutex _eventCallbackMutex; - - /// Subscription callbacks, only one per channel - std::unordered_map _cbs; - mutable std::mutex _cbsMutex; - - // Message Queue can be touched on control+background thread, - // protecting with a mutex. - // - // Message queue is used when there are problems sending messages so - // that sending can be retried later. - std::deque _messageQueue; - mutable std::mutex _queueMutex; - - // Cap the queue size (100 elems so far -> ~100k) - static constexpr size_t kQueueMaxSize = 256; - - // Each pdu sent should have an incremental unique id - std::atomic _id; - - // Frequency at which we send a websocket ping to the backing cobra connection - static constexpr int kPingIntervalSecs = 30; - }; - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraEvent.h b/ixcobra/ixcobra/IXCobraEvent.h deleted file mode 100644 index 5d00cf81..00000000 --- a/ixcobra/ixcobra/IXCobraEvent.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * IXCobraEvent.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include "IXCobraEventType.h" -#include -#include -#include -#include - -namespace ix -{ - struct CobraEvent - { - ix::CobraEventType type; - const std::string& errMsg; - const ix::WebSocketHttpHeaders& headers; - const std::string& subscriptionId; - uint64_t msgId; // CobraConnection::MsgId - const std::string& connectionId; - - CobraEvent(ix::CobraEventType t, - const std::string& e, - const ix::WebSocketHttpHeaders& h, - const std::string& s, - uint64_t m, - const std::string& c) - : type(t) - , errMsg(e) - , headers(h) - , subscriptionId(s) - , msgId(m) - , connectionId(c) - { - ; - } - }; - - using CobraEventPtr = std::unique_ptr; -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraEventType.h b/ixcobra/ixcobra/IXCobraEventType.h deleted file mode 100644 index e3038c5d..00000000 --- a/ixcobra/ixcobra/IXCobraEventType.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * IXCobraEventType.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -namespace ix -{ - enum class CobraEventType - { - Authenticated = 0, - Error = 1, - Open = 2, - Closed = 3, - Subscribed = 4, - UnSubscribed = 5, - Published = 6, - Pong = 7, - HandshakeError = 8, - AuthenticationError = 9, - SubscriptionError = 10, - Handshake = 11 - }; -} diff --git a/ixcobra/ixcobra/IXCobraMetricsPublisher.cpp b/ixcobra/ixcobra/IXCobraMetricsPublisher.cpp deleted file mode 100644 index 81429af9..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsPublisher.cpp +++ /dev/null @@ -1,232 +0,0 @@ -/* - * IXCobraMetricsPublisher.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#include "IXCobraMetricsPublisher.h" - -#include -#include -#include - - -namespace ix -{ - const int CobraMetricsPublisher::kVersion = 1; - const std::string CobraMetricsPublisher::kSetRateControlId = "sms_set_rate_control_id"; - const std::string CobraMetricsPublisher::kSetBlacklistId = "sms_set_blacklist_id"; - - CobraMetricsPublisher::CobraMetricsPublisher() - : _enabled(true) - { - } - - CobraMetricsPublisher::~CobraMetricsPublisher() - { - ; - } - - void CobraMetricsPublisher::configure(const CobraConfig& config, const std::string& channel) - { - // Configure the satori connection and start its publish background thread - _cobra_metrics_theaded_publisher.configure(config, channel); - _cobra_metrics_theaded_publisher.start(); - } - - Json::Value& CobraMetricsPublisher::getGenericAttributes() - { - std::lock_guard lock(_device_mutex); - return _device; - } - - void CobraMetricsPublisher::setGenericAttributes(const std::string& attrName, - const Json::Value& value) - { - std::lock_guard lock(_device_mutex); - _device[attrName] = value; - } - - void CobraMetricsPublisher::enable(bool enabled) - { - _enabled = enabled; - } - - void CobraMetricsPublisher::setBlacklist(const std::vector& blacklist) - { - _blacklist = blacklist; - std::sort(_blacklist.begin(), _blacklist.end()); - - // publish our blacklist - Json::Value data; - Json::Value metrics; - for (auto&& metric : blacklist) - { - metrics.append(metric); - } - data["blacklist"] = metrics; - push(kSetBlacklistId, data); - } - - bool CobraMetricsPublisher::isMetricBlacklisted(const std::string& id) const - { - return std::binary_search(_blacklist.begin(), _blacklist.end(), id); - } - - void CobraMetricsPublisher::setRateControl( - const std::unordered_map& rate_control) - { - for (auto&& it : rate_control) - { - if (it.second >= 0) - { - _rate_control[it.first] = it.second; - } - } - - // publish our rate_control - Json::Value data; - Json::Value metrics; - for (auto&& it : _rate_control) - { - metrics[it.first] = it.second; - } - data["rate_control"] = metrics; - push(kSetRateControlId, data); - } - - bool CobraMetricsPublisher::isAboveMaxUpdateRate(const std::string& id) const - { - // Is this metrics rate controlled ? - auto rate_control_it = _rate_control.find(id); - if (rate_control_it == _rate_control.end()) return false; - - // Was this metrics already sent ? - std::lock_guard lock(_last_update_mutex); - auto last_update = _last_update.find(id); - if (last_update == _last_update.end()) return false; - - auto timeDeltaFromLastSend = std::chrono::steady_clock::now() - last_update->second; - - return timeDeltaFromLastSend < std::chrono::seconds(rate_control_it->second); - } - - void CobraMetricsPublisher::setLastUpdate(const std::string& id) - { - std::lock_guard lock(_last_update_mutex); - _last_update[id] = std::chrono::steady_clock::now(); - } - - uint64_t CobraMetricsPublisher::getMillisecondsSinceEpoch() const - { - auto now = std::chrono::system_clock::now(); - auto ms = - std::chrono::duration_cast(now.time_since_epoch()).count(); - - return ms; - } - - CobraConnection::MsgId CobraMetricsPublisher::push(const std::string& id, - const std::string& data, - bool shouldPushTest) - { - if (!_enabled) return CobraConnection::kInvalidMsgId; - - Json::Value root; - Json::Reader reader; - if (!reader.parse(data, root)) return CobraConnection::kInvalidMsgId; - - return push(id, root, shouldPushTest); - } - - CobraConnection::MsgId CobraMetricsPublisher::push(const std::string& id, - const CobraMetricsPublisher::Message& data) - { - if (!_enabled) return CobraConnection::kInvalidMsgId; - - Json::Value root; - for (auto it : data) - { - root[it.first] = it.second; - } - - return push(id, root); - } - - bool CobraMetricsPublisher::shouldPush(const std::string& id) const - { - if (!_enabled) return false; - if (isMetricBlacklisted(id)) return false; - if (isAboveMaxUpdateRate(id)) return false; - - return true; - } - - CobraConnection::MsgId CobraMetricsPublisher::push(const std::string& id, - const Json::Value& data, - bool shouldPushTest) - { - if (shouldPushTest && !shouldPush(id)) return CobraConnection::kInvalidMsgId; - - setLastUpdate(id); - - Json::Value msg; - msg["id"] = id; - msg["data"] = data; - msg["session"] = _session; - msg["version"] = kVersion; - msg["timestamp"] = Json::UInt64(getMillisecondsSinceEpoch()); - - { - std::lock_guard lock(_device_mutex); - msg["device"] = _device; - } - - { - // - // Bump a counter for each id - // This is used to make sure that we are not - // dropping messages, by checking that all the ids is the list of - // all natural numbers until the last value sent (0, 1, 2, ..., N) - // - std::lock_guard lock(_device_mutex); - auto it = _counters.emplace(id, 0); - msg["per_id_counter"] = it.first->second; - it.first->second += 1; - } - - // Now actually enqueue the task - return _cobra_metrics_theaded_publisher.push(msg); - } - - void CobraMetricsPublisher::setPublishMode(CobraConnectionPublishMode publishMode) - { - _cobra_metrics_theaded_publisher.setPublishMode(publishMode); - } - - bool CobraMetricsPublisher::flushQueue() - { - return _cobra_metrics_theaded_publisher.flushQueue(); - } - - void CobraMetricsPublisher::suspend() - { - _cobra_metrics_theaded_publisher.suspend(); - } - - void CobraMetricsPublisher::resume() - { - _cobra_metrics_theaded_publisher.resume(); - } - - bool CobraMetricsPublisher::isConnected() const - { - return _cobra_metrics_theaded_publisher.isConnected(); - } - - bool CobraMetricsPublisher::isAuthenticated() const - { - return _cobra_metrics_theaded_publisher.isAuthenticated(); - } - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraMetricsPublisher.h b/ixcobra/ixcobra/IXCobraMetricsPublisher.h deleted file mode 100644 index 76d7760b..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsPublisher.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - * IXCobraMetricsPublisher.h - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#pragma once - -#include "IXCobraMetricsThreadedPublisher.h" -#include -#include -#include -#include -#include - -namespace ix -{ - struct SocketTLSOptions; - - class CobraMetricsPublisher - { - public: - CobraMetricsPublisher(); - ~CobraMetricsPublisher(); - - /// Thread safety notes: - /// - /// 1. _enabled, _blacklist and _rate_control read/writes are not protected by a mutex - /// to make shouldPush as fast as possible. _enabled default to false. - /// - /// The code that set those is ran only once at init, and - /// the last value to be set is _enabled, which is also the first value checked in - /// shouldPush, so there shouldn't be any race condition. - /// - /// 2. The queue of messages is thread safe, so multiple metrics can be safely pushed on - /// multiple threads - /// - /// 3. Access to _last_update is protected as it needs to be read/write. - /// - - /// Configuration / set keys, etc... - /// All input data but the channel name is encrypted with rc4 - void configure(const CobraConfig& config, const std::string& channel); - - /// Setter for the list of blacklisted metrics ids. - /// That list is sorted internally for fast lookups - void setBlacklist(const std::vector& blacklist); - - /// Set the maximum rate at which a metrics can be sent. Unit is seconds - /// if rate_control = { 'foo_id': 60 }, - /// the foo_id metric cannot be pushed more than once every 60 seconds - void setRateControl(const std::unordered_map& rate_control); - - /// Configuration / enable/disable - void enable(bool enabled); - - /// Simple interface, list of key value pairs where typeof(key) == typeof(value) == string - typedef std::unordered_map Message; - CobraConnection::MsgId push( - const std::string& id, - const CobraMetricsPublisher::Message& data = CobraMetricsPublisher::Message()); - - /// Richer interface using json, which supports types (bool, int, float) and hierarchies of - /// elements - /// - /// The shouldPushTest argument should be set to false, and used in combination with the - /// shouldPush method for places where we want to be as lightweight as possible when - /// collecting metrics. When set to false, it is used so that we don't do double work when - /// computing whether a metrics should be sent or not. - CobraConnection::MsgId push(const std::string& id, - const Json::Value& data, - bool shouldPushTest = true); - - /// Interface used by lua. msg is a json encoded string. - CobraConnection::MsgId push(const std::string& id, - const std::string& data, - bool shouldPushTest = true); - - /// Tells whether a metric can be pushed. - /// A metric can be pushed if it satisfies those conditions: - /// - /// 1. the metrics system should be enabled - /// 2. the metrics shouldn't be black-listed - /// 3. the metrics shouldn't have reached its rate control limit at this - /// "sampling"/"calling" time - bool shouldPush(const std::string& id) const; - - /// Get generic information json object - Json::Value& getGenericAttributes(); - - /// Set generic information values - void setGenericAttributes(const std::string& attrName, const Json::Value& value); - - /// Set a unique id for the session. A uuid can be used. - void setSession(const std::string& session) - { - _session = session; - } - - /// Get the unique id used to identify the current session - const std::string& getSession() const - { - return _session; - } - - /// Return the number of milliseconds since the epoch (~1970) - uint64_t getMillisecondsSinceEpoch() const; - - /// Set satori connection publish mode - void setPublishMode(CobraConnectionPublishMode publishMode); - - /// Flush the publish queue - bool flushQueue(); - - /// Lifecycle management. Free resources when backgrounding - void suspend(); - void resume(); - - /// Tells whether the socket connection is opened - bool isConnected() const; - - /// Returns true only if we're authenticated - bool isAuthenticated() const; - - private: - /// Lookup an id in our metrics to see whether it is blacklisted - /// Complexity is logarithmic - bool isMetricBlacklisted(const std::string& id) const; - - /// Tells whether we should drop a metrics or not as part of an enqueuing - /// because it exceed the max update rate (it is sent too often) - bool isAboveMaxUpdateRate(const std::string& id) const; - - /// Record when a metric was last sent. Used for rate control - void setLastUpdate(const std::string& id); - - /// - /// Member variables - /// - - CobraMetricsThreadedPublisher _cobra_metrics_theaded_publisher; - - /// A boolean to enable or disable this system - /// push becomes a no-op when _enabled is false - std::atomic _enabled; - - /// A uuid used to uniquely identify a session - std::string _session; - - /// The _device json blob is populated once when configuring this system - /// It record generic metadata about the client, run (version, device model, etc...) - Json::Value _device; - mutable std::mutex _device_mutex; // protect access to _device - - /// Metrics control (black list + rate control) - std::vector _blacklist; - std::unordered_map _rate_control; - std::unordered_map> - _last_update; - mutable std::mutex _last_update_mutex; // protect access to _last_update - - /// Bump a counter for each metric type - std::unordered_map _counters; - mutable std::mutex _counters_mutex; // protect access to _counters - - // const strings for internal ids - static const std::string kSetRateControlId; - static const std::string kSetBlacklistId; - - /// Our protocol version. Can be used by subscribers who would want to be backward - /// compatible if we change the way we arrange data - static const int kVersion; - }; - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp b/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp deleted file mode 100644 index ebd77a21..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* - * IXCobraMetricsThreadedPublisher.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#include "IXCobraMetricsThreadedPublisher.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace ix -{ - CobraMetricsThreadedPublisher::CobraMetricsThreadedPublisher() - : _stop(false) - { - _cobra_connection.setEventCallback([](const CobraEventPtr& event) { - std::stringstream ss; - ix::LogLevel logLevel = LogLevel::Info; - - if (event->type == ix::CobraEventType::Open) - { - ss << "Handshake headers" << std::endl; - - for (auto&& it : event->headers) - { - ss << it.first << ": " << it.second << std::endl; - } - } - else if (event->type == ix::CobraEventType::Handshake) - { - ss << "Cobra handshake connection id: " << event->connectionId; - } - else if (event->type == ix::CobraEventType::Authenticated) - { - ss << "Authenticated"; - } - else if (event->type == ix::CobraEventType::Error) - { - ss << "Error: " << event->errMsg; - logLevel = ix::LogLevel::Error; - } - else if (event->type == ix::CobraEventType::Closed) - { - ss << "Connection closed: " << event->errMsg; - } - else if (event->type == ix::CobraEventType::Subscribed) - { - ss << "Subscribed through subscription id: " << event->subscriptionId; - } - else if (event->type == ix::CobraEventType::UnSubscribed) - { - ss << "Unsubscribed through subscription id: " << event->subscriptionId; - } - else if (event->type == ix::CobraEventType::Published) - { - ss << "Published message " << event->msgId << " acked"; - logLevel = ix::LogLevel::Debug; - } - else if (event->type == ix::CobraEventType::Pong) - { - ss << "Received websocket pong"; - } - else if (event->type == ix::CobraEventType::HandshakeError) - { - ss << "Handshake error: " << event->errMsg; - logLevel = ix::LogLevel::Error; - } - else if (event->type == ix::CobraEventType::AuthenticationError) - { - ss << "Authentication error: " << event->errMsg; - logLevel = ix::LogLevel::Error; - } - else if (event->type == ix::CobraEventType::SubscriptionError) - { - ss << "Subscription error: " << event->errMsg; - logLevel = ix::LogLevel::Error; - } - - CoreLogger::log(ss.str().c_str(), logLevel); - }); - } - - CobraMetricsThreadedPublisher::~CobraMetricsThreadedPublisher() - { - // The background thread won't be joinable if it was never - // started by calling CobraMetricsThreadedPublisher::start - if (!_thread.joinable()) return; - - _stop = true; - _condition.notify_one(); - _thread.join(); - } - - void CobraMetricsThreadedPublisher::start() - { - if (_thread.joinable()) return; // we've already been started - - _thread = std::thread(&CobraMetricsThreadedPublisher::run, this); - } - - void CobraMetricsThreadedPublisher::configure(const CobraConfig& config, - const std::string& channel) - { - CoreLogger::log(config.socketTLSOptions.getDescription().c_str()); - - _channel = channel; - _cobra_connection.configure(config); - } - - void CobraMetricsThreadedPublisher::pushMessage(MessageKind messageKind) - { - { - std::unique_lock lock(_queue_mutex); - _queue.push(messageKind); - } - - // wake up one thread - _condition.notify_one(); - } - - void CobraMetricsThreadedPublisher::setPublishMode(CobraConnectionPublishMode publishMode) - { - _cobra_connection.setPublishMode(publishMode); - } - - bool CobraMetricsThreadedPublisher::flushQueue() - { - return _cobra_connection.flushQueue(); - } - - void CobraMetricsThreadedPublisher::run() - { - setThreadName("CobraMetricsPublisher"); - - _cobra_connection.connect(); - - while (true) - { - Json::Value msg; - MessageKind messageKind; - - { - std::unique_lock lock(_queue_mutex); - - while (!_stop && _queue.empty()) - { - _condition.wait(lock); - } - if (_stop) - { - _cobra_connection.disconnect(); - return; - } - - messageKind = _queue.front(); - _queue.pop(); - } - - switch (messageKind) - { - case MessageKind::Suspend: - { - _cobra_connection.suspend(); - continue; - }; - break; - - case MessageKind::Resume: - { - _cobra_connection.resume(); - continue; - }; - break; - - case MessageKind::Message: - { - if (_cobra_connection.getPublishMode() == CobraConnection_PublishMode_Immediate) - { - _cobra_connection.publishNext(); - } - }; - break; - } - } - } - - CobraConnection::MsgId CobraMetricsThreadedPublisher::push(const Json::Value& msg) - { - static const std::string messageIdKey("id"); - - // - // Publish to multiple channels. This let the consumer side - // easily subscribe to all message of a certain type, without having - // to do manipulations on the messages on the server side. - // - Json::Value channels; - - channels.append(_channel); - if (msg.isMember(messageIdKey)) - { - channels.append(msg[messageIdKey]); - } - auto res = _cobra_connection.prePublish(channels, msg, true); - auto msgId = res.first; - - pushMessage(MessageKind::Message); - - return msgId; - } - - void CobraMetricsThreadedPublisher::suspend() - { - pushMessage(MessageKind::Suspend); - } - - void CobraMetricsThreadedPublisher::resume() - { - pushMessage(MessageKind::Resume); - } - - bool CobraMetricsThreadedPublisher::isConnected() const - { - return _cobra_connection.isConnected(); - } - - bool CobraMetricsThreadedPublisher::isAuthenticated() const - { - return _cobra_connection.isAuthenticated(); - } - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h b/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h deleted file mode 100644 index 10c50415..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * IXCobraMetricsThreadedPublisher.h - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#pragma once - -#include "IXCobraConnection.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ix -{ - struct SocketTLSOptions; - - class CobraMetricsThreadedPublisher - { - public: - CobraMetricsThreadedPublisher(); - ~CobraMetricsThreadedPublisher(); - - /// Configuration / set keys, etc... - void configure(const CobraConfig& config, const std::string& channel); - - /// Start the worker thread, used for background publishing - void start(); - - /// Push a msg to our queue of messages to be published to cobra on the background - // thread. Main user right now is the Cobra Metrics System - CobraConnection::MsgId push(const Json::Value& msg); - - /// Set cobra connection publish mode - void setPublishMode(CobraConnectionPublishMode publishMode); - - /// Flush the publish queue - bool flushQueue(); - - /// Lifecycle management. Free resources when backgrounding - void suspend(); - void resume(); - - /// Tells whether the socket connection is opened - bool isConnected() const; - - /// Returns true only if we're authenticated - bool isAuthenticated() const; - - private: - enum class MessageKind - { - Message = 0, - Suspend = 1, - Resume = 2 - }; - - /// Push a message to be processed by the background thread - void pushMessage(MessageKind messageKind); - - /// Get a wait time which is increasing exponentially based on the number of retries - uint64_t getWaitTimeExp(int retry_count); - - /// Debugging routine to print the connection parameters to the console - void printInfo(); - - /// Publish a message to satory - /// Will retry multiple times (3) if a problem occurs. - /// - /// Right now, only called on the publish worker thread. - void safePublish(const Json::Value& msg); - - /// The worker thread "daemon" method. That method never returns unless _stop is set to true - void run(); - - /// Our connection to cobra. - CobraConnection _cobra_connection; - - /// The channel we are publishing to - std::string _channel; - - /// Internal data structures used to publish to cobra - /// Pending messages are stored into a queue, which is protected by a mutex - /// We used a condition variable to prevent the worker thread from busy polling - /// So we notify the condition variable when an incoming message arrives to signal - /// that it should wake up and take care of publishing it to cobra - /// To shutdown the worker thread one has to set the _stop boolean to true. - /// This is done in the destructor - std::queue _queue; - mutable std::mutex _queue_mutex; - std::condition_variable _condition; - std::atomic _stop; - std::thread _thread; - }; - -} // namespace ix diff --git a/ixcobra/ixcobra/README.md b/ixcobra/ixcobra/README.md deleted file mode 100644 index a852456e..00000000 --- a/ixcobra/ixcobra/README.md +++ /dev/null @@ -1 +0,0 @@ -Client code to publish to a real time analytic system, described in [https://bsergean.github.io/redis_conf_2019/slides.html#1](link). diff --git a/ixredis/CMakeLists.txt b/ixredis/CMakeLists.txt deleted file mode 100644 index b2118702..00000000 --- a/ixredis/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2020 Machine Zone, Inc. All rights reserved. -# - -set (IXREDIS_SOURCES - ixredis/IXRedisClient.cpp - ixredis/IXRedisServer.cpp -) - -set (IXREDIS_HEADERS - ixredis/IXRedisClient.h - ixredis/IXRedisServer.h -) - -add_library(ixredis STATIC - ${IXREDIS_SOURCES} - ${IXREDIS_HEADERS} -) - -set(IXREDIS_INCLUDE_DIRS - . - .. - ../ixcore - ../ixwebsocket) - -target_include_directories( ixredis PUBLIC ${IXREDIS_INCLUDE_DIRS} ) diff --git a/ixredis/ixredis/IXRedisClient.cpp b/ixredis/ixredis/IXRedisClient.cpp deleted file mode 100644 index 44d9c489..00000000 --- a/ixredis/ixredis/IXRedisClient.cpp +++ /dev/null @@ -1,457 +0,0 @@ -/* - * IXRedisClient.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXRedisClient.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ix -{ - bool RedisClient::connect(const std::string& hostname, int port) - { - bool tls = false; - std::string errorMsg; - SocketTLSOptions tlsOptions; - _socket = createSocket(tls, -1, errorMsg, tlsOptions); - - if (!_socket) - { - return false; - } - - CancellationRequest cancellationRequest = []() -> bool { return false; }; - - std::string errMsg; - return _socket->connect(hostname, port, errMsg, cancellationRequest); - } - - void RedisClient::stop() - { - _stop = true; - } - - bool RedisClient::auth(const std::string& password, std::string& response) - { - response.clear(); - - if (!_socket) return false; - - std::stringstream ss; - ss << "AUTH "; - ss << password; - ss << "\r\n"; - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - return false; - } - - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - return false; - } - - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - response = line; - return lineValid; - } - - std::string RedisClient::writeString(const std::string& str) - { - std::stringstream ss; - ss << "$"; - ss << str.size(); - ss << "\r\n"; - ss << str; - ss << "\r\n"; - - return ss.str(); - } - - bool RedisClient::publish(const std::string& channel, - const std::string& message, - std::string& errMsg) - { - errMsg.clear(); - - if (!_socket) - { - errMsg = "socket is not initialized"; - return false; - } - - std::stringstream ss; - ss << "*3\r\n"; - ss << writeString("PUBLISH"); - ss << writeString(channel); - ss << writeString(message); - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return false; - } - - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - errMsg = "Error while polling for result"; - return false; - } - - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - // A successful response starts with a : - if (line.empty() || line[0] != ':') - { - errMsg = line; - return false; - } - - return lineValid; - } - - // - // FIXME: we assume that redis never return errors... - // - bool RedisClient::subscribe(const std::string& channel, - const OnRedisSubscribeResponseCallback& responseCallback, - const OnRedisSubscribeCallback& callback) - { - _stop = false; - - if (!_socket) return false; - - std::stringstream ss; - ss << "*2\r\n"; - ss << writeString("SUBSCRIBE"); - ss << writeString(channel); - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - return false; - } - - // Wait 1s for the response - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - return false; - } - - // build the response as a single string - std::stringstream oss; - - // Read the first line of the response - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - oss << line; - - if (!lineValid) return false; - - // There are 5 items for the subscribe reply - for (int i = 0; i < 5; ++i) - { - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - oss << line; - - if (!lineValid) return false; - } - - responseCallback(oss.str()); - - // Wait indefinitely for new messages - while (true) - { - if (_stop) break; - - // Wait until something is ready to read - int timeoutMs = 10; - auto pollResult = _socket->isReadyToRead(timeoutMs); - if (pollResult == PollResultType::Error) - { - return false; - } - - if (pollResult == PollResultType::Timeout) - { - continue; - } - - // The first line of the response describe the return type, - // => *3 (an array of 3 elements) - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - int arraySize; - { - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> arraySize; - } - - // There are 6 items for each received message - for (int i = 0; i < arraySize; ++i) - { - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - // Messages are string, which start with a string size - // => $7 (7 bytes) - int stringSize; - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - - auto readResult = _socket->readBytes(stringSize, nullptr, nullptr); - if (!readResult.first) return false; - - if (i == 2) - { - // The message is the 3rd element. - callback(readResult.second); - } - - // read last 2 bytes (\r\n) - char c; - _socket->readByte(&c, nullptr); - _socket->readByte(&c, nullptr); - } - } - - return true; - } - - std::string RedisClient::prepareXaddCommand(const std::string& stream, - const std::string& message, - int maxLen) - { - std::stringstream ss; - ss << "*8\r\n"; - ss << writeString("XADD"); - ss << writeString(stream); - ss << writeString("MAXLEN"); - ss << writeString("~"); - ss << writeString(std::to_string(maxLen)); - ss << writeString("*"); - ss << writeString("field"); - ss << writeString(message); - - return ss.str(); - } - - std::string RedisClient::xadd(const std::string& stream, - const std::string& message, - int maxLen, - std::string& errMsg) - { - errMsg.clear(); - - if (!_socket) - { - errMsg = "socket is not initialized"; - return std::string(); - } - - std::string command = prepareXaddCommand(stream, message, maxLen); - - bool sent = _socket->writeBytes(command, nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return std::string(); - } - - return readXaddReply(errMsg); - } - - std::string RedisClient::readXaddReply(std::string& errMsg) - { - // Read result - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - errMsg = "Error while polling for result"; - return std::string(); - } - - // First line is the string length - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) - { - errMsg = "Error while polling for result"; - return std::string(); - } - - int stringSize; - { - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - } - - // Read the result, which is the stream id computed by the redis server - lineResult = _socket->readLine(nullptr); - lineValid = lineResult.first; - line = lineResult.second; - - std::string streamId = line.substr(0, stringSize - 1); - return streamId; - } - - bool RedisClient::sendCommand(const std::string& commands, - int commandsCount, - std::string& errMsg) - { - bool sent = _socket->writeBytes(commands, nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return false; - } - - bool success = true; - - for (int i = 0; i < commandsCount; ++i) - { - auto reply = readXaddReply(errMsg); - if (reply == std::string()) - { - success = false; - } - } - - return success; - } - - std::pair RedisClient::send( - const std::vector& args, - std::string& errMsg) - { - std::stringstream ss; - ss << "*"; - ss << std::to_string(args.size()); - ss << "\r\n"; - - for (auto&& arg : args) - { - ss << writeString(arg); - } - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return std::make_pair(RespType::Error, ""); - } - - return readResponse(errMsg); - } - - std::pair RedisClient::readResponse(std::string& errMsg) - { - // Read result - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - errMsg = "Error while polling for result"; - return std::make_pair(RespType::Error, ""); - } - - // First line is the string length - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) - { - errMsg = "Error while polling for result"; - return std::make_pair(RespType::Error, ""); - } - - std::string response; - - if (line[0] == '+') // Simple string - { - std::stringstream ss; - response = line.substr(1, line.size() - 3); - return std::make_pair(RespType::String, response); - } - else if (line[0] == '-') // Errors - { - std::stringstream ss; - response = line.substr(1, line.size() - 3); - return std::make_pair(RespType::Error, response); - } - else if (line[0] == ':') // Integers - { - std::stringstream ss; - response = line.substr(1, line.size() - 3); - return std::make_pair(RespType::Integer, response); - } - else if (line[0] == '$') // Bulk strings - { - int stringSize; - { - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - } - - // Read the result, which is the stream id computed by the redis server - lineResult = _socket->readLine(nullptr); - lineValid = lineResult.first; - line = lineResult.second; - - std::string str = line.substr(0, stringSize); - return std::make_pair(RespType::String, str); - } - else - { - errMsg = "Unhandled response type"; - return std::make_pair(RespType::Unknown, std::string()); - } - } - - std::string RedisClient::getRespTypeDescription(RespType respType) - { - switch (respType) - { - case RespType::Integer: return "integer"; - case RespType::Error: return "error"; - case RespType::String: return "string"; - default: return "unknown"; - } - } -} // namespace ix diff --git a/ixredis/ixredis/IXRedisClient.h b/ixredis/ixredis/IXRedisClient.h deleted file mode 100644 index c03629f7..00000000 --- a/ixredis/ixredis/IXRedisClient.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * IXRedisClient.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace ix -{ - enum class RespType : int - { - String = 0, - Error = 1, - Integer = 2, - Unknown = 3 - }; - - class RedisClient - { - public: - using OnRedisSubscribeResponseCallback = std::function; - using OnRedisSubscribeCallback = std::function; - - RedisClient() - : _stop(false) - { - } - ~RedisClient() = default; - - bool connect(const std::string& hostname, int port); - - bool auth(const std::string& password, std::string& response); - - // Publish / Subscribe - bool publish(const std::string& channel, const std::string& message, std::string& errMsg); - - bool subscribe(const std::string& channel, - const OnRedisSubscribeResponseCallback& responseCallback, - const OnRedisSubscribeCallback& callback); - - // XADD - std::string xadd(const std::string& channel, - const std::string& message, - int maxLen, - std::string& errMsg); - std::string prepareXaddCommand(const std::string& stream, - const std::string& message, - int maxLen); - std::string readXaddReply(std::string& errMsg); - bool sendCommand( - const std::string& commands, int commandsCount, std::string& errMsg); - - // Arbitrary commands - std::pair send( - const std::vector& args, - std::string& errMsg); - std::pair readResponse(std::string& errMsg); - - std::string getRespTypeDescription(RespType respType); - - void stop(); - - private: - std::string writeString(const std::string& str); - - std::unique_ptr _socket; - std::atomic _stop; - }; -} // namespace ix diff --git a/ixredis/ixredis/IXRedisServer.cpp b/ixredis/ixredis/IXRedisServer.cpp deleted file mode 100644 index 1267ca2a..00000000 --- a/ixredis/ixredis/IXRedisServer.cpp +++ /dev/null @@ -1,287 +0,0 @@ -/* - * IXRedisServer.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXRedisServer.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace ix -{ - RedisServer::RedisServer( - int port, const std::string& host, int backlog, size_t maxConnections, int addressFamily) - : SocketServer(port, host, backlog, maxConnections, addressFamily) - , _connectedClientsCount(0) - , _stopHandlingConnections(false) - { - ; - } - - RedisServer::~RedisServer() - { - stop(); - } - - void RedisServer::stop() - { - stopAcceptingConnections(); - - _stopHandlingConnections = true; - while (_connectedClientsCount != 0) - { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - _stopHandlingConnections = false; - - SocketServer::stop(); - } - - void RedisServer::handleConnection(std::unique_ptr socket, - std::shared_ptr connectionState) - { - logInfo("New connection from remote ip " + connectionState->getRemoteIp()); - - _connectedClientsCount++; - - while (!_stopHandlingConnections) - { - std::vector tokens; - if (!parseRequest(socket, tokens)) - { - if (_stopHandlingConnections) - { - logError("Cancellation requested"); - } - else - { - logError("Error parsing request"); - } - break; - } - - bool success = false; - - // publish - if (tokens[0] == "COMMAND") - { - success = handleCommand(socket, tokens); - } - else if (tokens[0] == "PUBLISH") - { - success = handlePublish(socket, tokens); - } - else if (tokens[0] == "SUBSCRIBE") - { - success = handleSubscribe(socket, tokens); - } - - if (!success) - { - if (_stopHandlingConnections) - { - logError("Cancellation requested"); - } - else - { - logError("Error processing request for command: " + tokens[0]); - } - break; - } - } - - cleanupSubscribers(socket); - - logInfo("Connection closed for connection id " + connectionState->getId()); - connectionState->setTerminated(); - - _connectedClientsCount--; - } - - void RedisServer::cleanupSubscribers(std::unique_ptr& socket) - { - std::lock_guard lock(_mutex); - - for (auto&& it : _subscribers) - { - it.second.erase(socket.get()); - } - - for (auto&& it : _subscribers) - { - std::stringstream ss; - ss << "Subscription id: " << it.first << " #subscribers: " << it.second.size(); - - logInfo(ss.str()); - } - } - - size_t RedisServer::getConnectedClientsCount() - { - return _connectedClientsCount; - } - - bool RedisServer::startsWith(const std::string& str, const std::string& start) - { - return str.compare(0, start.length(), start) == 0; - } - - std::string RedisServer::writeString(const std::string& str) - { - std::stringstream ss; - ss << "$"; - ss << str.size(); - ss << "\r\n"; - ss << str; - ss << "\r\n"; - - return ss.str(); - } - - bool RedisServer::parseRequest(std::unique_ptr& socket, - std::vector& tokens) - { - // Parse first line - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - auto lineResult = socket->readLine(cb); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - std::string str = line.substr(1); - std::stringstream ss; - ss << str; - int count; - ss >> count; - - for (int i = 0; i < count; ++i) - { - auto lineResult = socket->readLine(cb); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - int stringSize; - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - - auto readResult = socket->readBytes(stringSize, nullptr, nullptr); - - if (!readResult.first) return false; - - // read last 2 bytes (\r\n) - char c; - socket->readByte(&c, nullptr); - socket->readByte(&c, nullptr); - - tokens.push_back(readResult.second); - } - - return true; - } - - bool RedisServer::handleCommand(std::unique_ptr& socket, - const std::vector& tokens) - { - if (tokens.size() != 1) return false; - - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - std::stringstream ss; - - // return 2 nested arrays - ss << "*2\r\n"; - - // - // publish - // - ss << "*6\r\n"; - ss << writeString("publish"); // 1 - ss << ":3\r\n"; // 2 - ss << "*0\r\n"; // 3 - ss << ":1\r\n"; // 4 - ss << ":2\r\n"; // 5 - ss << ":1\r\n"; // 6 - - // - // subscribe - // - ss << "*6\r\n"; - ss << writeString("subscribe"); // 1 - ss << ":2\r\n"; // 2 - ss << "*0\r\n"; // 3 - ss << ":1\r\n"; // 4 - ss << ":1\r\n"; // 5 - ss << ":1\r\n"; // 6 - - socket->writeBytes(ss.str(), cb); - - return true; - } - - bool RedisServer::handleSubscribe(std::unique_ptr& socket, - const std::vector& tokens) - { - if (tokens.size() != 2) return false; - - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - std::string channel = tokens[1]; - - // Respond - socket->writeBytes("*3\r\n", cb); - socket->writeBytes(writeString("subscribe"), cb); - socket->writeBytes(writeString(channel), cb); - socket->writeBytes(":1\r\n", cb); - - std::lock_guard lock(_mutex); - _subscribers[channel].insert(socket.get()); - - return true; - } - - bool RedisServer::handlePublish(std::unique_ptr& socket, - const std::vector& tokens) - { - if (tokens.size() != 3) return false; - - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - std::string channel = tokens[1]; - std::string data = tokens[2]; - - // now dispatch the message to subscribers (write custom method) - std::lock_guard lock(_mutex); - auto it = _subscribers.find(channel); - if (it == _subscribers.end()) - { - // return the number of clients that received the message, 0 in that case - socket->writeBytes(":0\r\n", cb); - return true; - } - - auto subscribers = it->second; - for (auto jt : subscribers) - { - jt->writeBytes("*3\r\n", cb); - jt->writeBytes(writeString("message"), cb); - jt->writeBytes(writeString(channel), cb); - jt->writeBytes(writeString(data), cb); - } - - // return the number of clients that received the message. - std::stringstream ss; - ss << ":" << std::to_string(subscribers.size()) << "\r\n"; - socket->writeBytes(ss.str(), cb); - - return true; - } - -} // namespace ix diff --git a/ixredis/ixredis/IXRedisServer.h b/ixredis/ixredis/IXRedisServer.h deleted file mode 100644 index 75db0ee2..00000000 --- a/ixredis/ixredis/IXRedisServer.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * IXRedisServer.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include // pair -#include // pair - -namespace ix -{ - class RedisServer final : public SocketServer - { - public: - RedisServer(int port = SocketServer::kDefaultPort, - const std::string& host = SocketServer::kDefaultHost, - int backlog = SocketServer::kDefaultTcpBacklog, - size_t maxConnections = SocketServer::kDefaultMaxConnections, - int addressFamily = SocketServer::kDefaultAddressFamily); - virtual ~RedisServer(); - virtual void stop() final; - - private: - // Member variables - std::atomic _connectedClientsCount; - - // Subscribers - // We could store connection states in there, to add better debugging - // since a connection state has a readable ID - std::map> _subscribers; - std::mutex _mutex; - - std::atomic _stopHandlingConnections; - - // Methods - virtual void handleConnection(std::unique_ptr, - std::shared_ptr connectionState) final; - virtual size_t getConnectedClientsCount() final; - - bool startsWith(const std::string& str, const std::string& start); - std::string writeString(const std::string& str); - - bool parseRequest(std::unique_ptr& socket, std::vector& tokens); - - bool handlePublish(std::unique_ptr& socket, const std::vector& tokens); - - bool handleSubscribe(std::unique_ptr& socket, - const std::vector& tokens); - - bool handleCommand(std::unique_ptr& socket, const std::vector& tokens); - - void cleanupSubscribers(std::unique_ptr& socket); - }; -} // namespace ix diff --git a/ixsentry/CMakeLists.txt b/ixsentry/CMakeLists.txt deleted file mode 100644 index 3f22f140..00000000 --- a/ixsentry/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXSENTRY_SOURCES - ixsentry/IXSentryClient.cpp -) - -set (IXSENTRY_HEADERS - ixsentry/IXSentryClient.h -) - -add_library(ixsentry STATIC - ${IXSENTRY_SOURCES} - ${IXSENTRY_HEADERS} -) - -# -# Using try_compile or other techniques to detect std::regex -# availability is hard, so resorting to an ugly compiler and compiler -# version check. -# -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9.0") - else() - target_compile_definitions( ixsentry PUBLIC HAVE_STD_REGEX=1 ) - endif() -else() - target_compile_definitions( ixsentry PUBLIC HAVE_STD_REGEX=1 ) -endif() - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) -endif() - -set(IXSENTRY_INCLUDE_DIRS - . - .. - ../ixcore - ${JSONCPP_INCLUDE_DIRS}) - -target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} ) diff --git a/ixsentry/ixsentry/IXSentryClient.cpp b/ixsentry/ixsentry/IXSentryClient.cpp deleted file mode 100644 index d9abc600..00000000 --- a/ixsentry/ixsentry/IXSentryClient.cpp +++ /dev/null @@ -1,316 +0,0 @@ -/* - * IXSentryClient.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone. All rights reserved. - */ - -#include "IXSentryClient.h" - -#include -#include -#include -#include -#include -#include -#include - - -namespace ix -{ - SentryClient::SentryClient(const std::string& dsn) - : _dsn(dsn) - , _validDsn(false) -#ifdef HAVE_STD_REGEX - , _luaFrameRegex("\t([^/]+):([0-9]+): in function ['<]([^/]+)['>]") -#endif - , _httpClient(std::make_shared(true)) - { -#ifdef HAVE_STD_REGEX - const std::regex dsnRegex("(http[s]?)://([^:]+):([^@]+)@([^/]+)/([0-9]+)"); - std::smatch group; - - if (std::regex_match(dsn, group, dsnRegex) && group.size() == 6) - { - _validDsn = true; - - const auto scheme = group.str(1); - const auto host = group.str(4); - const auto project_id = group.str(5); - _url = scheme + "://" + host + "/api/" + project_id + "/store/"; - - _publicKey = group.str(2); - _secretKey = group.str(3); - } -#endif - } - - void SentryClient::setTLSOptions(const SocketTLSOptions& tlsOptions) - { - _httpClient->setTLSOptions(tlsOptions); - } - - int64_t SentryClient::getTimestamp() - { - const auto tp = std::chrono::system_clock::now(); - const auto dur = tp.time_since_epoch(); - return std::chrono::duration_cast(dur).count(); - } - - std::string SentryClient::getIso8601() - { - std::time_t now; - std::time(&now); - char buf[sizeof("2011-10-08T07:07:09Z")]; - std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", std::gmtime(&now)); - return buf; - } - - std::string SentryClient::computeAuthHeader() - { - std::string securityHeader("Sentry sentry_version=5"); - securityHeader += ",sentry_client=ws/"; - securityHeader += std::string(IX_WEBSOCKET_VERSION); - securityHeader += ",sentry_timestamp=" + std::to_string(SentryClient::getTimestamp()); - securityHeader += ",sentry_key=" + _publicKey; - securityHeader += ",sentry_secret=" + _secretKey; - - return securityHeader; - } - - Json::Value SentryClient::parseLuaStackTrace(const std::string& stack) - { - Json::Value frames; - -#ifdef HAVE_STD_REGEX - // Split by lines - std::string line; - std::stringstream tokenStream(stack); - - std::smatch group; - - while (std::getline(tokenStream, line)) - { - // MapScene.lua:2169: in function 'singleCB' - if (std::regex_match(line, group, _luaFrameRegex)) - { - const auto fileName = group.str(1); - const auto linenoStr = group.str(2); - const auto function = group.str(3); - - std::stringstream ss; - ss << linenoStr; - uint64_t lineno; - ss >> lineno; - - Json::Value frame; - frame["lineno"] = Json::UInt64(lineno); - frame["filename"] = fileName; - frame["function"] = function; - - frames.append(frame); - } - } - - std::reverse(frames.begin(), frames.end()); -#endif - - return frames; - } - - std::string parseExceptionName(const std::string& stack) - { - // Split by lines - std::string line; - std::stringstream tokenStream(stack); - - // Extract the first line - std::getline(tokenStream, line); - - return line; - } - - std::string SentryClient::computePayload(const Json::Value& msg) - { - Json::Value payload; - - // - // "tags": [ - // [ - // "a", - // "b" - // ], - // ] - // - Json::Value tags(Json::arrayValue); - - payload["platform"] = "python"; - payload["sdk"]["name"] = "ws"; - payload["sdk"]["version"] = IX_WEBSOCKET_VERSION; - payload["timestamp"] = SentryClient::getIso8601(); - - bool isNoisyTypes = msg["id"].asString() == "game_noisytypes_id"; - - std::string stackTraceFieldName = isNoisyTypes ? "traceback" : "stack"; - std::string stack; - std::string message; - - if (isNoisyTypes) - { - stack = msg["data"][stackTraceFieldName].asString(); - message = parseExceptionName(stack); - } - else // logging - { - if (msg["data"].isMember("info")) - { - stack = msg["data"]["info"][stackTraceFieldName].asString(); - message = msg["data"]["info"]["message"].asString(); - - if (msg["data"].isMember("tags")) - { - auto members = msg["data"]["tags"].getMemberNames(); - - for (auto member : members) - { - Json::Value tag; - tag.append(member); - tag.append(msg["data"]["tags"][member]); - tags.append(tag); - } - } - - if (msg["data"]["info"].isMember("level_str")) - { - // https://docs.sentry.io/enriching-error-data/context/?platform=python#setting-the-level - std::string level = msg["data"]["info"]["level_str"].asString(); - if (level == "critical") - { - level = "fatal"; - } - payload["level"] = level; - } - } - else - { - stack = msg["data"][stackTraceFieldName].asString(); - message = msg["data"]["message"].asString(); - } - } - - Json::Value exception; - exception["stacktrace"]["frames"] = parseLuaStackTrace(stack); - exception["value"] = message; - - payload["exception"].append(exception); - - Json::Value extra; - extra["cobra_event"] = msg; - - // Builtin tags - Json::Value gameTag; - gameTag.append("game"); - gameTag.append(msg["device"]["game"]); - tags.append(gameTag); - - Json::Value userIdTag; - userIdTag.append("userid"); - userIdTag.append(msg["device"]["user_id"]); - tags.append(userIdTag); - - Json::Value environmentTag; - environmentTag.append("environment"); - environmentTag.append(msg["device"]["environment"]); - tags.append(environmentTag); - - Json::Value clientVersionTag; - clientVersionTag.append("client_version"); - clientVersionTag.append(msg["device"]["app_version"]); - tags.append(clientVersionTag); - - payload["tags"] = tags; - - return _jsonWriter.write(payload); - } - - void SentryClient::send( - const Json::Value& msg, - bool verbose, - const OnResponseCallback& onResponseCallback) - { - auto args = _httpClient->createRequest(); - args->url = _url; - args->verb = HttpClient::kPost; - args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader(); - args->connectTimeout = 60; - args->transferTimeout = 5 * 60; - args->followRedirects = true; - args->verbose = verbose; - args->logger = [](const std::string& msg) { CoreLogger::log(msg.c_str()); }; - args->body = computePayload(msg); - - _httpClient->performRequest(args, onResponseCallback); - } - - // https://sentry.io/api/12345/minidump?sentry_key=abcdefgh"); - std::string SentryClient::computeUrl(const std::string& project, const std::string& key) - { - std::stringstream ss; - ss << "https://sentry.io/api/" << project << "/minidump?sentry_key=" << key; - - return ss.str(); - } - - // - // curl -v -X POST -F upload_file_minidump=@ws/crash.dmp - // 'https://sentry.io/api/123456/minidump?sentry_key=12344567890' - // - void SentryClient::uploadMinidump(const std::string& sentryMetadata, - const std::string& minidumpBytes, - const std::string& project, - const std::string& key, - bool verbose, - const OnResponseCallback& onResponseCallback) - { - std::string multipartBoundary = _httpClient->generateMultipartBoundary(); - - auto args = _httpClient->createRequest(); - args->verb = HttpClient::kPost; - args->connectTimeout = 60; - args->transferTimeout = 5 * 60; - args->followRedirects = true; - args->verbose = verbose; - args->multipartBoundary = multipartBoundary; - args->logger = [](const std::string& msg) { CoreLogger::log(msg.c_str()); }; - - HttpFormDataParameters httpFormDataParameters; - httpFormDataParameters["upload_file_minidump"] = minidumpBytes; - - HttpParameters httpParameters; - httpParameters["sentry"] = sentryMetadata; - - args->url = computeUrl(project, key); - args->body = _httpClient->serializeHttpFormDataParameters( - multipartBoundary, httpFormDataParameters, httpParameters); - - _httpClient->performRequest(args, onResponseCallback); - } - - void SentryClient::uploadPayload(const Json::Value& payload, - bool verbose, - const OnResponseCallback& onResponseCallback) - { - auto args = _httpClient->createRequest(); - args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader(); - args->verb = HttpClient::kPost; - args->connectTimeout = 60; - args->transferTimeout = 5 * 60; - args->followRedirects = true; - args->verbose = verbose; - args->logger = [](const std::string& msg) { CoreLogger::log(msg.c_str()); }; - - args->url = _url; - args->body = _jsonWriter.write(payload); - - _httpClient->performRequest(args, onResponseCallback); - } -} // namespace ix diff --git a/ixsentry/ixsentry/IXSentryClient.h b/ixsentry/ixsentry/IXSentryClient.h deleted file mode 100644 index fd7ee5cd..00000000 --- a/ixsentry/ixsentry/IXSentryClient.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * IXSentryClient.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include -#ifdef HAVE_STD_REGEX -#include -#endif - -namespace ix -{ - class SentryClient - { - public: - SentryClient(const std::string& dsn); - ~SentryClient() = default; - - void send(const Json::Value& msg, - bool verbose, - const OnResponseCallback& onResponseCallback); - - void uploadMinidump(const std::string& sentryMetadata, - const std::string& minidumpBytes, - const std::string& project, - const std::string& key, - bool verbose, - const OnResponseCallback& onResponseCallback); - - void uploadPayload(const Json::Value& payload, - bool verbose, - const OnResponseCallback& onResponseCallback); - - Json::Value parseLuaStackTrace(const std::string& stack); - - // Mostly for testing - void setTLSOptions(const SocketTLSOptions& tlsOptions); - - - private: - int64_t getTimestamp(); - std::string computeAuthHeader(); - std::string getIso8601(); - std::string computePayload(const Json::Value& msg); - - std::string computeUrl(const std::string& project, const std::string& key); - - void displayReponse(HttpResponsePtr response); - - std::string _dsn; - bool _validDsn; - std::string _url; - - // Used for authentication with a header - std::string _publicKey; - std::string _secretKey; - - Json::FastWriter _jsonWriter; - -#ifdef HAVE_STD_REGEX - std::regex _luaFrameRegex; -#endif - - std::shared_ptr _httpClient; - }; - -} // namespace ix diff --git a/ixsnake/CMakeLists.txt b/ixsnake/CMakeLists.txt deleted file mode 100644 index bd240d00..00000000 --- a/ixsnake/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXSNAKE_SOURCES - ixsnake/IXSnakeServer.cpp - ixsnake/IXSnakeProtocol.cpp - ixsnake/IXAppConfig.cpp - ixsnake/IXStreamSql.cpp -) - -set (IXSNAKE_HEADERS - ixsnake/IXSnakeServer.h - ixsnake/IXSnakeProtocol.h - ixsnake/IXAppConfig.h - ixsnake/IXStreamSql.h -) - -add_library(ixsnake STATIC - ${IXSNAKE_SOURCES} - ${IXSNAKE_HEADERS} -) - -set(IXSNAKE_INCLUDE_DIRS - . - .. - ../ixcore - ../ixcrypto - ../ixwebsocket - ../ixredis - ../third_party) - -target_include_directories( ixsnake PUBLIC ${IXSNAKE_INCLUDE_DIRS} ) diff --git a/ixsnake/ixsnake/IXAppConfig.cpp b/ixsnake/ixsnake/IXAppConfig.cpp deleted file mode 100644 index d1eb68b1..00000000 --- a/ixsnake/ixsnake/IXAppConfig.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * IXSnakeProtocol.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXAppConfig.h" - -#include "IXSnakeProtocol.h" -#include -#include - -namespace snake -{ - bool isAppKeyValid(const AppConfig& appConfig, std::string appkey) - { - return appConfig.apps.count(appkey) != 0; - } - - std::string getRoleSecret(const AppConfig& appConfig, std::string appkey, std::string role) - { - if (!isAppKeyValid(appConfig, appkey)) - { - std::cerr << "Missing appkey " << appkey << std::endl; - return std::string(); - } - - auto roles = appConfig.apps[appkey]["roles"]; - if (roles.count(role) == 0) - { - std::cerr << "Missing role " << role << std::endl; - return std::string(); - } - - auto channel = roles[role]["secret"]; - return channel; - } - - std::string generateNonce() - { - return ix::uuid4(); - } - - void dumpConfig(const AppConfig& appConfig) - { - for (auto&& host : appConfig.redisHosts) - { - std::cout << "redis host: " << host << std::endl; - } - - std::cout << "redis password: " << appConfig.redisPassword << std::endl; - std::cout << "redis port: " << appConfig.redisPort << std::endl; - } -} // namespace snake diff --git a/ixsnake/ixsnake/IXAppConfig.h b/ixsnake/ixsnake/IXAppConfig.h deleted file mode 100644 index bdc929df..00000000 --- a/ixsnake/ixsnake/IXAppConfig.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * IXAppConfig.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include - -namespace snake -{ - struct AppConfig - { - // Server - std::string hostname; - int port; - - // Redis - std::vector redisHosts; - int redisPort; - std::string redisPassword; - - // AppKeys - nlohmann::json apps; - - // TLS options - ix::SocketTLSOptions socketTLSOptions; - - // Misc - bool verbose; - bool disablePong; - - // If non empty, every published message gets republished to a given channel - std::string republishChannel; - }; - - bool isAppKeyValid(const AppConfig& appConfig, std::string appkey); - - std::string getRoleSecret(const AppConfig& appConfig, std::string appkey, std::string role); - - std::string generateNonce(); - - void dumpConfig(const AppConfig& appConfig); -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeConnectionState.h b/ixsnake/ixsnake/IXSnakeConnectionState.h deleted file mode 100644 index ea0817b3..00000000 --- a/ixsnake/ixsnake/IXSnakeConnectionState.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * IXSnakeConnectionState.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include "IXStreamSql.h" - -namespace snake -{ - class SnakeConnectionState : public ix::ConnectionState - { - public: - virtual ~SnakeConnectionState() - { - stopSubScriptionThread(); - } - - std::string getNonce() - { - return _nonce; - } - - void setNonce(const std::string& nonce) - { - _nonce = nonce; - } - - std::string appkey() - { - return _appkey; - } - - void setAppkey(const std::string& appkey) - { - _appkey = appkey; - } - - std::string role() - { - return _role; - } - - void setRole(const std::string& role) - { - _role = role; - } - - ix::RedisClient& redisClient() - { - return _redisClient; - } - - void stopSubScriptionThread() - { - if (subscriptionThread.joinable()) - { - subscriptionRedisClient.stop(); - subscriptionThread.join(); - } - } - - // We could make those accessible through methods - std::thread subscriptionThread; - std::string appChannel; - std::string subscriptionId; - uint64_t id; - std::unique_ptr streamSql; - ix::RedisClient subscriptionRedisClient; - ix::RedisClient::OnRedisSubscribeResponseCallback onRedisSubscribeResponseCallback; - ix::RedisClient::OnRedisSubscribeCallback onRedisSubscribeCallback; - - private: - std::string _nonce; - std::string _role; - std::string _appkey; - - ix::RedisClient _redisClient; - }; -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeProtocol.cpp b/ixsnake/ixsnake/IXSnakeProtocol.cpp deleted file mode 100644 index 40dace0e..00000000 --- a/ixsnake/ixsnake/IXSnakeProtocol.cpp +++ /dev/null @@ -1,320 +0,0 @@ -/* - * IXSnakeProtocol.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXSnakeProtocol.h" - -#include "IXAppConfig.h" -#include "IXSnakeConnectionState.h" -#include "nlohmann/json.hpp" -#include -#include -#include -#include -#include -#include - -namespace snake -{ - void handleError(const std::string& action, - ix::WebSocket& ws, - uint64_t pduId, - const std::string& errMsg) - { - std::string actionError(action); - actionError += "/error"; - - nlohmann::json response = { - {"action", actionError}, {"id", pduId}, {"body", {{"reason", errMsg}}}}; - ws.sendText(response.dump()); - } - - void handleHandshake(std::shared_ptr state, - ix::WebSocket& ws, - const nlohmann::json& pdu, - uint64_t pduId) - { - std::string role = pdu["body"]["data"]["role"]; - - state->setNonce(generateNonce()); - state->setRole(role); - - nlohmann::json response = { - {"action", "auth/handshake/ok"}, - {"id", pduId}, - {"body", - { - {"data", {{"nonce", state->getNonce()}, {"connection_id", state->getId()}}}, - }}}; - - auto serializedResponse = response.dump(); - - ws.sendText(serializedResponse); - } - - void handleAuth(std::shared_ptr state, - ix::WebSocket& ws, - const AppConfig& appConfig, - const nlohmann::json& pdu, - uint64_t pduId) - { - auto secret = getRoleSecret(appConfig, state->appkey(), state->role()); - - if (secret.empty()) - { - nlohmann::json response = { - {"action", "auth/authenticate/error"}, - {"id", pduId}, - {"body", {{"error", "authentication_failed"}, {"reason", "invalid secret"}}}}; - ws.sendText(response.dump()); - return; - } - - auto nonce = state->getNonce(); - auto serverHash = ix::hmac(nonce, secret); - std::string clientHash = pdu["body"]["credentials"]["hash"]; - - if (serverHash != clientHash) - { - nlohmann::json response = { - {"action", "auth/authenticate/error"}, - {"id", pdu.value("id", 1)}, - {"body", {{"error", "authentication_failed"}, {"reason", "invalid hash"}}}}; - ws.sendText(response.dump()); - return; - } - - nlohmann::json response = { - {"action", "auth/authenticate/ok"}, {"id", pdu.value("id", 1)}, {"body", {}}}; - - ws.sendText(response.dump()); - } - - void handlePublish(std::shared_ptr state, - ix::WebSocket& ws, - const AppConfig& appConfig, - const nlohmann::json& pdu, - uint64_t pduId) - { - std::vector channels; - - auto body = pdu["body"]; - if (body.find("channels") != body.end()) - { - for (auto&& channel : body["channels"]) - { - channels.push_back(channel); - } - } - else if (body.find("channel") != body.end()) - { - channels.push_back(body["channel"]); - } - else - { - std::stringstream ss; - ss << "Missing channels or channel field in publish data"; - handleError("rtm/publish", ws, pduId, ss.str()); - return; - } - - // add an extra channel if the config has one specified - if (!appConfig.republishChannel.empty()) - { - channels.push_back(appConfig.republishChannel); - } - - for (auto&& channel : channels) - { - std::stringstream ss; - ss << state->appkey() << "::" << channel; - - std::string errMsg; - if (!state->redisClient().publish(ss.str(), pdu.dump(), errMsg)) - { - std::stringstream ss; - ss << "Cannot publish to redis host " << errMsg; - handleError("rtm/publish", ws, pduId, ss.str()); - return; - } - } - - nlohmann::json response = { - {"action", "rtm/publish/ok"}, {"id", pdu.value("id", 1)}, {"body", {}}}; - - ws.sendText(response.dump()); - } - - // - // FIXME: this is not cancellable. We should be able to cancel the redis subscription - // - void handleSubscribe(std::shared_ptr state, - ix::WebSocket& ws, - const AppConfig& appConfig, - const nlohmann::json& pdu, - uint64_t pduId) - { - std::string channel = pdu["body"]["channel"]; - state->subscriptionId = channel; - - std::stringstream ss; - ss << state->appkey() << "::" << channel; - - state->appChannel = ss.str(); - - ix::RedisClient& redisClient = state->subscriptionRedisClient; - int port = appConfig.redisPort; - - auto urls = appConfig.redisHosts; - std::string hostname(urls[0]); - - // Connect to redis first - if (!redisClient.connect(hostname, port)) - { - std::stringstream ss; - ss << "Cannot connect to redis host " << hostname << ":" << port; - handleError("rtm/subscribe", ws, pduId, ss.str()); - return; - } - - // Now authenticate, if needed - if (!appConfig.redisPassword.empty()) - { - std::string authResponse; - if (!redisClient.auth(appConfig.redisPassword, authResponse)) - { - std::stringstream ss; - ss << "Cannot authenticated to redis"; - handleError("rtm/subscribe", ws, pduId, ss.str()); - return; - } - } - - std::string filterStr; - if (pdu["body"].find("filter") != pdu["body"].end()) - { - std::string filterStr = pdu["body"]["filter"]; - } - state->streamSql = ix::make_unique(filterStr); - state->id = 0; - state->onRedisSubscribeCallback = [&ws, state](const std::string& messageStr) { - auto msg = nlohmann::json::parse(messageStr); - - msg = msg["body"]["message"]; - - if (state->streamSql->valid() && !state->streamSql->match(msg)) - { - return; - } - - nlohmann::json response = { - {"action", "rtm/subscription/data"}, - {"id", state->id++}, - {"body", - {{"subscription_id", state->subscriptionId}, {"position", "0-0"}, {"messages", {msg}}}}}; - - ws.sendText(response.dump()); - }; - - state->onRedisSubscribeResponseCallback = [&ws, state, pduId](const std::string& redisResponse) { - std::stringstream ss; - ss << "Redis Response: " << redisResponse << "..."; - ix::CoreLogger::log(ss.str().c_str()); - - // Success - nlohmann::json response = {{"action", "rtm/subscribe/ok"}, - {"id", pduId}, - {"body", {{"subscription_id", state->subscriptionId}}}}; - ws.sendText(response.dump()); - }; - - { - std::stringstream ss; - ss << "Subscribing to " << state->appChannel << "..."; - ix::CoreLogger::log(ss.str().c_str()); - } - - auto subscription = [&redisClient, state, &ws, pduId] - { - if (!redisClient.subscribe(state->appChannel, - state->onRedisSubscribeResponseCallback, - state->onRedisSubscribeCallback)) - { - std::stringstream ss; - ss << "Error subscribing to channel " << state->appChannel; - handleError("rtm/subscribe", ws, pduId, ss.str()); - return; - } - }; - - state->subscriptionThread = std::thread(subscription); - } - - void handleUnSubscribe(std::shared_ptr state, - ix::WebSocket& ws, - const nlohmann::json& pdu, - uint64_t pduId) - { - // extract subscription_id - auto body = pdu["body"]; - auto subscriptionId = body["subscription_id"]; - - state->stopSubScriptionThread(); - - nlohmann::json response = {{"action", "rtm/unsubscribe/ok"}, - {"id", pduId}, - {"body", {{"subscription_id", subscriptionId}}}}; - ws.sendText(response.dump()); - } - - void processCobraMessage(std::shared_ptr state, - ix::WebSocket& ws, - const AppConfig& appConfig, - const std::string& str) - { - nlohmann::json pdu; - try - { - pdu = nlohmann::json::parse(str); - } - catch (const nlohmann::json::parse_error& e) - { - std::stringstream ss; - ss << "malformed json pdu: " << e.what() << " -> " << str << ""; - - nlohmann::json response = {{"body", {{"error", "invalid_json"}, {"reason", ss.str()}}}}; - ws.sendText(response.dump()); - return; - } - - auto action = pdu["action"]; - uint64_t pduId = pdu.value("id", 1); - - if (action == "auth/handshake") - { - handleHandshake(state, ws, pdu, pduId); - } - else if (action == "auth/authenticate") - { - handleAuth(state, ws, appConfig, pdu, pduId); - } - else if (action == "rtm/publish") - { - handlePublish(state, ws, appConfig, pdu, pduId); - } - else if (action == "rtm/subscribe") - { - handleSubscribe(state, ws, appConfig, pdu, pduId); - } - else if (action == "rtm/unsubscribe") - { - handleUnSubscribe(state, ws, pdu, pduId); - } - else - { - std::cerr << "Unhandled action: " << action << std::endl; - } - } -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeProtocol.h b/ixsnake/ixsnake/IXSnakeProtocol.h deleted file mode 100644 index 4f73ca7b..00000000 --- a/ixsnake/ixsnake/IXSnakeProtocol.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * IXSnakeProtocol.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include - -namespace ix -{ - class WebSocket; -} - -namespace snake -{ - class SnakeConnectionState; - struct AppConfig; - - void processCobraMessage(std::shared_ptr state, - ix::WebSocket& ws, - const AppConfig& appConfig, - const std::string& str); -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeServer.cpp b/ixsnake/ixsnake/IXSnakeServer.cpp deleted file mode 100644 index e1c28c02..00000000 --- a/ixsnake/ixsnake/IXSnakeServer.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* - * IXSnakeServer.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXSnakeServer.h" - -#include "IXAppConfig.h" -#include "IXSnakeConnectionState.h" -#include "IXSnakeProtocol.h" -#include -#include -#include - - -namespace snake -{ - SnakeServer::SnakeServer(const AppConfig& appConfig) - : _appConfig(appConfig) - , _server(appConfig.port, appConfig.hostname) - { - _server.setTLSOptions(appConfig.socketTLSOptions); - - if (appConfig.disablePong) - { - _server.disablePong(); - } - - std::stringstream ss; - ss << "Listening on " << appConfig.hostname << ":" << appConfig.port; - ix::CoreLogger::log(ss.str().c_str()); - } - - // - // Parse appkey from this uri. Won't work if multiple args are present in the uri - // Uri: /v2?appkey=FC2F10139A2BAc53BB72D9db967b024f - // - std::string SnakeServer::parseAppKey(const std::string& path) - { - std::string::size_type idx; - - idx = path.rfind('='); - if (idx != std::string::npos) - { - std::string appkey = path.substr(idx + 1); - return appkey; - } - else - { - return std::string(); - } - } - - bool SnakeServer::run() - { - auto factory = []() -> std::shared_ptr { - return std::make_shared(); - }; - _server.setConnectionStateFactory(factory); - - _server.setOnClientMessageCallback( - [this](std::shared_ptr connectionState, - ix::WebSocket& webSocket, - const ix::WebSocketMessagePtr& msg) { - auto state = std::dynamic_pointer_cast(connectionState); - auto remoteIp = connectionState->getRemoteIp(); - - std::stringstream ss; - ss << "[" << state->getId() << "] "; - - ix::LogLevel logLevel = ix::LogLevel::Debug; - if (msg->type == ix::WebSocketMessageType::Open) - { - ss << "New connection" << std::endl; - ss << "remote ip: " << remoteIp << std::endl; - ss << "id: " << state->getId() << std::endl; - ss << "Uri: " << msg->openInfo.uri << std::endl; - ss << "Headers:" << std::endl; - for (auto it : msg->openInfo.headers) - { - ss << it.first << ": " << it.second << std::endl; - } - - std::string appkey = parseAppKey(msg->openInfo.uri); - state->setAppkey(appkey); - - // Connect to redis first - if (!state->redisClient().connect(_appConfig.redisHosts[0], - _appConfig.redisPort)) - { - ss << "Cannot connect to redis host" << std::endl; - logLevel = ix::LogLevel::Error; - } - } - else if (msg->type == ix::WebSocketMessageType::Close) - { - ss << "Closed connection" - << " code " << msg->closeInfo.code << " reason " - << msg->closeInfo.reason << std::endl; - } - else if (msg->type == ix::WebSocketMessageType::Error) - { - std::stringstream ss; - ss << "Connection error: " << msg->errorInfo.reason << std::endl; - ss << "#retries: " << msg->errorInfo.retries << std::endl; - ss << "Wait time(ms): " << msg->errorInfo.wait_time << std::endl; - ss << "HTTP Status: " << msg->errorInfo.http_status << std::endl; - logLevel = ix::LogLevel::Error; - } - else if (msg->type == ix::WebSocketMessageType::Fragment) - { - ss << "Received message fragment" << std::endl; - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - ss << "Received " << msg->wireSize << " bytes" << " " << msg->str << std::endl; - processCobraMessage(state, webSocket, _appConfig, msg->str); - } - - ix::CoreLogger::log(ss.str().c_str(), logLevel); - }); - - auto res = _server.listen(); - if (!res.first) - { - std::cerr << res.second << std::endl; - return false; - } - - _server.start(); - return true; - } - - void SnakeServer::runForever() - { - if (run()) - { - _server.wait(); - } - } - - void SnakeServer::stop() - { - _server.stop(); - } -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeServer.h b/ixsnake/ixsnake/IXSnakeServer.h deleted file mode 100644 index 4be9a06a..00000000 --- a/ixsnake/ixsnake/IXSnakeServer.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * IXSnakeServer.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include "IXAppConfig.h" -#include -#include - -namespace snake -{ - class SnakeServer - { - public: - SnakeServer(const AppConfig& appConfig); - ~SnakeServer() = default; - - bool run(); - void runForever(); - void stop(); - - private: - std::string parseAppKey(const std::string& path); - - AppConfig _appConfig; - ix::WebSocketServer _server; - }; -} // namespace snake diff --git a/ixsnake/ixsnake/IXStreamSql.cpp b/ixsnake/ixsnake/IXStreamSql.cpp deleted file mode 100644 index 6431b34c..00000000 --- a/ixsnake/ixsnake/IXStreamSql.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * IXStreamSql.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - * - * Super simple hacked up version of a stream sql expression, - * that only supports non nested field evaluation - */ - -#include "IXStreamSql.h" -#include -#include - -namespace snake -{ - StreamSql::StreamSql(const std::string& sqlFilter) - : _valid(false) - { - std::string token; - std::stringstream tokenStream(sqlFilter); - std::vector tokens; - - // Split by ' ' - while (std::getline(tokenStream, token, ' ')) - { - tokens.push_back(token); - } - - _valid = tokens.size() == 8; - if (!_valid) return; - - _field = tokens[5]; - _operator = tokens[6]; - _value = tokens[7]; - - // remove single quotes - _value = _value.substr(1, _value.size() - 2); - - if (_operator == "LIKE") - { - _value = _value.substr(1, _value.size() - 2); - } - } - - bool StreamSql::valid() const - { - return _valid; - } - - bool StreamSql::match(const nlohmann::json& msg) - { - if (!_valid) return false; - - if (msg.find(_field) == msg.end()) - { - return false; - } - - std::string value = msg[_field]; - return value == _value; - } - -} // namespace snake diff --git a/ixsnake/ixsnake/IXStreamSql.h b/ixsnake/ixsnake/IXStreamSql.h deleted file mode 100644 index 812a2ad8..00000000 --- a/ixsnake/ixsnake/IXStreamSql.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * IXStreamSql.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include "nlohmann/json.hpp" - -namespace snake -{ - class StreamSql - { - public: - StreamSql(const std::string& sqlFilter = std::string()); - ~StreamSql() = default; - - bool match(const nlohmann::json& msg); - bool valid() const; - - private: - std::string _field; - std::string _operator; - std::string _value; - bool _valid; - }; -} diff --git a/ixsnake/ixsnake/appsConfig.json b/ixsnake/ixsnake/appsConfig.json deleted file mode 100644 index 14f8f48b..00000000 --- a/ixsnake/ixsnake/appsConfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "apps": { - "FC2F10139A2BAc53BB72D9db967b024f": { - "roles": { - "_sub": { - "secret": "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba" - }, - "_pub": { - "secret": "1c04DB8fFe76A4EeFE3E318C72d771db" - } - } - } - } -}