kapitan/IXWebSocket
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(redis cobra bots) update the cobra to redis bot to use the bot framework, and change it to report fps metrics into redis streams.

Browse Source
This commit is contained in:
Benjamin Sergeant
2020-06-10 22:30:55 -07:00
parent ecfca1f905
commit c5aadffa08
27 changed files with 225 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ws/CMakeLists.txt
View File

@ -51,7 +51,6 @@ add_executable(ws
ws_snake.cpp
ws_cobra_metrics_publish.cpp
ws_cobra_publish.cpp
ws_cobra_metrics_to_redis.cpp
ws_httpd.cpp
ws_autobahn.cpp
ws_proxy_server.cpp
@ -64,6 +63,7 @@ target_link_libraries(ws ixbots)
target_link_libraries(ws ixsnake)
target_link_libraries(ws ixcobra)
target_link_libraries(ws ixsentry)
target_link_libraries(ws ixredis)
target_link_libraries(ws ixwebsocket)
target_link_libraries(ws ixcrypto)
target_link_libraries(ws ixcore)

21
ws/ws.cpp
View File

@ -15,6 +15,8 @@
#include <ixbots/IXCobraToStatsdBot.h>
#include <ixbots/IXCobraToStdoutBot.h>
#include <ixbots/IXCobraMetricsToStatsdBot.h>
#include <ixbots/IXCobraMetricsToRedisBot.h>
#include <ixredis/IXRedisClient.h>
#include <ixcore/utils/IXCoreLogger.h>
#include <ixsentry/IXSentryClient.h>
#include <ixwebsocket/IXNetSystem.h>
@ -363,13 +365,10 @@ int main(int argc, char** argv)
CLI::App* cobra2redisApp =
app.add_subcommand("cobra_metrics_to_redis", "Cobra metrics to redis");
cobra2redisApp->fallthrough();
cobra2redisApp->add_option("channel", channel, "Channel")->required();
cobra2redisApp->add_option("--pidfile", pidfile, "Pid file");
cobra2redisApp->add_option("--filter", filter, "Stream SQL Filter");
cobra2redisApp->add_option("--position", position, "Stream position");
cobra2redisApp->add_option("--hostname", hostname, "Redis hostname");
cobra2redisApp->add_option("--port", redisPort, "Redis port");
cobra2redisApp->add_flag("-q", quiet, "Quiet / only display stats");
cobra2redisApp->add_flag("-v", verbose, "Verbose");
addTLSOptions(cobra2redisApp);
addCobraConfig(cobra2redisApp);
@ -605,8 +604,18 @@ int main(int argc, char** argv)
}
else if (app.got_subcommand("cobra_metrics_to_redis"))
{
ret = ix::ws_cobra_metrics_to_redis(
cobraConfig, channel, filter, position, hostname, redisPort);
ix::RedisClient redisClient;
if (!redisClient.connect(redisHosts, redisPort))
{
spdlog::error("Cannot connect to redis host {}:{}",
redisHosts, redisPort);
return 1;
}
else
{
ret = (int) ix::cobra_metrics_to_redis_bot(
cobraBotConfig, redisClient, verbose);
}
}
else if (app.got_subcommand("snake"))
{

166
ws/ws_cobra_metrics_to_redis.cpp
View File

@ -1,166 +0,0 @@
/*
* ws_cobra_metrics_to_redis.cpp
* Author: Benjamin Sergeant
* Copyright (c) 2019 Machine Zone, Inc. All rights reserved.
*/
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <ixcobra/IXCobraConnection.h>
#include <ixsnake/IXRedisClient.h>
#include <mutex>
#include <queue>
#include <spdlog/spdlog.h>
#include <sstream>
#include <thread>
namespace ix
{
int ws_cobra_metrics_to_redis(const ix::CobraConfig& config,
const std::string& channel,
const std::string& filter,
const std::string& position,
const std::string& host,
int port)
{
ix::CobraConnection conn;
conn.configure(config);
conn.connect();
// Display incoming messages
std::atomic<int> msgPerSeconds(0);
std::atomic<int> msgCount(0);
auto timer = [&msgPerSeconds, &msgCount] {
while (true)
{
spdlog::info("#messages {} msg/s {}", msgCount, msgPerSeconds);
msgPerSeconds = 0;
auto duration = std::chrono::seconds(1);
std::this_thread::sleep_for(duration);
}
};
ix::RedisClient redisClient;
if (!redisClient.connect(host, port))
{
spdlog::error("Cannot connect to redis host {}:{}", host, port);
return 0;
}
std::mutex conditionVariableMutex;
std::condition_variable condition;
std::queue<Json::Value> queue;
auto redisSender = [&condition, &queue, &conditionVariableMutex, &redisClient] {
Json::FastWriter jsonWriter;
int batchSize = 1000;
int i = 0;
std::stringstream pipe;
while (true)
{
Json::Value msg;
{
std::unique_lock<std::mutex> lock(conditionVariableMutex);
condition.wait(lock, [&queue] { return !queue.empty(); });
msg = queue.front();
queue.pop();
}
// compute channel
std::stringstream ss;
ss << "session=" << msg["session"].asString() << ";msgid=" << msg["id"].asString();
std::string channel = ss.str();
std::string errMsg;
pipe << redisClient.prepareXaddCommand(channel, jsonWriter.write(msg));
if (i++ == batchSize)
{
if (!redisClient.sendCommand(pipe.str(), batchSize, errMsg))
{
spdlog::error("error sending command: {}", errMsg);
}
i = 0;
}
}
};
std::thread t1(timer);
std::thread t2(redisSender);
conn.setEventCallback([&conn,
&channel,
&filter,
&position,
&msgCount,
&msgPerSeconds,
&conditionVariableMutex,
&condition,
&queue](const CobraEventPtr& event) {
if (event->type == ix::CobraEventType::Open)
{
spdlog::info("Subscriber connected");
for (auto&& it : event->headers)
{
spdlog::info("{}: {}", it.first, it.second);
}
}
else if (event->type == ix::CobraEventType::Authenticated)
{
spdlog::info("Subscriber authenticated");
conn.subscribe(
channel,
filter,
position,
[&msgPerSeconds, &msgCount, &conditionVariableMutex, &condition, &queue](
const Json::Value& msg, const std::string& /*position*/) {
{
std::unique_lock<std::mutex> lock(conditionVariableMutex);
queue.push(msg);
}
condition.notify_one();
msgPerSeconds++;
msgCount++;
});
}
else if (event->type == ix::CobraEventType::Subscribed)
{
spdlog::info("Subscriber: subscribed to channel {}", event->subscriptionId);
}
else if (event->type == ix::CobraEventType::UnSubscribed)
{
spdlog::info("Subscriber: unsubscribed from channel {}", event->subscriptionId);
}
else if (event->type == ix::CobraEventType::Error)
{
spdlog::error("Subscriber: error {}", event->errMsg);
}
else if (event->type == ix::CobraEventType::Published)
{
spdlog::error("Published message hacked: {}", event->msgId);
}
});
while (true)
{
std::chrono::duration<double, std::milli> duration(10);
std::this_thread::sleep_for(duration);
}
return 0;
}
} // namespace ix

2
ws/ws_redis_publish.cpp
View File

@ -4,7 +4,7 @@
* Copyright (c) 2019 Machine Zone, Inc. All rights reserved.
*/
#include <ixsnake/IXRedisClient.h>
#include <ixredis/IXRedisClient.h>
#include <spdlog/spdlog.h>
#include <sstream>

2
ws/ws_redis_server.cpp
View File

@ -4,7 +4,7 @@
* Copyright (c) 2019 Machine Zone, Inc. All rights reserved.
*/
#include <ixsnake/IXRedisServer.h>
#include <ixredis/IXRedisServer.h>
#include <spdlog/spdlog.h>
#include <sstream>

2
ws/ws_redis_subscribe.cpp
View File

@ -6,7 +6,7 @@
#include <atomic>
#include <chrono>
#include <ixsnake/IXRedisClient.h>
#include <ixredis/IXRedisClient.h>
#include <spdlog/spdlog.h>
#include <sstream>
#include <thread>