diff --git a/DOCKER_VERSION b/DOCKER_VERSION index e91f1044..18bb4182 100644 --- a/DOCKER_VERSION +++ b/DOCKER_VERSION @@ -1 +1 @@ -7.4.5 +7.5.0 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f9b02754..bfab2812 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,33 @@ # Changelog All notable changes to this project will be documented in this file. +## [7.5.0] - 2019-12-05 + +- (ws) cobra to sentry. Handle Error 429 Too Many Requests and politely wait before sending more data to sentry. + +In the example below sentry we are sending data too fast, sentry asks us to slow down which we do. Notice how the sent count stop increasing, while we are waiting for 41 seconds. + +``` +[2019-12-05 15:50:33.759] [info] messages received 2449 sent 3 +[2019-12-05 15:50:34.759] [info] messages received 5533 sent 7 +[2019-12-05 15:50:35.759] [info] messages received 8612 sent 11 +[2019-12-05 15:50:36.759] [info] messages received 11562 sent 15 +[2019-12-05 15:50:37.759] [info] messages received 14410 sent 19 +[2019-12-05 15:50:38.759] [info] messages received 17236 sent 23 +[2019-12-05 15:50:39.282] [error] Error sending data to sentry: 429 +[2019-12-05 15:50:39.282] [error] Body: {"exception":[{"stacktrace":{"frames":[{"filename":"WorldScene.lua","function":"WorldScene.lua:1935","lineno":1958},{"filename":"WorldScene.lua","function":"onUpdate_WorldCam","lineno":1921},{"filename":"WorldMapTile.lua","function":"__index","lineno":239}]},"value":"noisytypes: Attempt to call nil(nil,2224139838)!"}],"platform":"python","sdk":{"name":"ws","version":"1.0.0"},"tags":[["game","niso"],["userid","107638363"],["environment","live"]],"timestamp":"2019-12-05T23:50:39Z"} + +[2019-12-05 15:50:39.282] [error] Response: {"error_name":"rate_limit","error":"Creation of this event was denied due to rate limiting"} +[2019-12-05 15:50:39.282] [warning] Error 429 - Too Many Requests. ws will sleep and retry after 41 seconds +[2019-12-05 15:50:39.760] [info] messages received 18839 sent 25 +[2019-12-05 15:50:40.760] [info] messages received 18839 sent 25 +[2019-12-05 15:50:41.760] [info] messages received 18839 sent 25 +[2019-12-05 15:50:42.761] [info] messages received 18839 sent 25 +[2019-12-05 15:50:43.762] [info] messages received 18839 sent 25 +[2019-12-05 15:50:44.763] [info] messages received 18839 sent 25 +[2019-12-05 15:50:45.768] [info] messages received 18839 sent 25 +``` + ## [7.4.5] - 2019-12-03 - (ws) #125 / fix build problem when jsoncpp is not installed locally diff --git a/ixsentry/ixsentry/IXSentryClient.cpp b/ixsentry/ixsentry/IXSentryClient.cpp index 197ba8ec..b698abf6 100644 --- a/ixsentry/ixsentry/IXSentryClient.cpp +++ b/ixsentry/ixsentry/IXSentryClient.cpp @@ -229,7 +229,6 @@ namespace ix args->url = computeUrl(project, key); args->body = _httpClient->serializeHttpFormDataParameters(multipartBoundary, httpFormDataParameters, httpParameters); - _httpClient->performRequest(args, onResponseCallback); } } // namespace ix diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index 7c6ea33f..f3e64936 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "7.4.5" +#define IX_WEBSOCKET_VERSION "7.5.0" diff --git a/ws/ws_cobra_to_sentry.cpp b/ws/ws_cobra_to_sentry.cpp index bf54cb18..d2f96916 100644 --- a/ws/ws_cobra_to_sentry.cpp +++ b/ws/ws_cobra_to_sentry.cpp @@ -40,6 +40,7 @@ namespace ix std::atomic receivedCount(0); std::atomic errorSending(false); std::atomic stop(false); + std::atomic throttled(false); std::condition_variable condition; std::mutex conditionVariableMutex; @@ -64,6 +65,7 @@ namespace ix &errorSending, &sentCount, &stop, + &throttled, &dsn] { SentryClient sentryClient(dsn); @@ -110,6 +112,31 @@ namespace ix spdlog::error("Body: {}", ret.second); spdlog::error("Response: {}", response->payload); errorSending = true; + + // Error 429 Too Many Requests + if (response->statusCode == 429) + { + auto retryAfter = response->headers["Retry-After"]; + std::stringstream ss; + ss << retryAfter; + int seconds; + ss >> seconds; + + if (!ss.eof() || ss.fail()) + { + seconds = 30; + spdlog::warn("Error parsing Retry-After header. " + "Using {} for the sleep duration", seconds); + } + + spdlog::warn("Error 429 - Too Many Requests. ws will sleep " + "and retry after {} seconds", retryAfter); + + throttled = true; + auto duration = std::chrono::seconds(seconds); + std::this_thread::sleep_for(duration); + throttled = false; + } } else { @@ -133,8 +160,8 @@ namespace ix &filter, &jsonWriter, verbose, + &throttled, &receivedCount, - &sentCount, &condition, &conditionVariableMutex, &queue](ix::CobraConnectionEventType eventType, @@ -162,7 +189,7 @@ namespace ix filter, [&jsonWriter, verbose, - &sentCount, + &throttled, &receivedCount, &condition, &conditionVariableMutex, @@ -173,13 +200,8 @@ namespace ix } // If we cannot send to sentry fast enough, drop the message - const uint64_t scaleFactor = 2; - - if (sentCount != 0 && receivedCount != 0 && - (sentCount * scaleFactor < receivedCount)) + if (throttled) { - spdlog::warn("message dropped: sending is backlogged !"); - condition.notify_one(); return; }