(ws) cobra to sentry. Handle Error 429 Too Many Requests and politely wait before sending more data to sentry

DOCKER_VERSION

@@ -1 +1 @@
-7.4.4
+7.5.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine as build
+FROM alpine:edge as build
 
 RUN apk add --no-cache gcc g++ musl-dev linux-headers cmake openssl-dev 
 RUN apk add --no-cache make
@@ -16,7 +16,7 @@ WORKDIR /opt
 USER app
 RUN [ "make", "ws_install" ]
 
-FROM alpine as runtime
+FROM alpine:edge as runtime
 
 RUN apk add --no-cache libstdc++
 RUN apk add --no-cache strace

docker-compose.yml

@@ -18,50 +18,50 @@ services:
   #   networks:
   #     - ws-net
 
-  pyproxy:
-    image: bsergean/ws_proxy:build
-    entrypoint: /usr/bin/ws_proxy.py --remote_url 'wss://cobra.addsrv.com' --host 0.0.0.0 --port 8765
-    ports:
-      - "8765:8765"
-    networks:
-      - ws-net
+  #pyproxy:
+  #  image: bsergean/ws_proxy:build
+  #  entrypoint: /usr/bin/ws_proxy.py --remote_url 'wss://cobra.addsrv.com' --host 0.0.0.0 --port 8765
+  #  ports:
+  #    - "8765:8765"
+  #  networks:
+  #    - ws-net
 
-        #  ws:
-        #    security_opt:
-        #    - seccomp:unconfined
-        #    cap_add:
-        #    - SYS_PTRACE
-        #    stdin_open: true
-        #    tty: true
-        #    image: bsergean/ws:build
-        #    entrypoint: sh
-        #    networks:
-        #      - ws-net
-        #    depends_on:
-        #      - redis1
-        #
-        #  redis1:
-        #    image: redis:alpine
-        #    networks:
-        #      - ws-net
-        #
-        #  statsd:
-        #    image: jaconel/statsd
-        #    ports:
-        #      - "8125:8125"
-        #    environment:
-        #      - STATSD_DUMP_MSG=true
-        #      - GRAPHITE_HOST=127.0.0.1
-        #    networks:
-        #      - ws-net
+  #      #  ws:
+  #      #    security_opt:
+  #      #    - seccomp:unconfined
+  #      #    cap_add:
+  #      #    - SYS_PTRACE
+  #      #    stdin_open: true
+  #      #    tty: true
+  #      #    image: bsergean/ws:build
+  #      #    entrypoint: sh
+  #      #    networks:
+  #      #      - ws-net
+  #      #    depends_on:
+  #      #      - redis1
+  #      #
+  #      #  redis1:
+  #      #    image: redis:alpine
+  #      #    networks:
+  #      #      - ws-net
+  #      #
+  #      #  statsd:
+  #      #    image: jaconel/statsd
+  #      #    ports:
+  #      #      - "8125:8125"
+  #      #    environment:
+  #      #      - STATSD_DUMP_MSG=true
+  #      #      - GRAPHITE_HOST=127.0.0.1
+  #      #    networks:
+  #      #      - ws-net
 
-#  compile:
-#    image: alpine
-#    entrypoint: sh
-#    stdin_open: true
-#    tty: true
-#    volumes:
-#      - /Users/bsergeant/src/foss:/home/bsergean/src/foss
+  compile:
+    image: alpine
+    entrypoint: sh
+    stdin_open: true
+    tty: true
+    volumes:
+      - /Users/bsergeant/src/foss:/home/bsergean/src/foss
 
 networks:
   ws-net:

docs/CHANGELOG.md

@@ -1,6 +1,37 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [7.5.0] - 2019-12-05
+
+- (ws) cobra to sentry. Handle Error 429 Too Many Requests and politely wait before sending more data to sentry.
+
+In the example below sentry we are sending data too fast, sentry asks us to slow down which we do. Notice how the sent count stop increasing, while we are waiting for 41 seconds.
+
+```
+[2019-12-05 15:50:33.759] [info] messages received 2449 sent 3
+[2019-12-05 15:50:34.759] [info] messages received 5533 sent 7
+[2019-12-05 15:50:35.759] [info] messages received 8612 sent 11
+[2019-12-05 15:50:36.759] [info] messages received 11562 sent 15
+[2019-12-05 15:50:37.759] [info] messages received 14410 sent 19
+[2019-12-05 15:50:38.759] [info] messages received 17236 sent 23
+[2019-12-05 15:50:39.282] [error] Error sending data to sentry: 429
+[2019-12-05 15:50:39.282] [error] Body: {"exception":[{"stacktrace":{"frames":[{"filename":"WorldScene.lua","function":"WorldScene.lua:1935","lineno":1958},{"filename":"WorldScene.lua","function":"onUpdate_WorldCam","lineno":1921},{"filename":"WorldMapTile.lua","function":"__index","lineno":239}]},"value":"noisytypes: Attempt to call nil(nil,2224139838)!"}],"platform":"python","sdk":{"name":"ws","version":"1.0.0"},"tags":[["game","niso"],["userid","107638363"],["environment","live"]],"timestamp":"2019-12-05T23:50:39Z"}
+
+[2019-12-05 15:50:39.282] [error] Response: {"error_name":"rate_limit","error":"Creation of this event was denied due to rate limiting"}
+[2019-12-05 15:50:39.282] [warning] Error 429 - Too Many Requests. ws will sleep and retry after 41 seconds
+[2019-12-05 15:50:39.760] [info] messages received 18839 sent 25
+[2019-12-05 15:50:40.760] [info] messages received 18839 sent 25
+[2019-12-05 15:50:41.760] [info] messages received 18839 sent 25
+[2019-12-05 15:50:42.761] [info] messages received 18839 sent 25
+[2019-12-05 15:50:43.762] [info] messages received 18839 sent 25
+[2019-12-05 15:50:44.763] [info] messages received 18839 sent 25
+[2019-12-05 15:50:45.768] [info] messages received 18839 sent 25
+```
+
+## [7.4.5] - 2019-12-03
+
+- (ws) #125 / fix build problem when jsoncpp is not installed locally
+
 ## [7.4.4] - 2019-12-03
 
 - (ws) #125 / cmake detects an already installed jsoncpp and will try to use this one if present

ixcobra/CMakeLists.txt

@@ -20,10 +20,16 @@ add_library(ixcobra STATIC
     ${IXCOBRA_HEADERS}
 )
 
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp)
+endif()
+
 set(IXCOBRA_INCLUDE_DIRS
     .
     ..
     ../ixcore
-    ../ixcrypto)
+    ../ixcrypto
+    ${JSONCPP_INCLUDE_DIRS})
 
 target_include_directories( ixcobra PUBLIC ${IXCOBRA_INCLUDE_DIRS} )

ixsentry/CMakeLists.txt

@@ -16,9 +16,15 @@ add_library(ixsentry STATIC
   ${IXSENTRY_HEADERS}
 )
 
+find_package(JsonCpp)
+if (NOT JSONCPP_FOUND)
+  set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp)
+endif()
+
 set(IXSENTRY_INCLUDE_DIRS
     .
     ..
-    ../ixcore)
+    ../ixcore
+    ${JSONCPP_INCLUDE_DIRS})
 
 target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} )

ixsentry/ixsentry/IXSentryClient.cpp

@@ -229,7 +229,6 @@ namespace ix
         args->url = computeUrl(project, key);
         args->body = _httpClient->serializeHttpFormDataParameters(multipartBoundary, httpFormDataParameters, httpParameters);
 
-
         _httpClient->performRequest(args, onResponseCallback);
     }
 } // namespace ix

ixwebsocket/IXWebSocketVersion.h

@@ -6,4 +6,4 @@
 
 #pragma once
 
-#define IX_WEBSOCKET_VERSION "7.4.4"
+#define IX_WEBSOCKET_VERSION "7.5.0"

ws/ws_cobra_to_sentry.cpp

@@ -40,6 +40,7 @@ namespace ix
         std::atomic<uint64_t> receivedCount(0);
         std::atomic<bool> errorSending(false);
         std::atomic<bool> stop(false);
+        std::atomic<bool> throttled(false);
 
         std::condition_variable condition;
         std::mutex conditionVariableMutex;
@@ -64,6 +65,7 @@ namespace ix
                              &errorSending,
                              &sentCount,
                              &stop,
+                             &throttled,
                              &dsn] {
             SentryClient sentryClient(dsn);
 
@@ -110,6 +112,31 @@ namespace ix
                     spdlog::error("Body: {}", ret.second);
                     spdlog::error("Response: {}", response->payload);
                     errorSending = true;
+
+                    // Error 429 Too Many Requests
+                    if (response->statusCode == 429)
+                    {
+                        auto retryAfter = response->headers["Retry-After"];
+                        std::stringstream ss;
+                        ss << retryAfter;
+                        int seconds;
+                        ss >> seconds;
+
+                        if (!ss.eof() || ss.fail())
+                        {
+                            seconds = 30;
+                            spdlog::warn("Error parsing Retry-After header. "
+                                         "Using {} for the sleep duration", seconds);
+                        }
+
+                        spdlog::warn("Error 429 - Too Many Requests. ws will sleep "
+                                     "and retry after {} seconds", retryAfter);
+
+                        throttled = true;
+                        auto duration = std::chrono::seconds(seconds);
+                        std::this_thread::sleep_for(duration);
+                        throttled = false;
+                    }
                 }
                 else
                 {
@@ -133,8 +160,8 @@ namespace ix
                                &filter,
                                &jsonWriter,
                                verbose,
+                               &throttled,
                                &receivedCount,
-                               &sentCount,
                                &condition,
                                &conditionVariableMutex,
                                &queue](ix::CobraConnectionEventType eventType,
@@ -162,7 +189,7 @@ namespace ix
                                filter,
                                [&jsonWriter,
                                 verbose,
-                                &sentCount,
+                                &throttled,
                                 &receivedCount,
                                 &condition,
                                 &conditionVariableMutex,
@@ -173,13 +200,8 @@ namespace ix
                                    }
 
                                    // If we cannot send to sentry fast enough, drop the message
-                                   const uint64_t scaleFactor = 2;
-
-                                   if (sentCount != 0 && receivedCount != 0 &&
-                                       (sentCount * scaleFactor < receivedCount))
+                                   if (throttled)
                                    {
-                                       spdlog::warn("message dropped: sending is backlogged !");
-
                                        condition.notify_one();
                                        return;
                                    }

ws/ws_sentry_minidump_upload.cpp

@@ -94,10 +94,19 @@ namespace ix
                 done = true;
             });
 
+        int i = 0;
+
         while (!done)
         {
             std::chrono::duration<double, std::milli> duration(10);
             std::this_thread::sleep_for(duration);
+
+            if (i++ > 5000) break; // wait 5 seconds max
+        }
+
+        if (!done)
+        {
+            spdlog::error("Error: timing out trying to sent a crash to sentry");
         }
 
         return 0;