diff --git a/.dockerignore b/.dockerignore index 78ae78ea..538648ad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ build CMakeCache.txt ws/CMakeCache.txt test/build +makefile diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml deleted file mode 100644 index 2e82d4a4..00000000 --- a/.github/workflows/ccpp.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: unittest -on: - push: - paths-ignore: - - 'docs/**' - -jobs: - linux: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: make test_make - run: make test_make - - mac_tsan_sectransport: - runs-on: macOS-latest - steps: - - uses: actions/checkout@v1 - - name: make test_tsan - run: make test_tsan - - mac_tsan_openssl: - runs-on: macOS-latest - steps: - - uses: actions/checkout@v1 - - name: install openssl - run: brew install openssl@1.1 - - name: make test - run: make test_tsan_openssl - - mac_tsan_mbedtls: - runs-on: macOS-latest - steps: - - uses: actions/checkout@v1 - - name: install mbedtls - run: brew install mbedtls - - name: make test - run: make test_tsan_mbedtls - - windows: - runs-on: windows-latest - steps: - - uses: actions/checkout@v1 - - uses: seanmiddleditch/gha-setup-vsdevenv@master - - run: | - mkdir build - cd build - cmake -DCMAKE_CXX_COMPILER=cl.exe -DUSE_WS=1 -DUSE_TEST=1 .. - - run: cmake --build build - - # Running the unittest does not work, the binary cannot be found - #- run: ../build/test/ixwebsocket_unittest.exe - # working-directory: test - - uwp: - runs-on: windows-latest - steps: - - uses: actions/checkout@v1 - - uses: seanmiddleditch/gha-setup-vsdevenv@master - - run: | - mkdir build - cd build - cmake -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_CXX_COMPILER=cl.exe -DUSE_TEST=1 .. - - run: cmake --build build - -# -# Windows with OpenSSL is working but disabled as it takes 13 minutes (10 for openssl) to build with vcpkg -# -# windows_openssl: -# runs-on: windows-latest -# steps: -# - uses: actions/checkout@v1 -# - uses: seanmiddleditch/gha-setup-vsdevenv@master -# - run: | -# vcpkg install zlib:x64-windows -# vcpkg install openssl:x64-windows -# - run: | -# mkdir build -# cd build -# cmake -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_CXX_COMPILER=cl.exe -DUSE_OPEN_SSL=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. -# - run: cmake --build build -# -# # Running the unittest does not work, the binary cannot be found -# #- run: ../build/test/ixwebsocket_unittest.exe -# # working-directory: test - diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..bedb92b8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,66 @@ +name: docker + +# When its time to do a release do a build for amd64 +# and push all of them to Docker Hub. +# Only trigger on semver shaped tags. +on: + push: + tags: + - "v*.*.*" + +jobs: + login: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=machinezone/ws + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS,${DOCKER_IMAGE}:latest" + fi + echo ::set-output name=tags::${TAGS} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@master + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to GitHub Package Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2-build-push + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./Dockerfile + target: prod + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index fd3445aa..7000643c 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -21,5 +21,7 @@ jobs: pip install pygments - name: Build doc run: | + git clean -dfx . + git fetch git pull mkdocs gh-deploy diff --git a/.github/workflows/unittest_linux.yml b/.github/workflows/unittest_linux.yml new file mode 100644 index 00000000..9c6272a5 --- /dev/null +++ b/.github/workflows/unittest_linux.yml @@ -0,0 +1,15 @@ +name: linux +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - name: make test + run: make -f makefile.dev test diff --git a/.github/workflows/unittest_linux_asan.yml b/.github/workflows/unittest_linux_asan.yml new file mode 100644 index 00000000..613c618e --- /dev/null +++ b/.github/workflows/unittest_linux_asan.yml @@ -0,0 +1,15 @@ +name: linux_asan +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - name: make test_asan + run: make -f makefile.dev test_asan diff --git a/.github/workflows/unittest_mac_tsan_mbedtls.yml b/.github/workflows/unittest_mac_tsan_mbedtls.yml new file mode 100644 index 00000000..ab4d226b --- /dev/null +++ b/.github/workflows/unittest_mac_tsan_mbedtls.yml @@ -0,0 +1,17 @@ +name: mac_tsan_mbedtls +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + mac_tsan_mbedtls: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - name: install mbedtls + run: brew install mbedtls + - name: make test + run: make -f makefile.dev test_tsan_mbedtls diff --git a/.github/workflows/unittest_mac_tsan_openssl.yml b/.github/workflows/unittest_mac_tsan_openssl.yml new file mode 100644 index 00000000..ddc9c7df --- /dev/null +++ b/.github/workflows/unittest_mac_tsan_openssl.yml @@ -0,0 +1,17 @@ +name: mac_tsan_openssl +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + mac_tsan_openssl: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - name: install openssl + run: brew install openssl@1.1 + - name: make test + run: make -f makefile.dev test_tsan_openssl diff --git a/.github/workflows/unittest_mac_tsan_sectransport.yml b/.github/workflows/unittest_mac_tsan_sectransport.yml new file mode 100644 index 00000000..739c0c42 --- /dev/null +++ b/.github/workflows/unittest_mac_tsan_sectransport.yml @@ -0,0 +1,15 @@ +name: mac_tsan_sectransport +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + mac_tsan_sectransport: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - name: make test_tsan_sectransport + run: make -f makefile.dev test_tsan_sectransport diff --git a/.github/workflows/unittest_uwp.yml b/.github/workflows/unittest_uwp.yml new file mode 100644 index 00000000..a721e30a --- /dev/null +++ b/.github/workflows/unittest_uwp.yml @@ -0,0 +1,45 @@ +name: uwp +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + uwp: + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-vsdevenv@master + - uses: seanmiddleditch/gha-setup-ninja@master + - run: | + mkdir build + cd build + cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_CXX_COMPILER=cl.exe -DCMAKE_C_COMPILER=cl.exe -DUSE_TEST=1 -DUSE_ZLIB=0 .. + - run: | + cd build + ninja + - run: | + cd build + ninja test + +# +# Windows with OpenSSL is working but disabled as it takes 13 minutes (10 for openssl) to build with vcpkg +# +# windows_openssl: +# runs-on: windows-latest +# steps: +# - uses: actions/checkout@v1 +# - uses: seanmiddleditch/gha-setup-vsdevenv@master +# - run: | +# vcpkg install zlib:x64-windows +# vcpkg install openssl:x64-windows +# - run: | +# mkdir build +# cd build +# cmake -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_CXX_COMPILER=cl.exe -DUSE_OPEN_SSL=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. +# - run: cmake --build build +# +# # Running the unittest does not work, the binary cannot be found +# #- run: ../build/test/ixwebsocket_unittest.exe +# # working-directory: test diff --git a/.github/workflows/unittest_windows.yml b/.github/workflows/unittest_windows.yml new file mode 100644 index 00000000..cca5cc2a --- /dev/null +++ b/.github/workflows/unittest_windows.yml @@ -0,0 +1,27 @@ +name: windows +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-vsdevenv@master + - uses: seanmiddleditch/gha-setup-ninja@master + - run: | + mkdir build + cd build + cmake -GNinja -DCMAKE_CXX_COMPILER=cl.exe -DCMAKE_C_COMPILER=cl.exe -DUSE_WS=1 -DUSE_TEST=1 -DUSE_ZLIB=OFF -DBUILD_SHARED_LIBS=OFF .. + - run: | + cd build + ninja + - run: | + cd build + ninja test + +#- run: ../build/test/ixwebsocket_unittest.exe +# working-directory: test diff --git a/.github/workflows/unittest_windows_gcc.yml b/.github/workflows/unittest_windows_gcc.yml new file mode 100644 index 00000000..328cd6ec --- /dev/null +++ b/.github/workflows/unittest_windows_gcc.yml @@ -0,0 +1,28 @@ +name: windows_gcc +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - uses: egor-tensin/setup-mingw@v2 + - run: | + mkdir build + cd build + cmake -GNinja -DCMAKE_CXX_COMPILER=c++ -DCMAKE_C_COMPILER=cc -DUSE_WS=1 -DUSE_TEST=1 -DUSE_ZLIB=0 -DCMAKE_UNITY_BUILD=ON .. + - run: | + cd build + ninja + - run: | + cd build + ctest -V + # ninja test + +#- run: ../build/test/ixwebsocket_unittest.exe +# working-directory: test diff --git a/.gitignore b/.gitignore index 3f003614..892b7bf6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ ixsnake/ixsnake/.certs/ site/ ws/.certs/ ws/.srl +ixhttpd +makefile +a.out diff --git a/CMake/FindDeflate.cmake b/CMake/FindDeflate.cmake new file mode 100644 index 00000000..ebbd9fdf --- /dev/null +++ b/CMake/FindDeflate.cmake @@ -0,0 +1,19 @@ +# Find package structure taken from libcurl + +include(FindPackageHandleStandardArgs) + +find_path(DEFLATE_INCLUDE_DIRS libdeflate.h) +find_library(DEFLATE_LIBRARY deflate) + +find_package_handle_standard_args(Deflate + FOUND_VAR + DEFLATE_FOUND + REQUIRED_VARS + DEFLATE_LIBRARY + DEFLATE_INCLUDE_DIRS + FAIL_MESSAGE + "Could NOT find deflate" +) + +set(DEFLATE_INCLUDE_DIRS ${DEFLATE_INCLUDE_DIRS}) +set(DEFLATE_LIBRARIES ${DEFLATE_LIBRARY}) diff --git a/CMake/FindJsonCpp.cmake b/CMake/FindJsonCpp.cmake deleted file mode 100644 index 40032a19..00000000 --- a/CMake/FindJsonCpp.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# Find package structure taken from libcurl - -include(FindPackageHandleStandardArgs) - -find_path(JSONCPP_INCLUDE_DIRS json/json.h) -find_library(JSONCPP_LIBRARY jsoncpp) - -find_package_handle_standard_args(JsonCpp - FOUND_VAR - JSONCPP_FOUND - REQUIRED_VARS - JSONCPP_LIBRARY - JSONCPP_INCLUDE_DIRS - FAIL_MESSAGE - "Could NOT find jsoncpp" -) - -set(JSONCPP_INCLUDE_DIRS ${JSONCPP_INCLUDE_DIRS}) -set(JSONCPP_LIBRARIES ${JSONCPP_LIBRARY}) diff --git a/CMake/FindMbedTLS.cmake b/CMake/FindMbedTLS.cmake index a9163958..b6ef63ab 100644 --- a/CMake/FindMbedTLS.cmake +++ b/CMake/FindMbedTLS.cmake @@ -1,5 +1,8 @@ find_path(MBEDTLS_INCLUDE_DIRS mbedtls/ssl.h) +# mbedtls-3.0 changed headers files, and we need to ifdef'out a few things +find_path(MBEDTLS_VERSION_GREATER_THAN_3 mbedtls/build_info.h) + find_library(MBEDTLS_LIBRARY mbedtls) find_library(MBEDX509_LIBRARY mbedx509) find_library(MBEDCRYPTO_LIBRARY mbedcrypto) @@ -7,7 +10,7 @@ find_library(MBEDCRYPTO_LIBRARY mbedcrypto) set(MBEDTLS_LIBRARIES "${MBEDTLS_LIBRARY}" "${MBEDX509_LIBRARY}" "${MBEDCRYPTO_LIBRARY}") include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(MBEDTLS DEFAULT_MSG +find_package_handle_standard_args(MbedTLS DEFAULT_MSG MBEDTLS_INCLUDE_DIRS MBEDTLS_LIBRARY MBEDX509_LIBRARY MBEDCRYPTO_LIBRARY) mark_as_advanced(MBEDTLS_INCLUDE_DIRS MBEDTLS_LIBRARY MBEDX509_LIBRARY MBEDCRYPTO_LIBRARY) diff --git a/CMakeLists.txt b/CMakeLists.txt index 772b5751..72e5ac45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,15 +3,17 @@ # Copyright (c) 2018 Machine Zone, Inc. All rights reserved. # -cmake_minimum_required(VERSION 3.4.1) +cmake_minimum_required(VERSION 3.4.1...3.17.2) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}") project(ixwebsocket C CXX) -set (CMAKE_CXX_STANDARD 14) +set (CMAKE_CXX_STANDARD 11) set (CXX_STANDARD_REQUIRED ON) set (CMAKE_CXX_EXTENSIONS OFF) +option (BUILD_DEMO OFF) + if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() @@ -30,19 +32,25 @@ set( IXWEBSOCKET_SOURCES ixwebsocket/IXConnectionState.cpp ixwebsocket/IXDNSLookup.cpp ixwebsocket/IXExponentialBackoff.cpp + ixwebsocket/IXGetFreePort.cpp + ixwebsocket/IXGzipCodec.cpp ixwebsocket/IXHttp.cpp ixwebsocket/IXHttpClient.cpp ixwebsocket/IXHttpServer.cpp ixwebsocket/IXNetSystem.cpp ixwebsocket/IXSelectInterrupt.cpp ixwebsocket/IXSelectInterruptFactory.cpp + ixwebsocket/IXSelectInterruptPipe.cpp + ixwebsocket/IXSetThreadName.cpp ixwebsocket/IXSocket.cpp ixwebsocket/IXSocketConnect.cpp ixwebsocket/IXSocketFactory.cpp ixwebsocket/IXSocketServer.cpp ixwebsocket/IXSocketTLSOptions.cpp + ixwebsocket/IXStrCaseCompare.cpp ixwebsocket/IXUdpSocket.cpp ixwebsocket/IXUrlParser.cpp + ixwebsocket/IXUuid.cpp ixwebsocket/IXUserAgent.cpp ixwebsocket/IXWebSocket.cpp ixwebsocket/IXWebSocketCloseConstants.cpp @@ -51,6 +59,7 @@ set( IXWEBSOCKET_SOURCES ixwebsocket/IXWebSocketPerMessageDeflate.cpp ixwebsocket/IXWebSocketPerMessageDeflateCodec.cpp ixwebsocket/IXWebSocketPerMessageDeflateOptions.cpp + ixwebsocket/IXWebSocketProxyServer.cpp ixwebsocket/IXWebSocketServer.cpp ixwebsocket/IXWebSocketTransport.cpp ) @@ -61,6 +70,8 @@ set( IXWEBSOCKET_HEADERS ixwebsocket/IXConnectionState.h ixwebsocket/IXDNSLookup.h ixwebsocket/IXExponentialBackoff.h + ixwebsocket/IXGetFreePort.h + ixwebsocket/IXGzipCodec.h ixwebsocket/IXHttp.h ixwebsocket/IXHttpClient.h ixwebsocket/IXHttpServer.h @@ -68,14 +79,18 @@ set( IXWEBSOCKET_HEADERS ixwebsocket/IXProgressCallback.h ixwebsocket/IXSelectInterrupt.h ixwebsocket/IXSelectInterruptFactory.h + ixwebsocket/IXSelectInterruptPipe.h ixwebsocket/IXSetThreadName.h ixwebsocket/IXSocket.h ixwebsocket/IXSocketConnect.h ixwebsocket/IXSocketFactory.h ixwebsocket/IXSocketServer.h ixwebsocket/IXSocketTLSOptions.h + ixwebsocket/IXStrCaseCompare.h ixwebsocket/IXUdpSocket.h + ixwebsocket/IXUniquePtr.h ixwebsocket/IXUrlParser.h + ixwebsocket/IXUuid.h ixwebsocket/IXUtf8Validator.h ixwebsocket/IXUserAgent.h ixwebsocket/IXWebSocket.h @@ -92,29 +107,14 @@ set( IXWEBSOCKET_HEADERS ixwebsocket/IXWebSocketPerMessageDeflate.h ixwebsocket/IXWebSocketPerMessageDeflateCodec.h ixwebsocket/IXWebSocketPerMessageDeflateOptions.h + ixwebsocket/IXWebSocketProxyServer.h ixwebsocket/IXWebSocketSendInfo.h ixwebsocket/IXWebSocketServer.h ixwebsocket/IXWebSocketTransport.h ixwebsocket/IXWebSocketVersion.h ) -if (UNIX) - # Linux, Mac, iOS, Android - list( APPEND IXWEBSOCKET_SOURCES ixwebsocket/IXSelectInterruptPipe.cpp ) - list( APPEND IXWEBSOCKET_SOURCES ixwebsocket/IXSelectInterruptPipe.h ) -endif() - -# Platform specific code -if (APPLE) - list( APPEND IXWEBSOCKET_SOURCES ixwebsocket/apple/IXSetThreadName_apple.cpp) -elseif (WIN32) - list( APPEND IXWEBSOCKET_SOURCES ixwebsocket/windows/IXSetThreadName_windows.cpp) -elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") - list( APPEND IXWEBSOCKET_SOURCES ixwebsocket/freebsd/IXSetThreadName_freebsd.cpp) -else() - list( APPEND IXWEBSOCKET_SOURCES ixwebsocket/linux/IXSetThreadName_linux.cpp) -endif() - +option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" OFF) option(USE_TLS "Enable TLS support" FALSE) if (USE_TLS) @@ -148,13 +148,11 @@ if (USE_TLS) endif() endif() -add_library( ixwebsocket STATIC +add_library( ixwebsocket ${IXWEBSOCKET_SOURCES} ${IXWEBSOCKET_HEADERS} ) -add_library ( ixwebsocket::ixwebsocket ALIAS ixwebsocket ) - if (USE_TLS) target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_TLS) if (USE_MBED_TLS) @@ -184,48 +182,66 @@ if (USE_TLS) # This OPENSSL_FOUND check is to help find a cmake manually configured OpenSSL if (NOT OPENSSL_FOUND) - include(FindOpenSSL) + find_package(OpenSSL REQUIRED) endif() message(STATUS "OpenSSL: " ${OPENSSL_VERSION}) - target_link_libraries(ixwebsocket PUBLIC OpenSSL::SSL OpenSSL::Crypto) + add_definitions(${OPENSSL_DEFINITIONS}) + target_include_directories(ixwebsocket PUBLIC $) + target_link_libraries(ixwebsocket ${OPENSSL_LIBRARIES}) elseif (USE_MBED_TLS) message(STATUS "TLS configured to use mbedtls") - find_package(MbedTLS REQUIRED) - target_include_directories(ixwebsocket PUBLIC ${MBEDTLS_INCLUDE_DIRS}) - target_link_libraries(ixwebsocket PUBLIC ${MBEDTLS_LIBRARIES}) + # This MBEDTLS_FOUND check is to help find a cmake manually configured MbedTLS + if (NOT MBEDTLS_FOUND) + find_package(MbedTLS REQUIRED) + + if (MBEDTLS_VERSION_GREATER_THAN_3) + target_compile_definitions(ixwebsocket PRIVATE IXWEBSOCKET_USE_MBED_TLS_MIN_VERSION_3) + endif() + + endif() + target_include_directories(ixwebsocket PUBLIC $) + target_link_libraries(ixwebsocket ${MBEDTLS_LIBRARIES}) elseif (USE_SECURE_TRANSPORT) message(STATUS "TLS configured to use secure transport") - target_link_libraries(ixwebsocket PUBLIC "-framework foundation" "-framework security") + target_link_libraries(ixwebsocket "-framework Foundation" "-framework Security") endif() endif() -# This ZLIB_FOUND check is to help find a cmake manually configured zlib -if (NOT ZLIB_FOUND) - find_package(ZLIB) +option(USE_ZLIB "Enable zlib support" TRUE) + +if (USE_ZLIB) + # This ZLIB_FOUND check is to help find a cmake manually configured zlib + if (NOT ZLIB_FOUND) + find_package(ZLIB REQUIRED) + endif() + target_include_directories(ixwebsocket PUBLIC $) + target_link_libraries(ixwebsocket ${ZLIB_LIBRARIES}) + + target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_ZLIB) endif() -if (ZLIB_FOUND) - include_directories(${ZLIB_INCLUDE_DIRS}) - target_link_libraries(ixwebsocket PUBLIC ${ZLIB_LIBRARIES}) -else() - include_directories(third_party/zlib ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib) - add_subdirectory(third_party/zlib EXCLUDE_FROM_ALL) - target_link_libraries(ixwebsocket PRIVATE $) + +# brew install libdeflate +find_package(Deflate) +if (DEFLATE_FOUND) + include_directories(${DEFLATE_INCLUDE_DIRS}) + target_link_libraries(ixwebsocket ${DEFLATE_LIBRARIES}) + target_compile_definitions(ixwebsocket PUBLIC IXWEBSOCKET_USE_DEFLATE) endif() if (WIN32) - target_link_libraries(ixwebsocket PUBLIC wsock32 ws2_32 shlwapi) + target_link_libraries(ixwebsocket wsock32 ws2_32 shlwapi) add_definitions(-D_CRT_SECURE_NO_WARNINGS) if (USE_TLS) - target_link_libraries(ixwebsocket PUBLIC Crypt32) + target_link_libraries(ixwebsocket Crypt32) endif() endif() if (UNIX) find_package(Threads) - target_link_libraries(ixwebsocket PUBLIC ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(ixwebsocket ${CMAKE_THREAD_LIBS_INIT}) endif() @@ -238,32 +254,49 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") target_compile_options(ixwebsocket PRIVATE /MP) endif() -target_include_directories(ixwebsocket PUBLIC $ $) +include(GNUInstallDirs) + +target_include_directories(ixwebsocket PUBLIC + $ + $ +) set_target_properties(ixwebsocket PROPERTIES PUBLIC_HEADER "${IXWEBSOCKET_HEADERS}") -install(TARGETS ixwebsocket EXPORT ixwebsocket - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_PREFIX}/include/ixwebsocket/ -) +option(IXWEBSOCKET_INSTALL "Install IXWebSocket" TRUE) -install(EXPORT ixwebsocket NAMESPACE ixwebsocket:: DESTINATION lib/cmake/ixwebsocket) -export(EXPORT ixwebsocket NAMESPACE ixwebsocket:: FILE ixwebsocketConfig.cmake) +if (IXWEBSOCKET_INSTALL) + install(TARGETS ixwebsocket + EXPORT ixwebsocket + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/ixwebsocket/ + ) + + install(EXPORT ixwebsocket + FILE ixwebsocket-config.cmake + NAMESPACE ixwebsocket:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ixwebsocket) +endif() if (USE_WS OR USE_TEST) - add_subdirectory(ixcore) - add_subdirectory(ixcrypto) - add_subdirectory(ixcobra) - add_subdirectory(ixsnake) - add_subdirectory(ixsentry) - add_subdirectory(ixbots) + include(FetchContent) + FetchContent_Declare(spdlog + GIT_REPOSITORY "https://github.com/gabime/spdlog" + GIT_TAG "v1.8.0" + GIT_SHALLOW 1) - add_subdirectory(third_party/spdlog spdlog) + FetchContent_MakeAvailable(spdlog) if (USE_WS) - add_subdirectory(ws) + add_subdirectory(ws) endif() if (USE_TEST) - add_subdirectory(test) + enable_testing() + add_subdirectory(test) endif() endif() + +if (BUILD_DEMO) + add_executable(demo main.cpp) + target_link_libraries(demo ixwebsocket) +endif() diff --git a/README.md b/README.md index cf935c52..8f48931c 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,90 @@ ## Hello world -![Build status](https://github.com/machinezone/IXWebSocket/workflows/unittest/badge.svg) - IXWebSocket is a C++ library for WebSocket client and server development. It has minimal dependencies (no boost), is very simple to use and support everything you'll likely need for websocket dev (SSL, deflate compression, compiles on most platforms, etc...). HTTP client and server code is also available, but it hasn't received as much testing. -It is been used on big mobile video game titles sending and receiving tons of messages since 2017 (iOS and Android). It was tested on macOS, iOS, Linux, Android, Windows and FreeBSD. Two important design goals are simplicity and correctness. +It is been used on big mobile video game titles sending and receiving tons of messages since 2017 (iOS and Android). It was tested on macOS, iOS, Linux, Android, Windows and FreeBSD. Note that the MinGW compiler is not supported at this point. Two important design goals are simplicity and correctness. + +A bad security bug affecting users compiling with SSL enabled and OpenSSL as the backend was just fixed in newly released version 11.0.0. Please upgrade ! (more details in the [https://github.com/machinezone/IXWebSocket/pull/250](PR). ```cpp -// Required on Windows -ix::initNetSystem(); +/* + * main.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + * + * Super simple standalone example. See ws folder, unittest and doc/usage.md for more. + * + * On macOS + * $ mkdir -p build ; (cd build ; cmake -DUSE_TLS=1 .. ; make -j ; make install) + * $ clang++ --std=c++11 --stdlib=libc++ main.cpp -lixwebsocket -lz -framework Security -framework Foundation + * $ ./a.out + * + * Or use cmake -DBUILD_DEMO=ON option for other platforms + */ -// Our websocket object -ix::WebSocket webSocket; +#include +#include +#include +#include -std::string url("ws://localhost:8080/"); -webSocket.setUrl(url); +int main() +{ + // Required on Windows + ix::initNetSystem(); -// Setup a callback to be fired (in a background thread, watch out for race conditions !) -// when a message or an event (open, close, error) is received -webSocket.setOnMessageCallback([](const ix::WebSocketMessagePtr& msg) - { - if (msg->type == ix::WebSocketMessageType::Message) + // Our websocket object + ix::WebSocket webSocket; + + // Connect to a server with encryption + // See https://machinezone.github.io/IXWebSocket/usage/#tls-support-and-configuration + std::string url("wss://echo.websocket.org"); + webSocket.setUrl(url); + + std::cout << "Connecting to " << url << "..." << std::endl; + + // Setup a callback to be fired (in a background thread, watch out for race conditions !) + // when a message or an event (open, close, error) is received + webSocket.setOnMessageCallback([](const ix::WebSocketMessagePtr& msg) { - std::cout << msg->str << std::endl; + if (msg->type == ix::WebSocketMessageType::Message) + { + std::cout << "received message: " << msg->str << std::endl; + std::cout << "> " << std::flush; + } + else if (msg->type == ix::WebSocketMessageType::Open) + { + std::cout << "Connection established" << std::endl; + std::cout << "> " << std::flush; + } + else if (msg->type == ix::WebSocketMessageType::Error) + { + // Maybe SSL is not configured properly + std::cout << "Connection error: " << msg->errorInfo.reason << std::endl; + std::cout << "> " << std::flush; + } } + ); + + // Now that our callback is setup, we can start our background thread and receive messages + webSocket.start(); + + // Send a message to the server (default to TEXT mode) + webSocket.send("hello world"); + + // Display a prompt + std::cout << "> " << std::flush; + + std::string text; + // Read text from the console and send messages in text mode. + // Exit with Ctrl-D on Unix or Ctrl-Z on Windows. + while (std::getline(std::cin, text)) + { + webSocket.send(text); + std::cout << "> " << std::flush; } -); -// Now that our callback is setup, we can start our background thread and receive messages -webSocket.start(); - -// Send a message to the server (default to TEXT mode) -webSocket.send("hello world"); + return 0; +} ``` Interested? Go read the [docs](https://machinezone.github.io/IXWebSocket/)! If things don't work as expected, please create an issue on GitHub, or even better a pull request if you know how to fix your problem. @@ -40,12 +93,58 @@ IXWebSocket is actively being developed, check out the [changelog](https://machi IXWebSocket client code is autobahn compliant beginning with the 6.0.0 version. See the current [test results](https://bsergean.github.io/autobahn/reports/clients/index.html). Some tests are still failing in the server code. +Starting with the 11.0.8 release, IXWebSocket should be fully C++11 compatible. + ## Users If your company or project is using this library, feel free to open an issue or PR to amend this list. - [Machine Zone](https://www.mz.com) - [Tokio](https://gitlab.com/HCInk/tokio), a discord library focused on audio playback with node bindings. -- [libDiscordBot](https://github.com/tostc/libDiscordBot/tree/master), a work in progress discord library +- [libDiscordBot](https://github.com/tostc/libDiscordBot/tree/master), an easy to use Discord-bot framework. - [gwebsocket](https://github.com/norrbotten/gwebsocket), a websocket (lua) module for Garry's Mod - [DisCPP](https://github.com/DisCPP/DisCPP), a simple but feature rich Discord API wrapper +- [discord.cpp](https://github.com/luccanunes/discord.cpp), a discord library for making bots +- [Teleport](http://teleportconnect.com/), Teleport is your own personal remote robot avatar + +## Alternative libraries + +There are plenty of great websocket libraries out there, which might work for you. Here are a couple of serious ones. + +* [websocketpp](https://github.com/zaphoyd/websocketpp) - C++ +* [beast](https://github.com/boostorg/beast) - C++ +* [libwebsockets](https://libwebsockets.org/) - C +* [µWebSockets](https://github.com/uNetworking/uWebSockets) - C +* [wslay](https://github.com/tatsuhiro-t/wslay) - C + +[uvweb](https://github.com/bsergean/uvweb) is a library written by the IXWebSocket author which is built on top of [uvw](https://github.com/skypjack/uvw), which is a C++ wrapper for [libuv](https://libuv.org/). It has more dependencies and does not support SSL at this point, but it can be used to open multiple connections within a single OS thread thanks to libuv. + +To check the performance of a websocket library, you can look at the [autoroute](https://github.com/bsergean/autoroute) project. + +## Continuous Integration + +| OS | TLS | Sanitizer | Status | +|-------------------|-------------------|-------------------|-------------------| +| Linux | OpenSSL | None | [![Build2][1]][0] | +| macOS | Secure Transport | Thread Sanitizer | [![Build2][2]][0] | +| macOS | OpenSSL | Thread Sanitizer | [![Build2][3]][0] | +| macOS | MbedTLS | Thread Sanitizer | [![Build2][4]][0] | +| Windows | Disabled | None | [![Build2][5]][0] | +| UWP | Disabled | None | [![Build2][6]][0] | +| Linux | OpenSSL | Address Sanitizer | [![Build2][7]][0] | +| Mingw | Disabled | None | [![Build2][8]][0] | + +* ASAN fails on Linux because of a known problem, we need a +* Some tests are disabled on Windows/UWP because of a pathing problem +* TLS and ZLIB are disabled on Windows/UWP because enabling make the CI run takes a lot of time, for setting up vcpkg. + +[0]: https://github.com/machinezone/IXWebSocket +[1]: https://github.com/machinezone/IXWebSocket/workflows/linux/badge.svg +[2]: https://github.com/machinezone/IXWebSocket/workflows/mac_tsan_sectransport/badge.svg +[3]: https://github.com/machinezone/IXWebSocket/workflows/mac_tsan_openssl/badge.svg +[4]: https://github.com/machinezone/IXWebSocket/workflows/mac_tsan_mbedtls/badge.svg +[5]: https://github.com/machinezone/IXWebSocket/workflows/windows/badge.svg +[6]: https://github.com/machinezone/IXWebSocket/workflows/uwp/badge.svg +[7]: https://github.com/machinezone/IXWebSocket/workflows/linux_asan/badge.svg +[8]: https://github.com/machinezone/IXWebSocket/workflows/unittest_windows_gcc/badge.svg + diff --git a/docker-compose.yml b/docker-compose.yml index ee59c7cb..0caf0cb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,67 +1,11 @@ -version: "3" +version: "3.3" services: - # snake: - # image: bsergean/ws:build - # entrypoint: ws snake --port 8767 --host 0.0.0.0 --redis_hosts redis1 - # ports: - # - "8767:8767" - # networks: - # - ws-net - # depends_on: - # - redis1 + push: + entrypoint: ws push_server --host 0.0.0.0 + image: ${DOCKER_REPO}/ws:build - # proxy: - # image: bsergean/ws:build - # entrypoint: strace ws proxy_server --remote_host 'wss://cobra.addsrv.com' --host 0.0.0.0 --port 8765 -v - # ports: - # - "8765:8765" - # networks: - # - ws-net - - #pyproxy: - # image: bsergean/ws_proxy:build - # entrypoint: /usr/bin/ws_proxy.py --remote_url 'wss://cobra.addsrv.com' --host 0.0.0.0 --port 8765 - # ports: - # - "8765:8765" - # networks: - # - ws-net - - # # ws: - # # security_opt: - # # - seccomp:unconfined - # # cap_add: - # # - SYS_PTRACE - # # stdin_open: true - # # tty: true - # # image: bsergean/ws:build - # # entrypoint: sh - # # networks: - # # - ws-net - # # depends_on: - # # - redis1 - # # - # # redis1: - # # image: redis:alpine - # # networks: - # # - ws-net - # # - # # statsd: - # # image: jaconel/statsd - # # ports: - # # - "8125:8125" - # # environment: - # # - STATSD_DUMP_MSG=true - # # - GRAPHITE_HOST=127.0.0.1 - # # networks: - # # - ws-net - - compile: - image: alpine - entrypoint: sh - stdin_open: true - tty: true - volumes: - - /Users/bsergeant/src/foss:/home/bsergean/src/foss - -networks: - ws-net: + autoroute: + entrypoint: ws autoroute ws://push:8008 + image: ${DOCKER_REPO}/ws:build + depends_on: + - push diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 9515ab7a..e42528f0 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -1,8 +1,8 @@ -FROM alpine:3.11 as build +FROM alpine:3.12 as build RUN apk add --no-cache \ gcc g++ musl-dev linux-headers \ - cmake mbedtls-dev make zlib-dev ninja + cmake mbedtls-dev make zlib-dev python3-dev ninja git RUN addgroup -S app && \ adduser -S -G app app && \ @@ -15,12 +15,12 @@ COPY --chown=app:app . /opt WORKDIR /opt USER app -RUN make ws_mbedtls_install && \ +RUN make -f makefile.dev ws_mbedtls_install && \ sh tools/trim_repo_for_docker.sh -FROM alpine:3.11 as runtime +FROM alpine:3.12 as runtime -RUN apk add --no-cache libstdc++ mbedtls ca-certificates && \ +RUN apk add --no-cache libstdc++ mbedtls ca-certificates python3 strace && \ addgroup -S app && \ adduser -S -G app app diff --git a/docker/Dockerfile.centos7 b/docker/Dockerfile.centos7 new file mode 100644 index 00000000..adf5c651 --- /dev/null +++ b/docker/Dockerfile.centos7 @@ -0,0 +1,26 @@ +FROM centos:7 as build + +RUN yum install -y gcc-c++ make zlib-devel openssl-devel redhat-rpm-config + +RUN groupadd app && useradd -g app app +RUN chown -R app:app /opt +RUN chown -R app:app /usr/local + +WORKDIR /tmp +RUN curl -O https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.tar.gz +RUN tar zxvf cmake-3.14.0-Linux-x86_64.tar.gz +RUN cp -rf cmake-3.14.0-Linux-x86_64/* /usr/ + +RUN yum install -y git + +# There is a bug in CMake where we cannot build from the root top folder +# So we build from /opt +COPY --chown=app:app . /opt +WORKDIR /opt + +USER app +RUN [ "make", "ws_no_python" ] +RUN [ "rm", "-rf", "build" ] + +ENTRYPOINT ["ws"] +CMD ["--help"] diff --git a/docker/Dockerfile.ubuntu_groovy b/docker/Dockerfile.ubuntu_groovy new file mode 100644 index 00000000..a5e45a1b --- /dev/null +++ b/docker/Dockerfile.ubuntu_groovy @@ -0,0 +1,23 @@ +# Build time +FROM ubuntu:groovy as build + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update + +RUN apt-get -y install g++ libssl-dev libz-dev make python ninja-build +RUN apt-get -y install cmake +RUN apt-get -y install gdb + +COPY . /opt +WORKDIR /opt + +# +# To use the container interactively for debugging/building +# 1. Build with +# CMD ["ls"] +# 2. Run with +# docker run --entrypoint sh -it docker-game-eng-dev.addsrv.com/ws:9.10.6 +# + +RUN ["make", "test"] +# CMD ["ls"] diff --git a/docker/Dockerfile.ubuntu_precise b/docker/Dockerfile.ubuntu_precise new file mode 100644 index 00000000..da72e6db --- /dev/null +++ b/docker/Dockerfile.ubuntu_precise @@ -0,0 +1,27 @@ +# Build time +FROM ubuntu:precise as build + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update +RUN apt-get -y install wget +RUN mkdir -p /tmp/cmake +WORKDIR /tmp/cmake +RUN wget --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz +RUN tar zxf cmake-3.14.0-Linux-x86_64.tar.gz + +RUN apt-get -y install g++ +RUN apt-get -y install libssl-dev +RUN apt-get -y install libz-dev +RUN apt-get -y install make +RUN apt-get -y install python +RUN apt-get -y install git + +COPY . . + +ARG CMAKE_BIN_PATH=/tmp/cmake/cmake-3.14.0-Linux-x86_64/bin +ENV PATH="${CMAKE_BIN_PATH}:${PATH}" + +RUN ["make", "ws_no_python"] + +ENTRYPOINT ["ws"] +CMD ["--help"] diff --git a/docker/Dockerfile.ubuntu_trusty b/docker/Dockerfile.ubuntu_trusty new file mode 100644 index 00000000..a0701e1e --- /dev/null +++ b/docker/Dockerfile.ubuntu_trusty @@ -0,0 +1,22 @@ +# Build time +FROM ubuntu:trusty as build + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update +RUN apt-get -y install wget +RUN mkdir -p /tmp/cmake +WORKDIR /tmp/cmake +RUN wget --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz +RUN tar zxf cmake-3.14.0-Linux-x86_64.tar.gz + +RUN apt-get -y install g++ libssl-dev libz-dev make python git + +COPY . . + +ARG CMAKE_BIN_PATH=/tmp/cmake/cmake-3.14.0-Linux-x86_64/bin +ENV PATH="${CMAKE_BIN_PATH}:${PATH}" + +RUN ["make", "ws_no_python"] + +ENTRYPOINT ["ws"] +CMD ["--help"] diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 27167206..80d5daa2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,457 @@ # Changelog + All changes to this project will be documented in this file. +## [11.3.1] - 2021-10-22 + +(library/cmake) Compatible with MbedTLS 3 + fix a bug on Windows where the incorrect remote port is computed (#320) + +## [11.3.0] - 2021-09-20 + +(library/cmake) Only find OpenSSL, MbedTLS, zlib if they have not already been found, make CMake install optional (#317) + Use GNUInstallDirs in cmake (#318) + +## [11.2.10] - 2021-07-27 + +(ws) bump CLI command line parsing library from 1.8 to 2.0 + +## [11.2.9] - 2021-06-08 + +(ws) ws connect has a -g option to gzip decompress messages for API such as the websocket Huobi Global. + +## [11.2.8] - 2021-06-03 + +(websocket client + server) WebSocketMessage class tweak to fix unsafe patterns + +## [11.2.7] - 2021-05-27 + +(websocket server) Handle and accept firefox browser special upgrade value (keep-alive, Upgrade) + +## [11.2.6] - 2021-05-18 + +(Windows) move EINVAL (re)definition from IXSocket.h to IXNetSystem.h (fix #289) + +## [11.2.5] - 2021-04-04 + +(http client) DEL is not an HTTP method name, but DELETE is + +## [11.2.4] - 2021-03-25 + +(cmake) install IXUniquePtr.h + +## [11.2.3] - 2021-03-24 + +(ssl + windows) missing include for CertOpenStore function + +## [11.2.2] - 2021-03-23 + +(ixwebsocket) version bump + +## [11.2.1] - 2021-03-23 + +(ixwebsocket) version bump + +## [11.2.0] - 2021-03-23 + +(ixwebsocket) correct mingw support (gcc on windows) + +## [11.1.4] - 2021-03-23 + +(ixwebsocket) add getMinWaitBetweenReconnectionRetries + +## [11.1.3] - 2021-03-23 + +(ixwebsocket) New option to set the min wait between reconnection attempts. Still default to 1ms. (setMinWaitBetweenReconnectionRetries). + +## [11.1.2] - 2021-03-22 + +(ws) initialize maxWaitBetweenReconnectionRetries to a non zero value ; a zero value was causing spurious reconnections attempts + +## [11.1.1] - 2021-03-20 + +(cmake) Library can be built as a static or a dynamic library, controlled with BUILD_SHARED_LIBS. Default to static library + +## [11.1.0] - 2021-03-16 + +(ixwebsocket) Use LEAN_AND_MEAN Windows define to help with undefined link error when building a DLL. Support websocket server disablePerMessageDeflate option correctly. + +## [11.0.9] - 2021-03-07 + +(ixwebsocket) Expose setHandshakeTimeout method + +## [11.0.8] - 2020-12-25 + +(ws) trim ws dependencies no more ixcrypto and ixcore deps + +## [11.0.7] - 2020-12-25 + +(ws) trim ws dependencies, only depends on ixcrypto and ixcore + +## [11.0.6] - 2020-12-22 + +(build) rename makefile to makefile.dev to ease cmake BuildExternal (fix #261) + +## [11.0.5] - 2020-12-17 + +(ws) Implement simple header based websocket authorization technique to reject +client which do not supply a certain header ("Authorization") with a special +value (see doc). + +## [11.0.4] - 2020-11-16 + +(ixwebsocket) Handle EINTR return code in ix::poll and IXSelectInterrupt + +## [11.0.3] - 2020-11-16 + +(ixwebsocket) Fix #252 / regression in 11.0.2 with string comparisons + +## [11.0.2] - 2020-11-15 + +(ixwebsocket) use a C++11 compatible make_unique shim + +## [11.0.1] - 2020-11-11 + +(socket) replace a std::vector with an std::array used as a tmp buffer in Socket::readBytes + +## [11.0.0] - 2020-11-11 + +(openssl security fix) in the client to server connection, peer verification is not done in all cases. See https://github.com/machinezone/IXWebSocket/pull/250 + +## [10.5.7] - 2020-11-07 + +(docker) build docker container with zlib disabled + +## [10.5.6] - 2020-11-07 + +(cmake) DEFLATE -> Deflate in CMake to stop warnings about casing + +## [10.5.5] - 2020-11-07 + +(ws autoroute) Display result in compliant way (AUTOROUTE IXWebSocket :: N ms) so that result can be parsed easily + +## [10.5.4] - 2020-10-30 + +(ws gunzip + IXGZipCodec) Can decompress gziped data with libdeflate. ws gunzip computed output filename was incorrect (was the extension aka gz) instead of the file without the extension. Also check whether the output file is writeable. + +## [10.5.3] - 2020-10-19 + +(http code) With zlib disabled, some code should not be reached + +## [10.5.2] - 2020-10-12 + +(ws curl) Add support for --data-binary option, to set the request body. When present the request will be sent with the POST verb + +## [10.5.1] - 2020-10-09 + +(http client + server + ws) Add support for compressing http client requests with gzip. --compress_request argument is used in ws to enable this. The Content-Encoding is set to gzip, and decoded on the server side if present. + +## [10.5.0] - 2020-09-30 + +(http client + server + ws) Add support for uploading files with ws -F foo=@filename, new -D http server option to debug incoming client requests, internal api changed for http POST, PUT and PATCH to supply an HttpFormDataParameters + +## [10.4.9] - 2020-09-30 + +(http server + utility code) Add support for doing gzip compression with libdeflate library, if available + +## [10.4.8] - 2020-09-30 + +(cmake) Stop using FetchContent cmake module to retrieve jsoncpp third party dependency + +## [10.4.7] - 2020-09-28 + +(ws) add gzip and gunzip ws sub commands + +## [10.4.6] - 2020-09-26 + +(cmake) use FetchContent cmake module to retrieve jsoncpp third party dependency + +## [10.4.5] - 2020-09-26 + +(cmake) use FetchContent cmake module to retrieve spdlog third party dependency + +## [10.4.4] - 2020-09-22 + +(cobra connection) retrieve cobra server connection id from the cobra handshake message and display it in ws clients, metrics publisher and bots + +## [10.4.3] - 2020-09-22 + +(cobra 2 cobra) specify as an HTTP header which channel we will republish to + +## [10.4.2] - 2020-09-18 + +(cobra bots) change an error log to a warning log when reconnecting because no messages were received for a minute + +## [10.4.1] - 2020-09-18 + +(cobra connection and bots) set an HTTP header when connecting to help with debugging bots + +## [10.4.0] - 2020-09-12 + +(http server) read body request when the Content-Length is specified + set timeout to read the request to 30 seconds max by default, and make it configurable as a constructor parameter + +## [10.3.5] - 2020-09-09 + +(ws) autoroute command exit on its own once all messages have been received + +## [10.3.4] - 2020-09-04 + +(docker) ws docker file installs strace + +## [10.3.3] - 2020-09-02 + +(ws) echo_client command renamed to autoroute. Command exit once the server close the connection. push_server commands exit once N messages have been sent. + +## [10.3.2] - 2020-08-31 + +(ws + cobra bots) add a cobra_to_cobra ws subcommand to subscribe to a channel and republish received events to a different channel + +## [10.3.1] - 2020-08-28 + +(socket servers) merge the ConnectionInfo class with the ConnectionState one, which simplify all the server apis + +## [10.3.0] - 2020-08-26 + +(ws) set the main thread name, to help with debugging in XCode, gdb, lldb etc... + +## [10.2.9] - 2020-08-19 + +(ws) cobra to python bot / take a module python name as argument foo.bar.baz instead of a path foo/bar/baz.py + +## [10.2.8] - 2020-08-19 + +(ws) on Linux with mbedtls, when the system ca certs are specified (the default) pick up sensible OS supplied paths (tested with CentOS and Alpine) + +## [10.2.7] - 2020-08-18 + +(ws push_server) on the server side, stop sending and close the connection when the remote end has disconnected + +## [10.2.6] - 2020-08-17 + +(ixwebsocket) replace std::unique_ptr with std::array for some fixed arrays (which are in C++11) + +## [10.2.5] - 2020-08-15 + +(ws) merge all ws_*.cpp files into a single one to speedup compilation + +## [10.2.4] - 2020-08-15 + +(socket server) in the loop accepting connections, call select without a timeout on unix to avoid busy looping, and only wake up when a new connection happens + +## [10.2.3] - 2020-08-15 + +(socket server) instead of busy looping with a sleep, only wake up the GC thread when a new thread will have to be joined, (we know that thanks to the ConnectionState OnSetTerminated callback + +## [10.2.2] - 2020-08-15 + +(socket server) add a callback to the ConnectionState to be invoked when the connection is terminated. This will be used by the SocketServer in the future to know on time that the associated connection thread can be terminated. + +## [10.2.1] - 2020-08-15 + +(socket server) do not create a select interrupt object everytime when polling for notifications while waiting for new connections, instead use a persistent one which is a member variable + +## [10.2.0] - 2020-08-14 + +(ixwebsocket client) handle HTTP redirects + +## [10.2.0] - 2020-08-13 + +(ws) upgrade to latest version of nlohmann json (3.9.1 from 3.2.0) + +## [10.1.9] - 2020-08-13 + +(websocket proxy server) add ability to map different hosts to different websocket servers, using a json config file + +## [10.1.8] - 2020-08-12 + +(ws) on macOS, with OpenSSL or MbedTLS, use /etc/ssl/cert.pem as the system certs + +## [10.1.7] - 2020-08-11 + +(ws) -q option imply info log level, not warning log level + +## [10.1.6] - 2020-08-06 + +(websocket server) Handle programmer error when the server callback is not registered properly (fix #227) + +## [10.1.5] - 2020-08-02 + +(ws) Add a new ws sub-command, push_server. This command runs a server which sends many messages in a loop to a websocket client. We can receive above 200,000 messages per second (cf #235). + +## [10.1.4] - 2020-08-02 + +(ws) Add a new ws sub-command, echo_client. This command sends a message to an echo server, and send back to a server whatever message it does receive. When connecting to a local ws echo_server, on my MacBook Pro 2015 I can send/receive around 30,000 messages per second. (cf #235) + +## [10.1.3] - 2020-08-02 + +(ws) ws echo_server. Add a -q option to only enable warning and error log levels. This is useful for bench-marking so that we do not print a lot of things on the console. (cf #235) + +## [10.1.2] - 2020-07-31 + +(build) make using zlib optional, with the caveat that some http and websocket features are not available when zlib is absent + +## [10.1.1] - 2020-07-29 + +(websocket client) onProgressCallback not called for short messages on a websocket (fix #233) + +## [10.1.0] - 2020-07-29 + +(websocket client) heartbeat is not sent at the requested frequency (fix #232) + +## [10.0.3] - 2020-07-28 + +compiler warning fixes + +## [10.0.2] - 2020-07-28 + +(ixcobra) CobraConnection: unsubscribe from all subscriptions when disconnecting + +## [10.0.1] - 2020-07-27 + +(socket utility) move ix::getFreePort to ixwebsocket library + +## [10.0.0] - 2020-07-25 + +(ixwebsocket server) change legacy api with 2 nested callbacks, so that the first api takes a weak_ptr as its first argument + +## [9.10.7] - 2020-07-25 + +(ixwebsocket) add WebSocketProxyServer, from ws. Still need to make the interface better. + +## [9.10.6] - 2020-07-24 + +(ws) port broadcast_server sub-command to the new server API + +## [9.10.5] - 2020-07-24 + +(unittest) port most unittests to the new server API + +## [9.10.3] - 2020-07-24 + +(ws) port ws transfer to the new server API + +## [9.10.2] - 2020-07-24 + +(websocket client) reset WebSocketTransport onClose callback in the WebSocket destructor + +## [9.10.1] - 2020-07-24 + +(websocket server) reset client websocket callback when the connection is closed + +## [9.10.0] - 2020-07-23 + +(websocket server) add a new simpler API to handle client connections / that API does not trigger a memory leak while the previous one did + +## [9.9.3] - 2020-07-17 + +(build) merge platform specific files which were used to have different implementations for setting a thread name into a single file, to make it easier to include every source files and build the ixwebsocket library (fix #226) + +## [9.9.2] - 2020-07-10 + +(socket server) bump default max connection count from 32 to 128 + +## [9.9.1] - 2020-07-10 + +(snake) implement super simple stream sql expression support in snake server + +## [9.9.0] - 2020-07-08 + +(socket+websocket+http+redis+snake servers) expose the remote ip and remote port when a new connection is made + +## [9.8.6] - 2020-07-06 + +(cmake) change the way zlib and openssl are searched + +## [9.8.5] - 2020-07-06 + +(cobra python bots) remove the test which stop the bot when events do not follow cobra metrics system schema with an id and a device entry + +## [9.8.4] - 2020-06-26 + +(cobra bots) remove bots which is not required now that we can use Python extensions + +## [9.8.3] - 2020-06-25 + +(cmake) new python code is optional and enabled at cmake time with -DUSE_PYTHON=1 + +## [9.8.2] - 2020-06-24 + +(cobra bots) new cobra metrics bot to send data to statsd using Python for processing the message + +## [9.8.1] - 2020-06-19 + +(cobra metrics to statsd bot) fps slow frame info : do not include os name + +## [9.8.0] - 2020-06-19 + +(cobra metrics to statsd bot) send info about memory warnings + +## [9.7.9] - 2020-06-18 + +(http client) fix deadlock when following redirects + +## [9.7.8] - 2020-06-18 + +(cobra metrics to statsd bot) send info about net requests + +## [9.7.7] - 2020-06-17 + +(cobra client and bots) add batch_size subscription option for retrieving multiple messages at once + +## [9.7.6] - 2020-06-15 + +(websocket) WebSocketServer is not a final class, so that users can extend it (fix #215) + +## [9.7.5] - 2020-06-15 + +(cobra bots) minor aesthetic change, in how we display http headers with a : then space as key value separator instead of :: with no space + +## [9.7.4] - 2020-06-11 + +(cobra metrics to statsd bot) change from a statsd type of gauge to a timing one + +## [9.7.3] - 2020-06-11 + +(redis cobra bots) capture most used devices in a zset + +## [9.7.2] - 2020-06-11 + +(ws) add bare bone redis-cli like sub-command, with command line editing powered by libnoise + +## [9.7.1] - 2020-06-11 + +(redis cobra bots) ws cobra metrics to redis / hostname invalid parsing + +## [9.7.0] - 2020-06-11 + +(redis cobra bots) xadd with maxlen + fix bug in xadd client implementation and ws cobra metrics to redis command argument parsing + +## [9.6.9] - 2020-06-10 + +(redis cobra bots) update the cobra to redis bot to use the bot framework, and change it to report fps metrics into redis streams. + +## [9.6.6] - 2020-06-04 + +(statsd cobra bots) statsd improvement: prefix does not need a dot as a suffix, message size can be larger than 256 bytes, error handling was invalid, use core logger for logging instead of std::cerr + +## [9.6.5] - 2020-05-29 + +(http server) support gzip compression + +## [9.6.4] - 2020-05-20 + +(compiler fix) support clang 5 and earlier (contributed by @LunarWatcher) + +## [9.6.3] - 2020-05-18 + +(cmake) revert CMake changes to fix #203 and be able to use an external OpenSSL + +## [9.6.2] - 2020-05-17 + +(cmake) make install cmake files optional to not conflict with vcpkg + +## [9.6.1] - 2020-05-17 + +(windows + tls) mbedtls is the default windows tls backend + add ability to load system certificates with mbdetls on windows + ## [9.6.0] - 2020-05-12 (ixbots) add options to limit how many messages per minute should be processed diff --git a/docs/build.md b/docs/build.md index 897e4432..09c6f4dd 100644 --- a/docs/build.md +++ b/docs/build.md @@ -17,13 +17,15 @@ There is a unittest which can be executed by typing `make test`. Options for building: +* `-DBUILD_SHARED_LIBS=ON` will build the unittest as a shared libary instead of a static library, which is the default +* `-DUSE_ZLIB=1` will enable zlib support, required for http client + server + websocket per message deflate extension * `-DUSE_TLS=1` will enable TLS support -* `-DUSE_OPEN_SSL=1` will use [openssl](https://www.openssl.org/) for the TLS support (default on Linux and Windows) +* `-DUSE_OPEN_SSL=1` will use [openssl](https://www.openssl.org/) for the TLS support (default on Linux and Windows). When using a custom version of openssl (say a prebuilt version, odd runtime problems can happens, as in #319, and special cmake trickery will be required (see this [comment](https://github.com/machinezone/IXWebSocket/issues/175#issuecomment-620231032)) * `-DUSE_MBED_TLS=1` will use [mbedlts](https://tls.mbed.org/) for the TLS support * `-DUSE_WS=1` will build the ws interactive command line tool * `-DUSE_TEST=1` will build the unittest -If you are on Windows, look at the [appveyor](https://github.com/machinezone/IXWebSocket/blob/master/appveyor.yml) file (not maintained much though) or rather the [github actions](https://github.com/machinezone/IXWebSocket/blob/master/.github/workflows/ccpp.yml#L40) which have instructions for building dependencies. +If you are on Windows, look at the [appveyor](https://github.com/machinezone/IXWebSocket/blob/master/appveyor.yml) file (not maintained much though) or rather the [github actions](https://github.com/machinezone/IXWebSocket/blob/master/.github/workflows/unittest_windows.yml) which have instructions for building dependencies. It is also possible to externally include the project, so that everything is fetched over the wire when you build like so: @@ -42,6 +44,19 @@ It is possible to get IXWebSocket through Microsoft [vcpkg](https://github.com/m ``` vcpkg install ixwebsocket ``` +To use the installed package within a cmake project, use the following: +```cmake + set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "") # this is super important in order for cmake to include the vcpkg search/lib paths! + + # find library and its headers + find_path(IXWEBSOCKET_INCLUDE_DIR ixwebsocket/IXWebSocket.h) + find_library(IXWEBSOCKET_LIBRARY ixwebsocket) + # include headers + include_directories(${IXWEBSOCKET_INCLUDE_DIR}) + # ... + target_link_libraries(${PROJECT_NAME} ... ${IXWEBSOCKET_LIBRARY}) # Cmake will automatically fail the generation if the lib was not found, i.e is set to NOTFOUNS + +``` ### Conan diff --git a/docs/cobra.md b/docs/cobra.md deleted file mode 100644 index fd195774..00000000 --- a/docs/cobra.md +++ /dev/null @@ -1,81 +0,0 @@ -## General - -[cobra](https://github.com/machinezone/cobra) is a real time messaging server. The `ws` utility can run a cobra server (named snake), and has client to publish and subscribe to a cobra server. - -Bring up 3 terminals and run a server, a publisher and a subscriber in each one. As you publish data you should see it being received by the subscriber. You can run `redis-cli MONITOR` too to see how redis is being used. - -### Server - -You will need to have a redis server running locally. To run the server: - -```bash -$ cd /ixsnake/ixsnake -$ ws snake -{ - "apps": { - "FC2F10139A2BAc53BB72D9db967b024f": { - "roles": { - "_sub": { - "secret": "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba" - }, - "_pub": { - "secret": "1c04DB8fFe76A4EeFE3E318C72d771db" - } - } - } - } -} - -redis host: 127.0.0.1 -redis password: -redis port: 6379 -``` - -### Publisher - -```bash -$ cd /ws -$ ws cobra_publish --appkey FC2F10139A2BAc53BB72D9db967b024f --endpoint ws://127.0.0.1:8008 --rolename _pub --rolesecret 1c04DB8fFe76A4EeFE3E318C72d771db test_channel cobraMetricsSample.json -[2019-11-27 09:06:12.980] [info] Publisher connected -[2019-11-27 09:06:12.980] [info] Connection: Upgrade -[2019-11-27 09:06:12.980] [info] Sec-WebSocket-Accept: zTtQKMKbvwjdivURplYXwCVUCWM= -[2019-11-27 09:06:12.980] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -[2019-11-27 09:06:12.980] [info] Server: ixwebsocket/7.4.0 macos ssl/DarwinSSL zlib 1.2.11 -[2019-11-27 09:06:12.980] [info] Upgrade: websocket -[2019-11-27 09:06:12.982] [info] Publisher authenticated -[2019-11-27 09:06:12.982] [info] Published msg 3 -[2019-11-27 09:06:12.982] [info] Published message id 3 acked -``` - -### Subscriber - -```bash -$ ws cobra_subscribe --appkey FC2F10139A2BAc53BB72D9db967b024f --endpoint ws://127.0.0.1:8008 --rolename _pub --rolesecret 1c04DB8fFe76A4EeFE3E318C72d771db test_channel -#messages 0 msg/s 0 -[2019-11-27 09:07:39.341] [info] Subscriber connected -[2019-11-27 09:07:39.341] [info] Connection: Upgrade -[2019-11-27 09:07:39.341] [info] Sec-WebSocket-Accept: 9vkQWofz49qMCUlTSptCCwHWm+Q= -[2019-11-27 09:07:39.341] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -[2019-11-27 09:07:39.341] [info] Server: ixwebsocket/7.4.0 macos ssl/DarwinSSL zlib 1.2.11 -[2019-11-27 09:07:39.341] [info] Upgrade: websocket -[2019-11-27 09:07:39.342] [info] Subscriber authenticated -[2019-11-27 09:07:39.345] [info] Subscriber: subscribed to channel test_channel -#messages 0 msg/s 0 -#messages 0 msg/s 0 -#messages 0 msg/s 0 -{"baz":123,"foo":"bar"} - -#messages 1 msg/s 1 -#messages 1 msg/s 0 -#messages 1 msg/s 0 -{"baz":123,"foo":"bar"} - -{"baz":123,"foo":"bar"} - -#messages 3 msg/s 2 -#messages 3 msg/s 0 -{"baz":123,"foo":"bar"} - -#messages 4 msg/s 1 -^C -``` diff --git a/docs/index.md b/docs/index.md index 8439d04b..0f1f5a01 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,3 @@ -![Build status](https://github.com/machinezone/IXWebSocket/workflows/unittest/badge.svg) - ## Introduction [*WebSocket*](https://en.wikipedia.org/wiki/WebSocket) is a computer communications protocol, providing full-duplex and bi-directionnal communication channels over a single TCP connection. *IXWebSocket* is a C++ library for client and server Websocket communication, and for client and server HTTP communication. *TLS* aka *SSL* is supported. The code is derived from [easywsclient](https://github.com/dhbaird/easywsclient) and from the [Satori C SDK](https://github.com/satori-com/satori-rtm-sdk-c). It has been tested on the following platforms. diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 00000000..3720bc59 --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,37 @@ + +## WebSocket Client performance + +We will run a client and a server on the same machine, connecting to localhost. This bench is run on a MacBook Pro from 2015. We can receive over 200,000 (small) messages per second, another way to put it is that it takes 5 micro-second to receive and process one message. This is an indication about the minimal latency to receive messages. + +### Receiving messages + +By using the push_server ws sub-command, the server will send the same message in a loop to any connected client. + +``` +ws push_server -q --send_msg 'yo' +``` + +By using the echo_client ws sub-command, with the -m (mute or no_send), we will display statistics on how many messages we can receive per second. + +``` +$ ws echo_client -m ws://localhost:8008 +[2020-08-02 12:31:17.284] [info] ws_echo_client: connected +[2020-08-02 12:31:17.284] [info] Uri: / +[2020-08-02 12:31:17.284] [info] Headers: +[2020-08-02 12:31:17.284] [info] Connection: Upgrade +[2020-08-02 12:31:17.284] [info] Sec-WebSocket-Accept: byy/pMK2d0PtRwExaaiOnXJTQHo= +[2020-08-02 12:31:17.284] [info] Server: ixwebsocket/10.1.4 macos ssl/SecureTransport zlib 1.2.11 +[2020-08-02 12:31:17.284] [info] Upgrade: websocket +[2020-08-02 12:31:17.663] [info] messages received: 0 per second 2595307 total +[2020-08-02 12:31:18.668] [info] messages received: 79679 per second 2674986 total +[2020-08-02 12:31:19.668] [info] messages received: 207438 per second 2882424 total +[2020-08-02 12:31:20.673] [info] messages received: 209207 per second 3091631 total +[2020-08-02 12:31:21.676] [info] messages received: 216056 per second 3307687 total +[2020-08-02 12:31:22.680] [info] messages received: 214927 per second 3522614 total +[2020-08-02 12:31:23.684] [info] messages received: 216960 per second 3739574 total +[2020-08-02 12:31:24.688] [info] messages received: 215232 per second 3954806 total +[2020-08-02 12:31:25.691] [info] messages received: 212300 per second 4167106 total +[2020-08-02 12:31:26.694] [info] messages received: 212501 per second 4379607 total +[2020-08-02 12:31:27.699] [info] messages received: 212330 per second 4591937 total +[2020-08-02 12:31:28.702] [info] messages received: 216511 per second 4808448 total +``` diff --git a/docs/usage.md b/docs/usage.md index 99a38791..f3686ee0 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -67,9 +67,28 @@ webSocket.stop() ### Sending messages -`websocket.send("foo")` will send a message. +`WebSocketSendInfo result = websocket.send("foo")` will send a message. -If the connection was closed and sending failed, the return value will be set to false. +If the connection was closed, sending will fail, and the success field of the result object will be set to false. There could also be a compression error in which case the compressError field will be set to true. The payloadSize field and wireSize fields will tell you respectively how much bytes the message weight, and how many bytes were sent on the wire (potentially compressed + counting the message header (a few bytes). + +There is an optional progress callback that can be passed in as the second argument. If a message is large it will be fragmented into chunks which will be sent independantly. Everytime the we can write a fragment into the OS network cache, the callback will be invoked. If a user wants to cancel a slow send, false should be returned from within the callback. + +Here is an example code snippet copied from the ws send sub-command. Each fragment weights 32K, so the total integer is the wireSize divided by 32K. As an example if you are sending 32M of data, uncompressed, total will be 1000. current will be set to 0 for the first fragment, then 1, 2 etc... + +``` +auto result = + _webSocket.sendBinary(serializedMsg, [this, throttle](int current, int total) -> bool { + spdlog::info("ws_send: Step {} out of {}", current + 1, total); + + if (throttle) + { + std::chrono::duration duration(10); + std::this_thread::sleep_for(duration); + } + + return _connected; + }); +``` ### ReadyState @@ -237,15 +256,32 @@ Wait time(ms): 6400 Wait time(ms): 10000 ``` -The waiting time is capped by default at 10s between 2 attempts, but that value can be changed and queried. +The waiting time is capped by default at 10s between 2 attempts, but that value +can be changed and queried. The minimum waiting time can also be set. ```cpp webSocket.setMaxWaitBetweenReconnectionRetries(5 * 1000); // 5000ms = 5s uint32_t m = webSocket.getMaxWaitBetweenReconnectionRetries(); + +webSocket.setMinWaitBetweenReconnectionRetries(1000); // 1000ms = 1s +uint32_t m = webSocket.getMinWaitBetweenReconnectionRetries(); +``` + +## Handshake timeout + +You can control how long to wait until timing out while waiting for the websocket handshake to be performed. + +``` +int handshakeTimeoutSecs = 1; +setHandshakeTimeout(handshakeTimeoutSecs); ``` ## WebSocket server API +### Legacy api + +This api was actually changed to take a weak_ptr as the first argument to setOnConnectionCallback ; previously it would take a shared_ptr which was creating cycles and then memory leaks problems. + ```cpp #include @@ -256,38 +292,48 @@ uint32_t m = webSocket.getMaxWaitBetweenReconnectionRetries(); ix::WebSocketServer server(port); server.setOnConnectionCallback( - [&server](std::shared_ptr webSocket, + [&server](std::weak_ptr webSocket, std::shared_ptr connectionState) { - webSocket->setOnMessageCallback( - [webSocket, connectionState, &server](const ix::WebSocketMessagePtr msg) - { - if (msg->type == ix::WebSocketMessageType::Open) + std::cout << "Remote ip: " << connectionState->remoteIp << std::endl; + + auto ws = webSocket.lock(); + if (ws) + { + ws->setOnMessageCallback( + [webSocket, connectionState, &server](const ix::WebSocketMessagePtr msg) { - std::cerr << "New connection" << std::endl; - - // A connection state object is available, and has a default id - // You can subclass ConnectionState and pass an alternate factory - // to override it. It is useful if you want to store custom - // attributes per connection (authenticated bool flag, attributes, etc...) - std::cerr << "id: " << connectionState->getId() << std::endl; - - // The uri the client did connect to. - std::cerr << "Uri: " << msg->openInfo.uri << std::endl; - - std::cerr << "Headers:" << std::endl; - for (auto it : msg->openInfo.headers) + if (msg->type == ix::WebSocketMessageType::Open) { - std::cerr << it.first << ": " << it.second << std::endl; + std::cout << "New connection" << std::endl; + + // A connection state object is available, and has a default id + // You can subclass ConnectionState and pass an alternate factory + // to override it. It is useful if you want to store custom + // attributes per connection (authenticated bool flag, attributes, etc...) + std::cout << "id: " << connectionState->getId() << std::endl; + + // The uri the client did connect to. + std::cout << "Uri: " << msg->openInfo.uri << std::endl; + + std::cout << "Headers:" << std::endl; + for (auto it : msg->openInfo.headers) + { + std::cout << it.first << ": " << it.second << std::endl; + } + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + // For an echo server, we just send back to the client whatever was received by the server + // All connected clients are available in an std::set. See the broadcast cpp example. + // Second parameter tells whether we are sending the message in binary or text mode. + // Here we send it in the same mode as it was received. + auto ws = webSocket.lock(); + if (ws) + { + ws->send(msg->str, msg->binary); + } } - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - // For an echo server, we just send back to the client whatever was received by the server - // All connected clients are available in an std::set. See the broadcast cpp example. - // Second parameter tells whether we are sending the message in binary or text mode. - // Here we send it in the same mode as it was received. - webSocket->send(msg->str, msg->binary); } } ); @@ -301,6 +347,80 @@ if (!res.first) return 1; } +// Per message deflate connection is enabled by default. It can be disabled +// which might be helpful when running on low power devices such as a Rasbery Pi +server.disablePerMessageDeflate(); + +// Run the server in the background. Server can be stoped by calling server.stop() +server.start(); + +// Block until server.stop() is called. +server.wait(); + +``` + +### New api + +The new API does not require to use 2 nested callbacks, which is a bit annoying. The real fix is that there was a memory leak due to a shared_ptr cycle, due to passing down a shared_ptr down to the callbacks. + +The webSocket reference is guaranteed to be always valid ; by design the callback will never be invoked with a null webSocket object. + +```cpp +#include + +... + +// Run a server on localhost at a given port. +// Bound host name, max connections and listen backlog can also be passed in as parameters. +ix::WebSocketServer server(port); + +server.setOnClientMessageCallback([](std::shared_ptr connectionState, ix::WebSocket & webSocket, const ix::WebSocketMessagePtr & msg) { + // The ConnectionState object contains information about the connection, + // at this point only the client ip address and the port. + std::cout << "Remote ip: " << connectionState->getRemoteIp() << std::endl; + + if (msg->type == ix::WebSocketMessageType::Open) + { + std::cout << "New connection" << std::endl; + + // A connection state object is available, and has a default id + // You can subclass ConnectionState and pass an alternate factory + // to override it. It is useful if you want to store custom + // attributes per connection (authenticated bool flag, attributes, etc...) + std::cout << "id: " << connectionState->getId() << std::endl; + + // The uri the client did connect to. + std::cout << "Uri: " << msg->openInfo.uri << std::endl; + + std::cout << "Headers:" << std::endl; + for (auto it : msg->openInfo.headers) + { + std::cout << "\t" << it.first << ": " << it.second << std::endl; + } + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + // For an echo server, we just send back to the client whatever was received by the server + // All connected clients are available in an std::set. See the broadcast cpp example. + // Second parameter tells whether we are sending the message in binary or text mode. + // Here we send it in the same mode as it was received. + std::cout << "Received: " << msg->str << std::endl; + + webSocket.send(msg->str, msg->binary); + } +}); + +auto res = server.listen(); +if (!res.first) +{ + // Error handling + return 1; +} + +// Per message deflate connection is enabled by default. It can be disabled +// which might be helpful when running on low power devices such as a Rasbery Pi +server.disablePerMessageDeflate(); + // Run the server in the background. Server can be stoped by calling server.stop() server.start(); @@ -358,18 +478,25 @@ out = httpClient.get(url, args); // POST request with parameters HttpParameters httpParameters; httpParameters["foo"] = "bar"; -out = httpClient.post(url, httpParameters, args); + +// HTTP form data can be passed in as well, for multi-part upload of files +HttpFormDataParameters httpFormDataParameters; +httpParameters["baz"] = "booz"; + +out = httpClient.post(url, httpParameters, httpFormDataParameters, args); // POST request with a body out = httpClient.post(url, std::string("foo=bar"), args); +// PUT and PATCH are available too. + // // Result // auto statusCode = response->statusCode; // Can be HttpErrorCode::Ok, HttpErrorCode::UrlMalformed, etc... auto errorCode = response->errorCode; // 200, 404, etc... auto responseHeaders = response->headers; // All the headers in a special case-insensitive unordered_map of (string, string) -auto payload = response->payload; // All the bytes from the response as an std::string +auto body = response->body; // All the bytes from the response as an std::string auto errorMsg = response->errorMsg; // Descriptive error message in case of failure auto uploadSize = response->uploadSize; // Byte count of uploaded data auto downloadSize = response->downloadSize; // Byte count of downloaded data @@ -392,6 +519,8 @@ bool ok = httpClient.performRequest(args, [](const HttpResponsePtr& response) // ok will be false if your httpClient is not async ``` +See this [issue](https://github.com/machinezone/IXWebSocket/issues/209) for links about uploading files with HTTP multipart. + ## HTTP server API ```cpp @@ -415,11 +544,13 @@ If you want to handle how requests are processed, implement the setOnConnectionC ```cpp setOnConnectionCallback( [this](HttpRequestPtr request, - std::shared_ptr /*connectionState*/) -> HttpResponsePtr + std::shared_ptr connectionState) -> HttpResponsePtr { // Build a string for the response std::stringstream ss; - ss << request->method + ss << connectionState->getRemoteIp(); + << " " + << request->method << " " << request->uri; diff --git a/docs/ws.md b/docs/ws.md index 5d9d73a4..348e2baf 100644 --- a/docs/ws.md +++ b/docs/ws.md @@ -19,13 +19,6 @@ Subcommands: broadcast_server Broadcasting server ping Ping pong curl HTTP Client - redis_publish Redis publisher - redis_subscribe Redis subscriber - cobra_subscribe Cobra subscriber - cobra_publish Cobra publisher - cobra_to_statsd Cobra to statsd - cobra_to_sentry Cobra to sentry - snake Snake server httpd HTTP server ``` @@ -195,6 +188,63 @@ Server: Python/3.7 websockets/8.0.2 Upgrade: websocket ``` +It is possible to pass custom HTTP header when doing the connection handshake, +the remote server might process them to implement a simple authorization +scheme. + +``` +src$ ws connect -H Authorization:supersecret ws://localhost:8008 +Type Ctrl-D to exit prompt... +[2020-12-17 22:35:08.732] [info] Authorization: supersecret +Connecting to url: ws://localhost:8008 +> [2020-12-17 22:35:08.736] [info] ws_connect: connected +[2020-12-17 22:35:08.736] [info] Uri: / +[2020-12-17 22:35:08.736] [info] Headers: +[2020-12-17 22:35:08.736] [info] Connection: Upgrade +[2020-12-17 22:35:08.736] [info] Sec-WebSocket-Accept: 2yaTFcdwn8KL6IzSMj2u6Le7KTg= +[2020-12-17 22:35:08.736] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 +[2020-12-17 22:35:08.736] [info] Server: ixwebsocket/11.0.4 macos ssl/SecureTransport zlib 1.2.11 +[2020-12-17 22:35:08.736] [info] Upgrade: websocket +[2020-12-17 22:35:08.736] [info] Received 25 bytes +ws_connect: received message: Authorization suceeded! +[2020-12-17 22:35:08.736] [info] Received pong ixwebsocket::heartbeat::30s::0 +hello +> [2020-12-17 22:35:25.157] [info] Received 7 bytes +ws_connect: received message: hello +``` + +If the wrong header is passed in, the server would close the connection with a custom close code (>4000, and <4999). + +``` +[2020-12-17 22:39:37.044] [info] Upgrade: websocket +ws_connect: connection closed: code 4001 reason Permission denied +``` + +## echo server + +The ws echo server will respond what the client just sent him. If we use the +simple --http_authorization_header we can enforce that client need to pass a +special value in the Authorization header to connect. + +``` +$ ws echo_server --http_authorization_header supersecret +[2020-12-17 22:35:06.192] [info] Listening on 127.0.0.1:8008 +[2020-12-17 22:35:08.735] [info] New connection +[2020-12-17 22:35:08.735] [info] remote ip: 127.0.0.1 +[2020-12-17 22:35:08.735] [info] id: 0 +[2020-12-17 22:35:08.735] [info] Uri: / +[2020-12-17 22:35:08.735] [info] Headers: +[2020-12-17 22:35:08.735] [info] Authorization: supersecret +[2020-12-17 22:35:08.735] [info] Connection: Upgrade +[2020-12-17 22:35:08.735] [info] Host: localhost:8008 +[2020-12-17 22:35:08.735] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 +[2020-12-17 22:35:08.735] [info] Sec-WebSocket-Key: eFF2Gf25dC7eC15Ab1135G== +[2020-12-17 22:35:08.735] [info] Sec-WebSocket-Version: 13 +[2020-12-17 22:35:08.735] [info] Upgrade: websocket +[2020-12-17 22:35:08.735] [info] User-Agent: ixwebsocket/11.0.4 macos ssl/SecureTransport zlib 1.2.11 +[2020-12-17 22:35:25.157] [info] Received 7 bytes +``` + ## Websocket proxy ``` @@ -204,6 +254,20 @@ Listening on 127.0.0.1:8008 If you connect to ws://127.0.0.1:8008, the proxy will connect to ws://127.0.0.1:9000 and pass all traffic to this server. +You can also use a more complex setup if you want to redirect to different websocket servers based on the hostname your client is trying to connect to. If you have multiple CNAME aliases that point to the same server. + +A JSON config file is used to express that mapping ; here connecting to echo.jeanserge.com will proxy the client to ws://localhost:8008 on the local machine (which actually runs ws echo_server), while connecting to bavarde.jeanserge.com will proxy the client to ws://localhost:5678 where a cobra python server is running. As a side note you will need a wildcard SSL certificate if you want to have SSL enabled on that machine. + +``` +echo.jeanserge.com=ws://localhost:8008 +bavarde.jeanserge.com=ws://localhost:5678 +``` +The --config_path option is required to instruct ws proxy_server to read that file. + +``` +ws proxy_server --config_path proxyConfig.json --port 8765 +``` + ## File transfer ``` @@ -242,128 +306,3 @@ Options: --connect-timeout INT Connection timeout --transfer-timeout INT Transfer timeout ``` - -## Cobra client and server - -[cobra](https://github.com/machinezone/cobra) is a real time messenging server. ws has several sub-command to interact with cobra. There is also a minimal cobra compatible server named snake available. - -Below are examples on running a snake server and clients with TLS enabled (the server only works with the OpenSSL and the Mbed TLS backend for now). - -First, generate certificates. - -``` -$ cd /path/to/IXWebSocket -$ cd ixsnake/ixsnake -$ bash ../../ws/generate_certs.sh -Generating RSA private key, 2048 bit long modulus -.....+++ -.................+++ -e is 65537 (0x10001) -generated ./.certs/trusted-ca-key.pem -generated ./.certs/trusted-ca-crt.pem -Generating RSA private key, 2048 bit long modulus -..+++ -.......................................+++ -e is 65537 (0x10001) -generated ./.certs/trusted-server-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=trusted-server -Getting CA Private Key -generated ./.certs/trusted-server-crt.pem -Generating RSA private key, 2048 bit long modulus -...................................+++ -..................................................+++ -e is 65537 (0x10001) -generated ./.certs/trusted-client-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=trusted-client -Getting CA Private Key -generated ./.certs/trusted-client-crt.pem -Generating RSA private key, 2048 bit long modulus -..............+++ -.......................................+++ -e is 65537 (0x10001) -generated ./.certs/untrusted-ca-key.pem -generated ./.certs/untrusted-ca-crt.pem -Generating RSA private key, 2048 bit long modulus -..........+++ -................................................+++ -e is 65537 (0x10001) -generated ./.certs/untrusted-client-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=untrusted-client -Getting CA Private Key -generated ./.certs/untrusted-client-crt.pem -Generating RSA private key, 2048 bit long modulus -.....................................................................................+++ -...........+++ -e is 65537 (0x10001) -generated ./.certs/selfsigned-client-key.pem -Signature ok -subject=/O=machinezone/O=IXWebSocket/CN=selfsigned-client -Getting Private key -generated ./.certs/selfsigned-client-crt.pem -``` - -Now run the snake server. - -``` -$ export certs=.certs -$ ws snake --tls --port 8765 --cert-file ${certs}/trusted-server-crt.pem --key-file ${certs}/trusted-server-key.pem --ca-file ${certs}/trusted-ca-crt.pem -{ - "apps": { - "FC2F10139A2BAc53BB72D9db967b024f": { - "roles": { - "_sub": { - "secret": "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba" - }, - "_pub": { - "secret": "1c04DB8fFe76A4EeFE3E318C72d771db" - } - } - } - } -} - -redis host: 127.0.0.1 -redis password: -redis port: 6379 -``` - -As a new connection comes in, such output should be printed - -``` -[2019-12-19 20:27:19.724] [info] New connection -id: 0 -Uri: /v2?appkey=_health -Headers: -Connection: Upgrade -Host: 127.0.0.1:8765 -Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -Sec-WebSocket-Key: d747B0fE61Db73f7Eh47c0== -Sec-WebSocket-Protocol: json -Sec-WebSocket-Version: 13 -Upgrade: websocket -User-Agent: ixwebsocket/7.5.8 macos ssl/OpenSSL OpenSSL 1.0.2q 20 Nov 2018 zlib 1.2.11 -``` - -To connect and publish a message, do: - -``` -$ export certs=.certs -$ cd /path/to/ws/folder -$ ls cobraMetricsSample.json -cobraMetricsSample.json -$ ws cobra_publish --endpoint wss://127.0.0.1:8765 --appkey FC2F10139A2BAc53BB72D9db967b024f --rolename _pub --rolesecret 1c04DB8fFe76A4EeFE3E318C72d771db --channel foo --cert-file ${certs}/trusted-client-crt.pem --key-file ${certs}/trusted-client-key.pem --ca-file ${certs}/trusted-ca-crt.pem cobraMetricsSample.json -[2019-12-19 20:46:42.656] [info] Publisher connected -[2019-12-19 20:46:42.657] [info] Connection: Upgrade -[2019-12-19 20:46:42.657] [info] Sec-WebSocket-Accept: rs99IFThoBrhSg+k8G4ixH9yaq4= -[2019-12-19 20:46:42.657] [info] Sec-WebSocket-Extensions: permessage-deflate; server_max_window_bits=15; client_max_window_bits=15 -[2019-12-19 20:46:42.657] [info] Server: ixwebsocket/7.5.8 macos ssl/OpenSSL OpenSSL 1.0.2q 20 Nov 2018 zlib 1.2.11 -[2019-12-19 20:46:42.657] [info] Upgrade: websocket -[2019-12-19 20:46:42.658] [info] Publisher authenticated -[2019-12-19 20:46:42.658] [info] Published msg 3 -[2019-12-19 20:46:42.659] [info] Published message id 3 acked -``` - -To use OpenSSL on macOS, compile with `make ws_openssl`. First you will have to install OpenSSL libraries, which can be done with Homebrew. Use `make ws_mbedtls` accordingly to use MbedTLS. diff --git a/httpd.cpp b/httpd.cpp new file mode 100644 index 00000000..b1e580f1 --- /dev/null +++ b/httpd.cpp @@ -0,0 +1,46 @@ +/* + * httpd.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + * + * Buid with make httpd + */ + +#include +#include +#include + +int main(int argc, char** argv) +{ + if (argc != 3) + { + std::cerr << "Usage: " << argv[0] + << " " << std::endl; + std::cerr << " " << argv[0] << " 9090 127.0.0.1" << std::endl; + std::cerr << " " << argv[0] << " 9090 0.0.0.0" << std::endl; + return 1; + } + + int port; + std::stringstream ss; + ss << argv[1]; + ss >> port; + std::string hostname(argv[2]); + + std::cout << "Listening on " << hostname + << ":" << port << std::endl; + + ix::HttpServer server(port, hostname); + + auto res = server.listen(); + if (!res.first) + { + std::cout << res.second << std::endl; + return 1; + } + + server.start(); + server.wait(); + + return 0; +} diff --git a/ixbots/CMakeLists.txt b/ixbots/CMakeLists.txt deleted file mode 100644 index 2e5c1d3f..00000000 --- a/ixbots/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXBOTS_SOURCES - ixbots/IXCobraBot.cpp - ixbots/IXCobraToSentryBot.cpp - ixbots/IXCobraToStatsdBot.cpp - ixbots/IXCobraToStdoutBot.cpp - ixbots/IXStatsdClient.cpp -) - -set (IXBOTS_HEADERS - ixbots/IXCobraBot.h - ixbots/IXCobraBotConfig.h - ixbots/IXCobraToSentryBot.h - ixbots/IXCobraToStatsdBot.h - ixbots/IXCobraToStdoutBot.h - ixbots/IXStatsdClient.h -) - -add_library(ixbots STATIC - ${IXBOTS_SOURCES} - ${IXBOTS_HEADERS} -) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) -endif() - -set(IXBOTS_INCLUDE_DIRS - . - .. - ../ixcore - ../ixwebsocket - ../ixcobra - ../ixsentry - ${JSONCPP_INCLUDE_DIRS} - ${SPDLOG_INCLUDE_DIRS}) - -target_include_directories( ixbots PUBLIC ${IXBOTS_INCLUDE_DIRS} ) diff --git a/ixbots/ixbots/IXCobraBot.cpp b/ixbots/ixbots/IXCobraBot.cpp deleted file mode 100644 index 92d35072..00000000 --- a/ixbots/ixbots/IXCobraBot.cpp +++ /dev/null @@ -1,268 +0,0 @@ -/* - * IXCobraBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraBot.h" - -#include -#include - -#include -#include -#include -#include -#include - -namespace ix -{ - int64_t CobraBot::run(const CobraBotConfig& botConfig) - { - auto config = botConfig.cobraConfig; - auto channel = botConfig.channel; - auto filter = botConfig.filter; - auto position = botConfig.position; - auto enableHeartbeat = botConfig.enableHeartbeat; - auto heartBeatTimeout = botConfig.heartBeatTimeout; - auto runtime = botConfig.runtime; - auto maxEventsPerMinute = botConfig.maxEventsPerMinute; - auto limitReceivedEvents = botConfig.limitReceivedEvents; - - ix::CobraConnection conn; - conn.configure(config); - conn.connect(); - - std::atomic sentCount(0); - std::atomic receivedCount(0); - uint64_t sentCountTotal(0); - uint64_t receivedCountTotal(0); - uint64_t sentCountPerSecs(0); - uint64_t receivedCountPerSecs(0); - std::atomic receivedCountPerMinutes(0); - std::atomic stop(false); - std::atomic throttled(false); - std::atomic fatalCobraError(false); - int minuteCounter = 0; - - auto timer = [&sentCount, - &receivedCount, - &sentCountTotal, - &receivedCountTotal, - &sentCountPerSecs, - &receivedCountPerSecs, - &receivedCountPerMinutes, - &minuteCounter, - &stop] { - while (!stop) - { - // - // We cannot write to sentCount and receivedCount - // as those are used externally, so we need to introduce - // our own counters - // - std::stringstream ss; - ss << "messages received " - << receivedCountPerSecs - << " " - << receivedCountTotal - << " sent " - << sentCountPerSecs - << " " - << sentCountTotal; - CoreLogger::info(ss.str()); - - receivedCountPerSecs = receivedCount - receivedCountTotal; - sentCountPerSecs = sentCount - sentCountTotal; - - receivedCountTotal += receivedCountPerSecs; - sentCountTotal += sentCountPerSecs; - - auto duration = std::chrono::seconds(1); - std::this_thread::sleep_for(duration); - - if (minuteCounter++ == 60) - { - receivedCountPerMinutes = 0; - minuteCounter = 0; - } - } - - CoreLogger::info("timer thread done"); - }; - - std::thread t1(timer); - - auto heartbeat = [&sentCount, &receivedCount, &stop, &enableHeartbeat, &heartBeatTimeout, &fatalCobraError] { - std::string state("na"); - - if (!enableHeartbeat) return; - - while (!stop) - { - std::stringstream ss; - ss << "messages received " << receivedCount; - ss << "messages sent " << sentCount; - - std::string currentState = ss.str(); - - if (currentState == state) - { - CoreLogger::error("no messages received or sent for 1 minute, exiting"); - fatalCobraError = true; - break; - } - state = currentState; - - auto duration = std::chrono::seconds(heartBeatTimeout); - std::this_thread::sleep_for(duration); - } - - CoreLogger::info("heartbeat thread done"); - }; - - std::thread t2(heartbeat); - - std::string subscriptionPosition(position); - - conn.setEventCallback([this, - &conn, - &channel, - &filter, - &subscriptionPosition, - &throttled, - &receivedCount, - &receivedCountPerMinutes, - maxEventsPerMinute, - limitReceivedEvents, - &fatalCobraError, - &sentCount](const CobraEventPtr& event) { - if (event->type == ix::CobraEventType::Open) - { - CoreLogger::info("Subscriber connected"); - - for (auto&& it : event->headers) - { - CoreLogger::info(it.first + "::" + it.second); - } - } - else if (event->type == ix::CobraEventType::Closed) - { - CoreLogger::info("Subscriber closed: {}" + event->errMsg); - } - else if (event->type == ix::CobraEventType::Authenticated) - { - CoreLogger::info("Subscriber authenticated"); - CoreLogger::info("Subscribing to " + channel); - CoreLogger::info("Subscribing at position " + subscriptionPosition); - CoreLogger::info("Subscribing with filter " + filter); - conn.subscribe(channel, filter, subscriptionPosition, - [&sentCount, &receivedCountPerMinutes, - maxEventsPerMinute, limitReceivedEvents, - &throttled, &receivedCount, - &subscriptionPosition, &fatalCobraError, - this](const Json::Value& msg, const std::string& position) { - subscriptionPosition = position; - ++receivedCount; - - ++receivedCountPerMinutes; - if (limitReceivedEvents) - { - if (receivedCountPerMinutes > maxEventsPerMinute) - { - return; - } - } - - // If we cannot send to sentry fast enough, drop the message - if (throttled) - { - return; - } - - _onBotMessageCallback( - msg, position, throttled, - fatalCobraError, sentCount); - }); - } - else if (event->type == ix::CobraEventType::Subscribed) - { - CoreLogger::info("Subscriber: subscribed to channel " + event->subscriptionId); - } - else if (event->type == ix::CobraEventType::UnSubscribed) - { - CoreLogger::info("Subscriber: unsubscribed from channel " + event->subscriptionId); - } - else if (event->type == ix::CobraEventType::Error) - { - CoreLogger::error("Subscriber: error " + event->errMsg); - } - else if (event->type == ix::CobraEventType::Published) - { - CoreLogger::error("Published message hacked: " + std::to_string(event->msgId)); - } - else if (event->type == ix::CobraEventType::Pong) - { - CoreLogger::info("Received websocket pong: " + event->errMsg); - } - else if (event->type == ix::CobraEventType::HandshakeError) - { - CoreLogger::error("Subscriber: Handshake error: " + event->errMsg); - fatalCobraError = true; - } - else if (event->type == ix::CobraEventType::AuthenticationError) - { - CoreLogger::error("Subscriber: Authentication error: " + event->errMsg); - fatalCobraError = true; - } - else if (event->type == ix::CobraEventType::SubscriptionError) - { - CoreLogger::error("Subscriber: Subscription error: " + event->errMsg); - fatalCobraError = true; - } - }); - - // Run forever - if (runtime == -1) - { - while (true) - { - auto duration = std::chrono::seconds(1); - std::this_thread::sleep_for(duration); - - if (fatalCobraError) break; - } - } - // Run for a duration, used by unittesting now - else - { - for (int i = 0; i < runtime; ++i) - { - auto duration = std::chrono::seconds(1); - std::this_thread::sleep_for(duration); - - if (fatalCobraError) break; - } - } - - // - // Cleanup. - // join all the bg threads and stop them. - // - conn.disconnect(); - stop = true; - - // progress thread - t1.join(); - - // heartbeat thread - if (t2.joinable()) t2.join(); - - return fatalCobraError ? -1 : (int64_t) sentCount; - } - - void CobraBot::setOnBotMessageCallback(const OnBotMessageCallback& callback) - { - _onBotMessageCallback = callback; - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraBot.h b/ixbots/ixbots/IXCobraBot.h deleted file mode 100644 index e8c8b4f9..00000000 --- a/ixbots/ixbots/IXCobraBot.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * IXCobraBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - using OnBotMessageCallback = std::function&, - std::atomic&, - std::atomic&)>; - - class CobraBot - { - public: - CobraBot() = default; - - int64_t run(const CobraBotConfig& botConfig); - void setOnBotMessageCallback(const OnBotMessageCallback& callback); - - private: - OnBotMessageCallback _onBotMessageCallback; - }; -} // namespace ix diff --git a/ixbots/ixbots/IXCobraBotConfig.h b/ixbots/ixbots/IXCobraBotConfig.h deleted file mode 100644 index 15dab740..00000000 --- a/ixbots/ixbots/IXCobraBotConfig.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * IXCobraBotConfig.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include - -#ifdef max -#undef max -#endif - -namespace ix -{ - struct CobraBotConfig - { - CobraConfig cobraConfig; - std::string channel; - std::string filter; - std::string position = std::string("$"); - bool enableHeartbeat = true; - int heartBeatTimeout = 60; - int runtime = -1; - int maxEventsPerMinute = std::numeric_limits::max(); - bool limitReceivedEvents = false; - }; -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToSentryBot.cpp b/ixbots/ixbots/IXCobraToSentryBot.cpp deleted file mode 100644 index 906f612d..00000000 --- a/ixbots/ixbots/IXCobraToSentryBot.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * IXCobraToSentryBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToSentryBot.h" - -#include "IXCobraBot.h" -#include -#include - -#include -#include -#include - -namespace ix -{ - int64_t cobra_to_sentry_bot(const CobraBotConfig& config, - SentryClient& sentryClient, - bool verbose) - { - CobraBot bot; - bot.setOnBotMessageCallback([&sentryClient, &verbose](const Json::Value& msg, - const std::string& /*position*/, - std::atomic& throttled, - std::atomic& /*fatalCobraError*/, - std::atomic& sentCount) -> void { - sentryClient.send(msg, verbose, - [&sentCount, &throttled](const HttpResponsePtr& response) { - if (!response) - { - CoreLogger::warn("Null HTTP Response"); - return; - } - - if (response->statusCode == 200) - { - sentCount++; - } - else - { - CoreLogger::error("Error sending data to sentry: " + std::to_string(response->statusCode)); - CoreLogger::error("Response: " + response->payload); - - // Error 429 Too Many Requests - if (response->statusCode == 429) - { - auto retryAfter = response->headers["Retry-After"]; - std::stringstream ss; - ss << retryAfter; - int seconds; - ss >> seconds; - - if (!ss.eof() || ss.fail()) - { - seconds = 30; - CoreLogger::warn("Error parsing Retry-After header. " - "Using " + retryAfter + " for the sleep duration"); - } - - CoreLogger::warn("Error 429 - Too Many Requests. ws will sleep " - "and retry after " + retryAfter + " seconds"); - - throttled = true; - auto duration = std::chrono::seconds(seconds); - std::this_thread::sleep_for(duration); - throttled = false; - } - } - }); - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToSentryBot.h b/ixbots/ixbots/IXCobraToSentryBot.h deleted file mode 100644 index 7d9178b9..00000000 --- a/ixbots/ixbots/IXCobraToSentryBot.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * IXCobraToSentryBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_sentry_bot(const CobraBotConfig& config, - SentryClient& sentryClient, - bool verbose); -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStatsdBot.cpp b/ixbots/ixbots/IXCobraToStatsdBot.cpp deleted file mode 100644 index 138f2566..00000000 --- a/ixbots/ixbots/IXCobraToStatsdBot.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * IXCobraToStatsdBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToStatsdBot.h" - -#include "IXCobraBot.h" -#include "IXStatsdClient.h" -#include -#include -#include -#include -#include - -namespace ix -{ - // fields are command line argument that can be specified multiple times - std::vector parseFields(const std::string& fields) - { - std::vector tokens; - - // Split by \n - std::string token; - std::stringstream tokenStream(fields); - - while (std::getline(tokenStream, token)) - { - tokens.push_back(token); - } - - return tokens; - } - - // - // Extract an attribute from a Json Value. - // extractAttr("foo.bar", {"foo": {"bar": "baz"}}) => baz - // - Json::Value extractAttr(const std::string& attr, const Json::Value& jsonValue) - { - // Split by . - std::string token; - std::stringstream tokenStream(attr); - - Json::Value val(jsonValue); - - while (std::getline(tokenStream, token, '.')) - { - val = val[token]; - } - - return val; - } - - int64_t cobra_to_statsd_bot(const ix::CobraBotConfig& config, - StatsdClient& statsdClient, - const std::string& fields, - const std::string& gauge, - const std::string& timer, - bool verbose) - { - auto tokens = parseFields(fields); - - CobraBot bot; - bot.setOnBotMessageCallback( - [&statsdClient, &tokens, &gauge, &timer, &verbose](const Json::Value& msg, - const std::string& /*position*/, - std::atomic& /*throttled*/, - std::atomic& fatalCobraError, - std::atomic& sentCount) -> void { - std::string id; - for (auto&& attr : tokens) - { - id += "."; - auto val = extractAttr(attr, msg); - id += val.asString(); - } - - if (gauge.empty() && timer.empty()) - { - statsdClient.count(id, 1); - } - else - { - std::string attrName = (!gauge.empty()) ? gauge : timer; - auto val = extractAttr(attrName, msg); - size_t x; - - if (val.isInt()) - { - x = (size_t) val.asInt(); - } - else if (val.isInt64()) - { - x = (size_t) val.asInt64(); - } - else if (val.isUInt()) - { - x = (size_t) val.asUInt(); - } - else if (val.isUInt64()) - { - x = (size_t) val.asUInt64(); - } - else if (val.isDouble()) - { - x = (size_t) val.asUInt64(); - } - else - { - CoreLogger::error("Gauge " + gauge + " is not a numeric type"); - fatalCobraError = true; - return; - } - - if (verbose) - { - CoreLogger::info(id + " - " + attrName + " -> " + std::to_string(x)); - } - - if (!gauge.empty()) - { - statsdClient.gauge(id, x); - } - else - { - statsdClient.timing(id, x); - } - } - - sentCount++; - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStatsdBot.h b/ixbots/ixbots/IXCobraToStatsdBot.h deleted file mode 100644 index 50025957..00000000 --- a/ixbots/ixbots/IXCobraToStatsdBot.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * IXCobraToStatsdBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_statsd_bot(const ix::CobraBotConfig& config, - StatsdClient& statsdClient, - const std::string& fields, - const std::string& gauge, - const std::string& timer, - bool verbose); -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStdoutBot.cpp b/ixbots/ixbots/IXCobraToStdoutBot.cpp deleted file mode 100644 index df266f7b..00000000 --- a/ixbots/ixbots/IXCobraToStdoutBot.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * IXCobraToStdoutBot.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXCobraToStdoutBot.h" - -#include "IXCobraBot.h" -#include -#include -#include - -namespace ix -{ - using StreamWriterPtr = std::unique_ptr; - - StreamWriterPtr makeStreamWriter() - { - Json::StreamWriterBuilder builder; - builder["commentStyle"] = "None"; - builder["indentation"] = ""; // will make the JSON object compact - std::unique_ptr jsonWriter(builder.newStreamWriter()); - return jsonWriter; - } - - std::string timeSinceEpoch() - { - std::chrono::system_clock::time_point tp = std::chrono::system_clock::now(); - std::chrono::system_clock::duration dtn = tp.time_since_epoch(); - - std::stringstream ss; - ss << dtn.count() * std::chrono::system_clock::period::num / - std::chrono::system_clock::period::den; - return ss.str(); - } - - void writeToStdout(bool fluentd, - const StreamWriterPtr& jsonWriter, - const Json::Value& msg, - const std::string& position) - { - Json::Value enveloppe; - if (fluentd) - { - enveloppe["producer"] = "cobra"; - enveloppe["consumer"] = "fluentd"; - - Json::Value nestedMessage(msg); - nestedMessage["position"] = position; - nestedMessage["created_at"] = timeSinceEpoch(); - enveloppe["message"] = nestedMessage; - - jsonWriter->write(enveloppe, &std::cout); - std::cout << std::endl; // add lf and flush - } - else - { - enveloppe = msg; - std::cout << position << " "; - jsonWriter->write(enveloppe, &std::cout); - std::cout << std::endl; - } - } - - int64_t cobra_to_stdout_bot(const ix::CobraBotConfig& config, - bool fluentd, - bool quiet) - { - CobraBot bot; - auto jsonWriter = makeStreamWriter(); - - bot.setOnBotMessageCallback( - [&fluentd, &quiet, &jsonWriter](const Json::Value& msg, - const std::string& position, - std::atomic& /*throttled*/, - std::atomic& /*fatalCobraError*/, - std::atomic& sentCount) -> void { - if (!quiet) - { - writeToStdout(fluentd, jsonWriter, msg, position); - } - sentCount++; - }); - - return bot.run(config); - } -} // namespace ix diff --git a/ixbots/ixbots/IXCobraToStdoutBot.h b/ixbots/ixbots/IXCobraToStdoutBot.h deleted file mode 100644 index a99d83b0..00000000 --- a/ixbots/ixbots/IXCobraToStdoutBot.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * IXCobraToStdoutBot.h - * Author: Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ -#pragma once - -#include -#include "IXCobraBotConfig.h" -#include -#include - -namespace ix -{ - int64_t cobra_to_stdout_bot(const ix::CobraBotConfig& config, - bool fluentd, - bool quiet); -} // namespace ix diff --git a/ixbots/ixbots/IXStatsdClient.cpp b/ixbots/ixbots/IXStatsdClient.cpp deleted file mode 100644 index 762624fb..00000000 --- a/ixbots/ixbots/IXStatsdClient.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2014, Rex - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * * Neither the name of the {organization} nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * IXStatsdClient.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -// Adapted from statsd-client-cpp -// test with netcat as a server: `nc -ul 8125` - -#include "IXStatsdClient.h" - -#include -#include -#include -#include -#include - -namespace ix -{ - StatsdClient::StatsdClient(const std::string& host, int port, const std::string& prefix) - : _host(host) - , _port(port) - , _prefix(prefix) - , _stop(false) - { - _thread = std::thread([this] { - while (!_stop) - { - flushQueue(); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }); - } - - StatsdClient::~StatsdClient() - { - _stop = true; - if (_thread.joinable()) _thread.join(); - - _socket.close(); - } - - bool StatsdClient::init(std::string& errMsg) - { - return _socket.init(_host, _port, errMsg); - } - - /* will change the original string */ - void StatsdClient::cleanup(std::string& key) - { - size_t pos = key.find_first_of(":|@"); - while (pos != std::string::npos) - { - key[pos] = '_'; - pos = key.find_first_of(":|@"); - } - } - - int StatsdClient::dec(const std::string& key) - { - return count(key, -1); - } - - int StatsdClient::inc(const std::string& key) - { - return count(key, 1); - } - - int StatsdClient::count(const std::string& key, size_t value) - { - return send(key, value, "c"); - } - - int StatsdClient::gauge(const std::string& key, size_t value) - { - return send(key, value, "g"); - } - - int StatsdClient::timing(const std::string& key, size_t ms) - { - return send(key, ms, "ms"); - } - - int StatsdClient::send(std::string key, size_t value, const std::string& type) - { - cleanup(key); - - char buf[256]; - snprintf( - buf, sizeof(buf), "%s%s:%zd|%s\n", _prefix.c_str(), key.c_str(), value, type.c_str()); - - enqueue(buf); - return 0; - } - - void StatsdClient::enqueue(const std::string& message) - { - std::lock_guard lock(_mutex); - _queue.push_back(message); - } - - void StatsdClient::flushQueue() - { - std::lock_guard lock(_mutex); - - while (!_queue.empty()) - { - auto message = _queue.front(); - auto ret = _socket.sendto(message); - if (ret != 0) - { - std::cerr << "error: " << strerror(UdpSocket::getErrno()) << std::endl; - } - _queue.pop_front(); - } - } -} // end namespace ix diff --git a/ixbots/ixbots/IXStatsdClient.h b/ixbots/ixbots/IXStatsdClient.h deleted file mode 100644 index 9e61bd18..00000000 --- a/ixbots/ixbots/IXStatsdClient.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * IXStatsdClient.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace ix -{ - class StatsdClient - { - public: - StatsdClient(const std::string& host = "127.0.0.1", - int port = 8125, - const std::string& prefix = ""); - ~StatsdClient(); - - bool init(std::string& errMsg); - int inc(const std::string& key); - int dec(const std::string& key); - int count(const std::string& key, size_t value); - int gauge(const std::string& key, size_t value); - int timing(const std::string& key, size_t ms); - - private: - void enqueue(const std::string& message); - - /* (Low Level Api) manually send a message - * type = "c", "g" or "ms" - */ - int send(std::string key, size_t value, const std::string& type); - - void cleanup(std::string& key); - void flushQueue(); - - UdpSocket _socket; - - std::string _host; - int _port; - std::string _prefix; - - std::atomic _stop; - std::thread _thread; - std::mutex _mutex; // for the queue - - std::deque _queue; - }; - -} // end namespace ix diff --git a/ixcobra/CMakeLists.txt b/ixcobra/CMakeLists.txt deleted file mode 100644 index 8a03d1dc..00000000 --- a/ixcobra/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXCOBRA_SOURCES - ixcobra/IXCobraConnection.cpp - ixcobra/IXCobraMetricsThreadedPublisher.cpp - ixcobra/IXCobraMetricsPublisher.cpp -) - -set (IXCOBRA_HEADERS - ixcobra/IXCobraConnection.h - ixcobra/IXCobraMetricsThreadedPublisher.h - ixcobra/IXCobraMetricsPublisher.h - ixcobra/IXCobraConfig.h - ixcobra/IXCobraEventType.h -) - -add_library(ixcobra STATIC - ${IXCOBRA_SOURCES} - ${IXCOBRA_HEADERS} -) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) -endif() - -set(IXCOBRA_INCLUDE_DIRS - . - .. - ../ixcore - ../ixcrypto - ${JSONCPP_INCLUDE_DIRS}) - -target_include_directories( ixcobra PUBLIC ${IXCOBRA_INCLUDE_DIRS} ) diff --git a/ixcobra/ixcobra/IXCobraConfig.h b/ixcobra/ixcobra/IXCobraConfig.h deleted file mode 100644 index 26b85e41..00000000 --- a/ixcobra/ixcobra/IXCobraConfig.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * IXCobraConfig.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include - -namespace ix -{ - struct CobraConfig - { - std::string appkey; - std::string endpoint; - std::string rolename; - std::string rolesecret; - WebSocketPerMessageDeflateOptions webSocketPerMessageDeflateOptions; - SocketTLSOptions socketTLSOptions; - - CobraConfig(const std::string& a = std::string(), - const std::string& e = std::string(), - const std::string& r = std::string(), - const std::string& s = std::string()) - : appkey(a) - , endpoint(e) - , rolename(r) - , rolesecret(s) - { - ; - } - }; -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraConnection.cpp b/ixcobra/ixcobra/IXCobraConnection.cpp deleted file mode 100644 index 2b8240e3..00000000 --- a/ixcobra/ixcobra/IXCobraConnection.cpp +++ /dev/null @@ -1,676 +0,0 @@ -/* - * IXCobraConnection.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017-2018 Machine Zone. All rights reserved. - */ - -#include "IXCobraConnection.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace ix -{ - TrafficTrackerCallback CobraConnection::_trafficTrackerCallback = nullptr; - PublishTrackerCallback CobraConnection::_publishTrackerCallback = nullptr; - constexpr size_t CobraConnection::kQueueMaxSize; - constexpr CobraConnection::MsgId CobraConnection::kInvalidMsgId; - constexpr int CobraConnection::kPingIntervalSecs; - - CobraConnection::CobraConnection() - : _webSocket(new WebSocket()) - , _publishMode(CobraConnection_PublishMode_Immediate) - , _authenticated(false) - , _eventCallback(nullptr) - , _id(1) - { - _pdu["action"] = "rtm/publish"; - - _webSocket->addSubProtocol("json"); - initWebSocketOnMessageCallback(); - } - - CobraConnection::~CobraConnection() - { - disconnect(); - setEventCallback(nullptr); - } - - void CobraConnection::setTrafficTrackerCallback(const TrafficTrackerCallback& callback) - { - _trafficTrackerCallback = callback; - } - - void CobraConnection::resetTrafficTrackerCallback() - { - setTrafficTrackerCallback(nullptr); - } - - void CobraConnection::invokeTrafficTrackerCallback(size_t size, bool incoming) - { - if (_trafficTrackerCallback) - { - _trafficTrackerCallback(size, incoming); - } - } - - void CobraConnection::setPublishTrackerCallback(const PublishTrackerCallback& callback) - { - _publishTrackerCallback = callback; - } - - void CobraConnection::resetPublishTrackerCallback() - { - setPublishTrackerCallback(nullptr); - } - - void CobraConnection::invokePublishTrackerCallback(bool sent, bool acked) - { - if (_publishTrackerCallback) - { - _publishTrackerCallback(sent, acked); - } - } - - void CobraConnection::setEventCallback(const EventCallback& eventCallback) - { - std::lock_guard lock(_eventCallbackMutex); - _eventCallback = eventCallback; - } - - void CobraConnection::invokeEventCallback(ix::CobraEventType eventType, - const std::string& errorMsg, - const WebSocketHttpHeaders& headers, - const std::string& subscriptionId, - CobraConnection::MsgId msgId) - { - std::lock_guard lock(_eventCallbackMutex); - if (_eventCallback) - { - _eventCallback( - std::make_unique(eventType, errorMsg, headers, subscriptionId, msgId)); - } - } - - void CobraConnection::invokeErrorCallback(const std::string& errorMsg, - const std::string& serializedPdu) - { - std::stringstream ss; - ss << errorMsg << " : received pdu => " << serializedPdu; - invokeEventCallback(ix::CobraEventType::Error, ss.str()); - } - - void CobraConnection::disconnect() - { - _authenticated = false; - _webSocket->stop(); - } - - void CobraConnection::initWebSocketOnMessageCallback() - { - _webSocket->setOnMessageCallback([this](const ix::WebSocketMessagePtr& msg) { - CobraConnection::invokeTrafficTrackerCallback(msg->wireSize, true); - - std::stringstream ss; - if (msg->type == ix::WebSocketMessageType::Open) - { - invokeEventCallback(ix::CobraEventType::Open, std::string(), msg->openInfo.headers); - sendHandshakeMessage(); - } - else if (msg->type == ix::WebSocketMessageType::Close) - { - _authenticated = false; - - std::stringstream ss; - ss << "Close code " << msg->closeInfo.code; - ss << " reason " << msg->closeInfo.reason; - invokeEventCallback(ix::CobraEventType::Closed, ss.str()); - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - Json::Value data; - Json::Reader reader; - if (!reader.parse(msg->str, data)) - { - invokeErrorCallback("Invalid json", msg->str); - return; - } - - if (!data.isMember("action")) - { - invokeErrorCallback("Missing action", msg->str); - return; - } - - auto action = data["action"].asString(); - - if (action == "auth/handshake/ok") - { - if (!handleHandshakeResponse(data)) - { - invokeErrorCallback("Error extracting nonce from handshake response", - msg->str); - } - } - else if (action == "auth/handshake/error") - { - invokeEventCallback(ix::CobraEventType::HandshakeError, msg->str); - } - else if (action == "auth/authenticate/ok") - { - _authenticated = true; - invokeEventCallback(ix::CobraEventType::Authenticated); - flushQueue(); - } - else if (action == "auth/authenticate/error") - { - invokeEventCallback(ix::CobraEventType::AuthenticationError, msg->str); - } - else if (action == "rtm/subscription/data") - { - handleSubscriptionData(data); - } - else if (action == "rtm/subscribe/ok") - { - if (!handleSubscriptionResponse(data)) - { - invokeErrorCallback("Error processing subscribe response", msg->str); - } - } - else if (action == "rtm/subscribe/error") - { - invokeEventCallback(ix::CobraEventType::SubscriptionError, msg->str); - } - else if (action == "rtm/unsubscribe/ok") - { - if (!handleUnsubscriptionResponse(data)) - { - invokeErrorCallback("Error processing unsubscribe response", msg->str); - } - } - else if (action == "rtm/unsubscribe/error") - { - invokeErrorCallback("Unsubscription error", msg->str); - } - else if (action == "rtm/publish/ok") - { - if (!handlePublishResponse(data)) - { - invokeErrorCallback("Error processing publish response", msg->str); - } - } - else if (action == "rtm/publish/error") - { - invokeErrorCallback("Publish error", msg->str); - } - else - { - invokeErrorCallback("Un-handled message type", msg->str); - } - } - else if (msg->type == ix::WebSocketMessageType::Error) - { - std::stringstream ss; - ss << "Connection error: " << msg->errorInfo.reason << std::endl; - ss << "#retries: " << msg->errorInfo.retries << std::endl; - ss << "Wait time(ms): " << msg->errorInfo.wait_time << std::endl; - ss << "HTTP Status: " << msg->errorInfo.http_status << std::endl; - invokeErrorCallback(ss.str(), std::string()); - } - else if (msg->type == ix::WebSocketMessageType::Pong) - { - invokeEventCallback(ix::CobraEventType::Pong, msg->str); - } - }); - } - - void CobraConnection::setPublishMode(CobraConnectionPublishMode publishMode) - { - _publishMode = publishMode; - } - - CobraConnectionPublishMode CobraConnection::getPublishMode() - { - return _publishMode; - } - - void CobraConnection::configure( - const std::string& appkey, - const std::string& endpoint, - const std::string& rolename, - const std::string& rolesecret, - const WebSocketPerMessageDeflateOptions& webSocketPerMessageDeflateOptions, - const SocketTLSOptions& socketTLSOptions) - { - _roleName = rolename; - _roleSecret = rolesecret; - - std::stringstream ss; - ss << endpoint; - ss << "/v2?appkey="; - ss << appkey; - - std::string url = ss.str(); - _webSocket->setUrl(url); - _webSocket->setPerMessageDeflateOptions(webSocketPerMessageDeflateOptions); - _webSocket->setTLSOptions(socketTLSOptions); - - // Send a websocket ping every N seconds (N = 30) now - // This should keep the connection open and prevent some load balancers such as - // the Amazon one from shutting it down - _webSocket->setPingInterval(kPingIntervalSecs); - } - - void CobraConnection::configure(const ix::CobraConfig& config) - { - configure(config.appkey, - config.endpoint, - config.rolename, - config.rolesecret, - config.webSocketPerMessageDeflateOptions, - config.socketTLSOptions); - } - - // - // Handshake message schema. - // - // handshake = { - // "action": "auth/handshake", - // "body": { - // "data": { - // "role": role - // }, - // "method": "role_secret" - // }, - // } - // - // - bool CobraConnection::sendHandshakeMessage() - { - Json::Value data; - data["role"] = _roleName; - - Json::Value body; - body["data"] = data; - body["method"] = "role_secret"; - - Json::Value pdu; - pdu["action"] = "auth/handshake"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - std::string serializedJson = serializeJson(pdu); - CobraConnection::invokeTrafficTrackerCallback(serializedJson.size(), false); - - return _webSocket->send(serializedJson).success; - } - - // - // Extract the nonce from the handshake response - // use it to compute a hash during authentication - // - // { - // "action": "auth/handshake/ok", - // "body": { - // "data": { - // "nonce": "MTI0Njg4NTAyMjYxMzgxMzgzMg==", - // "version": "0.0.24" - // } - // } - // } - // - bool CobraConnection::handleHandshakeResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - if (!body.isMember("data")) return false; - Json::Value data = body["data"]; - - if (!data.isMember("nonce")) return false; - Json::Value nonce = data["nonce"]; - - if (!nonce.isString()) return false; - - return sendAuthMessage(nonce.asString()); - } - - // - // Authenticate message schema. - // - // challenge = { - // "action": "auth/authenticate", - // "body": { - // "method": "role_secret", - // "credentials": { - // "hash": computeHash(secret, nonce) - // } - // }, - // } - // - bool CobraConnection::sendAuthMessage(const std::string& nonce) - { - Json::Value credentials; - credentials["hash"] = hmac(nonce, _roleSecret); - - Json::Value body; - body["credentials"] = credentials; - body["method"] = "role_secret"; - - Json::Value pdu; - pdu["action"] = "auth/authenticate"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - std::string serializedJson = serializeJson(pdu); - CobraConnection::invokeTrafficTrackerCallback(serializedJson.size(), false); - - return _webSocket->send(serializedJson).success; - } - - bool CobraConnection::handleSubscriptionResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - if (!body.isMember("subscription_id")) return false; - Json::Value subscriptionId = body["subscription_id"]; - - if (!subscriptionId.isString()) return false; - - invokeEventCallback(ix::CobraEventType::Subscribed, - std::string(), - WebSocketHttpHeaders(), - subscriptionId.asString()); - return true; - } - - bool CobraConnection::handleUnsubscriptionResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - if (!body.isMember("subscription_id")) return false; - Json::Value subscriptionId = body["subscription_id"]; - - if (!subscriptionId.isString()) return false; - - invokeEventCallback(ix::CobraEventType::UnSubscribed, - std::string(), - WebSocketHttpHeaders(), - subscriptionId.asString()); - return true; - } - - bool CobraConnection::handleSubscriptionData(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("body")) return false; - Json::Value body = pdu["body"]; - - // Identify subscription_id, so that we can find - // which callback to execute - if (!body.isMember("subscription_id")) return false; - Json::Value subscriptionId = body["subscription_id"]; - - std::lock_guard lock(_cbsMutex); - auto cb = _cbs.find(subscriptionId.asString()); - if (cb == _cbs.end()) return false; // cannot find callback - - // Extract messages now - if (!body.isMember("messages")) return false; - Json::Value messages = body["messages"]; - - if (!body.isMember("position")) return false; - std::string position = body["position"].asString(); - - for (auto&& msg : messages) - { - cb->second(msg, position); - } - - return true; - } - - bool CobraConnection::handlePublishResponse(const Json::Value& pdu) - { - if (!pdu.isObject()) return false; - - if (!pdu.isMember("id")) return false; - Json::Value id = pdu["id"]; - - if (!id.isUInt64()) return false; - - uint64_t msgId = id.asUInt64(); - - invokeEventCallback(ix::CobraEventType::Published, - std::string(), - WebSocketHttpHeaders(), - std::string(), - msgId); - - invokePublishTrackerCallback(false, true); - - return true; - } - - bool CobraConnection::connect() - { - _webSocket->start(); - return true; - } - - bool CobraConnection::isConnected() const - { - return _webSocket->getReadyState() == ix::ReadyState::Open; - } - - bool CobraConnection::isAuthenticated() const - { - return isConnected() && _authenticated; - } - - std::string CobraConnection::serializeJson(const Json::Value& value) - { - std::lock_guard lock(_jsonWriterMutex); - return _jsonWriter.write(value); - } - - std::pair CobraConnection::prePublish( - const Json::Value& channels, const Json::Value& msg, bool addToQueue) - { - std::lock_guard lock(_prePublishMutex); - - invokePublishTrackerCallback(true, false); - - CobraConnection::MsgId msgId = _id; - - _body["channels"] = channels; - _body["message"] = msg; - _pdu["body"] = _body; - _pdu["id"] = Json::UInt64(_id++); - - std::string serializedJson = serializeJson(_pdu); - - if (addToQueue) - { - enqueue(serializedJson); - } - - return std::make_pair(msgId, serializedJson); - } - - bool CobraConnection::publishNext() - { - std::lock_guard lock(_queueMutex); - - if (_messageQueue.empty()) return true; - - auto&& msg = _messageQueue.back(); - if (!_authenticated || !publishMessage(msg)) - { - return false; - } - _messageQueue.pop_back(); - return true; - } - - // - // publish is not thread safe as we are trying to reuse some Json objects. - // - CobraConnection::MsgId CobraConnection::publish(const Json::Value& channels, - const Json::Value& msg) - { - auto p = prePublish(channels, msg, false); - auto msgId = p.first; - auto serializedJson = p.second; - - // - // 1. When we use batch mode, we just enqueue and will do the flush explicitely - // 2. When we aren't authenticated yet to the cobra server, we need to enqueue - // and retry later - // 3. If the network connection was droped (WebSocket::send will return false), - // it means the message won't be sent so we need to enqueue as well. - // - // The order of the conditionals is important. - // - if (_publishMode == CobraConnection_PublishMode_Batch || !_authenticated || - !publishMessage(serializedJson)) - { - enqueue(serializedJson); - } - - return msgId; - } - - void CobraConnection::subscribe(const std::string& channel, - const std::string& filter, - const std::string& position, - SubscriptionCallback cb) - { - // Create and send a subscribe pdu - Json::Value body; - body["channel"] = channel; - - if (!filter.empty()) - { - body["filter"] = filter; - } - - if (!position.empty()) - { - body["position"] = position; - } - - Json::Value pdu; - pdu["action"] = "rtm/subscribe"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - _webSocket->send(pdu.toStyledString()); - - // Set the callback - std::lock_guard lock(_cbsMutex); - _cbs[channel] = cb; - } - - void CobraConnection::unsubscribe(const std::string& channel) - { - { - std::lock_guard lock(_cbsMutex); - auto cb = _cbs.find(channel); - if (cb == _cbs.end()) return; - - _cbs.erase(cb); - } - - // Create and send an unsubscribe pdu - Json::Value body; - body["subscription_id"] = channel; - - Json::Value pdu; - pdu["action"] = "rtm/unsubscribe"; - pdu["body"] = body; - pdu["id"] = Json::UInt64(_id++); - - _webSocket->send(pdu.toStyledString()); - } - - // - // Enqueue strategy drops old messages when we are at full capacity - // - // If we want to keep only 3 items max in the queue: - // - // enqueue(A) -> [A] - // enqueue(B) -> [B, A] - // enqueue(C) -> [C, B, A] - // enqueue(D) -> [D, C, B] -- now we drop A, the oldest message, - // -- and keep the 'fresh ones' - // - void CobraConnection::enqueue(const std::string& msg) - { - std::lock_guard lock(_queueMutex); - - if (_messageQueue.size() == CobraConnection::kQueueMaxSize) - { - _messageQueue.pop_back(); - } - _messageQueue.push_front(msg); - } - - // - // We process messages back (oldest) to front (newest) to respect ordering - // when sending them. If we fail to send something, we put it back in the queue - // at the end we picked it up originally (at the end). - // - bool CobraConnection::flushQueue() - { - while (!isQueueEmpty()) - { - bool ok = publishNext(); - if (!ok) return false; - } - - return true; - } - - bool CobraConnection::isQueueEmpty() - { - std::lock_guard lock(_queueMutex); - return _messageQueue.empty(); - } - - bool CobraConnection::publishMessage(const std::string& serializedJson) - { - auto webSocketSendInfo = _webSocket->send(serializedJson); - CobraConnection::invokeTrafficTrackerCallback(webSocketSendInfo.wireSize, false); - return webSocketSendInfo.success; - } - - void CobraConnection::suspend() - { - disconnect(); - } - - void CobraConnection::resume() - { - connect(); - } - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraConnection.h b/ixcobra/ixcobra/IXCobraConnection.h deleted file mode 100644 index 50dbb08e..00000000 --- a/ixcobra/ixcobra/IXCobraConnection.h +++ /dev/null @@ -1,217 +0,0 @@ -/* - * IXCobraConnection.h - * Author: Benjamin Sergeant - * Copyright (c) 2017-2018 Machine Zone. All rights reserved. - */ - -#pragma once - -#include "IXCobraConfig.h" -#include "IXCobraEvent.h" -#include "IXCobraEventType.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef max -#undef max -#endif - -namespace ix -{ - class WebSocket; - struct SocketTLSOptions; - - enum CobraConnectionPublishMode - { - CobraConnection_PublishMode_Immediate = 0, - CobraConnection_PublishMode_Batch = 1 - }; - - using SubscriptionCallback = std::function; - using EventCallback = std::function; - - using TrafficTrackerCallback = std::function; - using PublishTrackerCallback = std::function; - - class CobraConnection - { - public: - using MsgId = uint64_t; - - CobraConnection(); - ~CobraConnection(); - - /// Configuration / set keys, etc... - /// All input data but the channel name is encrypted with rc4 - void configure(const std::string& appkey, - const std::string& endpoint, - const std::string& rolename, - const std::string& rolesecret, - const WebSocketPerMessageDeflateOptions& webSocketPerMessageDeflateOptions, - const SocketTLSOptions& socketTLSOptions); - - void configure(const ix::CobraConfig& config); - - /// Set the traffic tracker callback - static void setTrafficTrackerCallback(const TrafficTrackerCallback& callback); - - /// Reset the traffic tracker callback to an no-op one. - static void resetTrafficTrackerCallback(); - - /// Set the publish tracker callback - static void setPublishTrackerCallback(const PublishTrackerCallback& callback); - - /// Reset the publish tracker callback to an no-op one. - static void resetPublishTrackerCallback(); - - /// Set the closed callback - void setEventCallback(const EventCallback& eventCallback); - - /// Start the worker thread, used for background publishing - void start(); - - /// Publish a message to a channel - /// - /// No-op if the connection is not established - MsgId publish(const Json::Value& channels, const Json::Value& msg); - - // Subscribe to a channel, and execute a callback when an incoming - // message arrives. - void subscribe(const std::string& channel, - const std::string& filter = std::string(), - const std::string& position = std::string(), - SubscriptionCallback cb = nullptr); - - /// Unsubscribe from a channel - void unsubscribe(const std::string& channel); - - /// Close the connection - void disconnect(); - - /// Connect to Cobra and authenticate the connection - bool connect(); - - /// Returns true only if we're connected - bool isConnected() const; - - /// Returns true only if we're authenticated - bool isAuthenticated() const; - - /// Flush the publish queue - bool flushQueue(); - - /// Set the publish mode - void setPublishMode(CobraConnectionPublishMode publishMode); - - /// Query the publish mode - CobraConnectionPublishMode getPublishMode(); - - /// Lifecycle management. Free resources when backgrounding - void suspend(); - void resume(); - - /// Prepare a message for transmission - /// (update the pdu, compute a msgId, serialize json to a string) - std::pair prePublish(const Json::Value& channels, - const Json::Value& msg, - bool addToQueue); - - /// Attempt to send next message from the internal queue - bool publishNext(); - - // An invalid message id, signifying an error. - static constexpr MsgId kInvalidMsgId = 0; - - private: - bool sendHandshakeMessage(); - bool handleHandshakeResponse(const Json::Value& data); - bool sendAuthMessage(const std::string& nonce); - bool handleSubscriptionData(const Json::Value& pdu); - bool handleSubscriptionResponse(const Json::Value& pdu); - bool handleUnsubscriptionResponse(const Json::Value& pdu); - bool handlePublishResponse(const Json::Value& pdu); - - void initWebSocketOnMessageCallback(); - - bool publishMessage(const std::string& serializedJson); - void enqueue(const std::string& msg); - std::string serializeJson(const Json::Value& pdu); - - /// Invoke the traffic tracker callback - static void invokeTrafficTrackerCallback(size_t size, bool incoming); - - /// Invoke the publish tracker callback - static void invokePublishTrackerCallback(bool sent, bool acked); - - /// Invoke event callbacks - void invokeEventCallback(CobraEventType eventType, - const std::string& errorMsg = std::string(), - const WebSocketHttpHeaders& headers = WebSocketHttpHeaders(), - const std::string& subscriptionId = std::string(), - uint64_t msgId = std::numeric_limits::max()); - void invokeErrorCallback(const std::string& errorMsg, const std::string& serializedPdu); - - /// Tells whether the internal queue is empty or not - bool isQueueEmpty(); - - /// - /// Member variables - /// - std::unique_ptr _webSocket; - - /// Configuration data - std::string _roleName; - std::string _roleSecret; - std::atomic _publishMode; - - // Can be set on control+background thread, protecting with an atomic - std::atomic _authenticated; - - // Keep some objects around - Json::Value _body; - Json::Value _pdu; - Json::FastWriter _jsonWriter; - mutable std::mutex _jsonWriterMutex; - std::mutex _prePublishMutex; - - /// Traffic tracker callback - static TrafficTrackerCallback _trafficTrackerCallback; - - /// Publish tracker callback - static PublishTrackerCallback _publishTrackerCallback; - - /// Cobra events callbacks - EventCallback _eventCallback; - mutable std::mutex _eventCallbackMutex; - - /// Subscription callbacks, only one per channel - std::unordered_map _cbs; - mutable std::mutex _cbsMutex; - - // Message Queue can be touched on control+background thread, - // protecting with a mutex. - // - // Message queue is used when there are problems sending messages so - // that sending can be retried later. - std::deque _messageQueue; - mutable std::mutex _queueMutex; - - // Cap the queue size (100 elems so far -> ~100k) - static constexpr size_t kQueueMaxSize = 256; - - // Each pdu sent should have an incremental unique id - std::atomic _id; - - // Frequency at which we send a websocket ping to the backing cobra connection - static constexpr int kPingIntervalSecs = 30; - }; - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraEvent.h b/ixcobra/ixcobra/IXCobraEvent.h deleted file mode 100644 index 9227d743..00000000 --- a/ixcobra/ixcobra/IXCobraEvent.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * IXCobraEvent.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include "IXCobraEventType.h" -#include -#include -#include -#include - -namespace ix -{ - struct CobraEvent - { - ix::CobraEventType type; - const std::string& errMsg; - const ix::WebSocketHttpHeaders& headers; - const std::string& subscriptionId; - uint64_t msgId; // CobraConnection::MsgId - - CobraEvent(ix::CobraEventType t, - const std::string& e, - const ix::WebSocketHttpHeaders& h, - const std::string& s, - uint64_t m) - : type(t) - , errMsg(e) - , headers(h) - , subscriptionId(s) - , msgId(m) - { - ; - } - }; - - using CobraEventPtr = std::unique_ptr; -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraEventType.h b/ixcobra/ixcobra/IXCobraEventType.h deleted file mode 100644 index 1ccfbeb0..00000000 --- a/ixcobra/ixcobra/IXCobraEventType.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * IXCobraEventType.h - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -namespace ix -{ - enum class CobraEventType - { - Authenticated = 0, - Error = 1, - Open = 2, - Closed = 3, - Subscribed = 4, - UnSubscribed = 5, - Published = 6, - Pong = 7, - HandshakeError = 8, - AuthenticationError = 9, - SubscriptionError = 10 - }; -} diff --git a/ixcobra/ixcobra/IXCobraMetricsPublisher.cpp b/ixcobra/ixcobra/IXCobraMetricsPublisher.cpp deleted file mode 100644 index 81429af9..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsPublisher.cpp +++ /dev/null @@ -1,232 +0,0 @@ -/* - * IXCobraMetricsPublisher.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#include "IXCobraMetricsPublisher.h" - -#include -#include -#include - - -namespace ix -{ - const int CobraMetricsPublisher::kVersion = 1; - const std::string CobraMetricsPublisher::kSetRateControlId = "sms_set_rate_control_id"; - const std::string CobraMetricsPublisher::kSetBlacklistId = "sms_set_blacklist_id"; - - CobraMetricsPublisher::CobraMetricsPublisher() - : _enabled(true) - { - } - - CobraMetricsPublisher::~CobraMetricsPublisher() - { - ; - } - - void CobraMetricsPublisher::configure(const CobraConfig& config, const std::string& channel) - { - // Configure the satori connection and start its publish background thread - _cobra_metrics_theaded_publisher.configure(config, channel); - _cobra_metrics_theaded_publisher.start(); - } - - Json::Value& CobraMetricsPublisher::getGenericAttributes() - { - std::lock_guard lock(_device_mutex); - return _device; - } - - void CobraMetricsPublisher::setGenericAttributes(const std::string& attrName, - const Json::Value& value) - { - std::lock_guard lock(_device_mutex); - _device[attrName] = value; - } - - void CobraMetricsPublisher::enable(bool enabled) - { - _enabled = enabled; - } - - void CobraMetricsPublisher::setBlacklist(const std::vector& blacklist) - { - _blacklist = blacklist; - std::sort(_blacklist.begin(), _blacklist.end()); - - // publish our blacklist - Json::Value data; - Json::Value metrics; - for (auto&& metric : blacklist) - { - metrics.append(metric); - } - data["blacklist"] = metrics; - push(kSetBlacklistId, data); - } - - bool CobraMetricsPublisher::isMetricBlacklisted(const std::string& id) const - { - return std::binary_search(_blacklist.begin(), _blacklist.end(), id); - } - - void CobraMetricsPublisher::setRateControl( - const std::unordered_map& rate_control) - { - for (auto&& it : rate_control) - { - if (it.second >= 0) - { - _rate_control[it.first] = it.second; - } - } - - // publish our rate_control - Json::Value data; - Json::Value metrics; - for (auto&& it : _rate_control) - { - metrics[it.first] = it.second; - } - data["rate_control"] = metrics; - push(kSetRateControlId, data); - } - - bool CobraMetricsPublisher::isAboveMaxUpdateRate(const std::string& id) const - { - // Is this metrics rate controlled ? - auto rate_control_it = _rate_control.find(id); - if (rate_control_it == _rate_control.end()) return false; - - // Was this metrics already sent ? - std::lock_guard lock(_last_update_mutex); - auto last_update = _last_update.find(id); - if (last_update == _last_update.end()) return false; - - auto timeDeltaFromLastSend = std::chrono::steady_clock::now() - last_update->second; - - return timeDeltaFromLastSend < std::chrono::seconds(rate_control_it->second); - } - - void CobraMetricsPublisher::setLastUpdate(const std::string& id) - { - std::lock_guard lock(_last_update_mutex); - _last_update[id] = std::chrono::steady_clock::now(); - } - - uint64_t CobraMetricsPublisher::getMillisecondsSinceEpoch() const - { - auto now = std::chrono::system_clock::now(); - auto ms = - std::chrono::duration_cast(now.time_since_epoch()).count(); - - return ms; - } - - CobraConnection::MsgId CobraMetricsPublisher::push(const std::string& id, - const std::string& data, - bool shouldPushTest) - { - if (!_enabled) return CobraConnection::kInvalidMsgId; - - Json::Value root; - Json::Reader reader; - if (!reader.parse(data, root)) return CobraConnection::kInvalidMsgId; - - return push(id, root, shouldPushTest); - } - - CobraConnection::MsgId CobraMetricsPublisher::push(const std::string& id, - const CobraMetricsPublisher::Message& data) - { - if (!_enabled) return CobraConnection::kInvalidMsgId; - - Json::Value root; - for (auto it : data) - { - root[it.first] = it.second; - } - - return push(id, root); - } - - bool CobraMetricsPublisher::shouldPush(const std::string& id) const - { - if (!_enabled) return false; - if (isMetricBlacklisted(id)) return false; - if (isAboveMaxUpdateRate(id)) return false; - - return true; - } - - CobraConnection::MsgId CobraMetricsPublisher::push(const std::string& id, - const Json::Value& data, - bool shouldPushTest) - { - if (shouldPushTest && !shouldPush(id)) return CobraConnection::kInvalidMsgId; - - setLastUpdate(id); - - Json::Value msg; - msg["id"] = id; - msg["data"] = data; - msg["session"] = _session; - msg["version"] = kVersion; - msg["timestamp"] = Json::UInt64(getMillisecondsSinceEpoch()); - - { - std::lock_guard lock(_device_mutex); - msg["device"] = _device; - } - - { - // - // Bump a counter for each id - // This is used to make sure that we are not - // dropping messages, by checking that all the ids is the list of - // all natural numbers until the last value sent (0, 1, 2, ..., N) - // - std::lock_guard lock(_device_mutex); - auto it = _counters.emplace(id, 0); - msg["per_id_counter"] = it.first->second; - it.first->second += 1; - } - - // Now actually enqueue the task - return _cobra_metrics_theaded_publisher.push(msg); - } - - void CobraMetricsPublisher::setPublishMode(CobraConnectionPublishMode publishMode) - { - _cobra_metrics_theaded_publisher.setPublishMode(publishMode); - } - - bool CobraMetricsPublisher::flushQueue() - { - return _cobra_metrics_theaded_publisher.flushQueue(); - } - - void CobraMetricsPublisher::suspend() - { - _cobra_metrics_theaded_publisher.suspend(); - } - - void CobraMetricsPublisher::resume() - { - _cobra_metrics_theaded_publisher.resume(); - } - - bool CobraMetricsPublisher::isConnected() const - { - return _cobra_metrics_theaded_publisher.isConnected(); - } - - bool CobraMetricsPublisher::isAuthenticated() const - { - return _cobra_metrics_theaded_publisher.isAuthenticated(); - } - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraMetricsPublisher.h b/ixcobra/ixcobra/IXCobraMetricsPublisher.h deleted file mode 100644 index 76d7760b..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsPublisher.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - * IXCobraMetricsPublisher.h - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#pragma once - -#include "IXCobraMetricsThreadedPublisher.h" -#include -#include -#include -#include -#include - -namespace ix -{ - struct SocketTLSOptions; - - class CobraMetricsPublisher - { - public: - CobraMetricsPublisher(); - ~CobraMetricsPublisher(); - - /// Thread safety notes: - /// - /// 1. _enabled, _blacklist and _rate_control read/writes are not protected by a mutex - /// to make shouldPush as fast as possible. _enabled default to false. - /// - /// The code that set those is ran only once at init, and - /// the last value to be set is _enabled, which is also the first value checked in - /// shouldPush, so there shouldn't be any race condition. - /// - /// 2. The queue of messages is thread safe, so multiple metrics can be safely pushed on - /// multiple threads - /// - /// 3. Access to _last_update is protected as it needs to be read/write. - /// - - /// Configuration / set keys, etc... - /// All input data but the channel name is encrypted with rc4 - void configure(const CobraConfig& config, const std::string& channel); - - /// Setter for the list of blacklisted metrics ids. - /// That list is sorted internally for fast lookups - void setBlacklist(const std::vector& blacklist); - - /// Set the maximum rate at which a metrics can be sent. Unit is seconds - /// if rate_control = { 'foo_id': 60 }, - /// the foo_id metric cannot be pushed more than once every 60 seconds - void setRateControl(const std::unordered_map& rate_control); - - /// Configuration / enable/disable - void enable(bool enabled); - - /// Simple interface, list of key value pairs where typeof(key) == typeof(value) == string - typedef std::unordered_map Message; - CobraConnection::MsgId push( - const std::string& id, - const CobraMetricsPublisher::Message& data = CobraMetricsPublisher::Message()); - - /// Richer interface using json, which supports types (bool, int, float) and hierarchies of - /// elements - /// - /// The shouldPushTest argument should be set to false, and used in combination with the - /// shouldPush method for places where we want to be as lightweight as possible when - /// collecting metrics. When set to false, it is used so that we don't do double work when - /// computing whether a metrics should be sent or not. - CobraConnection::MsgId push(const std::string& id, - const Json::Value& data, - bool shouldPushTest = true); - - /// Interface used by lua. msg is a json encoded string. - CobraConnection::MsgId push(const std::string& id, - const std::string& data, - bool shouldPushTest = true); - - /// Tells whether a metric can be pushed. - /// A metric can be pushed if it satisfies those conditions: - /// - /// 1. the metrics system should be enabled - /// 2. the metrics shouldn't be black-listed - /// 3. the metrics shouldn't have reached its rate control limit at this - /// "sampling"/"calling" time - bool shouldPush(const std::string& id) const; - - /// Get generic information json object - Json::Value& getGenericAttributes(); - - /// Set generic information values - void setGenericAttributes(const std::string& attrName, const Json::Value& value); - - /// Set a unique id for the session. A uuid can be used. - void setSession(const std::string& session) - { - _session = session; - } - - /// Get the unique id used to identify the current session - const std::string& getSession() const - { - return _session; - } - - /// Return the number of milliseconds since the epoch (~1970) - uint64_t getMillisecondsSinceEpoch() const; - - /// Set satori connection publish mode - void setPublishMode(CobraConnectionPublishMode publishMode); - - /// Flush the publish queue - bool flushQueue(); - - /// Lifecycle management. Free resources when backgrounding - void suspend(); - void resume(); - - /// Tells whether the socket connection is opened - bool isConnected() const; - - /// Returns true only if we're authenticated - bool isAuthenticated() const; - - private: - /// Lookup an id in our metrics to see whether it is blacklisted - /// Complexity is logarithmic - bool isMetricBlacklisted(const std::string& id) const; - - /// Tells whether we should drop a metrics or not as part of an enqueuing - /// because it exceed the max update rate (it is sent too often) - bool isAboveMaxUpdateRate(const std::string& id) const; - - /// Record when a metric was last sent. Used for rate control - void setLastUpdate(const std::string& id); - - /// - /// Member variables - /// - - CobraMetricsThreadedPublisher _cobra_metrics_theaded_publisher; - - /// A boolean to enable or disable this system - /// push becomes a no-op when _enabled is false - std::atomic _enabled; - - /// A uuid used to uniquely identify a session - std::string _session; - - /// The _device json blob is populated once when configuring this system - /// It record generic metadata about the client, run (version, device model, etc...) - Json::Value _device; - mutable std::mutex _device_mutex; // protect access to _device - - /// Metrics control (black list + rate control) - std::vector _blacklist; - std::unordered_map _rate_control; - std::unordered_map> - _last_update; - mutable std::mutex _last_update_mutex; // protect access to _last_update - - /// Bump a counter for each metric type - std::unordered_map _counters; - mutable std::mutex _counters_mutex; // protect access to _counters - - // const strings for internal ids - static const std::string kSetRateControlId; - static const std::string kSetBlacklistId; - - /// Our protocol version. Can be used by subscribers who would want to be backward - /// compatible if we change the way we arrange data - static const int kVersion; - }; - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp b/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp deleted file mode 100644 index 86ddb220..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * IXCobraMetricsThreadedPublisher.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#include "IXCobraMetricsThreadedPublisher.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace ix -{ - CobraMetricsThreadedPublisher::CobraMetricsThreadedPublisher() - : _stop(false) - { - _cobra_connection.setEventCallback([](const CobraEventPtr& event) { - std::stringstream ss; - - if (event->type == ix::CobraEventType::Open) - { - ss << "Handshake headers" << std::endl; - - for (auto&& it : event->headers) - { - ss << it.first << ": " << it.second << std::endl; - } - } - else if (event->type == ix::CobraEventType::Authenticated) - { - ss << "Authenticated"; - } - else if (event->type == ix::CobraEventType::Error) - { - ss << "Error: " << event->errMsg; - } - else if (event->type == ix::CobraEventType::Closed) - { - ss << "Connection closed: " << event->errMsg; - } - else if (event->type == ix::CobraEventType::Subscribed) - { - ss << "Subscribed through subscription id: " << event->subscriptionId; - } - else if (event->type == ix::CobraEventType::UnSubscribed) - { - ss << "Unsubscribed through subscription id: " << event->subscriptionId; - } - else if (event->type == ix::CobraEventType::Published) - { - ss << "Published message " << event->msgId << " acked"; - } - else if (event->type == ix::CobraEventType::Pong) - { - ss << "Received websocket pong"; - } - else if (event->type == ix::CobraEventType::HandshakeError) - { - ss << "Handshake error: " << event->errMsg; - } - else if (event->type == ix::CobraEventType::AuthenticationError) - { - ss << "Authentication error: " << event->errMsg; - } - else if (event->type == ix::CobraEventType::SubscriptionError) - { - ss << "Subscription error: " << event->errMsg; - } - - CoreLogger::log(ss.str().c_str()); - }); - } - - CobraMetricsThreadedPublisher::~CobraMetricsThreadedPublisher() - { - // The background thread won't be joinable if it was never - // started by calling CobraMetricsThreadedPublisher::start - if (!_thread.joinable()) return; - - _stop = true; - _condition.notify_one(); - _thread.join(); - } - - void CobraMetricsThreadedPublisher::start() - { - if (_thread.joinable()) return; // we've already been started - - _thread = std::thread(&CobraMetricsThreadedPublisher::run, this); - } - - void CobraMetricsThreadedPublisher::configure(const CobraConfig& config, - const std::string& channel) - { - CoreLogger::log(config.socketTLSOptions.getDescription().c_str()); - - _channel = channel; - _cobra_connection.configure(config); - } - - void CobraMetricsThreadedPublisher::pushMessage(MessageKind messageKind) - { - { - std::unique_lock lock(_queue_mutex); - _queue.push(messageKind); - } - - // wake up one thread - _condition.notify_one(); - } - - void CobraMetricsThreadedPublisher::setPublishMode(CobraConnectionPublishMode publishMode) - { - _cobra_connection.setPublishMode(publishMode); - } - - bool CobraMetricsThreadedPublisher::flushQueue() - { - return _cobra_connection.flushQueue(); - } - - void CobraMetricsThreadedPublisher::run() - { - setThreadName("CobraMetricsPublisher"); - - _cobra_connection.connect(); - - while (true) - { - Json::Value msg; - MessageKind messageKind; - - { - std::unique_lock lock(_queue_mutex); - - while (!_stop && _queue.empty()) - { - _condition.wait(lock); - } - if (_stop) - { - _cobra_connection.disconnect(); - return; - } - - messageKind = _queue.front(); - _queue.pop(); - } - - switch (messageKind) - { - case MessageKind::Suspend: - { - _cobra_connection.suspend(); - continue; - }; - break; - - case MessageKind::Resume: - { - _cobra_connection.resume(); - continue; - }; - break; - - case MessageKind::Message: - { - if (_cobra_connection.getPublishMode() == CobraConnection_PublishMode_Immediate) - { - _cobra_connection.publishNext(); - } - }; - break; - } - } - } - - CobraConnection::MsgId CobraMetricsThreadedPublisher::push(const Json::Value& msg) - { - static const std::string messageIdKey("id"); - - // - // Publish to multiple channels. This let the consumer side - // easily subscribe to all message of a certain type, without having - // to do manipulations on the messages on the server side. - // - Json::Value channels; - - channels.append(_channel); - if (msg.isMember(messageIdKey)) - { - channels.append(msg[messageIdKey]); - } - auto res = _cobra_connection.prePublish(channels, msg, true); - auto msgId = res.first; - - pushMessage(MessageKind::Message); - - return msgId; - } - - void CobraMetricsThreadedPublisher::suspend() - { - pushMessage(MessageKind::Suspend); - } - - void CobraMetricsThreadedPublisher::resume() - { - pushMessage(MessageKind::Resume); - } - - bool CobraMetricsThreadedPublisher::isConnected() const - { - return _cobra_connection.isConnected(); - } - - bool CobraMetricsThreadedPublisher::isAuthenticated() const - { - return _cobra_connection.isAuthenticated(); - } - -} // namespace ix diff --git a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h b/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h deleted file mode 100644 index 10c50415..00000000 --- a/ixcobra/ixcobra/IXCobraMetricsThreadedPublisher.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * IXCobraMetricsThreadedPublisher.h - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#pragma once - -#include "IXCobraConnection.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ix -{ - struct SocketTLSOptions; - - class CobraMetricsThreadedPublisher - { - public: - CobraMetricsThreadedPublisher(); - ~CobraMetricsThreadedPublisher(); - - /// Configuration / set keys, etc... - void configure(const CobraConfig& config, const std::string& channel); - - /// Start the worker thread, used for background publishing - void start(); - - /// Push a msg to our queue of messages to be published to cobra on the background - // thread. Main user right now is the Cobra Metrics System - CobraConnection::MsgId push(const Json::Value& msg); - - /// Set cobra connection publish mode - void setPublishMode(CobraConnectionPublishMode publishMode); - - /// Flush the publish queue - bool flushQueue(); - - /// Lifecycle management. Free resources when backgrounding - void suspend(); - void resume(); - - /// Tells whether the socket connection is opened - bool isConnected() const; - - /// Returns true only if we're authenticated - bool isAuthenticated() const; - - private: - enum class MessageKind - { - Message = 0, - Suspend = 1, - Resume = 2 - }; - - /// Push a message to be processed by the background thread - void pushMessage(MessageKind messageKind); - - /// Get a wait time which is increasing exponentially based on the number of retries - uint64_t getWaitTimeExp(int retry_count); - - /// Debugging routine to print the connection parameters to the console - void printInfo(); - - /// Publish a message to satory - /// Will retry multiple times (3) if a problem occurs. - /// - /// Right now, only called on the publish worker thread. - void safePublish(const Json::Value& msg); - - /// The worker thread "daemon" method. That method never returns unless _stop is set to true - void run(); - - /// Our connection to cobra. - CobraConnection _cobra_connection; - - /// The channel we are publishing to - std::string _channel; - - /// Internal data structures used to publish to cobra - /// Pending messages are stored into a queue, which is protected by a mutex - /// We used a condition variable to prevent the worker thread from busy polling - /// So we notify the condition variable when an incoming message arrives to signal - /// that it should wake up and take care of publishing it to cobra - /// To shutdown the worker thread one has to set the _stop boolean to true. - /// This is done in the destructor - std::queue _queue; - mutable std::mutex _queue_mutex; - std::condition_variable _condition; - std::atomic _stop; - std::thread _thread; - }; - -} // namespace ix diff --git a/ixcobra/ixcobra/README.md b/ixcobra/ixcobra/README.md deleted file mode 100644 index a852456e..00000000 --- a/ixcobra/ixcobra/README.md +++ /dev/null @@ -1 +0,0 @@ -Client code to publish to a real time analytic system, described in [https://bsergean.github.io/redis_conf_2019/slides.html#1](link). diff --git a/ixcore/CMakeLists.txt b/ixcore/CMakeLists.txt deleted file mode 100644 index 54babb44..00000000 --- a/ixcore/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXCORE_SOURCES - ixcore/utils/IXCoreLogger.cpp -) - -set (IXCORE_HEADERS - ixcore/utils/IXCoreLogger.h -) - -add_library(ixcore STATIC - ${IXCORE_SOURCES} - ${IXCORE_HEADERS} -) - -target_include_directories( ixcore PUBLIC . ) diff --git a/ixcore/ixcore/utils/IXCoreLogger.cpp b/ixcore/ixcore/utils/IXCoreLogger.cpp deleted file mode 100644 index 89159d54..00000000 --- a/ixcore/ixcore/utils/IXCoreLogger.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * IXCoreLogger.cpp - * Author: Thomas Wells, Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ - -#include "ixcore/utils/IXCoreLogger.h" - -namespace ix -{ - // Default do a no-op logger - CoreLogger::LogFunc CoreLogger::_currentLogger = [](const char*, LogLevel) {}; - - void CoreLogger::log(const char* msg, LogLevel level) - { - _currentLogger(msg, level); - } - - void CoreLogger::debug(const std::string& msg) - { - _currentLogger(msg.c_str(), LogLevel::Debug); - } - - void CoreLogger::info(const std::string& msg) - { - _currentLogger(msg.c_str(), LogLevel::Info); - } - - void CoreLogger::warn(const std::string& msg) - { - _currentLogger(msg.c_str(), LogLevel::Warning); - } - - void CoreLogger::error(const std::string& msg) - { - _currentLogger(msg.c_str(), LogLevel::Error); - } - - void CoreLogger::critical(const std::string& msg) - { - _currentLogger(msg.c_str(), LogLevel::Critical); - } - -} // namespace ix diff --git a/ixcore/ixcore/utils/IXCoreLogger.h b/ixcore/ixcore/utils/IXCoreLogger.h deleted file mode 100644 index 18a23883..00000000 --- a/ixcore/ixcore/utils/IXCoreLogger.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * IXCoreLogger.h - * Author: Thomas Wells, Benjamin Sergeant - * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. - */ - -#pragma once -#include -#include - -namespace ix -{ - enum class LogLevel - { - Debug = 0, - Info = 1, - Warning = 2, - Error = 3, - Critical = 4 - }; - - class CoreLogger - { - public: - using LogFunc = std::function; - - static void log(const char* msg, LogLevel level = LogLevel::Debug); - - static void debug(const std::string& msg); - static void info(const std::string& msg); - static void warn(const std::string& msg); - static void error(const std::string& msg); - static void critical(const std::string& msg); - - static void setLogFunction(LogFunc& func) - { - _currentLogger = func; - } - - private: - static LogFunc _currentLogger; - }; - -} // namespace ix diff --git a/ixcrypto/CMakeLists.txt b/ixcrypto/CMakeLists.txt deleted file mode 100644 index fd6896b7..00000000 --- a/ixcrypto/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../CMake;${CMAKE_MODULE_PATH}") - -set (IXCRYPTO_SOURCES - ixcrypto/IXHMac.cpp - ixcrypto/IXBase64.cpp - ixcrypto/IXUuid.cpp - ixcrypto/IXHash.cpp -) - -set (IXCRYPTO_HEADERS - ixcrypto/IXHMac.h - ixcrypto/IXBase64.h - ixcrypto/IXUuid.h - ixcrypto/IXHash.h -) - -add_library(ixcrypto STATIC - ${IXCRYPTO_SOURCES} - ${IXCRYPTO_HEADERS} -) - -set(IXCRYPTO_INCLUDE_DIRS - . - ../ixcore) - -target_include_directories( ixcrypto PUBLIC ${IXCRYPTO_INCLUDE_DIRS} ) - -# hmac computation needs a crypto library - -target_compile_definitions(ixcrypto PUBLIC IXCRYPTO_USE_TLS) -if (USE_MBED_TLS) - find_package(MbedTLS REQUIRED) - target_include_directories(ixcrypto PUBLIC ${MBEDTLS_INCLUDE_DIRS}) - target_link_libraries(ixcrypto ${MBEDTLS_LIBRARIES}) - target_compile_definitions(ixcrypto PUBLIC IXCRYPTO_USE_MBED_TLS) -elseif (APPLE) -elseif (WIN32) -else() - find_package(OpenSSL REQUIRED) - add_definitions(${OPENSSL_DEFINITIONS}) - message(STATUS "OpenSSL: " ${OPENSSL_VERSION}) - include_directories(${OPENSSL_INCLUDE_DIR}) - target_link_libraries(ixcrypto ${OPENSSL_LIBRARIES}) - target_compile_definitions(ixcrypto PUBLIC IXCRYPTO_USE_OPEN_SSL) -endif() diff --git a/ixcrypto/ixcrypto/IXBase64.cpp b/ixcrypto/ixcrypto/IXBase64.cpp deleted file mode 100644 index 3248f497..00000000 --- a/ixcrypto/ixcrypto/IXBase64.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - base64.cpp and base64.h - - Copyright (C) 2004-2008 René Nyffenegger - - This source code is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - - 3. This notice may not be removed or altered from any source distribution. - - René Nyffenegger rene.nyffenegger@adp-gmbh.ch - - */ - -#include "IXBase64.h" - -namespace ix -{ - static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; - - std::string base64_encode(const std::string& data, size_t len) - { - const char* bytes_to_encode = data.c_str(); - return base64_encode(bytes_to_encode, len); - } - - std::string base64_encode(const char* bytes_to_encode, size_t len) - { - std::string ret; - ret.reserve(((len + 2) / 3) * 4); - - int i = 0; - int j = 0; - unsigned char char_array_3[3]; - unsigned char char_array_4[4]; - - while (len--) - { - char_array_3[i++] = *(bytes_to_encode++); - if (i == 3) - { - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; - - for (i = 0; (i < 4); i++) - ret += base64_chars[char_array_4[i]]; - - i = 0; - } - } - - if (i) - { - for (j = i; j < 3; j++) - char_array_3[j] = '\0'; - - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; - - for (j = 0; (j < i + 1); j++) - ret += base64_chars[char_array_4[j]]; - - while ((i++ < 3)) - ret += '='; - } - - return ret; - } - - static inline bool is_base64(unsigned char c) - { - return (isalnum(c) || (c == '+') || (c == '/')); - } - - std::string base64_decode(const std::string& encoded_string) - { - int in_len = (int) encoded_string.size(); - int i = 0; - int j = 0; - int in_ = 0; - unsigned char char_array_4[4], char_array_3[3]; - std::string ret; - ret.reserve(((in_len + 3) / 4) * 3); - - while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) - { - char_array_4[i++] = encoded_string[in_]; - in_++; - if (i == 4) - { - for (i = 0; i < 4; i++) - char_array_4[i] = base64_chars.find(char_array_4[i]); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (i = 0; (i < 3); i++) - ret += char_array_3[i]; - - i = 0; - } - } - - if (i) - { - for (j = i; j < 4; j++) - char_array_4[j] = 0; - - for (j = 0; j < 4; j++) - char_array_4[j] = base64_chars.find(char_array_4[j]); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (j = 0; (j < i - 1); j++) - ret += char_array_3[j]; - } - - return ret; - } -} // namespace ix diff --git a/ixcrypto/ixcrypto/IXBase64.h b/ixcrypto/ixcrypto/IXBase64.h deleted file mode 100644 index 07bad77b..00000000 --- a/ixcrypto/ixcrypto/IXBase64.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * base64.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone. All rights reserved. - */ - -#pragma once - -#include - -namespace ix -{ - std::string base64_encode(const std::string& data, size_t len); - std::string base64_encode(const char* data, size_t len); - std::string base64_decode(const std::string& encoded_string); -} // namespace ix diff --git a/ixcrypto/ixcrypto/IXHMac.cpp b/ixcrypto/ixcrypto/IXHMac.cpp deleted file mode 100644 index d798f80d..00000000 --- a/ixcrypto/ixcrypto/IXHMac.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * IXHMac.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone. All rights reserved. - */ - -#include "IXHMac.h" - -#include "IXBase64.h" - -#if defined(IXCRYPTO_USE_MBED_TLS) -#include -#elif defined(__APPLE__) -#include -#elif defined(IXCRYPTO_USE_OPEN_SSL) -#include -#else -#include -#endif - -namespace ix -{ - std::string hmac(const std::string& data, const std::string& key) - { - constexpr size_t hashSize = 16; - unsigned char hash[hashSize]; - -#if defined(IXCRYPTO_USE_MBED_TLS) - mbedtls_md_hmac(mbedtls_md_info_from_type(MBEDTLS_MD_MD5), - (unsigned char*) key.c_str(), - key.size(), - (unsigned char*) data.c_str(), - data.size(), - (unsigned char*) &hash); -#elif defined(__APPLE__) - CCHmac(kCCHmacAlgMD5, key.c_str(), key.size(), data.c_str(), data.size(), &hash); -#elif defined(IXCRYPTO_USE_OPEN_SSL) - HMAC(EVP_md5(), - key.c_str(), - (int) key.size(), - (unsigned char*) data.c_str(), - (int) data.size(), - (unsigned char*) hash, - nullptr); -#else - assert(false && "hmac not implemented on this platform"); -#endif - - std::string hashString(reinterpret_cast(hash), hashSize); - - return base64_encode(hashString, (uint32_t) hashString.size()); - } -} // namespace ix diff --git a/ixcrypto/ixcrypto/IXHMac.h b/ixcrypto/ixcrypto/IXHMac.h deleted file mode 100644 index a0f93eae..00000000 --- a/ixcrypto/ixcrypto/IXHMac.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * IXHMac.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone. All rights reserved. - */ - -#pragma once - -#include - -namespace ix -{ - std::string hmac(const std::string& data, const std::string& key); -} diff --git a/ixcrypto/ixcrypto/IXHash.cpp b/ixcrypto/ixcrypto/IXHash.cpp deleted file mode 100644 index a62e0871..00000000 --- a/ixcrypto/ixcrypto/IXHash.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * IXHash.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone. All rights reserved. - */ - -#include "IXHash.h" - -namespace ix -{ - uint64_t djb2Hash(const std::vector& data) - { - uint64_t hashAddress = 5381; - - for (auto&& c : data) - { - hashAddress = ((hashAddress << 5) + hashAddress) + c; - } - - return hashAddress; - } -} // namespace ix diff --git a/ixcrypto/ixcrypto/IXHash.h b/ixcrypto/ixcrypto/IXHash.h deleted file mode 100644 index 98d660e1..00000000 --- a/ixcrypto/ixcrypto/IXHash.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * IXHash.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone. All rights reserved. - */ - -#pragma once - -#include -#include - -namespace ix -{ - uint64_t djb2Hash(const std::vector& data); -} diff --git a/ixsentry/CMakeLists.txt b/ixsentry/CMakeLists.txt deleted file mode 100644 index 7a430d79..00000000 --- a/ixsentry/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXSENTRY_SOURCES - ixsentry/IXSentryClient.cpp -) - -set (IXSENTRY_HEADERS - ixsentry/IXSentryClient.h -) - -add_library(ixsentry STATIC - ${IXSENTRY_SOURCES} - ${IXSENTRY_HEADERS} -) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) -endif() - -set(IXSENTRY_INCLUDE_DIRS - . - .. - ../ixcore - ${JSONCPP_INCLUDE_DIRS}) - -target_include_directories( ixsentry PUBLIC ${IXSENTRY_INCLUDE_DIRS} ) diff --git a/ixsentry/ixsentry/IXSentryClient.cpp b/ixsentry/ixsentry/IXSentryClient.cpp deleted file mode 100644 index e6a5b773..00000000 --- a/ixsentry/ixsentry/IXSentryClient.cpp +++ /dev/null @@ -1,310 +0,0 @@ -/* - * IXSentryClient.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone. All rights reserved. - */ - -#include "IXSentryClient.h" - -#include -#include -#include -#include -#include -#include -#include - - -namespace ix -{ - SentryClient::SentryClient(const std::string& dsn) - : _dsn(dsn) - , _validDsn(false) - , _luaFrameRegex("\t([^/]+):([0-9]+): in function ['<]([^/]+)['>]") - , _httpClient(std::make_shared(true)) - { - const std::regex dsnRegex("(http[s]?)://([^:]+):([^@]+)@([^/]+)/([0-9]+)"); - std::smatch group; - - if (std::regex_match(dsn, group, dsnRegex) && group.size() == 6) - { - _validDsn = true; - - const auto scheme = group.str(1); - const auto host = group.str(4); - const auto project_id = group.str(5); - _url = scheme + "://" + host + "/api/" + project_id + "/store/"; - - _publicKey = group.str(2); - _secretKey = group.str(3); - } - } - - void SentryClient::setTLSOptions(const SocketTLSOptions& tlsOptions) - { - _httpClient->setTLSOptions(tlsOptions); - } - - int64_t SentryClient::getTimestamp() - { - const auto tp = std::chrono::system_clock::now(); - const auto dur = tp.time_since_epoch(); - return std::chrono::duration_cast(dur).count(); - } - - std::string SentryClient::getIso8601() - { - std::time_t now; - std::time(&now); - char buf[sizeof("2011-10-08T07:07:09Z")]; - std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", std::gmtime(&now)); - return buf; - } - - std::string SentryClient::computeAuthHeader() - { - std::string securityHeader("Sentry sentry_version=5"); - securityHeader += ",sentry_client=ws/"; - securityHeader += std::string(IX_WEBSOCKET_VERSION); - securityHeader += ",sentry_timestamp=" + std::to_string(SentryClient::getTimestamp()); - securityHeader += ",sentry_key=" + _publicKey; - securityHeader += ",sentry_secret=" + _secretKey; - - return securityHeader; - } - - Json::Value SentryClient::parseLuaStackTrace(const std::string& stack) - { - Json::Value frames; - - // Split by lines - std::string line; - std::stringstream tokenStream(stack); - - std::smatch group; - - while (std::getline(tokenStream, line)) - { - // MapScene.lua:2169: in function 'singleCB' - if (std::regex_match(line, group, _luaFrameRegex)) - { - const auto fileName = group.str(1); - const auto linenoStr = group.str(2); - const auto function = group.str(3); - - std::stringstream ss; - ss << linenoStr; - uint64_t lineno; - ss >> lineno; - - Json::Value frame; - frame["lineno"] = Json::UInt64(lineno); - frame["filename"] = fileName; - frame["function"] = function; - - frames.append(frame); - } - } - - std::reverse(frames.begin(), frames.end()); - - return frames; - } - - std::string parseExceptionName(const std::string& stack) - { - // Split by lines - std::string line; - std::stringstream tokenStream(stack); - - // Extract the first line - std::getline(tokenStream, line); - - return line; - } - - std::string SentryClient::computePayload(const Json::Value& msg) - { - Json::Value payload; - - // - // "tags": [ - // [ - // "a", - // "b" - // ], - // ] - // - Json::Value tags(Json::arrayValue); - - payload["platform"] = "python"; - payload["sdk"]["name"] = "ws"; - payload["sdk"]["version"] = IX_WEBSOCKET_VERSION; - payload["timestamp"] = SentryClient::getIso8601(); - - bool isNoisyTypes = msg["id"].asString() == "game_noisytypes_id"; - - std::string stackTraceFieldName = isNoisyTypes ? "traceback" : "stack"; - std::string stack; - std::string message; - - if (isNoisyTypes) - { - stack = msg["data"][stackTraceFieldName].asString(); - message = parseExceptionName(stack); - } - else // logging - { - if (msg["data"].isMember("info")) - { - stack = msg["data"]["info"][stackTraceFieldName].asString(); - message = msg["data"]["info"]["message"].asString(); - - if (msg["data"].isMember("tags")) - { - auto members = msg["data"]["tags"].getMemberNames(); - - for (auto member : members) - { - Json::Value tag; - tag.append(member); - tag.append(msg["data"]["tags"][member]); - tags.append(tag); - } - } - - if (msg["data"]["info"].isMember("level_str")) - { - // https://docs.sentry.io/enriching-error-data/context/?platform=python#setting-the-level - std::string level = msg["data"]["info"]["level_str"].asString(); - if (level == "critical") - { - level = "fatal"; - } - payload["level"] = level; - } - } - else - { - stack = msg["data"][stackTraceFieldName].asString(); - message = msg["data"]["message"].asString(); - } - } - - Json::Value exception; - exception["stacktrace"]["frames"] = parseLuaStackTrace(stack); - exception["value"] = message; - - payload["exception"].append(exception); - - Json::Value extra; - extra["cobra_event"] = msg; - - // Builtin tags - Json::Value gameTag; - gameTag.append("game"); - gameTag.append(msg["device"]["game"]); - tags.append(gameTag); - - Json::Value userIdTag; - userIdTag.append("userid"); - userIdTag.append(msg["device"]["user_id"]); - tags.append(userIdTag); - - Json::Value environmentTag; - environmentTag.append("environment"); - environmentTag.append(msg["device"]["environment"]); - tags.append(environmentTag); - - Json::Value clientVersionTag; - clientVersionTag.append("client_version"); - clientVersionTag.append(msg["device"]["app_version"]); - tags.append(clientVersionTag); - - payload["tags"] = tags; - - return _jsonWriter.write(payload); - } - - void SentryClient::send( - const Json::Value& msg, - bool verbose, - const OnResponseCallback& onResponseCallback) - { - auto args = _httpClient->createRequest(); - args->url = _url; - args->verb = HttpClient::kPost; - args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader(); - args->connectTimeout = 60; - args->transferTimeout = 5 * 60; - args->followRedirects = true; - args->verbose = verbose; - args->logger = [](const std::string& msg) { CoreLogger::log(msg.c_str()); }; - args->body = computePayload(msg); - - _httpClient->performRequest(args, onResponseCallback); - } - - // https://sentry.io/api/12345/minidump?sentry_key=abcdefgh"); - std::string SentryClient::computeUrl(const std::string& project, const std::string& key) - { - std::stringstream ss; - ss << "https://sentry.io/api/" << project << "/minidump?sentry_key=" << key; - - return ss.str(); - } - - // - // curl -v -X POST -F upload_file_minidump=@ws/crash.dmp - // 'https://sentry.io/api/123456/minidump?sentry_key=12344567890' - // - void SentryClient::uploadMinidump(const std::string& sentryMetadata, - const std::string& minidumpBytes, - const std::string& project, - const std::string& key, - bool verbose, - const OnResponseCallback& onResponseCallback) - { - std::string multipartBoundary = _httpClient->generateMultipartBoundary(); - - auto args = _httpClient->createRequest(); - args->verb = HttpClient::kPost; - args->connectTimeout = 60; - args->transferTimeout = 5 * 60; - args->followRedirects = true; - args->verbose = verbose; - args->multipartBoundary = multipartBoundary; - args->logger = [](const std::string& msg) { CoreLogger::log(msg.c_str()); }; - - HttpFormDataParameters httpFormDataParameters; - httpFormDataParameters["upload_file_minidump"] = minidumpBytes; - - HttpParameters httpParameters; - httpParameters["sentry"] = sentryMetadata; - - args->url = computeUrl(project, key); - args->body = _httpClient->serializeHttpFormDataParameters( - multipartBoundary, httpFormDataParameters, httpParameters); - - _httpClient->performRequest(args, onResponseCallback); - } - - void SentryClient::uploadPayload(const Json::Value& payload, - bool verbose, - const OnResponseCallback& onResponseCallback) - { - auto args = _httpClient->createRequest(); - args->extraHeaders["X-Sentry-Auth"] = SentryClient::computeAuthHeader(); - args->verb = HttpClient::kPost; - args->connectTimeout = 60; - args->transferTimeout = 5 * 60; - args->followRedirects = true; - args->verbose = verbose; - args->logger = [](const std::string& msg) { CoreLogger::log(msg.c_str()); }; - - args->url = _url; - args->body = _jsonWriter.write(payload); - - _httpClient->performRequest(args, onResponseCallback); - } -} // namespace ix diff --git a/ixsentry/ixsentry/IXSentryClient.h b/ixsentry/ixsentry/IXSentryClient.h deleted file mode 100644 index 81291988..00000000 --- a/ixsentry/ixsentry/IXSentryClient.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * IXSentryClient.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace ix -{ - class SentryClient - { - public: - SentryClient(const std::string& dsn); - ~SentryClient() = default; - - void send(const Json::Value& msg, - bool verbose, - const OnResponseCallback& onResponseCallback); - - void uploadMinidump(const std::string& sentryMetadata, - const std::string& minidumpBytes, - const std::string& project, - const std::string& key, - bool verbose, - const OnResponseCallback& onResponseCallback); - - void uploadPayload(const Json::Value& payload, - bool verbose, - const OnResponseCallback& onResponseCallback); - - Json::Value parseLuaStackTrace(const std::string& stack); - - // Mostly for testing - void setTLSOptions(const SocketTLSOptions& tlsOptions); - - - private: - int64_t getTimestamp(); - std::string computeAuthHeader(); - std::string getIso8601(); - std::string computePayload(const Json::Value& msg); - - std::string computeUrl(const std::string& project, const std::string& key); - - void displayReponse(HttpResponsePtr response); - - std::string _dsn; - bool _validDsn; - std::string _url; - - // Used for authentication with a header - std::string _publicKey; - std::string _secretKey; - - Json::FastWriter _jsonWriter; - - std::regex _luaFrameRegex; - - std::shared_ptr _httpClient; - }; - -} // namespace ix diff --git a/ixsnake/CMakeLists.txt b/ixsnake/CMakeLists.txt deleted file mode 100644 index 8a4e7a00..00000000 --- a/ixsnake/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -# -# Author: Benjamin Sergeant -# Copyright (c) 2019 Machine Zone, Inc. All rights reserved. -# - -set (IXSNAKE_SOURCES - ixsnake/IXSnakeServer.cpp - ixsnake/IXSnakeProtocol.cpp - ixsnake/IXAppConfig.cpp - ixsnake/IXRedisClient.cpp - ixsnake/IXRedisServer.cpp -) - -set (IXSNAKE_HEADERS - ixsnake/IXSnakeServer.h - ixsnake/IXSnakeProtocol.h - ixsnake/IXAppConfig.h - ixsnake/IXRedisClient.h - ixsnake/IXRedisServer.h -) - -add_library(ixsnake STATIC - ${IXSNAKE_SOURCES} - ${IXSNAKE_HEADERS} -) - -set(IXSNAKE_INCLUDE_DIRS - . - .. - ../ixcore - ../ixcrypto - ../ixwebsocket - ../third_party) - -target_include_directories( ixsnake PUBLIC ${IXSNAKE_INCLUDE_DIRS} ) diff --git a/ixsnake/ixsnake/IXAppConfig.cpp b/ixsnake/ixsnake/IXAppConfig.cpp deleted file mode 100644 index 8dd80b23..00000000 --- a/ixsnake/ixsnake/IXAppConfig.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * IXSnakeProtocol.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXAppConfig.h" - -#include "IXSnakeProtocol.h" -#include -#include - -namespace snake -{ - bool isAppKeyValid(const AppConfig& appConfig, std::string appkey) - { - return appConfig.apps.count(appkey) != 0; - } - - std::string getRoleSecret(const AppConfig& appConfig, std::string appkey, std::string role) - { - if (!isAppKeyValid(appConfig, appkey)) - { - std::cerr << "Missing appkey " << appkey << std::endl; - return std::string(); - } - - auto roles = appConfig.apps[appkey]["roles"]; - auto channel = roles[role]["secret"]; - return channel; - } - - std::string generateNonce() - { - return ix::uuid4(); - } - - void dumpConfig(const AppConfig& appConfig) - { - for (auto&& host : appConfig.redisHosts) - { - std::cout << "redis host: " << host << std::endl; - } - - std::cout << "redis password: " << appConfig.redisPassword << std::endl; - std::cout << "redis port: " << appConfig.redisPort << std::endl; - } -} // namespace snake diff --git a/ixsnake/ixsnake/IXAppConfig.h b/ixsnake/ixsnake/IXAppConfig.h deleted file mode 100644 index bad529da..00000000 --- a/ixsnake/ixsnake/IXAppConfig.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * IXAppConfig.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include - -namespace snake -{ - struct AppConfig - { - // Server - std::string hostname; - int port; - - // Redis - std::vector redisHosts; - int redisPort; - std::string redisPassword; - - // AppKeys - nlohmann::json apps; - - // TLS options - ix::SocketTLSOptions socketTLSOptions; - - // Misc - bool verbose; - bool disablePong; - }; - - bool isAppKeyValid(const AppConfig& appConfig, std::string appkey); - - std::string getRoleSecret(const AppConfig& appConfig, std::string appkey, std::string role); - - std::string generateNonce(); - - void dumpConfig(const AppConfig& appConfig); -} // namespace snake diff --git a/ixsnake/ixsnake/IXRedisClient.cpp b/ixsnake/ixsnake/IXRedisClient.cpp deleted file mode 100644 index 5429c44a..00000000 --- a/ixsnake/ixsnake/IXRedisClient.cpp +++ /dev/null @@ -1,351 +0,0 @@ -/* - * IXRedisClient.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXRedisClient.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace ix -{ - bool RedisClient::connect(const std::string& hostname, int port) - { - bool tls = false; - std::string errorMsg; - SocketTLSOptions tlsOptions; - _socket = createSocket(tls, -1, errorMsg, tlsOptions); - - if (!_socket) - { - return false; - } - - CancellationRequest cancellationRequest = []() -> bool { return false; }; - - std::string errMsg; - return _socket->connect(hostname, port, errMsg, cancellationRequest); - } - - void RedisClient::stop() - { - _stop = true; - } - - bool RedisClient::auth(const std::string& password, std::string& response) - { - response.clear(); - - if (!_socket) return false; - - std::stringstream ss; - ss << "AUTH "; - ss << password; - ss << "\r\n"; - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - return false; - } - - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - return false; - } - - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - response = line; - return lineValid; - } - - std::string RedisClient::writeString(const std::string& str) - { - std::stringstream ss; - ss << "$"; - ss << str.size(); - ss << "\r\n"; - ss << str; - ss << "\r\n"; - - return ss.str(); - } - - bool RedisClient::publish(const std::string& channel, - const std::string& message, - std::string& errMsg) - { - errMsg.clear(); - - if (!_socket) - { - errMsg = "socket is not initialized"; - return false; - } - - std::stringstream ss; - ss << "*3\r\n"; - ss << writeString("PUBLISH"); - ss << writeString(channel); - ss << writeString(message); - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return false; - } - - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - errMsg = "Error while polling for result"; - return false; - } - - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - // A successful response starts with a : - if (line.empty() || line[0] != ':') - { - errMsg = line; - return false; - } - - return lineValid; - } - - // - // FIXME: we assume that redis never return errors... - // - bool RedisClient::subscribe(const std::string& channel, - const OnRedisSubscribeResponseCallback& responseCallback, - const OnRedisSubscribeCallback& callback) - { - _stop = false; - - if (!_socket) return false; - - std::stringstream ss; - ss << "*2\r\n"; - ss << writeString("SUBSCRIBE"); - ss << writeString(channel); - - bool sent = _socket->writeBytes(ss.str(), nullptr); - if (!sent) - { - return false; - } - - // Wait 1s for the response - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - return false; - } - - // build the response as a single string - std::stringstream oss; - - // Read the first line of the response - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - oss << line; - - if (!lineValid) return false; - - // There are 5 items for the subscribe reply - for (int i = 0; i < 5; ++i) - { - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - oss << line; - - if (!lineValid) return false; - } - - responseCallback(oss.str()); - - // Wait indefinitely for new messages - while (true) - { - if (_stop) break; - - // Wait until something is ready to read - int timeoutMs = 10; - auto pollResult = _socket->isReadyToRead(timeoutMs); - if (pollResult == PollResultType::Error) - { - return false; - } - - if (pollResult == PollResultType::Timeout) - { - continue; - } - - // The first line of the response describe the return type, - // => *3 (an array of 3 elements) - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - int arraySize; - { - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> arraySize; - } - - // There are 6 items for each received message - for (int i = 0; i < arraySize; ++i) - { - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - // Messages are string, which start with a string size - // => $7 (7 bytes) - int stringSize; - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - - auto readResult = _socket->readBytes(stringSize, nullptr, nullptr); - if (!readResult.first) return false; - - if (i == 2) - { - // The message is the 3rd element. - callback(readResult.second); - } - - // read last 2 bytes (\r\n) - char c; - _socket->readByte(&c, nullptr); - _socket->readByte(&c, nullptr); - } - } - - return true; - } - - std::string RedisClient::prepareXaddCommand(const std::string& stream, - const std::string& message) - { - std::stringstream ss; - ss << "*5\r\n"; - ss << writeString("XADD"); - ss << writeString(stream); - ss << writeString("*"); - ss << writeString("field"); - ss << writeString(message); - - return ss.str(); - } - - std::string RedisClient::xadd(const std::string& stream, - const std::string& message, - std::string& errMsg) - { - errMsg.clear(); - - if (!_socket) - { - errMsg = "socket is not initialized"; - return std::string(); - } - - std::string command = prepareXaddCommand(stream, message); - - bool sent = _socket->writeBytes(command, nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return std::string(); - } - - return readXaddReply(errMsg); - } - - std::string RedisClient::readXaddReply(std::string& errMsg) - { - // Read result - auto pollResult = _socket->isReadyToRead(-1); - if (pollResult == PollResultType::Error) - { - errMsg = "Error while polling for result"; - return std::string(); - } - - // First line is the string length - auto lineResult = _socket->readLine(nullptr); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) - { - errMsg = "Error while polling for result"; - return std::string(); - } - - int stringSize; - { - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - } - - // Read the result, which is the stream id computed by the redis server - lineResult = _socket->readLine(nullptr); - lineValid = lineResult.first; - line = lineResult.second; - - std::string streamId = line.substr(0, stringSize - 1); - return streamId; - } - - bool RedisClient::sendCommand(const std::string& commands, - int commandsCount, - std::string& errMsg) - { - bool sent = _socket->writeBytes(commands, nullptr); - if (!sent) - { - errMsg = "Cannot write bytes to socket"; - return false; - } - - bool success = true; - - for (int i = 0; i < commandsCount; ++i) - { - auto reply = readXaddReply(errMsg); - if (reply == std::string()) - { - success = false; - } - } - - return success; - } -} // namespace ix diff --git a/ixsnake/ixsnake/IXRedisClient.h b/ixsnake/ixsnake/IXRedisClient.h deleted file mode 100644 index 1d1640c1..00000000 --- a/ixsnake/ixsnake/IXRedisClient.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * IXRedisClient.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace ix -{ - class RedisClient - { - public: - using OnRedisSubscribeResponseCallback = std::function; - using OnRedisSubscribeCallback = std::function; - - RedisClient() - : _stop(false) - { - } - ~RedisClient() = default; - - bool connect(const std::string& hostname, int port); - - bool auth(const std::string& password, std::string& response); - - // Publish / Subscribe - bool publish(const std::string& channel, const std::string& message, std::string& errMsg); - - bool subscribe(const std::string& channel, - const OnRedisSubscribeResponseCallback& responseCallback, - const OnRedisSubscribeCallback& callback); - - // XADD - std::string xadd(const std::string& channel, - const std::string& message, - std::string& errMsg); - - std::string prepareXaddCommand(const std::string& stream, const std::string& message); - - std::string readXaddReply(std::string& errMsg); - - bool sendCommand(const std::string& commands, int commandsCount, std::string& errMsg); - - void stop(); - - private: - std::string writeString(const std::string& str); - - std::unique_ptr _socket; - std::atomic _stop; - }; -} // namespace ix diff --git a/ixsnake/ixsnake/IXRedisServer.cpp b/ixsnake/ixsnake/IXRedisServer.cpp deleted file mode 100644 index aabfd2a6..00000000 --- a/ixsnake/ixsnake/IXRedisServer.cpp +++ /dev/null @@ -1,285 +0,0 @@ -/* - * IXRedisServer.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXRedisServer.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace ix -{ - RedisServer::RedisServer( - int port, const std::string& host, int backlog, size_t maxConnections, int addressFamily) - : SocketServer(port, host, backlog, maxConnections, addressFamily) - , _connectedClientsCount(0) - , _stopHandlingConnections(false) - { - ; - } - - RedisServer::~RedisServer() - { - stop(); - } - - void RedisServer::stop() - { - stopAcceptingConnections(); - - _stopHandlingConnections = true; - while (_connectedClientsCount != 0) - { - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - _stopHandlingConnections = false; - - SocketServer::stop(); - } - - void RedisServer::handleConnection(std::unique_ptr socket, - std::shared_ptr connectionState) - { - _connectedClientsCount++; - - while (!_stopHandlingConnections) - { - std::vector tokens; - if (!parseRequest(socket, tokens)) - { - if (_stopHandlingConnections) - { - logError("Cancellation requested"); - } - else - { - logError("Error parsing request"); - } - break; - } - - bool success = false; - - // publish - if (tokens[0] == "COMMAND") - { - success = handleCommand(socket, tokens); - } - else if (tokens[0] == "PUBLISH") - { - success = handlePublish(socket, tokens); - } - else if (tokens[0] == "SUBSCRIBE") - { - success = handleSubscribe(socket, tokens); - } - - if (!success) - { - if (_stopHandlingConnections) - { - logError("Cancellation requested"); - } - else - { - logError("Error processing request for command: " + tokens[0]); - } - break; - } - } - - cleanupSubscribers(socket); - - logInfo("Connection closed for connection id " + connectionState->getId()); - connectionState->setTerminated(); - - _connectedClientsCount--; - } - - void RedisServer::cleanupSubscribers(std::unique_ptr& socket) - { - std::lock_guard lock(_mutex); - - for (auto&& it : _subscribers) - { - it.second.erase(socket.get()); - } - - for (auto it : _subscribers) - { - std::stringstream ss; - ss << "Subscription id: " << it.first << " #subscribers: " << it.second.size(); - - logInfo(ss.str()); - } - } - - size_t RedisServer::getConnectedClientsCount() - { - return _connectedClientsCount; - } - - bool RedisServer::startsWith(const std::string& str, const std::string& start) - { - return str.compare(0, start.length(), start) == 0; - } - - std::string RedisServer::writeString(const std::string& str) - { - std::stringstream ss; - ss << "$"; - ss << str.size(); - ss << "\r\n"; - ss << str; - ss << "\r\n"; - - return ss.str(); - } - - bool RedisServer::parseRequest(std::unique_ptr& socket, - std::vector& tokens) - { - // Parse first line - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - auto lineResult = socket->readLine(cb); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - std::string str = line.substr(1); - std::stringstream ss; - ss << str; - int count; - ss >> count; - - for (int i = 0; i < count; ++i) - { - auto lineResult = socket->readLine(cb); - auto lineValid = lineResult.first; - auto line = lineResult.second; - - if (!lineValid) return false; - - int stringSize; - std::stringstream ss; - ss << line.substr(1, line.size() - 1); - ss >> stringSize; - - auto readResult = socket->readBytes(stringSize, nullptr, nullptr); - - if (!readResult.first) return false; - - // read last 2 bytes (\r\n) - char c; - socket->readByte(&c, nullptr); - socket->readByte(&c, nullptr); - - tokens.push_back(readResult.second); - } - - return true; - } - - bool RedisServer::handleCommand(std::unique_ptr& socket, - const std::vector& tokens) - { - if (tokens.size() != 1) return false; - - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - std::stringstream ss; - - // return 2 nested arrays - ss << "*2\r\n"; - - // - // publish - // - ss << "*6\r\n"; - ss << writeString("publish"); // 1 - ss << ":3\r\n"; // 2 - ss << "*0\r\n"; // 3 - ss << ":1\r\n"; // 4 - ss << ":2\r\n"; // 5 - ss << ":1\r\n"; // 6 - - // - // subscribe - // - ss << "*6\r\n"; - ss << writeString("subscribe"); // 1 - ss << ":2\r\n"; // 2 - ss << "*0\r\n"; // 3 - ss << ":1\r\n"; // 4 - ss << ":1\r\n"; // 5 - ss << ":1\r\n"; // 6 - - socket->writeBytes(ss.str(), cb); - - return true; - } - - bool RedisServer::handleSubscribe(std::unique_ptr& socket, - const std::vector& tokens) - { - if (tokens.size() != 2) return false; - - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - std::string channel = tokens[1]; - - // Respond - socket->writeBytes("*3\r\n", cb); - socket->writeBytes(writeString("subscribe"), cb); - socket->writeBytes(writeString(channel), cb); - socket->writeBytes(":1\r\n", cb); - - std::lock_guard lock(_mutex); - _subscribers[channel].insert(socket.get()); - - return true; - } - - bool RedisServer::handlePublish(std::unique_ptr& socket, - const std::vector& tokens) - { - if (tokens.size() != 3) return false; - - auto cb = makeCancellationRequestWithTimeout(30, _stopHandlingConnections); - std::string channel = tokens[1]; - std::string data = tokens[2]; - - // now dispatch the message to subscribers (write custom method) - std::lock_guard lock(_mutex); - auto it = _subscribers.find(channel); - if (it == _subscribers.end()) - { - // return the number of clients that received the message, 0 in that case - socket->writeBytes(":0\r\n", cb); - return true; - } - - auto subscribers = it->second; - for (auto jt : subscribers) - { - jt->writeBytes("*3\r\n", cb); - jt->writeBytes(writeString("message"), cb); - jt->writeBytes(writeString(channel), cb); - jt->writeBytes(writeString(data), cb); - } - - // return the number of clients that received the message. - std::stringstream ss; - ss << ":" << std::to_string(subscribers.size()) << "\r\n"; - socket->writeBytes(ss.str(), cb); - - return true; - } - -} // namespace ix diff --git a/ixsnake/ixsnake/IXRedisServer.h b/ixsnake/ixsnake/IXRedisServer.h deleted file mode 100644 index 6d2bfc00..00000000 --- a/ixsnake/ixsnake/IXRedisServer.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * IXRedisServer.h - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include "IXSocket.h" -#include "IXSocketServer.h" -#include -#include -#include -#include -#include -#include -#include -#include // pair - -namespace ix -{ - class RedisServer final : public SocketServer - { - public: - RedisServer(int port = SocketServer::kDefaultPort, - const std::string& host = SocketServer::kDefaultHost, - int backlog = SocketServer::kDefaultTcpBacklog, - size_t maxConnections = SocketServer::kDefaultMaxConnections, - int addressFamily = SocketServer::kDefaultAddressFamily); - virtual ~RedisServer(); - virtual void stop() final; - - private: - // Member variables - std::atomic _connectedClientsCount; - - // Subscribers - // We could store connection states in there, to add better debugging - // since a connection state has a readable ID - std::map> _subscribers; - std::mutex _mutex; - - std::atomic _stopHandlingConnections; - - // Methods - virtual void handleConnection(std::unique_ptr, - std::shared_ptr connectionState) final; - virtual size_t getConnectedClientsCount() final; - - bool startsWith(const std::string& str, const std::string& start); - std::string writeString(const std::string& str); - - bool parseRequest(std::unique_ptr& socket, std::vector& tokens); - - bool handlePublish(std::unique_ptr& socket, const std::vector& tokens); - - bool handleSubscribe(std::unique_ptr& socket, - const std::vector& tokens); - - bool handleCommand(std::unique_ptr& socket, const std::vector& tokens); - - void cleanupSubscribers(std::unique_ptr& socket); - }; -} // namespace ix diff --git a/ixsnake/ixsnake/IXSnakeConnectionState.h b/ixsnake/ixsnake/IXSnakeConnectionState.h deleted file mode 100644 index b4785608..00000000 --- a/ixsnake/ixsnake/IXSnakeConnectionState.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * IXSnakeConnectionState.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include "IXRedisClient.h" -#include -#include -#include - -namespace snake -{ - class SnakeConnectionState : public ix::ConnectionState - { - public: - std::string getNonce() - { - return _nonce; - } - - void setNonce(const std::string& nonce) - { - _nonce = nonce; - } - - std::string appkey() - { - return _appkey; - } - void setAppkey(const std::string& appkey) - { - _appkey = appkey; - } - - std::string role() - { - return _role; - } - void setRole(const std::string& role) - { - _role = role; - } - - ix::RedisClient& redisClient() - { - return _redisClient; - } - - std::future fut; - - private: - std::string _nonce; - std::string _role; - std::string _appkey; - - ix::RedisClient _redisClient; - }; -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeProtocol.cpp b/ixsnake/ixsnake/IXSnakeProtocol.cpp deleted file mode 100644 index d1941261..00000000 --- a/ixsnake/ixsnake/IXSnakeProtocol.cpp +++ /dev/null @@ -1,297 +0,0 @@ -/* - * IXSnakeProtocol.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXSnakeProtocol.h" - -#include "IXAppConfig.h" -#include "IXSnakeConnectionState.h" -#include "nlohmann/json.hpp" -#include -#include -#include -#include -#include - -namespace snake -{ - void handleError(const std::string& action, - std::shared_ptr ws, - nlohmann::json pdu, - const std::string& errMsg) - { - std::string actionError(action); - actionError += "/error"; - - nlohmann::json response = { - {"action", actionError}, {"id", pdu.value("id", 1)}, {"body", {{"reason", errMsg}}}}; - ws->sendText(response.dump()); - } - - void handleHandshake(std::shared_ptr state, - std::shared_ptr ws, - const nlohmann::json& pdu) - { - std::string role = pdu["body"]["data"]["role"]; - - state->setNonce(generateNonce()); - state->setRole(role); - - nlohmann::json response = { - {"action", "auth/handshake/ok"}, - {"id", pdu.value("id", 1)}, - {"body", - { - {"data", {{"nonce", state->getNonce()}, {"connection_id", state->getId()}}}, - }}}; - - auto serializedResponse = response.dump(); - - ws->sendText(serializedResponse); - } - - void handleAuth(std::shared_ptr state, - std::shared_ptr ws, - const AppConfig& appConfig, - const nlohmann::json& pdu) - { - auto secret = getRoleSecret(appConfig, state->appkey(), state->role()); - - if (secret.empty()) - { - nlohmann::json response = { - {"action", "auth/authenticate/error"}, - {"id", pdu.value("id", 1)}, - {"body", {{"error", "authentication_failed"}, {"reason", "invalid secret"}}}}; - ws->sendText(response.dump()); - return; - } - - auto nonce = state->getNonce(); - auto serverHash = ix::hmac(nonce, secret); - std::string clientHash = pdu["body"]["credentials"]["hash"]; - - if (serverHash != clientHash) - { - nlohmann::json response = { - {"action", "auth/authenticate/error"}, - {"id", pdu.value("id", 1)}, - {"body", {{"error", "authentication_failed"}, {"reason", "invalid hash"}}}}; - ws->sendText(response.dump()); - return; - } - - nlohmann::json response = { - {"action", "auth/authenticate/ok"}, {"id", pdu.value("id", 1)}, {"body", {}}}; - - ws->sendText(response.dump()); - } - - void handlePublish(std::shared_ptr state, - std::shared_ptr ws, - const nlohmann::json& pdu) - { - std::vector channels; - - auto body = pdu["body"]; - if (body.find("channels") != body.end()) - { - for (auto&& channel : body["channels"]) - { - channels.push_back(channel); - } - } - else if (body.find("channel") != body.end()) - { - channels.push_back(body["channel"]); - } - else - { - std::stringstream ss; - ss << "Missing channels or channel field in publish data"; - handleError("rtm/publish", ws, pdu, ss.str()); - return; - } - - for (auto&& channel : channels) - { - std::stringstream ss; - ss << state->appkey() << "::" << channel; - - std::string errMsg; - if (!state->redisClient().publish(ss.str(), pdu.dump(), errMsg)) - { - std::stringstream ss; - ss << "Cannot publish to redis host " << errMsg; - handleError("rtm/publish", ws, pdu, ss.str()); - return; - } - } - - nlohmann::json response = { - {"action", "rtm/publish/ok"}, {"id", pdu.value("id", 1)}, {"body", {}}}; - - ws->sendText(response.dump()); - } - - // - // FIXME: this is not cancellable. We should be able to cancel the redis subscription - // - void handleRedisSubscription(std::shared_ptr state, - std::shared_ptr ws, - const AppConfig& appConfig, - const nlohmann::json& pdu) - { - std::string channel = pdu["body"]["channel"]; - std::string subscriptionId = channel; - - std::stringstream ss; - ss << state->appkey() << "::" << channel; - - std::string appChannel(ss.str()); - - ix::RedisClient redisClient; - int port = appConfig.redisPort; - - auto urls = appConfig.redisHosts; - std::string hostname(urls[0]); - - // Connect to redis first - if (!redisClient.connect(hostname, port)) - { - std::stringstream ss; - ss << "Cannot connect to redis host " << hostname << ":" << port; - handleError("rtm/subscribe", ws, pdu, ss.str()); - return; - } - - // Now authenticate, if needed - if (!appConfig.redisPassword.empty()) - { - std::string authResponse; - if (!redisClient.auth(appConfig.redisPassword, authResponse)) - { - std::stringstream ss; - ss << "Cannot authenticated to redis"; - handleError("rtm/subscribe", ws, pdu, ss.str()); - return; - } - } - - int id = 0; - auto callback = [ws, &id, &subscriptionId](const std::string& messageStr) { - auto msg = nlohmann::json::parse(messageStr); - - msg = msg["body"]["message"]; - - nlohmann::json response = { - {"action", "rtm/subscription/data"}, - {"id", id++}, - {"body", - {{"subscription_id", subscriptionId}, {"position", "0-0"}, {"messages", {msg}}}}}; - - ws->sendText(response.dump()); - }; - - auto responseCallback = [ws, pdu, &subscriptionId](const std::string& redisResponse) { - std::stringstream ss; - ss << "Redis Response: " << redisResponse << "..."; - ix::CoreLogger::log(ss.str().c_str()); - - // Success - nlohmann::json response = {{"action", "rtm/subscribe/ok"}, - {"id", pdu.value("id", 1)}, - {"body", {{"subscription_id", subscriptionId}}}}; - ws->sendText(response.dump()); - }; - - { - std::stringstream ss; - ss << "Subscribing to " << appChannel << "..."; - ix::CoreLogger::log(ss.str().c_str()); - } - - if (!redisClient.subscribe(appChannel, responseCallback, callback)) - { - std::stringstream ss; - ss << "Error subscribing to channel " << appChannel; - handleError("rtm/subscribe", ws, pdu, ss.str()); - return; - } - } - - void handleSubscribe(std::shared_ptr state, - std::shared_ptr ws, - const AppConfig& appConfig, - const nlohmann::json& pdu) - { - state->fut = - std::async(std::launch::async, handleRedisSubscription, state, ws, appConfig, pdu); - } - - void handleUnSubscribe(std::shared_ptr state, - std::shared_ptr ws, - const nlohmann::json& pdu) - { - // extract subscription_id - auto body = pdu["body"]; - auto subscriptionId = body["subscription_id"]; - - state->redisClient().stop(); - - nlohmann::json response = {{"action", "rtm/unsubscribe/ok"}, - {"id", pdu.value("id", 1)}, - {"body", {{"subscription_id", subscriptionId}}}}; - ws->sendText(response.dump()); - } - - void processCobraMessage(std::shared_ptr state, - std::shared_ptr ws, - const AppConfig& appConfig, - const std::string& str) - { - nlohmann::json pdu; - try - { - pdu = nlohmann::json::parse(str); - } - catch (const nlohmann::json::parse_error& e) - { - std::stringstream ss; - ss << "malformed json pdu: " << e.what() << " -> " << str << ""; - - nlohmann::json response = {{"body", {{"error", "invalid_json"}, {"reason", ss.str()}}}}; - ws->sendText(response.dump()); - return; - } - - auto action = pdu["action"]; - - if (action == "auth/handshake") - { - handleHandshake(state, ws, pdu); - } - else if (action == "auth/authenticate") - { - handleAuth(state, ws, appConfig, pdu); - } - else if (action == "rtm/publish") - { - handlePublish(state, ws, pdu); - } - else if (action == "rtm/subscribe") - { - handleSubscribe(state, ws, appConfig, pdu); - } - else if (action == "rtm/unsubscribe") - { - handleUnSubscribe(state, ws, pdu); - } - else - { - std::cerr << "Unhandled action: " << action << std::endl; - } - } -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeProtocol.h b/ixsnake/ixsnake/IXSnakeProtocol.h deleted file mode 100644 index fd541d07..00000000 --- a/ixsnake/ixsnake/IXSnakeProtocol.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * IXSnakeProtocol.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include -#include - -namespace ix -{ - class WebSocket; -} - -namespace snake -{ - class SnakeConnectionState; - struct AppConfig; - - void processCobraMessage(std::shared_ptr state, - std::shared_ptr ws, - const AppConfig& appConfig, - const std::string& str); -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeServer.cpp b/ixsnake/ixsnake/IXSnakeServer.cpp deleted file mode 100644 index 49b2625c..00000000 --- a/ixsnake/ixsnake/IXSnakeServer.cpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * IXSnakeServer.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#include "IXSnakeServer.h" - -#include "IXAppConfig.h" -#include "IXSnakeConnectionState.h" -#include "IXSnakeProtocol.h" -#include -#include -#include - - -namespace snake -{ - SnakeServer::SnakeServer(const AppConfig& appConfig) - : _appConfig(appConfig) - , _server(appConfig.port, appConfig.hostname) - { - _server.setTLSOptions(appConfig.socketTLSOptions); - - if (appConfig.disablePong) - { - _server.disablePong(); - } - - std::stringstream ss; - ss << "Listening on " << appConfig.hostname << ":" << appConfig.port; - ix::CoreLogger::log(ss.str().c_str()); - } - - // - // Parse appkey from this uri. Won't work if multiple args are present in the uri - // Uri: /v2?appkey=FC2F10139A2BAc53BB72D9db967b024f - // - std::string SnakeServer::parseAppKey(const std::string& path) - { - std::string::size_type idx; - - idx = path.rfind('='); - if (idx != std::string::npos) - { - std::string appkey = path.substr(idx + 1); - return appkey; - } - else - { - return std::string(); - } - } - - bool SnakeServer::run() - { - auto factory = []() -> std::shared_ptr { - return std::make_shared(); - }; - _server.setConnectionStateFactory(factory); - - _server.setOnConnectionCallback( - [this](std::shared_ptr webSocket, - std::shared_ptr connectionState) { - auto state = std::dynamic_pointer_cast(connectionState); - - webSocket->setOnMessageCallback( - [this, webSocket, state](const ix::WebSocketMessagePtr& msg) { - std::stringstream ss; - ix::LogLevel logLevel = ix::LogLevel::Debug; - if (msg->type == ix::WebSocketMessageType::Open) - { - ss << "New connection" << std::endl; - ss << "id: " << state->getId() << std::endl; - ss << "Uri: " << msg->openInfo.uri << std::endl; - ss << "Headers:" << std::endl; - for (auto it : msg->openInfo.headers) - { - ss << it.first << ": " << it.second << std::endl; - } - - std::string appkey = parseAppKey(msg->openInfo.uri); - state->setAppkey(appkey); - - // Connect to redis first - if (!state->redisClient().connect(_appConfig.redisHosts[0], - _appConfig.redisPort)) - { - ss << "Cannot connect to redis host" << std::endl; - logLevel = ix::LogLevel::Error; - } - } - else if (msg->type == ix::WebSocketMessageType::Close) - { - ss << "Closed connection" - << " code " << msg->closeInfo.code << " reason " - << msg->closeInfo.reason << std::endl; - } - else if (msg->type == ix::WebSocketMessageType::Error) - { - std::stringstream ss; - ss << "Connection error: " << msg->errorInfo.reason << std::endl; - ss << "#retries: " << msg->errorInfo.retries << std::endl; - ss << "Wait time(ms): " << msg->errorInfo.wait_time << std::endl; - ss << "HTTP Status: " << msg->errorInfo.http_status << std::endl; - logLevel = ix::LogLevel::Error; - } - else if (msg->type == ix::WebSocketMessageType::Fragment) - { - ss << "Received message fragment" << std::endl; - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - ss << "Received " << msg->wireSize << " bytes" << std::endl; - processCobraMessage(state, webSocket, _appConfig, msg->str); - } - - ix::CoreLogger::log(ss.str().c_str(), logLevel); - }); - }); - - auto res = _server.listen(); - if (!res.first) - { - std::cerr << res.second << std::endl; - return false; - } - - _server.start(); - return true; - } - - void SnakeServer::runForever() - { - if (run()) - { - _server.wait(); - } - } - - void SnakeServer::stop() - { - _server.stop(); - } -} // namespace snake diff --git a/ixsnake/ixsnake/IXSnakeServer.h b/ixsnake/ixsnake/IXSnakeServer.h deleted file mode 100644 index 4be9a06a..00000000 --- a/ixsnake/ixsnake/IXSnakeServer.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * IXSnakeServer.h - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ - -#pragma once - -#include "IXAppConfig.h" -#include -#include - -namespace snake -{ - class SnakeServer - { - public: - SnakeServer(const AppConfig& appConfig); - ~SnakeServer() = default; - - bool run(); - void runForever(); - void stop(); - - private: - std::string parseAppKey(const std::string& path); - - AppConfig _appConfig; - ix::WebSocketServer _server; - }; -} // namespace snake diff --git a/ixsnake/ixsnake/appsConfig.json b/ixsnake/ixsnake/appsConfig.json deleted file mode 100644 index 14f8f48b..00000000 --- a/ixsnake/ixsnake/appsConfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "apps": { - "FC2F10139A2BAc53BB72D9db967b024f": { - "roles": { - "_sub": { - "secret": "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba" - }, - "_pub": { - "secret": "1c04DB8fFe76A4EeFE3E318C72d771db" - } - } - } - } -} diff --git a/ixwebsocket/IXBench.cpp b/ixwebsocket/IXBench.cpp index 64ca2eaf..75497f85 100644 --- a/ixwebsocket/IXBench.cpp +++ b/ixwebsocket/IXBench.cpp @@ -12,10 +12,8 @@ namespace ix { Bench::Bench(const std::string& description) : _description(description) - , _start(std::chrono::high_resolution_clock::now()) - , _reported(false) { - ; + reset(); } Bench::~Bench() @@ -26,19 +24,38 @@ namespace ix } } + void Bench::reset() + { + _start = std::chrono::high_resolution_clock::now(); + _reported = false; + } + void Bench::report() { auto now = std::chrono::high_resolution_clock::now(); - auto milliseconds = std::chrono::duration_cast(now - _start); + auto microseconds = std::chrono::duration_cast(now - _start); - _ms = milliseconds.count(); - std::cerr << _description << " completed in " << _ms << "ms" << std::endl; + _duration = microseconds.count(); + std::cerr << _description << " completed in " << _duration << " us" << std::endl; + setReported(); + } + + void Bench::record() + { + auto now = std::chrono::high_resolution_clock::now(); + auto microseconds = std::chrono::duration_cast(now - _start); + + _duration = microseconds.count(); + } + + void Bench::setReported() + { _reported = true; } uint64_t Bench::getDuration() const { - return _ms; + return _duration; } } // namespace ix diff --git a/ixwebsocket/IXBench.h b/ixwebsocket/IXBench.h index 5b3d3eb7..c4f904b7 100644 --- a/ixwebsocket/IXBench.h +++ b/ixwebsocket/IXBench.h @@ -3,6 +3,7 @@ * Author: Benjamin Sergeant * Copyright (c) 2017-2020 Machine Zone, Inc. All rights reserved. */ +#pragma once #include #include @@ -16,13 +17,16 @@ namespace ix Bench(const std::string& description); ~Bench(); + void reset(); + void record(); void report(); + void setReported(); uint64_t getDuration() const; private: std::string _description; std::chrono::time_point _start; - uint64_t _ms; + uint64_t _duration; bool _reported; }; } // namespace ix diff --git a/ixwebsocket/IXConnectionState.cpp b/ixwebsocket/IXConnectionState.cpp index a7c6c75b..8a559f19 100644 --- a/ixwebsocket/IXConnectionState.cpp +++ b/ixwebsocket/IXConnectionState.cpp @@ -31,6 +31,11 @@ namespace ix return std::make_shared(); } + void ConnectionState::setOnSetTerminatedCallback(const OnSetTerminatedCallback& callback) + { + _onSetTerminatedCallback = callback; + } + bool ConnectionState::isTerminated() const { return _terminated; @@ -39,5 +44,30 @@ namespace ix void ConnectionState::setTerminated() { _terminated = true; + + if (_onSetTerminatedCallback) + { + _onSetTerminatedCallback(); + } + } + + const std::string& ConnectionState::getRemoteIp() + { + return _remoteIp; + } + + int ConnectionState::getRemotePort() + { + return _remotePort; + } + + void ConnectionState::setRemoteIp(const std::string& remoteIp) + { + _remoteIp = remoteIp; + } + + void ConnectionState::setRemotePort(int remotePort) + { + _remotePort = remotePort; } } // namespace ix diff --git a/ixwebsocket/IXConnectionState.h b/ixwebsocket/IXConnectionState.h index 17df5bca..b7530d0b 100644 --- a/ixwebsocket/IXConnectionState.h +++ b/ixwebsocket/IXConnectionState.h @@ -7,12 +7,15 @@ #pragma once #include +#include #include #include #include namespace ix { + using OnSetTerminatedCallback = std::function; + class ConnectionState { public: @@ -25,12 +28,27 @@ namespace ix void setTerminated(); bool isTerminated() const; + const std::string& getRemoteIp(); + int getRemotePort(); + static std::shared_ptr createConnectionState(); + private: + void setOnSetTerminatedCallback(const OnSetTerminatedCallback& callback); + + void setRemoteIp(const std::string& remoteIp); + void setRemotePort(int remotePort); + protected: std::atomic _terminated; std::string _id; + OnSetTerminatedCallback _onSetTerminatedCallback; static std::atomic _globalId; + + std::string _remoteIp; + int _remotePort; + + friend class SocketServer; }; } // namespace ix diff --git a/ixwebsocket/IXDNSLookup.cpp b/ixwebsocket/IXDNSLookup.cpp index 09618704..9a2874f1 100644 --- a/ixwebsocket/IXDNSLookup.cpp +++ b/ixwebsocket/IXDNSLookup.cpp @@ -24,6 +24,12 @@ #include #include +// mingw build quirks +#if defined(_WIN32) && defined(__GNUC__) +#define AI_NUMERICSERV NI_NUMERICSERV +#define AI_ADDRCONFIG LUP_ADDRCONFIG +#endif + namespace ix { const int64_t DNSLookup::kDefaultWait = 1; // ms @@ -68,6 +74,11 @@ namespace ix : resolveUnCancellable(errMsg, isCancellationRequested); } + void DNSLookup::release(struct addrinfo* addr) + { + freeaddrinfo(addr); + } + struct addrinfo* DNSLookup::resolveUnCancellable( std::string& errMsg, const CancellationRequest& isCancellationRequested) { diff --git a/ixwebsocket/IXDNSLookup.h b/ixwebsocket/IXDNSLookup.h index 754da263..fcdd103d 100644 --- a/ixwebsocket/IXDNSLookup.h +++ b/ixwebsocket/IXDNSLookup.h @@ -31,6 +31,8 @@ namespace ix const CancellationRequest& isCancellationRequested, bool cancellable = true); + void release(struct addrinfo* addr); + private: struct addrinfo* resolveCancellable(std::string& errMsg, const CancellationRequest& isCancellationRequested); diff --git a/ixwebsocket/IXExponentialBackoff.cpp b/ixwebsocket/IXExponentialBackoff.cpp index 0cdb3fc9..1bb57ee5 100644 --- a/ixwebsocket/IXExponentialBackoff.cpp +++ b/ixwebsocket/IXExponentialBackoff.cpp @@ -1,5 +1,5 @@ /* - * IXExponentialBackoff.h + * IXExponentialBackoff.cpp * Author: Benjamin Sergeant * Copyright (c) 2017-2019 Machine Zone, Inc. All rights reserved. */ @@ -10,16 +10,22 @@ namespace ix { - uint32_t calculateRetryWaitMilliseconds(uint32_t retry_count, - uint32_t maxWaitBetweenReconnectionRetries) + uint32_t calculateRetryWaitMilliseconds(uint32_t retryCount, + uint32_t maxWaitBetweenReconnectionRetries, + uint32_t minWaitBetweenReconnectionRetries) { - uint32_t wait_time = (retry_count < 26) ? (std::pow(2, retry_count) * 100) : 0; + uint32_t waitTime = (retryCount < 26) ? (std::pow(2, retryCount) * 100) : 0; - if (wait_time > maxWaitBetweenReconnectionRetries || wait_time == 0) + if (waitTime < minWaitBetweenReconnectionRetries) { - wait_time = maxWaitBetweenReconnectionRetries; + waitTime = minWaitBetweenReconnectionRetries; } - return wait_time; + if (waitTime > maxWaitBetweenReconnectionRetries || waitTime == 0) + { + waitTime = maxWaitBetweenReconnectionRetries; + } + + return waitTime; } } // namespace ix diff --git a/ixwebsocket/IXExponentialBackoff.h b/ixwebsocket/IXExponentialBackoff.h index 100f6a60..79e19e9f 100644 --- a/ixwebsocket/IXExponentialBackoff.h +++ b/ixwebsocket/IXExponentialBackoff.h @@ -10,6 +10,7 @@ namespace ix { - uint32_t calculateRetryWaitMilliseconds(uint32_t retry_count, - uint32_t maxWaitBetweenReconnectionRetries); + uint32_t calculateRetryWaitMilliseconds(uint32_t retryCount, + uint32_t maxWaitBetweenReconnectionRetries, + uint32_t minWaitBetweenReconnectionRetries); } // namespace ix diff --git a/test/IXGetFreePort.cpp b/ixwebsocket/IXGetFreePort.cpp similarity index 98% rename from test/IXGetFreePort.cpp rename to ixwebsocket/IXGetFreePort.cpp index 3f806ddd..e69e8d3d 100644 --- a/test/IXGetFreePort.cpp +++ b/ixwebsocket/IXGetFreePort.cpp @@ -31,7 +31,7 @@ namespace ix int getAnyFreePort() { - int sockfd; + socket_t sockfd; if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { return getAnyFreePortRandom(); diff --git a/test/IXGetFreePort.h b/ixwebsocket/IXGetFreePort.h similarity index 100% rename from test/IXGetFreePort.h rename to ixwebsocket/IXGetFreePort.h diff --git a/ixwebsocket/IXGzipCodec.cpp b/ixwebsocket/IXGzipCodec.cpp new file mode 100644 index 00000000..fe59bd65 --- /dev/null +++ b/ixwebsocket/IXGzipCodec.cpp @@ -0,0 +1,183 @@ +/* + * IXGzipCodec.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + */ + +#include "IXGzipCodec.h" + +#include "IXBench.h" +#include +#include + +#ifdef IXWEBSOCKET_USE_ZLIB +#include +#endif + +#ifdef IXWEBSOCKET_USE_DEFLATE +#include +#endif + +namespace ix +{ + std::string gzipCompress(const std::string& str) + { +#ifndef IXWEBSOCKET_USE_ZLIB + return std::string(); +#else +#ifdef IXWEBSOCKET_USE_DEFLATE + int compressionLevel = 6; + struct libdeflate_compressor* compressor; + + compressor = libdeflate_alloc_compressor(compressionLevel); + + const void* uncompressed_data = str.data(); + size_t uncompressed_size = str.size(); + void* compressed_data; + size_t actual_compressed_size; + size_t max_compressed_size; + + max_compressed_size = libdeflate_gzip_compress_bound(compressor, uncompressed_size); + compressed_data = malloc(max_compressed_size); + + if (compressed_data == NULL) + { + return std::string(); + } + + actual_compressed_size = libdeflate_gzip_compress( + compressor, uncompressed_data, uncompressed_size, compressed_data, max_compressed_size); + + libdeflate_free_compressor(compressor); + + if (actual_compressed_size == 0) + { + free(compressed_data); + return std::string(); + } + + std::string out; + out.assign(reinterpret_cast(compressed_data), actual_compressed_size); + free(compressed_data); + + return out; +#else + z_stream zs; // z_stream is zlib's control structure + memset(&zs, 0, sizeof(zs)); + + // deflateInit2 configure the file format: request gzip instead of deflate + const int windowBits = 15; + const int GZIP_ENCODING = 16; + + deflateInit2(&zs, + Z_DEFAULT_COMPRESSION, + Z_DEFLATED, + windowBits | GZIP_ENCODING, + 8, + Z_DEFAULT_STRATEGY); + + zs.next_in = (Bytef*) str.data(); + zs.avail_in = (uInt) str.size(); // set the z_stream's input + + int ret; + char outbuffer[32768]; + std::string outstring; + + // retrieve the compressed bytes blockwise + do + { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = deflate(&zs, Z_FINISH); + + if (outstring.size() < zs.total_out) + { + // append the block to the output string + outstring.append(outbuffer, zs.total_out - outstring.size()); + } + } while (ret == Z_OK); + + deflateEnd(&zs); + + return outstring; +#endif // IXWEBSOCKET_USE_DEFLATE +#endif // IXWEBSOCKET_USE_ZLIB + } + +#ifdef IXWEBSOCKET_USE_DEFLATE + static uint32_t loadDecompressedGzipSize(const uint8_t* p) + { + return ((uint32_t) p[0] << 0) | ((uint32_t) p[1] << 8) | ((uint32_t) p[2] << 16) | + ((uint32_t) p[3] << 24); + } +#endif + + bool gzipDecompress(const std::string& in, std::string& out) + { +#ifndef IXWEBSOCKET_USE_ZLIB + return false; +#else +#ifdef IXWEBSOCKET_USE_DEFLATE + struct libdeflate_decompressor* decompressor; + decompressor = libdeflate_alloc_decompressor(); + + const void* compressed_data = in.data(); + size_t compressed_size = in.size(); + + // Retrieve uncompressed size from the trailer of the gziped data + const uint8_t* ptr = reinterpret_cast(&in.front()); + auto uncompressed_size = loadDecompressedGzipSize(&ptr[compressed_size - 4]); + + // Use it to redimension our output buffer + out.resize(uncompressed_size); + + libdeflate_result result = libdeflate_gzip_decompress( + decompressor, compressed_data, compressed_size, &out.front(), uncompressed_size, NULL); + + libdeflate_free_decompressor(decompressor); + return result == LIBDEFLATE_SUCCESS; +#else + z_stream inflateState; + memset(&inflateState, 0, sizeof(inflateState)); + + inflateState.zalloc = Z_NULL; + inflateState.zfree = Z_NULL; + inflateState.opaque = Z_NULL; + inflateState.avail_in = 0; + inflateState.next_in = Z_NULL; + + if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK) + { + return false; + } + + inflateState.avail_in = (uInt) in.size(); + inflateState.next_in = (unsigned char*) (const_cast(in.data())); + + const int kBufferSize = 1 << 14; + std::array compressBuffer; + + do + { + inflateState.avail_out = (uInt) kBufferSize; + inflateState.next_out = &compressBuffer.front(); + + int ret = inflate(&inflateState, Z_SYNC_FLUSH); + + if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) + { + inflateEnd(&inflateState); + return false; + } + + out.append(reinterpret_cast(&compressBuffer.front()), + kBufferSize - inflateState.avail_out); + } while (inflateState.avail_out == 0); + + inflateEnd(&inflateState); + return true; +#endif // IXWEBSOCKET_USE_DEFLATE +#endif // IXWEBSOCKET_USE_ZLIB + } +} // namespace ix diff --git a/ixwebsocket/IXGzipCodec.h b/ixwebsocket/IXGzipCodec.h new file mode 100644 index 00000000..8a5fc113 --- /dev/null +++ b/ixwebsocket/IXGzipCodec.h @@ -0,0 +1,15 @@ +/* + * IXGzipCodec.h + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + */ + +#pragma once + +#include + +namespace ix +{ + std::string gzipCompress(const std::string& str); + bool gzipDecompress(const std::string& in, std::string& out); +} // namespace ix diff --git a/ixwebsocket/IXHttp.cpp b/ixwebsocket/IXHttp.cpp index 0f858ac2..56a466c0 100644 --- a/ixwebsocket/IXHttp.cpp +++ b/ixwebsocket/IXHttp.cpp @@ -7,6 +7,7 @@ #include "IXHttp.h" #include "IXCancellationRequest.h" +#include "IXGzipCodec.h" #include "IXSocket.h" #include #include @@ -93,14 +94,12 @@ namespace ix } std::tuple Http::parseRequest( - std::unique_ptr& socket) + std::unique_ptr& socket, int timeoutSecs) { HttpRequestPtr httpRequest; std::atomic requestInitCancellation(false); - int timeoutSecs = 5; // FIXME - auto isCancellationRequested = makeCancellationRequestWithTimeout(timeoutSecs, requestInitCancellation); @@ -130,7 +129,53 @@ namespace ix return std::make_tuple(false, "Error parsing HTTP headers", httpRequest); } - httpRequest = std::make_shared(uri, method, httpVersion, headers); + std::string body; + if (headers.find("Content-Length") != headers.end()) + { + int contentLength = 0; + try + { + contentLength = std::stoi(headers["Content-Length"]); + } + catch (const std::exception&) + { + return std::make_tuple( + false, "Error parsing HTTP Header 'Content-Length'", httpRequest); + } + + if (contentLength < 0) + { + return std::make_tuple( + false, "Error: 'Content-Length' should be a positive integer", httpRequest); + } + + auto res = socket->readBytes(contentLength, nullptr, isCancellationRequested); + if (!res.first) + { + return std::make_tuple( + false, std::string("Error reading request: ") + res.second, httpRequest); + } + body = res.second; + } + + // If the content was compressed with gzip, decode it + if (headers["Content-Encoding"] == "gzip") + { +#ifdef IXWEBSOCKET_USE_ZLIB + std::string decompressedPayload; + if (!gzipDecompress(body, decompressedPayload)) + { + return std::make_tuple( + false, std::string("Error during gzip decompression of the body"), httpRequest); + } + body = decompressedPayload; +#else + std::string errorMsg("ixwebsocket was not compiled with gzip support on"); + return std::make_tuple(false, errorMsg, httpRequest); +#endif + } + + httpRequest = std::make_shared(uri, method, httpVersion, body, headers); return std::make_tuple(true, "", httpRequest); } @@ -151,7 +196,7 @@ namespace ix // Write headers ss.str(""); - ss << "Content-Length: " << response->payload.size() << "\r\n"; + ss << "Content-Length: " << response->body.size() << "\r\n"; for (auto&& it : response->headers) { ss << it.first << ": " << it.second << "\r\n"; @@ -163,6 +208,6 @@ namespace ix return false; } - return response->payload.empty() ? true : socket->writeBytes(response->payload, nullptr); + return response->body.empty() ? true : socket->writeBytes(response->body, nullptr); } } // namespace ix diff --git a/ixwebsocket/IXHttp.h b/ixwebsocket/IXHttp.h index 71a0db69..bfdaefcc 100644 --- a/ixwebsocket/IXHttp.h +++ b/ixwebsocket/IXHttp.h @@ -39,7 +39,7 @@ namespace ix std::string description; HttpErrorCode errorCode; WebSocketHttpHeaders headers; - std::string payload; + std::string body; std::string errorMsg; uint64_t uploadSize; uint64_t downloadSize; @@ -48,7 +48,7 @@ namespace ix const std::string& des = std::string(), const HttpErrorCode& c = HttpErrorCode::Ok, const WebSocketHttpHeaders& h = WebSocketHttpHeaders(), - const std::string& p = std::string(), + const std::string& b = std::string(), const std::string& e = std::string(), uint64_t u = 0, uint64_t d = 0) @@ -56,7 +56,7 @@ namespace ix , description(des) , errorCode(c) , headers(h) - , payload(p) + , body(b) , errorMsg(e) , uploadSize(u) , downloadSize(d) @@ -84,6 +84,7 @@ namespace ix int maxRedirects = 5; bool verbose = false; bool compress = true; + bool compressRequest = false; Logger logger; OnProgressCallback onProgressCallback; }; @@ -95,15 +96,18 @@ namespace ix std::string uri; std::string method; std::string version; + std::string body; WebSocketHttpHeaders headers; HttpRequest(const std::string& u, const std::string& m, const std::string& v, + const std::string& b, const WebSocketHttpHeaders& h = WebSocketHttpHeaders()) : uri(u) , method(m) , version(v) + , body(b) , headers(h) { } @@ -115,7 +119,7 @@ namespace ix { public: static std::tuple parseRequest( - std::unique_ptr& socket); + std::unique_ptr& socket, int timeoutSecs); static bool sendResponse(HttpResponsePtr response, std::unique_ptr& socket); static std::pair parseStatusLine(const std::string& line); diff --git a/ixwebsocket/IXHttpClient.cpp b/ixwebsocket/IXHttpClient.cpp index ef8da7da..3be8b8b7 100644 --- a/ixwebsocket/IXHttpClient.cpp +++ b/ixwebsocket/IXHttpClient.cpp @@ -6,6 +6,7 @@ #include "IXHttpClient.h" +#include "IXGzipCodec.h" #include "IXSocketFactory.h" #include "IXUrlParser.h" #include "IXUserAgent.h" @@ -16,14 +17,14 @@ #include #include #include -#include namespace ix { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods const std::string HttpClient::kPost = "POST"; const std::string HttpClient::kGet = "GET"; const std::string HttpClient::kHead = "HEAD"; - const std::string HttpClient::kDel = "DEL"; + const std::string HttpClient::kDelete = "DELETE"; const std::string HttpClient::kPut = "PUT"; const std::string HttpClient::kPatch = "PATCH"; @@ -127,7 +128,7 @@ namespace ix { // We only have one socket connection, so we cannot // make multiple requests concurrently. - std::lock_guard lock(_mutex); + std::lock_guard lock(_mutex); uint64_t uploadSize = 0; uint64_t downloadSize = 0; @@ -174,11 +175,13 @@ namespace ix ss << verb << " " << path << " HTTP/1.1\r\n"; ss << "Host: " << host << "\r\n"; +#ifdef IXWEBSOCKET_USE_ZLIB if (args->compress) { ss << "Accept-Encoding: gzip" << "\r\n"; } +#endif // Append extra headers for (auto&& it : args->extraHeaders) @@ -187,20 +190,29 @@ namespace ix } // Set a default Accept header if none is present - if (headers.find("Accept") == headers.end()) + if (args->extraHeaders.find("Accept") == args->extraHeaders.end()) { ss << "Accept: */*" << "\r\n"; } // Set a default User agent if none is present - if (headers.find("User-Agent") == headers.end()) + if (args->extraHeaders.find("User-Agent") == args->extraHeaders.end()) { ss << "User-Agent: " << userAgent() << "\r\n"; } if (verb == kPost || verb == kPut || verb == kPatch || _forceBody) { + // Set request compression header +#ifdef IXWEBSOCKET_USE_ZLIB + if (args->compressRequest) + { + ss << "Content-Encoding: gzip" + << "\r\n"; + } +#endif + ss << "Content-Length: " << body.size() << "\r\n"; // Set default Content-Type if unspecified @@ -498,8 +510,9 @@ namespace ix // If the content was compressed with gzip, decode it if (headers["Content-Encoding"] == "gzip") { +#ifdef IXWEBSOCKET_USE_ZLIB std::string decompressedPayload; - if (!gzipInflate(payload, decompressedPayload)) + if (!gzipDecompress(payload, decompressedPayload)) { std::string errorMsg("Error decompressing payload"); return std::make_shared(code, @@ -512,6 +525,17 @@ namespace ix downloadSize); } payload = decompressedPayload; +#else + std::string errorMsg("ixwebsocket was not compiled with gzip support on"); + return std::make_shared(code, + description, + HttpErrorCode::Gzip, + headers, + payload, + errorMsg, + uploadSize, + downloadSize); +#endif } return std::make_shared(code, @@ -534,16 +558,47 @@ namespace ix return request(url, kHead, std::string(), args); } - HttpResponsePtr HttpClient::del(const std::string& url, HttpRequestArgsPtr args) + HttpResponsePtr HttpClient::Delete(const std::string& url, HttpRequestArgsPtr args) { - return request(url, kDel, std::string(), args); + return request(url, kDelete, std::string(), args); + } + + HttpResponsePtr HttpClient::request(const std::string& url, + const std::string& verb, + const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, + HttpRequestArgsPtr args) + { + std::string body; + + if (httpFormDataParameters.empty()) + { + body = serializeHttpParameters(httpParameters); + } + else + { + std::string multipartBoundary = generateMultipartBoundary(); + args->multipartBoundary = multipartBoundary; + body = serializeHttpFormDataParameters( + multipartBoundary, httpFormDataParameters, httpParameters); + } + +#ifdef IXWEBSOCKET_USE_ZLIB + if (args->compressRequest) + { + body = gzipCompress(body); + } +#endif + + return request(url, verb, body, args); } HttpResponsePtr HttpClient::post(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - return request(url, kPost, serializeHttpParameters(httpParameters), args); + return request(url, kPost, httpParameters, httpFormDataParameters, args); } HttpResponsePtr HttpClient::post(const std::string& url, @@ -555,9 +610,10 @@ namespace ix HttpResponsePtr HttpClient::put(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - return request(url, kPut, serializeHttpParameters(httpParameters), args); + return request(url, kPut, httpParameters, httpFormDataParameters, args); } HttpResponsePtr HttpClient::put(const std::string& url, @@ -569,9 +625,10 @@ namespace ix HttpResponsePtr HttpClient::patch(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args) { - return request(url, kPatch, serializeHttpParameters(httpParameters), args); + return request(url, kPatch, httpParameters, httpFormDataParameters, args); } HttpResponsePtr HttpClient::patch(const std::string& url, @@ -672,51 +729,6 @@ namespace ix return ss.str(); } - bool HttpClient::gzipInflate(const std::string& in, std::string& out) - { - z_stream inflateState; - std::memset(&inflateState, 0, sizeof(inflateState)); - - inflateState.zalloc = Z_NULL; - inflateState.zfree = Z_NULL; - inflateState.opaque = Z_NULL; - inflateState.avail_in = 0; - inflateState.next_in = Z_NULL; - - if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK) - { - return false; - } - - inflateState.avail_in = (uInt) in.size(); - inflateState.next_in = (unsigned char*) (const_cast(in.data())); - - const int kBufferSize = 1 << 14; - - std::unique_ptr compressBuffer = - std::make_unique(kBufferSize); - - do - { - inflateState.avail_out = (uInt) kBufferSize; - inflateState.next_out = compressBuffer.get(); - - int ret = inflate(&inflateState, Z_SYNC_FLUSH); - - if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) - { - inflateEnd(&inflateState); - return false; - } - - out.append(reinterpret_cast(compressBuffer.get()), - kBufferSize - inflateState.avail_out); - } while (inflateState.avail_out == 0); - - inflateEnd(&inflateState); - return true; - } - void HttpClient::log(const std::string& msg, HttpRequestArgsPtr args) { if (args->logger) diff --git a/ixwebsocket/IXHttpClient.h b/ixwebsocket/IXHttpClient.h index f591b1e9..c4b05845 100644 --- a/ixwebsocket/IXHttpClient.h +++ b/ixwebsocket/IXHttpClient.h @@ -30,10 +30,11 @@ namespace ix HttpResponsePtr get(const std::string& url, HttpRequestArgsPtr args); HttpResponsePtr head(const std::string& url, HttpRequestArgsPtr args); - HttpResponsePtr del(const std::string& url, HttpRequestArgsPtr args); + HttpResponsePtr Delete(const std::string& url, HttpRequestArgsPtr args); HttpResponsePtr post(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args); HttpResponsePtr post(const std::string& url, const std::string& body, @@ -41,6 +42,7 @@ namespace ix HttpResponsePtr put(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args); HttpResponsePtr put(const std::string& url, const std::string& body, @@ -48,6 +50,7 @@ namespace ix HttpResponsePtr patch(const std::string& url, const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, HttpRequestArgsPtr args); HttpResponsePtr patch(const std::string& url, const std::string& body, @@ -58,7 +61,15 @@ namespace ix const std::string& body, HttpRequestArgsPtr args, int redirects = 0); + + HttpResponsePtr request(const std::string& url, + const std::string& verb, + const HttpParameters& httpParameters, + const HttpFormDataParameters& httpFormDataParameters, + HttpRequestArgsPtr args); + void setForceBody(bool value); + // Async API HttpRequestArgsPtr createRequest(const std::string& url = std::string(), const std::string& verb = HttpClient::kGet); @@ -83,15 +94,13 @@ namespace ix const static std::string kPost; const static std::string kGet; const static std::string kHead; - const static std::string kDel; + const static std::string kDelete; const static std::string kPut; const static std::string kPatch; private: void log(const std::string& msg, HttpRequestArgsPtr args); - bool gzipInflate(const std::string& in, std::string& out); - // Async API background thread runner void run(); // Async API @@ -103,7 +112,9 @@ namespace ix std::thread _thread; std::unique_ptr _socket; - std::mutex _mutex; // to protect accessing the _socket (only one socket per client) + std::recursive_mutex _mutex; // to protect accessing the _socket (only one socket per + // client) the mutex needs to be recursive as this function + // might be called recursively to follow HTTP redirections SocketTLSOptions _tlsOptions; diff --git a/ixwebsocket/IXHttpServer.cpp b/ixwebsocket/IXHttpServer.cpp index d4cf951f..563dca91 100644 --- a/ixwebsocket/IXHttpServer.cpp +++ b/ixwebsocket/IXHttpServer.cpp @@ -6,9 +6,11 @@ #include "IXHttpServer.h" +#include "IXGzipCodec.h" #include "IXNetSystem.h" #include "IXSocketConnect.h" #include "IXUserAgent.h" +#include #include #include #include @@ -42,10 +44,17 @@ namespace namespace ix { - HttpServer::HttpServer( - int port, const std::string& host, int backlog, size_t maxConnections, int addressFamily) + const int HttpServer::kDefaultTimeoutSecs(30); + + HttpServer::HttpServer(int port, + const std::string& host, + int backlog, + size_t maxConnections, + int addressFamily, + int timeoutSecs) : SocketServer(port, host, backlog, maxConnections, addressFamily) , _connectedClientsCount(0) + , _timeoutSecs(timeoutSecs) { setDefaultConnectionCallback(); } @@ -74,7 +83,7 @@ namespace ix { _connectedClientsCount++; - auto ret = Http::parseRequest(socket); + auto ret = Http::parseRequest(socket, _timeoutSecs); // FIXME: handle errors in parseRequest if (std::get<0>(ret)) @@ -99,7 +108,7 @@ namespace ix { setOnConnectionCallback( [this](HttpRequestPtr request, - std::shared_ptr /*connectionState*/) -> HttpResponsePtr { + std::shared_ptr connectionState) -> HttpResponsePtr { std::string uri(request->uri); if (uri.empty() || uri == "/") { @@ -120,9 +129,19 @@ namespace ix std::string content = res.second; +#ifdef IXWEBSOCKET_USE_ZLIB + std::string acceptEncoding = request->headers["Accept-encoding"]; + if (acceptEncoding == "*" || acceptEncoding.find("gzip") != std::string::npos) + { + content = gzipCompress(content); + headers["Content-Encoding"] = "gzip"; + } +#endif + // Log request std::stringstream ss; - ss << request->method << " " << request->headers["User-Agent"] << " " + ss << connectionState->getRemoteIp() << ":" << connectionState->getRemotePort() + << " " << request->method << " " << request->headers["User-Agent"] << " " << request->uri << " " << content.size(); logInfo(ss.str()); @@ -148,13 +167,14 @@ namespace ix setOnConnectionCallback( [this, redirectUrl](HttpRequestPtr request, - std::shared_ptr /*connectionState*/) -> HttpResponsePtr { + std::shared_ptr connectionState) -> HttpResponsePtr { WebSocketHttpHeaders headers; headers["Server"] = userAgent(); // Log request std::stringstream ss; - ss << request->method << " " << request->headers["User-Agent"] << " " + ss << connectionState->getRemoteIp() << ":" << connectionState->getRemotePort() + << " " << request->method << " " << request->headers["User-Agent"] << " " << request->uri; logInfo(ss.str()); @@ -170,4 +190,46 @@ namespace ix 301, "OK", HttpErrorCode::Ok, headers, std::string()); }); } + + // + // Display the client parameter and body on the console + // + void HttpServer::makeDebugServer() + { + setOnConnectionCallback( + [this](HttpRequestPtr request, + std::shared_ptr connectionState) -> HttpResponsePtr { + WebSocketHttpHeaders headers; + headers["Server"] = userAgent(); + + // Log request + std::stringstream ss; + ss << connectionState->getRemoteIp() << ":" << connectionState->getRemotePort() + << " " << request->method << " " << request->headers["User-Agent"] << " " + << request->uri; + logInfo(ss.str()); + + logInfo("== Headers == "); + for (auto&& it : request->headers) + { + std::ostringstream oss; + oss << it.first << ": " << it.second; + logInfo(oss.str()); + } + logInfo(""); + + logInfo("== Body == "); + logInfo(request->body); + logInfo(""); + + return std::make_shared( + 200, "OK", HttpErrorCode::Ok, headers, std::string("OK")); + }); + } + + int HttpServer::getTimeoutSecs() + { + return _timeoutSecs; + } + } // namespace ix diff --git a/ixwebsocket/IXHttpServer.h b/ixwebsocket/IXHttpServer.h index 96b22d3f..7de67631 100644 --- a/ixwebsocket/IXHttpServer.h +++ b/ixwebsocket/IXHttpServer.h @@ -29,7 +29,8 @@ namespace ix const std::string& host = SocketServer::kDefaultHost, int backlog = SocketServer::kDefaultTcpBacklog, size_t maxConnections = SocketServer::kDefaultMaxConnections, - int addressFamily = SocketServer::kDefaultAddressFamily); + int addressFamily = SocketServer::kDefaultAddressFamily, + int timeoutSecs = HttpServer::kDefaultTimeoutSecs); virtual ~HttpServer(); virtual void stop() final; @@ -37,11 +38,17 @@ namespace ix void makeRedirectServer(const std::string& redirectUrl); + void makeDebugServer(); + + int getTimeoutSecs(); private: // Member variables OnConnectionCallback _onConnectionCallback; std::atomic _connectedClientsCount; + const static int kDefaultTimeoutSecs; + int _timeoutSecs; + // Methods virtual void handleConnection(std::unique_ptr, std::shared_ptr connectionState) final; diff --git a/ixwebsocket/IXNetSystem.cpp b/ixwebsocket/IXNetSystem.cpp index 195d1294..ab7b2a38 100644 --- a/ixwebsocket/IXNetSystem.cpp +++ b/ixwebsocket/IXNetSystem.cpp @@ -5,6 +5,8 @@ */ #include "IXNetSystem.h" +#include +#include namespace ix { @@ -45,7 +47,7 @@ namespace ix int poll(struct pollfd* fds, nfds_t nfds, int timeout) { #ifdef _WIN32 - int maxfd = 0; + socket_t maxfd = 0; fd_set readfds, writefds, errorfds; FD_ZERO(&readfds); FD_ZERO(&writefds); @@ -105,8 +107,190 @@ namespace ix return ret; #else - return ::poll(fds, nfds, timeout); + // + // It was reported that on Android poll can fail and return -1 with + // errno == EINTR, which should be a temp error and should typically + // be handled by retrying in a loop. + // Maybe we need to put all syscall / C functions in + // a new IXSysCalls.cpp and wrap them all. + // + // The style from libuv is as such. + // + int ret = -1; + do + { + ret = ::poll(fds, nfds, timeout); + } while (ret == -1 && errno == EINTR); + + return ret; #endif } + // + // mingw does not have inet_ntop, which were taken as is from the musl C library. + // + const char* inet_ntop(int af, const void* a0, char* s, socklen_t l) + { +#if defined(_WIN32) && defined(__GNUC__) + const unsigned char* a = (const unsigned char*) a0; + int i, j, max, best; + char buf[100]; + + switch (af) + { + case AF_INET: + if (snprintf(s, l, "%d.%d.%d.%d", a[0], a[1], a[2], a[3]) < l) return s; + break; + case AF_INET6: + if (memcmp(a, "\0\0\0\0\0\0\0\0\0\0\377\377", 12)) + snprintf(buf, + sizeof buf, + "%x:%x:%x:%x:%x:%x:%x:%x", + 256 * a[0] + a[1], + 256 * a[2] + a[3], + 256 * a[4] + a[5], + 256 * a[6] + a[7], + 256 * a[8] + a[9], + 256 * a[10] + a[11], + 256 * a[12] + a[13], + 256 * a[14] + a[15]); + else + snprintf(buf, + sizeof buf, + "%x:%x:%x:%x:%x:%x:%d.%d.%d.%d", + 256 * a[0] + a[1], + 256 * a[2] + a[3], + 256 * a[4] + a[5], + 256 * a[6] + a[7], + 256 * a[8] + a[9], + 256 * a[10] + a[11], + a[12], + a[13], + a[14], + a[15]); + /* Replace longest /(^0|:)[:0]{2,}/ with "::" */ + for (i = best = 0, max = 2; buf[i]; i++) + { + if (i && buf[i] != ':') continue; + j = strspn(buf + i, ":0"); + if (j > max) best = i, max = j; + } + if (max > 3) + { + buf[best] = buf[best + 1] = ':'; + memmove(buf + best + 2, buf + best + max, i - best - max + 1); + } + if (strlen(buf) < l) + { + strcpy(s, buf); + return s; + } + break; + default: errno = EAFNOSUPPORT; return 0; + } + errno = ENOSPC; + return 0; +#else + return ::inet_ntop(af, a0, s, l); +#endif + } + +#if defined(_WIN32) && defined(__GNUC__) + static int hexval(unsigned c) + { + if (c - '0' < 10) return c - '0'; + c |= 32; + if (c - 'a' < 6) return c - 'a' + 10; + return -1; + } +#endif + + // + // mingw does not have inet_pton, which were taken as is from the musl C library. + // + int inet_pton(int af, const char* s, void* a0) + { +#if defined(_WIN32) && defined(__GNUC__) + uint16_t ip[8]; + unsigned char* a = (unsigned char*) a0; + int i, j, v, d, brk = -1, need_v4 = 0; + + if (af == AF_INET) + { + for (i = 0; i < 4; i++) + { + for (v = j = 0; j < 3 && isdigit(s[j]); j++) + v = 10 * v + s[j] - '0'; + if (j == 0 || (j > 1 && s[0] == '0') || v > 255) return 0; + a[i] = v; + if (s[j] == 0 && i == 3) return 1; + if (s[j] != '.') return 0; + s += j + 1; + } + return 0; + } + else if (af != AF_INET6) + { + errno = EAFNOSUPPORT; + return -1; + } + + if (*s == ':' && *++s != ':') return 0; + + for (i = 0;; i++) + { + if (s[0] == ':' && brk < 0) + { + brk = i; + ip[i & 7] = 0; + if (!*++s) break; + if (i == 7) return 0; + continue; + } + for (v = j = 0; j < 4 && (d = hexval(s[j])) >= 0; j++) + v = 16 * v + d; + if (j == 0) return 0; + ip[i & 7] = v; + if (!s[j] && (brk >= 0 || i == 7)) break; + if (i == 7) return 0; + if (s[j] != ':') + { + if (s[j] != '.' || (i < 6 && brk < 0)) return 0; + need_v4 = 1; + i++; + break; + } + s += j + 1; + } + if (brk >= 0) + { + memmove(ip + brk + 7 - i, ip + brk, 2 * (i + 1 - brk)); + for (j = 0; j < 7 - i; j++) + ip[brk + j] = 0; + } + for (j = 0; j < 8; j++) + { + *a++ = ip[j] >> 8; + *a++ = ip[j]; + } + if (need_v4 && inet_pton(AF_INET, (const char*) s, a - 4) <= 0) return 0; + return 1; +#else + return ::inet_pton(af, s, a0); +#endif + } + + // Convert network bytes to host bytes. Copied from the ASIO library + unsigned short network_to_host_short(unsigned short value) + { + #if defined(_WIN32) + unsigned char* value_p = reinterpret_cast(&value); + unsigned short result = (static_cast(value_p[0]) << 8) + | static_cast(value_p[1]); + return result; + #else // defined(_WIN32) + return ntohs(value); + #endif // defined(_WIN32) + } + } // namespace ix diff --git a/ixwebsocket/IXNetSystem.h b/ixwebsocket/IXNetSystem.h index 465f43de..b2220a95 100644 --- a/ixwebsocket/IXNetSystem.h +++ b/ixwebsocket/IXNetSystem.h @@ -7,15 +7,49 @@ #pragma once #ifdef _WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + #include #include #include #include #include +#undef EWOULDBLOCK +#undef EAGAIN +#undef EINPROGRESS +#undef EBADF +#undef EINVAL + +// map to WSA error codes +#define EWOULDBLOCK WSAEWOULDBLOCK +#define EAGAIN WSATRY_AGAIN +#define EINPROGRESS WSAEINPROGRESS +#define EBADF WSAEBADF +#define EINVAL WSAEINVAL + // Define our own poll on Windows, as a wrapper on top of select typedef unsigned long int nfds_t; +// pollfd is not defined by some versions of mingw64 since _WIN32_WINNT is too low +#if _WIN32_WINNT < 0x0600 +struct pollfd +{ + int fd; /* file descriptor */ + short events; /* requested events */ + short revents; /* returned events */ +}; + +#define POLLIN 0x001 /* There is data to read. */ +#define POLLOUT 0x004 /* Writing now will not block. */ +#define POLLERR 0x008 /* Error condition. */ +#define POLLHUP 0x010 /* Hung up. */ +#define POLLNVAL 0x020 /* Invalid polling request. */ +#endif + #else #include #include @@ -34,8 +68,19 @@ typedef unsigned long int nfds_t; namespace ix { +#ifdef _WIN32 + typedef SOCKET socket_t; +#else + typedef int socket_t; +#endif + bool initNetSystem(); bool uninitNetSystem(); int poll(struct pollfd* fds, nfds_t nfds, int timeout); + + const char* inet_ntop(int af, const void* src, char* dst, socklen_t size); + int inet_pton(int af, const char* src, void* dst); + + unsigned short network_to_host_short(unsigned short value); } // namespace ix diff --git a/ixwebsocket/IXSelectInterrupt.cpp b/ixwebsocket/IXSelectInterrupt.cpp index 3c7afd8a..36dc66c9 100644 --- a/ixwebsocket/IXSelectInterrupt.cpp +++ b/ixwebsocket/IXSelectInterrupt.cpp @@ -8,6 +8,9 @@ namespace ix { + const uint64_t SelectInterrupt::kSendRequest = 1; + const uint64_t SelectInterrupt::kCloseRequest = 2; + SelectInterrupt::SelectInterrupt() { ; diff --git a/ixwebsocket/IXSelectInterrupt.h b/ixwebsocket/IXSelectInterrupt.h index 6d03f32d..c3bb7f3f 100644 --- a/ixwebsocket/IXSelectInterrupt.h +++ b/ixwebsocket/IXSelectInterrupt.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include @@ -23,5 +24,11 @@ namespace ix virtual bool clear(); virtual uint64_t read(); virtual int getFd() const; + + // Used as special codes for pipe communication + static const uint64_t kSendRequest; + static const uint64_t kCloseRequest; }; + + using SelectInterruptPtr = std::unique_ptr; } // namespace ix diff --git a/ixwebsocket/IXSelectInterruptFactory.cpp b/ixwebsocket/IXSelectInterruptFactory.cpp index 0cd4a278..9018810d 100644 --- a/ixwebsocket/IXSelectInterruptFactory.cpp +++ b/ixwebsocket/IXSelectInterruptFactory.cpp @@ -6,6 +6,7 @@ #include "IXSelectInterruptFactory.h" +#include "IXUniquePtr.h" #if defined(__linux__) || defined(__APPLE__) #include "IXSelectInterruptPipe.h" #else @@ -17,9 +18,9 @@ namespace ix SelectInterruptPtr createSelectInterrupt() { #if defined(__linux__) || defined(__APPLE__) - return std::make_unique(); + return ix::make_unique(); #else - return std::make_unique(); + return ix::make_unique(); #endif } } // namespace ix diff --git a/ixwebsocket/IXSelectInterruptPipe.cpp b/ixwebsocket/IXSelectInterruptPipe.cpp index 6cf53d19..75c42f27 100644 --- a/ixwebsocket/IXSelectInterruptPipe.cpp +++ b/ixwebsocket/IXSelectInterruptPipe.cpp @@ -5,8 +5,10 @@ */ // -// On macOS we use UNIX pipes to wake up select. +// On UNIX we use pipes to wake up select. There is no way to do that +// on Windows so this file is compiled out on Windows. // +#ifndef _WIN32 #include "IXSelectInterruptPipe.h" @@ -115,8 +117,14 @@ namespace ix int fd = _fildes[kPipeWriteIndex]; if (fd == -1) return false; + ssize_t ret = -1; + do + { + ret = ::write(fd, &value, sizeof(value)); + } while (ret == -1 && errno == EINTR); + // we should write 8 bytes for an uint64_t - return write(fd, &value, sizeof(value)) == 8; + return ret == 8; } // TODO: return max uint64_t for errors ? @@ -127,7 +135,12 @@ namespace ix int fd = _fildes[kPipeReadIndex]; uint64_t value = 0; - ::read(fd, &value, sizeof(value)); + + ssize_t ret = -1; + do + { + ret = ::read(fd, &value, sizeof(value)); + } while (ret == -1 && errno == EINTR); return value; } @@ -144,3 +157,5 @@ namespace ix return _fildes[kPipeReadIndex]; } } // namespace ix + +#endif // !_WIN32 diff --git a/ixwebsocket/IXSetThreadName.cpp b/ixwebsocket/IXSetThreadName.cpp new file mode 100644 index 00000000..40faf9d9 --- /dev/null +++ b/ixwebsocket/IXSetThreadName.cpp @@ -0,0 +1,83 @@ +/* + * IXSetThreadName.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2018 2020 Machine Zone, Inc. All rights reserved. + */ +#include "IXSetThreadName.h" + +// unix systems +#if defined(__APPLE__) || defined(__linux__) || defined(BSD) +#include +#endif + +// freebsd needs this header as well +#if defined(BSD) +#include +#endif + +// Windows +#ifdef _WIN32 +#include +#endif + +namespace ix +{ +#ifdef _WIN32 + const DWORD MS_VC_EXCEPTION = 0x406D1388; + +#pragma pack(push, 8) + typedef struct tagTHREADNAME_INFO + { + DWORD dwType; // Must be 0x1000. + LPCSTR szName; // Pointer to name (in user addr space). + DWORD dwThreadID; // Thread ID (-1=caller thread). + DWORD dwFlags; // Reserved for future use, must be zero. + } THREADNAME_INFO; +#pragma pack(pop) + + void SetThreadName(DWORD dwThreadID, const char* threadName) + { +#ifndef __GNUC__ + THREADNAME_INFO info; + info.dwType = 0x1000; + info.szName = threadName; + info.dwThreadID = dwThreadID; + info.dwFlags = 0; + + __try + { + RaiseException( + MS_VC_EXCEPTION, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR*) &info); + } + __except (EXCEPTION_EXECUTE_HANDLER) + { + } +#endif + } +#endif + + void setThreadName(const std::string& name) + { +#if defined(__APPLE__) + // + // Apple reserves 16 bytes for its thread names + // Notice that the Apple version of pthread_setname_np + // does not take a pthread_t argument + // + pthread_setname_np(name.substr(0, 63).c_str()); +#elif defined(__linux__) + // + // Linux only reserves 16 bytes for its thread names + // See prctl and PR_SET_NAME property in + // http://man7.org/linux/man-pages/man2/prctl.2.html + // + pthread_setname_np(pthread_self(), name.substr(0, 15).c_str()); +#elif defined(_WIN32) + SetThreadName(-1, name.c_str()); +#elif defined(BSD) + pthread_set_name_np(pthread_self(), name.substr(0, 15).c_str()); +#else + // ... assert here ? +#endif + } +} // namespace ix diff --git a/ixwebsocket/IXSocket.cpp b/ixwebsocket/IXSocket.cpp index 9ef78bb7..bccfe7d9 100644 --- a/ixwebsocket/IXSocket.cpp +++ b/ixwebsocket/IXSocket.cpp @@ -11,6 +11,7 @@ #include "IXSelectInterruptFactory.h" #include "IXSocketConnect.h" #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #ifdef min #undef min @@ -27,9 +29,6 @@ namespace ix { const int Socket::kDefaultPollNoTimeout = -1; // No poll timeout by default const int Socket::kDefaultPollTimeout = kDefaultPollNoTimeout; - const uint64_t Socket::kSendRequest = 1; - const uint64_t Socket::kCloseRequest = 2; - constexpr size_t Socket::kChunkSize; Socket::Socket(int fd) : _sockfd(fd) @@ -96,11 +95,11 @@ namespace ix { uint64_t value = selectInterrupt->read(); - if (value == kSendRequest) + if (value == SelectInterrupt::kSendRequest) { pollResult = PollResultType::SendRequest; } - else if (value == kCloseRequest) + else if (value == SelectInterrupt::kCloseRequest) { pollResult = PollResultType::CloseRequest; } @@ -366,10 +365,7 @@ namespace ix const OnProgressCallback& onProgressCallback, const CancellationRequest& isCancellationRequested) { - if (_readBuffer.empty()) - { - _readBuffer.resize(kChunkSize); - } + std::array readBuffer; std::vector output; while (output.size() != length) @@ -380,12 +376,12 @@ namespace ix return std::make_pair(false, errorMsg); } - size_t size = std::min(kChunkSize, length - output.size()); - ssize_t ret = recv((char*) &_readBuffer[0], size); + size_t size = std::min(readBuffer.size(), length - output.size()); + ssize_t ret = recv((char*) &readBuffer[0], size); if (ret > 0) { - output.insert(output.end(), _readBuffer.begin(), _readBuffer.begin() + ret); + output.insert(output.end(), readBuffer.begin(), readBuffer.begin() + ret); } else if (ret <= 0 && !Socket::isWaitNeeded()) { diff --git a/ixwebsocket/IXSocket.h b/ixwebsocket/IXSocket.h index 84b0b737..5604c40d 100644 --- a/ixwebsocket/IXSocket.h +++ b/ixwebsocket/IXSocket.h @@ -11,35 +11,18 @@ #include #include #include -#include #ifdef _WIN32 #include typedef SSIZE_T ssize_t; - -#undef EWOULDBLOCK -#undef EAGAIN -#undef EINPROGRESS -#undef EBADF -#undef EINVAL - -// map to WSA error codes -#define EWOULDBLOCK WSAEWOULDBLOCK -#define EAGAIN WSATRY_AGAIN -#define EINPROGRESS WSAEINPROGRESS -#define EBADF WSAEBADF -#define EINVAL WSAEINVAL - #endif #include "IXCancellationRequest.h" #include "IXProgressCallback.h" +#include "IXSelectInterrupt.h" namespace ix { - class SelectInterrupt; - using SelectInterruptPtr = std::unique_ptr; - enum class PollResultType { ReadyForRead = 0, @@ -96,11 +79,6 @@ namespace ix int sockfd, const SelectInterruptPtr& selectInterrupt); - - // Used as special codes for pipe communication - static const uint64_t kSendRequest; - static const uint64_t kCloseRequest; - protected: std::atomic _sockfd; std::mutex _socketMutex; @@ -109,10 +87,6 @@ namespace ix static const int kDefaultPollTimeout; static const int kDefaultPollNoTimeout; - // Buffer for reading from our socket. That buffer is never resized. - std::vector _readBuffer; - static constexpr size_t kChunkSize = 1 << 15; - SelectInterruptPtr _selectInterrupt; }; } // namespace ix diff --git a/ixwebsocket/IXSocketConnect.cpp b/ixwebsocket/IXSocketConnect.cpp index fea01897..94ebc406 100644 --- a/ixwebsocket/IXSocketConnect.cpp +++ b/ixwebsocket/IXSocketConnect.cpp @@ -10,6 +10,7 @@ #include "IXNetSystem.h" #include "IXSelectInterrupt.h" #include "IXSocket.h" +#include "IXUniquePtr.h" #include #include #include @@ -34,7 +35,7 @@ namespace ix { errMsg = "no error"; - int fd = socket(address->ai_family, address->ai_socktype, address->ai_protocol); + socket_t fd = socket(address->ai_family, address->ai_socktype, address->ai_protocol); if (fd < 0) { errMsg = "Cannot create a socket"; @@ -65,7 +66,7 @@ namespace ix int timeoutMs = 10; bool readyToRead = false; - auto selectInterrupt = std::make_unique(); + auto selectInterrupt = ix::make_unique(); PollResultType pollResult = Socket::poll(readyToRead, timeoutMs, fd, selectInterrupt); if (pollResult == PollResultType::Timeout) diff --git a/ixwebsocket/IXSocketFactory.cpp b/ixwebsocket/IXSocketFactory.cpp index bf2d24fe..0273d683 100644 --- a/ixwebsocket/IXSocketFactory.cpp +++ b/ixwebsocket/IXSocketFactory.cpp @@ -6,6 +6,7 @@ #include "IXSocketFactory.h" +#include "IXUniquePtr.h" #ifdef IXWEBSOCKET_USE_TLS #ifdef IXWEBSOCKET_USE_MBED_TLS @@ -35,17 +36,17 @@ namespace ix if (!tls) { - socket = std::make_unique(fd); + socket = ix::make_unique(fd); } else { #ifdef IXWEBSOCKET_USE_TLS #if defined(IXWEBSOCKET_USE_MBED_TLS) - socket = std::make_unique(tlsOptions, fd); + socket = ix::make_unique(tlsOptions, fd); #elif defined(IXWEBSOCKET_USE_OPEN_SSL) - socket = std::make_unique(tlsOptions, fd); + socket = ix::make_unique(tlsOptions, fd); #elif defined(__APPLE__) - socket = std::make_unique(tlsOptions, fd); + socket = ix::make_unique(tlsOptions, fd); #endif #else errorMsg = "TLS support is not enabled on this platform."; diff --git a/ixwebsocket/IXSocketMbedTLS.cpp b/ixwebsocket/IXSocketMbedTLS.cpp index 22620d1a..01f8c870 100644 --- a/ixwebsocket/IXSocketMbedTLS.cpp +++ b/ixwebsocket/IXSocketMbedTLS.cpp @@ -16,6 +16,11 @@ #include "IXSocketConnect.h" #include +#ifdef _WIN32 +// For manipulating the certificate store +#include +#endif + namespace ix { SocketMbedTLS::SocketMbedTLS(const SocketTLSOptions& tlsOptions, int fd) @@ -127,7 +132,11 @@ namespace ix errMsg = "Cannot parse cert file '" + _tlsOptions.certFile + "'"; return false; } +#ifdef IXWEBSOCKET_USE_MBED_TLS_MIN_VERSION_3 + if (mbedtls_pk_parse_keyfile(&_pkey, _tlsOptions.keyFile.c_str(), "", mbedtls_ctr_drbg_random, &_ctr_drbg) < 0) +#else if (mbedtls_pk_parse_keyfile(&_pkey, _tlsOptions.keyFile.c_str(), "") < 0) +#endif { errMsg = "Cannot parse key file '" + _tlsOptions.keyFile + "'"; return false; diff --git a/ixwebsocket/IXSocketMbedTLS.h b/ixwebsocket/IXSocketMbedTLS.h index 032560b9..9dd73f50 100644 --- a/ixwebsocket/IXSocketMbedTLS.h +++ b/ixwebsocket/IXSocketMbedTLS.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ixwebsocket/IXSocketOpenSSL.cpp b/ixwebsocket/IXSocketOpenSSL.cpp index a504a5cb..c92477a1 100644 --- a/ixwebsocket/IXSocketOpenSSL.cpp +++ b/ixwebsocket/IXSocketOpenSSL.cpp @@ -10,8 +10,10 @@ #include "IXSocketOpenSSL.h" #include "IXSocketConnect.h" +#include "IXUniquePtr.h" #include #include +#include #ifdef _WIN32 #include #else @@ -22,6 +24,11 @@ #endif #define socketerrno errno +#ifdef _WIN32 +// For manipulating the certificate store +#include +#endif + #ifdef _WIN32 namespace { @@ -85,8 +92,7 @@ namespace ix std::atomic SocketOpenSSL::_openSSLInitializationSuccessful(false); std::once_flag SocketOpenSSL::_openSSLInitFlag; - std::unique_ptr SocketOpenSSL::_openSSLMutexes = - std::make_unique(CRYPTO_num_locks()); + std::vector> openSSLMutexes; SocketOpenSSL::SocketOpenSSL(const SocketTLSOptions& tlsOptions, int fd) : Socket(fd) @@ -111,6 +117,11 @@ namespace ix if (CRYPTO_get_locking_callback() == nullptr) { + openSSLMutexes.clear(); + for (int i = 0; i < CRYPTO_num_locks(); ++i) + { + openSSLMutexes.push_back(ix::make_unique()); + } CRYPTO_set_locking_callback(SocketOpenSSL::openSSLLockingCallback); } #endif @@ -128,11 +139,11 @@ namespace ix { if (mode & CRYPTO_LOCK) { - _openSSLMutexes[type].lock(); + openSSLMutexes[type]->lock(); } else { - _openSSLMutexes[type].unlock(); + openSSLMutexes[type]->unlock(); } } @@ -503,14 +514,13 @@ namespace ix errMsg += ERR_error_string(sslErr, nullptr); return false; } - - SSL_CTX_set_verify( - _ssl_context, SSL_VERIFY_PEER, [](int preverify, X509_STORE_CTX*) -> int { - return preverify; - }); - SSL_CTX_set_verify_depth(_ssl_context, 4); } } + + SSL_CTX_set_verify(_ssl_context, + SSL_VERIFY_PEER, + [](int preverify, X509_STORE_CTX*) -> int { return preverify; }); + SSL_CTX_set_verify_depth(_ssl_context, 4); } else { diff --git a/ixwebsocket/IXSocketOpenSSL.h b/ixwebsocket/IXSocketOpenSSL.h index 92d9935a..dea1ffd6 100644 --- a/ixwebsocket/IXSocketOpenSSL.h +++ b/ixwebsocket/IXSocketOpenSSL.h @@ -61,7 +61,6 @@ namespace ix static std::once_flag _openSSLInitFlag; static std::atomic _openSSLInitializationSuccessful; - static std::unique_ptr _openSSLMutexes; }; } // namespace ix diff --git a/ixwebsocket/IXSocketServer.cpp b/ixwebsocket/IXSocketServer.cpp index f1d34e43..ed04bb0c 100644 --- a/ixwebsocket/IXSocketServer.cpp +++ b/ixwebsocket/IXSocketServer.cpp @@ -8,6 +8,7 @@ #include "IXNetSystem.h" #include "IXSelectInterrupt.h" +#include "IXSelectInterruptFactory.h" #include "IXSetThreadName.h" #include "IXSocket.h" #include "IXSocketConnect.h" @@ -22,7 +23,7 @@ namespace ix const int SocketServer::kDefaultPort(8080); const std::string SocketServer::kDefaultHost("127.0.0.1"); const int SocketServer::kDefaultTcpBacklog(5); - const size_t SocketServer::kDefaultMaxConnections(32); + const size_t SocketServer::kDefaultMaxConnections(128); const int SocketServer::kDefaultAddressFamily(AF_INET); SocketServer::SocketServer( @@ -36,6 +37,7 @@ namespace ix , _stop(false) , _stopGc(false) , _connectionStateFactory(&ConnectionState::createConnectionState) + , _acceptSelectInterrupt(createSelectInterrupt()) { } @@ -58,6 +60,16 @@ namespace ix std::pair SocketServer::listen() { + std::string acceptSelectInterruptInitErrorMsg; + if (!_acceptSelectInterrupt->init(acceptSelectInterruptInitErrorMsg)) + { + std::stringstream ss; + ss << "SocketServer::listen() error in SelectInterrupt::init: " + << acceptSelectInterruptInitErrorMsg; + + return std::make_pair(false, ss.str()); + } + if (_addressFamily != AF_INET && _addressFamily != AF_INET6) { std::string errMsg("SocketServer::listen() AF_INET and AF_INET6 are currently " @@ -92,7 +104,7 @@ namespace ix server.sin_family = _addressFamily; server.sin_port = htons(_port); - if (inet_pton(_addressFamily, _host.c_str(), &server.sin_addr.s_addr) <= 0) + if (ix::inet_pton(_addressFamily, _host.c_str(), &server.sin_addr.s_addr) <= 0) { std::stringstream ss; ss << "SocketServer::listen() error calling inet_pton " @@ -121,7 +133,7 @@ namespace ix server.sin6_family = _addressFamily; server.sin6_port = htons(_port); - if (inet_pton(_addressFamily, _host.c_str(), &server.sin6_addr) <= 0) + if (ix::inet_pton(_addressFamily, _host.c_str(), &server.sin6_addr) <= 0) { std::stringstream ss; ss << "SocketServer::listen() error calling inet_pton " @@ -193,6 +205,12 @@ namespace ix if (_thread.joinable()) { _stop = true; + // Wake up select + if (!_acceptSelectInterrupt->notify(SelectInterrupt::kCloseRequest)) + { + logError("SocketServer::stop: Cannot wake up from select"); + } + _thread.join(); _stop = false; } @@ -201,6 +219,7 @@ namespace ix if (_gcThread.joinable()) { _stopGc = true; + _conditionVariableGC.notify_one(); _gcThread.join(); _stopGc = false; } @@ -249,18 +268,22 @@ namespace ix // Set the socket to non blocking mode, so that accept calls are not blocking SocketConnect::configure(_serverFd); - setThreadName("SocketServer::listen"); + setThreadName("SocketServer::accept"); for (;;) { if (_stop) return; // Use poll to check whether a new connection is in progress - int timeoutMs = 10; + int timeoutMs = -1; +#ifdef _WIN32 + // select cannot be interrupted on Windows so we need to pass a small timeout + timeoutMs = 10; +#endif + bool readyToRead = true; - auto selectInterrupt = std::make_unique(); PollResultType pollResult = - Socket::poll(readyToRead, timeoutMs, _serverFd, selectInterrupt); + Socket::poll(readyToRead, timeoutMs, _serverFd, _acceptSelectInterrupt); if (pollResult == PollResultType::Error) { @@ -276,6 +299,7 @@ namespace ix } // Accept a connection. + // FIXME: Is this working for ipv6 ? struct sockaddr_in client; // client address information int clientFd; // socket connected to client socklen_t addressLen = sizeof(client); @@ -307,11 +331,58 @@ namespace ix continue; } + // Retrieve connection info, the ip address of the remote peer/client) + std::string remoteIp; + int remotePort; + + if (_addressFamily == AF_INET) + { + char remoteIp4[INET_ADDRSTRLEN]; + if (ix::inet_ntop(AF_INET, &client.sin_addr, remoteIp4, INET_ADDRSTRLEN) == nullptr) + { + int err = Socket::getErrno(); + std::stringstream ss; + ss << "SocketServer::run() error calling inet_ntop (ipv4): " << err << ", " + << strerror(err); + logError(ss.str()); + + Socket::closeSocket(clientFd); + + continue; + } + + remotePort = ix::network_to_host_short(client.sin_port); + remoteIp = remoteIp4; + } + else // AF_INET6 + { + char remoteIp6[INET6_ADDRSTRLEN]; + if (ix::inet_ntop(AF_INET6, &client.sin_addr, remoteIp6, INET6_ADDRSTRLEN) == + nullptr) + { + int err = Socket::getErrno(); + std::stringstream ss; + ss << "SocketServer::run() error calling inet_ntop (ipv6): " << err << ", " + << strerror(err); + logError(ss.str()); + + Socket::closeSocket(clientFd); + + continue; + } + + remotePort = ix::network_to_host_short(client.sin_port); + remoteIp = remoteIp6; + } + std::shared_ptr connectionState; if (_connectionStateFactory) { connectionState = _connectionStateFactory(); } + connectionState->setOnSetTerminatedCallback([this] { onSetTerminatedCallback(); }); + connectionState->setRemoteIp(remoteIp); + connectionState->setRemotePort(remotePort); if (_stop) return; @@ -368,8 +439,14 @@ namespace ix break; } - // Sleep a little bit then keep cleaning up - std::this_thread::sleep_for(std::chrono::milliseconds(10)); + // Unless we are stopping the server, wait for a connection + // to be terminated to run the threads GC, instead of busy waiting + // with a sleep + if (!_stopGc) + { + std::unique_lock lock(_conditionVariableMutexGC); + _conditionVariableGC.wait(lock); + } } } @@ -377,4 +454,36 @@ namespace ix { _socketTLSOptions = socketTLSOptions; } + + void SocketServer::onSetTerminatedCallback() + { + // a connection got terminated, we can run the connection thread GC, + // so wake up the thread responsible for that + _conditionVariableGC.notify_one(); + } + + int SocketServer::getPort() + { + return _port; + } + + std::string SocketServer::getHost() + { + return _host; + } + + int SocketServer::getBacklog() + { + return _backlog; + } + + std::size_t SocketServer::getMaxConnections() + { + return _maxConnections; + } + + int SocketServer::getAddressFamily() + { + return _addressFamily; + } } // namespace ix diff --git a/ixwebsocket/IXSocketServer.h b/ixwebsocket/IXSocketServer.h index 8874587b..fe0f7e28 100644 --- a/ixwebsocket/IXSocketServer.h +++ b/ixwebsocket/IXSocketServer.h @@ -7,6 +7,8 @@ #pragma once #include "IXConnectionState.h" +#include "IXNetSystem.h" +#include "IXSelectInterrupt.h" #include "IXSocketTLSOptions.h" #include #include @@ -58,6 +60,11 @@ namespace ix void setTLSOptions(const SocketTLSOptions& socketTLSOptions); + int getPort(); + std::string getHost(); + int getBacklog(); + std::size_t getMaxConnections(); + int getAddressFamily(); protected: // Logging void logError(const std::string& str); @@ -74,7 +81,7 @@ namespace ix int _addressFamily; // socket for accepting connections - int _serverFd; + socket_t _serverFd; std::atomic _stop; @@ -83,6 +90,7 @@ namespace ix // background thread to wait for incoming connections std::thread _thread; void run(); + void onSetTerminatedCallback(); // background thread to cleanup (join) terminated threads std::atomic _stopGc; @@ -110,5 +118,13 @@ namespace ix size_t getConnectionsThreadsCount(); SocketTLSOptions _socketTLSOptions; + + // to wake up from select + SelectInterruptPtr _acceptSelectInterrupt; + + // used by the gc thread, to know that a thread needs to be garbage collected + // as a connection + std::condition_variable _conditionVariableGC; + std::mutex _conditionVariableMutexGC; }; } // namespace ix diff --git a/ixwebsocket/IXStrCaseCompare.cpp b/ixwebsocket/IXStrCaseCompare.cpp new file mode 100644 index 00000000..833815ff --- /dev/null +++ b/ixwebsocket/IXStrCaseCompare.cpp @@ -0,0 +1,37 @@ +/* + * IXStrCaseCompare.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone. All rights reserved. + */ + +#include "IXStrCaseCompare.h" + +#include +#include + +namespace ix +{ + bool CaseInsensitiveLess::NocaseCompare::operator()(const unsigned char& c1, + const unsigned char& c2) const + { +#if defined(_WIN32) && !defined(__GNUC__) + return std::tolower(c1, std::locale()) < std::tolower(c2, std::locale()); +#else + return std::tolower(c1) < std::tolower(c2); +#endif + } + + bool CaseInsensitiveLess::cmp(const std::string& s1, const std::string& s2) + { + return std::lexicographical_compare(s1.begin(), + s1.end(), // source range + s2.begin(), + s2.end(), // dest range + NocaseCompare()); // comparison + } + + bool CaseInsensitiveLess::operator()(const std::string& s1, const std::string& s2) const + { + return CaseInsensitiveLess::cmp(s1, s2); + } +} // namespace ix diff --git a/ixwebsocket/IXStrCaseCompare.h b/ixwebsocket/IXStrCaseCompare.h new file mode 100644 index 00000000..8a55de0e --- /dev/null +++ b/ixwebsocket/IXStrCaseCompare.h @@ -0,0 +1,25 @@ +/* + * IXStrCaseCompare.h + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone. All rights reserved. + */ + +#pragma once + +#include + +namespace ix +{ + struct CaseInsensitiveLess + { + // Case Insensitive compare_less binary function + struct NocaseCompare + { + bool operator()(const unsigned char& c1, const unsigned char& c2) const; + }; + + static bool cmp(const std::string& s1, const std::string& s2); + + bool operator()(const std::string& s1, const std::string& s2) const; + }; +} // namespace ix diff --git a/ixwebsocket/IXUniquePtr.h b/ixwebsocket/IXUniquePtr.h new file mode 100644 index 00000000..d88ce9bd --- /dev/null +++ b/ixwebsocket/IXUniquePtr.h @@ -0,0 +1,18 @@ +/* + * IXUniquePtr.h + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + */ + +#pragma once + +#include + +namespace ix +{ + template + std::unique_ptr make_unique(Args&&... args) + { + return std::unique_ptr(new T(std::forward(args)...)); + } +} // namespace ix diff --git a/ixwebsocket/IXUrlParser.cpp b/ixwebsocket/IXUrlParser.cpp index 9018f893..9963052c 100644 --- a/ixwebsocket/IXUrlParser.cpp +++ b/ixwebsocket/IXUrlParser.cpp @@ -32,6 +32,7 @@ #include "IXUrlParser.h" #include +#include #include namespace diff --git a/ixwebsocket/IXUserAgent.cpp b/ixwebsocket/IXUserAgent.cpp index 85fb9992..89dcb0f3 100644 --- a/ixwebsocket/IXUserAgent.cpp +++ b/ixwebsocket/IXUserAgent.cpp @@ -8,7 +8,9 @@ #include "IXWebSocketVersion.h" #include +#ifdef IXWEBSOCKET_USE_ZLIB #include +#endif // Platform name #if defined(_WIN32) @@ -77,8 +79,10 @@ namespace ix ss << " nossl"; #endif +#ifdef IXWEBSOCKET_USE_ZLIB // Zlib version ss << " zlib " << ZLIB_VERSION; +#endif return ss.str(); } diff --git a/ixcrypto/ixcrypto/IXUuid.cpp b/ixwebsocket/IXUuid.cpp similarity index 100% rename from ixcrypto/ixcrypto/IXUuid.cpp rename to ixwebsocket/IXUuid.cpp diff --git a/ixcrypto/ixcrypto/IXUuid.h b/ixwebsocket/IXUuid.h similarity index 100% rename from ixcrypto/ixcrypto/IXUuid.h rename to ixwebsocket/IXUuid.h diff --git a/ixwebsocket/IXWebSocket.cpp b/ixwebsocket/IXWebSocket.cpp index 864482ea..e272521a 100644 --- a/ixwebsocket/IXWebSocket.cpp +++ b/ixwebsocket/IXWebSocket.cpp @@ -8,12 +8,19 @@ #include "IXExponentialBackoff.h" #include "IXSetThreadName.h" +#include "IXUniquePtr.h" #include "IXUtf8Validator.h" #include "IXWebSocketHandshake.h" #include #include +namespace +{ + const std::string emptyMsg; +} // namespace + + namespace ix { OnTrafficTrackerCallback WebSocket::_onTrafficTrackerCallback = nullptr; @@ -21,12 +28,14 @@ namespace ix const int WebSocket::kDefaultPingIntervalSecs(-1); const bool WebSocket::kDefaultEnablePong(true); const uint32_t WebSocket::kDefaultMaxWaitBetweenReconnectionRetries(10 * 1000); // 10s + const uint32_t WebSocket::kDefaultMinWaitBetweenReconnectionRetries(1); // 1 ms WebSocket::WebSocket() : _onMessageCallback(OnMessageCallback()) , _stop(false) , _automaticReconnection(true) , _maxWaitBetweenReconnectionRetries(kDefaultMaxWaitBetweenReconnectionRetries) + , _minWaitBetweenReconnectionRetries(kDefaultMinWaitBetweenReconnectionRetries) , _handshakeTimeoutSecs(kDefaultHandShakeTimeoutSecs) , _enablePong(kDefaultEnablePong) , _pingIntervalSecs(kDefaultPingIntervalSecs) @@ -34,18 +43,19 @@ namespace ix _ws.setOnCloseCallback( [this](uint16_t code, const std::string& reason, size_t wireSize, bool remote) { _onMessageCallback( - std::make_unique(WebSocketMessageType::Close, - "", - wireSize, - WebSocketErrorInfo(), - WebSocketOpenInfo(), - WebSocketCloseInfo(code, reason, remote))); + ix::make_unique(WebSocketMessageType::Close, + emptyMsg, + wireSize, + WebSocketErrorInfo(), + WebSocketOpenInfo(), + WebSocketCloseInfo(code, reason, remote))); }); } WebSocket::~WebSocket() { stop(); + _ws.setOnCloseCallback(nullptr); } void WebSocket::setUrl(const std::string& url) @@ -53,13 +63,19 @@ namespace ix std::lock_guard lock(_configMutex); _url = url; } + + void WebSocket::setHandshakeTimeout(int handshakeTimeoutSecs) + { + _handshakeTimeoutSecs = handshakeTimeoutSecs; + } + void WebSocket::setExtraHeaders(const WebSocketHttpHeaders& headers) { std::lock_guard lock(_configMutex); _extraHeaders = headers; } - const std::string& WebSocket::getUrl() const + const std::string WebSocket::getUrl() const { std::lock_guard lock(_configMutex); return _url; @@ -78,7 +94,7 @@ namespace ix _socketTLSOptions = socketTLSOptions; } - const WebSocketPerMessageDeflateOptions& WebSocket::getPerMessageDeflateOptions() const + const WebSocketPerMessageDeflateOptions WebSocket::getPerMessageDeflateOptions() const { std::lock_guard lock(_configMutex); return _perMessageDeflateOptions; @@ -128,12 +144,24 @@ namespace ix _maxWaitBetweenReconnectionRetries = maxWaitBetweenReconnectionRetries; } + void WebSocket::setMinWaitBetweenReconnectionRetries(uint32_t minWaitBetweenReconnectionRetries) + { + std::lock_guard lock(_configMutex); + _minWaitBetweenReconnectionRetries = minWaitBetweenReconnectionRetries; + } + uint32_t WebSocket::getMaxWaitBetweenReconnectionRetries() const { std::lock_guard lock(_configMutex); return _maxWaitBetweenReconnectionRetries; } + uint32_t WebSocket::getMinWaitBetweenReconnectionRetries() const + { + std::lock_guard lock(_configMutex); + return _minWaitBetweenReconnectionRetries; + } + void WebSocket::start() { if (_thread.joinable()) return; // we've already been started @@ -193,9 +221,9 @@ namespace ix return status; } - _onMessageCallback(std::make_unique( + _onMessageCallback(ix::make_unique( WebSocketMessageType::Open, - "", + emptyMsg, 0, WebSocketErrorInfo(), WebSocketOpenInfo(status.uri, status.headers, status.protocol), @@ -210,7 +238,9 @@ namespace ix return status; } - WebSocketInitResult WebSocket::connectToSocket(std::unique_ptr socket, int timeoutSecs) + WebSocketInitResult WebSocket::connectToSocket(std::unique_ptr socket, + int timeoutSecs, + bool enablePerMessageDeflate) { { std::lock_guard lock(_configMutex); @@ -218,19 +248,20 @@ namespace ix _perMessageDeflateOptions, _socketTLSOptions, _enablePong, _pingIntervalSecs); } - WebSocketInitResult status = _ws.connectToSocket(std::move(socket), timeoutSecs); + WebSocketInitResult status = + _ws.connectToSocket(std::move(socket), timeoutSecs, enablePerMessageDeflate); if (!status.success) { return status; } _onMessageCallback( - std::make_unique(WebSocketMessageType::Open, - "", - 0, - WebSocketErrorInfo(), - WebSocketOpenInfo(status.uri, status.headers), - WebSocketCloseInfo())); + ix::make_unique(WebSocketMessageType::Open, + emptyMsg, + 0, + WebSocketErrorInfo(), + WebSocketOpenInfo(status.uri, status.headers), + WebSocketCloseInfo())); if (_pingIntervalSecs > 0) { @@ -300,8 +331,10 @@ namespace ix if (_automaticReconnection) { - duration = millis(calculateRetryWaitMilliseconds( - retries++, _maxWaitBetweenReconnectionRetries)); + duration = + millis(calculateRetryWaitMilliseconds(retries++, + _maxWaitBetweenReconnectionRetries, + _minWaitBetweenReconnectionRetries)); connectErr.wait_time = duration.count(); connectErr.retries = retries; @@ -310,12 +343,12 @@ namespace ix connectErr.reason = status.errorStr; connectErr.http_status = status.http_status; - _onMessageCallback(std::make_unique(WebSocketMessageType::Error, - "", - 0, - connectErr, - WebSocketOpenInfo(), - WebSocketCloseInfo())); + _onMessageCallback(ix::make_unique(WebSocketMessageType::Error, + emptyMsg, + 0, + connectErr, + WebSocketOpenInfo(), + WebSocketCloseInfo())); } } } @@ -352,7 +385,7 @@ namespace ix size_t wireSize, bool decompressionError, WebSocketTransport::MessageKind messageKind) { - WebSocketMessageType webSocketMessageType; + WebSocketMessageType webSocketMessageType{WebSocketMessageType::Error}; switch (messageKind) { case WebSocketTransport::MessageKind::MSG_TEXT: @@ -386,13 +419,13 @@ namespace ix bool binary = messageKind == WebSocketTransport::MessageKind::MSG_BINARY; - _onMessageCallback(std::make_unique(webSocketMessageType, - msg, - wireSize, - webSocketErrorInfo, - WebSocketOpenInfo(), - WebSocketCloseInfo(), - binary)); + _onMessageCallback(ix::make_unique(webSocketMessageType, + msg, + wireSize, + webSocketErrorInfo, + WebSocketOpenInfo(), + WebSocketCloseInfo(), + binary)); WebSocket::invokeTrafficTrackerCallback(wireSize, true); }); @@ -404,6 +437,11 @@ namespace ix _onMessageCallback = callback; } + bool WebSocket::isOnMessageCallbackRegistered() const + { + return _onMessageCallback != nullptr; + } + void WebSocket::setTrafficTrackerCallback(const OnTrafficTrackerCallback& callback) { _onTrafficTrackerCallback = callback; diff --git a/ixwebsocket/IXWebSocket.h b/ixwebsocket/IXWebSocket.h index 0c288cd5..7cfe0088 100644 --- a/ixwebsocket/IXWebSocket.h +++ b/ixwebsocket/IXWebSocket.h @@ -58,6 +58,7 @@ namespace ix void enablePerMessageDeflate(); void disablePerMessageDeflate(); void addSubProtocol(const std::string& subProtocol); + void setHandshakeTimeout(int handshakeTimeoutSecs); // Run asynchronously, by calling start and stop. void start(); @@ -84,14 +85,15 @@ namespace ix const std::string& reason = WebSocketCloseConstants::kNormalClosureMessage); void setOnMessageCallback(const OnMessageCallback& callback); + bool isOnMessageCallbackRegistered() const; static void setTrafficTrackerCallback(const OnTrafficTrackerCallback& callback); static void resetTrafficTrackerCallback(); ReadyState getReadyState() const; static std::string readyStateToString(ReadyState readyState); - const std::string& getUrl() const; - const WebSocketPerMessageDeflateOptions& getPerMessageDeflateOptions() const; + const std::string getUrl() const; + const WebSocketPerMessageDeflateOptions getPerMessageDeflateOptions() const; int getPingInterval() const; size_t bufferedAmount() const; @@ -99,7 +101,9 @@ namespace ix void disableAutomaticReconnection(); bool isAutomaticReconnectionEnabled() const; void setMaxWaitBetweenReconnectionRetries(uint32_t maxWaitBetweenReconnectionRetries); + void setMinWaitBetweenReconnectionRetries(uint32_t minWaitBetweenReconnectionRetries); uint32_t getMaxWaitBetweenReconnectionRetries() const; + uint32_t getMinWaitBetweenReconnectionRetries() const; const std::vector& getSubProtocols(); private: @@ -113,7 +117,9 @@ namespace ix static void invokeTrafficTrackerCallback(size_t size, bool incoming); // Server - WebSocketInitResult connectToSocket(std::unique_ptr, int timeoutSecs); + WebSocketInitResult connectToSocket(std::unique_ptr, + int timeoutSecs, + bool enablePerMessageDeflate); WebSocketTransport _ws; @@ -136,7 +142,9 @@ namespace ix // Automatic reconnection std::atomic _automaticReconnection; static const uint32_t kDefaultMaxWaitBetweenReconnectionRetries; + static const uint32_t kDefaultMinWaitBetweenReconnectionRetries; uint32_t _maxWaitBetweenReconnectionRetries; + uint32_t _minWaitBetweenReconnectionRetries; // Make the sleeping in the automatic reconnection cancellable std::mutex _sleepMutex; diff --git a/ixwebsocket/IXWebSocketHandshake.cpp b/ixwebsocket/IXWebSocketHandshake.cpp index 2c972c80..9a8de059 100644 --- a/ixwebsocket/IXWebSocketHandshake.cpp +++ b/ixwebsocket/IXWebSocketHandshake.cpp @@ -8,6 +8,7 @@ #include "IXHttp.h" #include "IXSocketConnect.h" +#include "IXStrCaseCompare.h" #include "IXUrlParser.h" #include "IXUserAgent.h" #include "IXWebSocketHandshakeKeyGen.h" @@ -35,9 +36,7 @@ namespace ix bool WebSocketHandshake::insensitiveStringCompare(const std::string& a, const std::string& b) { - return std::equal(a.begin(), a.end(), b.begin(), b.end(), [](char a, char b) { - return tolower(a) == tolower(b); - }); + return CaseInsensitiveLess::cmp(a, b) == 0; } std::string WebSocketHandshake::genRandomString(const int len) @@ -170,20 +169,11 @@ namespace ix { std::stringstream ss; ss << "Expecting HTTP/1.1, got " << httpVersion << ". " - << "Rejecting connection to " << host << ":" << port << ", status: " << status + << "Rejecting connection to " << url << ", status: " << status << ", HTTP Status line: " << line; return WebSocketInitResult(false, status, ss.str()); } - // We want an 101 HTTP status - if (status != 101) - { - std::stringstream ss; - ss << "Expecting status 101 (Switching Protocol), got " << status - << " status connecting to " << host << ":" << port << ", HTTP Status line: " << line; - return WebSocketInitResult(false, status, ss.str()); - } - auto result = parseHttpHeaders(_socket, isCancellationRequested); auto headersValid = result.first; auto headers = result.second; @@ -193,6 +183,17 @@ namespace ix return WebSocketInitResult(false, status, "Error parsing HTTP headers"); } + // We want an 101 HTTP status for websocket, otherwise it could be + // a redirection (like 301) + if (status != 101) + { + std::stringstream ss; + ss << "Expecting status 101 (Switching Protocol), got " << status + << " status connecting to " << url << ", HTTP Status line: " << line; + + return WebSocketInitResult(false, status, ss.str(), headers, path); + } + // Check the presence of the connection field if (headers.find("connection") == headers.end()) { @@ -203,6 +204,9 @@ namespace ix // Check the value of the connection field // Some websocket servers (Go/Gorilla?) send lowercase values for the // connection header, so do a case insensitive comparison + // + // See https://github.com/apache/thrift/commit/7c4bdf9914fcba6c89e0f69ae48b9675578f084a + // if (!insensitiveStringCompare(headers["connection"], "Upgrade")) { std::stringstream ss; @@ -240,7 +244,8 @@ namespace ix return WebSocketInitResult(true, status, "", headers, path); } - WebSocketInitResult WebSocketHandshake::serverHandshake(int timeoutSecs) + WebSocketInitResult WebSocketHandshake::serverHandshake(int timeoutSecs, + bool enablePerMessageDeflate) { _requestInitCancellation = false; @@ -294,7 +299,8 @@ namespace ix return sendErrorResponse(400, "Missing Upgrade header"); } - if (!insensitiveStringCompare(headers["upgrade"], "WebSocket")) + if (!insensitiveStringCompare(headers["upgrade"], "WebSocket") && + headers["Upgrade"] != "keep-alive, Upgrade") // special case for firefox { return sendErrorResponse(400, "Invalid Upgrade header, " @@ -337,7 +343,7 @@ namespace ix WebSocketPerMessageDeflateOptions webSocketPerMessageDeflateOptions(header); // If the client has requested that extension, - if (webSocketPerMessageDeflateOptions.enabled()) + if (webSocketPerMessageDeflateOptions.enabled() && enablePerMessageDeflate) { _enablePerMessageDeflate = true; diff --git a/ixwebsocket/IXWebSocketHandshake.h b/ixwebsocket/IXWebSocketHandshake.h index 0c8ed289..0a275e43 100644 --- a/ixwebsocket/IXWebSocketHandshake.h +++ b/ixwebsocket/IXWebSocketHandshake.h @@ -35,7 +35,7 @@ namespace ix int port, int timeoutSecs); - WebSocketInitResult serverHandshake(int timeoutSecs); + WebSocketInitResult serverHandshake(int timeoutSecs, bool enablePerMessageDeflate); private: std::string genRandomString(const int len); diff --git a/ixwebsocket/IXWebSocketHttpHeaders.cpp b/ixwebsocket/IXWebSocketHttpHeaders.cpp index 77fd7955..55634301 100644 --- a/ixwebsocket/IXWebSocketHttpHeaders.cpp +++ b/ixwebsocket/IXWebSocketHttpHeaders.cpp @@ -12,25 +12,6 @@ namespace ix { - bool CaseInsensitiveLess::NocaseCompare::operator()(const unsigned char& c1, - const unsigned char& c2) const - { -#ifdef _WIN32 - return std::tolower(c1, std::locale()) < std::tolower(c2, std::locale()); -#else - return std::tolower(c1) < std::tolower(c2); -#endif - } - - bool CaseInsensitiveLess::operator()(const std::string& s1, const std::string& s2) const - { - return std::lexicographical_compare(s1.begin(), - s1.end(), // source range - s2.begin(), - s2.end(), // dest range - NocaseCompare()); // comparison - } - std::pair parseHttpHeaders( std::unique_ptr& socket, const CancellationRequest& isCancellationRequested) { diff --git a/ixwebsocket/IXWebSocketHttpHeaders.h b/ixwebsocket/IXWebSocketHttpHeaders.h index 777e7a9d..7ba8c4ef 100644 --- a/ixwebsocket/IXWebSocketHttpHeaders.h +++ b/ixwebsocket/IXWebSocketHttpHeaders.h @@ -7,6 +7,7 @@ #pragma once #include "IXCancellationRequest.h" +#include "IXStrCaseCompare.h" #include #include #include @@ -15,17 +16,6 @@ namespace ix { class Socket; - struct CaseInsensitiveLess - { - // Case Insensitive compare_less binary function - struct NocaseCompare - { - bool operator()(const unsigned char& c1, const unsigned char& c2) const; - }; - - bool operator()(const std::string& s1, const std::string& s2) const; - }; - using WebSocketHttpHeaders = std::map; std::pair parseHttpHeaders( diff --git a/ixwebsocket/IXWebSocketMessage.h b/ixwebsocket/IXWebSocketMessage.h index 6dcc3d64..25a00ce7 100644 --- a/ixwebsocket/IXWebSocketMessage.h +++ b/ixwebsocket/IXWebSocketMessage.h @@ -12,7 +12,6 @@ #include "IXWebSocketOpenInfo.h" #include #include -#include namespace ix { @@ -43,6 +42,18 @@ namespace ix { ; } + + /** + * @brief Deleted overload to prevent binding `str` to a temporary, which would cause + * undefined behavior since class members don't extend lifetime beyond the constructor call. + */ + WebSocketMessage(WebSocketMessageType t, + std::string&& s, + size_t w, + WebSocketErrorInfo e, + WebSocketOpenInfo o, + WebSocketCloseInfo c, + bool b = false) = delete; }; using WebSocketMessagePtr = std::unique_ptr; diff --git a/ixwebsocket/IXWebSocketPerMessageDeflate.cpp b/ixwebsocket/IXWebSocketPerMessageDeflate.cpp index 6392768b..80435870 100644 --- a/ixwebsocket/IXWebSocketPerMessageDeflate.cpp +++ b/ixwebsocket/IXWebSocketPerMessageDeflate.cpp @@ -48,14 +48,15 @@ #include "IXWebSocketPerMessageDeflate.h" +#include "IXUniquePtr.h" #include "IXWebSocketPerMessageDeflateCodec.h" #include "IXWebSocketPerMessageDeflateOptions.h" namespace ix { WebSocketPerMessageDeflate::WebSocketPerMessageDeflate() - : _compressor(std::make_unique()) - , _decompressor(std::make_unique()) + : _compressor(ix::make_unique()) + , _decompressor(ix::make_unique()) { ; } diff --git a/ixwebsocket/IXWebSocketPerMessageDeflateCodec.cpp b/ixwebsocket/IXWebSocketPerMessageDeflateCodec.cpp index eb51a82d..d641c47c 100644 --- a/ixwebsocket/IXWebSocketPerMessageDeflateCodec.cpp +++ b/ixwebsocket/IXWebSocketPerMessageDeflateCodec.cpp @@ -16,8 +16,6 @@ namespace // is treated as a char* and the null termination (\x00) makes it // look like an empty string. const std::string kEmptyUncompressedBlock = std::string("\x00\x00\xff\xff", 4); - - const int kBufferSize = 1 << 14; } // namespace namespace ix @@ -26,23 +24,27 @@ namespace ix // Compressor // WebSocketPerMessageDeflateCompressor::WebSocketPerMessageDeflateCompressor() - : _compressBufferSize(kBufferSize) { +#ifdef IXWEBSOCKET_USE_ZLIB memset(&_deflateState, 0, sizeof(_deflateState)); _deflateState.zalloc = Z_NULL; _deflateState.zfree = Z_NULL; _deflateState.opaque = Z_NULL; +#endif } WebSocketPerMessageDeflateCompressor::~WebSocketPerMessageDeflateCompressor() { +#ifdef IXWEBSOCKET_USE_ZLIB deflateEnd(&_deflateState); +#endif } bool WebSocketPerMessageDeflateCompressor::init(uint8_t deflateBits, bool clientNoContextTakeOver) { +#ifdef IXWEBSOCKET_USE_ZLIB int ret = deflateInit2(&_deflateState, Z_DEFAULT_COMPRESSION, Z_DEFLATED, @@ -52,22 +54,52 @@ namespace ix if (ret != Z_OK) return false; - _compressBuffer = std::make_unique(_compressBufferSize); - _flush = (clientNoContextTakeOver) ? Z_FULL_FLUSH : Z_SYNC_FLUSH; return true; +#else + return false; +#endif } - bool WebSocketPerMessageDeflateCompressor::endsWith(const std::string& value, - const std::string& ending) + template + bool WebSocketPerMessageDeflateCompressor::endsWithEmptyUnCompressedBlock(const T& value) { - if (ending.size() > value.size()) return false; - return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); + if (kEmptyUncompressedBlock.size() > value.size()) return false; + auto N = value.size(); + return value[N - 1] == kEmptyUncompressedBlock[3] && + value[N - 2] == kEmptyUncompressedBlock[2] && + value[N - 3] == kEmptyUncompressedBlock[1] && + value[N - 4] == kEmptyUncompressedBlock[0]; } bool WebSocketPerMessageDeflateCompressor::compress(const std::string& in, std::string& out) { + return compressData(in, out); + } + + bool WebSocketPerMessageDeflateCompressor::compress(const std::string& in, + std::vector& out) + { + return compressData(in, out); + } + + bool WebSocketPerMessageDeflateCompressor::compress(const std::vector& in, + std::string& out) + { + return compressData(in, out); + } + + bool WebSocketPerMessageDeflateCompressor::compress(const std::vector& in, + std::vector& out) + { + return compressData(in, out); + } + + template + bool WebSocketPerMessageDeflateCompressor::compressData(const T& in, S& out) + { +#ifdef IXWEBSOCKET_USE_ZLIB // // 7.2.1. Compression // @@ -96,7 +128,8 @@ namespace ix // The normal buffer size should be 6 but // we remove the 4 octets from the tail (#4) uint8_t buf[2] = {0x02, 0x00}; - out.append((char*) (buf), 2); + out.push_back(buf[0]); + out.push_back(buf[1]); return true; } @@ -107,30 +140,33 @@ namespace ix do { // Output to local buffer - _deflateState.avail_out = (uInt) _compressBufferSize; - _deflateState.next_out = _compressBuffer.get(); + _deflateState.avail_out = (uInt) _compressBuffer.size(); + _deflateState.next_out = &_compressBuffer.front(); deflate(&_deflateState, _flush); - output = _compressBufferSize - _deflateState.avail_out; + output = _compressBuffer.size() - _deflateState.avail_out; - out.append((char*) (_compressBuffer.get()), output); + out.insert(out.end(), _compressBuffer.begin(), _compressBuffer.begin() + output); } while (_deflateState.avail_out == 0); - if (endsWith(out, kEmptyUncompressedBlock)) + if (endsWithEmptyUnCompressedBlock(out)) { out.resize(out.size() - 4); } return true; +#else + return false; +#endif } // // Decompressor // WebSocketPerMessageDeflateDecompressor::WebSocketPerMessageDeflateDecompressor() - : _compressBufferSize(kBufferSize) { +#ifdef IXWEBSOCKET_USE_ZLIB memset(&_inflateState, 0, sizeof(_inflateState)); _inflateState.zalloc = Z_NULL; @@ -138,29 +174,35 @@ namespace ix _inflateState.opaque = Z_NULL; _inflateState.avail_in = 0; _inflateState.next_in = Z_NULL; +#endif } WebSocketPerMessageDeflateDecompressor::~WebSocketPerMessageDeflateDecompressor() { +#ifdef IXWEBSOCKET_USE_ZLIB inflateEnd(&_inflateState); +#endif } bool WebSocketPerMessageDeflateDecompressor::init(uint8_t inflateBits, bool clientNoContextTakeOver) { +#ifdef IXWEBSOCKET_USE_ZLIB int ret = inflateInit2(&_inflateState, -1 * inflateBits); if (ret != Z_OK) return false; - _compressBuffer = std::make_unique(_compressBufferSize); - _flush = (clientNoContextTakeOver) ? Z_FULL_FLUSH : Z_SYNC_FLUSH; return true; +#else + return false; +#endif } bool WebSocketPerMessageDeflateDecompressor::decompress(const std::string& in, std::string& out) { +#ifdef IXWEBSOCKET_USE_ZLIB // // 7.2.2. Decompression // @@ -182,8 +224,8 @@ namespace ix do { - _inflateState.avail_out = (uInt) _compressBufferSize; - _inflateState.next_out = _compressBuffer.get(); + _inflateState.avail_out = (uInt) _compressBuffer.size(); + _inflateState.next_out = &_compressBuffer.front(); int ret = inflate(&_inflateState, Z_SYNC_FLUSH); @@ -192,10 +234,13 @@ namespace ix return false; // zlib error } - out.append(reinterpret_cast(_compressBuffer.get()), - _compressBufferSize - _inflateState.avail_out); + out.append(reinterpret_cast(&_compressBuffer.front()), + _compressBuffer.size() - _inflateState.avail_out); } while (_inflateState.avail_out == 0); return true; +#else + return false; +#endif } } // namespace ix diff --git a/ixwebsocket/IXWebSocketPerMessageDeflateCodec.h b/ixwebsocket/IXWebSocketPerMessageDeflateCodec.h index 3e70fa54..2d06517c 100644 --- a/ixwebsocket/IXWebSocketPerMessageDeflateCodec.h +++ b/ixwebsocket/IXWebSocketPerMessageDeflateCodec.h @@ -6,9 +6,12 @@ #pragma once +#ifdef IXWEBSOCKET_USE_ZLIB #include "zlib.h" -#include +#endif +#include #include +#include namespace ix { @@ -20,14 +23,22 @@ namespace ix bool init(uint8_t deflateBits, bool clientNoContextTakeOver); bool compress(const std::string& in, std::string& out); + bool compress(const std::string& in, std::vector& out); + bool compress(const std::vector& in, std::string& out); + bool compress(const std::vector& in, std::vector& out); private: - static bool endsWith(const std::string& value, const std::string& ending); + template + bool compressData(const T& in, S& out); + template + bool endsWithEmptyUnCompressedBlock(const T& value); int _flush; - size_t _compressBufferSize; - std::unique_ptr _compressBuffer; + std::array _compressBuffer; + +#ifdef IXWEBSOCKET_USE_ZLIB z_stream _deflateState; +#endif }; class WebSocketPerMessageDeflateDecompressor @@ -41,9 +52,11 @@ namespace ix private: int _flush; - size_t _compressBufferSize; - std::unique_ptr _compressBuffer; + std::array _compressBuffer; + +#ifdef IXWEBSOCKET_USE_ZLIB z_stream _inflateState; +#endif }; } // namespace ix diff --git a/ixwebsocket/IXWebSocketPerMessageDeflateOptions.cpp b/ixwebsocket/IXWebSocketPerMessageDeflateOptions.cpp index da81693a..c41a8c3d 100644 --- a/ixwebsocket/IXWebSocketPerMessageDeflateOptions.cpp +++ b/ixwebsocket/IXWebSocketPerMessageDeflateOptions.cpp @@ -14,12 +14,12 @@ namespace ix { /// Default values as defined in the RFC const uint8_t WebSocketPerMessageDeflateOptions::kDefaultServerMaxWindowBits = 15; - static const int minServerMaxWindowBits = 8; - static const int maxServerMaxWindowBits = 15; + static const uint8_t minServerMaxWindowBits = 8; + static const uint8_t maxServerMaxWindowBits = 15; const uint8_t WebSocketPerMessageDeflateOptions::kDefaultClientMaxWindowBits = 15; - static const int minClientMaxWindowBits = 8; - static const int maxClientMaxWindowBits = 15; + static const uint8_t minClientMaxWindowBits = 8; + static const uint8_t maxClientMaxWindowBits = 15; WebSocketPerMessageDeflateOptions::WebSocketPerMessageDeflateOptions( bool enabled, @@ -61,6 +61,7 @@ namespace ix _clientMaxWindowBits = kDefaultClientMaxWindowBits; _serverMaxWindowBits = kDefaultServerMaxWindowBits; +#ifdef IXWEBSOCKET_USE_ZLIB // Split by ; std::string token; std::stringstream tokenStream(extension); @@ -84,11 +85,7 @@ namespace ix if (startsWith(token, "server_max_window_bits=")) { - std::string val = token.substr(token.find_last_of("=") + 1); - std::stringstream ss; - ss << val; - int x; - ss >> x; + uint8_t x = strtol(token.substr(token.find_last_of("=") + 1).c_str(), nullptr, 10); // Sanitize values to be in the proper range [8, 15] in // case a server would give us bogus values @@ -98,11 +95,7 @@ namespace ix if (startsWith(token, "client_max_window_bits=")) { - std::string val = token.substr(token.find_last_of("=") + 1); - std::stringstream ss; - ss << val; - int x; - ss >> x; + uint8_t x = strtol(token.substr(token.find_last_of("=") + 1).c_str(), nullptr, 10); // Sanitize values to be in the proper range [8, 15] in // case a server would give us bogus values @@ -112,6 +105,7 @@ namespace ix sanitizeClientMaxWindowBits(); } } +#endif } void WebSocketPerMessageDeflateOptions::sanitizeClientMaxWindowBits() @@ -126,6 +120,7 @@ namespace ix std::string WebSocketPerMessageDeflateOptions::generateHeader() { +#ifdef IXWEBSOCKET_USE_ZLIB std::stringstream ss; ss << "Sec-WebSocket-Extensions: permessage-deflate"; @@ -138,11 +133,18 @@ namespace ix ss << "\r\n"; return ss.str(); +#else + return std::string(); +#endif } bool WebSocketPerMessageDeflateOptions::enabled() const { +#ifdef IXWEBSOCKET_USE_ZLIB return _enabled; +#else + return false; +#endif } bool WebSocketPerMessageDeflateOptions::getClientNoContextTakeover() const diff --git a/ixwebsocket/IXWebSocketPerMessageDeflateOptions.h b/ixwebsocket/IXWebSocketPerMessageDeflateOptions.h index 3e960f11..7cd33c0c 100644 --- a/ixwebsocket/IXWebSocketPerMessageDeflateOptions.h +++ b/ixwebsocket/IXWebSocketPerMessageDeflateOptions.h @@ -39,8 +39,8 @@ namespace ix bool _enabled; bool _clientNoContextTakeover; bool _serverNoContextTakeover; - int _clientMaxWindowBits; - int _serverMaxWindowBits; + uint8_t _clientMaxWindowBits; + uint8_t _serverMaxWindowBits; void sanitizeClientMaxWindowBits(); }; diff --git a/ixwebsocket/IXWebSocketProxyServer.cpp b/ixwebsocket/IXWebSocketProxyServer.cpp new file mode 100644 index 00000000..4b78c63b --- /dev/null +++ b/ixwebsocket/IXWebSocketProxyServer.cpp @@ -0,0 +1,137 @@ +/* + * IXWebSocketProxyServer.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2018 Machine Zone, Inc. All rights reserved. + */ + +#include "IXWebSocketProxyServer.h" + +#include "IXWebSocketServer.h" +#include + +namespace ix +{ + class ProxyConnectionState : public ix::ConnectionState + { + public: + ProxyConnectionState() + : _connected(false) + { + } + + ix::WebSocket& webSocket() + { + return _serverWebSocket; + } + + bool isConnected() + { + return _connected; + } + + void setConnected() + { + _connected = true; + } + + private: + ix::WebSocket _serverWebSocket; + bool _connected; + }; + + int websocket_proxy_server_main(int port, + const std::string& hostname, + const ix::SocketTLSOptions& tlsOptions, + const std::string& remoteUrl, + const RemoteUrlsMapping& remoteUrlsMapping, + bool /*verbose*/) + { + ix::WebSocketServer server(port, hostname); + server.setTLSOptions(tlsOptions); + + auto factory = []() -> std::shared_ptr { + return std::make_shared(); + }; + server.setConnectionStateFactory(factory); + + server.setOnConnectionCallback( + [remoteUrl, remoteUrlsMapping](std::weak_ptr webSocket, + std::shared_ptr connectionState) { + auto state = std::dynamic_pointer_cast(connectionState); + auto remoteIp = connectionState->getRemoteIp(); + + // Server connection + state->webSocket().setOnMessageCallback( + [webSocket, state, remoteIp](const WebSocketMessagePtr& msg) { + if (msg->type == ix::WebSocketMessageType::Close) + { + state->setTerminated(); + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + auto ws = webSocket.lock(); + if (ws) + { + ws->send(msg->str, msg->binary); + } + } + }); + + // Client connection + auto ws = webSocket.lock(); + if (ws) + { + ws->setOnMessageCallback([state, remoteUrl, remoteUrlsMapping]( + const WebSocketMessagePtr& msg) { + if (msg->type == ix::WebSocketMessageType::Open) + { + // Connect to the 'real' server + std::string url(remoteUrl); + + // maybe we want a different url based on the mapping + std::string host = msg->openInfo.headers["Host"]; + auto it = remoteUrlsMapping.find(host); + if (it != remoteUrlsMapping.end()) + { + url = it->second; + } + + // append the uri to form the full url + // (say ws://localhost:1234/foo/?bar=baz) + url += msg->openInfo.uri; + + state->webSocket().setUrl(url); + state->webSocket().disableAutomaticReconnection(); + state->webSocket().start(); + + // we should sleep here for a bit until we've established the + // connection with the remote server + while (state->webSocket().getReadyState() != ReadyState::Open) + { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + state->webSocket().close(msg->closeInfo.code, msg->closeInfo.reason); + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + state->webSocket().send(msg->str, msg->binary); + } + }); + } + }); + + auto res = server.listen(); + if (!res.first) + { + return 1; + } + + server.start(); + server.wait(); + + return 0; + } +} // namespace ix diff --git a/ixwebsocket/IXWebSocketProxyServer.h b/ixwebsocket/IXWebSocketProxyServer.h new file mode 100644 index 00000000..7a55e472 --- /dev/null +++ b/ixwebsocket/IXWebSocketProxyServer.h @@ -0,0 +1,24 @@ +/* + * IXWebSocketProxyServer.h + * Author: Benjamin Sergeant + * Copyright (c) 2019-2020 Machine Zone, Inc. All rights reserved. + */ +#pragma once + +#include "IXSocketTLSOptions.h" +#include +#include +#include +#include + +namespace ix +{ + using RemoteUrlsMapping = std::map; + + int websocket_proxy_server_main(int port, + const std::string& hostname, + const ix::SocketTLSOptions& tlsOptions, + const std::string& remoteUrl, + const RemoteUrlsMapping& remoteUrlsMapping, + bool verbose); +} // namespace ix diff --git a/ixwebsocket/IXWebSocketServer.cpp b/ixwebsocket/IXWebSocketServer.cpp index 86b0242a..64830a5f 100644 --- a/ixwebsocket/IXWebSocketServer.cpp +++ b/ixwebsocket/IXWebSocketServer.cpp @@ -71,13 +71,46 @@ namespace ix _onConnectionCallback = callback; } + void WebSocketServer::setOnClientMessageCallback(const OnClientMessageCallback& callback) + { + _onClientMessageCallback = callback; + } + void WebSocketServer::handleConnection(std::unique_ptr socket, std::shared_ptr connectionState) { setThreadName("WebSocketServer::" + connectionState->getId()); auto webSocket = std::make_shared(); - _onConnectionCallback(webSocket, connectionState); + if (_onConnectionCallback) + { + _onConnectionCallback(webSocket, connectionState); + + if (!webSocket->isOnMessageCallbackRegistered()) + { + logError("WebSocketServer Application developer error: Server callback improperly " + "registerered."); + logError("Missing call to setOnMessageCallback inside setOnConnectionCallback."); + connectionState->setTerminated(); + return; + } + } + else if (_onClientMessageCallback) + { + WebSocket* webSocketRawPtr = webSocket.get(); + webSocket->setOnMessageCallback( + [this, webSocketRawPtr, connectionState](const WebSocketMessagePtr& msg) { + _onClientMessageCallback(connectionState, *webSocketRawPtr, msg); + }); + } + else + { + logError( + "WebSocketServer Application developer error: No server callback is registerered."); + logError("Missing call to setOnConnectionCallback or setOnClientMessageCallback."); + connectionState->setTerminated(); + return; + } webSocket->disableAutomaticReconnection(); @@ -96,7 +129,8 @@ namespace ix _clients.insert(webSocket); } - auto status = webSocket->connectToSocket(std::move(socket), _handshakeTimeoutSecs); + auto status = webSocket->connectToSocket( + std::move(socket), _handshakeTimeoutSecs, _enablePerMessageDeflate); if (status.success) { // Process incoming messages and execute callbacks @@ -111,6 +145,8 @@ namespace ix logError(ss.str()); } + webSocket->setOnMessageCallback(nullptr); + // Remove this client from our client set { std::lock_guard lock(_clientsMutex); @@ -134,4 +170,60 @@ namespace ix std::lock_guard lock(_clientsMutex); return _clients.size(); } + + // + // Classic servers + // + void WebSocketServer::makeBroadcastServer() + { + setOnClientMessageCallback([this](std::shared_ptr connectionState, + WebSocket& webSocket, + const WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Message) + { + for (auto&& client : getClients()) + { + if (client.get() != &webSocket) + { + client->send(msg->str, msg->binary); + + // Make sure the OS send buffer is flushed before moving on + do + { + std::chrono::duration duration(500); + std::this_thread::sleep_for(duration); + } while (client->bufferedAmount() != 0); + } + } + } + }); + } + + bool WebSocketServer::listenAndStart() + { + auto res = listen(); + if (!res.first) + { + return false; + } + + start(); + return true; + } + + int WebSocketServer::getHandshakeTimeoutSecs() + { + return _handshakeTimeoutSecs; + } + + bool WebSocketServer::isPongEnabled() + { + return _enablePong; + } + + bool WebSocketServer::isPerMessageDeflateEnabled() + { + return _enablePerMessageDeflate; + } } // namespace ix diff --git a/ixwebsocket/IXWebSocketServer.h b/ixwebsocket/IXWebSocketServer.h index 22bfa5b0..6cae6331 100644 --- a/ixwebsocket/IXWebSocketServer.h +++ b/ixwebsocket/IXWebSocketServer.h @@ -19,11 +19,14 @@ namespace ix { - class WebSocketServer final : public SocketServer + class WebSocketServer : public SocketServer { public: using OnConnectionCallback = - std::function, std::shared_ptr)>; + std::function, std::shared_ptr)>; + + using OnClientMessageCallback = std::function, WebSocket&, const WebSocketMessagePtr&)>; WebSocketServer(int port = SocketServer::kDefaultPort, const std::string& host = SocketServer::kDefaultHost, @@ -39,12 +42,19 @@ namespace ix void disablePerMessageDeflate(); void setOnConnectionCallback(const OnConnectionCallback& callback); + void setOnClientMessageCallback(const OnClientMessageCallback& callback); // Get all the connected clients std::set> getClients(); + void makeBroadcastServer(); + bool listenAndStart(); + const static int kDefaultHandShakeTimeoutSecs; + int getHandshakeTimeoutSecs(); + bool isPongEnabled(); + bool isPerMessageDeflateEnabled(); private: // Member variables int _handshakeTimeoutSecs; @@ -52,6 +62,7 @@ namespace ix bool _enablePerMessageDeflate; OnConnectionCallback _onConnectionCallback; + OnClientMessageCallback _onClientMessageCallback; std::mutex _clientsMutex; std::set> _clients; @@ -60,7 +71,7 @@ namespace ix // Methods virtual void handleConnection(std::unique_ptr socket, - std::shared_ptr connectionState) final; + std::shared_ptr connectionState); virtual size_t getConnectedClientsCount() final; }; } // namespace ix diff --git a/ixwebsocket/IXWebSocketTransport.cpp b/ixwebsocket/IXWebSocketTransport.cpp index ff72b886..8537a95e 100644 --- a/ixwebsocket/IXWebSocketTransport.cpp +++ b/ixwebsocket/IXWebSocketTransport.cpp @@ -36,6 +36,7 @@ #include "IXSocketFactory.h" #include "IXSocketTLSOptions.h" +#include "IXUniquePtr.h" #include "IXUrlParser.h" #include "IXUtf8Validator.h" #include "IXWebSocketHandshake.h" @@ -65,7 +66,6 @@ namespace ix , _receivedMessageCompressed(false) , _readyState(ReadyState::CLOSED) , _closeCode(WebSocketCloseConstants::kInternalErrorCode) - , _closeReason(WebSocketCloseConstants::kInternalErrorMessage) , _closeWireSize(0) , _closeRemote(false) , _enablePerMessageDeflate(false) @@ -77,6 +77,7 @@ namespace ix , _pingCount(0) , _lastSendPingTimePoint(std::chrono::steady_clock::now()) { + setCloseReason(WebSocketCloseConstants::kInternalErrorMessage); _readbuf.resize(kChunkSize); } @@ -107,42 +108,69 @@ namespace ix std::string protocol, host, path, query; int port; + std::string remoteUrl(url); - if (!UrlParser::parse(url, protocol, host, path, query, port)) + WebSocketInitResult result; + const int maxRedirections = 10; + + for (int i = 0; i < maxRedirections; ++i) { - std::stringstream ss; - ss << "Could not parse url: '" << url << "'"; - return WebSocketInitResult(false, 0, ss.str()); + if (!UrlParser::parse(remoteUrl, protocol, host, path, query, port)) + { + std::stringstream ss; + ss << "Could not parse url: '" << url << "'"; + return WebSocketInitResult(false, 0, ss.str()); + } + + std::string errorMsg; + bool tls = protocol == "wss"; + _socket = createSocket(tls, -1, errorMsg, _socketTLSOptions); + _perMessageDeflate = ix::make_unique(); + + if (!_socket) + { + return WebSocketInitResult(false, 0, errorMsg); + } + + WebSocketHandshake webSocketHandshake(_requestInitCancellation, + _socket, + _perMessageDeflate, + _perMessageDeflateOptions, + _enablePerMessageDeflate); + + result = webSocketHandshake.clientHandshake( + remoteUrl, headers, host, path, port, timeoutSecs); + + if (result.http_status >= 300 && result.http_status < 400) + { + auto it = result.headers.find("Location"); + if (it == result.headers.end()) + { + std::stringstream ss; + ss << "Missing Location Header for HTTP Redirect response. " + << "Rejecting connection to " << url << ", status: " << result.http_status; + result.errorStr = ss.str(); + break; + } + + remoteUrl = it->second; + continue; + } + + if (result.success) + { + setReadyState(ReadyState::OPEN); + } + return result; } - std::string errorMsg; - bool tls = protocol == "wss"; - _socket = createSocket(tls, -1, errorMsg, _socketTLSOptions); - _perMessageDeflate = std::make_unique(); - - if (!_socket) - { - return WebSocketInitResult(false, 0, errorMsg); - } - - WebSocketHandshake webSocketHandshake(_requestInitCancellation, - _socket, - _perMessageDeflate, - _perMessageDeflateOptions, - _enablePerMessageDeflate); - - auto result = - webSocketHandshake.clientHandshake(url, headers, host, path, port, timeoutSecs); - if (result.success) - { - setReadyState(ReadyState::OPEN); - } return result; } // Server WebSocketInitResult WebSocketTransport::connectToSocket(std::unique_ptr socket, - int timeoutSecs) + int timeoutSecs, + bool enablePerMessageDeflate) { std::lock_guard lock(_socketMutex); @@ -151,7 +179,7 @@ namespace ix _blockingSend = true; _socket = std::move(socket); - _perMessageDeflate = std::make_unique(); + _perMessageDeflate = ix::make_unique(); WebSocketHandshake webSocketHandshake(_requestInitCancellation, _socket, @@ -159,7 +187,7 @@ namespace ix _perMessageDeflateOptions, _enablePerMessageDeflate); - auto result = webSocketHandshake.serverHandshake(timeoutSecs); + auto result = webSocketHandshake.serverHandshake(timeoutSecs, enablePerMessageDeflate); if (result.success) { setReadyState(ReadyState::OPEN); @@ -179,10 +207,12 @@ namespace ix if (readyState == ReadyState::CLOSED) { - std::lock_guard lock(_closeDataMutex); - _onCloseCallback(_closeCode, _closeReason, _closeWireSize, _closeRemote); + if (_onCloseCallback) + { + _onCloseCallback(_closeCode, getCloseReason(), _closeWireSize, _closeRemote); + } + setCloseReason(WebSocketCloseConstants::kInternalErrorMessage); _closeCode = WebSocketCloseConstants::kInternalErrorCode; - _closeReason = WebSocketCloseConstants::kInternalErrorMessage; _closeWireSize = 0; _closeRemote = false; } @@ -261,9 +291,10 @@ namespace ix { // compute lasting delay to wait for next ping / timeout, if at least one set auto now = std::chrono::steady_clock::now(); - lastingTimeoutDelayInMs = (int) std::chrono::duration_cast( + int timeSinceLastPingMs = (int) std::chrono::duration_cast( now - _lastSendPingTimePoint) .count(); + lastingTimeoutDelayInMs = (1000 * _pingIntervalSecs) - timeSinceLastPingMs; } #ifdef _WIN32 @@ -326,9 +357,10 @@ namespace ix return _txbuf.empty(); } + template void WebSocketTransport::appendToSendBuffer(const std::vector& header, - std::string::const_iterator begin, - std::string::const_iterator end, + Iterator begin, + Iterator end, uint64_t message_size, uint8_t masking_key[4]) { @@ -629,7 +661,7 @@ namespace ix // send back the CLOSE frame sendCloseFrame(code, reason); - wakeUpFromPoll(Socket::kCloseRequest); + wakeUpFromPoll(SelectInterrupt::kCloseRequest); bool remote = true; closeSocketAndSwitchToClosedState(code, reason, _rxbuf.size(), remote); @@ -638,11 +670,7 @@ namespace ix { // we got the CLOSE frame answer from our close, so we can close the connection // if the code/reason are the same - bool identicalReason; - { - std::lock_guard lock(_closeDataMutex); - identicalReason = _closeCode == code && _closeReason == reason; - } + bool identicalReason = _closeCode == code && getCloseReason() == reason; if (identicalReason) { @@ -750,8 +778,9 @@ namespace ix return static_cast(seconds); } + template WebSocketSendInfo WebSocketTransport::sendData(wsheader_type::opcode_type type, - const std::string& message, + const T& message, bool compress, const OnProgressCallback& onProgressCallback) { @@ -764,8 +793,8 @@ namespace ix size_t wireSize = message.size(); bool compressionError = false; - std::string::const_iterator message_begin = message.begin(); - std::string::const_iterator message_end = message.end(); + auto message_begin = message.cbegin(); + auto message_end = message.cend(); if (compress) { @@ -780,8 +809,8 @@ namespace ix compressionError = false; wireSize = _compressedMessage.size(); - message_begin = _compressedMessage.begin(); - message_end = _compressedMessage.end(); + message_begin = _compressedMessage.cbegin(); + message_end = _compressedMessage.cend(); } { @@ -795,6 +824,11 @@ namespace ix if (wireSize < kChunkSize) { success = sendFragment(type, true, message_begin, message_end, compress); + + if (onProgressCallback) + { + onProgressCallback(0, 1); + } } else { @@ -847,7 +881,7 @@ namespace ix // Request to flush the send buffer on the background thread if it isn't empty if (!isSendBufferEmpty()) { - wakeUpFromPoll(Socket::kSendRequest); + wakeUpFromPoll(SelectInterrupt::kSendRequest); // FIXME: we should have a timeout when sending large messages: see #131 if (_blockingSend && !flushSendBuffer()) @@ -859,10 +893,11 @@ namespace ix return WebSocketSendInfo(success, compressionError, payloadSize, wireSize); } + template bool WebSocketTransport::sendFragment(wsheader_type::opcode_type type, bool fin, - std::string::const_iterator message_begin, - std::string::const_iterator message_end, + Iterator message_begin, + Iterator message_end, bool compress) { uint64_t message_size = static_cast(message_end - message_begin); @@ -1055,7 +1090,7 @@ namespace ix else { // no close code/reason set - sendData(wsheader_type::CLOSE, "", compress); + sendData(wsheader_type::CLOSE, std::string(""), compress); } } @@ -1078,13 +1113,10 @@ namespace ix { closeSocket(); - { - std::lock_guard lock(_closeDataMutex); - _closeCode = code; - _closeReason = reason; - _closeWireSize = closeWireSize; - _closeRemote = remote; - } + setCloseReason(reason); + _closeCode = code; + _closeWireSize = closeWireSize; + _closeRemote = remote; setReadyState(ReadyState::CLOSED); _requestInitCancellation = false; @@ -1104,13 +1136,11 @@ namespace ix closeWireSize = reason.size(); } - { - std::lock_guard lock(_closeDataMutex); - _closeCode = code; - _closeReason = reason; - _closeWireSize = closeWireSize; - _closeRemote = remote; - } + setCloseReason(reason); + _closeCode = code; + _closeWireSize = closeWireSize; + _closeRemote = remote; + { std::lock_guard lock(_closingTimePointMutex); _closingTimePoint = std::chrono::steady_clock::now(); @@ -1120,7 +1150,7 @@ namespace ix sendCloseFrame(code, reason); // wake up the poll, but do not close yet - wakeUpFromPoll(Socket::kSendRequest); + wakeUpFromPoll(SelectInterrupt::kSendRequest); } size_t WebSocketTransport::bufferedAmount() const @@ -1155,4 +1185,15 @@ namespace ix return true; } + void WebSocketTransport::setCloseReason(const std::string& reason) + { + std::lock_guard lock(_closeReasonMutex); + _closeReason = reason; + } + + const std::string& WebSocketTransport::getCloseReason() const + { + std::lock_guard lock(_closeReasonMutex); + return _closeReason; + } } // namespace ix diff --git a/ixwebsocket/IXWebSocketTransport.h b/ixwebsocket/IXWebSocketTransport.h index 09b88b6b..777e29c0 100644 --- a/ixwebsocket/IXWebSocketTransport.h +++ b/ixwebsocket/IXWebSocketTransport.h @@ -83,7 +83,9 @@ namespace ix int timeoutSecs); // Server - WebSocketInitResult connectToSocket(std::unique_ptr socket, int timeoutSecs); + WebSocketInitResult connectToSocket(std::unique_ptr socket, + int timeoutSecs, + bool enablePerMessageDeflate); PollResult poll(); WebSocketSendInfo sendBinary(const std::string& message, @@ -146,7 +148,7 @@ namespace ix // Contains all messages that were fetched in the last socket read. // This could be a mix of control messages (Close, Ping, etc...) and - // data messages. That buffer + // data messages. That buffer is resized std::vector _rxbuf; // Contains all messages that are waiting to be sent @@ -178,11 +180,11 @@ namespace ix std::atomic _readyState; OnCloseCallback _onCloseCallback; - uint16_t _closeCode; std::string _closeReason; - size_t _closeWireSize; - bool _closeRemote; - mutable std::mutex _closeDataMutex; + mutable std::mutex _closeReasonMutex; + std::atomic _closeCode; + std::atomic _closeWireSize; + std::atomic _closeRemote; // Data used for Per Message Deflate compression (with zlib) WebSocketPerMessageDeflatePtr _perMessageDeflate; @@ -239,16 +241,15 @@ namespace ix bool sendOnSocket(); bool receiveFromSocket(); + template WebSocketSendInfo sendData(wsheader_type::opcode_type type, - const std::string& message, + const T& message, bool compress, const OnProgressCallback& onProgressCallback = nullptr); - bool sendFragment(wsheader_type::opcode_type type, - bool fin, - std::string::const_iterator begin, - std::string::const_iterator end, - bool compress); + template + bool sendFragment( + wsheader_type::opcode_type type, bool fin, Iterator begin, Iterator end, bool compress); void emitMessage(MessageKind messageKind, const std::string& message, @@ -256,9 +257,11 @@ namespace ix const OnMessageCallback& onMessageCallback); bool isSendBufferEmpty() const; + + template void appendToSendBuffer(const std::vector& header, - std::string::const_iterator begin, - std::string::const_iterator end, + Iterator begin, + Iterator end, uint64_t message_size, uint8_t masking_key[4]); @@ -266,5 +269,8 @@ namespace ix void unmaskReceiveBuffer(const wsheader_type& ws); std::string getMergedChunks() const; + + void setCloseReason(const std::string& reason); + const std::string& getCloseReason() const; }; } // namespace ix diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index f81819d9..5c338a7e 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "9.6.0" +#define IX_WEBSOCKET_VERSION "11.3.1" diff --git a/ixwebsocket/apple/IXSetThreadName_apple.cpp b/ixwebsocket/apple/IXSetThreadName_apple.cpp deleted file mode 100644 index c4291db2..00000000 --- a/ixwebsocket/apple/IXSetThreadName_apple.cpp +++ /dev/null @@ -1,20 +0,0 @@ -/* - * IXSetThreadName_apple.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone, Inc. All rights reserved. - */ -#include "../IXSetThreadName.h" -#include - -namespace ix -{ - void setThreadName(const std::string& name) - { - // - // Apple reserves 16 bytes for its thread names - // Notice that the Apple version of pthread_setname_np - // does not take a pthread_t argument - // - pthread_setname_np(name.substr(0, 63).c_str()); - } -} // namespace ix diff --git a/ixwebsocket/freebsd/IXSetThreadName_freebsd.cpp b/ixwebsocket/freebsd/IXSetThreadName_freebsd.cpp deleted file mode 100644 index 9047e2c0..00000000 --- a/ixwebsocket/freebsd/IXSetThreadName_freebsd.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * IXSetThreadName_freebsd.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ -#include "../IXSetThreadName.h" -#include -#include - -namespace ix -{ - void setThreadName(const std::string& name) - { - pthread_set_name_np(pthread_self(), name.substr(0, 15).c_str()); - } -} // namespace ix diff --git a/ixwebsocket/linux/IXSetThreadName_linux.cpp b/ixwebsocket/linux/IXSetThreadName_linux.cpp deleted file mode 100644 index 1e548efa..00000000 --- a/ixwebsocket/linux/IXSetThreadName_linux.cpp +++ /dev/null @@ -1,20 +0,0 @@ -/* - * IXSetThreadName_linux.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone, Inc. All rights reserved. - */ -#include "../IXSetThreadName.h" -#include - -namespace ix -{ - void setThreadName(const std::string& name) - { - // - // Linux only reserves 16 bytes for its thread names - // See prctl and PR_SET_NAME property in - // http://man7.org/linux/man-pages/man2/prctl.2.html - // - pthread_setname_np(pthread_self(), name.substr(0, 15).c_str()); - } -} // namespace ix diff --git a/ixwebsocket/windows/IXSetThreadName_windows.cpp b/ixwebsocket/windows/IXSetThreadName_windows.cpp deleted file mode 100644 index 7773e5f5..00000000 --- a/ixwebsocket/windows/IXSetThreadName_windows.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * IXSetThreadName_windows.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2019 Machine Zone, Inc. All rights reserved. - */ -#include "../IXSetThreadName.h" - -#include - -namespace ix -{ - const DWORD MS_VC_EXCEPTION = 0x406D1388; - -#pragma pack(push, 8) - typedef struct tagTHREADNAME_INFO - { - DWORD dwType; // Must be 0x1000. - LPCSTR szName; // Pointer to name (in user addr space). - DWORD dwThreadID; // Thread ID (-1=caller thread). - DWORD dwFlags; // Reserved for future use, must be zero. - } THREADNAME_INFO; -#pragma pack(pop) - - void SetThreadName(DWORD dwThreadID, const char* threadName) - { - THREADNAME_INFO info; - info.dwType = 0x1000; - info.szName = threadName; - info.dwThreadID = dwThreadID; - info.dwFlags = 0; - - __try - { - RaiseException( - MS_VC_EXCEPTION, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR*) &info); - } - __except (EXCEPTION_EXECUTE_HANDLER) - { - } - } - - void setThreadName(const std::string& name) - { - SetThreadName(-1, name.c_str()); - } -} // namespace ix diff --git a/main.cpp b/main.cpp new file mode 100644 index 00000000..8512537f --- /dev/null +++ b/main.cpp @@ -0,0 +1,79 @@ +/* + * main.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + * + * Super simple standalone example. See ws folder, unittest and doc/usage.md for more. + * + * On macOS + * $ mkdir -p build ; cd build ; cmake -DUSE_TLS=1 .. ; make -j ; make install + * $ clang++ --std=c++14 --stdlib=libc++ main.cpp -lixwebsocket -lz -framework Security -framework Foundation + * $ ./a.out + * + * Or use cmake -DBUILD_DEMO=ON option for other platform + */ + +#include +#include +#include +#include + +int main() +{ + // Required on Windows + ix::initNetSystem(); + + // Our websocket object + ix::WebSocket webSocket; + + // Connect to a server with encryption + // See https://machinezone.github.io/IXWebSocket/usage/#tls-support-and-configuration + std::string url("wss://echo.websocket.org"); + webSocket.setUrl(url); + + std::cout << ix::userAgent() << std::endl; + std::cout << "Connecting to " << url << "..." << std::endl; + + // Setup a callback to be fired (in a background thread, watch out for race conditions !) + // when a message or an event (open, close, error) is received + webSocket.setOnMessageCallback([](const ix::WebSocketMessagePtr& msg) + { + if (msg->type == ix::WebSocketMessageType::Message) + { + std::cout << "received message: " << msg->str << std::endl; + std::cout << "> " << std::flush; + } + else if (msg->type == ix::WebSocketMessageType::Open) + { + std::cout << "Connection established" << std::endl; + std::cout << "> " << std::flush; + } + else if (msg->type == ix::WebSocketMessageType::Error) + { + // Maybe SSL is not configured properly + std::cout << "Connection error: " << msg->errorInfo.reason << std::endl; + std::cout << "> " << std::flush; + } + } + ); + + // Now that our callback is setup, we can start our background thread and receive messages + webSocket.start(); + + // Send a message to the server (default to TEXT mode) + webSocket.send("hello world"); + + // Display a prompt + std::cout << "> " << std::flush; + + std::string text; + // Read text from the console and send messages in text mode. + // Exit with Ctrl-D on Unix or Ctrl-Z on Windows. + while (std::getline(std::cin, text)) + { + webSocket.send(text); + std::cout << "> " << std::flush; + } + + return 0; +} diff --git a/makefile b/makefile deleted file mode 100644 index 7a5e23dc..00000000 --- a/makefile +++ /dev/null @@ -1,191 +0,0 @@ -# -# This makefile is used for convenience, and wrap simple cmake commands -# You don't need to use it as an end user, it is more for developer. -# -# * work with docker (linux build) -# * execute the unittest -# -# The default target will install ws, the command line tool coming with -# IXWebSocket into /usr/local/bin -# -# -all: brew - -install: brew - -# Use -DCMAKE_INSTALL_PREFIX= to install into another location -# on osx it is good practice to make /usr/local user writable -# sudo chown -R `whoami`/staff /usr/local -# -# Release, Debug, MinSizeRel, RelWithDebInfo are the build types -# -brew: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; ninja install) - -# Docker default target. We've add problem with OpenSSL and TLS 1.3 (on the -# server side ?) and I can't work-around it easily, so we're using mbedtls on -# Linux for the SSL backend, which works great. -ws_mbedtls_install: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; ninja install) - -ws: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 .. ; make -j 4) - -ws_install: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 .. ; make -j 4 install) - -ws_openssl: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_OPEN_SSL=1 .. ; make -j 4) - -ws_mbedtls: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; make -j 4) - -ws_no_ssl: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_WS=1 .. ; make -j 4) - -uninstall: - xargs rm -fv < build/install_manifest.txt - -tag: - git tag v"`sh tools/extract_version.sh`" - -xcode: - cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 -GXcode && open ixwebsocket.xcodeproj - -xcode_openssl: - cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 -GXcode && open ixwebsocket.xcodeproj - -.PHONY: docker - -NAME := ${DOCKER_REPO}/ws -TAG := $(shell sh tools/extract_version.sh) -IMG := ${NAME}:${TAG} -LATEST := ${NAME}:latest -BUILD := ${NAME}:build - -print_version: - @echo 'IXWebSocket version =>' ${TAG} - -set_version: - sh tools/update_version.sh ${VERSION} - -docker_test: - docker build -f docker/Dockerfile.debian -t bsergean/ixwebsocket_test:build . - -docker: - git clean -dfx - docker build -t ${IMG} . - docker tag ${IMG} ${BUILD} - -docker_push: - docker tag ${IMG} ${LATEST} - docker push ${LATEST} - docker push ${IMG} - -deploy: docker docker_push - -run: - docker run --cap-add sys_ptrace --entrypoint=sh -it bsergean/ws:build - -# this is helpful to remove trailing whitespaces -trail: - sh third_party/remote_trailing_whitespaces.sh - -format: - clang-format -i `find test ixwebsocket ws -name '*.cpp' -o -name '*.h'` - -# That target is used to start a node server, but isn't required as we have -# a builtin C++ server started in the unittest now -test_server: - (cd test && npm i ws && node broadcast-server.js) - -# env TEST=Websocket_server make test -# env TEST=Websocket_chat make test -# env TEST=heartbeat make test -test: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; ninja install) - (cd test ; python2.7 run.py -r) - -test_make: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; make -j 4) - (cd test ; python2.7 run.py -r) - -test_tsan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_ubsan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_asan: build_test_asan - (cd test ; python2.7 run.py -r) - -build_test_asan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableAddressSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - -test_tsan_openssl: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_ubsan_openssl: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_tsan_openssl_release: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Release -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -configuration Release -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Release/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_tsan_mbedtls: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_MBED_TLS=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -build_test_openssl: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_OPEN_SSL=1 -DUSE_TEST=1 .. ; ninja install) - -test_openssl: build_test_openssl - (cd test ; python2.7 run.py -r) - -build_test_mbedtls: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_MBED_TLS=1 -DUSE_TEST=1 .. ; make -j 4) - -test_mbedtls: build_test_mbedtls - (cd test ; python2.7 run.py -r) - -test_no_ssl: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TEST=1 .. ; make -j 4) - (cd test ; python2.7 run.py -r) - -ws_test: ws - (cd ws ; env DEBUG=1 PATH=../ws/build:$$PATH bash test_ws.sh) - -autobahn_report: - cp -rvf ~/sandbox/reports/clients/* ../bsergean.github.io/IXWebSocket/autobahn/ - -# For the fork that is configured with appveyor -rebase_upstream: - git fetch upstream - git checkout master - git reset --hard upstream/master - git push origin master --force - -install_cmake_for_linux: - mkdir -p /tmp/cmake - (cd /tmp/cmake ; curl -L -O https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz ; tar zxf cmake-3.14.0-Linux-x86_64.tar.gz) - -# python -m venv venv -# source venv/bin/activate -# pip install mkdocs -doc: - mkdocs gh-deploy - -.PHONY: test -.PHONY: build -.PHONY: ws diff --git a/makefile.dev b/makefile.dev new file mode 100644 index 00000000..e4c57c37 --- /dev/null +++ b/makefile.dev @@ -0,0 +1,220 @@ +# +# This makefile is used for convenience, and wrap simple cmake commands +# You don't need to use it as an end user, it is more for developer. +# +# * work with docker (linux build) +# * execute the unittest +# +# The default target will install ws, the command line tool coming with +# IXWebSocket into /usr/local/bin +# +# +all: brew + +install: brew + +# Use -DCMAKE_INSTALL_PREFIX= to install into another location +# on osx it is good practice to make /usr/local user writable +# sudo chown -R `whoami`/staff /usr/local +# +# Release, Debug, MinSizeRel, RelWithDebInfo are the build types +# +# Default rule does not use python as that requires first time users to have Python3 installed +# +brew: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_UNITY_BUILD=OFF -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; ninja install) + +# Docker default target. We've had problems with OpenSSL and TLS 1.3 (on the +# server side ?) and I can't work-around it easily, so we're using mbedtls on +# Linux for the SSL backend, which works great. +ws_mbedtls_install: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_UNITY_BUILD=ON -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_ZLIB=OFF -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; ninja install) + +ws: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 .. && ninja install) + +ws_unity: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_UNITY_BUILD=ON -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 .. && ninja install) + +ws_install: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 .. -DUSE_TEST=0 && ninja install) + +ws_install_release: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 .. -DUSE_TEST=0 && ninja install) + +ws_openssl_install: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_OPEN_SSL=1 .. ; ninja install) + +ws_mbedtls: + mkdir -p build && (cd build ; cmake -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; make -j 4) + +ws_no_ssl: + mkdir -p build && (cd build ; cmake -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=Debug -DUSE_WS=1 .. ; make -j 4) + +ws_no_python: + mkdir -p build && (cd build ; cmake -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 .. ; make -j4 install) + +uninstall: + xargs rm -fv < build/install_manifest.txt + +tag: + git tag v"`sh tools/extract_version.sh`" + +xcode: + cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 -GXcode && open ixwebsocket.xcodeproj + +xcode_openssl: + cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 -GXcode && open ixwebsocket.xcodeproj + +.PHONY: docker + +NAME := ${DOCKER_REPO}/ws +TAG := $(shell sh tools/extract_version.sh) +IMG := ${NAME}:${TAG} +LATEST := ${NAME}:latest +BUILD := ${NAME}:build + +print_version: + @echo 'IXWebSocket version =>' ${TAG} + +set_version: + sh tools/update_version.sh ${VERSION} + +docker_test: + docker build -f docker/Dockerfile.debian -t bsergean/ixwebsocket_test:build . + +docker: + git clean -dfx + docker build -t ${IMG} . + docker tag ${IMG} ${BUILD} + +docker_push: + docker tag ${IMG} ${LATEST} + docker push ${LATEST} + docker push ${IMG} + +deploy: docker docker_push + +run: + docker run --cap-add sys_ptrace --entrypoint=sh -it bsergean/ws:build + +# this is helpful to remove trailing whitespaces +trail: + sh third_party/remote_trailing_whitespaces.sh + +format: + clang-format -i `find test ixwebsocket ws -name '*.cpp' -o -name '*.h'` + +# That target is used to start a node server, but isn't required as we have +# a builtin C++ server started in the unittest now +test_server: + (cd test && npm i ws && node broadcast-server.js) + +test: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_UNITY_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 ..) + (cd build ; ninja) + (cd build ; ninja -v test) + +test_asan: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_UNITY_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ctest -V .) + +test_tsan_mbedtls: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_UNITY_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_MBED_TLS=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=thread -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) + +test_tsan_openssl: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_UNITY_BUILD=ON DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_OPENS_SSL=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=thread -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) + +test_tsan_sectransport: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_INSTALL_MESSAGE=LAZY -DCMAKE_UNITY_BUILD=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_OPENS_SSL=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=thread -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) + +ws_test: ws + (cd ws ; env DEBUG=1 PATH=../ws/build:$$PATH bash test_ws.sh) + +autobahn_report: + cp -rvf ~/sandbox/reports/clients/* ../bsergean.github.io/IXWebSocket/autobahn/ + +httpd: + clang++ --std=c++14 --stdlib=libc++ -o ixhttpd httpd.cpp \ + ixwebsocket/IXSelectInterruptFactory.cpp \ + ixwebsocket/IXCancellationRequest.cpp \ + ixwebsocket/IXSocketTLSOptions.cpp \ + ixwebsocket/IXUserAgent.cpp \ + ixwebsocket/IXDNSLookup.cpp \ + ixwebsocket/IXBench.cpp \ + ixwebsocket/IXWebSocketHttpHeaders.cpp \ + ixwebsocket/IXSelectInterruptPipe.cpp \ + ixwebsocket/IXHttp.cpp \ + ixwebsocket/IXSocketConnect.cpp \ + ixwebsocket/IXSocket.cpp \ + ixwebsocket/IXSocketServer.cpp \ + ixwebsocket/IXNetSystem.cpp \ + ixwebsocket/IXHttpServer.cpp \ + ixwebsocket/IXSocketFactory.cpp \ + ixwebsocket/IXConnectionState.cpp \ + ixwebsocket/IXUrlParser.cpp \ + ixwebsocket/IXSelectInterrupt.cpp \ + ixwebsocket/IXSetThreadName.cpp \ + -lz + +httpd_linux: + g++ --std=c++14 -o ixhttpd httpd.cpp -Iixwebsocket \ + ixwebsocket/IXSelectInterruptFactory.cpp \ + ixwebsocket/IXCancellationRequest.cpp \ + ixwebsocket/IXSocketTLSOptions.cpp \ + ixwebsocket/IXUserAgent.cpp \ + ixwebsocket/IXDNSLookup.cpp \ + ixwebsocket/IXBench.cpp \ + ixwebsocket/IXWebSocketHttpHeaders.cpp \ + ixwebsocket/IXSelectInterruptPipe.cpp \ + ixwebsocket/IXHttp.cpp \ + ixwebsocket/IXSocketConnect.cpp \ + ixwebsocket/IXSocket.cpp \ + ixwebsocket/IXSocketServer.cpp \ + ixwebsocket/IXNetSystem.cpp \ + ixwebsocket/IXHttpServer.cpp \ + ixwebsocket/IXSocketFactory.cpp \ + ixwebsocket/IXConnectionState.cpp \ + ixwebsocket/IXUrlParser.cpp \ + ixwebsocket/IXSelectInterrupt.cpp \ + ixwebsocket/IXSetThreadName.cpp \ + -lz -lpthread + cp -f ixhttpd /usr/local/bin + +# For the fork that is configured with appveyor +rebase_upstream: + git fetch upstream + git checkout master + git reset --hard upstream/master + git push origin master --force + +# Legacy target +install_cmake_for_linux: + mkdir -p /tmp/cmake + (cd /tmp/cmake ; curl -L -O https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz ; tar zxf cmake-3.14.0-Linux-x86_64.tar.gz) + +# python -m venv venv +# source venv/bin/activate +# pip install mkdocs +doc: + mkdocs gh-deploy + +change: format + vim ixwebsocket/IXWebSocketVersion.h docs/CHANGELOG.md + +change_no_format: + vim ixwebsocket/IXWebSocketVersion.h docs/CHANGELOG.md + +commit: + git commit -am "`sh tools/extract_latest_change.sh`" + +.PHONY: test +.PHONY: build +.PHONY: ws diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 70599804..661aebb1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -2,71 +2,45 @@ # Author: Benjamin Sergeant # Copyright (c) 2018 Machine Zone, Inc. All rights reserved. # -cmake_minimum_required (VERSION 3.4.1) +cmake_minimum_required (VERSION 3.14) project (ixwebsocket_unittest) -set (CMAKE_CXX_STANDARD 14) +set (CMAKE_CXX_STANDARD 11) -if (MAC) - set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../third_party/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH}) - find_package(Sanitizers) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") - set(CMAKE_LD_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") - option(USE_TLS "Add TLS support" ON) -endif() - -include_directories( - ${PROJECT_SOURCE_DIR}/Catch2/single_include - ../third_party - ../third_party/msgpack11 - ../third_party/spdlog/include - ../ws -) - -add_definitions(-DSPDLOG_COMPILED_LIB=1) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - include_directories(../third_party/jsoncpp) - set(JSONCPP_SOURCES ../third_party/jsoncpp/jsoncpp.cpp) -endif() +option(USE_TLS "Add TLS support" ON) # Shared sources -set (SOURCES - ${JSONCPP_SOURCES} - - test_runner.cpp - IXTest.cpp - IXGetFreePort.cpp - ../third_party/msgpack11/msgpack11.cpp - - IXSocketTest.cpp - IXSocketConnectTest.cpp - IXWebSocketServerTest.cpp - IXWebSocketTestConnectionDisconnection.cpp - IXUrlParserTest.cpp - IXWebSocketServerTest.cpp - IXHttpClientTest.cpp - IXHttpServerTest.cpp - IXUnityBuildsTest.cpp - IXHttpTest.cpp - IXDNSLookupTest.cpp - IXWebSocketSubProtocolTest.cpp - IXSentryClientTest.cpp - IXWebSocketChatTest.cpp - IXWebSocketBroadcastTest.cpp +set (TEST_TARGET_NAMES + IXSocketTest + IXSocketConnectTest + IXWebSocketServerTest + IXWebSocketTestConnectionDisconnection + IXUrlParserTest + IXHttpClientTest + IXUnityBuildsTest + IXHttpTest + IXDNSLookupTest + IXWebSocketSubProtocolTest + # IXWebSocketBroadcastTest ## FIXME was depending on cobra / take a broadcast server from ws + IXStrCaseCompareTest ) # Some unittest don't work on windows yet # Windows without TLS does not have hmac yet if (UNIX) - list(APPEND SOURCES - IXWebSocketCloseTest.cpp - IXCobraChatTest.cpp - IXCobraMetricsPublisherTest.cpp - IXCobraToSentryBotTest.cpp - IXCobraToStatsdBotTest.cpp - IXCobraToStdoutBotTest.cpp + list(APPEND TEST_TARGET_NAMES + IXWebSocketCloseTest + + # Fail on Windows in CI probably because the pathing is wrong and + # some resource files cannot be found + IXHttpServerTest + IXWebSocketChatTest + ) +endif() + +if (USE_ZLIB) + list(APPEND TEST_TARGET_NAMES + IXWebSocketPerMessageDeflateCompressorTest ) endif() @@ -74,32 +48,51 @@ endif() # IXWebSocketPingTest.cpp # IXWebSocketPingTimeoutTest.cpp +# IXWebSocketLeakTest.cpp # commented until we have a fix for #224 / +# that was was fixed but now the test does not compile + # Disable tests for now that are failing or not reliable -add_executable(ixwebsocket_unittest ${SOURCES}) +add_library(ixwebsocket_test) +target_sources(ixwebsocket_test PRIVATE + ${JSONCPP_SOURCES} + test_runner.cpp + IXTest.cpp + ../third_party/msgpack11/msgpack11.cpp +) +target_compile_definitions(ixwebsocket_test PRIVATE ${TEST_PROGRAMS_DEFINITIONS}) +target_include_directories(ixwebsocket_test PRIVATE + ${PROJECT_SOURCE_DIR}/Catch2/single_include + ../third_party +) +target_link_libraries(ixwebsocket_test ixwebsocket) +target_link_libraries(ixwebsocket_test spdlog) -if (MAC) - add_sanitizers(ixwebsocket_unittest) -endif() +foreach(TEST_TARGET_NAME ${TEST_TARGET_NAMES}) + add_executable(${TEST_TARGET_NAME} + ${TEST_TARGET_NAME}.cpp + ) -if (APPLE AND USE_TLS) - target_link_libraries(ixwebsocket_unittest "-framework foundation" "-framework security") -endif() + target_include_directories(${TEST_TARGET_NAME} PRIVATE + ${PROJECT_SOURCE_DIR}/Catch2/single_include + ../third_party + ../third_party/msgpack11 + ) -if (JSONCPP_FOUND) - target_include_directories(ixwebsocket_unittest PUBLIC ${JSONCPP_INCLUDE_DIRS}) - target_link_libraries(ixwebsocket_unittest ${JSONCPP_LIBRARIES}) -endif() + target_compile_definitions(${TEST_TARGET_NAME} PRIVATE SPDLOG_COMPILED_LIB=1) -# library with the most dependencies come first -target_link_libraries(ixwebsocket_unittest ixbots) -target_link_libraries(ixwebsocket_unittest ixsnake) -target_link_libraries(ixwebsocket_unittest ixcobra) -target_link_libraries(ixwebsocket_unittest ixsentry) -target_link_libraries(ixwebsocket_unittest ixwebsocket) -target_link_libraries(ixwebsocket_unittest ixcrypto) -target_link_libraries(ixwebsocket_unittest ixcore) + if (APPLE AND USE_TLS) + target_link_libraries(${TEST_TARGET_NAME} "-framework foundation" "-framework security") + endif() -target_link_libraries(ixwebsocket_unittest spdlog) + # library with the most dependencies come first + target_link_libraries(${TEST_TARGET_NAME} ixwebsocket_test) + target_link_libraries(${TEST_TARGET_NAME} ixwebsocket) -install(TARGETS ixwebsocket_unittest DESTINATION bin) + target_link_libraries(${TEST_TARGET_NAME} spdlog) + + add_test(NAME ${TEST_TARGET_NAME} + COMMAND ${TEST_TARGET_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +endforeach() diff --git a/test/IXCobraChatTest.cpp b/test/IXCobraChatTest.cpp deleted file mode 100644 index 6ffbd80f..00000000 --- a/test/IXCobraChatTest.cpp +++ /dev/null @@ -1,349 +0,0 @@ -/* - * cmd_satori_chat.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2017 Machine Zone. All rights reserved. - */ - -#include "IXTest.h" -#include "catch.hpp" -#include -#include -#include -#include -#include -#include - -using namespace ix; - -namespace -{ - std::atomic incomingBytes(0); - std::atomic outgoingBytes(0); - - void setupTrafficTrackerCallback() - { - ix::CobraConnection::setTrafficTrackerCallback([](size_t size, bool incoming) { - if (incoming) - { - incomingBytes += size; - } - else - { - outgoingBytes += size; - } - }); - } - - class CobraChat - { - public: - CobraChat(const std::string& user, - const std::string& session, - const ix::CobraConfig& config); - - void subscribe(const std::string& channel); - void start(); - void stop(); - void run(); - bool isReady() const; - - void sendMessage(const std::string& text); - size_t getReceivedMessagesCount() const; - - bool hasPendingMessages() const; - Json::Value popMessage(); - - private: - std::string _user; - std::string _session; - ix::CobraConfig _cobraConfig; - - std::queue _publish_queue; - mutable std::mutex _queue_mutex; - - std::thread _thread; - std::atomic _stop; - - ix::CobraConnection _conn; - std::atomic _connectedAndSubscribed; - - std::queue _receivedQueue; - - std::mutex _logMutex; - }; - - CobraChat::CobraChat(const std::string& user, - const std::string& session, - const ix::CobraConfig& config) - : _user(user) - , _session(session) - , _cobraConfig(config) - , _stop(false) - , _connectedAndSubscribed(false) - { - } - - void CobraChat::start() - { - _thread = std::thread(&CobraChat::run, this); - } - - void CobraChat::stop() - { - _stop = true; - _thread.join(); - } - - bool CobraChat::isReady() const - { - return _connectedAndSubscribed; - } - - size_t CobraChat::getReceivedMessagesCount() const - { - return _receivedQueue.size(); - } - - bool CobraChat::hasPendingMessages() const - { - std::unique_lock lock(_queue_mutex); - return !_publish_queue.empty(); - } - - Json::Value CobraChat::popMessage() - { - std::unique_lock lock(_queue_mutex); - auto msg = _publish_queue.front(); - _publish_queue.pop(); - return msg; - } - - // - // Callback to handle received messages, that are printed on the console - // - void CobraChat::subscribe(const std::string& channel) - { - std::string filter; - std::string position("$"); - - _conn.subscribe(channel, - filter, - position, - [this](const Json::Value& msg, const std::string& /*position*/) { - spdlog::info("receive {}", msg.toStyledString()); - - if (!msg.isObject()) return; - if (!msg.isMember("user")) return; - if (!msg.isMember("text")) return; - if (!msg.isMember("session")) return; - - std::string msg_user = msg["user"].asString(); - std::string msg_text = msg["text"].asString(); - std::string msg_session = msg["session"].asString(); - - // We are not interested in messages - // from a different session. - if (msg_session != _session) return; - - // We are not interested in our own messages - if (msg_user == _user) return; - - _receivedQueue.push(msg); - - std::stringstream ss; - ss << std::endl - << msg_user << " > " << msg_text << std::endl - << _user << " > "; - log(ss.str()); - }); - } - - void CobraChat::sendMessage(const std::string& text) - { - Json::Value msg; - msg["user"] = _user; - msg["session"] = _session; - msg["text"] = text; - - std::unique_lock lock(_queue_mutex); - _publish_queue.push(msg); - } - - // - // Do satori communication on a background thread, where we can have - // something like an event loop that publish, poll and receive data - // - void CobraChat::run() - { - std::string channel = _session; - - _conn.configure(_cobraConfig); - _conn.connect(); - - _conn.setEventCallback([this, channel](const CobraEventPtr& event) { - if (event->type == ix::CobraEventType::Open) - { - log("Subscriber connected: " + _user); - for (auto&& it : event->headers) - { - log("Headers " + it.first + " " + it.second); - } - } - else if (event->type == ix::CobraEventType::Authenticated) - { - log("Subscriber authenticated: " + _user); - subscribe(channel); - } - else if (event->type == ix::CobraEventType::Error) - { - log(event->errMsg + _user); - } - else if (event->type == ix::CobraEventType::Closed) - { - log("Connection closed: " + _user); - } - else if (event->type == ix::CobraEventType::Subscribed) - { - log("Subscription ok: " + _user + " subscription_id " + event->subscriptionId); - _connectedAndSubscribed = true; - } - else if (event->type == ix::CobraEventType::UnSubscribed) - { - log("Unsubscription ok: " + _user + " subscription_id " + event->subscriptionId); - } - else if (event->type == ix::CobraEventType::Published) - { - TLogger() << "Subscriber: published message acked: " << event->msgId; - } - }); - - while (!_stop) - { - { - while (hasPendingMessages()) - { - auto msg = popMessage(); - - std::string text = msg["text"].asString(); - - std::stringstream ss; - ss << "Sending msg [" << text << "]"; - log(ss.str()); - - Json::Value channels; - channels.append(channel); - _conn.publish(channels, msg); - } - } - - ix::msleep(50); - } - - _conn.unsubscribe(channel); - - ix::msleep(50); - _conn.disconnect(); - - _conn.setEventCallback([](const CobraEventPtr& /*event*/) {}); - } -} // namespace - -TEST_CASE("Cobra_chat", "[cobra_chat]") -{ - SECTION("Exchange and count sent/received messages.") - { - int port = getFreePort(); - snake::AppConfig appConfig = makeSnakeServerConfig(port, true); - - // Start a redis server - ix::RedisServer redisServer(appConfig.redisPort); - auto res = redisServer.listen(); - REQUIRE(res.first); - redisServer.start(); - - // Start a snake server - snake::SnakeServer snakeServer(appConfig); - snakeServer.run(); - - int timeout; - setupTrafficTrackerCallback(); - - std::string session = ix::generateSessionId(); - std::string appkey("FC2F10139A2BAc53BB72D9db967b024f"); - std::string role = "_sub"; - std::string secret = "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba"; - std::string endpoint = makeCobraEndpoint(port, true); - - ix::CobraConfig config; - config.endpoint = endpoint; - config.appkey = appkey; - config.rolename = role; - config.rolesecret = secret; - config.socketTLSOptions = makeClientTLSOptions(); - - CobraChat chatA("jean", session, config); - CobraChat chatB("paul", session, config); - - chatA.start(); - chatB.start(); - - // Wait for all chat instance to be ready - timeout = 10 * 1000; // 10s - while (true) - { - if (chatA.isReady() && chatB.isReady()) break; - ix::msleep(10); - - timeout -= 10; - if (timeout <= 0) - { - snakeServer.stop(); - redisServer.stop(); - REQUIRE(false); // timeout - } - } - - // Add a bit of extra time, for the subscription to be active - ix::msleep(1000); - - chatA.sendMessage("from A1"); - chatA.sendMessage("from A2"); - chatA.sendMessage("from A3"); - - chatB.sendMessage("from B1"); - chatB.sendMessage("from B2"); - - // 1. Wait for all messages to be sent - timeout = 10 * 1000; // 10s - while (chatA.hasPendingMessages() || chatB.hasPendingMessages()) - { - ix::msleep(10); - - timeout -= 10; - if (timeout <= 0) - { - snakeServer.stop(); - redisServer.stop(); - REQUIRE(false); // timeout - } - } - - // Give us 1s for all messages to be received - ix::msleep(1000); - - chatA.stop(); - chatB.stop(); - - REQUIRE(chatA.getReceivedMessagesCount() == 2); - REQUIRE(chatB.getReceivedMessagesCount() == 3); - - spdlog::info("Incoming bytes {}", incomingBytes); - spdlog::info("Outgoing bytes {}", outgoingBytes); - - spdlog::info("Stopping snake server..."); - snakeServer.stop(); - - spdlog::info("Stopping redis server..."); - redisServer.stop(); - } -} diff --git a/test/IXCobraMetricsPublisherTest.cpp b/test/IXCobraMetricsPublisherTest.cpp deleted file mode 100644 index 10653bd3..00000000 --- a/test/IXCobraMetricsPublisherTest.cpp +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Author: Benjamin Sergeant - * Copyright (c) 2018 Machine Zone. All rights reserved. - */ - -#include "IXTest.h" -#include "catch.hpp" -#include -#include -#include -#include -#include -#include - -using namespace ix; - -namespace -{ - std::atomic incomingBytes(0); - std::atomic outgoingBytes(0); - - void setupTrafficTrackerCallback() - { - ix::CobraConnection::setTrafficTrackerCallback([](size_t size, bool incoming) { - if (incoming) - { - incomingBytes += size; - } - else - { - outgoingBytes += size; - } - }); - } - - std::atomic gStop; - std::atomic gSubscriberConnectedAndSubscribed; - std::atomic gUniqueMessageIdsCount; - std::atomic gMessageCount; - - std::set gIds; - std::mutex gProtectIds; // std::set is no thread-safe, so protect access with this mutex. - - // - // Background thread subscribe to the channel and validates what was sent - // - void startSubscriber(const ix::CobraConfig& config, const std::string& channel) - { - gSubscriberConnectedAndSubscribed = false; - gUniqueMessageIdsCount = 0; - gMessageCount = 0; - - ix::CobraConnection conn; - conn.configure(config); - conn.connect(); - - conn.setEventCallback([&conn, &channel](const CobraEventPtr& event) { - if (event->type == ix::CobraEventType::Open) - { - TLogger() << "Subscriber connected:"; - for (auto&& it : event->headers) - { - log("Headers " + it.first + " " + it.second); - } - } - else if (event->type == ix::CobraEventType::Closed) - { - TLogger() << "Subscriber closed:" << event->errMsg; - } - else if (event->type == ix::CobraEventType::Error) - { - TLogger() << "Subscriber error:" << event->errMsg; - } - else if (event->type == ix::CobraEventType::Authenticated) - { - log("Subscriber authenticated"); - std::string filter; - std::string position("$"); - - conn.subscribe(channel, - filter, - position, - [](const Json::Value& msg, const std::string& /*position*/) { - log(msg.toStyledString()); - - std::string id = msg["id"].asString(); - { - std::lock_guard guard(gProtectIds); - gIds.insert(id); - } - - gMessageCount++; - }); - } - else if (event->type == ix::CobraEventType::Subscribed) - { - TLogger() << "Subscriber: subscribed to channel " << event->subscriptionId; - if (event->subscriptionId == channel) - { - gSubscriberConnectedAndSubscribed = true; - } - else - { - TLogger() << "Subscriber: unexpected channel " << event->subscriptionId; - } - } - else if (event->type == ix::CobraEventType::UnSubscribed) - { - TLogger() << "Subscriber: ununexpected from channel " << event->subscriptionId; - if (event->subscriptionId != channel) - { - TLogger() << "Subscriber: unexpected channel " << event->subscriptionId; - } - } - else if (event->type == ix::CobraEventType::Published) - { - TLogger() << "Subscriber: published message acked: " << event->msgId; - } - }); - - while (!gStop) - { - std::chrono::duration duration(10); - std::this_thread::sleep_for(duration); - } - - conn.unsubscribe(channel); - conn.disconnect(); - - gUniqueMessageIdsCount = gIds.size(); - } - - // publish 100 messages, during roughly 100ms - // this is used to test thread safety of CobraMetricsPublisher::push - void runAdditionalPublisher(ix::CobraMetricsPublisher* cobraMetricsPublisher) - { - Json::Value data; - data["foo"] = "bar"; - - for (int i = 0; i < 100; ++i) - { - cobraMetricsPublisher->push("sms_metric_F_id", data); - ix::msleep(1); - } - } - -} // namespace - -TEST_CASE("Cobra_Metrics_Publisher", "[cobra]") -{ - int port = getFreePort(); - bool preferTLS = true; - snake::AppConfig appConfig = makeSnakeServerConfig(port, preferTLS); - - // Start a redis server - ix::RedisServer redisServer(appConfig.redisPort); - auto res = redisServer.listen(); - REQUIRE(res.first); - redisServer.start(); - - // Start a snake server - snake::SnakeServer snakeServer(appConfig); - snakeServer.run(); - - setupTrafficTrackerCallback(); - - std::string channel = ix::generateSessionId(); - std::string endpoint = makeCobraEndpoint(port, preferTLS); - std::string appkey("FC2F10139A2BAc53BB72D9db967b024f"); - std::string role = "_sub"; - std::string secret = "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba"; - - ix::CobraConfig config; - config.endpoint = endpoint; - config.appkey = appkey; - config.rolename = role; - config.rolesecret = secret; - config.socketTLSOptions = makeClientTLSOptions(); - - gStop = false; - std::thread subscriberThread(&startSubscriber, config, channel); - - int timeout = 10 * 1000; // 10s - - // Wait until the subscriber is ready (authenticated + subscription successful) - while (!gSubscriberConnectedAndSubscribed) - { - std::chrono::duration duration(10); - std::this_thread::sleep_for(duration); - - timeout -= 10; - if (timeout <= 0) - { - snakeServer.stop(); - redisServer.stop(); - REQUIRE(false); // timeout - } - } - - ix::CobraMetricsPublisher cobraMetricsPublisher; - cobraMetricsPublisher.configure(config, channel); - cobraMetricsPublisher.setSession(uuid4()); - cobraMetricsPublisher.enable(true); - - Json::Value data; - data["foo"] = "bar"; - - // (1) Publish without restrictions - cobraMetricsPublisher.push("sms_metric_A_id", data); // (msg #1) - cobraMetricsPublisher.push("sms_metric_B_id", data); // (msg #2) - - // (2) Restrict what is sent using a blacklist - // Add one entry to the blacklist - // (will send msg #3) - cobraMetricsPublisher.setBlacklist({ - "sms_metric_B_id" // this id will not be sent - }); - // (msg #4) - cobraMetricsPublisher.push("sms_metric_A_id", data); - // ... - cobraMetricsPublisher.push("sms_metric_B_id", data); // this won't be sent - - // Reset the blacklist - // (msg #5) - cobraMetricsPublisher.setBlacklist({}); // 4. - - // (3) Restrict what is sent using rate control - - // (msg #6) - cobraMetricsPublisher.setRateControl({ - {"sms_metric_C_id", 1}, // published once per minute (60 seconds) max - }); - // (msg #7) - cobraMetricsPublisher.push("sms_metric_C_id", data); - cobraMetricsPublisher.push("sms_metric_C_id", data); // this won't be sent - - ix::msleep(1400); - - // (msg #8) - cobraMetricsPublisher.push("sms_metric_C_id", data); // now this will be sent - - ix::msleep(600); // wait a bit so that the last message is sent and can be received - - log("Testing suspend/resume now, which will disconnect the cobraMetricsPublisher."); - - // Test suspend + resume - for (int i = 0; i < 3; ++i) - { - cobraMetricsPublisher.suspend(); - ix::msleep(500); - REQUIRE(!cobraMetricsPublisher.isConnected()); // Check that we are not connected anymore - - cobraMetricsPublisher.push("sms_metric_D_id", data); // will not be sent this time - - cobraMetricsPublisher.resume(); - ix::msleep(2000); // give cobra 2s to connect - REQUIRE(cobraMetricsPublisher.isConnected()); // Check that we are connected now - - cobraMetricsPublisher.push("sms_metric_E_id", data); - } - - ix::msleep(500); - - // Test multi-threaded publish - std::thread bgPublisher1(&runAdditionalPublisher, &cobraMetricsPublisher); - std::thread bgPublisher2(&runAdditionalPublisher, &cobraMetricsPublisher); - std::thread bgPublisher3(&runAdditionalPublisher, &cobraMetricsPublisher); - std::thread bgPublisher4(&runAdditionalPublisher, &cobraMetricsPublisher); - std::thread bgPublisher5(&runAdditionalPublisher, &cobraMetricsPublisher); - - bgPublisher1.join(); - bgPublisher2.join(); - bgPublisher3.join(); - bgPublisher4.join(); - bgPublisher5.join(); - - // Now stop the thread - gStop = true; - subscriberThread.join(); - - // - // Validate that we received all message kinds, and the correct number of messages - // - CHECK(gIds.count("sms_metric_A_id") == 1); - CHECK(gIds.count("sms_metric_B_id") == 1); - CHECK(gIds.count("sms_metric_C_id") == 1); - CHECK(gIds.count("sms_metric_D_id") == 1); - CHECK(gIds.count("sms_metric_E_id") == 1); - CHECK(gIds.count("sms_metric_F_id") == 1); - CHECK(gIds.count("sms_set_rate_control_id") == 1); - CHECK(gIds.count("sms_set_blacklist_id") == 1); - - spdlog::info("Incoming bytes {}", incomingBytes); - spdlog::info("Outgoing bytes {}", outgoingBytes); - - spdlog::info("Stopping snake server..."); - snakeServer.stop(); - - spdlog::info("Stopping redis server..."); - redisServer.stop(); -} diff --git a/test/IXCobraToSentryBotTest.cpp b/test/IXCobraToSentryBotTest.cpp deleted file mode 100644 index b61c06fe..00000000 --- a/test/IXCobraToSentryBotTest.cpp +++ /dev/null @@ -1,182 +0,0 @@ -/* - * IXCobraToSentryTest.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone. All rights reserved. - */ - -#include "IXTest.h" -#include "catch.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace ix; - -namespace -{ - std::atomic incomingBytes(0); - std::atomic outgoingBytes(0); - - void setupTrafficTrackerCallback() - { - ix::CobraConnection::setTrafficTrackerCallback([](size_t size, bool incoming) { - if (incoming) - { - incomingBytes += size; - } - else - { - outgoingBytes += size; - } - }); - } - - void runPublisher(const ix::CobraConfig& config, const std::string& channel) - { - ix::CobraMetricsPublisher cobraMetricsPublisher; - cobraMetricsPublisher.configure(config, channel); - cobraMetricsPublisher.setSession(uuid4()); - cobraMetricsPublisher.enable(true); - - Json::Value msg; - msg["fps"] = 60; - - cobraMetricsPublisher.setGenericAttributes("game", "ody"); - - // Wait a bit - ix::msleep(500); - - // publish some messages - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #1) - cobraMetricsPublisher.push("sms_metric_B_id", msg); // (msg #2) - ix::msleep(500); - - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #3) - cobraMetricsPublisher.push("sms_metric_D_id", msg); // (msg #4) - ix::msleep(500); - - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #4) - cobraMetricsPublisher.push("sms_metric_F_id", msg); // (msg #5) - ix::msleep(500); - } -} // namespace - -TEST_CASE("Cobra_to_sentry_bot", "[cobra_bots]") -{ - SECTION("Exchange and count sent/received messages.") - { - int port = getFreePort(); - snake::AppConfig appConfig = makeSnakeServerConfig(port, true); - - // Start a redis server - ix::RedisServer redisServer(appConfig.redisPort); - auto res = redisServer.listen(); - REQUIRE(res.first); - redisServer.start(); - - // Start a snake server - snake::SnakeServer snakeServer(appConfig); - snakeServer.run(); - - // Start a fake sentry http server - SocketTLSOptions tlsOptionsServer = makeServerTLSOptions(true); - - int sentryPort = getFreePort(); - ix::HttpServer sentryServer(sentryPort, "127.0.0.1"); - sentryServer.setTLSOptions(tlsOptionsServer); - - sentryServer.setOnConnectionCallback( - [](HttpRequestPtr request, - std::shared_ptr /*connectionState*/) -> HttpResponsePtr { - WebSocketHttpHeaders headers; - headers["Server"] = userAgent(); - - // Log request - std::stringstream ss; - ss << request->method << " " << request->headers["User-Agent"] << " " - << request->uri; - - if (request->method == "POST") - { - return std::make_shared( - 200, "OK", HttpErrorCode::Ok, headers, std::string()); - } - else - { - return std::make_shared( - 405, "OK", HttpErrorCode::Invalid, headers, std::string("Invalid method")); - } - }); - - res = sentryServer.listen(); - REQUIRE(res.first); - sentryServer.start(); - - setupTrafficTrackerCallback(); - - // Run the bot for a small amount of time - std::string channel = ix::generateSessionId(); - std::string appkey("FC2F10139A2BAc53BB72D9db967b024f"); - std::string role = "_sub"; - std::string secret = "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba"; - std::string endpoint = makeCobraEndpoint(port, true); - - ix::CobraConfig config; - config.endpoint = endpoint; - config.appkey = appkey; - config.rolename = role; - config.rolesecret = secret; - config.socketTLSOptions = makeClientTLSOptions(); - - std::thread publisherThread(runPublisher, config, channel); - - ix::CobraBotConfig cobraBotConfig; - cobraBotConfig.cobraConfig = config; - cobraBotConfig.channel = channel; - cobraBotConfig.runtime = 3; // Only run the bot for 3 seconds - cobraBotConfig.enableHeartbeat = false; - bool verbose = true; - - // FIXME: try to get this working with https instead of http - // to regress the TLS 1.3 OpenSSL bug - // -> https://github.com/openssl/openssl/issues/7967 - // https://xxxxx:yyyyyy@sentry.io/1234567 - std::stringstream oss; - oss << getHttpScheme() << "xxxxxxx:yyyyyyy@localhost:" << sentryPort << "/1234567"; - std::string dsn = oss.str(); - - SocketTLSOptions tlsOptionsClient = makeClientTLSOptions(); - - SentryClient sentryClient(dsn); - sentryClient.setTLSOptions(tlsOptionsClient); - - int64_t sentCount = cobra_to_sentry_bot(cobraBotConfig, sentryClient, verbose); - // - // We want at least 2 messages to be sent - // - REQUIRE(sentCount >= 2); - - // Give us 1s for all messages to be received - ix::msleep(1000); - - spdlog::info("Incoming bytes {}", incomingBytes); - spdlog::info("Outgoing bytes {}", outgoingBytes); - - spdlog::info("Stopping snake server..."); - snakeServer.stop(); - - spdlog::info("Stopping redis server..."); - redisServer.stop(); - - publisherThread.join(); - sentryServer.stop(); - } -} diff --git a/test/IXCobraToStatsdBotTest.cpp b/test/IXCobraToStatsdBotTest.cpp deleted file mode 100644 index ff436c08..00000000 --- a/test/IXCobraToStatsdBotTest.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * IXCobraToStatsdTest.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone. All rights reserved. - */ - -#include "IXTest.h" -#include "catch.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace ix; - -namespace -{ - void runPublisher(const ix::CobraConfig& config, const std::string& channel) - { - ix::CobraMetricsPublisher cobraMetricsPublisher; - cobraMetricsPublisher.configure(config, channel); - cobraMetricsPublisher.setSession(uuid4()); - cobraMetricsPublisher.enable(true); - - Json::Value msg; - msg["fps"] = 60; - - cobraMetricsPublisher.setGenericAttributes("game", "ody"); - - // Wait a bit - ix::msleep(500); - - // publish some messages - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #1) - cobraMetricsPublisher.push("sms_metric_B_id", msg); // (msg #2) - ix::msleep(500); - - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #3) - cobraMetricsPublisher.push("sms_metric_D_id", msg); // (msg #4) - ix::msleep(500); - - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #4) - cobraMetricsPublisher.push("sms_metric_F_id", msg); // (msg #5) - ix::msleep(500); - } -} // namespace - -TEST_CASE("Cobra_to_statsd_bot", "[cobra_bots]") -{ - SECTION("Exchange and count sent/received messages.") - { - int port = getFreePort(); - snake::AppConfig appConfig = makeSnakeServerConfig(port, true); - - // Start a redis server - ix::RedisServer redisServer(appConfig.redisPort); - auto res = redisServer.listen(); - REQUIRE(res.first); - redisServer.start(); - - // Start a snake server - snake::SnakeServer snakeServer(appConfig); - snakeServer.run(); - - // Start a fake statsd server (ultimately) - - // Run the bot for a small amount of time - std::string channel = ix::generateSessionId(); - std::string appkey("FC2F10139A2BAc53BB72D9db967b024f"); - std::string role = "_sub"; - std::string secret = "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba"; - std::string endpoint = makeCobraEndpoint(port, true); - - ix::CobraConfig config; - config.endpoint = endpoint; - config.appkey = appkey; - config.rolename = role; - config.rolesecret = secret; - config.socketTLSOptions = makeClientTLSOptions(); - - std::thread publisherThread(runPublisher, config, channel); - - ix::CobraBotConfig cobraBotConfig; - cobraBotConfig.cobraConfig = config; - cobraBotConfig.channel = channel; - cobraBotConfig.runtime = 3; // Only run the bot for 3 seconds - cobraBotConfig.enableHeartbeat = false; - - std::string hostname("127.0.0.1"); - // std::string hostname("www.google.com"); - int statsdPort = 8125; - std::string prefix("ix.test"); - StatsdClient statsdClient(hostname, statsdPort, prefix); - - std::string errMsg; - bool initialized = statsdClient.init(errMsg); - if (!initialized) - { - spdlog::error(errMsg); - } - REQUIRE(initialized); - - std::string fields("device.game\ndevice.os_name"); - std::string gauge; - std::string timer; - bool verbose = true; - - int64_t sentCount = - ix::cobra_to_statsd_bot(cobraBotConfig, statsdClient, fields, gauge, timer, verbose); - // - // We want at least 2 messages to be sent - // - REQUIRE(sentCount >= 2); - - // Give us 1s for all messages to be received - ix::msleep(1000); - - spdlog::info("Stopping snake server..."); - snakeServer.stop(); - - spdlog::info("Stopping redis server..."); - redisServer.stop(); - - publisherThread.join(); - } -} diff --git a/test/IXCobraToStdoutBotTest.cpp b/test/IXCobraToStdoutBotTest.cpp deleted file mode 100644 index 1361d68c..00000000 --- a/test/IXCobraToStdoutBotTest.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * IXCobraToStdoutTest.cpp - * Author: Benjamin Sergeant - * Copyright (c) 2020 Machine Zone. All rights reserved. - */ - -#include "IXTest.h" -#include "catch.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace ix; - -namespace -{ - void runPublisher(const ix::CobraConfig& config, const std::string& channel) - { - ix::CobraMetricsPublisher cobraMetricsPublisher; - cobraMetricsPublisher.configure(config, channel); - cobraMetricsPublisher.setSession(uuid4()); - cobraMetricsPublisher.enable(true); - - Json::Value msg; - msg["fps"] = 60; - - cobraMetricsPublisher.setGenericAttributes("game", "ody"); - - // Wait a bit - ix::msleep(500); - - // publish some messages - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #1) - cobraMetricsPublisher.push("sms_metric_B_id", msg); // (msg #2) - ix::msleep(500); - - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #3) - cobraMetricsPublisher.push("sms_metric_D_id", msg); // (msg #4) - ix::msleep(500); - - cobraMetricsPublisher.push("sms_metric_A_id", msg); // (msg #4) - cobraMetricsPublisher.push("sms_metric_F_id", msg); // (msg #5) - ix::msleep(500); - } -} // namespace - -TEST_CASE("Cobra_to_stdout_bot", "[cobra_bots]") -{ - SECTION("Exchange and count sent/received messages.") - { - int port = getFreePort(); - snake::AppConfig appConfig = makeSnakeServerConfig(port, true); - - // Start a redis server - ix::RedisServer redisServer(appConfig.redisPort); - auto res = redisServer.listen(); - REQUIRE(res.first); - redisServer.start(); - - // Start a snake server - snake::SnakeServer snakeServer(appConfig); - snakeServer.run(); - - // Run the bot for a small amount of time - std::string channel = ix::generateSessionId(); - std::string appkey("FC2F10139A2BAc53BB72D9db967b024f"); - std::string role = "_sub"; - std::string secret = "66B1dA3ED5fA074EB5AE84Dd8CE3b5ba"; - std::string endpoint = makeCobraEndpoint(port, true); - - ix::CobraConfig config; - config.endpoint = endpoint; - config.appkey = appkey; - config.rolename = role; - config.rolesecret = secret; - config.socketTLSOptions = makeClientTLSOptions(); - - std::thread publisherThread(runPublisher, config, channel); - - ix::CobraBotConfig cobraBotConfig; - cobraBotConfig.cobraConfig = config; - cobraBotConfig.channel = channel; - cobraBotConfig.runtime = 3; // Only run the bot for 3 seconds - cobraBotConfig.enableHeartbeat = false; - bool quiet = false; - - // We could try to capture the output ... not sure how. - bool fluentd = true; - - int64_t sentCount = ix::cobra_to_stdout_bot(cobraBotConfig, fluentd, quiet); - // - // We want at least 2 messages to be sent - // - REQUIRE(sentCount >= 2); - - // Give us 1s for all messages to be received - ix::msleep(1000); - - spdlog::info("Stopping snake server..."); - snakeServer.stop(); - - spdlog::info("Stopping redis server..."); - redisServer.stop(); - - publisherThread.join(); - } -} diff --git a/test/IXDNSLookupTest.cpp b/test/IXDNSLookupTest.cpp index 31a7e0aa..a2405042 100644 --- a/test/IXDNSLookupTest.cpp +++ b/test/IXDNSLookupTest.cpp @@ -24,6 +24,8 @@ TEST_CASE("dns", "[net]") res = dnsLookup->resolve(errMsg, [] { return false; }); std::cerr << "Error message: " << errMsg << std::endl; REQUIRE(res != nullptr); + + dnsLookup->release(res); } SECTION("Test resolving a non-existing hostname") @@ -31,11 +33,7 @@ TEST_CASE("dns", "[net]") auto dnsLookup = std::make_shared("wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww", 80); std::string errMsg; - struct addrinfo* res = dnsLookup->resolve(errMsg, - [] - { - return false; - }); + struct addrinfo* res = dnsLookup->resolve(errMsg, [] { return false; }); std::cerr << "Error message: " << errMsg << std::endl; REQUIRE(res == nullptr); } @@ -46,11 +44,7 @@ TEST_CASE("dns", "[net]") std::string errMsg; // The callback returning true means we are requesting cancellation - struct addrinfo* res = dnsLookup->resolve(errMsg, - [] - { - return true; - }); + struct addrinfo* res = dnsLookup->resolve(errMsg, [] { return true; }); std::cerr << "Error message: " << errMsg << std::endl; REQUIRE(res == nullptr); } diff --git a/test/IXHttpServerTest.cpp b/test/IXHttpServerTest.cpp index 89b6351c..e06f958c 100644 --- a/test/IXHttpServerTest.cpp +++ b/test/IXHttpServerTest.cpp @@ -4,9 +4,9 @@ * Copyright (c) 2019 Machine Zone. All rights reserved. */ -#include "IXGetFreePort.h" #include "catch.hpp" #include +#include #include #include @@ -63,11 +63,60 @@ TEST_CASE("http server", "[httpd]") server.stop(); } + + SECTION("Posting plain text data to a local HTTP server") + { + int port = getFreePort(); + ix::HttpServer server(port, "127.0.0.1"); + + server.setOnConnectionCallback( + [](HttpRequestPtr request, std::shared_ptr) -> HttpResponsePtr { + if (request->method == "POST") + { + return std::make_shared( + 200, "OK", HttpErrorCode::Ok, WebSocketHttpHeaders(), request->body); + } + + return std::make_shared(400, "BAD REQUEST"); + }); + + auto res = server.listen(); + REQUIRE(res.first); + server.start(); + + HttpClient httpClient; + WebSocketHttpHeaders headers; + headers["Content-Type"] = "text/plain"; + + std::string url("http://127.0.0.1:"); + url += std::to_string(port); + auto args = httpClient.createRequest(url); + + args->extraHeaders = headers; + args->connectTimeout = 60; + args->transferTimeout = 60; + args->verbose = true; + args->logger = [](const std::string& msg) { std::cout << msg; }; + args->body = "Hello World!"; + + auto response = httpClient.post(url, args->body, args); + + std::cerr << "Status: " << response->statusCode << std::endl; + std::cerr << "Error message: " << response->errorMsg << std::endl; + std::cerr << "Body: " << response->body << std::endl; + + REQUIRE(response->errorCode == HttpErrorCode::Ok); + REQUIRE(response->statusCode == 200); + REQUIRE(response->body == args->body); + + server.stop(); + } } TEST_CASE("http server redirection", "[httpd_redirect]") { - SECTION("Connect to a local HTTP server, with redirection enabled") + SECTION( + "Connect to a local HTTP server, with redirection enabled, but we do not follow redirects") { int port = getFreePort(); ix::HttpServer server(port, "127.0.0.1"); @@ -117,4 +166,54 @@ TEST_CASE("http server redirection", "[httpd_redirect]") server.stop(); } + + SECTION("Connect to a local HTTP server, with redirection enabled, but we do follow redirects") + { + int port = getFreePort(); + ix::HttpServer server(port, "127.0.0.1"); + server.makeRedirectServer("http://www.google.com"); + + auto res = server.listen(); + REQUIRE(res.first); + server.start(); + + HttpClient httpClient; + WebSocketHttpHeaders headers; + + std::string url("http://127.0.0.1:"); + url += std::to_string(port); + url += "/data/foo.txt"; + auto args = httpClient.createRequest(url); + + args->extraHeaders = headers; + args->connectTimeout = 60; + args->transferTimeout = 60; + args->followRedirects = true; + args->maxRedirects = 10; + args->verbose = true; + args->compress = true; + args->logger = [](const std::string& msg) { std::cout << msg; }; + args->onProgressCallback = [](int current, int total) -> bool { + std::cerr << "\r" + << "Downloaded " << current << " bytes out of " << total; + return true; + }; + + auto response = httpClient.get(url, args); + + for (auto it : response->headers) + { + std::cerr << it.first << ": " << it.second << std::endl; + } + + std::cerr << "Upload size: " << response->uploadSize << std::endl; + std::cerr << "Download size: " << response->downloadSize << std::endl; + std::cerr << "Status: " << response->statusCode << std::endl; + std::cerr << "Error message: " << response->errorMsg << std::endl; + + REQUIRE(response->errorCode == HttpErrorCode::Ok); + REQUIRE(response->statusCode == 200); + + server.stop(); + } } diff --git a/test/IXStrCaseCompareTest.cpp b/test/IXStrCaseCompareTest.cpp new file mode 100644 index 00000000..8a4408d4 --- /dev/null +++ b/test/IXStrCaseCompareTest.cpp @@ -0,0 +1,47 @@ +/* + * IXStrCaseCompareTest.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone. All rights reserved. + */ + +#include "IXTest.h" +#include "catch.hpp" +#include +#include +#include + +using namespace ix; + +namespace ix +{ + TEST_CASE("str_case_compare", "[str_case_compare]") + { + SECTION("1") + { + using HttpHeaders = std::map; + + HttpHeaders httpHeaders; + + httpHeaders["foo"] = "foo"; + + REQUIRE(httpHeaders["foo"] == "foo"); + REQUIRE(httpHeaders["missing"] == ""); + + // Comparison should be case insensitive + REQUIRE(httpHeaders["Foo"] == "foo"); + REQUIRE(httpHeaders["Foo"] != "bar"); + } + + SECTION("2") + { + using HttpHeaders = std::map; + + HttpHeaders headers; + + headers["Upgrade"] = "webSocket"; + + REQUIRE(!CaseInsensitiveLess::cmp(headers["upGRADE"], "webSocket")); + } + } + +} // namespace ix diff --git a/test/IXStreamSqlTest.cpp b/test/IXStreamSqlTest.cpp new file mode 100644 index 00000000..e14c126d --- /dev/null +++ b/test/IXStreamSqlTest.cpp @@ -0,0 +1,42 @@ +/* + * IXStreamSqlTest.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone. All rights reserved. + */ + +#include "IXTest.h" +#include "catch.hpp" +#include +#include +#include + +using namespace ix; + +namespace ix +{ + TEST_CASE("stream_sql", "[streamsql]") + { + SECTION("expression A") + { + snake::StreamSql streamSql( + "select * from subscriber_republished_v1_neo where session LIKE '%123456%'"); + + nlohmann::json msg = {{"session", "123456"}, {"id", "foo_id"}, {"timestamp", 12}}; + + CHECK(streamSql.match(msg)); + } + + SECTION("expression A") + { + snake::StreamSql streamSql("select * from `subscriber_republished_v1_neo` where " + "session = '30091320ed8d4e50b758f8409b83bed7'"); + + nlohmann::json msg = {{"session", "30091320ed8d4e50b758f8409b83bed7"}, + {"id", "foo_id"}, + {"timestamp", 12}}; + + CHECK(streamSql.match(msg)); + } + } + +} // namespace ix diff --git a/test/IXTest.cpp b/test/IXTest.cpp index 26cfb033..a07caa68 100644 --- a/test/IXTest.cpp +++ b/test/IXTest.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -84,36 +85,37 @@ namespace ix bool startWebSocketEchoServer(ix::WebSocketServer& server) { - server.setOnConnectionCallback([&server](std::shared_ptr webSocket, - std::shared_ptr connectionState) { - webSocket->setOnMessageCallback( - [webSocket, connectionState, &server](const ix::WebSocketMessagePtr& msg) { - if (msg->type == ix::WebSocketMessageType::Open) + server.setOnClientMessageCallback( + [&server](std::shared_ptr connectionState, + WebSocket& webSocket, + const ix::WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Open) + { + TLogger() << "New connection"; + TLogger() << "Remote ip: " << remoteIp; + TLogger() << "Uri: " << msg->openInfo.uri; + TLogger() << "Headers:"; + for (auto it : msg->openInfo.headers) { - TLogger() << "New connection"; - TLogger() << "Uri: " << msg->openInfo.uri; - TLogger() << "Headers:"; - for (auto it : msg->openInfo.headers) + TLogger() << it.first << ": " << it.second; + } + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + TLogger() << "Closed connection"; + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + for (auto&& client : server.getClients()) + { + if (client.get() != &webSocket) { - TLogger() << it.first << ": " << it.second; + client->send(msg->str, msg->binary); } } - else if (msg->type == ix::WebSocketMessageType::Close) - { - TLogger() << "Closed connection"; - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - for (auto&& client : server.getClients()) - { - if (client != webSocket) - { - client->send(msg->str, msg->binary); - } - } - } - }); - }); + } + }); auto res = server.listen(); if (!res.first) @@ -137,8 +139,9 @@ namespace ix std::streamoff size = file.tellg(); file.seekg(0, file.beg); - memblock.resize((size_t) size); - file.read((char*) &memblock.front(), static_cast(size)); + memblock.reserve((size_t) size); + memblock.insert( + memblock.begin(), std::istream_iterator(file), std::istream_iterator()); return memblock; } @@ -202,44 +205,4 @@ namespace ix #endif return scheme; } - - snake::AppConfig makeSnakeServerConfig(int port, bool preferTLS) - { - snake::AppConfig appConfig; - appConfig.port = port; - appConfig.hostname = "127.0.0.1"; - appConfig.verbose = true; - appConfig.redisPort = getFreePort(); - appConfig.redisPassword = ""; - appConfig.redisHosts.push_back("localhost"); // only one host supported now - appConfig.socketTLSOptions = makeServerTLSOptions(preferTLS); - - std::string appsConfigPath("appsConfig.json"); - - // Parse config file - auto str = readAsString(appsConfigPath); - if (str.empty()) - { - std::cout << "Cannot read content of " << appsConfigPath << std::endl; - return appConfig; - } - - std::cout << str << std::endl; - auto apps = nlohmann::json::parse(str); - appConfig.apps = apps["apps"]; - - // Display config on the terminal for debugging - dumpConfig(appConfig); - - return appConfig; - } - - std::string makeCobraEndpoint(int port, bool preferTLS) - { - std::stringstream ss; - ss << getWsScheme(preferTLS) << "localhost:" << port; - std::string endpoint = ss.str(); - - return endpoint; - } } // namespace ix diff --git a/test/IXTest.h b/test/IXTest.h index 4a49d38b..27b3a93c 100644 --- a/test/IXTest.h +++ b/test/IXTest.h @@ -6,9 +6,8 @@ #pragma once -#include "IXGetFreePort.h" #include -#include +#include #include #include #include @@ -51,12 +50,8 @@ namespace ix bool startWebSocketEchoServer(ix::WebSocketServer& server); - snake::AppConfig makeSnakeServerConfig(int port, bool preferTLS); - SocketTLSOptions makeClientTLSOptions(); SocketTLSOptions makeServerTLSOptions(bool preferTLS); std::string getHttpScheme(); std::string getWsScheme(bool preferTLS); - - std::string makeCobraEndpoint(int port, bool preferTLS); } // namespace ix diff --git a/test/IXWebSocketBroadcastTest.cpp b/test/IXWebSocketBroadcastTest.cpp index 879df7a2..40324476 100644 --- a/test/IXWebSocketBroadcastTest.cpp +++ b/test/IXWebSocketBroadcastTest.cpp @@ -18,10 +18,10 @@ using namespace ix; namespace { - class WebSocketChat + class WebSocketBroadcastChat { public: - WebSocketChat(const std::string& user, const std::string& session, int port); + WebSocketBroadcastChat(const std::string& user, const std::string& session, int port); void subscribe(const std::string& channel); void start(); @@ -47,7 +47,9 @@ namespace mutable std::mutex _mutex; }; - WebSocketChat::WebSocketChat(const std::string& user, const std::string& session, int port) + WebSocketBroadcastChat::WebSocketBroadcastChat(const std::string& user, + const std::string& session, + int port) : _user(user) , _session(session) , _port(port) @@ -55,42 +57,40 @@ namespace _webSocket.setTLSOptions(makeClientTLSOptions()); } - size_t WebSocketChat::getReceivedMessagesCount() const + size_t WebSocketBroadcastChat::getReceivedMessagesCount() const { std::lock_guard lock(_mutex); return _receivedMessages.size(); } - const std::vector& WebSocketChat::getReceivedMessages() const + const std::vector& WebSocketBroadcastChat::getReceivedMessages() const { std::lock_guard lock(_mutex); return _receivedMessages; } - void WebSocketChat::appendMessage(const std::string& message) + void WebSocketBroadcastChat::appendMessage(const std::string& message) { std::lock_guard lock(_mutex); _receivedMessages.push_back(message); } - bool WebSocketChat::isReady() const + bool WebSocketBroadcastChat::isReady() const { return _webSocket.getReadyState() == ix::ReadyState::Open; } - void WebSocketChat::stop() + void WebSocketBroadcastChat::stop() { _webSocket.stop(); } - void WebSocketChat::start() + void WebSocketBroadcastChat::start() { + // + // Which server ?? + // std::string url; - { - bool preferTLS = true; - url = makeCobraEndpoint(_port, preferTLS); - } - _webSocket.setUrl(url); std::stringstream ss; @@ -156,7 +156,8 @@ namespace _webSocket.start(); } - std::pair WebSocketChat::decodeMessage(const std::string& str) + std::pair WebSocketBroadcastChat::decodeMessage( + const std::string& str) { std::string errMsg; MsgPack msg = MsgPack::parse(str, errMsg); @@ -167,7 +168,7 @@ namespace return std::pair(msg_user, msg_text); } - std::string WebSocketChat::encodeMessage(const std::string& text) + std::string WebSocketBroadcastChat::encodeMessage(const std::string& text) { std::map obj; obj["user"] = _user; @@ -179,7 +180,7 @@ namespace return output; } - void WebSocketChat::sendMessage(const std::string& text) + void WebSocketBroadcastChat::sendMessage(const std::string& text) { _webSocket.sendBinary(encodeMessage(text)); } @@ -189,15 +190,17 @@ namespace bool preferTLS = true; server.setTLSOptions(makeServerTLSOptions(preferTLS)); - server.setOnConnectionCallback([&server, &connectionId]( - std::shared_ptr webSocket, - std::shared_ptr connectionState) { - webSocket->setOnMessageCallback([webSocket, connectionState, &connectionId, &server]( - const ix::WebSocketMessagePtr& msg) { + server.setOnClientMessageCallback( + [&server, &connectionId](std::shared_ptr connectionState, + WebSocket& webSocket, + const ix::WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Open) { TLogger() << "New connection"; connectionState->computeId(); + TLogger() << "remote ip: " << remoteIp; TLogger() << "id: " << connectionState->getId(); TLogger() << "Uri: " << msg->openInfo.uri; TLogger() << "Headers:"; @@ -216,14 +219,13 @@ namespace { for (auto&& client : server.getClients()) { - if (client != webSocket) + if (client.get() != &webSocket) { client->send(msg->str, msg->binary); } } } }); - }); auto res = server.listen(); if (!res.first) @@ -247,11 +249,11 @@ TEST_CASE("Websocket_broadcast_server", "[websocket_server]") REQUIRE(startServer(server, connectionId)); std::string session = ix::generateSessionId(); - std::vector> chatClients; + std::vector> chatClients; for (int i = 0; i < 10; ++i) { std::string user("user_" + std::to_string(i)); - chatClients.push_back(std::make_shared(user, session, port)); + chatClients.push_back(std::make_shared(user, session, port)); chatClients[i]->start(); ix::msleep(50); } diff --git a/test/IXWebSocketChatTest.cpp b/test/IXWebSocketChatTest.cpp index 8c531079..16c81319 100644 --- a/test/IXWebSocketChatTest.cpp +++ b/test/IXWebSocketChatTest.cpp @@ -193,37 +193,38 @@ namespace bool startServer(ix::WebSocketServer& server) { - server.setOnConnectionCallback([&server](std::shared_ptr webSocket, - std::shared_ptr connectionState) { - webSocket->setOnMessageCallback( - [webSocket, connectionState, &server](const ix::WebSocketMessagePtr& msg) { - if (msg->type == ix::WebSocketMessageType::Open) + server.setOnClientMessageCallback( + [&server](std::shared_ptr connectionState, + WebSocket& webSocket, + const ix::WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Open) + { + TLogger() << "New connection"; + TLogger() << "remote ip: " << remoteIp; + TLogger() << "id: " << connectionState->getId(); + TLogger() << "Uri: " << msg->openInfo.uri; + TLogger() << "Headers:"; + for (auto it : msg->openInfo.headers) { - TLogger() << "New connection"; - TLogger() << "id: " << connectionState->getId(); - TLogger() << "Uri: " << msg->openInfo.uri; - TLogger() << "Headers:"; - for (auto it : msg->openInfo.headers) + TLogger() << it.first << ": " << it.second; + } + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + log("Closed connection"); + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + for (auto&& client : server.getClients()) + { + if (client.get() != &webSocket) { - TLogger() << it.first << ": " << it.second; + client->sendBinary(msg->str); } } - else if (msg->type == ix::WebSocketMessageType::Close) - { - log("Closed connection"); - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - for (auto&& client : server.getClients()) - { - if (client != webSocket) - { - client->sendBinary(msg->str); - } - } - } - }); - }); + } + }); auto res = server.listen(); if (!res.first) @@ -284,27 +285,27 @@ TEST_CASE("Websocket_chat", "[websocket_chat]") int attempts = 0; while (chatA.getReceivedMessagesCount() != 3 || chatB.getReceivedMessagesCount() != 3) { - REQUIRE(attempts++ < 10); + CHECK(attempts++ < 10); ix::msleep(1000); } chatA.stop(); chatB.stop(); - REQUIRE(chatA.getReceivedMessagesCount() == 3); - REQUIRE(chatB.getReceivedMessagesCount() == 3); + CHECK(chatA.getReceivedMessagesCount() == 3); + CHECK(chatB.getReceivedMessagesCount() == 3); - REQUIRE(chatB.getReceivedMessages()[0] == "from A1"); - REQUIRE(chatB.getReceivedMessages()[1] == "from A2"); - REQUIRE(chatB.getReceivedMessages()[2] == "from A3"); + CHECK(chatB.getReceivedMessages()[0] == "from A1"); + CHECK(chatB.getReceivedMessages()[1] == "from A2"); + CHECK(chatB.getReceivedMessages()[2] == "from A3"); - REQUIRE(chatA.getReceivedMessages()[0] == "from B1"); - REQUIRE(chatA.getReceivedMessages()[1] == "from B2"); - REQUIRE(chatA.getReceivedMessages()[2].size() == bigMessage.size()); + CHECK(chatA.getReceivedMessages()[0] == "from B1"); + CHECK(chatA.getReceivedMessages()[1] == "from B2"); + CHECK(chatA.getReceivedMessages()[2].size() == bigMessage.size()); // Give us 1000ms for the server to notice that clients went away ix::msleep(1000); - REQUIRE(server.getClients().size() == 0); + CHECK(server.getClients().size() == 0); ix::reportWebSocketTraffic(); } diff --git a/test/IXWebSocketCloseTest.cpp b/test/IXWebSocketCloseTest.cpp index b52ab2c5..76404d98 100644 --- a/test/IXWebSocketCloseTest.cpp +++ b/test/IXWebSocketCloseTest.cpp @@ -168,41 +168,37 @@ namespace std::mutex& mutexWrite) { // A dev/null server - server.setOnConnectionCallback( + server.setOnClientMessageCallback( [&receivedCloseCode, &receivedCloseReason, &receivedCloseRemote, &mutexWrite]( - std::shared_ptr webSocket, - std::shared_ptr connectionState) { - webSocket->setOnMessageCallback([webSocket, - connectionState, - &receivedCloseCode, - &receivedCloseReason, - &receivedCloseRemote, - &mutexWrite](const ix::WebSocketMessagePtr& msg) { - if (msg->type == ix::WebSocketMessageType::Open) + std::shared_ptr connectionState, + WebSocket& /*webSocket*/, + const ix::WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Open) + { + TLogger() << "New server connection"; + TLogger() << "remote ip: " << remoteIp; + TLogger() << "id: " << connectionState->getId(); + TLogger() << "Uri: " << msg->openInfo.uri; + TLogger() << "Headers:"; + for (auto it : msg->openInfo.headers) { - TLogger() << "New server connection"; - TLogger() << "id: " << connectionState->getId(); - TLogger() << "Uri: " << msg->openInfo.uri; - TLogger() << "Headers:"; - for (auto it : msg->openInfo.headers) - { - TLogger() << it.first << ": " << it.second; - } + TLogger() << it.first << ": " << it.second; } - else if (msg->type == ix::WebSocketMessageType::Close) - { - std::stringstream ss; - ss << "Server closed connection(" << msg->closeInfo.code << "," - << msg->closeInfo.reason << ")"; - log(ss.str()); + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + std::stringstream ss; + ss << "Server closed connection(" << msg->closeInfo.code << "," + << msg->closeInfo.reason << ")"; + log(ss.str()); - std::lock_guard lck(mutexWrite); + std::lock_guard lck(mutexWrite); - receivedCloseCode = msg->closeInfo.code; - receivedCloseReason = std::string(msg->closeInfo.reason); - receivedCloseRemote = msg->closeInfo.remote; - } - }); + receivedCloseCode = msg->closeInfo.code; + receivedCloseReason = std::string(msg->closeInfo.reason); + receivedCloseRemote = msg->closeInfo.remote; + } }); auto res = server.listen(); diff --git a/test/IXWebSocketLeakTest.cpp b/test/IXWebSocketLeakTest.cpp new file mode 100644 index 00000000..abbb4e2c --- /dev/null +++ b/test/IXWebSocketLeakTest.cpp @@ -0,0 +1,183 @@ +/* + * IXWebSocketServer.cpp + * Author: Benjamin Sergeant, @marcelkauf + * Copyright (c) 2020 Machine Zone, Inc. All rights reserved. + */ + +#include "IXTest.h" +#include "catch.hpp" +#include +#include +#include +#include + +using namespace ix; + +namespace +{ + class WebSocketClient + { + public: + WebSocketClient(int port); + void start(); + void stop(); + bool isReady() const; + bool hasConnectionError() const; + + private: + ix::WebSocket _webSocket; + int _port; + std::atomic _connectionError; + }; + + WebSocketClient::WebSocketClient(int port) + : _port(port) + , _connectionError(false) + { + } + + bool WebSocketClient::hasConnectionError() const + { + return _connectionError; + } + + bool WebSocketClient::isReady() const + { + return _webSocket.getReadyState() == ix::ReadyState::Open; + } + + void WebSocketClient::stop() + { + _webSocket.stop(); + } + + void WebSocketClient::start() + { + std::string url; + { + std::stringstream ss; + ss << "ws://localhost:" << _port << "/"; + + url = ss.str(); + } + + _webSocket.setUrl(url); + _webSocket.disableAutomaticReconnection(); + + std::stringstream ss; + log(std::string("Connecting to url: ") + url); + + _webSocket.setOnMessageCallback([this](const ix::WebSocketMessagePtr& msg) { + std::stringstream ss; + if (msg->type == ix::WebSocketMessageType::Open) + { + log("client connected"); + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + log("client disconnected"); + } + else if (msg->type == ix::WebSocketMessageType::Error) + { + _connectionError = true; + log("error"); + } + else if (msg->type == ix::WebSocketMessageType::Pong) + { + log("pong"); + } + else if (msg->type == ix::WebSocketMessageType::Ping) + { + log("ping"); + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + log("message"); + } + else + { + log("invalid type"); + } + }); + + _webSocket.start(); + } +} // namespace + +TEST_CASE("Websocket leak test") +{ + SECTION("Websocket destructor is called when closing the connection.") + { + // stores the server websocket in order to check the use_count + std::shared_ptr webSocketPtr; + + { + int port = getFreePort(); + WebSocketServer server(port); + + server.setOnConnectionCallback( + [&webSocketPtr](std::shared_ptr webSocket, + std::shared_ptr connectionState, + std::unique_ptr connectionInfo) { + // original ptr in WebSocketServer::handleConnection and the callback argument + REQUIRE(webSocket.use_count() == 2); + webSocket->setOnMessageCallback([&webSocketPtr, webSocket, connectionState]( + const ix::WebSocketMessagePtr& msg) { + if (msg->type == ix::WebSocketMessageType::Open) + { + log(std::string("New connection id: ") + connectionState->getId()); + // original ptr in WebSocketServer::handleConnection, captured ptr of + // this callback, and ptr in WebSocketServer::_clients + REQUIRE(webSocket.use_count() == 3); + webSocketPtr = std::shared_ptr(webSocket); + REQUIRE(webSocket.use_count() == 4); + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + log(std::string("Client closed connection id: ") + + connectionState->getId()); + } + else + { + log(std::string(msg->str)); + } + }); + // original ptr in WebSocketServer::handleConnection, argument of this callback, + // and captured ptr in websocket callback + REQUIRE(webSocket.use_count() == 3); + }); + + server.listen(); + server.start(); + + WebSocketClient webSocketClient(port); + webSocketClient.start(); + + while (true) + { + REQUIRE(!webSocketClient.hasConnectionError()); + if (webSocketClient.isReady()) break; + ix::msleep(10); + } + + REQUIRE(server.getClients().size() == 1); + // same value as in Open-handler above + REQUIRE(webSocketPtr.use_count() == 4); + + ix::msleep(500); + webSocketClient.stop(); + ix::msleep(500); + REQUIRE(server.getClients().size() == 0); + + // websocket should only be referenced by webSocketPtr but is still used by the + // websocket callback + REQUIRE(webSocketPtr.use_count() == 1); + webSocketPtr->setOnMessageCallback(nullptr); + // websocket should only be referenced by webSocketPtr + REQUIRE(webSocketPtr.use_count() == 1); + server.stop(); + } + // websocket should only be referenced by webSocketPtr + REQUIRE(webSocketPtr.use_count() == 1); + } +} diff --git a/test/IXWebSocketPerMessageDeflateCompressorTest.cpp b/test/IXWebSocketPerMessageDeflateCompressorTest.cpp new file mode 100644 index 00000000..b067b825 --- /dev/null +++ b/test/IXWebSocketPerMessageDeflateCompressorTest.cpp @@ -0,0 +1,76 @@ +/* + * IXWebSocketPerMessageDeflateCodecTest.cpp + * Author: Benjamin Sergeant + * Copyright (c) 2020 Machine Zone. All rights reserved. + * + * make build_test && build/test/ixwebsocket_unittest per-message-deflate-codec + */ + +#include "IXTest.h" +#include "catch.hpp" +#include +#include +#include + +using namespace ix; + +namespace ix +{ + std::string compressAndDecompress(const std::string& a) + { + std::string b, c; + + WebSocketPerMessageDeflateCompressor compressor; + compressor.init(11, true); + compressor.compress(a, b); + + WebSocketPerMessageDeflateDecompressor decompressor; + decompressor.init(11, true); + decompressor.decompress(b, c); + + return c; + } + + std::string compressAndDecompressVector(const std::string& a) + { + std::string b, c; + + std::vector vec(a.begin(), a.end()); + + WebSocketPerMessageDeflateCompressor compressor; + compressor.init(11, true); + compressor.compress(vec, b); + + WebSocketPerMessageDeflateDecompressor decompressor; + decompressor.init(11, true); + decompressor.decompress(b, c); + + return c; + } + + TEST_CASE("per-message-deflate-codec", "[zlib]") + { + SECTION("string api") + { + REQUIRE(compressAndDecompress("") == ""); + REQUIRE(compressAndDecompress("foo") == "foo"); + REQUIRE(compressAndDecompress("bar") == "bar"); + REQUIRE(compressAndDecompress("asdcaseqw`21897dehqwed") == "asdcaseqw`21897dehqwed"); + REQUIRE(compressAndDecompress("/usr/local/include/ixwebsocket/IXSocketAppleSSL.h") == + "/usr/local/include/ixwebsocket/IXSocketAppleSSL.h"); + } + + SECTION("vector api") + { + REQUIRE(compressAndDecompressVector("") == ""); + REQUIRE(compressAndDecompressVector("foo") == "foo"); + REQUIRE(compressAndDecompressVector("bar") == "bar"); + REQUIRE(compressAndDecompressVector("asdcaseqw`21897dehqwed") == + "asdcaseqw`21897dehqwed"); + REQUIRE( + compressAndDecompressVector("/usr/local/include/ixwebsocket/IXSocketAppleSSL.h") == + "/usr/local/include/ixwebsocket/IXSocketAppleSSL.h"); + } + } + +} // namespace ix diff --git a/test/IXWebSocketPingTest.cpp b/test/IXWebSocketPingTest.cpp index b845345b..b484f9e3 100644 --- a/test/IXWebSocketPingTest.cpp +++ b/test/IXWebSocketPingTest.cpp @@ -19,7 +19,7 @@ namespace class WebSocketClient { public: - WebSocketClient(int port, bool useHeartBeatMethod); + WebSocketClient(int port); void start(); void stop(); @@ -29,12 +29,10 @@ namespace private: ix::WebSocket _webSocket; int _port; - bool _useHeartBeatMethod; }; - WebSocketClient::WebSocketClient(int port, bool useHeartBeatMethod) + WebSocketClient::WebSocketClient(int port) : _port(port) - , _useHeartBeatMethod(useHeartBeatMethod) { ; } @@ -63,49 +61,37 @@ namespace // The important bit for this test. // Set a 1 second heartbeat with the setter method to test - if (_useHeartBeatMethod) - { - _webSocket.setPingInterval(1); - } - else - { - _webSocket.setPingInterval(1); - } + _webSocket.setPingInterval(1); std::stringstream ss; log(std::string("Connecting to url: ") + url); - _webSocket.setOnMessageCallback([](ix::WebSocketMessageType messageType, - const std::string& str, - size_t wireSize, - const ix::WebSocketErrorInfo& error, - const ix::WebSocketOpenInfo& openInfo, - const ix::WebSocketCloseInfo& closeInfo) { + _webSocket.setOnMessageCallback([](const ix::WebSocketMessagePtr& msg) { std::stringstream ss; - if (messageType == ix::WebSocketMessageType::Open) + if (msg->type == ix::WebSocketMessageType::Open) { log("client connected"); } - else if (messageType == ix::WebSocketMessageType::Close) + else if (msg->type == ix::WebSocketMessageType::Close) { log("client disconnected"); } - else if (messageType == ix::WebSocketMessageType::Error) + else if (msg->type == ix::WebSocketMessageType::Error) { - ss << "Error ! " << error.reason; + ss << "Error ! " << msg->errorInfo.reason; log(ss.str()); } - else if (messageType == ix::WebSocketMessageType::Pong) + else if (msg->type == ix::WebSocketMessageType::Pong) { - ss << "Received pong message " << str; + ss << "Received pong message " << msg->str; log(ss.str()); } - else if (messageType == ix::WebSocketMessageType::Ping) + else if (msg->type == ix::WebSocketMessageType::Ping) { - ss << "Received ping message " << str; + ss << "Received ping message " << msg->str; log(ss.str()); } - else if (messageType == ix::WebSocketMessageType::Message) + else if (msg->type == ix::WebSocketMessageType::Message) { // too many messages to log } @@ -132,33 +118,28 @@ namespace std::shared_ptr connectionState) { webSocket->setOnMessageCallback( [webSocket, connectionState, &server, &receivedPingMessages]( - ix::WebSocketMessageType messageType, - const std::string& str, - size_t wireSize, - const ix::WebSocketErrorInfo& error, - const ix::WebSocketOpenInfo& openInfo, - const ix::WebSocketCloseInfo& closeInfo) { - if (messageType == ix::WebSocketMessageType::Open) + const ix::WebSocketMessagePtr& msg) { + if (msg->type == ix::WebSocketMessageType::Open) { TLogger() << "New server connection"; TLogger() << "id: " << connectionState->getId(); - TLogger() << "Uri: " << openInfo.uri; + TLogger() << "Uri: " << msg->openInfo.uri; TLogger() << "Headers:"; - for (auto it : openInfo.headers) + for (auto it : msg->openInfo.headers) { TLogger() << it.first << ": " << it.second; } } - else if (messageType == ix::WebSocketMessageType::Close) + else if (msg->type == ix::WebSocketMessageType::Close) { log("Server closed connection"); } - else if (messageType == ix::WebSocketMessageType::Ping) + else if (msg->type == ix::WebSocketMessageType::Ping) { log("Server received a ping"); receivedPingMessages++; } - else if (messageType == ix::WebSocketMessageType::Message) + else if (msg->type == ix::WebSocketMessageType::Message) { // to many messages to log for (auto client : server.getClients()) @@ -193,8 +174,7 @@ TEST_CASE("Websocket_ping_no_data_sent_setPingInterval", "[setPingInterval]") REQUIRE(startServer(server, serverReceivedPingMessages)); std::string session = ix::generateSessionId(); - bool useSetHeartBeatPeriodMethod = false; // so use setPingInterval - WebSocketClient webSocketClient(port, useSetHeartBeatPeriodMethod); + WebSocketClient webSocketClient(port); webSocketClient.start(); @@ -236,8 +216,7 @@ TEST_CASE("Websocket_ping_data_sent_setPingInterval", "[setPingInterval]") REQUIRE(startServer(server, serverReceivedPingMessages)); std::string session = ix::generateSessionId(); - bool useSetHeartBeatPeriodMethod = false; // so use setPingInterval - WebSocketClient webSocketClient(port, useSetHeartBeatPeriodMethod); + WebSocketClient webSocketClient(port); webSocketClient.start(); @@ -261,7 +240,7 @@ TEST_CASE("Websocket_ping_data_sent_setPingInterval", "[setPingInterval]") // Here we test ping interval // client has sent data, but ping should have been sent no matter what // -> expected ping messages == 3 as 900+900+1300 = 3100 seconds, 1 ping sent every second - REQUIRE(serverReceivedPingMessages == 3); + REQUIRE(serverReceivedPingMessages >= 2); // Give us 1000ms for the server to notice that clients went away ix::msleep(1000); @@ -284,8 +263,7 @@ TEST_CASE("Websocket_ping_data_sent_setPingInterval_half_full", "[setPingInterva REQUIRE(startServer(server, serverReceivedPingMessages)); std::string session = ix::generateSessionId(); - bool useSetHeartBeatPeriodMethod = false; // so use setPingInterval - WebSocketClient webSocketClient(port, useSetHeartBeatPeriodMethod); + WebSocketClient webSocketClient(port); webSocketClient.start(); @@ -338,8 +316,7 @@ TEST_CASE("Websocket_ping_data_sent_setPingInterval_full", "[setPingInterval]") REQUIRE(startServer(server, serverReceivedPingMessages)); std::string session = ix::generateSessionId(); - bool useSetHeartBeatPeriodMethod = false; // so use setPingInterval - WebSocketClient webSocketClient(port, useSetHeartBeatPeriodMethod); + WebSocketClient webSocketClient(port); webSocketClient.start(); @@ -363,8 +340,9 @@ TEST_CASE("Websocket_ping_data_sent_setPingInterval_full", "[setPingInterval]") // Here we test ping interval // client has sent data, but ping should have been sent no matter what - // -> expected ping messages == 1, 1 ping sent every second - REQUIRE(serverReceivedPingMessages == 1); + // -> expected ping messages == 2, 1 ping sent every second + // The first ping is sent right away on connect + REQUIRE(serverReceivedPingMessages == 2); ix::msleep(100); @@ -392,8 +370,7 @@ TEST_CASE("Websocket_ping_no_data_sent_setHeartBeatPeriod", "[setPingInterval]") REQUIRE(startServer(server, serverReceivedPingMessages)); std::string session = ix::generateSessionId(); - bool useSetHeartBeatPeriodMethod = true; - WebSocketClient webSocketClient(port, useSetHeartBeatPeriodMethod); + WebSocketClient webSocketClient(port); webSocketClient.start(); @@ -406,14 +383,13 @@ TEST_CASE("Websocket_ping_no_data_sent_setHeartBeatPeriod", "[setPingInterval]") REQUIRE(server.getClients().size() == 1); - ix::msleep(1900); + ix::msleep(2100); webSocketClient.stop(); - // Here we test ping interval - // -> expected ping messages == 1 as 1900 seconds, 1 ping sent every second - REQUIRE(serverReceivedPingMessages == 1); + // -> expected ping messages == 2 as 2100 seconds, 1 ping sent every second + REQUIRE(serverReceivedPingMessages == 2); // Give us 1000ms for the server to notice that clients went away ix::msleep(1000); @@ -436,8 +412,7 @@ TEST_CASE("Websocket_ping_data_sent_setHeartBeatPeriod", "[setPingInterval]") REQUIRE(startServer(server, serverReceivedPingMessages)); std::string session = ix::generateSessionId(); - bool useSetHeartBeatPeriodMethod = true; - WebSocketClient webSocketClient(port, useSetHeartBeatPeriodMethod); + WebSocketClient webSocketClient(port); webSocketClient.start(); @@ -464,7 +439,7 @@ TEST_CASE("Websocket_ping_data_sent_setHeartBeatPeriod", "[setPingInterval]") // Here we test ping interval // client has sent data, but ping should have been sent no matter what // -> expected ping messages == 2 as 900+900+1100 = 2900 seconds, 1 ping sent every second - REQUIRE(serverReceivedPingMessages == 2); + REQUIRE(serverReceivedPingMessages >= 2); // Give us 1000ms for the server to notice that clients went away ix::msleep(1000); diff --git a/test/IXWebSocketServerTest.cpp b/test/IXWebSocketServerTest.cpp index a53c697d..5ff58f64 100644 --- a/test/IXWebSocketServerTest.cpp +++ b/test/IXWebSocketServerTest.cpp @@ -33,15 +33,17 @@ namespace ix }; server.setConnectionStateFactory(factory); - server.setOnConnectionCallback([&server, &connectionId]( - std::shared_ptr webSocket, - std::shared_ptr connectionState) { - webSocket->setOnMessageCallback([webSocket, connectionState, &connectionId, &server]( - const ix::WebSocketMessagePtr& msg) { + server.setOnClientMessageCallback( + [&server, &connectionId](std::shared_ptr connectionState, + WebSocket& webSocket, + const ix::WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Open) { TLogger() << "New connection"; connectionState->computeId(); + TLogger() << "remote ip: " << remoteIp; TLogger() << "id: " << connectionState->getId(); TLogger() << "Uri: " << msg->openInfo.uri; TLogger() << "Headers:"; @@ -60,14 +62,13 @@ namespace ix { for (auto&& client : server.getClients()) { - if (client != webSocket) + if (client.get() != &webSocket) { client->send(msg->str, msg->binary); } } } }); - }); auto res = server.listen(); if (!res.first) diff --git a/test/IXWebSocketSubProtocolTest.cpp b/test/IXWebSocketSubProtocolTest.cpp index e4430c13..8c18f7b2 100644 --- a/test/IXWebSocketSubProtocolTest.cpp +++ b/test/IXWebSocketSubProtocolTest.cpp @@ -16,39 +16,39 @@ using namespace ix; bool startServer(ix::WebSocketServer& server, std::string& subProtocols) { - server.setOnConnectionCallback( - [&server, &subProtocols](std::shared_ptr webSocket, - std::shared_ptr connectionState) { - webSocket->setOnMessageCallback([webSocket, connectionState, &server, &subProtocols]( - const ix::WebSocketMessagePtr& msg) { - if (msg->type == ix::WebSocketMessageType::Open) + server.setOnClientMessageCallback( + [&server, &subProtocols](std::shared_ptr connectionState, + WebSocket& webSocket, + const ix::WebSocketMessagePtr& msg) { + auto remoteIp = connectionState->getRemoteIp(); + if (msg->type == ix::WebSocketMessageType::Open) + { + TLogger() << "New connection"; + TLogger() << "remote ip: " << remoteIp; + TLogger() << "id: " << connectionState->getId(); + TLogger() << "Uri: " << msg->openInfo.uri; + TLogger() << "Headers:"; + for (auto it : msg->openInfo.headers) { - TLogger() << "New connection"; - TLogger() << "id: " << connectionState->getId(); - TLogger() << "Uri: " << msg->openInfo.uri; - TLogger() << "Headers:"; - for (auto it : msg->openInfo.headers) - { - TLogger() << it.first << ": " << it.second; - } + TLogger() << it.first << ": " << it.second; + } - subProtocols = msg->openInfo.headers["Sec-WebSocket-Protocol"]; - } - else if (msg->type == ix::WebSocketMessageType::Close) + subProtocols = msg->openInfo.headers["Sec-WebSocket-Protocol"]; + } + else if (msg->type == ix::WebSocketMessageType::Close) + { + log("Closed connection"); + } + else if (msg->type == ix::WebSocketMessageType::Message) + { + for (auto&& client : server.getClients()) { - log("Closed connection"); - } - else if (msg->type == ix::WebSocketMessageType::Message) - { - for (auto&& client : server.getClients()) + if (client.get() != &webSocket) { - if (client != webSocket) - { - client->sendBinary(msg->str); - } + client->sendBinary(msg->str); } } - }); + } }); auto res = server.listen(); diff --git a/test/compatibility/cpp/libwebsockets/devnull_client.cpp b/test/compatibility/cpp/libwebsockets/devnull_client.cpp new file mode 100644 index 00000000..60412544 --- /dev/null +++ b/test/compatibility/cpp/libwebsockets/devnull_client.cpp @@ -0,0 +1,171 @@ +/* + * lws-minimal-ws-client + * + * Written in 2010-2019 by Andy Green + * + * This file is made available under the Creative Commons CC0 1.0 + * Universal Public Domain Dedication. + * + * This demonstrates the a minimal ws client using lws. + * + * Original programs connects to https://libwebsockets.org/ and makes a + * wss connection to the dumb-increment protocol there. While + * connected, it prints the numbers it is being sent by + * dumb-increment protocol. + * + * This is modified to make a test client which counts how much messages + * per second can be received. + * + * libwebsockets$ make && ./a.out + * g++ --std=c++14 -I/usr/local/opt/openssl/include devnull_client.cpp -lwebsockets + * messages received: 0 per second 0 total + * [2020/08/02 19:22:21:4774] U: LWS minimal ws client rx [-d ] [--h2] + * [2020/08/02 19:22:21:4814] U: callback_dumb_increment: established + * messages received: 0 per second 0 total + * messages received: 180015 per second 180015 total + * messages received: 172866 per second 352881 total + * messages received: 176177 per second 529058 total + * messages received: 174191 per second 703249 total + * messages received: 193397 per second 896646 total + * messages received: 196385 per second 1093031 total + * messages received: 194593 per second 1287624 total + * messages received: 189484 per second 1477108 total + * messages received: 200825 per second 1677933 total + * messages received: 183542 per second 1861475 total + * ^C[2020/08/02 19:22:33:4450] U: Completed OK + * + */ + +#include +#include +#include +#include +#include +#include + +static int interrupted; +static struct lws* client_wsi; + +std::atomic receivedCount(0); + +static int callback_dumb_increment( + struct lws* wsi, enum lws_callback_reasons reason, void* user, void* in, size_t len) +{ + switch (reason) + { + /* because we are protocols[0] ... */ + case LWS_CALLBACK_CLIENT_CONNECTION_ERROR: + lwsl_err("CLIENT_CONNECTION_ERROR: %s\n", in ? (char*) in : "(null)"); + client_wsi = NULL; + break; + + case LWS_CALLBACK_CLIENT_ESTABLISHED: lwsl_user("%s: established\n", __func__); break; + + case LWS_CALLBACK_CLIENT_RECEIVE: receivedCount++; break; + + case LWS_CALLBACK_CLIENT_CLOSED: client_wsi = NULL; break; + + default: break; + } + + return lws_callback_http_dummy(wsi, reason, user, in, len); +} + +static const struct lws_protocols protocols[] = {{ + "dumb-increment-protocol", + callback_dumb_increment, + 0, + 0, + }, + {NULL, NULL, 0, 0}}; + +static void sigint_handler(int sig) +{ + interrupted = 1; +} + +int main(int argc, const char** argv) +{ + uint64_t receivedCountTotal(0); + uint64_t receivedCountPerSecs(0); + + auto timer = [&receivedCountTotal, &receivedCountPerSecs] { + while (!interrupted) + { + std::cerr << "messages received: " << receivedCountPerSecs << " per second " + << receivedCountTotal << " total" << std::endl; + + receivedCountPerSecs = receivedCount - receivedCountTotal; + receivedCountTotal += receivedCountPerSecs; + + auto duration = std::chrono::seconds(1); + std::this_thread::sleep_for(duration); + } + }; + + std::thread t1(timer); + + struct lws_context_creation_info info; + struct lws_client_connect_info i; + struct lws_context* context; + const char* p; + int n = 0, logs = LLL_USER | LLL_ERR | LLL_WARN | LLL_NOTICE + /* for LLL_ verbosity above NOTICE to be built into lws, lws + * must have been configured with -DCMAKE_BUILD_TYPE=DEBUG + * instead of =RELEASE */ + /* | LLL_INFO */ /* | LLL_PARSER */ /* | LLL_HEADER */ + /* | LLL_EXT */ /* | LLL_CLIENT */ /* | LLL_LATENCY */ + /* | LLL_DEBUG */; + + signal(SIGINT, sigint_handler); + if ((p = lws_cmdline_option(argc, argv, "-d"))) logs = atoi(p); + + lws_set_log_level(logs, NULL); + lwsl_user("LWS minimal ws client rx [-d ] [--h2]\n"); + + memset(&info, 0, sizeof info); /* otherwise uninitialized garbage */ + info.port = CONTEXT_PORT_NO_LISTEN; /* we do not run any server */ + info.protocols = protocols; + info.timeout_secs = 10; + + /* + * since we know this lws context is only ever going to be used with + * one client wsis / fds / sockets at a time, let lws know it doesn't + * have to use the default allocations for fd tables up to ulimit -n. + * It will just allocate for 1 internal and 1 (+ 1 http2 nwsi) that we + * will use. + */ + info.fd_limit_per_thread = 1 + 1 + 1; + + context = lws_create_context(&info); + if (!context) + { + lwsl_err("lws init failed\n"); + return 1; + } + + memset(&i, 0, sizeof i); /* otherwise uninitialized garbage */ + i.context = context; + i.port = 8008; + i.address = "127.0.0.1"; + i.path = "/"; + i.host = i.address; + i.origin = i.address; + i.protocol = protocols[0].name; /* "dumb-increment-protocol" */ + i.pwsi = &client_wsi; + + if (lws_cmdline_option(argc, argv, "--h2")) i.alpn = "h2"; + + lws_client_connect_via_info(&i); + + while (n >= 0 && client_wsi && !interrupted) + n = lws_service(context, 0); + + lws_context_destroy(context); + + lwsl_user("Completed %s\n", receivedCount > 10 ? "OK" : "Failed"); + + t1.join(); + + return receivedCount > 10; +} diff --git a/test/compatibility/csharp/.gitignore b/test/compatibility/csharp/.gitignore new file mode 100644 index 00000000..1746e326 --- /dev/null +++ b/test/compatibility/csharp/.gitignore @@ -0,0 +1,2 @@ +bin +obj diff --git a/test/compatibility/csharp/Main.cs b/test/compatibility/csharp/Main.cs new file mode 100644 index 00000000..f6979e94 --- /dev/null +++ b/test/compatibility/csharp/Main.cs @@ -0,0 +1,99 @@ +// +// Main.cs +// Author: Benjamin Sergeant +// Copyright (c) 2020 Machine Zone, Inc. All rights reserved. +// +// In a different terminal, start a push server: +// $ ws push_server -q +// +// $ dotnet run +// messages received per second: 145157 +// messages received per second: 141405 +// messages received per second: 152202 +// messages received per second: 157149 +// messages received per second: 157673 +// messages received per second: 153594 +// messages received per second: 157830 +// messages received per second: 158422 +// + +using System; +using System.Net.WebSockets; +using System.Threading; +using System.Threading.Tasks; + +public class DevNullClientCli +{ + private static int receivedMessage = 0; + + public static async Task ReceiveAsync(ClientWebSocket ws, CancellationToken token) + { + int bufferSize = 8192; // 8K + var buffer = new byte[bufferSize]; + var offset = 0; + var free = buffer.Length; + + while (true) + { + var result = await ws.ReceiveAsync(new ArraySegment(buffer, offset, free), token).ConfigureAwait(false); + + offset += result.Count; + free -= result.Count; + if (result.EndOfMessage) break; + + if (free == 0) + { + // No free space + // Resize the outgoing buffer + var newSize = buffer.Length + bufferSize; + + var newBuffer = new byte[newSize]; + Array.Copy(buffer, 0, newBuffer, 0, offset); + buffer = newBuffer; + free = buffer.Length - offset; + } + } + + return buffer; + } + + private static void OnTimedEvent(object source, EventArgs e) + { + Console.WriteLine($"messages received per second: {receivedMessage}"); + receivedMessage = 0; + } + + public static async Task ReceiveMessagesAsync(string url) + { + var ws = new ClientWebSocket(); + + System.Uri uri = new System.Uri(url); + var cancellationToken = CancellationToken.None; + + try + { + await ws.ConnectAsync(uri, cancellationToken).ConfigureAwait(false); + while (true) + { + var data = await DevNullClientCli.ReceiveAsync(ws, cancellationToken); + receivedMessage += 1; + } + } + catch (System.Net.WebSockets.WebSocketException e) + { + Console.WriteLine($"WebSocket error: {e}"); + return; + } + } + + public static async Task Main() + { + var timer = new System.Timers.Timer(1000); + timer.Elapsed += OnTimedEvent; + timer.Enabled = true; + timer.Start(); + + var url = "ws://localhost:8008"; + await ReceiveMessagesAsync(url); + } +} diff --git a/test/compatibility/csharp/devnull_client.csproj b/test/compatibility/csharp/devnull_client.csproj new file mode 100644 index 00000000..afa7bad5 --- /dev/null +++ b/test/compatibility/csharp/devnull_client.csproj @@ -0,0 +1,6 @@ + + + Exe + netcoreapp3.1 + + diff --git a/test/compatibility/node/devnull_client.js b/test/compatibility/node/devnull_client.js new file mode 100644 index 00000000..a65b493a --- /dev/null +++ b/test/compatibility/node/devnull_client.js @@ -0,0 +1,42 @@ +// +// With ws@7.3.1 +// and +// node --version +// v13.11.0 +// +// In a different terminal, start a push server: +// $ ws push_server -q +// +// $ node devnull_client.js +// messages received per second: 16643 +// messages received per second: 28065 +// messages received per second: 28432 +// messages received per second: 22207 +// messages received per second: 28805 +// messages received per second: 28694 +// messages received per second: 28180 +// messages received per second: 28601 +// messages received per second: 28698 +// messages received per second: 28931 +// messages received per second: 27975 +// +const WebSocket = require('ws'); + +const ws = new WebSocket('ws://localhost:8008'); + +ws.on('open', function open() { + ws.send('hello from node'); +}); + +var receivedMessages = 0; + +setInterval(function timeout() { + console.log(`messages received per second: ${receivedMessages}`) + receivedMessages = 0; +}, 1000); + +ws.on('message', function incoming(data) { + receivedMessages += 1; +}); + + diff --git a/test/compatibility/python/websockets/devnull_client.py b/test/compatibility/python/websockets/devnull_client.py new file mode 100644 index 00000000..b4325fe7 --- /dev/null +++ b/test/compatibility/python/websockets/devnull_client.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# websocket send client + +import argparse +import asyncio +import websockets + +try: + import uvloop + uvloop.install() +except ImportError: + print('uvloop not available') + pass + +msgCount = 0 + +async def timer(): + global msgCount + + while True: + print(f'Received messages: {msgCount}') + msgCount = 0 + + await asyncio.sleep(1) + + +async def client(url): + global msgCount + + asyncio.ensure_future(timer()) + + async with websockets.connect(url) as ws: + async for message in ws: + msgCount += 1 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='websocket proxy.') + parser.add_argument('--url', help='Remote websocket url', + default='wss://echo.websocket.org') + args = parser.parse_args() + + asyncio.get_event_loop().run_until_complete(client(args.url)) diff --git a/test/compatibility/python/websockets/echo_server.py b/test/compatibility/python/websockets/echo_server.py index 22a44cc6..e609aa8d 100644 --- a/test/compatibility/python/websockets/echo_server.py +++ b/test/compatibility/python/websockets/echo_server.py @@ -10,7 +10,7 @@ import websockets async def echo(websocket, path): while True: msg = await websocket.recv() - print(f'Received {len(msg)} bytes') + # print(f'Received {len(msg)} bytes') await websocket.send(msg) host = os.getenv('BIND_HOST', 'localhost') diff --git a/test/compatibility/ruby/README.md b/test/compatibility/ruby/README.md new file mode 100644 index 00000000..bc0ce6a7 --- /dev/null +++ b/test/compatibility/ruby/README.md @@ -0,0 +1,6 @@ +``` +export GEM_HOME=$HOME/local/gems +bundle install faye-websocket +``` + +https://stackoverflow.com/questions/486995/ruby-equivalent-of-virtualenv diff --git a/test/compatibility/ruby/devnull_client.rb b/test/compatibility/ruby/devnull_client.rb new file mode 100644 index 00000000..1037fe34 --- /dev/null +++ b/test/compatibility/ruby/devnull_client.rb @@ -0,0 +1,59 @@ +# +# $ ruby --version +# ruby 2.6.3p62 (2019-04-16 revision 67580) [universal.x86_64-darwin19] +# +# Install a gem locally by setting GEM_HOME +# https://stackoverflow.com/questions/486995/ruby-equivalent-of-virtualenv +# export GEM_HOME=$HOME/local/gems +# bundle install faye-websocket +# +# In a different terminal, start a push server: +# $ ws push_server -q +# +# $ ruby devnull_client.rb +# [:open] +# Connected to server +# messages received per second: 115926 +# messages received per second: 119156 +# messages received per second: 119156 +# messages received per second: 119157 +# messages received per second: 119156 +# messages received per second: 119156 +# messages received per second: 119157 +# messages received per second: 119156 +# messages received per second: 119156 +# messages received per second: 119157 +# messages received per second: 119156 +# messages received per second: 119157 +# messages received per second: 119156 +# ^C[:close, 1006, ""] +# +require 'faye/websocket' +require 'eventmachine' + +EM.run { + ws = Faye::WebSocket::Client.new('ws://127.0.0.1:8008') + + counter = 0 + + EM.add_periodic_timer(1) do + print "messages received per second: #{counter}\n" + counter = 0 # reset counter + end + + ws.on :open do |event| + p [:open] + print "Connected to server\n" + end + + ws.on :message do |event| + # Uncomment the next line to validate that we receive something correct + # p [:message, event.data] + counter += 1 + end + + ws.on :close do |event| + p [:close, event.code, event.reason] + ws = nil + end +} diff --git a/test/test_runner.cpp b/test/test_runner.cpp index 351708a7..5c000e1e 100644 --- a/test/test_runner.cpp +++ b/test/test_runner.cpp @@ -6,49 +6,21 @@ #define CATCH_CONFIG_RUNNER #include "catch.hpp" -#include #include #include +#ifndef _WIN32 +#include +#endif + int main(int argc, char* argv[]) { ix::initNetSystem(); - ix::CoreLogger::LogFunc logFunc = [](const char* msg, ix::LogLevel level) { - switch (level) - { - case ix::LogLevel::Debug: - { - spdlog::debug(msg); - } - break; - - case ix::LogLevel::Info: - { - spdlog::info(msg); - } - break; - - case ix::LogLevel::Warning: - { - spdlog::warn(msg); - } - break; - - case ix::LogLevel::Error: - { - spdlog::error(msg); - } - break; - - case ix::LogLevel::Critical: - { - spdlog::critical(msg); - } - break; - } - }; - ix::CoreLogger::setLogFunction(logFunc); +#ifndef _WIN32 + signal(SIGPIPE, SIG_IGN); +#endif + spdlog::set_level(spdlog::level::debug); int result = Catch::Session().run(argc, argv); diff --git a/third_party/cli11/CLI11.hpp b/third_party/cli11/CLI11.hpp index 5c880875..ce1d06e3 100644 --- a/third_party/cli11/CLI11.hpp +++ b/third_party/cli11/CLI11.hpp @@ -1,20 +1,16 @@ -#pragma once - -// CLI11: Version 1.8.0 +// CLI11: Version 2.0.0 // Originally designed by Henry Schreiner // https://github.com/CLIUtils/CLI11 // // This is a standalone header file generated by MakeSingleHeader.py in CLI11/scripts -// from: v1.8.0 +// from: v2.0.0 (added include gaurd) // -// From LICENSE: -// -// CLI11 1.8 Copyright (c) 2017-2019 University of Cincinnati, developed by Henry +// CLI11 2.0.0 Copyright (c) 2017-2020 University of Cincinnati, developed by Henry // Schreiner under NSF AWARD 1414736. All rights reserved. -// +// // Redistribution and use in source and binary forms of CLI11, with or without // modification, are permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. // 2. Redistributions in binary form must reproduce the above copyright notice, @@ -23,7 +19,7 @@ // 3. Neither the name of the copyright holder nor the names of its contributors // may be used to endorse or promote products derived from this software without // specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -35,50 +31,42 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once // Standard combined includes: - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -// Verbatim copy from CLI/Version.hpp: - - -#define CLI11_VERSION_MAJOR 1 -#define CLI11_VERSION_MINOR 8 +#define CLI11_VERSION_MAJOR 2 +#define CLI11_VERSION_MINOR 0 #define CLI11_VERSION_PATCH 0 -#define CLI11_VERSION "1.8.0" +#define CLI11_VERSION "2.0.0" -// Verbatim copy from CLI/Macros.hpp: - - -// The following version macro is very similar to the one in PyBind11 +// The following version macro is very similar to the one in pybind11 #if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER) #if __cplusplus >= 201402L #define CLI11_CPP14 @@ -114,109 +102,42 @@ -// Verbatim copy from CLI/Optional.hpp: - - -// You can explicitly enable or disable support -// by defining to 1 or 0. Extra check here to ensure it's in the stdlib too. -// We nest the check for __has_include and it's usage -#ifndef CLI11_STD_OPTIONAL -#ifdef __has_include -#if defined(CLI11_CPP17) && __has_include() -#define CLI11_STD_OPTIONAL 1 +// C standard library +// Only needed for existence checking +#if defined CLI11_CPP17 && defined __has_include && !defined CLI11_HAS_FILESYSTEM +#if __has_include() +// Filesystem cannot be used if targeting macOS < 10.15 +#if defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 +#define CLI11_HAS_FILESYSTEM 0 #else -#define CLI11_STD_OPTIONAL 0 +#include +#if defined __cpp_lib_filesystem && __cpp_lib_filesystem >= 201703 +#if defined _GLIBCXX_RELEASE && _GLIBCXX_RELEASE >= 9 +#define CLI11_HAS_FILESYSTEM 1 +#elif defined(__GLIBCXX__) +// if we are using gcc and Version <9 default to no filesystem +#define CLI11_HAS_FILESYSTEM 0 +#else +#define CLI11_HAS_FILESYSTEM 1 #endif #else -#define CLI11_STD_OPTIONAL 0 +#define CLI11_HAS_FILESYSTEM 0 +#endif +#endif #endif #endif -#ifndef CLI11_EXPERIMENTAL_OPTIONAL -#define CLI11_EXPERIMENTAL_OPTIONAL 0 -#endif - -#ifndef CLI11_BOOST_OPTIONAL -#define CLI11_BOOST_OPTIONAL 0 -#endif - -#if CLI11_BOOST_OPTIONAL -#include -#if BOOST_VERSION < 106100 -#error "This boost::optional version is not supported, use 1.61 or better" -#endif -#endif - -#if CLI11_STD_OPTIONAL -#include -#endif -#if CLI11_EXPERIMENTAL_OPTIONAL -#include -#endif -#if CLI11_BOOST_OPTIONAL -#include -#include +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +#include // NOLINT(build/include) +#else +#include +#include #endif -// From CLI/Version.hpp: - - - -// From CLI/Macros.hpp: - - - -// From CLI/Optional.hpp: namespace CLI { -#if CLI11_STD_OPTIONAL -template std::istream &operator>>(std::istream &in, std::optional &val) { - T v; - in >> v; - val = v; - return in; -} -#endif - -#if CLI11_EXPERIMENTAL_OPTIONAL -template std::istream &operator>>(std::istream &in, std::experimental::optional &val) { - T v; - in >> v; - val = v; - return in; -} -#endif - -#if CLI11_BOOST_OPTIONAL -template std::istream &operator>>(std::istream &in, boost::optional &val) { - T v; - in >> v; - val = v; - return in; -} -#endif - -// Export the best optional to the CLI namespace -#if CLI11_STD_OPTIONAL -using std::optional; -#elif CLI11_EXPERIMENTAL_OPTIONAL -using std::experimental::optional; -#elif CLI11_BOOST_OPTIONAL -using boost::optional; -#endif - -// This is true if any optional is found -#if CLI11_STD_OPTIONAL || CLI11_EXPERIMENTAL_OPTIONAL || CLI11_BOOST_OPTIONAL -#define CLI11_OPTIONAL 1 -#endif - -} // namespace CLI - -// From CLI/StringTools.hpp: - -namespace CLI { /// Include the items in this namespace to get free conversion of enums to/from streams. /// (This is available inside CLI as well, so CLI11 will use this without a using statement). @@ -229,29 +150,23 @@ std::ostream &operator<<(std::ostream &in, const T &item) { return in << static_cast::type>(item); } -/// input streaming for enumerations -template ::value>::type> -std::istream &operator>>(std::istream &in, T &item) { - typename std::underlying_type::type i; - in >> i; - item = static_cast(i); - return in; -} -} // namespace enums +} // namespace enums /// Export to CLI namespace -using namespace enums; +using enums::operator<<; namespace detail { - +/// a constant defining an expected max vector size defined to be a big number that could be multiplied by 4 and not +/// produce overflow for some expected uses +constexpr int expected_max_vector_size{1 << 29}; // Based on http://stackoverflow.com/questions/236129/split-a-string-in-c /// Split a string by a delim inline std::vector split(const std::string &s, char delim) { std::vector elems; // Check to see if empty string, give consistent result - if(s.empty()) + if(s.empty()) { elems.emplace_back(); - else { + } else { std::stringstream ss; ss.str(s); std::string item; @@ -261,17 +176,6 @@ inline std::vector split(const std::string &s, char delim) { } return elems; } -/// simple utility to convert various types to a string -template inline std::string as_string(const T &v) { - std::ostringstream s; - s << v; - return s.str(); -} -// if the data type is already a string just forward it -template ::value>::type> -inline auto as_string(T &&v) -> decltype(std::forward(v)) { - return std::forward(v); -} /// Simple function to join a string template std::string join(const T &v, std::string delim = ",") { @@ -294,10 +198,14 @@ std::string join(const T &v, Callable func, std::string delim = ",") { std::ostringstream s; auto beg = std::begin(v); auto end = std::end(v); - if(beg != end) - s << func(*beg++); + auto loc = s.tellp(); while(beg != end) { - s << delim << func(*beg++); + auto nloc = s.tellp(); + if(nloc > loc) { + s << delim; + loc = nloc; + } + s << func(*beg++); } return s.str(); } @@ -305,7 +213,7 @@ std::string join(const T &v, Callable func, std::string delim = ",") { /// Join a string in reverse order template std::string rjoin(const T &v, std::string delim = ",") { std::ostringstream s; - for(size_t start = 0; start < v.size(); start++) { + for(std::size_t start = 0; start < v.size(); start++) { if(start > 0) s << delim; s << v[v.size() - start - 1]; @@ -356,13 +264,24 @@ inline std::string trim_copy(const std::string &str) { return trim(s); } +/// remove quotes at the front and back of a string either '"' or '\'' +inline std::string &remove_quotes(std::string &str) { + if(str.length() > 1 && (str.front() == '"' || str.front() == '\'')) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + /// Make a copy of the string and then trim it, any filter string can be used (any char in string is filtered) inline std::string trim_copy(const std::string &str, const std::string &filter) { std::string s = str; return trim(s, filter); } /// Print a two part "help" string -inline std::ostream &format_help(std::ostream &out, std::string name, std::string description, size_t wid) { +inline std::ostream &format_help(std::ostream &out, std::string name, const std::string &description, std::size_t wid) { name = " " + name; out << std::setw(static_cast(wid)) << std::left << name; if(!description.empty()) { @@ -379,6 +298,24 @@ inline std::ostream &format_help(std::ostream &out, std::string name, std::strin return out; } +/// Print subcommand aliases +inline std::ostream &format_aliases(std::ostream &out, const std::vector &aliases, std::size_t wid) { + if(!aliases.empty()) { + out << std::setw(static_cast(wid)) << " aliases: "; + bool front = true; + for(const auto &alias : aliases) { + if(!front) { + out << ", "; + } else { + front = false; + } + out << alias; + } + out << "\n"; + } + return out; +} + /// Verify the first character of an option template bool valid_first_char(T c) { return std::isalnum(c, std::locale()) || c == '_' || c == '?' || c == '@'; @@ -397,6 +334,12 @@ inline bool valid_name_string(const std::string &str) { return true; } +/// check if a string is a container segment separator (empty or "%%") +inline bool is_separator(const std::string &str) { + static const std::string sep("%%"); + return (str.empty() || str == sep); +} + /// Verify that str consists of letters only inline bool isalpha(const std::string &str) { return std::all_of(str.begin(), str.end(), [](char c) { return std::isalpha(c, std::locale()); }); @@ -419,7 +362,7 @@ inline std::string remove_underscore(std::string str) { /// Find and replace a substring with another substring inline std::string find_and_replace(std::string str, std::string from, std::string to) { - size_t start_pos = 0; + std::size_t start_pos = 0; while((start_pos = str.find(from, start_pos)) != std::string::npos) { str.replace(start_pos, from.length(), to); @@ -471,8 +414,9 @@ inline std::ptrdiff_t find_member(std::string name, it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { return detail::remove_underscore(local_name) == name; }); - } else + } else { it = std::find(std::begin(names), std::end(names), name); + } return (it != std::end(names)) ? (it - std::begin(names)) : (-1); } @@ -480,7 +424,7 @@ inline std::ptrdiff_t find_member(std::string name, /// Find a trigger string and call a modify callable function that takes the current string and starting position of the /// trigger and returns the position in the string to search for the next trigger string template inline std::string find_and_modify(std::string str, std::string trigger, Callable modify) { - size_t start_pos = 0; + std::size_t start_pos = 0; while((start_pos = str.find(trigger, start_pos)) != std::string::npos) { start_pos = modify(str, start_pos); } @@ -489,10 +433,12 @@ template inline std::string find_and_modify(std::string str, /// Split a string '"one two" "three"' into 'one two', 'three' /// Quote characters can be ` ' or " -inline std::vector split_up(std::string str) { +inline std::vector split_up(std::string str, char delimiter = '\0') { const std::string delims("\'\"`"); - auto find_ws = [](char ch) { return std::isspace(ch, std::locale()); }; + auto find_ws = [delimiter](char ch) { + return (delimiter == '\0') ? (std::isspace(ch, std::locale()) != 0) : (ch == delimiter); + }; trim(str); std::vector output; @@ -502,13 +448,18 @@ inline std::vector split_up(std::string str) { if(delims.find_first_of(str[0]) != std::string::npos) { keyChar = str[0]; auto end = str.find_first_of(keyChar, 1); - while((end != std::string::npos) && (str[end - 1] == '\\')) { // deal with escaped quotes + while((end != std::string::npos) && (str[end - 1] == '\\')) { // deal with escaped quotes end = str.find_first_of(keyChar, end + 1); embeddedQuote = true; } if(end != std::string::npos) { output.push_back(str.substr(1, end - 1)); - str = str.substr(end + 1); + if(end + 2 < str.size()) { + str = str.substr(end + 2); + } else { + str.clear(); + } + } else { output.push_back(str.substr(1)); str = ""; @@ -518,7 +469,7 @@ inline std::vector split_up(std::string str) { if(it != std::end(str)) { std::string value = std::string(str.begin(), it); output.push_back(value); - str = std::string(it, str.end()); + str = std::string(it + 1, str.end()); } else { output.push_back(str); str = ""; @@ -538,7 +489,7 @@ inline std::vector split_up(std::string str) { /// at the start of the first line). `"; "` would be for ini files /// /// Can't use Regex, or this would be a subs. -inline std::string fix_newlines(std::string leader, std::string input) { +inline std::string fix_newlines(const std::string &leader, std::string input) { std::string::size_type n = 0; while(n != std::string::npos && n < input.size()) { n = input.find('\n', n); @@ -554,13 +505,13 @@ inline std::string fix_newlines(std::string leader, std::string input) { /// then modifies the string to replace the equality with a space. This is needed /// to allow the split up function to work properly and is intended to be used with the find_and_modify function /// the return value is the offset+1 which is required by the find_and_modify function. -inline size_t escape_detect(std::string &str, size_t offset) { +inline std::size_t escape_detect(std::string &str, std::size_t offset) { auto next = str[offset + 1]; if((next == '\"') || (next == '\'') || (next == '`')) { auto astart = str.find_last_of("-/ \"\'`", offset - 1); if(astart != std::string::npos) { if(str[astart] == ((str[offset] == '=') ? '-' : '/')) - str[offset] = ' '; // interpret this as a space so the split_up works properly + str[offset] = ' '; // interpret this as a space so the split_up works properly } } return offset + 1; @@ -578,13 +529,10 @@ inline std::string &add_quotes_if_needed(std::string &str) { return str; } -} // namespace detail +} // namespace detail -} // namespace CLI -// From CLI/Error.hpp: -namespace CLI { // Use one of these on all error classes. // These are temporary and are undef'd at the end of this file. @@ -726,19 +674,26 @@ class Success : public ParseError { }; /// -h or --help on command line -class CallForHelp : public ParseError { - CLI11_ERROR_DEF(ParseError, CallForHelp) +class CallForHelp : public Success { + CLI11_ERROR_DEF(Success, CallForHelp) CallForHelp() : CallForHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} }; /// Usually something like --help-all on command line -class CallForAllHelp : public ParseError { - CLI11_ERROR_DEF(ParseError, CallForAllHelp) +class CallForAllHelp : public Success { + CLI11_ERROR_DEF(Success, CallForAllHelp) CallForAllHelp() : CallForAllHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} }; -/// Does not output a diagnostic in CLI11_PARSE, but allows to return from main() with a specific error code. +/// -v or --version on command line +class CallForVersion : public Success { + CLI11_ERROR_DEF(Success, CallForVersion) + CallForVersion() + : CallForVersion("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Does not output a diagnostic in CLI11_PARSE, but allows main() to return with a specific error code. class RuntimeError : public ParseError { CLI11_ERROR_DEF(ParseError, RuntimeError) explicit RuntimeError(int exit_code = 1) : RuntimeError("Runtime error", exit_code) {} @@ -778,33 +733,36 @@ class ValidationError : public ParseError { class RequiredError : public ParseError { CLI11_ERROR_DEF(ParseError, RequiredError) explicit RequiredError(std::string name) : RequiredError(name + " is required", ExitCodes::RequiredError) {} - static RequiredError Subcommand(size_t min_subcom) { - if(min_subcom == 1) + static RequiredError Subcommand(std::size_t min_subcom) { + if(min_subcom == 1) { return RequiredError("A subcommand"); - else - return RequiredError("Requires at least " + std::to_string(min_subcom) + " subcommands", - ExitCodes::RequiredError); + } + return RequiredError("Requires at least " + std::to_string(min_subcom) + " subcommands", + ExitCodes::RequiredError); } - static RequiredError Option(size_t min_option, size_t max_option, size_t used, const std::string &option_list) { + static RequiredError + Option(std::size_t min_option, std::size_t max_option, std::size_t used, const std::string &option_list) { if((min_option == 1) && (max_option == 1) && (used == 0)) return RequiredError("Exactly 1 option from [" + option_list + "]"); - else if((min_option == 1) && (max_option == 1) && (used > 1)) + if((min_option == 1) && (max_option == 1) && (used > 1)) { return RequiredError("Exactly 1 option from [" + option_list + "] is required and " + std::to_string(used) + " were given", ExitCodes::RequiredError); - else if((min_option == 1) && (used == 0)) + } + if((min_option == 1) && (used == 0)) return RequiredError("At least 1 option from [" + option_list + "]"); - else if(used < min_option) + if(used < min_option) { return RequiredError("Requires at least " + std::to_string(min_option) + " options used and only " + std::to_string(used) + "were given from [" + option_list + "]", ExitCodes::RequiredError); - else if(max_option == 1) + } + if(max_option == 1) return RequiredError("Requires at most 1 options be given from [" + option_list + "]", ExitCodes::RequiredError); - else - return RequiredError("Requires at most " + std::to_string(max_option) + " options be used and " + - std::to_string(used) + "were given from [" + option_list + "]", - ExitCodes::RequiredError); + + return RequiredError("Requires at most " + std::to_string(max_option) + " options be used and " + + std::to_string(used) + "were given from [" + option_list + "]", + ExitCodes::RequiredError); } }; @@ -812,15 +770,20 @@ class RequiredError : public ParseError { class ArgumentMismatch : public ParseError { CLI11_ERROR_DEF(ParseError, ArgumentMismatch) CLI11_ERROR_SIMPLE(ArgumentMismatch) - ArgumentMismatch(std::string name, int expected, size_t recieved) + ArgumentMismatch(std::string name, int expected, std::size_t received) : ArgumentMismatch(expected > 0 ? ("Expected exactly " + std::to_string(expected) + " arguments to " + name + - ", got " + std::to_string(recieved)) + ", got " + std::to_string(received)) : ("Expected at least " + std::to_string(-expected) + " arguments to " + name + - ", got " + std::to_string(recieved)), + ", got " + std::to_string(received)), ExitCodes::ArgumentMismatch) {} - static ArgumentMismatch AtLeast(std::string name, int num) { - return ArgumentMismatch(name + ": At least " + std::to_string(num) + " required"); + static ArgumentMismatch AtLeast(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At least " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch AtMost(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At Most " + std::to_string(num) + " required but received " + + std::to_string(received)); } static ArgumentMismatch TypedAtLeast(std::string name, int num, std::string type) { return ArgumentMismatch(name + ": " + std::to_string(num) + " required " + type + " missing"); @@ -852,6 +815,12 @@ class ExtrasError : public ParseError { : "The following argument was not expected: ") + detail::rjoin(args, " "), ExitCodes::ExtrasError) {} + ExtrasError(const std::string &name, std::vector args) + : ExtrasError(name, + (args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} }; /// Thrown when extra values are found in an INI file @@ -892,11 +861,8 @@ class OptionNotFound : public Error { /// @} -} // namespace CLI -// From CLI/TypeTools.hpp: -namespace CLI { // Type tools @@ -908,7 +874,7 @@ enum class enabler {}; /// An instance to use in EnableIf constexpr enabler dummy = {}; -} // namespace detail +} // namespace detail /// A copy of enable_if_t from C++14, compatible with C++11. /// @@ -926,12 +892,6 @@ template using void_t = typename make_void::type; /// A copy of std::conditional_t from C++14 - same reasoning as enable_if_t, it does not hurt to redefine template using conditional_t = typename std::conditional::type; -/// Check to see if something is a vector (fail check by default) -template struct is_vector : std::false_type {}; - -/// Check to see if something is a vector (true if actually a vector) -template struct is_vector> : std::true_type {}; - /// Check to see if something is bool (fail check by default) template struct is_bool : std::false_type {}; @@ -960,13 +920,16 @@ template <> struct IsMemberType { using type = std::string; }; namespace detail { -// These are utilities for IsMember +// These are utilities for IsMember and other transforming objects /// Handy helper to access the element_type generically. This is not part of is_copyable_ptr because it requires that /// pointer_traits be valid. -template struct element_type { - using type = - typename std::conditional::value, typename std::pointer_traits::element_type, T>::type; + +/// not a pointer +template struct element_type { using type = T; }; + +template struct element_type::value>::type> { + using type = typename std::pointer_traits::element_type; }; /// Combination of the element type and value type - remove pointer (including smart pointers) and get the value_type of @@ -1010,28 +973,158 @@ struct pair_adaptor< } }; -// Check for streamability +// Warning is suppressed due to "bug" in gcc<5.0 and gcc 7.0 with c++17 enabled that generates a Wnarrowing warning +// in the unevaluated context even if the function that was using this wasn't used. The standard says narrowing in +// brace initialization shouldn't be allowed but for backwards compatibility gcc allows it in some contexts. It is a +// little fuzzy what happens in template constructs and I think that was something GCC took a little while to work out. +// But regardless some versions of gcc generate a warning when they shouldn't from the following code so that should be +// suppressed +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif +// check for constructibility from a specific type and copy assignable used in the parse detection +template class is_direct_constructible { + template + static auto test(int, std::true_type) -> decltype( +// NVCC warns about narrowing conversions here +#ifdef __CUDACC__ +#pragma diag_suppress 2361 +#endif + TT { std::declval() } +#ifdef __CUDACC__ +#pragma diag_default 2361 +#endif + , + std::is_move_assignable()); + + template static auto test(int, std::false_type) -> std::false_type; + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0, typename std::is_constructible::type()))::value; +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +// Check for output streamability // Based on https://stackoverflow.com/questions/22758291/how-can-i-detect-if-a-type-can-be-streamed-to-an-stdostream -template class is_streamable { - template +template class is_ostreamable { + template static auto test(int) -> decltype(std::declval() << std::declval(), std::true_type()); template static auto test(...) -> std::false_type; public: - static const bool value = decltype(test(0))::value; + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for input streamability +template class is_istreamable { + template + static auto test(int) -> decltype(std::declval() >> std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for complex +template class is_complex { + template + static auto test(int) -> decltype(std::declval().real(), std::declval().imag(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Templated operation to get a value from a stream +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string &istring, T &obj) { + std::istringstream is; + is.str(istring); + is >> obj; + return !is.fail() && !is.rdbuf()->in_avail(); +} + +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string & /*istring*/, T & /*obj*/) { + return false; +} + +// check to see if an object is a mutable container (fail by default) +template struct is_mutable_container : std::false_type {}; + +/// type trait to test if a type is a mutable container meaning it has a value_type, it has an iterator, a clear, and +/// end methods and an insert function. And for our purposes we exclude std::string and types that can be constructed +/// from a std::string +template +struct is_mutable_container< + T, + conditional_t().end()), + decltype(std::declval().clear()), + decltype(std::declval().insert(std::declval().end())>(), + std::declval()))>, + void>> + : public conditional_t::value, std::false_type, std::true_type> {}; + +// check to see if an object is a mutable container (fail by default) +template struct is_readable_container : std::false_type {}; + +/// type trait to test if a type is a container meaning it has a value_type, it has an iterator, a clear, and an end +/// methods and an insert function. And for our purposes we exclude std::string and types that can be constructed from +/// a std::string +template +struct is_readable_container< + T, + conditional_t().end()), decltype(std::declval().begin())>, void>> + : public std::true_type {}; + +// check to see if an object is a wrapper (fail by default) +template struct is_wrapper : std::false_type {}; + +// check if an object is a wrapper (it has a value_type defined) +template +struct is_wrapper, void>> : public std::true_type {}; + +// Check for tuple like types, as in classes with a tuple_size type trait +template class is_tuple_like { + template + // static auto test(int) + // -> decltype(std::conditional<(std::tuple_size::value > 0), std::true_type, std::false_type>::type()); + static auto test(int) -> decltype(std::tuple_size::type>::value, std::true_type{}); + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; }; /// Convert an object to a string (directly forward if this can become a string) -template ::value, detail::enabler> = detail::dummy> +template ::value, detail::enabler> = detail::dummy> auto to_string(T &&value) -> decltype(std::forward(value)) { return std::forward(value); } +/// Construct a string from the object +template ::value && !std::is_convertible::value, + detail::enabler> = detail::dummy> +std::string to_string(const T &value) { + return std::string(value); +} + /// Convert an object to a string (streaming must be supported for that type) template ::value && is_streamable::value, + enable_if_t::value && !std::is_constructible::value && + is_ostreamable::value, detail::enabler> = detail::dummy> std::string to_string(T &&value) { std::stringstream stream; @@ -1041,12 +1134,380 @@ std::string to_string(T &&value) { /// If conversion is not supported, return an empty string (streaming is not supported for that type) template ::value && !is_streamable::value, + enable_if_t::value && !is_ostreamable::value && + !is_readable_container::type>::value, detail::enabler> = detail::dummy> std::string to_string(T &&) { return std::string{}; } +/// convert a readable container to a string +template ::value && !is_ostreamable::value && + is_readable_container::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&variable) { + std::vector defaults; + auto cval = variable.begin(); + auto end = variable.end(); + while(cval != end) { + defaults.emplace_back(CLI::detail::to_string(*cval)); + ++cval; + } + return std::string("[" + detail::join(defaults) + "]"); +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +auto checked_to_string(T &&value) -> decltype(to_string(std::forward(value))) { + return to_string(std::forward(value)); +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +std::string checked_to_string(T &&) { + return std::string{}; +} +/// get a string as a convertible value for arithmetic types +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(value); +} +/// get a string as a convertible value for enumerations +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(static_cast::type>(value)); +} +/// for other types just use the regular to_string function +template ::value && !std::is_arithmetic::value, detail::enabler> = detail::dummy> +auto value_string(const T &value) -> decltype(to_string(value)) { + return to_string(value); +} + +/// template to get the underlying value type if it exists or use a default +template struct wrapped_type { using type = def; }; + +/// Type size for regular object types that do not look like a tuple +template struct wrapped_type::value>::type> { + using type = typename T::value_type; +}; + +/// This will only trigger for actual void type +template struct type_count_base { static const int value{0}; }; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count_base::value && !is_mutable_container::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// the base tuple size +template +struct type_count_base::value && !is_mutable_container::value>::type> { + static constexpr int value{std::tuple_size::value}; +}; + +/// Type count base for containers is the type_count_base of the individual element +template struct type_count_base::value>::type> { + static constexpr int value{type_count_base::value}; +}; + +/// Set of overloads to get the type size of an object + +/// forward declare the subtype_count structure +template struct subtype_count; + +/// forward declare the subtype_count_min structure +template struct subtype_count_min; + +/// This will only trigger for actual void type +template struct type_count { static const int value{0}; }; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count::value && !is_tuple_like::value && !is_complex::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// Type size for complex since it sometimes looks like a wrapper +template struct type_count::value>::type> { + static constexpr int value{2}; +}; + +/// Type size of types that are wrappers,except complex and tuples(which can also be wrappers sometimes) +template struct type_count::value>::type> { + static constexpr int value{subtype_count::value}; +}; + +/// Type size of types that are wrappers,except containers complex and tuples(which can also be wrappers sometimes) +template +struct type_count::value && !is_complex::value && !is_tuple_like::value && + !is_mutable_container::value>::type> { + static constexpr int value{type_count::value}; +}; + +/// 0 if the index > tuple size +template +constexpr typename std::enable_if::value, int>::type tuple_type_size() { + return 0; +} + +/// Recursively generate the tuple type name +template + constexpr typename std::enable_if < I::value, int>::type tuple_type_size() { + return subtype_count::type>::value + tuple_type_size(); +} + +/// Get the type size of the sum of type sizes for all the individual tuple types +template struct type_count::value>::type> { + static constexpr int value{tuple_type_size()}; +}; + +/// definition of subtype count +template struct subtype_count { + static constexpr int value{is_mutable_container::value ? expected_max_vector_size : type_count::value}; +}; + +/// This will only trigger for actual void type +template struct type_count_min { static const int value{0}; }; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count_min< + T, + typename std::enable_if::value && !is_tuple_like::value && !is_wrapper::value && + !is_complex::value && !std::is_void::value>::type> { + static constexpr int value{type_count::value}; +}; + +/// Type size for complex since it sometimes looks like a wrapper +template struct type_count_min::value>::type> { + static constexpr int value{1}; +}; + +/// Type size min of types that are wrappers,except complex and tuples(which can also be wrappers sometimes) +template +struct type_count_min< + T, + typename std::enable_if::value && !is_complex::value && !is_tuple_like::value>::type> { + static constexpr int value{subtype_count_min::value}; +}; + +/// 0 if the index > tuple size +template +constexpr typename std::enable_if::value, int>::type tuple_type_size_min() { + return 0; +} + +/// Recursively generate the tuple type name +template + constexpr typename std::enable_if < I::value, int>::type tuple_type_size_min() { + return subtype_count_min::type>::value + tuple_type_size_min(); +} + +/// Get the type size of the sum of type sizes for all the individual tuple types +template struct type_count_min::value>::type> { + static constexpr int value{tuple_type_size_min()}; +}; + +/// definition of subtype count +template struct subtype_count_min { + static constexpr int value{is_mutable_container::value + ? ((type_count::value < expected_max_vector_size) ? type_count::value : 0) + : type_count_min::value}; +}; + +/// This will only trigger for actual void type +template struct expected_count { static const int value{0}; }; + +/// For most types the number of expected items is 1 +template +struct expected_count::value && !is_wrapper::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; +/// number of expected items in a vector +template struct expected_count::value>::type> { + static constexpr int value{expected_max_vector_size}; +}; + +/// number of expected items in a vector +template +struct expected_count::value && is_wrapper::value>::type> { + static constexpr int value{expected_count::value}; +}; + +// Enumeration of the different supported categorizations of objects +enum class object_category : int { + char_value = 1, + integral_value = 2, + unsigned_integral = 4, + enumeration = 6, + boolean_value = 8, + floating_point = 10, + number_constructible = 12, + double_constructible = 14, + integer_constructible = 16, + // string like types + string_assignable = 23, + string_constructible = 24, + other = 45, + // special wrapper or container types + wrapper_value = 50, + complex_number = 60, + tuple_value = 70, + container_value = 80, + +}; + +/// Set of overloads to classify an object according to type + +/// some type that is not otherwise recognized +template struct classify_object { + static constexpr object_category value{object_category::other}; +}; + +/// Signed integers +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_same::value && std::is_signed::value && + !is_bool::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::integral_value}; +}; + +/// Unsigned integers +template +struct classify_object::value && std::is_unsigned::value && + !std::is_same::value && !is_bool::value>::type> { + static constexpr object_category value{object_category::unsigned_integral}; +}; + +/// single character values +template +struct classify_object::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::char_value}; +}; + +/// Boolean values +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::boolean_value}; +}; + +/// Floats +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::floating_point}; +}; + +/// String and similar direct assignment +template +struct classify_object::value && !std::is_integral::value && + std::is_assignable::value>::type> { + static constexpr object_category value{object_category::string_assignable}; +}; + +/// String and similar constructible and copy assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + !std::is_assignable::value && (type_count::value == 1) && + std::is_constructible::value>::type> { + static constexpr object_category value{object_category::string_constructible}; +}; + +/// Enumerations +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::enumeration}; +}; + +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::complex_number}; +}; + +/// Handy helper to contain a bunch of checks that rule out many common types (integers, string like, floating point, +/// vectors, and enumerations +template struct uncommon_type { + using type = typename std::conditional::value && !std::is_integral::value && + !std::is_assignable::value && + !std::is_constructible::value && !is_complex::value && + !is_mutable_container::value && !std::is_enum::value, + std::true_type, + std::false_type>::type; + static constexpr bool value = type::value; +}; + +/// wrapper type +template +struct classify_object::value && is_wrapper::value && + !is_tuple_like::value && uncommon_type::value)>::type> { + static constexpr object_category value{object_category::wrapper_value}; +}; + +/// Assignable from double or int +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::number_constructible}; +}; + +/// Assignable from int +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && !is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::integer_constructible}; +}; + +/// Assignable from double +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && is_direct_constructible::value && + !is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::double_constructible}; +}; + +/// Tuple type +template +struct classify_object< + T, + typename std::enable_if::value && + ((type_count::value >= 2 && !is_wrapper::value) || + (uncommon_type::value && !is_direct_constructible::value && + !is_direct_constructible::value))>::type> { + static constexpr object_category value{object_category::tuple_value}; + // the condition on this class requires it be like a tuple, but on some compilers (like Xcode) tuples can be + // constructed from just the first element so tuples of can be constructed from a string, which + // could lead to issues so there are two variants of the condition, the first isolates things with a type size >=2 + // mainly to get tuples on Xcode with the exception of wrappers, the second is the main one and just separating out + // those cases that are caught by other object classifications +}; + +/// container type +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::container_value}; +}; + // Type name print /// Was going to be based on @@ -1054,46 +1515,147 @@ std::string to_string(T &&) { /// But this is cleaner and works better in this case template ::value && std::is_signed::value, detail::enabler> = detail::dummy> + enable_if_t::value == object_category::char_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "CHAR"; +} + +template ::value == object_category::integral_value || + classify_object::value == object_category::integer_constructible, + detail::enabler> = detail::dummy> constexpr const char *type_name() { return "INT"; } template ::value && std::is_unsigned::value, detail::enabler> = detail::dummy> + enable_if_t::value == object_category::unsigned_integral, detail::enabler> = detail::dummy> constexpr const char *type_name() { return "UINT"; } -template ::value, detail::enabler> = detail::dummy> +template ::value == object_category::floating_point || + classify_object::value == object_category::number_constructible || + classify_object::value == object_category::double_constructible, + detail::enabler> = detail::dummy> constexpr const char *type_name() { return "FLOAT"; } -/// This one should not be used, since vector types print the internal type -template ::value, detail::enabler> = detail::dummy> -constexpr const char *type_name() { - return "VECTOR"; -} /// Print name for enumeration types -template ::value, detail::enabler> = detail::dummy> +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> constexpr const char *type_name() { return "ENUM"; } +/// Print name for enumeration types +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "BOOLEAN"; +} + +/// Print name for enumeration types +template ::value == object_category::complex_number, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "COMPLEX"; +} + /// Print for all other types template ::value && !std::is_integral::value && !is_vector::value && - !std::is_enum::value, + enable_if_t::value >= object_category::string_assignable && + classify_object::value <= object_category::other, detail::enabler> = detail::dummy> constexpr const char *type_name() { return "TEXT"; } +/// typename for tuple value +template ::value == object_category::tuple_value && type_count_base::value >= 2, + detail::enabler> = detail::dummy> +std::string type_name(); // forward declaration + +/// Generate type name for a wrapper or container value +template ::value == object_category::container_value || + classify_object::value == object_category::wrapper_value, + detail::enabler> = detail::dummy> +std::string type_name(); // forward declaration + +/// Print name for single element tuple types +template ::value == object_category::tuple_value && type_count_base::value == 1, + detail::enabler> = detail::dummy> +inline std::string type_name() { + return type_name::type>::type>(); +} + +/// Empty string if the index > tuple size +template +inline typename std::enable_if::value, std::string>::type tuple_name() { + return std::string{}; +} + +/// Recursively generate the tuple type name +template +inline typename std::enable_if<(I < type_count_base::value), std::string>::type tuple_name() { + std::string str = std::string(type_name::type>::type>()) + + ',' + tuple_name(); + if(str.back() == ',') + str.pop_back(); + return str; +} + +/// Print type name for tuples with 2 or more elements +template ::value == object_category::tuple_value && type_count_base::value >= 2, + detail::enabler>> +inline std::string type_name() { + auto tname = std::string(1, '[') + tuple_name(); + tname.push_back(']'); + return tname; +} + +/// get the type name for a type that has a value_type member +template ::value == object_category::container_value || + classify_object::value == object_category::wrapper_value, + detail::enabler>> +inline std::string type_name() { + return type_name(); +} // Lexical cast +/// Convert to an unsigned integral +template ::value, detail::enabler> = detail::dummy> +bool integral_conversion(const std::string &input, T &output) noexcept { + if(input.empty()) { + return false; + } + char *val = nullptr; + std::uint64_t output_ll = std::strtoull(input.c_str(), &val, 0); + output = static_cast(output_ll); + return val == (input.c_str() + input.size()) && static_cast(output) == output_ll; +} + +/// Convert to a signed integral +template ::value, detail::enabler> = detail::dummy> +bool integral_conversion(const std::string &input, T &output) noexcept { + if(input.empty()) { + return false; + } + char *val = nullptr; + std::int64_t output_ll = std::strtoll(input.c_str(), &val, 0); + output = static_cast(output_ll); + return val == (input.c_str() + input.size()) && static_cast(output) == output_ll; +} + /// Convert a flag into an integer value typically binary flags -inline int64_t to_flag_value(std::string val) { +inline std::int64_t to_flag_value(std::string val) { static const std::string trueString("true"); static const std::string falseString("false"); if(val == trueString) { @@ -1103,8 +1665,11 @@ inline int64_t to_flag_value(std::string val) { return -1; } val = detail::to_lower(val); - int64_t ret; + std::int64_t ret; if(val.size() == 1) { + if(val[0] >= '1' && val[0] <= '9') { + return (static_cast(val[0]) - '0'); + } switch(val[0]) { case '0': case 'f': @@ -1112,22 +1677,11 @@ inline int64_t to_flag_value(std::string val) { case '-': ret = -1; break; - case '1': case 't': case 'y': case '+': ret = 1; break; - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - ret = val[0] - '0'; - break; default: throw std::invalid_argument("unrecognized character"); } @@ -1143,113 +1697,648 @@ inline int64_t to_flag_value(std::string val) { return ret; } -/// Signed integers -template < - typename T, - enable_if_t::value && std::is_signed::value && !is_bool::value && !std::is_enum::value, - detail::enabler> = detail::dummy> -bool lexical_cast(std::string input, T &output) { - try { - size_t n = 0; - long long output_ll = std::stoll(input, &n, 0); - output = static_cast(output_ll); - return n == input.size() && static_cast(output) == output_ll; - } catch(const std::invalid_argument &) { - return false; - } catch(const std::out_of_range &) { - return false; - } +/// Integer conversion +template ::value == object_category::integral_value || + classify_object::value == object_category::unsigned_integral, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + return integral_conversion(input, output); } -/// Unsigned integers +/// char values template ::value && std::is_unsigned::value && !is_bool::value, detail::enabler> = - detail::dummy> -bool lexical_cast(std::string input, T &output) { - if(!input.empty() && input.front() == '-') - return false; // std::stoull happily converts negative values to junk without any errors. - - try { - size_t n = 0; - unsigned long long output_ll = std::stoull(input, &n, 0); - output = static_cast(output_ll); - return n == input.size() && static_cast(output) == output_ll; - } catch(const std::invalid_argument &) { - return false; - } catch(const std::out_of_range &) { - return false; + enable_if_t::value == object_category::char_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(input.size() == 1) { + output = static_cast(input[0]); + return true; } + return integral_conversion(input, output); } /// Boolean values -template ::value, detail::enabler> = detail::dummy> -bool lexical_cast(std::string input, T &output) { +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { try { auto out = to_flag_value(input); output = (out > 0); return true; } catch(const std::invalid_argument &) { return false; + } catch(const std::out_of_range &) { + // if the number is out of the range of a 64 bit value then it is still a number and for this purpose is still + // valid all we care about the sign + output = (input[0] != '-'); + return true; } } /// Floats -template ::value, detail::enabler> = detail::dummy> -bool lexical_cast(std::string input, T &output) { - try { - size_t n = 0; - output = static_cast(std::stold(input, &n)); - return n == input.size(); - } catch(const std::invalid_argument &) { - return false; - } catch(const std::out_of_range &) { +template ::value == object_category::floating_point, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(input.empty()) { return false; } + char *val = nullptr; + auto output_ld = std::strtold(input.c_str(), &val); + output = static_cast(output_ld); + return val == (input.c_str() + input.size()); } -/// String and similar +/// complex template ::value && !std::is_integral::value && - std::is_assignable::value, - detail::enabler> = detail::dummy> -bool lexical_cast(std::string input, T &output) { + enable_if_t::value == object_category::complex_number, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + using XC = typename wrapped_type::type; + XC x{0.0}, y{0.0}; + auto str1 = input; + bool worked = false; + auto nloc = str1.find_last_of("+-"); + if(nloc != std::string::npos && nloc > 0) { + worked = detail::lexical_cast(str1.substr(0, nloc), x); + str1 = str1.substr(nloc); + if(str1.back() == 'i' || str1.back() == 'j') + str1.pop_back(); + worked = worked && detail::lexical_cast(str1, y); + } else { + if(str1.back() == 'i' || str1.back() == 'j') { + str1.pop_back(); + worked = detail::lexical_cast(str1, y); + x = XC{0}; + } else { + worked = detail::lexical_cast(str1, x); + y = XC{0}; + } + } + if(worked) { + output = T{x, y}; + return worked; + } + return from_stream(input, output); +} + +/// String and similar direct assignment +template ::value == object_category::string_assignable, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { output = input; return true; } +/// String and similar constructible and copy assignment +template < + typename T, + enable_if_t::value == object_category::string_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = T(input); + return true; +} + /// Enumerations -template ::value, detail::enabler> = detail::dummy> -bool lexical_cast(std::string input, T &output) { +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { typename std::underlying_type::type val; - bool retval = detail::lexical_cast(input, val); - if(!retval) { + if(!integral_conversion(input, val)) { return false; } output = static_cast(val); return true; } -/// Non-string parsable +/// wrapper types template ::value && !std::is_integral::value && - !std::is_assignable::value && !std::is_enum::value, + enable_if_t::value == object_category::wrapper_value && + std::is_assignable::value, detail::enabler> = detail::dummy> -bool lexical_cast(std::string input, T &output) { - std::istringstream is; +bool lexical_cast(const std::string &input, T &output) { + typename T::value_type val; + if(lexical_cast(input, val)) { + output = val; + return true; + } + return from_stream(input, output); +} - is.str(input); - is >> output; - return !is.fail() && !is.rdbuf()->in_avail(); +template ::value == object_category::wrapper_value && + !std::is_assignable::value && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename T::value_type val; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Assignable from double or int +template < + typename T, + enable_if_t::value == object_category::number_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val; + if(integral_conversion(input, val)) { + output = T(val); + return true; + } else { + double dval; + if(lexical_cast(input, dval)) { + output = T{dval}; + return true; + } + } + return from_stream(input, output); +} + +/// Assignable from int +template < + typename T, + enable_if_t::value == object_category::integer_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val; + if(integral_conversion(input, val)) { + output = T(val); + return true; + } + return from_stream(input, output); +} + +/// Assignable from double +template < + typename T, + enable_if_t::value == object_category::double_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + double val; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Non-string convertible from an int +template ::value == object_category::other && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val; + if(integral_conversion(input, val)) { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4800) +#endif + // with Atomic this could produce a warning due to the conversion but if atomic gets here it is an old style + // so will most likely still work + output = val; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + return true; + } + // LCOV_EXCL_START + // This version of cast is only used for odd cases in an older compilers the fail over + // from_stream is tested elsewhere an not relevant for coverage here + return from_stream(input, output); + // LCOV_EXCL_STOP +} + +/// Non-string parsable by a stream +template ::value == object_category::other && !std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + static_assert(is_istreamable::value, + "option object type must have a lexical cast overload or streaming input operator(>>) defined, if it " + "is convertible from another type use the add_option(...) with XC being the known type"); + return from_stream(input, output); +} + +/// Assign a value through lexical cast operations +/// Strings can be empty so we need to do a little different +template ::value && + (classify_object::value == object_category::string_assignable || + classify_object::value == object_category::string_constructible), + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && std::is_assignable::value && + classify_object::value != object_category::string_assignable && + classify_object::value != object_category::string_constructible, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + output = AssignTo{}; + return true; + } + + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && !std::is_assignable::value && + classify_object::value == object_category::wrapper_value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + typename AssignTo::value_type emptyVal{}; + output = emptyVal; + return true; + } + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations for int compatible values +/// mainly for atomic operations on some compilers +template ::value && !std::is_assignable::value && + classify_object::value != object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + output = 0; + return true; + } + int val; + if(lexical_cast(input, val)) { + output = val; + return true; + } + return false; +} + +/// Assign a value converted from a string in lexical cast to the output value directly +template ::value && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + ConvertTo val{}; + bool parse_result = (!input.empty()) ? lexical_cast(input, val) : true; + if(parse_result) { + output = val; + } + return parse_result; +} + +/// Assign a value from a lexical cast through constructing a value and move assigning it +template < + typename AssignTo, + typename ConvertTo, + enable_if_t::value && !std::is_assignable::value && + std::is_move_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + ConvertTo val{}; + bool parse_result = input.empty() ? true : lexical_cast(input, val); + if(parse_result) { + output = AssignTo(val); // use () form of constructor to allow some implicit conversions + } + return parse_result; +} + +/// primary lexical conversion operation, 1 string to 1 type of some kind +template ::value <= object_category::other && + classify_object::value <= object_category::wrapper_value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + return lexical_assign(strings[0], output); +} + +/// Lexical conversion if there is only one element but the conversion type is for two, then call a two element +/// constructor +template ::value <= 2) && expected_count::value == 1 && + is_tuple_like::value && type_count_base::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + // the remove const is to handle pair types coming from a container + typename std::remove_const::type>::type v1; + typename std::tuple_element<1, ConvertTo>::type v2; + bool retval = lexical_assign(strings[0], v1); + if(strings.size() > 1) { + retval = retval && lexical_assign(strings[1], v2); + } + if(retval) { + output = AssignTo{v1, v2}; + } + return retval; +} + +/// Lexical conversion of a container types of single elements +template ::value && is_mutable_container::value && + type_count::value == 1, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + output.erase(output.begin(), output.end()); + for(const auto &elem : strings) { + typename AssignTo::value_type out; + bool retval = lexical_assign(elem, out); + if(!retval) { + return false; + } + output.insert(output.end(), std::move(out)); + } + return (!output.empty()); +} + +/// Lexical conversion for complex types +template ::value, detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + + if(strings.size() >= 2 && !strings[1].empty()) { + using XC2 = typename wrapped_type::type; + XC2 x{0.0}, y{0.0}; + auto str1 = strings[1]; + if(str1.back() == 'i' || str1.back() == 'j') { + str1.pop_back(); + } + auto worked = detail::lexical_cast(strings[0], x) && detail::lexical_cast(str1, y); + if(worked) { + output = ConvertTo{x, y}; + } + return worked; + } else { + return lexical_assign(strings[0], output); + } +} + +/// Conversion to a vector type using a particular single type as the conversion type +template ::value && (expected_count::value == 1) && + (type_count::value == 1), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + bool retval = true; + output.clear(); + output.reserve(strings.size()); + for(const auto &elem : strings) { + + output.emplace_back(); + retval = retval && lexical_assign(elem, output.back()); + } + return (!output.empty()) && retval; +} + +// forward declaration + +/// Lexical conversion of a container types with conversion type of two elements +template ::value && is_mutable_container::value && + type_count_base::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(std::vector strings, AssignTo &output); + +/// Lexical conversion of a vector types with type_size >2 forward declaration +template ::value && is_mutable_container::value && + type_count_base::value != 2 && + ((type_count::value > 2) || + (type_count::value > type_count_base::value)), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output); + +/// Conversion for tuples +template ::value && is_tuple_like::value && + (type_count_base::value != type_count::value || + type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output); // forward declaration + +/// Conversion for operations where the assigned type is some class but the conversion is a mutable container or large +/// tuple +template ::value && !is_mutable_container::value && + classify_object::value != object_category::wrapper_value && + (is_mutable_container::value || type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + + if(strings.size() > 1 || (!strings.empty() && !(strings.front().empty()))) { + ConvertTo val; + auto retval = lexical_conversion(strings, val); + output = AssignTo{val}; + return retval; + } + output = AssignTo{}; + return true; +} + +/// function template for converting tuples if the static Index is greater than the tuple size +template +inline typename std::enable_if<(I >= type_count_base::value), bool>::type +tuple_conversion(const std::vector &, AssignTo &) { + return true; +} + +/// Conversion of a tuple element where the type size ==1 and not a mutable container +template +inline typename std::enable_if::value && type_count::value == 1, bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + auto retval = lexical_assign(strings[0], output); + strings.erase(strings.begin()); + return retval; +} + +/// Conversion of a tuple element where the type size !=1 but the size is fixed and not a mutable container +template +inline typename std::enable_if::value && (type_count::value > 1) && + type_count::value == type_count_min::value, + bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + auto retval = lexical_conversion(strings, output); + strings.erase(strings.begin(), strings.begin() + type_count::value); + return retval; +} + +/// Conversion of a tuple element where the type is a mutable container or a type with different min and max type sizes +template +inline typename std::enable_if::value || + type_count::value != type_count_min::value, + bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + + std::size_t index{subtype_count_min::value}; + const std::size_t mx_count{subtype_count::value}; + const std::size_t mx{(std::max)(mx_count, strings.size())}; + + while(index < mx) { + if(is_separator(strings[index])) { + break; + } + ++index; + } + bool retval = lexical_conversion( + std::vector(strings.begin(), strings.begin() + static_cast(index)), output); + strings.erase(strings.begin(), strings.begin() + static_cast(index) + 1); + return retval; +} + +/// Tuple conversion operation +template +inline typename std::enable_if<(I < type_count_base::value), bool>::type +tuple_conversion(std::vector strings, AssignTo &output) { + bool retval = true; + using ConvertToElement = typename std:: + conditional::value, typename std::tuple_element::type, ConvertTo>::type; + if(!strings.empty()) { + retval = retval && tuple_type_conversion::type, ConvertToElement>( + strings, std::get(output)); + } + retval = retval && tuple_conversion(std::move(strings), output); + return retval; +} + +/// Lexical conversion of a container types with tuple elements of size 2 +template ::value && is_mutable_container::value && + type_count_base::value == 2, + detail::enabler>> +bool lexical_conversion(std::vector strings, AssignTo &output) { + output.clear(); + while(!strings.empty()) { + + typename std::remove_const::type>::type v1; + typename std::tuple_element<1, typename ConvertTo::value_type>::type v2; + bool retval = tuple_type_conversion(strings, v1); + if(!strings.empty()) { + retval = retval && tuple_type_conversion(strings, v2); + } + if(retval) { + output.insert(output.end(), typename AssignTo::value_type{v1, v2}); + } else { + return false; + } + } + return (!output.empty()); +} + +/// lexical conversion of tuples with type count>2 or tuples of types of some element with a type size>=2 +template ::value && is_tuple_like::value && + (type_count_base::value != type_count::value || + type_count::value > 2), + detail::enabler>> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + static_assert( + !is_tuple_like::value || type_count_base::value == type_count_base::value, + "if the conversion type is defined as a tuple it must be the same size as the type you are converting to"); + return tuple_conversion(strings, output); +} + +/// Lexical conversion of a vector types for everything but tuples of two elements and types of size 1 +template ::value && is_mutable_container::value && + type_count_base::value != 2 && + ((type_count::value > 2) || + (type_count::value > type_count_base::value)), + detail::enabler>> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + bool retval = true; + output.clear(); + std::vector temp; + std::size_t ii{0}; + std::size_t icount{0}; + std::size_t xcm{type_count::value}; + auto ii_max = strings.size(); + while(ii < ii_max) { + temp.push_back(strings[ii]); + ++ii; + ++icount; + if(icount == xcm || is_separator(temp.back()) || ii == ii_max) { + if(static_cast(xcm) > type_count_min::value && is_separator(temp.back())) { + temp.pop_back(); + } + typename AssignTo::value_type temp_out; + retval = retval && + lexical_conversion(temp, temp_out); + temp.clear(); + if(!retval) { + return false; + } + output.insert(output.end(), std::move(temp_out)); + icount = 0; + } + } + return retval; +} + +/// conversion for wrapper types +template ::value == object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + if(strings.empty() || strings.front().empty()) { + output = ConvertTo{}; + return true; + } + typename ConvertTo::value_type val; + if(lexical_conversion(strings, val)) { + output = ConvertTo{val}; + return true; + } + return false; +} + +/// conversion for wrapper types +template ::value == object_category::wrapper_value && + !std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + using ConvertType = typename ConvertTo::value_type; + if(strings.empty() || strings.front().empty()) { + output = ConvertType{}; + return true; + } + ConvertType val; + if(lexical_conversion(strings, val)) { + output = val; + return true; + } + return false; } /// Sum a vector of flag representations -/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is by +/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is +/// by /// "-1" an if numbers are passed by some fashion they are captured as well so the function just checks for the most /// common true and false strings then uses stoll to convert the rest for summing -template ::value && std::is_unsigned::value, detail::enabler> = detail::dummy> +template ::value, detail::enabler> = detail::dummy> void sum_flag_vector(const std::vector &flags, T &output) { - int64_t count{0}; + std::int64_t count{0}; for(auto &flag : flags) { count += detail::to_flag_value(flag); } @@ -1257,25 +2346,50 @@ void sum_flag_vector(const std::vector &flags, T &output) { } /// Sum a vector of flag representations -/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is by +/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is +/// by /// "-1" an if numbers are passed by some fashion they are captured as well so the function just checks for the most /// common true and false strings then uses stoll to convert the rest for summing -template ::value && std::is_signed::value, detail::enabler> = detail::dummy> +template ::value, detail::enabler> = detail::dummy> void sum_flag_vector(const std::vector &flags, T &output) { - int64_t count{0}; + std::int64_t count{0}; for(auto &flag : flags) { count += detail::to_flag_value(flag); } output = static_cast(count); } -} // namespace detail -} // namespace CLI +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4800) +#endif +// with Atomic this could produce a warning due to the conversion but if atomic gets here it is an old style so will +// most likely still work + +/// Sum a vector of flag representations +/// The flag vector produces a series of strings in a vector, simple true is represented by a "1", simple false is +/// by +/// "-1" an if numbers are passed by some fashion they are captured as well so the function just checks for the most +/// common true and false strings then uses stoll to convert the rest for summing +template ::value && !std::is_unsigned::value, detail::enabler> = detail::dummy> +void sum_flag_vector(const std::vector &flags, T &output) { + std::int64_t count{0}; + for(auto &flag : flags) { + count += detail::to_flag_value(flag); + } + std::string out = detail::to_string(count); + lexical_cast(out, output); +} + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +} // namespace detail + -// From CLI/Split.hpp: -namespace CLI { namespace detail { // Returns false if not a short option. Otherwise, sets opt name and rest and returns true @@ -1284,8 +2398,8 @@ inline bool split_short(const std::string ¤t, std::string &name, std::stri name = current.substr(1, 1); rest = current.substr(2); return true; - } else - return false; + } + return false; } // Returns false if not a long option. Otherwise, sets opt name and other side of = and returns true @@ -1300,8 +2414,8 @@ inline bool split_long(const std::string ¤t, std::string &name, std::strin value = ""; } return true; - } else - return false; + } + return false; } // Returns false if not a windows style option. Otherwise, sets opt name and value and returns true @@ -1316,14 +2430,14 @@ inline bool split_windows_style(const std::string ¤t, std::string &name, s value = ""; } return true; - } else - return false; + } + return false; } // Splits a string into multiple long and short names inline std::vector split_names(std::string current) { std::vector output; - size_t val; + std::size_t val; while((val = current.find(",")) != std::string::npos) { output.push_back(trim_copy(current.substr(0, val))); current = current.substr(val + 1); @@ -1368,9 +2482,10 @@ get_names(const std::vector &input) { std::string pos_name; for(std::string name : input) { - if(name.length() == 0) + if(name.length() == 0) { continue; - else if(name.length() > 1 && name[0] == '-' && name[1] != '-') { + } + if(name.length() > 1 && name[0] == '-' && name[1] != '-') { if(name.length() == 2 && valid_first_char(name[1])) short_names.emplace_back(1, name[1]); else @@ -1394,49 +2509,22 @@ get_names(const std::vector &input) { short_names, long_names, pos_name); } -} // namespace detail -} // namespace CLI +} // namespace detail -// From CLI/ConfigFwd.hpp: -namespace CLI { class App; -namespace detail { - -/// Comma separated join, adds quotes if needed -inline std::string ini_join(std::vector args) { - std::ostringstream s; - size_t start = 0; - for(const auto &arg : args) { - if(start++ > 0) - s << " "; - - auto it = std::find_if(arg.begin(), arg.end(), [](char ch) { return std::isspace(ch, std::locale()); }); - if(it == arg.end()) - s << arg; - else if(arg.find_first_of('\"') == std::string::npos) - s << '\"' << arg << '\"'; - else - s << '\'' << arg << '\''; - } - - return s.str(); -} - -} // namespace detail - /// Holds values to load into Options struct ConfigItem { /// This is the list of parents - std::vector parents; + std::vector parents{}; /// This is the name - std::string name; + std::string name{}; /// Listing of inputs - std::vector inputs; + std::vector inputs{}; /// The list of parents and name joined by "." std::string fullname() const { @@ -1449,7 +2537,7 @@ struct ConfigItem { /// This class provides a converter for configuration files. class Config { protected: - std::vector items; + std::vector items{}; public: /// Convert an app into a configuration @@ -1479,62 +2567,75 @@ class Config { virtual ~Config() = default; }; -/// This converter works with INI files -class ConfigINI : public Config { +/// This converter works with INI/TOML files; to write INI files use ConfigINI +class ConfigBase : public Config { + protected: + /// the character used for comments + char commentChar = '#'; + /// the character used to start an array '\0' is a default to not use + char arrayStart = '['; + /// the character used to end an array '\0' is a default to not use + char arrayEnd = ']'; + /// the character used to separate elements in an array + char arraySeparator = ','; + /// the character used separate the name from the value + char valueDelimiter = '='; + /// the character to use around strings + char stringQuote = '"'; + /// the character to use around single characters + char characterQuote = '\''; + public: - std::string to_config(const App *, bool default_also, bool write_description, std::string prefix) const override; + std::string + to_config(const App * /*app*/, bool default_also, bool write_description, std::string prefix) const override; - std::vector from_config(std::istream &input) const override { - std::string line; - std::string section = "default"; - - std::vector output; - - while(getline(input, line)) { - std::vector items_buffer; - - detail::trim(line); - size_t len = line.length(); - if(len > 1 && line[0] == '[' && line[len - 1] == ']') { - section = line.substr(1, len - 2); - } else if(len > 0 && line[0] != ';') { - output.emplace_back(); - ConfigItem &out = output.back(); - - // Find = in string, split and recombine - auto pos = line.find('='); - if(pos != std::string::npos) { - out.name = detail::trim_copy(line.substr(0, pos)); - std::string item = detail::trim_copy(line.substr(pos + 1)); - items_buffer = detail::split_up(item); - } else { - out.name = detail::trim_copy(line); - items_buffer = {"ON"}; - } - - if(detail::to_lower(section) != "default") { - out.parents = {section}; - } - - if(out.name.find('.') != std::string::npos) { - std::vector plist = detail::split(out.name, '.'); - out.name = plist.back(); - plist.pop_back(); - out.parents.insert(out.parents.end(), plist.begin(), plist.end()); - } - - out.inputs.insert(std::end(out.inputs), std::begin(items_buffer), std::end(items_buffer)); - } - } - return output; + std::vector from_config(std::istream &input) const override; + /// Specify the configuration for comment characters + ConfigBase *comment(char cchar) { + commentChar = cchar; + return this; + } + /// Specify the start and end characters for an array + ConfigBase *arrayBounds(char aStart, char aEnd) { + arrayStart = aStart; + arrayEnd = aEnd; + return this; + } + /// Specify the delimiter character for an array + ConfigBase *arrayDelimiter(char aSep) { + arraySeparator = aSep; + return this; + } + /// Specify the delimiter between a name and value + ConfigBase *valueSeparator(char vSep) { + valueDelimiter = vSep; + return this; + } + /// Specify the quote characters used around strings and characters + ConfigBase *quoteCharacter(char qString, char qChar) { + stringQuote = qString; + characterQuote = qChar; + return this; } }; -} // namespace CLI +/// the default Config is the TOML file format +using ConfigTOML = ConfigBase; + +/// ConfigINI generates a "standard" INI compliant output +class ConfigINI : public ConfigTOML { + + public: + ConfigINI() { + commentChar = ';'; + arrayStart = '\0'; + arrayEnd = '\0'; + arraySeparator = ' '; + valueDelimiter = '='; + } +}; -// From CLI/Validators.hpp: -namespace CLI { class Option; @@ -1553,11 +2654,13 @@ class Validator { /// This is the description function, if empty the description_ will be used std::function desc_function_{[]() { return std::string{}; }}; - /// This it the base function that is to be called. + /// This is the base function that is to be called. /// Returns a string error message if validation fails. std::function func_{[](std::string &) { return std::string{}; }}; /// The name for search purposes of the Validator - std::string name_; + std::string name_{}; + /// A Validator will only apply to an indexed value (-1 is all elements) + int application_index_ = -1; /// Enable for Validator to allow it to be disabled if need be bool active_{true}; /// specify that a validator should not modify the input @@ -1567,7 +2670,7 @@ class Validator { Validator() = default; /// Construct a Validator with just the description string explicit Validator(std::string validator_desc) : desc_function_([validator_desc]() { return validator_desc; }) {} - // Construct Validator from basic information + /// Construct Validator from basic information Validator(std::function op, std::string validator_desc, std::string validator_name = "") : desc_function_([validator_desc]() { return validator_desc; }), func_(std::move(op)), name_(std::move(validator_name)) {} @@ -1589,20 +2692,26 @@ class Validator { } } return retstring; - }; + } /// This is the required operator for a Validator - provided to help /// users (CLI11 uses the member `func` directly) std::string operator()(const std::string &str) const { std::string value = str; return (active_) ? func_(value) : std::string{}; - }; + } /// Specify the type string Validator &description(std::string validator_desc) { desc_function_ = [validator_desc]() { return validator_desc; }; return *this; } + /// Specify the type string + Validator description(std::string validator_desc) const { + Validator newval(*this); + newval.desc_function_ = [validator_desc]() { return validator_desc; }; + return newval; + } /// Generate type description information for the Validator std::string get_description() const { if(active_) { @@ -1615,6 +2724,12 @@ class Validator { name_ = std::move(validator_name); return *this; } + /// Specify the type string + Validator name(std::string validator_name) const { + Validator newval(*this); + newval.name_ = std::move(validator_name); + return newval; + } /// Get the name of the Validator const std::string &get_name() const { return name_; } /// Specify whether the Validator is active or not @@ -1622,13 +2737,31 @@ class Validator { active_ = active_val; return *this; } + /// Specify whether the Validator is active or not + Validator active(bool active_val = true) const { + Validator newval(*this); + newval.active_ = active_val; + return newval; + } /// Specify whether the Validator can be modifying or not Validator &non_modifying(bool no_modify = true) { non_modifying_ = no_modify; return *this; } - + /// Specify the application index of a validator + Validator &application_index(int app_index) { + application_index_ = app_index; + return *this; + } + /// Specify the application index of a validator + Validator application_index(int app_index) const { + Validator newval(*this); + newval.application_index_ = app_index; + return newval; + } + /// Get the current value of the application index + int get_application_index() const { return application_index_; } /// Get a boolean if the validator is active bool get_active() const { return active_; } @@ -1656,6 +2789,7 @@ class Validator { }; newval.active_ = (active_ & other.active_); + newval.application_index_ = application_index_; return newval; } @@ -1675,10 +2809,11 @@ class Validator { std::string s2 = f2(input); if(s1.empty() || s2.empty()) return std::string(); - else - return std::string("(") + s1 + ") OR (" + s2 + ")"; + + return std::string("(") + s1 + ") OR (" + s2 + ")"; }; newval.active_ = (active_ & other.active_); + newval.application_index_ = application_index_; return newval; } @@ -1697,10 +2832,11 @@ class Validator { std::string s1 = f1(test); if(s1.empty()) { return std::string("check ") + dfunc1() + " succeeded improperly"; - } else - return std::string{}; + } + return std::string{}; }; newval.active_ = active_; + newval.application_index_ = application_index_; return newval; } @@ -1716,10 +2852,10 @@ class Validator { if((f1.empty()) || (f2.empty())) { return f1 + f2; } - return std::string("(") + f1 + ")" + merger + "(" + f2 + ")"; + return std::string(1, '(') + f1 + ')' + merger + '(' + f2 + ')'; }; } -}; +}; // namespace CLI /// Class wrapping some of the accessors of Validator class CustomValidator : public Validator { @@ -1730,17 +2866,61 @@ class CustomValidator : public Validator { // Therefore, this is in detail. namespace detail { +/// CLI enumeration of different file types +enum class path_type { nonexistent, file, directory }; + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +/// get the type of the path from a file name +inline path_type check_path(const char *file) noexcept { + std::error_code ec; + auto stat = std::filesystem::status(file, ec); + if(ec) { + return path_type::nonexistent; + } + switch(stat.type()) { + case std::filesystem::file_type::none: + case std::filesystem::file_type::not_found: + return path_type::nonexistent; + case std::filesystem::file_type::directory: + return path_type::directory; + case std::filesystem::file_type::symlink: + case std::filesystem::file_type::block: + case std::filesystem::file_type::character: + case std::filesystem::file_type::fifo: + case std::filesystem::file_type::socket: + case std::filesystem::file_type::regular: + case std::filesystem::file_type::unknown: + default: + return path_type::file; + } +} +#else +/// get the type of the path from a file name +inline path_type check_path(const char *file) noexcept { +#if defined(_MSC_VER) + struct __stat64 buffer; + if(_stat64(file, &buffer) == 0) { + return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; + } +#else + struct stat buffer; + if(stat(file, &buffer) == 0) { + return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; + } +#endif + return path_type::nonexistent; +} +#endif /// Check for an existing file (returns error message if check fails) class ExistingFileValidator : public Validator { public: ExistingFileValidator() : Validator("FILE") { func_ = [](std::string &filename) { - struct stat buffer; - bool exist = stat(filename.c_str(), &buffer) == 0; - bool is_dir = (buffer.st_mode & S_IFDIR) != 0; - if(!exist) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { return "File does not exist: " + filename; - } else if(is_dir) { + } + if(path_result == path_type::directory) { return "File is actually a directory: " + filename; } return std::string(); @@ -1753,12 +2933,11 @@ class ExistingDirectoryValidator : public Validator { public: ExistingDirectoryValidator() : Validator("DIR") { func_ = [](std::string &filename) { - struct stat buffer; - bool exist = stat(filename.c_str(), &buffer) == 0; - bool is_dir = (buffer.st_mode & S_IFDIR) != 0; - if(!exist) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { return "Directory does not exist: " + filename; - } else if(!is_dir) { + } + if(path_result == path_type::file) { return "Directory is actually a file: " + filename; } return std::string(); @@ -1771,9 +2950,8 @@ class ExistingPathValidator : public Validator { public: ExistingPathValidator() : Validator("PATH(existing)") { func_ = [](std::string &filename) { - struct stat buffer; - bool const exist = stat(filename.c_str(), &buffer) == 0; - if(!exist) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { return "Path does not exist: " + filename; } return std::string(); @@ -1786,9 +2964,8 @@ class NonexistentPathValidator : public Validator { public: NonexistentPathValidator() : Validator("PATH(non-existing)") { func_ = [](std::string &filename) { - struct stat buffer; - bool exist = stat(filename.c_str(), &buffer) == 0; - if(exist) { + auto path_result = check_path(filename.c_str()); + if(path_result != path_type::nonexistent) { return "Path already exists: " + filename; } return std::string(); @@ -1803,17 +2980,16 @@ class IPV4Validator : public Validator { func_ = [](std::string &ip_addr) { auto result = CLI::detail::split(ip_addr, '.'); if(result.size() != 4) { - return "Invalid IPV4 address must have four parts " + ip_addr; + return std::string("Invalid IPV4 address must have four parts (") + ip_addr + ')'; } int num; - bool retval = true; for(const auto &var : result) { - retval &= detail::lexical_cast(var, num); + bool retval = detail::lexical_cast(var, num); if(!retval) { - return "Failed parsing number " + var; + return std::string("Failed parsing number (") + var + ')'; } if(num < 0 || num > 255) { - return "Each IP number must be between 0 and 255 " + var; + return std::string("Each IP number must be between 0 and 255 ") + var; } } return std::string(); @@ -1821,38 +2997,7 @@ class IPV4Validator : public Validator { } }; -/// Validate the argument is a number and greater than or equal to 0 -class PositiveNumber : public Validator { - public: - PositiveNumber() : Validator("POSITIVE") { - func_ = [](std::string &number_str) { - int number; - if(!detail::lexical_cast(number_str, number)) { - return "Failed parsing number " + number_str; - } - if(number < 0) { - return "Number less then 0 " + number_str; - } - return std::string(); - }; - } -}; - -/// Validate the argument is a number and greater than or equal to 0 -class Number : public Validator { - public: - Number() : Validator("NUMBER") { - func_ = [](std::string &number_str) { - double number; - if(!detail::lexical_cast(number_str, number)) { - return "Failed parsing as a number " + number_str; - } - return std::string(); - }; - } -}; - -} // namespace detail +} // namespace detail // Static is not needed here, because global const implies static. @@ -1871,11 +3016,23 @@ const detail::NonexistentPathValidator NonexistentPath; /// Check for an IP4 address const detail::IPV4Validator ValidIPV4; -/// Check for a positive number -const detail::PositiveNumber PositiveNumber; +/// Validate the input as a particular type +template class TypeValidator : public Validator { + public: + explicit TypeValidator(const std::string &validator_name) : Validator(validator_name) { + func_ = [](std::string &input_string) { + auto val = DesiredType(); + if(!detail::lexical_cast(input_string, val)) { + return std::string("Failed parsing ") + input_string + " as a " + detail::type_name(); + } + return std::string(); + }; + } + TypeValidator() : TypeValidator(detail::type_name()) {} +}; /// Check for a number -const detail::Number Number; +const TypeValidator Number("NUMBER"); /// Produce a range (factory). Min and max are inclusive. class Range : public Validator { @@ -1884,25 +3041,37 @@ class Range : public Validator { /// /// Note that the constructor is templated, but the struct is not, so C++17 is not /// needed to provide nice syntax for Range(a,b). - template Range(T min, T max) { - std::stringstream out; - out << detail::type_name() << " in [" << min << " - " << max << "]"; - description(out.str()); + template + Range(T min, T max, const std::string &validator_name = std::string{}) : Validator(validator_name) { + if(validator_name.empty()) { + std::stringstream out; + out << detail::type_name() << " in [" << min << " - " << max << "]"; + description(out.str()); + } func_ = [min, max](std::string &input) { T val; bool converted = detail::lexical_cast(input, val); if((!converted) || (val < min || val > max)) - return "Value " + input + " not in range " + std::to_string(min) + " to " + std::to_string(max); + return std::string("Value ") + input + " not in range " + std::to_string(min) + " to " + + std::to_string(max); return std::string(); }; } /// Range of one value is 0 to value - template explicit Range(T max) : Range(static_cast(0), max) {} + template + explicit Range(T max, const std::string &validator_name = std::string{}) + : Range(static_cast(0), max, validator_name) {} }; +/// Check for a non negative number +const Range NonNegativeNumber(std::numeric_limits::max(), "NONNEGATIVE"); + +/// Check for a positive valued number (val>0.0), min() her is the smallest positive number +const Range PositiveNumber(std::numeric_limits::min(), std::numeric_limits::max(), "POSITIVE"); + /// Produce a bounded range (factory). Min and max are inclusive. class Bound : public Validator { public: @@ -1919,14 +3088,14 @@ class Bound : public Validator { T val; bool converted = detail::lexical_cast(input, val); if(!converted) { - return "Value " + input + " could not be converted"; + return std::string("Value ") + input + " could not be converted"; } if(val < min) - input = detail::as_string(min); + input = detail::to_string(min); else if(val > max) - input = detail::as_string(max); + input = detail::to_string(max); - return std::string(); + return std::string{}; }; } @@ -1950,11 +3119,12 @@ typename std::remove_reference::type &smart_deref(T &value) { /// Generate a string representation of a set template std::string generate_set(const T &set) { using element_t = typename detail::element_type::type; - using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair std::string out(1, '{'); - out.append(detail::join(detail::smart_deref(set), - [](const iteration_type_t &v) { return detail::pair_adaptor::first(v); }, - ",")); + out.append(detail::join( + detail::smart_deref(set), + [](const iteration_type_t &v) { return detail::pair_adaptor::first(v); }, + ",")); out.push_back('}'); return out; } @@ -1962,29 +3132,32 @@ template std::string generate_set(const T &set) { /// Generate a string representation of a map template std::string generate_map(const T &map, bool key_only = false) { using element_t = typename detail::element_type::type; - using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair std::string out(1, '{'); - out.append(detail::join(detail::smart_deref(map), - [key_only](const iteration_type_t &v) { - auto res = detail::as_string(detail::pair_adaptor::first(v)); - if(!key_only) { - res += "->" + detail::as_string(detail::pair_adaptor::second(v)); - } - return res; - }, - ",")); + out.append(detail::join( + detail::smart_deref(map), + [key_only](const iteration_type_t &v) { + std::string res{detail::to_string(detail::pair_adaptor::first(v))}; + + if(!key_only) { + res.append("->"); + res += detail::to_string(detail::pair_adaptor::second(v)); + } + return res; + }, + ",")); out.push_back('}'); return out; } -template struct sfinae_true : std::true_type {}; -/// Function to check for the existence of a member find function which presumably is more efficient than looping over -/// everything -template -static auto test_find(int) -> sfinae_true().find(std::declval()))>; -template static auto test_find(long) -> std::false_type; +template struct has_find { + template + static auto test(int) -> decltype(std::declval().find(std::declval()), std::true_type()); + template static auto test(...) -> decltype(std::false_type()); -template struct has_find : decltype(test_find(0)) {}; + static const auto value = decltype(test(0))::value; + using type = std::integral_constant; +}; /// A search function template ::value, detail::enabler> = detail::dummy> @@ -2018,24 +3191,44 @@ auto search(const T &set, const V &val, const std::function &filter_functi // if we haven't found it do the longer linear search with all the element translations auto &setref = detail::smart_deref(set); auto it = std::find_if(std::begin(setref), std::end(setref), [&](decltype(*std::begin(setref)) v) { - V a = detail::pair_adaptor::first(v); + V a{detail::pair_adaptor::first(v)}; a = filter_function(a); return (a == val); }); return {(it != std::end(setref)), it}; } +// the following suggestion was made by Nikita Ofitserov(@himikof) +// done in templates to prevent compiler warnings on negation of unsigned numbers + +/// Do a check for overflow on signed numbers +template +inline typename std::enable_if::value, T>::type overflowCheck(const T &a, const T &b) { + if((a > 0) == (b > 0)) { + return ((std::numeric_limits::max)() / (std::abs)(a) < (std::abs)(b)); + } else { + return ((std::numeric_limits::min)() / (std::abs)(a) > -(std::abs)(b)); + } +} +/// Do a check for overflow on unsigned numbers +template +inline typename std::enable_if::value, T>::type overflowCheck(const T &a, const T &b) { + return ((std::numeric_limits::max)() / a < b); +} + /// Performs a *= b; if it doesn't cause integer overflow. Returns false otherwise. template typename std::enable_if::value, bool>::type checked_multiply(T &a, T b) { - if(a == 0 || b == 0) { + if(a == 0 || b == 0 || a == 1 || b == 1) { a *= b; return true; } - T c = a * b; - if(c / a != b) { + if(a == (std::numeric_limits::min)() || b == (std::numeric_limits::min)()) { return false; } - a = c; + if(overflowCheck(a, b)) { + return false; + } + a *= b; return true; } @@ -2050,7 +3243,7 @@ typename std::enable_if::value, bool>::type checked_mu return true; } -} // namespace detail +} // namespace detail /// Verify items are in a set class IsMember : public Validator { public: @@ -2058,7 +3251,7 @@ class IsMember : public Validator { /// This allows in-place construction using an initializer list template - explicit IsMember(std::initializer_list values, Args &&... args) + IsMember(std::initializer_list values, Args &&... args) : IsMember(std::vector(values), std::forward(args)...) {} /// This checks to see if an item is in a set (empty function) @@ -2070,11 +3263,11 @@ class IsMember : public Validator { // Get the type of the contained item - requires a container have ::value_type // if the type does not have first_type and second_type, these are both value_type - using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed - using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map - using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones - // (const char * to std::string) + using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones + // (const char * to std::string) // Make a local copy of the filter function, using a std::function if not one already std::function filter_fn = filter_function; @@ -2087,7 +3280,7 @@ class IsMember : public Validator { func_ = [set, filter_fn](std::string &input) { local_item_t b; if(!detail::lexical_cast(input, b)) { - throw ValidationError(input); // name is added later + throw ValidationError(input); // name is added later } if(filter_fn) { b = filter_fn(b); @@ -2096,7 +3289,7 @@ class IsMember : public Validator { if(res.first) { // Make sure the version in the input string is identical to the one in the set if(filter_fn) { - input = detail::as_string(detail::pair_adaptor::first(*(res.second))); + input = detail::value_string(detail::pair_adaptor::first(*(res.second))); } // Return empty error string (success) @@ -2104,18 +3297,17 @@ class IsMember : public Validator { } // If you reach this point, the result was not found - std::string out(" not in "); - out += detail::generate_set(detail::smart_deref(set)); - return out; + return input + " not in " + detail::generate_set(detail::smart_deref(set)); }; } /// You can pass in as many filter functions as you like, they nest (string only currently) template IsMember(T &&set, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&... other) - : IsMember(std::forward(set), - [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, - other...) {} + : IsMember( + std::forward(set), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} }; /// definition of the default transformation object @@ -2128,7 +3320,7 @@ class Transformer : public Validator { /// This allows in-place construction template - explicit Transformer(std::initializer_list> values, Args &&... args) + Transformer(std::initializer_list> values, Args &&... args) : Transformer(TransformPairs(values), std::forward(args)...) {} /// direct map of std::string to std::string @@ -2142,10 +3334,10 @@ class Transformer : public Validator { "mapping must produce value pairs"); // Get the type of the contained item - requires a container have ::value_type // if the type does not have first_type and second_type, these are both value_type - using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed - using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map - using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones - // (const char * to std::string) + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) // Make a local copy of the filter function, using a std::function if not one already std::function filter_fn = filter_function; @@ -2164,7 +3356,7 @@ class Transformer : public Validator { } auto res = detail::search(mapping, b, filter_fn); if(res.first) { - input = detail::as_string(detail::pair_adaptor::second(*res.second)); + input = detail::value_string(detail::pair_adaptor::second(*res.second)); } return std::string{}; }; @@ -2173,9 +3365,10 @@ class Transformer : public Validator { /// You can pass in as many filter functions as you like, they nest template Transformer(T &&mapping, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&... other) - : Transformer(std::forward(mapping), - [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, - other...) {} + : Transformer( + std::forward(mapping), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} }; /// translate named items to other or a value set @@ -2185,7 +3378,7 @@ class CheckedTransformer : public Validator { /// This allows in-place construction template - explicit CheckedTransformer(std::initializer_list> values, Args &&... args) + CheckedTransformer(std::initializer_list> values, Args &&... args) : CheckedTransformer(TransformPairs(values), std::forward(args)...) {} /// direct map of std::string to std::string @@ -2199,12 +3392,11 @@ class CheckedTransformer : public Validator { "mapping must produce value pairs"); // Get the type of the contained item - requires a container have ::value_type // if the type does not have first_type and second_type, these are both value_type - using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed - using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map - using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones - // (const char * to std::string) - using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair // - // the type of the object pair + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair // Make a local copy of the filter function, using a std::function if not one already std::function filter_fn = filter_function; @@ -2214,7 +3406,7 @@ class CheckedTransformer : public Validator { out += detail::generate_map(detail::smart_deref(mapping)) + " OR {"; out += detail::join( detail::smart_deref(mapping), - [](const iteration_type_t &v) { return detail::as_string(detail::pair_adaptor::second(v)); }, + [](const iteration_type_t &v) { return detail::to_string(detail::pair_adaptor::second(v)); }, ","); out.push_back('}'); return out; @@ -2231,12 +3423,12 @@ class CheckedTransformer : public Validator { } auto res = detail::search(mapping, b, filter_fn); if(res.first) { - input = detail::as_string(detail::pair_adaptor::second(*res.second)); + input = detail::value_string(detail::pair_adaptor::second(*res.second)); return std::string{}; } } for(const auto &v : detail::smart_deref(mapping)) { - auto output_string = detail::as_string(detail::pair_adaptor::second(v)); + auto output_string = detail::value_string(detail::pair_adaptor::second(v)); if(output_string == input) { return std::string(); } @@ -2249,9 +3441,10 @@ class CheckedTransformer : public Validator { /// You can pass in as many filter functions as you like, they nest template CheckedTransformer(T &&mapping, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&... other) - : CheckedTransformer(std::forward(mapping), - [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, - other...) {} + : CheckedTransformer( + std::forward(mapping), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} }; /// Helper function to allow ignore_case to be passed to IsMember or Transform @@ -2315,7 +3508,7 @@ class AsNumberWithUnit : public Validator { } std::string unit{unit_begin, input.end()}; - input.resize(static_cast(std::distance(input.begin(), unit_begin))); + input.resize(static_cast(std::distance(input.begin(), unit_begin))); detail::trim(input); if(opts & UNIT_REQUIRED && unit.empty()) { @@ -2324,13 +3517,11 @@ class AsNumberWithUnit : public Validator { if(opts & CASE_INSENSITIVE) { unit = detail::to_lower(unit); } - - bool converted = detail::lexical_cast(input, num); - if(!converted) { - throw ValidationError("Value " + input + " could not be converted to " + detail::type_name()); - } - if(unit.empty()) { + if(!detail::lexical_cast(input, num)) { + throw ValidationError(std::string("Value ") + input + " could not be converted to " + + detail::type_name()); + } // No need to modify input if no unit passed return {}; } @@ -2344,13 +3535,23 @@ class AsNumberWithUnit : public Validator { detail::generate_map(mapping, true)); } - // perform safe multiplication - bool ok = detail::checked_multiply(num, it->second); - if(!ok) { - throw ValidationError(detail::as_string(num) + " multiplied by " + unit + - " factor would cause number overflow. Use smaller value."); + if(!input.empty()) { + bool converted = detail::lexical_cast(input, num); + if(!converted) { + throw ValidationError(std::string("Value ") + input + " could not be converted to " + + detail::type_name()); + } + // perform safe multiplication + bool ok = detail::checked_multiply(num, it->second); + if(!ok) { + throw ValidationError(detail::to_string(num) + " multiplied by " + unit + + " factor would cause number overflow. Use smaller value."); + } + } else { + num = static_cast(it->second); } - input = detail::as_string(num); + + input = detail::to_string(num); return {}; }; @@ -2375,7 +3576,8 @@ class AsNumberWithUnit : public Validator { for(auto &kv : mapping) { auto s = detail::to_lower(kv.first); if(lower_mapping.count(s)) { - throw ValidationError("Several matching lowercase unit representations are found: " + s); + throw ValidationError(std::string("Several matching lowercase unit representations are found: ") + + s); } lower_mapping[detail::to_lower(kv.first)] = kv.second; } @@ -2409,7 +3611,7 @@ class AsNumberWithUnit : public Validator { /// "2 EiB" => 2^61 // Units up to exibyte are supported class AsSizeValue : public AsNumberWithUnit { public: - using result_t = uint64_t; + using result_t = std::uint64_t; /// If kb_is_1000 is true, /// interpret 'kb', 'k' as 1000 and 'kib', 'ki' as 1024 @@ -2468,31 +3670,52 @@ inline std::pair split_program_name(std::string comman std::pair vals; trim(commandline); auto esp = commandline.find_first_of(' ', 1); - while(!ExistingFile(commandline.substr(0, esp)).empty()) { + while(detail::check_path(commandline.substr(0, esp).c_str()) != path_type::file) { esp = commandline.find_first_of(' ', esp + 1); if(esp == std::string::npos) { // if we have reached the end and haven't found a valid file just assume the first argument is the // program name - esp = commandline.find_first_of(' ', 1); + if(commandline[0] == '"' || commandline[0] == '\'' || commandline[0] == '`') { + bool embeddedQuote = false; + auto keyChar = commandline[0]; + auto end = commandline.find_first_of(keyChar, 1); + while((end != std::string::npos) && (commandline[end - 1] == '\\')) { // deal with escaped quotes + end = commandline.find_first_of(keyChar, end + 1); + embeddedQuote = true; + } + if(end != std::string::npos) { + vals.first = commandline.substr(1, end - 1); + esp = end + 1; + if(embeddedQuote) { + vals.first = find_and_replace(vals.first, std::string("\\") + keyChar, std::string(1, keyChar)); + embeddedQuote = false; + } + } else { + esp = commandline.find_first_of(' ', 1); + } + } else { + esp = commandline.find_first_of(' ', 1); + } + break; } } - vals.first = commandline.substr(0, esp); - rtrim(vals.first); + if(vals.first.empty()) { + vals.first = commandline.substr(0, esp); + rtrim(vals.first); + } + // strip the program name vals.second = (esp != std::string::npos) ? commandline.substr(esp + 1) : std::string{}; ltrim(vals.second); return vals; } -} // namespace detail +} // namespace detail /// @} -} // namespace CLI -// From CLI/FormatterFwd.hpp: -namespace CLI { class Option; class App; @@ -2503,9 +3726,9 @@ class App; /// the second argument. enum class AppFormatMode { - Normal, //< The normal, detailed help - All, //< A fully expanded help - Sub, //< Used when printed as part of expanded subcommand + Normal, ///< The normal, detailed help + All, ///< A fully expanded help + Sub, ///< Used when printed as part of expanded subcommand }; /// This is the minimum requirements to run a formatter. @@ -2518,11 +3741,11 @@ class FormatterBase { ///@{ /// The width of the first column - size_t column_width_{30}; + std::size_t column_width_{30}; /// @brief The required help printout labels (user changeable) /// Values are Needs, Excludes, etc. - std::map labels_; + std::map labels_{}; ///@} /// @name Basic @@ -2534,7 +3757,7 @@ class FormatterBase { FormatterBase(FormatterBase &&) = default; /// Adding a destructor in this form to work around bug in GCC 4.7 - virtual ~FormatterBase() noexcept {} // NOLINT(modernize-use-equals-default) + virtual ~FormatterBase() noexcept {} // NOLINT(modernize-use-equals-default) /// This is the key method that puts together help virtual std::string make_help(const App *, std::string, AppFormatMode) const = 0; @@ -2547,7 +3770,7 @@ class FormatterBase { void label(std::string key, std::string val) { labels_[key] = val; } /// Set the column width - void column_width(size_t val) { column_width_ = val; } + void column_width(std::size_t val) { column_width_ = val; } ///@} /// @name Getters @@ -2562,7 +3785,7 @@ class FormatterBase { } /// Get the current column width - size_t get_column_width() const { return column_width_; } + std::size_t get_column_width() const { return column_width_; } ///@} }; @@ -2579,7 +3802,7 @@ class FormatterLambda final : public FormatterBase { explicit FormatterLambda(funct_t funct) : lambda_(std::move(funct)) {} /// Adding a destructor (mostly to make GCC 4.7 happy) - ~FormatterLambda() noexcept override {} // NOLINT(modernize-use-equals-default) + ~FormatterLambda() noexcept override {} // NOLINT(modernize-use-equals-default) /// This will simply call the lambda function std::string make_help(const App *app, std::string name, AppFormatMode mode) const override { @@ -2627,7 +3850,7 @@ class Formatter : public FormatterBase { virtual std::string make_usage(const App *app, std::string name) const; /// This puts everything together - std::string make_help(const App *, std::string, AppFormatMode) const override; + std::string make_help(const App * /*app*/, std::string, AppFormatMode) const override; ///@} /// @name Options @@ -2656,21 +3879,25 @@ class Formatter : public FormatterBase { ///@} }; -} // namespace CLI -// From CLI/Option.hpp: -namespace CLI { using results_t = std::vector; -using callback_t = std::function; +/// callback function definition +using callback_t = std::function; class Option; class App; using Option_p = std::unique_ptr