From f60485d9c2e1d2408d152ae468a03bf69673f252 Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Wed, 11 Nov 2020 11:11:34 -0800 Subject: [PATCH] use ctest for testing --- .github/workflows/unittest_linux.yml | 5 +- .github/workflows/unittest_linux_asan.yml | 14 ++ .../workflows/unittest_mac_tsan_mbedtls.yml | 1 + .../workflows/unittest_mac_tsan_openssl.yml | 1 + .../unittest_mac_tsan_sectransport.yml | 5 +- CMakeLists.txt | 5 +- README.md | 16 +- makefile | 79 ++------- test/CMakeLists.txt | 162 ++++++++++-------- 9 files changed, 142 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/unittest_linux_asan.yml diff --git a/.github/workflows/unittest_linux.yml b/.github/workflows/unittest_linux.yml index 40b9a8a2..bb321c69 100644 --- a/.github/workflows/unittest_linux.yml +++ b/.github/workflows/unittest_linux.yml @@ -9,5 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - name: make test_make - run: make test_make + - uses: seanmiddleditch/gha-setup-ninja@master + - name: make test + run: make test diff --git a/.github/workflows/unittest_linux_asan.yml b/.github/workflows/unittest_linux_asan.yml new file mode 100644 index 00000000..1bec10b3 --- /dev/null +++ b/.github/workflows/unittest_linux_asan.yml @@ -0,0 +1,14 @@ +name: linux +on: + push: + paths-ignore: + - 'docs/**' + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master + - name: make test_asan + run: make test_asan diff --git a/.github/workflows/unittest_mac_tsan_mbedtls.yml b/.github/workflows/unittest_mac_tsan_mbedtls.yml index 794672ee..be80e2c7 100644 --- a/.github/workflows/unittest_mac_tsan_mbedtls.yml +++ b/.github/workflows/unittest_mac_tsan_mbedtls.yml @@ -9,6 +9,7 @@ jobs: runs-on: macOS-latest steps: - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master - name: install mbedtls run: brew install mbedtls - name: make test diff --git a/.github/workflows/unittest_mac_tsan_openssl.yml b/.github/workflows/unittest_mac_tsan_openssl.yml index 9926ec29..74f113cb 100644 --- a/.github/workflows/unittest_mac_tsan_openssl.yml +++ b/.github/workflows/unittest_mac_tsan_openssl.yml @@ -9,6 +9,7 @@ jobs: runs-on: macOS-latest steps: - uses: actions/checkout@v1 + - uses: seanmiddleditch/gha-setup-ninja@master - name: install openssl run: brew install openssl@1.1 - name: make test diff --git a/.github/workflows/unittest_mac_tsan_sectransport.yml b/.github/workflows/unittest_mac_tsan_sectransport.yml index 79c6dd81..c34e51dc 100644 --- a/.github/workflows/unittest_mac_tsan_sectransport.yml +++ b/.github/workflows/unittest_mac_tsan_sectransport.yml @@ -9,5 +9,6 @@ jobs: runs-on: macOS-latest steps: - uses: actions/checkout@v1 - - name: make test_tsan - run: make test_tsan + - uses: seanmiddleditch/gha-setup-ninja@master + - name: make test_tsan_sectransport + run: make test_tsan_sectransport diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e818947..1cbfdd1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -270,9 +270,10 @@ if (USE_WS OR USE_TEST) FetchContent_MakeAvailable(spdlog) if (USE_WS) - add_subdirectory(ws) + add_subdirectory(ws) endif() if (USE_TEST) - add_subdirectory(test) + enable_testing() + add_subdirectory(test) endif() endif() diff --git a/README.md b/README.md index d1eee7be..e8579337 100644 --- a/README.md +++ b/README.md @@ -92,18 +92,20 @@ If your company or project is using this library, feel free to open an issue or | OS | TLS | Sanitizer | Status | |-------------------|-------------------|-------------------|-------------------| -| Linux | OpenSSL | None | [![Build2][1]][7] | -| macOS | Secure Transport | Thread Sanitizer | [![Build2][2]][7] | -| macOS | OpenSSL | Thread Sanitizer | [![Build2][3]][7] | -| macOS | MbedTLS | Thread Sanitizer | [![Build2][4]][7] | -| Windows | Disabled | None | [![Build2][5]][7] | -| UWP | Disabled | None | [![Build2][6]][7] | +| Linux | OpenSSL | None | [![Build2][1]][0] | +| macOS | Secure Transport | Thread Sanitizer | [![Build2][2]][0] | +| macOS | OpenSSL | Thread Sanitizer | [![Build2][3]][0] | +| macOS | MbedTLS | Thread Sanitizer | [![Build2][4]][0] | +| Windows | Disabled | None | [![Build2][5]][0] | +| UWP | Disabled | None | [![Build2][6]][0] | +| Linux | OpenSSL | Address Sanitizer | [![Build2][7]][0] | +[0]: https://github.com/machinezone/IXWebSocket [1]: https://github.com/machinezone/IXWebSocket/workflows/linux/badge.svg [2]: https://github.com/machinezone/IXWebSocket/workflows/mac_tsan_sectransport/badge.svg [3]: https://github.com/machinezone/IXWebSocket/workflows/mac_tsan_openssl/badge.svg [4]: https://github.com/machinezone/IXWebSocket/workflows/mac_tsan_mbedtls/badge.svg [5]: https://github.com/machinezone/IXWebSocket/workflows/windows/badge.svg [6]: https://github.com/machinezone/IXWebSocket/workflows/uwp/badge.svg -[7]: https://github.com/machinezone/IXWebSocket +[7]: https://github.com/machinezone/IXWebSocket/workflows/linux_asan/badge.svg diff --git a/makefile b/makefile index b0837b24..6921756c 100644 --- a/makefile +++ b/makefile @@ -110,71 +110,30 @@ format: test_server: (cd test && npm i ws && node broadcast-server.js) -# env TEST=Websocket_server make test -# env TEST=Websocket_chat make test -# env TEST=heartbeat make test -build_test: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. ; ninja install) +test: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 ..) + (cd build ; ninja) + (cd build ; ninja test) -test: build_test - (cd test ; python2.7 run.py -r) - -test_make: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; make -j 4) - (cd test ; python2.7 run.py -r) - -test_tsan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_ubsan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_asan: build_test_asan - (cd test ; python2.7 run.py -r) - -build_test_asan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableAddressSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - -test_tsan_openssl: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_ubsan_openssl: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) - -test_tsan_openssl_release: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Release -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -configuration Release -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Release/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) +test_asan: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) test_tsan_mbedtls: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_MBED_TLS=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) - (cd build/test ; ln -sf Debug/ixwebsocket_unittest) - (cd test ; python2.7 run.py -r) + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_MBED_TLS=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=thread -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) -build_test_openssl: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_OPEN_SSL=1 -DUSE_TEST=1 .. ; ninja install) +test_tsan_openssl: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_OPENS_SSL=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=thread -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) -test_openssl: build_test_openssl - (cd test ; python2.7 run.py -r) - -build_test_mbedtls: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_MBED_TLS=1 -DUSE_TEST=1 .. ; make -j 4) - -test_mbedtls: build_test_mbedtls - (cd test ; python2.7 run.py -r) - -test_no_ssl: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TEST=1 .. ; make -j 4) - (cd test ; python2.7 run.py -r) +test_tsan_sectransport: + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_OPENS_SSL=1 -DUSE_TEST=1 .. -DCMAKE_C_FLAGS="-fsanitize=thread -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer") + (cd build ; ninja) + (cd build ; ninja test) ws_test: ws (cd ws ; env DEBUG=1 PATH=../ws/build:$$PATH bash test_ws.sh) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5e49714f..855d876e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,59 +9,37 @@ set (CMAKE_CXX_STANDARD 14) option(USE_TLS "Add TLS support" ON) -include_directories( - ${PROJECT_SOURCE_DIR}/Catch2/single_include - ../third_party - ../third_party/msgpack11 - ../ws -) - -add_definitions(-DSPDLOG_COMPILED_LIB=1) - -find_package(JsonCpp) -if (NOT JSONCPP_FOUND) - include_directories(../third_party/jsoncpp) - set(JSONCPP_SOURCES ../third_party/jsoncpp/jsoncpp.cpp) -endif() - # Shared sources -set (SOURCES - ${JSONCPP_SOURCES} - - test_runner.cpp - IXTest.cpp - ../third_party/msgpack11/msgpack11.cpp - - IXSocketTest.cpp - IXSocketConnectTest.cpp +set (TEST_TARGET_NAMES + IXSocketTest + IXSocketConnectTest # IXWebSocketLeakTest.cpp # commented until we have a fix for #224 - IXWebSocketServerTest.cpp - IXWebSocketTestConnectionDisconnection.cpp - IXUrlParserTest.cpp - IXWebSocketServerTest.cpp - IXHttpClientTest.cpp - IXHttpServerTest.cpp - IXUnityBuildsTest.cpp - IXHttpTest.cpp - IXDNSLookupTest.cpp - IXWebSocketSubProtocolTest.cpp - IXSentryClientTest.cpp - IXWebSocketChatTest.cpp - IXWebSocketBroadcastTest.cpp - IXWebSocketPerMessageDeflateCompressorTest.cpp - IXStreamSqlTest.cpp + IXWebSocketServerTest + IXWebSocketTestConnectionDisconnection + IXUrlParserTest + IXHttpClientTest + IXHttpServerTest + IXUnityBuildsTest + IXHttpTest + IXDNSLookupTest + IXWebSocketSubProtocolTest + IXSentryClientTest + IXWebSocketChatTest + IXWebSocketBroadcastTest + IXWebSocketPerMessageDeflateCompressorTest + IXStreamSqlTest ) # Some unittest don't work on windows yet # Windows without TLS does not have hmac yet if (UNIX) - list(APPEND SOURCES - IXWebSocketCloseTest.cpp - IXCobraChatTest.cpp - IXCobraMetricsPublisherTest.cpp - IXCobraToSentryBotTest.cpp - IXCobraToStatsdBotTest.cpp - IXCobraToStdoutBotTest.cpp + list(APPEND TEST_TARGET_NAMES + IXWebSocketCloseTest + IXCobraChatTest + IXCobraMetricsPublisherTest + IXCobraToSentryBotTest + IXCobraToStatsdBotTest + IXCobraToStdoutBotTest ) endif() @@ -71,19 +49,9 @@ endif() # Disable tests for now that are failing or not reliable -add_executable(ixwebsocket_unittest ${SOURCES}) - -if (MAC) - add_sanitizers(ixwebsocket_unittest) -endif() - -if (APPLE AND USE_TLS) - target_link_libraries(ixwebsocket_unittest "-framework foundation" "-framework security") -endif() - -if (JSONCPP_FOUND) - target_include_directories(ixwebsocket_unittest PUBLIC ${JSONCPP_INCLUDE_DIRS}) - target_link_libraries(ixwebsocket_unittest ${JSONCPP_LIBRARIES}) +find_package(JsonCpp) +if (NOT JSONCPP_FOUND) + set(JSONCPP_SOURCES ../third_party/jsoncpp/jsoncpp.cpp) endif() if (USE_PYTHON) @@ -97,19 +65,67 @@ if (USE_PYTHON) message("Python_LIBRARIES:${Python_LIBRARIES}") endif() -# library with the most dependencies come first -target_link_libraries(ixwebsocket_unittest ixbots) -target_link_libraries(ixwebsocket_unittest ixsnake) -target_link_libraries(ixwebsocket_unittest ixcobra) -target_link_libraries(ixwebsocket_unittest ixsentry) -target_link_libraries(ixwebsocket_unittest ixredis) -target_link_libraries(ixwebsocket_unittest ixwebsocket) -target_link_libraries(ixwebsocket_unittest ixcrypto) -target_link_libraries(ixwebsocket_unittest ixcore) +add_library(ixwebsocket_test) +target_sources(ixwebsocket_test PRIVATE + ${JSONCPP_SOURCES} + test_runner.cpp + IXTest.cpp + ../third_party/msgpack11/msgpack11.cpp +) +target_compile_definitions(ixwebsocket_test PRIVATE ${TEST_PROGRAMS_DEFINITIONS}) +target_include_directories(ixwebsocket_test PRIVATE + ${PROJECT_SOURCE_DIR}/Catch2/single_include + ../third_party +) +target_link_libraries(ixwebsocket_test ixsnake) +target_link_libraries(ixwebsocket_test ixcobra) +target_link_libraries(ixwebsocket_test ixwebsocket) +target_link_libraries(ixwebsocket_test spdlog) -target_link_libraries(ixwebsocket_unittest spdlog) -if (USE_PYTHON) - target_link_libraries(ixwebsocket_unittest ${Python_LIBRARIES}) -endif() +foreach(TEST_TARGET_NAME ${TEST_TARGET_NAMES}) + add_executable(${TEST_TARGET_NAME} + ${TEST_TARGET_NAME}.cpp + ) -install(TARGETS ixwebsocket_unittest DESTINATION bin) + target_include_directories(${TEST_TARGET_NAME} PRIVATE + ${PROJECT_SOURCE_DIR}/Catch2/single_include + ../third_party + ../third_party/msgpack11 + ) + + target_compile_definitions(${TEST_TARGET_NAME} PRIVATE SPDLOG_COMPILED_LIB=1) + + if (NOT JSONCPP_FOUND) + target_include_directories(${TEST_TARGET_NAME} PRIVATE ../third_party/jsoncpp) + endif() + + if (APPLE AND USE_TLS) + target_link_libraries(${TEST_TARGET_NAME} "-framework foundation" "-framework security") + endif() + + if (JSONCPP_FOUND) + target_include_directories(${TEST_TARGET_NAME} PRIVATE ${JSONCPP_INCLUDE_DIRS}) + target_link_libraries(${TEST_TARGET_NAME} ${JSONCPP_LIBRARIES}) + endif() + + # library with the most dependencies come first + target_link_libraries(${TEST_TARGET_NAME} ixbots) + target_link_libraries(${TEST_TARGET_NAME} ixsnake) + target_link_libraries(${TEST_TARGET_NAME} ixcobra) + target_link_libraries(${TEST_TARGET_NAME} ixsentry) + target_link_libraries(${TEST_TARGET_NAME} ixredis) + target_link_libraries(${TEST_TARGET_NAME} ixwebsocket_test) + target_link_libraries(${TEST_TARGET_NAME} ixwebsocket) + target_link_libraries(${TEST_TARGET_NAME} ixcrypto) + target_link_libraries(${TEST_TARGET_NAME} ixcore) + + target_link_libraries(${TEST_TARGET_NAME} spdlog) + if (USE_PYTHON) + target_link_libraries(${TEST_TARGET_NAME} ${Python_LIBRARIES}) + endif() + + add_test(NAME ${TEST_TARGET_NAME} + COMMAND ${TEST_TARGET_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +endforeach()