From 68c97da51856df3385a7fd88744d798df24e011d Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Thu, 25 Jun 2020 10:05:02 -0700 Subject: [PATCH] (cmake) new python code is optional and enabled at cmake time with -DUSE_PYTHON=1 --- docs/CHANGELOG.md | 5 +++- docs/build.md | 3 ++- ixbots/CMakeLists.txt | 12 +++++++--- ixbots/ixbots/IXCobraToPythonBot.cpp | 13 ++++++----- ixwebsocket/IXWebSocketVersion.h | 2 +- makefile | 35 +++++++++++++++------------- test/CMakeLists.txt | 18 +++++++------- ws/CMakeLists.txt | 18 +++++++------- 8 files changed, 62 insertions(+), 44 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 98eb82b9..dc98fd28 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,10 @@ # Changelog All changes to this project will be documented in this file. -======= +## [9.8.3] - 2020-06-25 + +(cmake) new python code is optional and enabled at cmake time with -DUSE_PYTHON=1 + ## [9.8.2] - 2020-06-24 (cobra bots) new cobra metrics bot to send data to statsd using Python for processing the message diff --git a/docs/build.md b/docs/build.md index f76b5c30..00e5895d 100644 --- a/docs/build.md +++ b/docs/build.md @@ -22,8 +22,9 @@ Options for building: * `-DUSE_MBED_TLS=1` will use [mbedlts](https://tls.mbed.org/) for the TLS support * `-DUSE_WS=1` will build the ws interactive command line tool * `-DUSE_TEST=1` will build the unittest +* `-DUSE_PYTHON=1` will use Python3 for cobra bots, require Python3 to be installed. -If you are on Windows, look at the [appveyor](https://github.com/machinezone/IXWebSocket/blob/master/appveyor.yml) file (not maintained much though) or rather the [github actions](https://github.com/machinezone/IXWebSocket/blob/master/.github/workflows/ccpp.yml#L40) which have instructions for building dependencies. +If you are on Windows, look at the [appveyor](https://github.com/machinezone/IXWebSocket/blob/master/appveyor.yml) file (not maintained much though) or rather the [github actions](https://github.com/machinezone/IXWebSocket/blob/master/.github/workflows/unittest_windows.yml) which have instructions for building dependencies. It is also possible to externally include the project, so that everything is fetched over the wire when you build like so: diff --git a/ixbots/CMakeLists.txt b/ixbots/CMakeLists.txt index 6e4041cc..dc424a44 100644 --- a/ixbots/CMakeLists.txt +++ b/ixbots/CMakeLists.txt @@ -36,7 +36,10 @@ if (NOT JSONCPP_FOUND) set(JSONCPP_INCLUDE_DIRS ../third_party/jsoncpp) endif() -find_package(Python COMPONENTS Development) +if (USE_PYTHON) + target_compile_definitions(ixbots PUBLIC IXBOTS_USE_PYTHON) + find_package(Python COMPONENTS Development) +endif() set(IXBOTS_INCLUDE_DIRS . @@ -47,7 +50,10 @@ set(IXBOTS_INCLUDE_DIRS ../ixredis ../ixsentry ${JSONCPP_INCLUDE_DIRS} - ${SPDLOG_INCLUDE_DIRS} - ${Python_INCLUDE_DIRS}) + ${SPDLOG_INCLUDE_DIRS}) + +if (USE_PYTHON) + set(IXBOTS_INCLUDE_DIRS ${IXBOTS_INCLUDE_DIRS} ${Python_INCLUDE_DIRS}) +endif() target_include_directories( ixbots PUBLIC ${IXBOTS_INCLUDE_DIRS} ) diff --git a/ixbots/ixbots/IXCobraToPythonBot.cpp b/ixbots/ixbots/IXCobraToPythonBot.cpp index e867880e..fd137983 100644 --- a/ixbots/ixbots/IXCobraToPythonBot.cpp +++ b/ixbots/ixbots/IXCobraToPythonBot.cpp @@ -23,18 +23,18 @@ // (linking error about missing debug build) // -#ifndef _WIN32 +#ifdef IXBOTS_USE_PYTHON #define PY_SSIZE_T_CLEAN #include #endif -#ifndef _WIN32 +#ifdef IXBOTS_USE_PYTHON namespace { // // This function is unused at this point. It produce a correct output, - // but triggers memory leaks when called repeateadly, as the reference counting - // Python functions are not used properly + // but triggers memory leaks when called repeateadly, as I cannot figure out how to + // make the reference counting Python functions to work properly (Py_DECREF and friends) // PyObject* jsonToPythonObject(const Json::Value& val) { @@ -104,8 +104,9 @@ namespace ix StatsdClient& statsdClient, const std::string& scriptPath) { -#ifdef _WIN32 - CoreLogger::error("Command is disabled on Windows."); +#ifndef IXBOTS_USE_PYTHON + CoreLogger::error("Command is disabled. " + "Needs to be configured with USE_PYTHON=1"); return -1; #else CobraBot bot; diff --git a/ixwebsocket/IXWebSocketVersion.h b/ixwebsocket/IXWebSocketVersion.h index 461c6055..a6f7fd17 100644 --- a/ixwebsocket/IXWebSocketVersion.h +++ b/ixwebsocket/IXWebSocketVersion.h @@ -6,4 +6,4 @@ #pragma once -#define IX_WEBSOCKET_VERSION "9.8.2" +#define IX_WEBSOCKET_VERSION "9.8.3" diff --git a/makefile b/makefile index 22e391c8..e1ed7c34 100644 --- a/makefile +++ b/makefile @@ -19,26 +19,28 @@ install: brew # # Release, Debug, MinSizeRel, RelWithDebInfo are the build types # +# Default rule does not use python as that requires first time users to have Python3 installed +# brew: mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; ninja install) -# Docker default target. We've add problem with OpenSSL and TLS 1.3 (on the +# Docker default target. We've had problems with OpenSSL and TLS 1.3 (on the # server side ?) and I can't work-around it easily, so we're using mbedtls on # Linux for the SSL backend, which works great. ws_mbedtls_install: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; ninja install) + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; ninja install) ws: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 .. && ninja install) + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 .. && ninja install) ws_install: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_TLS=1 -DUSE_WS=1 .. && make -j 4 install) + mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=MinSizeRel -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 .. && make -j 4 install) ws_openssl: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_OPEN_SSL=1 .. ; make -j 4) + mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_OPEN_SSL=1 .. ; make -j 4) ws_mbedtls: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; make -j 4) + mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_MBED_TLS=1 .. ; make -j 4) ws_no_ssl: mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_WS=1 .. ; make -j 4) @@ -107,16 +109,16 @@ test: (cd test ; python2.7 run.py -r) test_make: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; make -j 4) + mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_WS=1 -DUSE_TEST=1 .. ; make -j 4) (cd test ; python2.7 run.py -r) test_tsan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) (cd build/test ; ln -sf Debug/ixwebsocket_unittest) (cd test ; python2.7 run.py -r) test_ubsan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) (cd build/test ; ln -sf Debug/ixwebsocket_unittest) (cd test ; python2.7 run.py -r) @@ -124,37 +126,37 @@ test_asan: build_test_asan (cd test ; python2.7 run.py -r) build_test_asan: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableAddressSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableAddressSanitizer YES) (cd build/test ; ln -sf Debug/ixwebsocket_unittest) test_tsan_openssl: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) (cd build/test ; ln -sf Debug/ixwebsocket_unittest) (cd test ; python2.7 run.py -r) test_ubsan_openssl: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableUndefinedBehaviorSanitizer YES) (cd build/test ; ln -sf Debug/ixwebsocket_unittest) (cd test ; python2.7 run.py -r) test_tsan_openssl_release: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Release -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -configuration Release -target ixwebsocket_unittest -enableThreadSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Release -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_OPEN_SSL=1 .. && xcodebuild -project ixwebsocket.xcodeproj -configuration Release -target ixwebsocket_unittest -enableThreadSanitizer YES) (cd build/test ; ln -sf Release/ixwebsocket_unittest) (cd test ; python2.7 run.py -r) test_tsan_mbedtls: - mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_MBED_TLS=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) + mkdir -p build && (cd build && cmake -GXcode -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_TEST=1 -DUSE_MBED_TLS=1 .. && xcodebuild -project ixwebsocket.xcodeproj -target ixwebsocket_unittest -enableThreadSanitizer YES) (cd build/test ; ln -sf Debug/ixwebsocket_unittest) (cd test ; python2.7 run.py -r) build_test_openssl: - mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_OPEN_SSL=1 -DUSE_TEST=1 .. ; ninja install) + mkdir -p build && (cd build ; cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_OPEN_SSL=1 -DUSE_TEST=1 .. ; ninja install) test_openssl: build_test_openssl (cd test ; python2.7 run.py -r) build_test_mbedtls: - mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_TLS=1 -DUSE_MBED_TLS=1 -DUSE_TEST=1 .. ; make -j 4) + mkdir -p build && (cd build ; cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_PYTHON=1 -DUSE_TLS=1 -DUSE_MBED_TLS=1 -DUSE_TEST=1 .. ; make -j 4) test_mbedtls: build_test_mbedtls (cd test ; python2.7 run.py -r) @@ -223,6 +225,7 @@ rebase_upstream: git reset --hard upstream/master git push origin master --force +# Legacy target install_cmake_for_linux: mkdir -p /tmp/cmake (cd /tmp/cmake ; curl -L -O https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz ; tar zxf cmake-3.14.0-Linux-x86_64.tar.gz) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 64cb80ff..a7081b0c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -91,14 +91,16 @@ if (JSONCPP_FOUND) target_link_libraries(ixwebsocket_unittest ${JSONCPP_LIBRARIES}) endif() -find_package(Python COMPONENTS Development) -if (NOT Python_FOUND) - message(FATAL_ERROR "Python3 not found") +if (USE_PYTHON) + find_package(Python COMPONENTS Development) + if (NOT Python_FOUND) + message(FATAL_ERROR "Python3 not found") + endif() + message("Python_FOUND:${Python_FOUND}") + message("Python_VERSION:${Python_VERSION}") + message("Python_Development_FOUND:${Python_Development_FOUND}") + message("Python_LIBRARIES:${Python_LIBRARIES}") endif() -message("Python_FOUND:${Python_FOUND}") -message("Python_VERSION:${Python_VERSION}") -message("Python_Development_FOUND:${Python_Development_FOUND}") -message("Python_LIBRARIES:${Python_LIBRARIES}") # library with the most dependencies come first target_link_libraries(ixwebsocket_unittest ixbots) @@ -111,7 +113,7 @@ target_link_libraries(ixwebsocket_unittest ixcrypto) target_link_libraries(ixwebsocket_unittest ixcore) target_link_libraries(ixwebsocket_unittest spdlog) -if (NOT WIN32) +if (USE_PYTHON) target_link_libraries(ixwebsocket_unittest ${Python_LIBRARIES}) endif() diff --git a/ws/CMakeLists.txt b/ws/CMakeLists.txt index 1543655a..cd477eb7 100644 --- a/ws/CMakeLists.txt +++ b/ws/CMakeLists.txt @@ -32,14 +32,16 @@ if (NOT JSONCPP_FOUND) set(JSONCPP_SOURCES ../third_party/jsoncpp/jsoncpp.cpp) endif() -find_package(Python COMPONENTS Development) -if (NOT Python_FOUND) - message(FATAL_ERROR "Python3 not found") +if (USE_PYTHON) + find_package(Python COMPONENTS Development) + if (NOT Python_FOUND) + message(FATAL_ERROR "Python3 not found") + endif() + message("Python_FOUND:${Python_FOUND}") + message("Python_VERSION:${Python_VERSION}") + message("Python_Development_FOUND:${Python_Development_FOUND}") + message("Python_LIBRARIES:${Python_LIBRARIES}") endif() -message("Python_FOUND:${Python_FOUND}") -message("Python_VERSION:${Python_VERSION}") -message("Python_Development_FOUND:${Python_Development_FOUND}") -message("Python_LIBRARIES:${Python_LIBRARIES}") add_executable(ws ../third_party/msgpack11/msgpack11.cpp @@ -80,7 +82,7 @@ target_link_libraries(ws ixcrypto) target_link_libraries(ws ixcore) target_link_libraries(ws spdlog) -if (NOT WIN32) +if (USE_PYTHON) target_link_libraries(ws ${Python_LIBRARIES}) endif()