diff --git a/.gitignore b/.gitignore index 378eac25..6f97ca1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ build +*.pyc diff --git a/test/.gitignore b/test/.gitignore index 477ba279..910eaa65 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,9 +1,10 @@ CMakeCache.txt package-lock.json -CMakeFiles -ixwebsocket_unittest -cmake_install.cmake +CMakeFiles +ixwebsocket_unittest +cmake_install.cmake node_modules ixwebsocket Makefile build +ixwebsocket_unittest.xml diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dd47a970..677edd20 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -5,12 +5,11 @@ cmake_minimum_required (VERSION 3.4.1) project (ixwebsocket_unittest) -set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../third_party/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH}) -find_package(Sanitizers) - set (CMAKE_CXX_STANDARD 14) if (NOT WIN32) + set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../third_party/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH}) + find_package(Sanitizers) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") set(CMAKE_LD_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") option(USE_TLS "Add TLS support" ON) @@ -51,7 +50,10 @@ if (NOT WIN32) endif() add_executable(ixwebsocket_unittest ${SOURCES}) -add_sanitizers(ixwebsocket_unittest) + +if (NOT WIN32) + add_sanitizers(ixwebsocket_unittest) +endif() if (APPLE AND USE_TLS) target_link_libraries(ixwebsocket_unittest "-framework foundation" "-framework security") diff --git a/test/run.py b/test/run.py index 32234d42..30b441ce 100755 --- a/test/run.py +++ b/test/run.py @@ -28,9 +28,9 @@ try: except ImportError: hasClick = False - -DEFAULT_EXE = 'ixwebsocket_unittest' - +BUILD_TYPE = 'Debug' +XML_OUTPUT_FILE = 'ixwebsocket_unittest.xml' +TEST_EXE_PATH = None class Command(object): """Run system commands with timeout @@ -65,7 +65,7 @@ class Command(object): return True, self.process.returncode -def runCommand(cmd, assertOnFailure=True, timeout=None): +def runCommand(cmd, abortOnFailure=True, timeout=None): '''Small wrapper to run a command and make sure it succeed''' if timeout is None: @@ -73,16 +73,13 @@ def runCommand(cmd, assertOnFailure=True, timeout=None): print('\nRunning', cmd) command = Command(cmd) - timedout, ret = command.run(timeout) + succeed, ret = command.run(timeout) - if timedout: - print('Unittest timed out') - - msg = 'cmd {} failed with error code {}'.format(cmd, ret) - if ret != 0: + if not succeed or ret != 0: + msg = 'cmd {}\nfailed with error code {}'.format(cmd, ret) print(msg) - if assertOnFailure: - assert False + if abortOnFailure: + sys.exit(-1) def runCMake(sanitizer, buildDir): @@ -91,12 +88,6 @@ def runCMake(sanitizer, buildDir): (remove build sub-folder). ''' - # CMake installed via Self Service ends up here. - cmake_executable = '/Applications/CMake.app/Contents/bin/cmake' - - if not os.path.exists(cmake_executable): - cmake_executable = 'cmake' - sanitizersFlags = { 'asan': '-DSANITIZE_ADDRESS=On', 'ubsan': '-DSANITIZE_UNDEFINED=On', @@ -110,19 +101,22 @@ def runCMake(sanitizer, buildDir): if not os.path.exists(cmakeExecutable): cmakeExecutable = 'cmake' - generator = '"Unix Makefiles"' if platform.system() == 'Windows': - generator = '"NMake Makefiles"' + #generator = '"NMake Makefiles"' + generator = '"Visual Studio 16 2019"' + else: + generator = '"Unix Makefiles"' - fmt = ''' -{cmakeExecutable} -H. \ + CMAKE_BUILD_TYPE = BUILD_TYPE + + fmt = '{cmakeExecutable} -H. \ {sanitizerFlag} \ - -B{buildDir} \ - -DCMAKE_BUILD_TYPE=Debug \ + -B"{buildDir}" \ + -DCMAKE_BUILD_TYPE={CMAKE_BUILD_TYPE} \ -DUSE_TLS=1 \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -G{generator} -''' + -G{generator}' + cmakeCmd = fmt.format(**locals()) runCommand(cmakeCmd) @@ -133,10 +127,10 @@ def runTest(args, buildDir, xmlOutput, testRunName): if args is None: args = '' - fmt = '{buildDir}/{DEFAULT_EXE} -o {xmlOutput} -n "{testRunName}" -r junit "{args}"' - testCommand = fmt.format(**locals()) + testCommand = '{} -o {} -n "{}" -r junit "{}"'.format(TEST_EXE_PATH, xmlOutput, testRunName, args) + runCommand(testCommand, - assertOnFailure=False) + abortOnFailure=False) def validateTestSuite(xmlOutput): @@ -296,8 +290,7 @@ def executeJobs(jobs): def computeAllTestNames(buildDir): '''Compute all test case names, by executing the unittest in a custom mode''' - executable = os.path.join(buildDir, DEFAULT_EXE) - cmd = '"{}" --list-test-names-only'.format(executable) + cmd = '"{}" --list-test-names-only'.format(TEST_EXE_PATH) names = os.popen(cmd).read().splitlines() names.sort() # Sort test names for execution determinism return names @@ -344,7 +337,7 @@ def generateXmlOutput(results, xmlOutput, testRunName, runTime): }) systemOut = ET.Element('system-out') - systemOut.text = result['output'].decode('utf-8') + systemOut.text = result['output'].decode('utf-8', 'ignore') testCase.append(systemOut) if not result['success']: @@ -365,16 +358,19 @@ def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLD runCMake(sanitizer, buildDir) # build with make - makeCmd = 'make' - jobs = '-j8' + #makeCmd = 'cmake --build ' + #jobs = '-j8' - if platform.system() == 'Windows': - makeCmd = 'nmake' + #if platform.system() == 'Windows': + # makeCmd = 'nmake' # nmake does not have a -j option - jobs = '' + # jobs = '' - runCommand('{} -C {} {}'.format(makeCmd, buildDir, jobs)) + #runCommand('{} -C {} {}'.format(makeCmd, buildDir, jobs)) + + # build with cmake + runCommand('cmake --build ' + buildDir) if buildOnly: return @@ -409,12 +405,7 @@ def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLD continue # testName can contains spaces, so we enclose them in double quotes - executable = os.path.join(buildDir, DEFAULT_EXE) - - if platform.system() == 'Windows': - executable += '.exe' - - cmd = '{} "{}" "{}" > "{}" 2>&1'.format(lldb, executable, testName, outputPath) + cmd = '{} "{}" "{}" > "{}" 2>&1'.format(lldb, TEST_EXE_PATH, testName, outputPath) jobs.append({ 'name': testName, @@ -454,8 +445,6 @@ def main(): if not os.path.exists(buildDir): os.makedirs(buildDir) - defaultOutput = DEFAULT_EXE + '.xml' - parser = argparse.ArgumentParser(description='Build and Run the engine unittest') sanitizers = ['tsan', 'asan', 'ubsan', 'none'] @@ -481,14 +470,29 @@ def main(): # Default sanitizer is tsan sanitizer = args.sanitizer - if args.sanitizer is None: + + if args.no_sanitizer: + sanitizer = 'none' + elif args.sanitizer is None: sanitizer = 'tsan' + # Sanitizers display lots of strange errors on Linux on CI, + # which looks like false positives + if platform.system() != 'Darwin': + sanitizer = 'none' + defaultRunName = 'ixengine_{}_{}'.format(platform.system(), sanitizer) - xmlOutput = args.output or defaultOutput + xmlOutput = args.output or XML_OUTPUT_FILE testRunName = args.run_name or os.getenv('IXENGINE_TEST_RUN_NAME') or defaultRunName + global TEST_EXE_PATH + + if platform.system() == 'Windows': + TEST_EXE_PATH = os.path.join(buildDir, BUILD_TYPE, 'ixwebsocket_unittest.exe') + else: + TEST_EXE_PATH = os.path.join(buildDir, 'ixwebsocket_unittest') + if args.list: # catch2 exit with a different error code when requesting the list of files try: @@ -505,11 +509,6 @@ def main(): print('LLDB is only supported on Apple at this point') args.lldb = False - # Sanitizers display lots of strange errors on Linux on CI, - # which looks like false positives - if platform.system() != 'Darwin': - sanitizer = 'none' - return run(args.test, buildDir, sanitizer, xmlOutput, testRunName, args.build_only, args.lldb)