Fix run.py (#71)

* fix run.py

* run.py: fix Windows support

* fix test listing

.gitignore

@@ -1 +1,2 @@
 build
+*.pyc

test/.gitignore

@@ -7,3 +7,4 @@ node_modules
 ixwebsocket
 Makefile
 build
+ixwebsocket_unittest.xml

test/CMakeLists.txt

@@ -5,12 +5,11 @@
 cmake_minimum_required (VERSION 3.4.1)
 project (ixwebsocket_unittest)
 
-set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../third_party/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH})
-find_package(Sanitizers)
-
 set (CMAKE_CXX_STANDARD 14)
 
 if (NOT WIN32)
+  set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../third_party/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH})
+  find_package(Sanitizers)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
   set(CMAKE_LD_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
   option(USE_TLS "Add TLS support" ON)
@@ -51,7 +50,10 @@ if (NOT WIN32)
 endif()
 
 add_executable(ixwebsocket_unittest ${SOURCES})
-add_sanitizers(ixwebsocket_unittest)
+
+if (NOT WIN32)
+  add_sanitizers(ixwebsocket_unittest)
+endif()
 
 if (APPLE AND USE_TLS)
     target_link_libraries(ixwebsocket_unittest "-framework foundation" "-framework security")

test/run.py

@@ -28,9 +28,9 @@ try:
 except ImportError:
     hasClick = False
 
-
-DEFAULT_EXE = 'ixwebsocket_unittest'
-
+BUILD_TYPE = 'Debug'
+XML_OUTPUT_FILE = 'ixwebsocket_unittest.xml'
+TEST_EXE_PATH = None
 
 class Command(object):
     """Run system commands with timeout
@@ -65,7 +65,7 @@ class Command(object):
             return True, self.process.returncode
 
 
-def runCommand(cmd, assertOnFailure=True, timeout=None):
+def runCommand(cmd, abortOnFailure=True, timeout=None):
     '''Small wrapper to run a command and make sure it succeed'''
 
     if timeout is None:
@@ -73,16 +73,13 @@ def runCommand(cmd, assertOnFailure=True, timeout=None):
 
     print('\nRunning', cmd)
     command = Command(cmd)
-    timedout, ret = command.run(timeout)
+    succeed, ret = command.run(timeout)
 
-    if timedout:
-        print('Unittest timed out')
-
-    msg = 'cmd {} failed with error code {}'.format(cmd, ret)
-    if ret != 0:
+    if not succeed or ret != 0:
+        msg = 'cmd {}\nfailed with error code {}'.format(cmd, ret)
         print(msg)
-        if assertOnFailure:
-            assert False
+        if abortOnFailure:
+            sys.exit(-1)
 
 
 def runCMake(sanitizer, buildDir):
@@ -91,12 +88,6 @@ def runCMake(sanitizer, buildDir):
     (remove build sub-folder).
     '''
 
-    # CMake installed via Self Service ends up here.
-    cmake_executable = '/Applications/CMake.app/Contents/bin/cmake'
-
-    if not os.path.exists(cmake_executable):
-        cmake_executable = 'cmake'
-
     sanitizersFlags = {
         'asan': '-DSANITIZE_ADDRESS=On',
         'ubsan': '-DSANITIZE_UNDEFINED=On',
@@ -110,19 +101,22 @@ def runCMake(sanitizer, buildDir):
     if not os.path.exists(cmakeExecutable):
         cmakeExecutable = 'cmake'
 
-    generator = '"Unix Makefiles"'
     if platform.system() == 'Windows':
-        generator = '"NMake Makefiles"'
+        #generator = '"NMake Makefiles"'
+        generator = '"Visual Studio 16 2019"'
+    else:
+        generator = '"Unix Makefiles"'
 
-    fmt = '''
-{cmakeExecutable} -H. \
+    CMAKE_BUILD_TYPE = BUILD_TYPE
+
+    fmt = '{cmakeExecutable} -H. \
     {sanitizerFlag} \
-    -B{buildDir} \
-    -DCMAKE_BUILD_TYPE=Debug \
+    -B"{buildDir}" \
+    -DCMAKE_BUILD_TYPE={CMAKE_BUILD_TYPE} \
     -DUSE_TLS=1 \
     -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-    -G{generator}
-'''
+    -G{generator}'
+
     cmakeCmd = fmt.format(**locals())
     runCommand(cmakeCmd)
 
@@ -133,10 +127,10 @@ def runTest(args, buildDir, xmlOutput, testRunName):
     if args is None:
         args = ''
 
-    fmt = '{buildDir}/{DEFAULT_EXE} -o {xmlOutput} -n "{testRunName}" -r junit "{args}"'
-    testCommand = fmt.format(**locals())
+    testCommand = '{} -o {} -n "{}" -r junit "{}"'.format(TEST_EXE_PATH, xmlOutput, testRunName, args)
+
     runCommand(testCommand,
-               assertOnFailure=False)
+               abortOnFailure=False)
 
 
 def validateTestSuite(xmlOutput):
@@ -296,8 +290,7 @@ def executeJobs(jobs):
 def computeAllTestNames(buildDir):
     '''Compute all test case names, by executing the unittest in a custom mode'''
 
-    executable = os.path.join(buildDir, DEFAULT_EXE)
-    cmd = '"{}" --list-test-names-only'.format(executable)
+    cmd = '"{}" --list-test-names-only'.format(TEST_EXE_PATH)
     names = os.popen(cmd).read().splitlines()
     names.sort()  # Sort test names for execution determinism
     return names
@@ -344,7 +337,7 @@ def generateXmlOutput(results, xmlOutput, testRunName, runTime):
         })
 
         systemOut = ET.Element('system-out')
-        systemOut.text = result['output'].decode('utf-8')
+        systemOut.text = result['output'].decode('utf-8', 'ignore')
         testCase.append(systemOut)
 
         if not result['success']:
@@ -365,16 +358,19 @@ def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLD
     runCMake(sanitizer, buildDir)
 
     # build with make
-    makeCmd = 'make'
-    jobs = '-j8'
+    #makeCmd = 'cmake --build '
+    #jobs = '-j8'
 
-    if platform.system() == 'Windows':
-        makeCmd = 'nmake'
+    #if platform.system() == 'Windows':
+    #    makeCmd = 'nmake'
 
         # nmake does not have a -j option
-        jobs = ''
+    #    jobs = ''
 
-    runCommand('{} -C {} {}'.format(makeCmd, buildDir, jobs))
+    #runCommand('{} -C {} {}'.format(makeCmd, buildDir, jobs))
+
+    # build with cmake
+    runCommand('cmake --build ' + buildDir)
 
     if buildOnly:
         return
@@ -409,12 +405,7 @@ def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLD
             continue
 
         # testName can contains spaces, so we enclose them in double quotes
-        executable = os.path.join(buildDir, DEFAULT_EXE)
-
-        if platform.system() == 'Windows':
-            executable += '.exe'
-
-        cmd = '{} "{}" "{}" > "{}" 2>&1'.format(lldb, executable, testName, outputPath)
+        cmd = '{} "{}" "{}" > "{}" 2>&1'.format(lldb, TEST_EXE_PATH, testName, outputPath)
 
         jobs.append({
             'name': testName,
@@ -454,8 +445,6 @@ def main():
     if not os.path.exists(buildDir):
         os.makedirs(buildDir)
 
-    defaultOutput = DEFAULT_EXE + '.xml'
-
     parser = argparse.ArgumentParser(description='Build and Run the engine unittest')
 
     sanitizers = ['tsan', 'asan', 'ubsan', 'none']
@@ -481,14 +470,29 @@ def main():
 
     # Default sanitizer is tsan
     sanitizer = args.sanitizer
-    if args.sanitizer is None:
+
+    if args.no_sanitizer:
+        sanitizer = 'none'
+    elif args.sanitizer is None:
         sanitizer = 'tsan'
 
+    # Sanitizers display lots of strange errors on Linux on CI,
+    # which looks like false positives
+    if platform.system() != 'Darwin':
+        sanitizer = 'none'
+
     defaultRunName = 'ixengine_{}_{}'.format(platform.system(), sanitizer)
 
-    xmlOutput = args.output or defaultOutput
+    xmlOutput = args.output or XML_OUTPUT_FILE
     testRunName = args.run_name or os.getenv('IXENGINE_TEST_RUN_NAME') or defaultRunName
 
+    global TEST_EXE_PATH
+
+    if platform.system() == 'Windows':
+        TEST_EXE_PATH = os.path.join(buildDir, BUILD_TYPE, 'ixwebsocket_unittest.exe')
+    else:
+        TEST_EXE_PATH = os.path.join(buildDir, 'ixwebsocket_unittest')
+
     if args.list:
         # catch2 exit with a different error code when requesting the list of files
         try:
@@ -505,11 +509,6 @@ def main():
         print('LLDB is only supported on Apple at this point')
         args.lldb = False
 
-    # Sanitizers display lots of strange errors on Linux on CI,
-    # which looks like false positives
-    if platform.system() != 'Darwin':
-        sanitizer = 'none'
-
     return run(args.test, buildDir, sanitizer, xmlOutput, 
                testRunName, args.build_only, args.lldb)