better unittest runner / can run through lldb and produce a junit XML artifact

makefile

@@ -48,7 +48,7 @@ test_server:
 # env TEST=Websocket_chat make test
 # env TEST=heartbeat make test
 test:
-	python test/run.py
+	(cd test ; python2.7 run.py)
 
 ws_test: all
 	(cd ws ; bash test_ws.sh)

test/run.py

@@ -1,9 +1,31 @@
-import os
+#!/usr/bin/env python2.7
-import platform
+'''
-import shutil
+'''
 
-import subprocess
+from __future__ import print_function
+
+import os
+import sys
+import platform
+import argparse
+import multiprocessing
+import tempfile
+import time
+import datetime
 import threading
+import subprocess
+import re
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+
+hasClick = True
+try:
+    import click
+except ImportError:
+    hasClick = False
+
+
+DEFAULT_EXE = 'ixwebsocket_unittest'
 
 
 class Command(object):
@@ -16,12 +38,16 @@ class Command(object):
         self.cmd = cmd
         self.process = None
 
-    def run_command(self, capture = False):
+    def run_command(self):
         self.process = subprocess.Popen(self.cmd, shell=True)
         self.process.communicate()
 
-    def run(self, timeout = 5 * 60):
+    def run(self, timeout=None):
         '''5 minutes default timeout'''
+        
+        if timeout is None:
+            timeout = 5 * 60
+
         thread = threading.Thread(target=self.run_command, args=())
         thread.start()
         thread.join(timeout)
@@ -35,85 +61,427 @@ class Command(object):
             return True, self.process.returncode
 
 
-osName = platform.system()
+def runCommand(cmd, assertOnFailure=True, timeout=None):
-print('os name = {}'.format(osName))
+    '''Small wrapper to run a command and make sure it succeed'''
 
-root = os.path.dirname(os.path.realpath(__file__))
+    if timeout is None:
-buildDir = os.path.join(root, 'build', osName)
+        timeout = 30 * 60 # 30 minute default timeout
 
-if not os.path.exists(buildDir):
+    print('\nRunning', cmd)
-    os.makedirs(buildDir)
+    command = Command(cmd)
+    timedout, ret = command.run(timeout)
 
-os.chdir(buildDir)
+    if timedout:
+        print('Unittest timed out')
 
-if osName == 'Windows':
+    msg = 'cmd {} failed with error code {}'.format(cmd, ret)
-    generator = '-G"NMake Makefiles"'
+    if ret != 0:
-    make = 'nmake'
+        print(msg)
-    testBinary ='ixwebsocket_unittest.exe'
+        if assertOnFailure:
-else:
+            assert False
-    generator = ''
-    make = 'make -j6'
-    testBinary ='./ixwebsocket_unittest'
 
-sanitizersFlags = {
-    'asan': '-DSANITIZE_ADDRESS=On',
-    'ubsan': '-DSANITIZE_UNDEFINED=On',
-    'tsan': '-DSANITIZE_THREAD=On',
-    'none': ''
-}
-sanitizer = 'tsan'
-if osName == 'Linux':
-    sanitizer = 'none'
 
-sanitizerFlags = sanitizersFlags[sanitizer]
+def runCMake(sanitizer, buildDir):
+    '''Generate a makefile from CMake.
+    We do an out of dir build, so that cleaning up is easy
+    (remove build sub-folder).
+    '''
 
-# if osName == 'Windows':
+    # CMake installed via Self Service ends up here.
-#     os.environ['CC'] = 'clang-cl'
+    cmake_executable = '/Applications/CMake.app/Contents/bin/cmake'
-#     os.environ['CXX'] = 'clang-cl'
 
-cmakeCmd = 'cmake -DCMAKE_BUILD_TYPE=Debug {} {} ../..'.format(generator, sanitizerFlags)
+    if not os.path.exists(cmake_executable):
-print(cmakeCmd)
+        cmake_executable = 'cmake'
-ret = os.system(cmakeCmd)
-assert ret == 0, 'CMake failed, exiting'
 
-ret = os.system(make)
+    sanitizersFlags = {
-assert ret == 0, 'Make failed, exiting'
+        'asan': '-DSANITIZE_ADDRESS=On',
+        'ubsan': '-DSANITIZE_UNDEFINED=On',
+        'tsan': '-DSANITIZE_THREAD=On',
+        'none': ''
+    }
+    sanitizerFlag = sanitizersFlags[sanitizer]
 
-def findFiles(prefix):
+    # CMake installed via Self Service ends up here.
-    '''Find all files under a given directory'''
+    cmakeExecutable = '/Applications/CMake.app/Contents/bin/cmake'
+    if not os.path.exists(cmakeExecutable):
+        cmakeExecutable = 'cmake'
 
-    paths = []
+    fmt = '''
+{cmakeExecutable} -H. \
+    {sanitizerFlag} \
+    -B{buildDir} \
+    -DCMAKE_BUILD_TYPE=Debug \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+'''
+    cmakeCmd = fmt.format(**locals())
+    runCommand(cmakeCmd)
 
-    for root, _, files in os.walk(prefix):
-        for path in files:
-            fullPath = os.path.join(root, path)
 
-            if os.path.islink(fullPath):
+def runTest(args, buildDir, xmlOutput, testRunName):
-                continue
+    '''Execute the unittest.
+    '''
+    if args is None:
+        args = ''
 
-            paths.append(fullPath)
+    fmt = '{buildDir}/{DEFAULT_EXE} -o {xmlOutput} -n "{testRunName}" -r junit "{args}"'
+    testCommand = fmt.format(**locals())
+    runCommand(testCommand,
+               assertOnFailure=False)
 
-    return paths
 
-#for path in findFiles('.'):
+def validateTestSuite(xmlOutput):
-#    print(path)
+    '''
+    Parse the output XML file to validate that all tests passed.
 
-# We need to copy the zlib DLL in the current work directory
+    Assume that the XML file contains only one testsuite.
-shutil.copy(os.path.join(
+    (which is true when generate by catch2)
-    '..',
+    '''
-    '..',
+    tree = ET.parse(xmlOutput)
-    '..',
+    root = tree.getroot()
-    'third_party',
+    testSuite = root[0]
-    'ZLIB-Windows',
+    testSuiteAttributes = testSuite.attrib
-    'zlib-1.2.11_deploy_v140',
-    'release_dynamic',
-    'x64',
-    'bin',
-    'zlib.dll'), '.')
 
-# lldb = "lldb --batch -o 'run' -k 'thread backtrace all' -k 'quit 1'"
+    tests = testSuiteAttributes['tests']
-lldb = ""  # Disabled for now
+
-testCommand = '{} {} {}'.format(lldb, testBinary, os.getenv('TEST', ''))
+    success = True
-command = Command(testCommand)
+
-timedout, ret = command.run()
+    for testcase in testSuite:
-assert ret == 0, 'Test command failed'
+        if testcase.tag != 'testcase':
+            continue
+
+        testName = testcase.attrib['name']
+        systemOutput = None
+
+        for child in testcase:
+            if child.tag == 'system-out':
+                systemOutput = child.text
+
+            if child.tag == 'failure':
+                success = False
+
+                print("Testcase '{}' failed".format(testName))
+                print(' ', systemOutput)
+
+    return success, tests
+
+
+def log(msg, color):
+    if hasClick:
+        click.secho(msg, fg=color)
+    else:
+        print(msg)
+
+
+def isSuccessFullRun(output):
+    '''When being run from lldb, we cannot capture the exit code 
+    so we have to parse the output which is produced in a 
+    consistent way. Whenever we'll be on a recent enough version of lldb we 
+    won't have to do this.
+    '''
+    pid = None
+    matchingPids = False
+    exitCode = -1
+
+    # 'Process 279 exited with status = 1 (0x00000001) ',
+    exitPattern = re.compile('^Process (?P<pid>[0-9]+) exited with status = (?P<exitCode>[0-9]+)')
+
+    # "Process 99232 launched: '/Users/bse...
+    launchedPattern = re.compile('^Process (?P<pid>[0-9]+) launched: ')
+
+    for line in output:
+        match = exitPattern.match(line)
+        if match:
+            exitCode = int(match.group('exitCode'))
+            pid = match.group('pid')
+
+        match = launchedPattern.match(line)
+        if match:
+            matchingPids = (pid == match.group('pid'))
+
+    return exitCode == 0 and matchingPids
+
+
+def testLLDBOutput():
+    failedOutputWithCrashLines = [
+        '    frame #15: 0x00007fff73f4d305 libsystem_pthread.dylib`_pthread_body + 126',
+        '    frame #16: 0x00007fff73f5026f libsystem_pthread.dylib`_pthread_start + 70',
+        '    frame #17: 0x00007fff73f4c415 libsystem_pthread.dylib`thread_start + 13',
+        '(lldb) quit 1'
+    ]
+
+    failedOutputWithFailedUnittest = [
+        '===============================================================================', 
+        'test cases:  1 |  0 passed | 1 failed', 'assertions: 15 | 14 passed | 1 failed', 
+        '', 
+        'Process 279 exited with status = 1 (0x00000001) ',
+        '',
+        "Process 279 launched: '/Users/bsergeant/src/foss/ixwebsocket/test/build/Darwin/ixwebsocket_unittest' (x86_64)"
+    ]
+
+    successLines = [
+        '...',
+        '...',
+        'All tests passed (16 assertions in 1 test case)',
+        '',
+        'Process 99232 exited with status = 0 (0x00000000) ',
+        '',
+        "Process 99232 launched: '/Users/bsergeant/src/foss/ixwebsocket/test/build/Darwin/ixwebsocket_unittest' (x86_64)"
+    ]
+
+    assert not isSuccessFullRun(failedOutputWithCrashLines)
+    assert not isSuccessFullRun(failedOutputWithFailedUnittest)
+    assert isSuccessFullRun(successLines)
+
+
+def executeJob(job):
+    '''Execute a unittest and capture info about it (runtime, success, etc...)'''
+
+    start = time.time()
+
+    sys.stderr.write('.')
+
+    # 2 minutes of timeout for a single test
+    timeout = 2 * 60
+    command = Command(job['cmd'])
+    timedout, ret = command.run(timeout)
+
+    job['exit_code'] = ret
+    job['success'] = ret == 0
+    job['runtime'] = time.time() - start
+
+    # Record unittest console output
+    job['output'] = ''
+    path = job['output_path']
+
+    if os.path.exists(path):
+        with open(path) as f:
+            output = f.read()
+            job['output'] = output
+
+        outputLines = output.splitlines()
+
+        if job['use_lldb']:
+            job['success'] = isSuccessFullRun(outputLines)
+
+        # Cleanup tmp file now that its content was read
+        os.unlink(path)
+
+    return job
+
+
+def executeJobs(jobs):
+    '''Execute a list of job concurrently on multiple CPU/cores'''
+
+    poolSize = multiprocessing.cpu_count()
+
+    pool = multiprocessing.Pool(poolSize)
+    results = pool.map(executeJob, jobs)
+    pool.close()
+    pool.join()
+
+    return results
+
+
+def computeAllTestNames(buildDir):
+    '''Compute all test case names, by executing the unittest in a custom mode'''
+
+    executable = os.path.join(buildDir, DEFAULT_EXE)
+    cmd = '"{}" --list-test-names-only'.format(executable)
+    names = os.popen(cmd).read().splitlines()
+    names.sort()  # Sort test names for execution determinism
+    return names
+
+
+def prettyPrintXML(root):
+    '''Pretty print an XML file. Default writer write it on a single line
+    which makes it hard for human to inspect.'''
+
+    serializedXml = ET.tostring(root, encoding='utf-8')
+    reparsed = minidom.parseString(serializedXml)
+    prettyPrinted = reparsed.toprettyxml(indent="  ")
+    return prettyPrinted
+
+
+def generateXmlOutput(results, xmlOutput, testRunName, runTime):
+    '''Generate a junit compatible XML file
+    
+    We prefer doing this ourself instead of letting Catch2 do it.
+    When the test is crashing (as has happened on Jenkins), an invalid file 
+    with no trailer can be created which trigger an XML reading error in validateTestSuite.
+
+    Something like that:
+    ```
+    <testsuite>
+      <foo>
+    ```
+    '''
+
+    root = ET.Element('testsuites')
+    testSuite = ET.Element('testsuite', {
+        'name': testRunName,
+        'tests': str(len(results)),
+        'failures': str(sum(1 for result in results if not result['success'])),
+        'time': str(runTime),
+        'timestamp': datetime.datetime.utcnow().isoformat(),
+    })
+    root.append(testSuite)
+
+    for result in results:
+        testCase = ET.Element('testcase', {
+            'name': result['name'],
+            'time': str(result['runtime'])
+        })
+
+        systemOut = ET.Element('system-out')
+        systemOut.text = result['output'].decode('utf-8')
+        testCase.append(systemOut)
+
+        if not result['success']:
+            failure = ET.Element('failure')
+            testCase.append(failure)
+
+        testSuite.append(testCase)
+
+    with open(xmlOutput, 'w') as f:
+        content = prettyPrintXML(root)
+        f.write(content.encode('utf-8'))
+
+
+def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLDB):
+    '''Main driver. Run cmake, compiles, execute and validate the testsuite.'''
+
+    # Override CXX for Linux
+    os.environ['CXX'] = "/usr/bin/clang++ --std=c++14 --stdlib=libc++"
+
+    runCMake(sanitizer, buildDir)
+    runCommand('make -C {} -j8'.format(buildDir))
+
+    if buildOnly:
+        return
+
+    # A specific test case can be provided on the command line
+    if testName:
+        testNames = [testName]
+    else:
+        # Default case
+        testNames = computeAllTestNames(buildDir)
+
+    # This should be empty. It is useful to have a blacklist during transitions
+    # We could add something for asan as well.
+    blackLists = {
+        'ubsan': []
+    }
+    blackList = blackLists.get(sanitizer, [])
+
+    # Run through LLDB to capture crashes
+    lldb = ''
+    if useLLDB:
+        lldb = "lldb --batch -o 'run' -k 'thread backtrace all' -k 'quit 1'"
+
+    # Jobs is a list of python dicts
+    jobs = []
+
+    for testName in testNames:
+        outputPath = tempfile.mktemp(suffix=testName + '.log')
+
+        if testName in blackList:
+            log('Skipping blacklisted test {}'.format(testName), 'yellow')
+            continue
+
+        # testName can contains spaces, so we enclose them in double quotes
+        executable = os.path.join(buildDir, DEFAULT_EXE)
+
+        cmd = '{} "{}" "{}" >& "{}"'.format(lldb, executable, testName, outputPath)
+
+        jobs.append({
+            'name': testName,
+            'cmd': cmd,
+            'output_path': outputPath,
+            'use_lldb': useLLDB
+        })
+
+    start = time.time()
+    results = executeJobs(jobs)
+    runTime = time.time() - start
+    generateXmlOutput(results, xmlOutput, testRunName, runTime)
+
+    # Validate and report results
+    print('\nParsing junit test result file: {}'.format(xmlOutput))
+    log('## Results', 'blue')
+    success, tests = validateTestSuite(xmlOutput)
+
+    if success:
+        label = 'tests' if int(tests) > 1 else 'test'
+        msg = 'All test passed (#{} {})'.format(tests, label)
+        color = 'green'
+    else:
+        msg = 'unittest failed'
+        color = 'red'
+
+    log(msg, color)
+    log('Execution time: %.2fs' % (runTime), 'blue')
+    sys.exit(0 if success else 1)
+
+
+def main():
+    buildDir = 'build/' + platform.system()
+    if not os.path.exists(buildDir):
+        os.makedirs(buildDir)
+
+    defaultOutput = DEFAULT_EXE + '.xml'
+
+    parser = argparse.ArgumentParser(description='Build and Run the engine unittest')
+
+    sanitizers = ['tsan', 'asan', 'ubsan', 'none']
+
+    parser.add_argument('--sanitizer', choices=sanitizers,
+                        help='Run a clang sanitizer.')
+    parser.add_argument('--test', '-t', help='Test name.')
+    parser.add_argument('--list', '-l', action='store_true',
+                        help='Print test names and exit.')
+    parser.add_argument('--no_sanitizer', action='store_true',
+                        help='Do not execute a clang sanitizer.')
+    parser.add_argument('--validate', action='store_true',
+                        help='Validate XML output.')
+    parser.add_argument('--build_only', '-b', action='store_true',
+                        help='Stop after building. Do not run the unittest.')
+    parser.add_argument('--output', '-o', help='Output XML file.')
+    parser.add_argument('--lldb', action='store_true',
+                        help='Run the test through lldb.')
+    parser.add_argument('--run_name', '-n',
+                        help='Name of the test run.')
+
+    args = parser.parse_args()
+
+    # Default sanitizer is tsan
+    sanitizer = args.sanitizer
+    if args.sanitizer is None:
+        sanitizer = 'tsan'
+
+    defaultRunName = 'ixengine_{}_{}'.format(platform.system(), sanitizer)
+
+    xmlOutput = args.output or defaultOutput
+    testRunName = args.run_name or os.getenv('IXENGINE_TEST_RUN_NAME') or defaultRunName
+
+    if args.list:
+        # catch2 exit with a different error code when requesting the list of files
+        try:
+            runTest('--list-test-names-only', buildDir, xmlOutput, testRunName)
+        except AssertionError:
+            pass
+        return
+
+    if args.validate:
+        validateTestSuite(xmlOutput)
+        return
+
+    if platform.system() != 'Darwin' and args.lldb:
+        print('LLDB is only supported on Apple at this point')
+        args.lldb = False
+
+    return run(args.test, buildDir, sanitizer, xmlOutput, 
+               testRunName, args.build_only, args.lldb)
+
+
+if __name__ == '__main__':
+    main()