diff --git a/makefile b/makefile index 1098117d..76251df5 100644 --- a/makefile +++ b/makefile @@ -48,7 +48,7 @@ test_server: # env TEST=Websocket_chat make test # env TEST=heartbeat make test test: - python test/run.py + (cd test ; python2.7 run.py) ws_test: all (cd ws ; bash test_ws.sh) diff --git a/test/run.py b/test/run.py old mode 100644 new mode 100755 index decdc261..ec27f865 --- a/test/run.py +++ b/test/run.py @@ -1,9 +1,31 @@ -import os -import platform -import shutil +#!/usr/bin/env python2.7 +''' +''' -import subprocess +from __future__ import print_function + +import os +import sys +import platform +import argparse +import multiprocessing +import tempfile +import time +import datetime import threading +import subprocess +import re +import xml.etree.ElementTree as ET +from xml.dom import minidom + +hasClick = True +try: + import click +except ImportError: + hasClick = False + + +DEFAULT_EXE = 'ixwebsocket_unittest' class Command(object): @@ -16,12 +38,16 @@ class Command(object): self.cmd = cmd self.process = None - def run_command(self, capture = False): + def run_command(self): self.process = subprocess.Popen(self.cmd, shell=True) self.process.communicate() - def run(self, timeout = 5 * 60): + def run(self, timeout=None): '''5 minutes default timeout''' + + if timeout is None: + timeout = 5 * 60 + thread = threading.Thread(target=self.run_command, args=()) thread.start() thread.join(timeout) @@ -35,85 +61,427 @@ class Command(object): return True, self.process.returncode -osName = platform.system() -print('os name = {}'.format(osName)) +def runCommand(cmd, assertOnFailure=True, timeout=None): + '''Small wrapper to run a command and make sure it succeed''' -root = os.path.dirname(os.path.realpath(__file__)) -buildDir = os.path.join(root, 'build', osName) + if timeout is None: + timeout = 30 * 60 # 30 minute default timeout -if not os.path.exists(buildDir): - os.makedirs(buildDir) + print('\nRunning', cmd) + command = Command(cmd) + timedout, ret = command.run(timeout) -os.chdir(buildDir) + if timedout: + print('Unittest timed out') -if osName == 'Windows': - generator = '-G"NMake Makefiles"' - make = 'nmake' - testBinary ='ixwebsocket_unittest.exe' -else: - generator = '' - make = 'make -j6' - testBinary ='./ixwebsocket_unittest' + msg = 'cmd {} failed with error code {}'.format(cmd, ret) + if ret != 0: + print(msg) + if assertOnFailure: + assert False -sanitizersFlags = { - 'asan': '-DSANITIZE_ADDRESS=On', - 'ubsan': '-DSANITIZE_UNDEFINED=On', - 'tsan': '-DSANITIZE_THREAD=On', - 'none': '' -} -sanitizer = 'tsan' -if osName == 'Linux': - sanitizer = 'none' -sanitizerFlags = sanitizersFlags[sanitizer] +def runCMake(sanitizer, buildDir): + '''Generate a makefile from CMake. + We do an out of dir build, so that cleaning up is easy + (remove build sub-folder). + ''' -# if osName == 'Windows': -# os.environ['CC'] = 'clang-cl' -# os.environ['CXX'] = 'clang-cl' + # CMake installed via Self Service ends up here. + cmake_executable = '/Applications/CMake.app/Contents/bin/cmake' -cmakeCmd = 'cmake -DCMAKE_BUILD_TYPE=Debug {} {} ../..'.format(generator, sanitizerFlags) -print(cmakeCmd) -ret = os.system(cmakeCmd) -assert ret == 0, 'CMake failed, exiting' + if not os.path.exists(cmake_executable): + cmake_executable = 'cmake' -ret = os.system(make) -assert ret == 0, 'Make failed, exiting' + sanitizersFlags = { + 'asan': '-DSANITIZE_ADDRESS=On', + 'ubsan': '-DSANITIZE_UNDEFINED=On', + 'tsan': '-DSANITIZE_THREAD=On', + 'none': '' + } + sanitizerFlag = sanitizersFlags[sanitizer] -def findFiles(prefix): - '''Find all files under a given directory''' + # CMake installed via Self Service ends up here. + cmakeExecutable = '/Applications/CMake.app/Contents/bin/cmake' + if not os.path.exists(cmakeExecutable): + cmakeExecutable = 'cmake' - paths = [] + fmt = ''' +{cmakeExecutable} -H. \ + {sanitizerFlag} \ + -B{buildDir} \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +''' + cmakeCmd = fmt.format(**locals()) + runCommand(cmakeCmd) - for root, _, files in os.walk(prefix): - for path in files: - fullPath = os.path.join(root, path) - if os.path.islink(fullPath): - continue +def runTest(args, buildDir, xmlOutput, testRunName): + '''Execute the unittest. + ''' + if args is None: + args = '' - paths.append(fullPath) + fmt = '{buildDir}/{DEFAULT_EXE} -o {xmlOutput} -n "{testRunName}" -r junit "{args}"' + testCommand = fmt.format(**locals()) + runCommand(testCommand, + assertOnFailure=False) - return paths -#for path in findFiles('.'): -# print(path) +def validateTestSuite(xmlOutput): + ''' + Parse the output XML file to validate that all tests passed. -# We need to copy the zlib DLL in the current work directory -shutil.copy(os.path.join( - '..', - '..', - '..', - 'third_party', - 'ZLIB-Windows', - 'zlib-1.2.11_deploy_v140', - 'release_dynamic', - 'x64', - 'bin', - 'zlib.dll'), '.') + Assume that the XML file contains only one testsuite. + (which is true when generate by catch2) + ''' + tree = ET.parse(xmlOutput) + root = tree.getroot() + testSuite = root[0] + testSuiteAttributes = testSuite.attrib -# lldb = "lldb --batch -o 'run' -k 'thread backtrace all' -k 'quit 1'" -lldb = "" # Disabled for now -testCommand = '{} {} {}'.format(lldb, testBinary, os.getenv('TEST', '')) -command = Command(testCommand) -timedout, ret = command.run() -assert ret == 0, 'Test command failed' + tests = testSuiteAttributes['tests'] + + success = True + + for testcase in testSuite: + if testcase.tag != 'testcase': + continue + + testName = testcase.attrib['name'] + systemOutput = None + + for child in testcase: + if child.tag == 'system-out': + systemOutput = child.text + + if child.tag == 'failure': + success = False + + print("Testcase '{}' failed".format(testName)) + print(' ', systemOutput) + + return success, tests + + +def log(msg, color): + if hasClick: + click.secho(msg, fg=color) + else: + print(msg) + + +def isSuccessFullRun(output): + '''When being run from lldb, we cannot capture the exit code + so we have to parse the output which is produced in a + consistent way. Whenever we'll be on a recent enough version of lldb we + won't have to do this. + ''' + pid = None + matchingPids = False + exitCode = -1 + + # 'Process 279 exited with status = 1 (0x00000001) ', + exitPattern = re.compile('^Process (?P[0-9]+) exited with status = (?P[0-9]+)') + + # "Process 99232 launched: '/Users/bse... + launchedPattern = re.compile('^Process (?P[0-9]+) launched: ') + + for line in output: + match = exitPattern.match(line) + if match: + exitCode = int(match.group('exitCode')) + pid = match.group('pid') + + match = launchedPattern.match(line) + if match: + matchingPids = (pid == match.group('pid')) + + return exitCode == 0 and matchingPids + + +def testLLDBOutput(): + failedOutputWithCrashLines = [ + ' frame #15: 0x00007fff73f4d305 libsystem_pthread.dylib`_pthread_body + 126', + ' frame #16: 0x00007fff73f5026f libsystem_pthread.dylib`_pthread_start + 70', + ' frame #17: 0x00007fff73f4c415 libsystem_pthread.dylib`thread_start + 13', + '(lldb) quit 1' + ] + + failedOutputWithFailedUnittest = [ + '===============================================================================', + 'test cases: 1 | 0 passed | 1 failed', 'assertions: 15 | 14 passed | 1 failed', + '', + 'Process 279 exited with status = 1 (0x00000001) ', + '', + "Process 279 launched: '/Users/bsergeant/src/foss/ixwebsocket/test/build/Darwin/ixwebsocket_unittest' (x86_64)" + ] + + successLines = [ + '...', + '...', + 'All tests passed (16 assertions in 1 test case)', + '', + 'Process 99232 exited with status = 0 (0x00000000) ', + '', + "Process 99232 launched: '/Users/bsergeant/src/foss/ixwebsocket/test/build/Darwin/ixwebsocket_unittest' (x86_64)" + ] + + assert not isSuccessFullRun(failedOutputWithCrashLines) + assert not isSuccessFullRun(failedOutputWithFailedUnittest) + assert isSuccessFullRun(successLines) + + +def executeJob(job): + '''Execute a unittest and capture info about it (runtime, success, etc...)''' + + start = time.time() + + sys.stderr.write('.') + + # 2 minutes of timeout for a single test + timeout = 2 * 60 + command = Command(job['cmd']) + timedout, ret = command.run(timeout) + + job['exit_code'] = ret + job['success'] = ret == 0 + job['runtime'] = time.time() - start + + # Record unittest console output + job['output'] = '' + path = job['output_path'] + + if os.path.exists(path): + with open(path) as f: + output = f.read() + job['output'] = output + + outputLines = output.splitlines() + + if job['use_lldb']: + job['success'] = isSuccessFullRun(outputLines) + + # Cleanup tmp file now that its content was read + os.unlink(path) + + return job + + +def executeJobs(jobs): + '''Execute a list of job concurrently on multiple CPU/cores''' + + poolSize = multiprocessing.cpu_count() + + pool = multiprocessing.Pool(poolSize) + results = pool.map(executeJob, jobs) + pool.close() + pool.join() + + return results + + +def computeAllTestNames(buildDir): + '''Compute all test case names, by executing the unittest in a custom mode''' + + executable = os.path.join(buildDir, DEFAULT_EXE) + cmd = '"{}" --list-test-names-only'.format(executable) + names = os.popen(cmd).read().splitlines() + names.sort() # Sort test names for execution determinism + return names + + +def prettyPrintXML(root): + '''Pretty print an XML file. Default writer write it on a single line + which makes it hard for human to inspect.''' + + serializedXml = ET.tostring(root, encoding='utf-8') + reparsed = minidom.parseString(serializedXml) + prettyPrinted = reparsed.toprettyxml(indent=" ") + return prettyPrinted + + +def generateXmlOutput(results, xmlOutput, testRunName, runTime): + '''Generate a junit compatible XML file + + We prefer doing this ourself instead of letting Catch2 do it. + When the test is crashing (as has happened on Jenkins), an invalid file + with no trailer can be created which trigger an XML reading error in validateTestSuite. + + Something like that: + ``` + + + ``` + ''' + + root = ET.Element('testsuites') + testSuite = ET.Element('testsuite', { + 'name': testRunName, + 'tests': str(len(results)), + 'failures': str(sum(1 for result in results if not result['success'])), + 'time': str(runTime), + 'timestamp': datetime.datetime.utcnow().isoformat(), + }) + root.append(testSuite) + + for result in results: + testCase = ET.Element('testcase', { + 'name': result['name'], + 'time': str(result['runtime']) + }) + + systemOut = ET.Element('system-out') + systemOut.text = result['output'].decode('utf-8') + testCase.append(systemOut) + + if not result['success']: + failure = ET.Element('failure') + testCase.append(failure) + + testSuite.append(testCase) + + with open(xmlOutput, 'w') as f: + content = prettyPrintXML(root) + f.write(content.encode('utf-8')) + + +def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLDB): + '''Main driver. Run cmake, compiles, execute and validate the testsuite.''' + + # Override CXX for Linux + os.environ['CXX'] = "/usr/bin/clang++ --std=c++14 --stdlib=libc++" + + runCMake(sanitizer, buildDir) + runCommand('make -C {} -j8'.format(buildDir)) + + if buildOnly: + return + + # A specific test case can be provided on the command line + if testName: + testNames = [testName] + else: + # Default case + testNames = computeAllTestNames(buildDir) + + # This should be empty. It is useful to have a blacklist during transitions + # We could add something for asan as well. + blackLists = { + 'ubsan': [] + } + blackList = blackLists.get(sanitizer, []) + + # Run through LLDB to capture crashes + lldb = '' + if useLLDB: + lldb = "lldb --batch -o 'run' -k 'thread backtrace all' -k 'quit 1'" + + # Jobs is a list of python dicts + jobs = [] + + for testName in testNames: + outputPath = tempfile.mktemp(suffix=testName + '.log') + + if testName in blackList: + log('Skipping blacklisted test {}'.format(testName), 'yellow') + continue + + # testName can contains spaces, so we enclose them in double quotes + executable = os.path.join(buildDir, DEFAULT_EXE) + + cmd = '{} "{}" "{}" >& "{}"'.format(lldb, executable, testName, outputPath) + + jobs.append({ + 'name': testName, + 'cmd': cmd, + 'output_path': outputPath, + 'use_lldb': useLLDB + }) + + start = time.time() + results = executeJobs(jobs) + runTime = time.time() - start + generateXmlOutput(results, xmlOutput, testRunName, runTime) + + # Validate and report results + print('\nParsing junit test result file: {}'.format(xmlOutput)) + log('## Results', 'blue') + success, tests = validateTestSuite(xmlOutput) + + if success: + label = 'tests' if int(tests) > 1 else 'test' + msg = 'All test passed (#{} {})'.format(tests, label) + color = 'green' + else: + msg = 'unittest failed' + color = 'red' + + log(msg, color) + log('Execution time: %.2fs' % (runTime), 'blue') + sys.exit(0 if success else 1) + + +def main(): + buildDir = 'build/' + platform.system() + if not os.path.exists(buildDir): + os.makedirs(buildDir) + + defaultOutput = DEFAULT_EXE + '.xml' + + parser = argparse.ArgumentParser(description='Build and Run the engine unittest') + + sanitizers = ['tsan', 'asan', 'ubsan', 'none'] + + parser.add_argument('--sanitizer', choices=sanitizers, + help='Run a clang sanitizer.') + parser.add_argument('--test', '-t', help='Test name.') + parser.add_argument('--list', '-l', action='store_true', + help='Print test names and exit.') + parser.add_argument('--no_sanitizer', action='store_true', + help='Do not execute a clang sanitizer.') + parser.add_argument('--validate', action='store_true', + help='Validate XML output.') + parser.add_argument('--build_only', '-b', action='store_true', + help='Stop after building. Do not run the unittest.') + parser.add_argument('--output', '-o', help='Output XML file.') + parser.add_argument('--lldb', action='store_true', + help='Run the test through lldb.') + parser.add_argument('--run_name', '-n', + help='Name of the test run.') + + args = parser.parse_args() + + # Default sanitizer is tsan + sanitizer = args.sanitizer + if args.sanitizer is None: + sanitizer = 'tsan' + + defaultRunName = 'ixengine_{}_{}'.format(platform.system(), sanitizer) + + xmlOutput = args.output or defaultOutput + testRunName = args.run_name or os.getenv('IXENGINE_TEST_RUN_NAME') or defaultRunName + + if args.list: + # catch2 exit with a different error code when requesting the list of files + try: + runTest('--list-test-names-only', buildDir, xmlOutput, testRunName) + except AssertionError: + pass + return + + if args.validate: + validateTestSuite(xmlOutput) + return + + if platform.system() != 'Darwin' and args.lldb: + print('LLDB is only supported on Apple at this point') + args.lldb = False + + return run(args.test, buildDir, sanitizer, xmlOutput, + testRunName, args.build_only, args.lldb) + + +if __name__ == '__main__': + main()