better unittest runner / can run through lldb and produce a junit XML artifact
This commit is contained in:
Benjamin Sergeant
508
test/run.py
Normal file → Executable file
508
test/run.py
Normal file → Executable file
@@ -1,9 +1,31 @@
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
#!/usr/bin/env python2.7
|
||||
'''
|
||||
'''
|
||||
|
||||
import subprocess
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import platform
|
||||
import argparse
|
||||
import multiprocessing
|
||||
import tempfile
|
||||
import time
|
||||
import datetime
|
||||
import threading
|
||||
import subprocess
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from xml.dom import minidom
|
||||
|
||||
hasClick = True
|
||||
try:
|
||||
import click
|
||||
except ImportError:
|
||||
hasClick = False
|
||||
|
||||
|
||||
DEFAULT_EXE = 'ixwebsocket_unittest'
|
||||
|
||||
|
||||
class Command(object):
|
||||
@@ -16,12 +38,16 @@ class Command(object):
|
||||
self.cmd = cmd
|
||||
self.process = None
|
||||
|
||||
def run_command(self, capture = False):
|
||||
def run_command(self):
|
||||
self.process = subprocess.Popen(self.cmd, shell=True)
|
||||
self.process.communicate()
|
||||
|
||||
def run(self, timeout = 5 * 60):
|
||||
def run(self, timeout=None):
|
||||
'''5 minutes default timeout'''
|
||||
|
||||
if timeout is None:
|
||||
timeout = 5 * 60
|
||||
|
||||
thread = threading.Thread(target=self.run_command, args=())
|
||||
thread.start()
|
||||
thread.join(timeout)
|
||||
@@ -35,85 +61,427 @@ class Command(object):
|
||||
return True, self.process.returncode
|
||||
|
||||
|
||||
osName = platform.system()
|
||||
print('os name = {}'.format(osName))
|
||||
def runCommand(cmd, assertOnFailure=True, timeout=None):
|
||||
'''Small wrapper to run a command and make sure it succeed'''
|
||||
|
||||
root = os.path.dirname(os.path.realpath(__file__))
|
||||
buildDir = os.path.join(root, 'build', osName)
|
||||
if timeout is None:
|
||||
timeout = 30 * 60 # 30 minute default timeout
|
||||
|
||||
if not os.path.exists(buildDir):
|
||||
os.makedirs(buildDir)
|
||||
print('\nRunning', cmd)
|
||||
command = Command(cmd)
|
||||
timedout, ret = command.run(timeout)
|
||||
|
||||
os.chdir(buildDir)
|
||||
if timedout:
|
||||
print('Unittest timed out')
|
||||
|
||||
if osName == 'Windows':
|
||||
generator = '-G"NMake Makefiles"'
|
||||
make = 'nmake'
|
||||
testBinary ='ixwebsocket_unittest.exe'
|
||||
else:
|
||||
generator = ''
|
||||
make = 'make -j6'
|
||||
testBinary ='./ixwebsocket_unittest'
|
||||
msg = 'cmd {} failed with error code {}'.format(cmd, ret)
|
||||
if ret != 0:
|
||||
print(msg)
|
||||
if assertOnFailure:
|
||||
assert False
|
||||
|
||||
sanitizersFlags = {
|
||||
'asan': '-DSANITIZE_ADDRESS=On',
|
||||
'ubsan': '-DSANITIZE_UNDEFINED=On',
|
||||
'tsan': '-DSANITIZE_THREAD=On',
|
||||
'none': ''
|
||||
}
|
||||
sanitizer = 'tsan'
|
||||
if osName == 'Linux':
|
||||
sanitizer = 'none'
|
||||
|
||||
sanitizerFlags = sanitizersFlags[sanitizer]
|
||||
def runCMake(sanitizer, buildDir):
|
||||
'''Generate a makefile from CMake.
|
||||
We do an out of dir build, so that cleaning up is easy
|
||||
(remove build sub-folder).
|
||||
'''
|
||||
|
||||
# if osName == 'Windows':
|
||||
# os.environ['CC'] = 'clang-cl'
|
||||
# os.environ['CXX'] = 'clang-cl'
|
||||
# CMake installed via Self Service ends up here.
|
||||
cmake_executable = '/Applications/CMake.app/Contents/bin/cmake'
|
||||
|
||||
cmakeCmd = 'cmake -DCMAKE_BUILD_TYPE=Debug {} {} ../..'.format(generator, sanitizerFlags)
|
||||
print(cmakeCmd)
|
||||
ret = os.system(cmakeCmd)
|
||||
assert ret == 0, 'CMake failed, exiting'
|
||||
if not os.path.exists(cmake_executable):
|
||||
cmake_executable = 'cmake'
|
||||
|
||||
ret = os.system(make)
|
||||
assert ret == 0, 'Make failed, exiting'
|
||||
sanitizersFlags = {
|
||||
'asan': '-DSANITIZE_ADDRESS=On',
|
||||
'ubsan': '-DSANITIZE_UNDEFINED=On',
|
||||
'tsan': '-DSANITIZE_THREAD=On',
|
||||
'none': ''
|
||||
}
|
||||
sanitizerFlag = sanitizersFlags[sanitizer]
|
||||
|
||||
def findFiles(prefix):
|
||||
'''Find all files under a given directory'''
|
||||
# CMake installed via Self Service ends up here.
|
||||
cmakeExecutable = '/Applications/CMake.app/Contents/bin/cmake'
|
||||
if not os.path.exists(cmakeExecutable):
|
||||
cmakeExecutable = 'cmake'
|
||||
|
||||
paths = []
|
||||
fmt = '''
|
||||
{cmakeExecutable} -H. \
|
||||
{sanitizerFlag} \
|
||||
-B{buildDir} \
|
||||
-DCMAKE_BUILD_TYPE=Debug \
|
||||
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
|
||||
'''
|
||||
cmakeCmd = fmt.format(**locals())
|
||||
runCommand(cmakeCmd)
|
||||
|
||||
for root, _, files in os.walk(prefix):
|
||||
for path in files:
|
||||
fullPath = os.path.join(root, path)
|
||||
|
||||
if os.path.islink(fullPath):
|
||||
continue
|
||||
def runTest(args, buildDir, xmlOutput, testRunName):
|
||||
'''Execute the unittest.
|
||||
'''
|
||||
if args is None:
|
||||
args = ''
|
||||
|
||||
paths.append(fullPath)
|
||||
fmt = '{buildDir}/{DEFAULT_EXE} -o {xmlOutput} -n "{testRunName}" -r junit "{args}"'
|
||||
testCommand = fmt.format(**locals())
|
||||
runCommand(testCommand,
|
||||
assertOnFailure=False)
|
||||
|
||||
return paths
|
||||
|
||||
#for path in findFiles('.'):
|
||||
# print(path)
|
||||
def validateTestSuite(xmlOutput):
|
||||
'''
|
||||
Parse the output XML file to validate that all tests passed.
|
||||
|
||||
# We need to copy the zlib DLL in the current work directory
|
||||
shutil.copy(os.path.join(
|
||||
'..',
|
||||
'..',
|
||||
'..',
|
||||
'third_party',
|
||||
'ZLIB-Windows',
|
||||
'zlib-1.2.11_deploy_v140',
|
||||
'release_dynamic',
|
||||
'x64',
|
||||
'bin',
|
||||
'zlib.dll'), '.')
|
||||
Assume that the XML file contains only one testsuite.
|
||||
(which is true when generate by catch2)
|
||||
'''
|
||||
tree = ET.parse(xmlOutput)
|
||||
root = tree.getroot()
|
||||
testSuite = root[0]
|
||||
testSuiteAttributes = testSuite.attrib
|
||||
|
||||
# lldb = "lldb --batch -o 'run' -k 'thread backtrace all' -k 'quit 1'"
|
||||
lldb = "" # Disabled for now
|
||||
testCommand = '{} {} {}'.format(lldb, testBinary, os.getenv('TEST', ''))
|
||||
command = Command(testCommand)
|
||||
timedout, ret = command.run()
|
||||
assert ret == 0, 'Test command failed'
|
||||
tests = testSuiteAttributes['tests']
|
||||
|
||||
success = True
|
||||
|
||||
for testcase in testSuite:
|
||||
if testcase.tag != 'testcase':
|
||||
continue
|
||||
|
||||
testName = testcase.attrib['name']
|
||||
systemOutput = None
|
||||
|
||||
for child in testcase:
|
||||
if child.tag == 'system-out':
|
||||
systemOutput = child.text
|
||||
|
||||
if child.tag == 'failure':
|
||||
success = False
|
||||
|
||||
print("Testcase '{}' failed".format(testName))
|
||||
print(' ', systemOutput)
|
||||
|
||||
return success, tests
|
||||
|
||||
|
||||
def log(msg, color):
|
||||
if hasClick:
|
||||
click.secho(msg, fg=color)
|
||||
else:
|
||||
print(msg)
|
||||
|
||||
|
||||
def isSuccessFullRun(output):
|
||||
'''When being run from lldb, we cannot capture the exit code
|
||||
so we have to parse the output which is produced in a
|
||||
consistent way. Whenever we'll be on a recent enough version of lldb we
|
||||
won't have to do this.
|
||||
'''
|
||||
pid = None
|
||||
matchingPids = False
|
||||
exitCode = -1
|
||||
|
||||
# 'Process 279 exited with status = 1 (0x00000001) ',
|
||||
exitPattern = re.compile('^Process (?P<pid>[0-9]+) exited with status = (?P<exitCode>[0-9]+)')
|
||||
|
||||
# "Process 99232 launched: '/Users/bse...
|
||||
launchedPattern = re.compile('^Process (?P<pid>[0-9]+) launched: ')
|
||||
|
||||
for line in output:
|
||||
match = exitPattern.match(line)
|
||||
if match:
|
||||
exitCode = int(match.group('exitCode'))
|
||||
pid = match.group('pid')
|
||||
|
||||
match = launchedPattern.match(line)
|
||||
if match:
|
||||
matchingPids = (pid == match.group('pid'))
|
||||
|
||||
return exitCode == 0 and matchingPids
|
||||
|
||||
|
||||
def testLLDBOutput():
|
||||
failedOutputWithCrashLines = [
|
||||
' frame #15: 0x00007fff73f4d305 libsystem_pthread.dylib`_pthread_body + 126',
|
||||
' frame #16: 0x00007fff73f5026f libsystem_pthread.dylib`_pthread_start + 70',
|
||||
' frame #17: 0x00007fff73f4c415 libsystem_pthread.dylib`thread_start + 13',
|
||||
'(lldb) quit 1'
|
||||
]
|
||||
|
||||
failedOutputWithFailedUnittest = [
|
||||
'===============================================================================',
|
||||
'test cases: 1 | 0 passed | 1 failed', 'assertions: 15 | 14 passed | 1 failed',
|
||||
'',
|
||||
'Process 279 exited with status = 1 (0x00000001) ',
|
||||
'',
|
||||
"Process 279 launched: '/Users/bsergeant/src/foss/ixwebsocket/test/build/Darwin/ixwebsocket_unittest' (x86_64)"
|
||||
]
|
||||
|
||||
successLines = [
|
||||
'...',
|
||||
'...',
|
||||
'All tests passed (16 assertions in 1 test case)',
|
||||
'',
|
||||
'Process 99232 exited with status = 0 (0x00000000) ',
|
||||
'',
|
||||
"Process 99232 launched: '/Users/bsergeant/src/foss/ixwebsocket/test/build/Darwin/ixwebsocket_unittest' (x86_64)"
|
||||
]
|
||||
|
||||
assert not isSuccessFullRun(failedOutputWithCrashLines)
|
||||
assert not isSuccessFullRun(failedOutputWithFailedUnittest)
|
||||
assert isSuccessFullRun(successLines)
|
||||
|
||||
|
||||
def executeJob(job):
|
||||
'''Execute a unittest and capture info about it (runtime, success, etc...)'''
|
||||
|
||||
start = time.time()
|
||||
|
||||
sys.stderr.write('.')
|
||||
|
||||
# 2 minutes of timeout for a single test
|
||||
timeout = 2 * 60
|
||||
command = Command(job['cmd'])
|
||||
timedout, ret = command.run(timeout)
|
||||
|
||||
job['exit_code'] = ret
|
||||
job['success'] = ret == 0
|
||||
job['runtime'] = time.time() - start
|
||||
|
||||
# Record unittest console output
|
||||
job['output'] = ''
|
||||
path = job['output_path']
|
||||
|
||||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
output = f.read()
|
||||
job['output'] = output
|
||||
|
||||
outputLines = output.splitlines()
|
||||
|
||||
if job['use_lldb']:
|
||||
job['success'] = isSuccessFullRun(outputLines)
|
||||
|
||||
# Cleanup tmp file now that its content was read
|
||||
os.unlink(path)
|
||||
|
||||
return job
|
||||
|
||||
|
||||
def executeJobs(jobs):
|
||||
'''Execute a list of job concurrently on multiple CPU/cores'''
|
||||
|
||||
poolSize = multiprocessing.cpu_count()
|
||||
|
||||
pool = multiprocessing.Pool(poolSize)
|
||||
results = pool.map(executeJob, jobs)
|
||||
pool.close()
|
||||
pool.join()
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def computeAllTestNames(buildDir):
|
||||
'''Compute all test case names, by executing the unittest in a custom mode'''
|
||||
|
||||
executable = os.path.join(buildDir, DEFAULT_EXE)
|
||||
cmd = '"{}" --list-test-names-only'.format(executable)
|
||||
names = os.popen(cmd).read().splitlines()
|
||||
names.sort() # Sort test names for execution determinism
|
||||
return names
|
||||
|
||||
|
||||
def prettyPrintXML(root):
|
||||
'''Pretty print an XML file. Default writer write it on a single line
|
||||
which makes it hard for human to inspect.'''
|
||||
|
||||
serializedXml = ET.tostring(root, encoding='utf-8')
|
||||
reparsed = minidom.parseString(serializedXml)
|
||||
prettyPrinted = reparsed.toprettyxml(indent=" ")
|
||||
return prettyPrinted
|
||||
|
||||
|
||||
def generateXmlOutput(results, xmlOutput, testRunName, runTime):
|
||||
'''Generate a junit compatible XML file
|
||||
|
||||
We prefer doing this ourself instead of letting Catch2 do it.
|
||||
When the test is crashing (as has happened on Jenkins), an invalid file
|
||||
with no trailer can be created which trigger an XML reading error in validateTestSuite.
|
||||
|
||||
Something like that:
|
||||
```
|
||||
<testsuite>
|
||||
<foo>
|
||||
```
|
||||
'''
|
||||
|
||||
root = ET.Element('testsuites')
|
||||
testSuite = ET.Element('testsuite', {
|
||||
'name': testRunName,
|
||||
'tests': str(len(results)),
|
||||
'failures': str(sum(1 for result in results if not result['success'])),
|
||||
'time': str(runTime),
|
||||
'timestamp': datetime.datetime.utcnow().isoformat(),
|
||||
})
|
||||
root.append(testSuite)
|
||||
|
||||
for result in results:
|
||||
testCase = ET.Element('testcase', {
|
||||
'name': result['name'],
|
||||
'time': str(result['runtime'])
|
||||
})
|
||||
|
||||
systemOut = ET.Element('system-out')
|
||||
systemOut.text = result['output'].decode('utf-8')
|
||||
testCase.append(systemOut)
|
||||
|
||||
if not result['success']:
|
||||
failure = ET.Element('failure')
|
||||
testCase.append(failure)
|
||||
|
||||
testSuite.append(testCase)
|
||||
|
||||
with open(xmlOutput, 'w') as f:
|
||||
content = prettyPrintXML(root)
|
||||
f.write(content.encode('utf-8'))
|
||||
|
||||
|
||||
def run(testName, buildDir, sanitizer, xmlOutput, testRunName, buildOnly, useLLDB):
|
||||
'''Main driver. Run cmake, compiles, execute and validate the testsuite.'''
|
||||
|
||||
# Override CXX for Linux
|
||||
os.environ['CXX'] = "/usr/bin/clang++ --std=c++14 --stdlib=libc++"
|
||||
|
||||
runCMake(sanitizer, buildDir)
|
||||
runCommand('make -C {} -j8'.format(buildDir))
|
||||
|
||||
if buildOnly:
|
||||
return
|
||||
|
||||
# A specific test case can be provided on the command line
|
||||
if testName:
|
||||
testNames = [testName]
|
||||
else:
|
||||
# Default case
|
||||
testNames = computeAllTestNames(buildDir)
|
||||
|
||||
# This should be empty. It is useful to have a blacklist during transitions
|
||||
# We could add something for asan as well.
|
||||
blackLists = {
|
||||
'ubsan': []
|
||||
}
|
||||
blackList = blackLists.get(sanitizer, [])
|
||||
|
||||
# Run through LLDB to capture crashes
|
||||
lldb = ''
|
||||
if useLLDB:
|
||||
lldb = "lldb --batch -o 'run' -k 'thread backtrace all' -k 'quit 1'"
|
||||
|
||||
# Jobs is a list of python dicts
|
||||
jobs = []
|
||||
|
||||
for testName in testNames:
|
||||
outputPath = tempfile.mktemp(suffix=testName + '.log')
|
||||
|
||||
if testName in blackList:
|
||||
log('Skipping blacklisted test {}'.format(testName), 'yellow')
|
||||
continue
|
||||
|
||||
# testName can contains spaces, so we enclose them in double quotes
|
||||
executable = os.path.join(buildDir, DEFAULT_EXE)
|
||||
|
||||
cmd = '{} "{}" "{}" >& "{}"'.format(lldb, executable, testName, outputPath)
|
||||
|
||||
jobs.append({
|
||||
'name': testName,
|
||||
'cmd': cmd,
|
||||
'output_path': outputPath,
|
||||
'use_lldb': useLLDB
|
||||
})
|
||||
|
||||
start = time.time()
|
||||
results = executeJobs(jobs)
|
||||
runTime = time.time() - start
|
||||
generateXmlOutput(results, xmlOutput, testRunName, runTime)
|
||||
|
||||
# Validate and report results
|
||||
print('\nParsing junit test result file: {}'.format(xmlOutput))
|
||||
log('## Results', 'blue')
|
||||
success, tests = validateTestSuite(xmlOutput)
|
||||
|
||||
if success:
|
||||
label = 'tests' if int(tests) > 1 else 'test'
|
||||
msg = 'All test passed (#{} {})'.format(tests, label)
|
||||
color = 'green'
|
||||
else:
|
||||
msg = 'unittest failed'
|
||||
color = 'red'
|
||||
|
||||
log(msg, color)
|
||||
log('Execution time: %.2fs' % (runTime), 'blue')
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
def main():
|
||||
buildDir = 'build/' + platform.system()
|
||||
if not os.path.exists(buildDir):
|
||||
os.makedirs(buildDir)
|
||||
|
||||
defaultOutput = DEFAULT_EXE + '.xml'
|
||||
|
||||
parser = argparse.ArgumentParser(description='Build and Run the engine unittest')
|
||||
|
||||
sanitizers = ['tsan', 'asan', 'ubsan', 'none']
|
||||
|
||||
parser.add_argument('--sanitizer', choices=sanitizers,
|
||||
help='Run a clang sanitizer.')
|
||||
parser.add_argument('--test', '-t', help='Test name.')
|
||||
parser.add_argument('--list', '-l', action='store_true',
|
||||
help='Print test names and exit.')
|
||||
parser.add_argument('--no_sanitizer', action='store_true',
|
||||
help='Do not execute a clang sanitizer.')
|
||||
parser.add_argument('--validate', action='store_true',
|
||||
help='Validate XML output.')
|
||||
parser.add_argument('--build_only', '-b', action='store_true',
|
||||
help='Stop after building. Do not run the unittest.')
|
||||
parser.add_argument('--output', '-o', help='Output XML file.')
|
||||
parser.add_argument('--lldb', action='store_true',
|
||||
help='Run the test through lldb.')
|
||||
parser.add_argument('--run_name', '-n',
|
||||
help='Name of the test run.')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Default sanitizer is tsan
|
||||
sanitizer = args.sanitizer
|
||||
if args.sanitizer is None:
|
||||
sanitizer = 'tsan'
|
||||
|
||||
defaultRunName = 'ixengine_{}_{}'.format(platform.system(), sanitizer)
|
||||
|
||||
xmlOutput = args.output or defaultOutput
|
||||
testRunName = args.run_name or os.getenv('IXENGINE_TEST_RUN_NAME') or defaultRunName
|
||||
|
||||
if args.list:
|
||||
# catch2 exit with a different error code when requesting the list of files
|
||||
try:
|
||||
runTest('--list-test-names-only', buildDir, xmlOutput, testRunName)
|
||||
except AssertionError:
|
||||
pass
|
||||
return
|
||||
|
||||
if args.validate:
|
||||
validateTestSuite(xmlOutput)
|
||||
return
|
||||
|
||||
if platform.system() != 'Darwin' and args.lldb:
|
||||
print('LLDB is only supported on Apple at this point')
|
||||
args.lldb = False
|
||||
|
||||
return run(args.test, buildDir, sanitizer, xmlOutput,
|
||||
testRunName, args.build_only, args.lldb)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user