15129044dSC.J. Collier#!/usr/bin/python
25129044dSC.J. Collier
35129044dSC.J. Collier#   BSD LICENSE
45129044dSC.J. Collier#
55129044dSC.J. Collier#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
65129044dSC.J. Collier#   All rights reserved.
75129044dSC.J. Collier#
85129044dSC.J. Collier#   Redistribution and use in source and binary forms, with or without
95129044dSC.J. Collier#   modification, are permitted provided that the following conditions
105129044dSC.J. Collier#   are met:
115129044dSC.J. Collier#
125129044dSC.J. Collier#     * Redistributions of source code must retain the above copyright
135129044dSC.J. Collier#       notice, this list of conditions and the following disclaimer.
145129044dSC.J. Collier#     * Redistributions in binary form must reproduce the above copyright
155129044dSC.J. Collier#       notice, this list of conditions and the following disclaimer in
165129044dSC.J. Collier#       the documentation and/or other materials provided with the
175129044dSC.J. Collier#       distribution.
185129044dSC.J. Collier#     * Neither the name of Intel Corporation nor the names of its
195129044dSC.J. Collier#       contributors may be used to endorse or promote products derived
205129044dSC.J. Collier#       from this software without specific prior written permission.
215129044dSC.J. Collier#
225129044dSC.J. Collier#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
235129044dSC.J. Collier#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
245129044dSC.J. Collier#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
255129044dSC.J. Collier#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
265129044dSC.J. Collier#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
275129044dSC.J. Collier#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
285129044dSC.J. Collier#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
295129044dSC.J. Collier#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
305129044dSC.J. Collier#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
315129044dSC.J. Collier#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
325129044dSC.J. Collier#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
335129044dSC.J. Collier
345129044dSC.J. Collier# The main logic behind running autotests in parallel
355129044dSC.J. Collier
369ecc306dSRicardo Salvetiimport multiprocessing, subprocess, sys, pexpect, re, time, os, StringIO, csv
375129044dSC.J. Collier
385129044dSC.J. Collier# wait for prompt
395129044dSC.J. Collierdef wait_prompt(child):
405129044dSC.J. Collier	try:
415129044dSC.J. Collier		child.sendline()
425129044dSC.J. Collier		result = child.expect(["RTE>>", pexpect.TIMEOUT, pexpect.EOF],
435129044dSC.J. Collier			timeout = 120)
445129044dSC.J. Collier	except:
455129044dSC.J. Collier		return False
465129044dSC.J. Collier	if result == 0:
475129044dSC.J. Collier		return True
485129044dSC.J. Collier	else:
495129044dSC.J. Collier		return False
505129044dSC.J. Collier
515129044dSC.J. Collier# run a test group
525129044dSC.J. Collier# each result tuple in results list consists of:
535129044dSC.J. Collier#   result value (0 or -1)
545129044dSC.J. Collier#   result string
555129044dSC.J. Collier#   test name
565129044dSC.J. Collier#   total test run time (double)
575129044dSC.J. Collier#   raw test log
585129044dSC.J. Collier#   test report (if not available, should be None)
595129044dSC.J. Collier#
605129044dSC.J. Collier# this function needs to be outside AutotestRunner class
615129044dSC.J. Collier# because otherwise Pool won't work (or rather it will require
625129044dSC.J. Collier# quite a bit of effort to make it work).
635129044dSC.J. Collierdef run_test_group(cmdline, test_group):
645129044dSC.J. Collier	results = []
655129044dSC.J. Collier	child = None
665129044dSC.J. Collier	start_time = time.time()
675129044dSC.J. Collier	startuplog = None
685129044dSC.J. Collier
695129044dSC.J. Collier	# run test app
705129044dSC.J. Collier	try:
715129044dSC.J. Collier		# prepare logging of init
725129044dSC.J. Collier		startuplog = StringIO.StringIO()
735129044dSC.J. Collier
745129044dSC.J. Collier		print >>startuplog, "\n%s %s\n" % ("="*20, test_group["Prefix"])
755129044dSC.J. Collier		print >>startuplog, "\ncmdline=%s" % cmdline
765129044dSC.J. Collier
775129044dSC.J. Collier		child = pexpect.spawn(cmdline, logfile=startuplog)
785129044dSC.J. Collier
795129044dSC.J. Collier		# wait for target to boot
805129044dSC.J. Collier		if not wait_prompt(child):
815129044dSC.J. Collier			child.close()
825129044dSC.J. Collier
835129044dSC.J. Collier			results.append((-1, "Fail [No prompt]", "Start %s" % test_group["Prefix"],
845129044dSC.J. Collier				time.time() - start_time, startuplog.getvalue(), None))
855129044dSC.J. Collier
865129044dSC.J. Collier			# mark all tests as failed
875129044dSC.J. Collier			for test in test_group["Tests"]:
885129044dSC.J. Collier				results.append((-1, "Fail [No prompt]", test["Name"],
895129044dSC.J. Collier				time.time() - start_time, "", None))
905129044dSC.J. Collier			# exit test
915129044dSC.J. Collier			return results
925129044dSC.J. Collier
935129044dSC.J. Collier	except:
945129044dSC.J. Collier		results.append((-1, "Fail [Can't run]", "Start %s" % test_group["Prefix"],
955129044dSC.J. Collier				time.time() - start_time, startuplog.getvalue(), None))
965129044dSC.J. Collier
975129044dSC.J. Collier		# mark all tests as failed
985129044dSC.J. Collier		for t in test_group["Tests"]:
995129044dSC.J. Collier			results.append((-1, "Fail [Can't run]", t["Name"],
1005129044dSC.J. Collier				time.time() - start_time, "", None))
1015129044dSC.J. Collier		# exit test
1025129044dSC.J. Collier		return results
1035129044dSC.J. Collier
1045129044dSC.J. Collier	# startup was successful
1055129044dSC.J. Collier	results.append((0, "Success", "Start %s" % test_group["Prefix"],
1065129044dSC.J. Collier		time.time() - start_time, startuplog.getvalue(), None))
1075129044dSC.J. Collier
1089ecc306dSRicardo Salveti	# parse the binary for available test commands
1099ecc306dSRicardo Salveti	binary = cmdline.split()[0]
1105b1ff351SRicardo Salveti	stripped = 'not stripped' not in subprocess.check_output(['file', binary])
1115b1ff351SRicardo Salveti	if not stripped:
1125b1ff351SRicardo Salveti		symbols = subprocess.check_output(['nm', binary]).decode('utf-8')
1135b1ff351SRicardo Salveti		avail_cmds = re.findall('test_register_(\w+)', symbols)
1149ecc306dSRicardo Salveti
1155129044dSC.J. Collier	# run all tests in test group
1165129044dSC.J. Collier	for test in test_group["Tests"]:
1175129044dSC.J. Collier
1185129044dSC.J. Collier		# create log buffer for each test
1195129044dSC.J. Collier		# in multiprocessing environment, the logging would be
1205129044dSC.J. Collier		# interleaved and will create a mess, hence the buffering
1215129044dSC.J. Collier		logfile = StringIO.StringIO()
1225129044dSC.J. Collier		child.logfile = logfile
1235129044dSC.J. Collier
1245129044dSC.J. Collier		result = ()
1255129044dSC.J. Collier
1265129044dSC.J. Collier		# make a note when the test started
1275129044dSC.J. Collier		start_time = time.time()
1285129044dSC.J. Collier
1295129044dSC.J. Collier		try:
1305129044dSC.J. Collier			# print test name to log buffer
1315129044dSC.J. Collier			print >>logfile, "\n%s %s\n" % ("-"*20, test["Name"])
1325129044dSC.J. Collier
1335129044dSC.J. Collier			# run test function associated with the test
1345b1ff351SRicardo Salveti			if stripped or test["Command"] in avail_cmds:
1359ecc306dSRicardo Salveti				result = test["Func"](child, test["Command"])
1369ecc306dSRicardo Salveti			else:
1379ecc306dSRicardo Salveti				result = (0, "Skipped [Not Available]")
1385129044dSC.J. Collier
1395129044dSC.J. Collier			# make a note when the test was finished
1405129044dSC.J. Collier			end_time = time.time()
1415129044dSC.J. Collier
1425129044dSC.J. Collier			# append test data to the result tuple
1435129044dSC.J. Collier			result += (test["Name"], end_time - start_time,
1445129044dSC.J. Collier				logfile.getvalue())
1455129044dSC.J. Collier
1465129044dSC.J. Collier			# call report function, if any defined, and supply it with
1475129044dSC.J. Collier			# target and complete log for test run
1485129044dSC.J. Collier			if test["Report"]:
1495129044dSC.J. Collier				report = test["Report"](self.target, log)
1505129044dSC.J. Collier
1515129044dSC.J. Collier				# append report to results tuple
1525129044dSC.J. Collier				result += (report,)
1535129044dSC.J. Collier			else:
1545129044dSC.J. Collier				# report is None
1555129044dSC.J. Collier				result += (None,)
1565129044dSC.J. Collier		except:
1575129044dSC.J. Collier			# make a note when the test crashed
1585129044dSC.J. Collier			end_time = time.time()
1595129044dSC.J. Collier
1605129044dSC.J. Collier			# mark test as failed
1615129044dSC.J. Collier			result = (-1, "Fail [Crash]", test["Name"],
1625129044dSC.J. Collier				end_time - start_time, logfile.getvalue(), None)
1635129044dSC.J. Collier		finally:
1645129044dSC.J. Collier			# append the results to the results list
1655129044dSC.J. Collier			results.append(result)
1665129044dSC.J. Collier
1675129044dSC.J. Collier	# regardless of whether test has crashed, try quitting it
1685129044dSC.J. Collier	try:
1695129044dSC.J. Collier		child.sendline("quit")
1705129044dSC.J. Collier		child.close()
1715129044dSC.J. Collier	# if the test crashed, just do nothing instead
1725129044dSC.J. Collier	except:
1735129044dSC.J. Collier		# nop
1745129044dSC.J. Collier		pass
1755129044dSC.J. Collier
1765129044dSC.J. Collier	# return test results
1775129044dSC.J. Collier	return results
1785129044dSC.J. Collier
1795129044dSC.J. Collier
1805129044dSC.J. Collier
1815129044dSC.J. Collier
1825129044dSC.J. Collier
1835129044dSC.J. Collier# class representing an instance of autotests run
1845129044dSC.J. Collierclass AutotestRunner:
1855129044dSC.J. Collier	cmdline = ""
1865129044dSC.J. Collier	parallel_test_groups = []
1875129044dSC.J. Collier	non_parallel_test_groups = []
1885129044dSC.J. Collier	logfile = None
1895129044dSC.J. Collier	csvwriter = None
1905129044dSC.J. Collier	target = ""
1915129044dSC.J. Collier	start = None
1925129044dSC.J. Collier	n_tests = 0
1935129044dSC.J. Collier	fails = 0
1945129044dSC.J. Collier	log_buffers = []
1955129044dSC.J. Collier	blacklist = []
1965129044dSC.J. Collier	whitelist = []
1975129044dSC.J. Collier
1985129044dSC.J. Collier
1995129044dSC.J. Collier	def __init__(self, cmdline, target, blacklist, whitelist):
2005129044dSC.J. Collier		self.cmdline = cmdline
2015129044dSC.J. Collier		self.target = target
2025129044dSC.J. Collier		self.blacklist = blacklist
2035129044dSC.J. Collier		self.whitelist = whitelist
2045129044dSC.J. Collier
2055129044dSC.J. Collier		# log file filename
2065129044dSC.J. Collier		logfile = "%s.log" % target
2075129044dSC.J. Collier		csvfile = "%s.csv" % target
2085129044dSC.J. Collier
2095129044dSC.J. Collier		self.logfile = open(logfile, "w")
2105129044dSC.J. Collier		csvfile = open(csvfile, "w")
2115129044dSC.J. Collier		self.csvwriter = csv.writer(csvfile)
2125129044dSC.J. Collier
2135129044dSC.J. Collier		# prepare results table
2145129044dSC.J. Collier		self.csvwriter.writerow(["test_name","test_result","result_str"])
2155129044dSC.J. Collier
2165129044dSC.J. Collier
2175129044dSC.J. Collier
2185129044dSC.J. Collier	# set up cmdline string
2195129044dSC.J. Collier	def __get_cmdline(self, test):
2205129044dSC.J. Collier		cmdline = self.cmdline
2215129044dSC.J. Collier
2225129044dSC.J. Collier		# append memory limitations for each test
2235129044dSC.J. Collier		# otherwise tests won't run in parallel
2245129044dSC.J. Collier		if not "i686" in self.target:
2255129044dSC.J. Collier			cmdline += " --socket-mem=%s"% test["Memory"]
2265129044dSC.J. Collier		else:
2275129044dSC.J. Collier			# affinitize startup so that tests don't fail on i686
2285129044dSC.J. Collier			cmdline = "taskset 1 " + cmdline
2295129044dSC.J. Collier			cmdline += " -m " + str(sum(map(int,test["Memory"].split(","))))
2305129044dSC.J. Collier
2315129044dSC.J. Collier		# set group prefix for autotest group
2325129044dSC.J. Collier		# otherwise they won't run in parallel
2335129044dSC.J. Collier		cmdline += " --file-prefix=%s"% test["Prefix"]
2345129044dSC.J. Collier
2355129044dSC.J. Collier		return cmdline
2365129044dSC.J. Collier
2375129044dSC.J. Collier
2385129044dSC.J. Collier
2395129044dSC.J. Collier	def add_parallel_test_group(self,test_group):
2405129044dSC.J. Collier		self.parallel_test_groups.append(test_group)
2415129044dSC.J. Collier
2425129044dSC.J. Collier	def add_non_parallel_test_group(self,test_group):
2435129044dSC.J. Collier		self.non_parallel_test_groups.append(test_group)
2445129044dSC.J. Collier
2455129044dSC.J. Collier
2465129044dSC.J. Collier	def __process_results(self, results):
2475129044dSC.J. Collier		# this iterates over individual test results
2485129044dSC.J. Collier		for i, result in enumerate(results):
2495129044dSC.J. Collier
2505129044dSC.J. Collier			# increase total number of tests that were run
2515129044dSC.J. Collier			# do not include "start" test
2525129044dSC.J. Collier			if i > 0:
2535129044dSC.J. Collier				self.n_tests += 1
2545129044dSC.J. Collier
2555129044dSC.J. Collier			# unpack result tuple
2565129044dSC.J. Collier			test_result, result_str, test_name, \
2575129044dSC.J. Collier				test_time, log, report = result
2585129044dSC.J. Collier
2595129044dSC.J. Collier			# get total run time
2605129044dSC.J. Collier			cur_time = time.time()
2615129044dSC.J. Collier			total_time = int(cur_time - self.start)
2625129044dSC.J. Collier
2635129044dSC.J. Collier			# print results, test run time and total time since start
2645129044dSC.J. Collier			print ("%s:" % test_name).ljust(30),
2655129044dSC.J. Collier			print result_str.ljust(29),
2665129044dSC.J. Collier			print "[%02dm %02ds]" % (test_time / 60, test_time % 60),
2675129044dSC.J. Collier
2685129044dSC.J. Collier			# don't print out total time every line, it's the same anyway
2695129044dSC.J. Collier			if i == len(results) - 1:
2705129044dSC.J. Collier				print "[%02dm %02ds]" % (total_time / 60, total_time % 60)
2715129044dSC.J. Collier			else:
2725129044dSC.J. Collier				print ""
2735129044dSC.J. Collier
2745129044dSC.J. Collier			# if test failed and it wasn't a "start" test
2755129044dSC.J. Collier			if test_result < 0 and not i == 0:
2765129044dSC.J. Collier				self.fails += 1
2775129044dSC.J. Collier
2785129044dSC.J. Collier			# collect logs
2795129044dSC.J. Collier			self.log_buffers.append(log)
2805129044dSC.J. Collier
2815129044dSC.J. Collier			# create report if it exists
2825129044dSC.J. Collier			if report:
2835129044dSC.J. Collier				try:
2845129044dSC.J. Collier					f = open("%s_%s_report.rst" % (self.target,test_name), "w")
2855129044dSC.J. Collier				except IOError:
2865129044dSC.J. Collier					print "Report for %s could not be created!" % test_name
2875129044dSC.J. Collier				else:
2885129044dSC.J. Collier					with f:
2895129044dSC.J. Collier						f.write(report)
2905129044dSC.J. Collier
2915129044dSC.J. Collier			# write test result to CSV file
2925129044dSC.J. Collier			if i != 0:
2935129044dSC.J. Collier				self.csvwriter.writerow([test_name, test_result, result_str])
2945129044dSC.J. Collier
2955129044dSC.J. Collier
2965129044dSC.J. Collier
2975129044dSC.J. Collier
2985129044dSC.J. Collier	# this function iterates over test groups and removes each
2995129044dSC.J. Collier	# test that is not in whitelist/blacklist
3005129044dSC.J. Collier	def __filter_groups(self, test_groups):
3015129044dSC.J. Collier		groups_to_remove = []
3025129044dSC.J. Collier
3035129044dSC.J. Collier		# filter out tests from parallel test groups
3045129044dSC.J. Collier		for i, test_group in enumerate(test_groups):
3055129044dSC.J. Collier
3065129044dSC.J. Collier			# iterate over a copy so that we could safely delete individual tests
3075129044dSC.J. Collier			for test in test_group["Tests"][:]:
3085129044dSC.J. Collier				test_id = test["Command"]
3095129044dSC.J. Collier
3105129044dSC.J. Collier				# dump tests are specified in full e.g. "Dump_mempool"
3115129044dSC.J. Collier				if "_autotest" in test_id:
3125129044dSC.J. Collier					test_id = test_id[:-len("_autotest")]
3135129044dSC.J. Collier
3145129044dSC.J. Collier				# filter out blacklisted/whitelisted tests
3155129044dSC.J. Collier				if self.blacklist and test_id in self.blacklist:
3165129044dSC.J. Collier					test_group["Tests"].remove(test)
3175129044dSC.J. Collier					continue
3185129044dSC.J. Collier				if self.whitelist and test_id not in self.whitelist:
3195129044dSC.J. Collier					test_group["Tests"].remove(test)
3205129044dSC.J. Collier					continue
3215129044dSC.J. Collier
3225129044dSC.J. Collier			# modify or remove original group
3235129044dSC.J. Collier			if len(test_group["Tests"]) > 0:
3245129044dSC.J. Collier				test_groups[i] = test_group
3255129044dSC.J. Collier			else:
3265129044dSC.J. Collier				# remember which groups should be deleted
3275129044dSC.J. Collier				# put the numbers backwards so that we start
3285129044dSC.J. Collier				# deleting from the end, not from the beginning
3295129044dSC.J. Collier				groups_to_remove.insert(0, i)
3305129044dSC.J. Collier
3315129044dSC.J. Collier		# remove test groups that need to be removed
3325129044dSC.J. Collier		for i in groups_to_remove:
3335129044dSC.J. Collier			del test_groups[i]
3345129044dSC.J. Collier
3355129044dSC.J. Collier		return test_groups
3365129044dSC.J. Collier
3375129044dSC.J. Collier
3385129044dSC.J. Collier
3395129044dSC.J. Collier	# iterate over test groups and run tests associated with them
3405129044dSC.J. Collier	def run_all_tests(self):
3415129044dSC.J. Collier		# filter groups
3425129044dSC.J. Collier		self.parallel_test_groups = \
3435129044dSC.J. Collier			self.__filter_groups(self.parallel_test_groups)
3445129044dSC.J. Collier		self.non_parallel_test_groups = \
3455129044dSC.J. Collier			self.__filter_groups(self.non_parallel_test_groups)
3465129044dSC.J. Collier
3475129044dSC.J. Collier		# create a pool of worker threads
3485129044dSC.J. Collier		pool = multiprocessing.Pool(processes=1)
3495129044dSC.J. Collier
3505129044dSC.J. Collier		results = []
3515129044dSC.J. Collier
3525129044dSC.J. Collier		# whatever happens, try to save as much logs as possible
3535129044dSC.J. Collier		try:
3545129044dSC.J. Collier
3555129044dSC.J. Collier			# create table header
3565129044dSC.J. Collier			print ""
3575129044dSC.J. Collier			print "Test name".ljust(30),
3585129044dSC.J. Collier			print "Test result".ljust(29),
3595129044dSC.J. Collier			print "Test".center(9),
3605129044dSC.J. Collier			print "Total".center(9)
3615129044dSC.J. Collier			print "=" * 80
3625129044dSC.J. Collier
3635129044dSC.J. Collier			# make a note of tests start time
3645129044dSC.J. Collier			self.start = time.time()
3655129044dSC.J. Collier
3665129044dSC.J. Collier			# assign worker threads to run test groups
3675129044dSC.J. Collier			for test_group in self.parallel_test_groups:
3685129044dSC.J. Collier				result = pool.apply_async(run_test_group,
3695129044dSC.J. Collier					[self.__get_cmdline(test_group), test_group])
3705129044dSC.J. Collier				results.append(result)
3715129044dSC.J. Collier
3725129044dSC.J. Collier			# iterate while we have group execution results to get
3735129044dSC.J. Collier			while len(results) > 0:
3745129044dSC.J. Collier
3755129044dSC.J. Collier				# iterate over a copy to be able to safely delete results
3765129044dSC.J. Collier				# this iterates over a list of group results
3775129044dSC.J. Collier				for group_result in results[:]:
3785129044dSC.J. Collier
3795129044dSC.J. Collier					# if the thread hasn't finished yet, continue
3805129044dSC.J. Collier					if not group_result.ready():
3815129044dSC.J. Collier						continue
3825129044dSC.J. Collier
3835129044dSC.J. Collier					res = group_result.get()
3845129044dSC.J. Collier
3855129044dSC.J. Collier					self.__process_results(res)
3865129044dSC.J. Collier
3875129044dSC.J. Collier					# remove result from results list once we're done with it
3885129044dSC.J. Collier					results.remove(group_result)
3895129044dSC.J. Collier
3905129044dSC.J. Collier			# run non_parallel tests. they are run one by one, synchronously
3915129044dSC.J. Collier			for test_group in self.non_parallel_test_groups:
3925129044dSC.J. Collier				group_result = run_test_group(self.__get_cmdline(test_group), test_group)
3935129044dSC.J. Collier
3945129044dSC.J. Collier				self.__process_results(group_result)
3955129044dSC.J. Collier
3965129044dSC.J. Collier			# get total run time
3975129044dSC.J. Collier			cur_time = time.time()
3985129044dSC.J. Collier			total_time = int(cur_time - self.start)
3995129044dSC.J. Collier
4005129044dSC.J. Collier			# print out summary
4015129044dSC.J. Collier			print "=" * 80
4025129044dSC.J. Collier			print "Total run time: %02dm %02ds" % (total_time / 60, total_time % 60)
4035129044dSC.J. Collier			if self.fails != 0:
4045129044dSC.J. Collier				print "Number of failed tests: %s" % str(self.fails)
4055129044dSC.J. Collier
4065129044dSC.J. Collier			# write summary to logfile
4075129044dSC.J. Collier			self.logfile.write("Summary\n")
4085129044dSC.J. Collier			self.logfile.write("Target: ".ljust(15) + "%s\n" % self.target)
4095129044dSC.J. Collier			self.logfile.write("Tests: ".ljust(15) + "%i\n" % self.n_tests)
4105129044dSC.J. Collier			self.logfile.write("Failed tests: ".ljust(15) + "%i\n" % self.fails)
4115129044dSC.J. Collier		except:
4125129044dSC.J. Collier			print "Exception occured"
4135129044dSC.J. Collier			print sys.exc_info()
4145129044dSC.J. Collier			self.fails = 1
4155129044dSC.J. Collier
4165129044dSC.J. Collier		# drop logs from all executions to a logfile
4175129044dSC.J. Collier		for buf in self.log_buffers:
4185129044dSC.J. Collier			self.logfile.write(buf.replace("\r",""))
4195129044dSC.J. Collier
4205129044dSC.J. Collier		log_buffers = []
4215129044dSC.J. Collier
4225129044dSC.J. Collier		return self.fails
423