Github_Repos/cvw
1
0
Fork 1
mirror of https://github.com/openhwgroup/cvw synced 2025-02-02 17:55:19 +00:00
Code Issues Packages Projects Releases Wiki Activity
19a61e301e
cvw/sim/regression-wally
Kevin Kim 19a61e301e formatting
2024-02-20 17:24:15 -08:00

455 lines
21 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python3
##################################
#
# regression-wally
# David_Harris@Hmc.edu 25 January 2021
# Modified by Jarred Allen <jaallen@g.hmc.edu>
#
# Run a regression with multiple configurations in parallel and exit with
# non-zero status code if an error happened, as well as printing human-readable
# output.
#
##################################
import sys,os,shutil
import multiprocessing
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
from collections import namedtuple
regressionDir = os.path.dirname(os.path.abspath(__file__))
os.chdir(regressionDir)
coverage = '-coverage' in sys.argv
fp = '-fp' in sys.argv
nightly = '-nightly' in sys.argv
softfloat = '-softfloat' in sys.argv
TestCase = namedtuple("TestCase", ['name', 'variant', 'cmd', 'grepstr'])
# name: the name of this test configuration (used in printing human-readable
# output and picking logfile names)
# cmd: the command to run to test (should include the logfile as '{}', and
# the command needs to write to that file)
# grepstr: the string to grep through the log file for. The test succeeds iff
# grep finds that string in the logfile (is used by grep, so it may
# be any pattern grep accepts, see `man 1 grep` for more info).
# edit this list to add more test cases
if (nightly):
nightMode = "-nightly";
configs = []
else:
nightMode = "";
configs = [
TestCase(
name="lints",
variant="all",
cmd="./lint-wally " + nightMode + " | tee {}",
grepstr="lints run with no errors or warnings"
)
]
def getBuildrootTC(boot):
INSTR_LIMIT = 1000000 # multiple of 100000; 4M is interesting because it gets into the kernel and enabling VM
MAX_EXPECTED = 246000000 # *** TODO: replace this with a search for the login prompt.
if boot:
name="buildrootboot"
BRcmd="vsim > {} -c <<!\ndo wally.do buildroot buildroot-no-trace $RISCV 0 1 0\n!"
BRgrepstr="WallyHostname login:"
else:
name="buildroot"
if (coverage):
print( "buildroot coverage")
BRcmd="vsim > {} -c <<!\ndo wally-batch.do buildroot buildroot $RISCV "+str(INSTR_LIMIT)+" 1 0 -coverage\n!"
else:
print( "buildroot no coverage")
BRcmd="vsim > {} -c <<!\ndo wally-batch.do buildroot buildroot configOptions -GINSTR_LIMIT=" +str(INSTR_LIMIT) + " \n!"
BRgrepstr=str(INSTR_LIMIT)+" instructions"
return TestCase(name,variant="rv64gc",cmd=BRcmd,grepstr=BRgrepstr)
tests64gcimperas = ["imperas64i", "imperas64f", "imperas64d", "imperas64m", "imperas64c"] # unused
tests64i = ["arch64i"]
for test in tests64i:
tc = TestCase(
name=test,
variant="rv64i",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv64i "+test+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
tests32gcimperas = ["imperas32i", "imperas32f", "imperas32m", "imperas32c"] # unused
tests32gc = ["arch32f", "arch32d", "arch32f_fma", "arch32d_fma", "arch32f_divsqrt", "arch32d_divsqrt", "arch32i", "arch32priv", "arch32c", "arch32m", "arch32a", "arch32zifencei", "arch32zicond", "arch32zba", "arch32zbb", "arch32zbs", "arch32zfh", "arch32zfh_fma", "arch32zfh_divsqrt", "arch32zfaf", "wally32a", "wally32priv", "wally32periph"] # "arch32zbc", "arch32zfad",
#tests32gc = ["arch32f", "arch32d", "arch32f_fma", "arch32d_fma", "arch32i", "arch32priv", "arch32c", "arch32m", "arch32a", "arch32zifencei", "arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zicboz", "arch32zcb", "wally32a", "wally32priv", "wally32periph"]
for test in tests32gc:
tc = TestCase(
name=test,
variant="rv32gc",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32gc "+test+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
tests32imcimperas = ["imperas32i", "imperas32c"] # unused
tests32imc = ["arch32i", "arch32c", "arch32m", "wally32periph"]
for test in tests32imc:
tc = TestCase(
name=test,
variant="rv32imc",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32imc "+test+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
tests32i = ["arch32i"]
for test in tests32i:
tc = TestCase(
name=test,
variant="rv32i",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32i "+test+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
tests32e = ["arch32e"]
for test in tests32e:
tc = TestCase(
name=test,
variant="rv32e",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32e "+test+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
tests64gc = ["arch64f", "arch64d", "arch64f_fma", "arch64d_fma", "arch64f_divsqrt", "arch64d_divsqrt", "arch64i", "arch64zba", "arch64zbb", "arch64zbc", "arch64zbs", "arch64zfh", "arch64zfh_divsqrt", "arch64zfh_fma", "arch64zfaf", "arch64zfad",
"arch64priv", "arch64c", "arch64m", "arch64a", "arch64zifencei", "arch64zicond", "wally64a", "wally64periph", "wally64priv"] # add arch64zfh_fma when available; arch64zicobz, arch64zcb when working
#tests64gc = ["arch64f", "arch64d", "arch64f_fma", "arch64d_fma", "arch64i", "arch64zba", "arch64zbb", "arch64zbc", "arch64zbs",
# "arch64priv", "arch64c", "arch64m", "arch64a", "arch64zifencei", "wally64a", "wally64periph", "wally64priv", "arch64zicboz", "arch64zcb"]
if (coverage): # delete all but 64gc tests when running coverage
configs = []
tests64gc = ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m",
"arch64zifencei", "arch64zicond", "arch64a", "wally64a", "wally64periph", "wally64priv",
"arch64zba", "arch64zbb", "arch64zbc", "arch64zbs"] # add when working: "arch64zcb", "arch64zicboz"
if (fp):
tests64gc.append("arch64f")
tests64gc.append("arch64d")
tests64gc.append("arch64zfh")
tests64gc.append("arch64f_fma")
tests64gc.append("arch64d_fma")
tests64gc.append("arch64zfh_fma")
tests64gc.append("arch64f_divsqrt")
tests64gc.append("arch64d_divsqrt")
tests64gc.append("arch64zfh_divsqrt")
tests64gc.append("arch64zfaf")
tests64gc.append("arch64zfad")
coverStr = '-coverage'
else:
coverStr = ''
for test in tests64gc:
tc = TestCase(
name=test,
variant="rv64gc",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv64gc "+test+" " + coverStr + "\n!",
grepstr="All tests ran without failures")
configs.append(tc)
# run derivative configurations if requested
if (nightly):
derivconfigtests = [
["tlb2_rv32gc", ["wally32priv"]],
["tlb16_rv32gc", ["wally32priv"]],
["tlb2_rv64gc", ["wally64priv"]],
["tlb16_rv64gc", ["wally64priv"]],
["way_1_4096_512_rv32gc", ["arch32i"]],
["way_2_4096_512_rv32gc", ["arch32i"]],
["way_8_4096_512_rv32gc", ["arch32i"]],
["way_4_2048_512_rv32gc", ["arch32i"]],
["way_4_4096_256_rv32gc", ["arch32i"]],
["way_1_4096_512_rv64gc", ["arch64i"]],
["way_2_4096_512_rv64gc", ["arch64i"]],
["way_8_4096_512_rv64gc", ["arch64i"]],
["way_4_2048_512_rv64gc", ["arch64i"]],
["way_4_4096_256_rv64gc", ["arch64i"]],
["way_4_4096_1024_rv64gc", ["arch64i"]],
["ram_0_0_rv64gc", ["ahb64"]],
["ram_1_0_rv64gc", ["ahb64"]],
["ram_1_1_rv64gc", ["ahb64"]],
["ram_2_0_rv64gc", ["ahb64"]],
["ram_2_1_rv64gc", ["ahb64"]],
["noicache_rv32gc", ["ahb32"]],
# cacheless designs will not work until DTIM supports FLEN > XLEN
# ["nodcache_rv32gc", ["ahb32"]],
# ["nocache_rv32gc", ["ahb32"]],
["noicache_rv64gc", ["ahb64"]],
["nodcache_rv64gc", ["ahb64"]],
["nocache_rv64gc", ["ahb64"]],
### add misaligned tests
["div_2_1_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_2_1i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_2_2_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_2_2i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_2_4_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_2_4i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_4_1_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_4_1i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_4_2_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_4_2i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_4_4_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_4_4i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]],
["div_2_1_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_2_1i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_2_2_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_2_2i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_2_4_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_2_4i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_4_1_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_4_1i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_4_2_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_4_2i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_4_4_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
["div_4_4i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]],
### branch predictor simulation
["bpred_TWOBIT_6_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_8_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_10_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_12_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_14_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_16_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_6_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_8_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_10_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_12_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_14_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_TWOBIT_16_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_6_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_6_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_8_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_8_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_12_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_12_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_14_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_14_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_16_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_16_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
# btb
["bpred_GSHARE_10_16_6_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_6_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_8_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_8_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_12_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_16_12_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
# ras
["bpred_GSHARE_10_2_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_2_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_3_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_3_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_4_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_4_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_6_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_6_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_10_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
["bpred_GSHARE_10_10_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"],
# enable floating-point tests when lint is fixed
["f_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma"]],
["fh_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32zfh", "arch32zfh_divsqrt"]],
["fdh_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32d", "arch32d_divsqrt", "arch32d_fma", "arch32zfh", "arch32zfh_divsqrt"]],
["fdq_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32d", "arch32d_divsqrt", "arch32d_fma", "arch32zfh", "arch32zfh_divsqrt"]],
["fdqh_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32d", "arch32d_divsqrt", "arch32d_fma", "arch32zfh", "arch32zfh_divsqrt"]],
["f_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma"]],
["fh_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64zfh", "arch64zfh_divsqrt"]], # hanging 1/31/24 dh; try again when lint is fixed
["fdh_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64d", "arch64d_divsqrt", "arch64d_fma", "arch64zfh", "arch64zfh_divsqrt"]],
["fdq_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64d", "arch64d_divsqrt", "arch64d_fma", "arch64zfh", "arch64zfh_divsqrt"]],
["fdqh_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64d", "arch64d_divsqrt", "arch64d_fma", "arch64zfh", "arch64zfh_divsqrt"]],
]
for test in derivconfigtests:
config = test[0];
tests = test[1];
if(len(test) >= 4 and test[2] == "configOptions"):
configOptions = test[3]
cmdPrefix = "vsim > {} -c <<!\ndo wally-batch.do "+config
else:
configOptions = ""
cmdPrefix = "vsim > {} -c <<!\ndo wally-batch.do "+config
for t in tests:
tc = TestCase(
name=t,
variant=config,
cmd=cmdPrefix+" "+t+" configOptions "+configOptions+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
tests32e = ["arch32e"]
for test in tests32e:
tc = TestCase(
name=test,
variant="rv32e",
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32e "+test+"\n!",
grepstr="All tests ran without failures")
configs.append(tc)
# softfloat tests
if (softfloat):
configs = []
softfloatconfigs = ['fdh_ieee_rv32gc', 'fdqh_ieee_rv32gc', 'fdq_ieee_rv32gc', \
'fh_ieee_v32gc', 'f_ieee_rv64gc', 'fdqh_ieee_rv64gc', \
'fdq_ieee_rv64gc', 'div_2_1_rv32gc', 'div_2_2_rv32gc', \
'div_2_4_rv32gc', 'div_4_1_rv32gc', 'div_4_2_rv32gc', \
'div_4_4_rv32gc', 'fd_ieee_rv32gc', 'fh_ieee_rv32gc', \
'div_2_1_rv64gc', 'div_2_2_rv64gc', 'div_2_4_rv64gc', \
'div_4_1_rv64gc', 'div_4_2_rv64gc', 'div_4_4_rv64gc', \
'fd_ieee_rv64gc', 'fh_ieee_rv64gc', 'f_ieee_rv32gc']
for config in softfloatconfigs:
# div test case
divtest = TestCase(
name="div",
variant=config,
cmd="vsim > {} -c <<!\ndo testfloat.do " + config + " div \n!",
grepstr="All Tests completed with 0 errors"
)
configs.append(divtest)
# sqrt test case
sqrttest = TestCase(
name="sqrt",
variant=config,
cmd="vsim > {} -c <<!\ndo testfloat.do " + config + " sqrt \n!",
grepstr="All Tests completed with 0 errors"
)
configs.append(sqrttest)
# skip if divider variant config
if ("ieee" in config[0]):
# cvtint test case
cvtinttest = TestCase(
name="cvtint",
variant=config,
cmd="vsim > {} -c <<!\ndo testfloat.do " + config + " cvtint \n!",
grepstr="All Tests completed with 0 errors"
)
configs.append(cvtinttest)
# cvtfp test case
cvtfptest = TestCase(
name="cvtfp",
variant=config,
cmd="vsim > {} -c <<!\ndo testfloat.do " + config + " cvtfp \n!",
grepstr="All Tests completed with 0 errors"
)
configs.append(cvtfptest)
import os
from multiprocessing import Pool, TimeoutError
def search_log_for_text(text, logfile):
"""Search through the given log file for text, returning True if it is found or False if it is not"""
grepcmd = "grep -e '%s' '%s' > /dev/null" % (text, logfile)
return os.system(grepcmd) == 0
def run_test_case(config):
"""Run the given test case, and return 0 if the test suceeds and 1 if it fails"""
logname = "logs/"+config.variant+"_"+config.name+".log"
cmd = config.cmd.format(logname)
# print(cmd)
os.chdir(regressionDir)
os.system(cmd)
if search_log_for_text(config.grepstr, logname):
print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name))
return 0
else:
print(f"{bcolors.FAIL}%s_%s: Failures detected in output{bcolors.ENDC}" % (config.variant, config.name))
print(" Check %s" % logname)
return 1
def main():
"""Run the tests and count the failures"""
global configs, coverage
try:
os.chdir(regressionDir)
os.mkdir("logs")
except:
pass
try:
shutil.rmtree("wkdir")
except:
pass
finally:
os.mkdir("wkdir")
if '-makeTests' in sys.argv:
os.chdir(regressionDir)
os.system('./make-tests.sh | tee ./logs/make-tests.log')
if '-all' in sys.argv:
TIMEOUT_DUR = 30*7200 # seconds
configs.append(getBuildrootTC(boot=True))
elif '-buildroot' in sys.argv:
TIMEOUT_DUR = 30*7200 # seconds
configs=[getBuildrootTC(boot=True)]
elif '-coverage' in sys.argv:
TIMEOUT_DUR = 20*60 # seconds
# Presently don't run buildroot because it has a different config and can't be merged with the rv64gc coverage.
# Also it is slow to run.
# configs.append(getBuildrootTC(boot=False))
os.system('rm -f cov/*.ucdb')
elif '-nightly' in sys.argv:
TIMEOUT_DUR = 60*1440 # 1 day
configs.append(getBuildrootTC(boot=False))
elif '-softfloat' in sys.argv:
TIMEOUT_DUR = 60*60 # seconds
else:
TIMEOUT_DUR = 10*60 # seconds
configs.append(getBuildrootTC(boot=False))
# Scale the number of concurrent processes to the number of test cases, but
# max out at a limited number of concurrent processes to not overwhelm the system
with Pool(processes=min(len(configs),multiprocessing.cpu_count())) as pool:
num_fail = 0
results = {}
for config in configs:
results[config] = pool.apply_async(run_test_case,(config,))
for (config,result) in results.items():
try:
num_fail+=result.get(timeout=TIMEOUT_DUR)
except TimeoutError:
num_fail+=1
print(f"{bcolors.FAIL}%s_%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.variant, config.name, TIMEOUT_DUR))
# Coverage report
if coverage:
os.system('make coverage')
# Count the number of failures
if num_fail:
print(f"{bcolors.FAIL}Regression failed with %s failed configurations{bcolors.ENDC}" % num_fail)
else:
print(f"{bcolors.OKGREEN}SUCCESS! All tests ran without failures{bcolors.ENDC}")
return num_fail
if __name__ == '__main__':
exit(main())
Reference in New Issue View Git Blame Copy Permalink