mirror of
https://github.com/openhwgroup/cvw
synced 2025-02-11 06:05:49 +00:00
243 lines
8.9 KiB
Python
Executable File
243 lines
8.9 KiB
Python
Executable File
#!/usr/bin/python3
|
|
##################################
|
|
#
|
|
# regression-wally
|
|
# David_Harris@Hmc.edu 25 January 2021
|
|
# Modified by Jarred Allen <jaallen@g.hmc.edu>
|
|
#
|
|
# Run a regression with multiple configurations in parallel and exit with
|
|
# non-zero status code if an error happened, as well as printing human-readable
|
|
# output.
|
|
#
|
|
##################################
|
|
import sys,os,shutil
|
|
|
|
class bcolors:
|
|
HEADER = '\033[95m'
|
|
OKBLUE = '\033[94m'
|
|
OKCYAN = '\033[96m'
|
|
OKGREEN = '\033[92m'
|
|
WARNING = '\033[93m'
|
|
FAIL = '\033[91m'
|
|
ENDC = '\033[0m'
|
|
BOLD = '\033[1m'
|
|
UNDERLINE = '\033[4m'
|
|
|
|
from collections import namedtuple
|
|
regressionDir = os.path.dirname(os.path.abspath(__file__))
|
|
os.chdir(regressionDir)
|
|
|
|
coverage = '-coverage' in sys.argv
|
|
fp = '-fp' in sys.argv
|
|
|
|
TestCase = namedtuple("TestCase", ['name', 'variant', 'cmd', 'grepstr'])
|
|
# name: the name of this test configuration (used in printing human-readable
|
|
# output and picking logfile names)
|
|
# cmd: the command to run to test (should include the logfile as '{}', and
|
|
# the command needs to write to that file)
|
|
# grepstr: the string to grep through the log file for. The test succeeds iff
|
|
# grep finds that string in the logfile (is used by grep, so it may
|
|
# be any pattern grep accepts, see `man 1 grep` for more info).
|
|
|
|
# edit this list to add more test cases
|
|
configs = [
|
|
TestCase(
|
|
name="lints",
|
|
variant="all",
|
|
cmd="./lint-wally | tee {}",
|
|
grepstr="All lints run with no errors or warnings"
|
|
)
|
|
]
|
|
def getBuildrootTC(boot):
|
|
INSTR_LIMIT = 4000000 # multiple of 100000; 4M is interesting because it gets into the kernel and enabling VM
|
|
MAX_EXPECTED = 246000000 # *** TODO: replace this with a search for the login prompt.
|
|
if boot:
|
|
name="buildrootboot"
|
|
BRcmd="vsim > {} -c <<!\ndo wally.do buildroot buildroot-no-trace $RISCV 0 1 0\n!"
|
|
BRgrepstr="WallyHostname login:"
|
|
else:
|
|
name="buildroot"
|
|
if (coverage):
|
|
print( "buildroot coverage")
|
|
BRcmd="vsim > {} -c <<!\ndo wally-batch.do buildroot buildroot $RISCV "+str(INSTR_LIMIT)+" 1 0 -coverage\n!"
|
|
else:
|
|
print( "buildroot no coverage")
|
|
BRcmd="vsim > {} -c <<!\ndo wally-batch.do buildroot buildroot $RISCV "+str(INSTR_LIMIT)+" 1 0\n!"
|
|
BRgrepstr=str(INSTR_LIMIT)+" instructions"
|
|
return TestCase(name,variant="rv64gc",cmd=BRcmd,grepstr=BRgrepstr)
|
|
|
|
tc = TestCase(
|
|
name="buildroot-checkpoint",
|
|
variant="rv64gc",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do buildroot buildroot-checkpoint $RISCV 400100000 400000001 400000000\n!", # *** will this work with rv64gc rather than buildroot config?
|
|
grepstr="400100000 instructions")
|
|
configs.append(tc)
|
|
|
|
tests64gcimperas = ["imperas64i", "imperas64f", "imperas64d", "imperas64m", "imperas64c"] # unused
|
|
|
|
tests64i = ["arch64i"]
|
|
for test in tests64i:
|
|
tc = TestCase(
|
|
name=test,
|
|
variant="rv64i",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv64i "+test+"\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
tests32gcimperas = ["imperas32i", "imperas32f", "imperas32m", "imperas32c"] # unused
|
|
tests32gc = ["arch32f", "arch32d", "arch32f_fma", "arch32d_fma", "arch32i", "arch32priv", "arch32c", "arch32m", "arch32zi", "arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "wally32a", "wally32priv", "wally32periph"]
|
|
for test in tests32gc:
|
|
tc = TestCase(
|
|
name=test,
|
|
variant="rv32gc",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32gc "+test+"\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
tests32imcimperas = ["imperas32i", "imperas32c"] # unused
|
|
tests32imc = ["arch32i", "arch32c", "arch32m", "wally32periph"]
|
|
for test in tests32imc:
|
|
tc = TestCase(
|
|
name=test,
|
|
variant="rv32imc",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32imc "+test+"\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
tests32i = ["arch32i"]
|
|
for test in tests32i:
|
|
tc = TestCase(
|
|
name=test,
|
|
variant="rv32i",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32i "+test+"\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
|
|
tests32e = ["wally32e"]
|
|
for test in tests32e:
|
|
tc = TestCase(
|
|
name=test,
|
|
variant="rv32e",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv32e "+test+"\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
ahbTests = [("0", "0"), ("0", "1"), ("1", "0"), ("1", "1"), ("2", "0"), ("2", "1")]
|
|
for test in ahbTests:
|
|
tc = TestCase(
|
|
name="ram_latency_" + test[0] + "_burst_en_" + test[1],
|
|
variant="ahb",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv64gc ahb "+test[0]+" "+test[1]+"\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
tests64gc = ["arch64f", "arch64d", "arch64f_fma", "arch64d_fma", "arch64i", "arch64zba", "arch64zbb", "arch64zbc", "arch64zbs",
|
|
"arch64priv", "arch64c", "arch64m", "arch64zi", "wally64a", "wally64periph", "wally64priv"]
|
|
if (coverage): # delete all but 64gc tests when running coverage
|
|
configs = []
|
|
# tests64gc = ["coverage64gc", "arch64f", "arch64d", "arch64i", "arch64priv", "arch64c", "arch64m",
|
|
tests64gc = ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m",
|
|
"arch64zi", "wally64a", "wally64periph", "wally64priv",
|
|
"arch64zba", "arch64zbb", "arch64zbc", "arch64zbs",
|
|
"imperas64f", "imperas64d", "imperas64c", "imperas64i"]
|
|
if (fp):
|
|
tests64gc.append("arch64f")
|
|
tests64gc.append("arch64d")
|
|
coverStr = '-coverage'
|
|
else:
|
|
coverStr = ''
|
|
for test in tests64gc:
|
|
tc = TestCase(
|
|
name=test,
|
|
variant="rv64gc",
|
|
cmd="vsim > {} -c <<!\ndo wally-batch.do rv64gc "+test+" " + coverStr + "\n!",
|
|
grepstr="All tests ran without failures")
|
|
configs.append(tc)
|
|
|
|
|
|
import os
|
|
from multiprocessing import Pool, TimeoutError
|
|
|
|
def search_log_for_text(text, logfile):
|
|
"""Search through the given log file for text, returning True if it is found or False if it is not"""
|
|
grepcmd = "grep -e '%s' '%s' > /dev/null" % (text, logfile)
|
|
return os.system(grepcmd) == 0
|
|
|
|
def run_test_case(config):
|
|
"""Run the given test case, and return 0 if the test suceeds and 1 if it fails"""
|
|
logname = "logs/"+config.variant+"_"+config.name+".log"
|
|
cmd = config.cmd.format(logname)
|
|
# print(cmd)
|
|
os.chdir(regressionDir)
|
|
os.system(cmd)
|
|
if search_log_for_text(config.grepstr, logname):
|
|
print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name))
|
|
return 0
|
|
else:
|
|
print(f"{bcolors.FAIL}%s_%s: Failures detected in output{bcolors.ENDC}" % (config.variant, config.name))
|
|
print(" Check %s" % logname)
|
|
return 1
|
|
|
|
def main():
|
|
"""Run the tests and count the failures"""
|
|
global configs, coverage
|
|
try:
|
|
os.chdir(regressionDir)
|
|
os.mkdir("logs")
|
|
except:
|
|
pass
|
|
try:
|
|
shutil.rmtree("wkdir")
|
|
except:
|
|
pass
|
|
finally:
|
|
os.mkdir("wkdir")
|
|
|
|
if '-makeTests' in sys.argv:
|
|
os.chdir(regressionDir)
|
|
os.system('./make-tests.sh | tee ./logs/make-tests.log')
|
|
|
|
if '-all' in sys.argv:
|
|
TIMEOUT_DUR = 30*7200 # seconds
|
|
configs.append(getBuildrootTC(boot=True))
|
|
elif '-buildroot' in sys.argv:
|
|
TIMEOUT_DUR = 30*7200 # seconds
|
|
configs=[getBuildrootTC(boot=True)]
|
|
elif '-coverage' in sys.argv:
|
|
TIMEOUT_DUR = 20*60 # seconds
|
|
# Presently don't run buildroot because it has a different config and can't be merged with the rv64gc coverage.
|
|
# Also it is slow to run.
|
|
# configs.append(getBuildrootTC(boot=False))
|
|
os.system('rm -f cov/*.ucdb')
|
|
else:
|
|
TIMEOUT_DUR = 10*60 # seconds
|
|
configs.append(getBuildrootTC(boot=False))
|
|
|
|
# Scale the number of concurrent processes to the number of test cases, but
|
|
# max out at a limited number of concurrent processes to not overwhelm the system
|
|
with Pool(processes=min(len(configs),40)) as pool:
|
|
num_fail = 0
|
|
results = {}
|
|
for config in configs:
|
|
results[config] = pool.apply_async(run_test_case,(config,))
|
|
for (config,result) in results.items():
|
|
try:
|
|
num_fail+=result.get(timeout=TIMEOUT_DUR)
|
|
except TimeoutError:
|
|
num_fail+=1
|
|
print(f"{bcolors.FAIL}%s_%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.variant, config.name, TIMEOUT_DUR))
|
|
|
|
# Coverage report
|
|
if coverage:
|
|
os.system('make coverage')
|
|
# Count the number of failures
|
|
if num_fail:
|
|
print(f"{bcolors.FAIL}Regression failed with %s failed configurations{bcolors.ENDC}" % num_fail)
|
|
else:
|
|
print(f"{bcolors.OKGREEN}SUCCESS! All tests ran without failures{bcolors.ENDC}")
|
|
return num_fail
|
|
|
|
if __name__ == '__main__':
|
|
exit(main())
|