Github_Repos/cvw
1
0
Fork 1
mirror of https://github.com/openhwgroup/cvw synced 2025-02-11 06:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

regression-wally passing pylint. Still needs a larger refactoring

Browse Source
This commit is contained in:
Jordan Carlin 2024-12-11 22:53:26 -08:00
parent 1e8d4c2cce
commit 57d723664f
No known key found for this signature in database
1 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
bin/regression-wally
View File

@ -11,11 +11,12 @@
# output. # output.
# #
################################## ##################################
import sys,os,shutil import sys
import os
import argparse import argparse
import multiprocessing import multiprocessing
from collections import namedtuple from collections import namedtuple
from multiprocessing import Pool, TimeoutError from multiprocessing import Pool, TimeoutError as MPTimeoutError
################################## ##################################
# Define lists of configurations and tests to run on each configuration # Define lists of configurations and tests to run on each configuration
@ -39,18 +40,18 @@ tests = [
# Separate test for short buildroot run through OpenSBI UART output # Separate test for short buildroot run through OpenSBI UART output
tests_buildrootshort = [ tests_buildrootshort = [
["buildroot", ["buildroot"], [f"+INSTR_LIMIT=1400000"], # Instruction limit gets to first OpenSBI UART output ["buildroot", ["buildroot"], ["+INSTR_LIMIT=1400000"], # Instruction limit gets to first OpenSBI UART output
"OpenSBI v", "buildroot_uart.out"] "OpenSBI v", "buildroot_uart.out"]
] ]
# Separate test for full buildroot run # Separate test for full buildroot run
tests_buildrootboot = [ tests_buildrootboot = [
["buildroot", ["buildroot"], [f"+INSTR_LIMIT=600000000"], # boot entire buildroot Linux to login prompt ["buildroot", ["buildroot"], ["+INSTR_LIMIT=600000000"], # boot entire buildroot Linux to login prompt
"WallyHostname login: ", "buildroot_uart.out"] "WallyHostname login: ", "buildroot_uart.out"]
] ]
tests_buildrootbootlockstep = [ tests_buildrootbootlockstep = [
["buildroot", ["buildroot"], [f"+INSTR_LIMIT=600000000 --lockstep"], # boot entire buildroot Linux to login prompt ["buildroot", ["buildroot"], ["+INSTR_LIMIT=600000000 --lockstep"], # boot entire buildroot Linux to login prompt
"WallyHostname login: ", "buildroot_uart.out"] "WallyHostname login: ", "buildroot_uart.out"]
] ]
@ -267,18 +268,18 @@ def addTests(tests, sim):
for test in tests: for test in tests:
config = test[0] config = test[0]
suites = test[1] suites = test[1]
if (len(test) >= 3): if len(test) >= 3:
args = " --args " + " ".join(test[2]) args = " --args " + " ".join(test[2])
else: else:
args = "" args = ""
if (len(test) >= 4): if len(test) >= 4:
gs = test[3] gs = test[3]
else: else:
gs = "All tests ran without failures" gs = "All tests ran without failures"
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
for t in suites: for t in suites:
sim_log = sim_logdir + config + "_" + t + ".log" sim_log = sim_logdir + config + "_" + t + ".log"
if (len(test) >= 5): if len(test) >= 5:
grepfile = sim_logdir + test[4] grepfile = sim_logdir + test[4]
else: else:
grepfile = sim_log grepfile = sim_log
@ -291,20 +292,20 @@ def addTests(tests, sim):
configs.append(tc) configs.append(tc)
def addTestsByDir(dir, config, sim, lockstepMode=0): def addTestsByDir(testDir, config, sim, lockstepMode=0):
if os.path.isdir(dir): if os.path.isdir(testDir):
sim_logdir = WALLY+ "/sim/" + sim + "/logs/" sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
if coverStr == "--fcov": # use --fcov in place of --lockstep if coverStr == "--fcov": # use --fcov in place of --lockstep
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
gs="Mismatches : 0" gs="Mismatches : 0"
if ("cvw-arch-verif/tests" in dir and not "priv" in dir): if ("cvw-arch-verif/tests" in testDir and not "priv" in testDir):
fileEnd = "ALL.elf" fileEnd = "ALL.elf"
else: else:
fileEnd = ".elf" fileEnd = ".elf"
elif coverStr == "--ccov": elif coverStr == "--ccov":
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
gs="Single Elf file tests are not signatured verified." gs="Single Elf file tests are not signatured verified."
if ("cvw-arch-verif/tests" in dir and not "priv" in dir): if ("cvw-arch-verif/tests" in testDir and not "priv" in testDir):
fileEnd = "ALL.elf" fileEnd = "ALL.elf"
else: else:
fileEnd = ".elf" fileEnd = ".elf"
@ -316,21 +317,20 @@ def addTestsByDir(dir, config, sim, lockstepMode=0):
cmdPrefix="wsim --sim " + sim + " " + config cmdPrefix="wsim --sim " + sim + " " + config
gs="Single Elf file tests are not signatured verified." gs="Single Elf file tests are not signatured verified."
fileEnd = ".elf" fileEnd = ".elf"
for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)): for dirpath, _, filenames in os.walk(os.path.abspath(testDir)):
for file in filenames: for file in filenames:
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files # fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
if file.endswith(fileEnd): if file.endswith(fileEnd):
fullfile = os.path.join(dirpath, file) fullfile = os.path.join(dirpath, file)
fields = fullfile.rsplit('/', 3) fields = fullfile.rsplit('/', 3)
if (fields[2] == "ref"): if fields[2] == "ref":
shortelf = fields[1] + "_" + fields[3] shortelf = fields[1] + "_" + fields[3]
else: else:
shortelf = fields[2] + "_" + fields[3] shortelf = fields[2] + "_" + fields[3]
if (shortelf in lockstepwaivers): # skip tests that itch bugs in ImperasDV if shortelf in lockstepwaivers: # skip tests that itch bugs in ImperasDV
print(f"{bcolors.WARNING}Skipping waived test {shortelf}{bcolors.ENDC}") print(f"{bcolors.WARNING}Skipping waived test {shortelf}{bcolors.ENDC}")
continue continue
sim_log = sim_logdir + config + "_" + shortelf + ".log" sim_log = sim_logdir + config + "_" + shortelf + ".log"
grepstring = ""
tc = TestCase( tc = TestCase(
name=file, name=file,
variant=config, variant=config,
@ -339,8 +339,8 @@ def addTestsByDir(dir, config, sim, lockstepMode=0):
grepfile = sim_log) grepfile = sim_log)
configs.append(tc) configs.append(tc)
else: else:
print("Error: Directory not found: " + dir) print("Error: Directory not found: " + testDir)
exit(1) sys.exit(1)
def search_log_for_text(text, grepfile): def search_log_for_text(text, grepfile):
"""Search through the given log file for text, returning True if it is found or False if it is not""" """Search through the given log file for text, returning True if it is found or False if it is not"""
@ -348,8 +348,7 @@ def search_log_for_text(text, grepfile):
os.system(grepwarn) os.system(grepwarn)
greperr = "grep -i -H Error: " + grepfile greperr = "grep -i -H Error: " + grepfile
os.system(greperr) os.system(greperr)
grepcmd = "grep -a -e '%s' '%s' > /dev/null" % (text, grepfile) grepcmd = f"grep -a -e '{text}' '{grepfile}' > /dev/null"
# print(" search_log_for_text invoking %s" % grepcmd)
return os.system(grepcmd) == 0 return os.system(grepcmd) == 0
def run_test_case(config, dryrun: bool = False): def run_test_case(config, dryrun: bool = False):
@ -368,12 +367,11 @@ def run_test_case(config, dryrun: bool = False):
os.system(cmd) os.system(cmd)
if search_log_for_text(config.grepstr, grepfile): if search_log_for_text(config.grepstr, grepfile):
# Flush is needed to flush output to stdout when running in multiprocessing Pool # Flush is needed to flush output to stdout when running in multiprocessing Pool
# print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name), flush=True)
print(f"{bcolors.OKGREEN}%s: Success{bcolors.ENDC}" % (config.cmd), flush=True) print(f"{bcolors.OKGREEN}%s: Success{bcolors.ENDC}" % (config.cmd), flush=True)
return 0 return 0
else: else:
print(f"{bcolors.FAIL}%s: Failures detected in output{bcolors.ENDC}" % (config.cmd), flush=True) print(f"{bcolors.FAIL}%s: Failures detected in output{bcolors.ENDC}" % (config.cmd), flush=True)
print(" Check %s" % grepfile) print(f" Check {grepfile}", flush=True)
return 1 return 1
################################## ##################################
@ -400,16 +398,16 @@ parser.add_argument("--fp", help="Include floating-point tests in coverage (slow
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true") parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
args = parser.parse_args() args = parser.parse_args()
if (args.nightly): if args.nightly:
nightMode = "--nightly" nightMode = "--nightly"
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
else: else:
nightMode = "" nightMode = ""
sims = [defaultsim] sims = [defaultsim]
if (args.ccov): # only run RV64GC tests in coverage mode if args.ccov: # only run RV64GC tests in coverage mode
coverStr = '--ccov' coverStr = '--ccov'
elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode elif args.fcov: # only run RV64GC tests in lockstep in coverage mode
coverStr = '--fcov' coverStr = '--fcov'
else: else:
coverStr = '' coverStr = ''
@ -428,15 +426,15 @@ configs = [
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests # run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
if (args.buildroot): if args.buildroot:
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours # addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode if args.ccov: # only run RV64GC tests on Questa in code coverage mode
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim) addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim)
elif (args.fcov): # run tests in lockstep in functional coverage mode elif args.fcov: # run tests in lockstep in functional coverage mode
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim)
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim)
@ -444,14 +442,14 @@ elif (args.fcov): # run tests in lockstep in functional coverage mode
else: else:
for sim in sims: for sim in sims:
if (not (args.buildroot and sim == lockstepsim)): # skip short buildroot sim if running long one if not (args.buildroot and sim == lockstepsim): # skip short buildroot sim if running long one
addTests(tests_buildrootshort, sim) addTests(tests_buildrootshort, sim)
addTests(tests, sim) addTests(tests, sim)
addTests(tests64gc_nofp, sim) addTests(tests64gc_nofp, sim)
addTests(tests64gc_fp, sim) addTests(tests64gc_fp, sim)
# run derivative configurations and lockstep tests in nightly regression # run derivative configurations and lockstep tests in nightly regression
if (args.nightly): if args.nightly:
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1) addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1)
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1) addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1)
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1) addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1)
@ -459,14 +457,14 @@ if (args.nightly):
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script. # addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
# testfloat tests # testfloat tests
if (args.testfloat): # for testfloat alone, just run testfloat tests if args.testfloat: # for testfloat alone, just run testfloat tests
configs = [] configs = []
if (args.testfloat or args.nightly): # for nightly, run testfloat along with others if (args.testfloat or args.nightly): # for nightly, run testfloat along with others
testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved
testfloatconfigs = ["fdqh_rv64gc", "fdq_rv64gc", "fdh_rv64gc", "fd_rv64gc", "fh_rv64gc", "f_rv64gc", "fdqh_rv32gc", "f_rv32gc"] testfloatconfigs = ["fdqh_rv64gc", "fdq_rv64gc", "fdh_rv64gc", "fd_rv64gc", "fh_rv64gc", "f_rv64gc", "fdqh_rv32gc", "f_rv32gc"]
for config in testfloatconfigs: for config in testfloatconfigs:
tests = ["div", "sqrt", "add", "sub", "mul", "cvtint", "cvtfp", "fma", "cmp"] tests = ["div", "sqrt", "add", "sub", "mul", "cvtint", "cvtfp", "fma", "cmp"]
if ("f_" in config): if "f_" in config:
tests.remove("cvtfp") tests.remove("cvtfp")
for test in tests: for test in tests:
sim_log = WALLY + "/sim/" + testfloatsim + "/logs/"+config+"_"+test+".log" sim_log = WALLY + "/sim/" + testfloatsim + "/logs/"+config+"_"+test+".log"
@ -508,7 +506,7 @@ if (args.testfloat or args.nightly): # for nightly, run testfloat along with oth
for config in testfloatdivconfigs: for config in testfloatdivconfigs:
# div test case # div test case
tests = ["div", "sqrt", "cvtint", "cvtfp"] tests = ["div", "sqrt", "cvtint", "cvtfp"]
if ("f_" in config): if "f_" in config:
tests.remove("cvtfp") tests.remove("cvtfp")
for test in tests: for test in tests:
sim_log = WALLY + "/sim/questa/logs/"+config+"_"+test+".log" sim_log = WALLY + "/sim/questa/logs/"+config+"_"+test+".log"
@ -523,15 +521,12 @@ if (args.testfloat or args.nightly): # for nightly, run testfloat along with oth
def main(): def main():
"""Run the tests and count the failures""" """Run the tests and count the failures"""
global configs, args # global configs, args
os.chdir(regressionDir) os.chdir(regressionDir)
dirs = ["questa/logs", "questa/wkdir", "verilator/logs", "verilator/wkdir", "vcs/logs", "vcs/wkdir"] dirs = ["questa/logs", "questa/wkdir", "verilator/logs", "verilator/wkdir", "vcs/logs", "vcs/wkdir"]
for d in dirs: for d in dirs:
try: os.system(f'rm -rf {d}')
os.system('rm -rf %s' % d) os.makedirs(d, exist_ok=True)
os.mkdir(d)
except:
pass
if args.ccov: if args.ccov:
TIMEOUT_DUR = 20*60 # seconds TIMEOUT_DUR = 20*60 # seconds
os.system('rm -f questa/ucdb/* questa/cov/*') os.system('rm -f questa/ucdb/* questa/cov/*')
@ -562,7 +557,7 @@ def main():
for (config,result) in results.items(): for (config,result) in results.items():
try: try:
num_fail+=result.get(timeout=TIMEOUT_DUR) num_fail+=result.get(timeout=TIMEOUT_DUR)
except TimeoutError: except MPTimeoutError:
pool.terminate() pool.terminate()
num_fail+=1 num_fail+=1
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR)) print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))
@ -580,4 +575,4 @@ def main():
return num_fail return num_fail
if __name__ == '__main__': if __name__ == '__main__':
exit(main()) sys.exit(main())