mirror of
https://github.com/openhwgroup/cvw
synced 2025-02-02 17:55:19 +00:00
Begin refactoring regression-wally
This commit is contained in:
Jordan Carlin
parent
32b62e9d0c
commit
1a2569e6f1
@ -4,6 +4,7 @@
|
||||
# regression-wally
|
||||
# David_Harris@Hmc.edu 25 January 2021
|
||||
# Modified by Jarred Allen <jaallen@g.hmc.edu> and many others
|
||||
# jcarlin@hmc.edu December 2024
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
|
||||
#
|
||||
# Run a regression with multiple configurations in parallel and exit with
|
||||
@ -18,6 +19,14 @@ import multiprocessing
|
||||
from collections import namedtuple
|
||||
from multiprocessing import Pool, TimeoutError as MPTimeoutError
|
||||
|
||||
# Globals
|
||||
WALLY = os.environ.get('WALLY')
|
||||
regressionDir = WALLY + '/sim'
|
||||
coveragesim = "questa" # Questa is required for code/functional coverage
|
||||
defaultsim = "verilator" # Default simulator for all other tests
|
||||
lockstepsim = "questa"
|
||||
|
||||
|
||||
##################################
|
||||
# Define lists of configurations and tests to run on each configuration
|
||||
##################################
|
||||
@ -38,6 +47,21 @@ tests = [
|
||||
["rv64i", ["arch64i"]]
|
||||
]
|
||||
|
||||
tests64gc_fp = [
|
||||
["rv64gc", ["arch64f", "arch64d", "arch64zfh",
|
||||
"arch64f_fma", "arch64d_fma", "arch64zfh_fma",
|
||||
"arch64f_divsqrt", "arch64d_divsqrt", "arch64zfh_divsqrt",
|
||||
"arch64zfaf", "arch64zfad"]]
|
||||
]
|
||||
|
||||
# Separate out floating-point tests for RV64 to speed up coverage
|
||||
tests64gc_nofp = [
|
||||
["rv64gc", ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m", "arch64zcb",
|
||||
"arch64zifencei", "arch64zicond", "arch64a_amo", "wally64a_lrsc", "wally64periph", "wally64priv",
|
||||
"arch64zbkb", "arch64zbkc", "arch64zbkx", "arch64zknd", "arch64zkne", "arch64zknh",
|
||||
"arch64zba", "arch64zbb", "arch64zbc", "arch64zbs"]] # add when working: "arch64zicboz"
|
||||
]
|
||||
|
||||
# Separate test for short buildroot run through OpenSBI UART output
|
||||
tests_buildrootshort = [
|
||||
["buildroot", ["buildroot"], ["+INSTR_LIMIT=1400000"], # Instruction limit gets to first OpenSBI UART output
|
||||
@ -55,22 +79,6 @@ tests_buildrootbootlockstep = [
|
||||
"WallyHostname login: ", "buildroot_uart.out"]
|
||||
]
|
||||
|
||||
|
||||
# Separate out floating-point tests for RV64 to speed up coverage
|
||||
tests64gc_nofp = [
|
||||
["rv64gc", ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m", "arch64zcb",
|
||||
"arch64zifencei", "arch64zicond", "arch64a_amo", "wally64a_lrsc", "wally64periph", "wally64priv",
|
||||
"arch64zbkb", "arch64zbkc", "arch64zbkx", "arch64zknd", "arch64zkne", "arch64zknh",
|
||||
"arch64zba", "arch64zbb", "arch64zbc", "arch64zbs"]] # add when working: "arch64zicboz"
|
||||
]
|
||||
|
||||
tests64gc_fp = [
|
||||
["rv64gc", ["arch64f", "arch64d", "arch64zfh",
|
||||
"arch64f_fma", "arch64d_fma", "arch64zfh_fma",
|
||||
"arch64f_divsqrt", "arch64d_divsqrt", "arch64zfh_divsqrt",
|
||||
"arch64zfaf", "arch64zfad"]]
|
||||
]
|
||||
|
||||
derivconfigtests = [
|
||||
# memory system
|
||||
["tlb2_rv32gc", ["wally32priv"]],
|
||||
@ -93,21 +101,21 @@ derivconfigtests = [
|
||||
["ram_1_1_rv64gc", ["ahb64"]],
|
||||
["ram_2_0_rv64gc", ["ahb64"]],
|
||||
["ram_2_1_rv64gc", ["ahb64"]],
|
||||
# RV32 cacheless designs will not work unless DTIM supports FLEN > XLEN. This support is not planned.
|
||||
# ["nodcache_rv32gc", ["ahb32"]],
|
||||
# ["nocache_rv32gc", ["ahb32"]],
|
||||
# RV32 cacheless designs will not work unless DTIM supports FLEN > XLEN. This support is not planned.
|
||||
# ["nodcache_rv32gc", ["ahb32"]],
|
||||
# ["nocache_rv32gc", ["ahb32"]],
|
||||
["noicache_rv32gc", ["ahb32"]],
|
||||
["noicache_rv64gc", ["ahb64"]],
|
||||
["nodcache_rv64gc", ["ahb64"]],
|
||||
["nocache_rv64gc", ["ahb64"]],
|
||||
|
||||
# Atomic variants
|
||||
# Atomic variants
|
||||
["zaamo_rv64gc", ["arch64i", "arch64a_amo"]],
|
||||
["zalrsc_rv64gc", ["arch64i", "wally64a_lrsc"]],
|
||||
["zaamo_rv32gc", ["arch32i", "arch32a_amo"]],
|
||||
["zalrsc_rv32gc", ["arch32i", "wally32a_lrsc"]],
|
||||
|
||||
# Bit manipulation and crypto variants
|
||||
# Bit manipulation and crypto variants
|
||||
["zba_rv32gc", ["arch32i", "arch32zba"]],
|
||||
["zbb_rv32gc", ["arch32i", "arch32zbb"]],
|
||||
["zbc_rv32gc", ["arch32i", "arch32zbc"]],
|
||||
@ -130,7 +138,7 @@ derivconfigtests = [
|
||||
["zknd_rv64gc", ["arch64i", "arch64zknd"]],
|
||||
["zknh_rv64gc", ["arch64i", "arch64zknh"]],
|
||||
|
||||
# No privilege modes variants
|
||||
# No privilege modes variants
|
||||
["noS_rv32gc", ["arch32i", "arch32f", "arch32priv", "arch32c", "arch32m", "arch32a_amo", "arch32zifencei", "arch32zicond",
|
||||
"arch32zba", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "arch32zcb", "arch32zbkx", "arch32zknd"]],
|
||||
["noS_rv64gc", ["arch64i", "arch64f", "arch64priv", "arch64c", "arch64m", "arch64a_amo", "arch64zifencei", "arch64zicond",
|
||||
@ -182,7 +190,6 @@ derivconfigtests = [
|
||||
]
|
||||
|
||||
bpredtests = [
|
||||
|
||||
["nobpred_rv32gc", ["rv32i"]],
|
||||
["bpred_TWOBIT_6_16_10_0_rv32gc", ["embench"], "-GPrintHPMCounters=1"],
|
||||
["bpred_TWOBIT_8_16_10_0_rv32gc", ["embench"], "-GPrintHPMCounters=1"],
|
||||
@ -231,11 +238,38 @@ bpredtests = [
|
||||
["bpred_GSHARE_10_10_10_1_rv32gc", ["embench"], "-GPrintHPMCounters=1"]
|
||||
]
|
||||
|
||||
testfloatdivconfigs = [
|
||||
"fdh_div_2_1_rv32gc", "fdh_div_2_1_rv64gc", "fdh_div_2_2_rv32gc",
|
||||
"fdh_div_2_2_rv64gc", "fdh_div_2_4_rv32gc", "fdh_div_2_4_rv64gc",
|
||||
"fdh_div_4_1_rv32gc", "fdh_div_4_1_rv64gc", "fdh_div_4_2_rv32gc",
|
||||
"fdh_div_4_2_rv64gc", "fdh_div_4_4_rv32gc", "fdh_div_4_4_rv64gc",
|
||||
"fd_div_2_1_rv32gc", "fd_div_2_1_rv64gc", "fd_div_2_2_rv32gc",
|
||||
"fd_div_2_2_rv64gc", "fd_div_2_4_rv32gc", "fd_div_2_4_rv64gc",
|
||||
"fd_div_4_1_rv32gc", "fd_div_4_1_rv64gc", "fd_div_4_2_rv32gc",
|
||||
"fd_div_4_2_rv64gc", "fd_div_4_4_rv32gc", "fd_div_4_4_rv64gc",
|
||||
"fdqh_div_2_1_rv32gc", "fdqh_div_2_1_rv64gc", "fdqh_div_2_2_rv32gc",
|
||||
"fdqh_div_2_2_rv64gc", "fdqh_div_2_4_rv32gc", "fdqh_div_2_4_rv64gc",
|
||||
"fdqh_div_4_1_rv32gc", "fdqh_div_4_1_rv64gc", "fdqh_div_4_2_rv32gc",
|
||||
"fdqh_div_4_2_rv64gc", "fdqh_div_4_4_rv32gc", "fdqh_div_4_4_rv64gc",
|
||||
"fdq_div_2_1_rv32gc", "fdq_div_2_1_rv64gc", "fdq_div_2_2_rv32gc",
|
||||
"fdq_div_2_2_rv64gc", "fdq_div_2_4_rv32gc", "fdq_div_2_4_rv64gc",
|
||||
"fdq_div_4_1_rv32gc", "fdq_div_4_1_rv64gc", "fdq_div_4_2_rv32gc",
|
||||
"fdq_div_4_2_rv64gc", "fdq_div_4_4_rv32gc", "fdq_div_4_4_rv64gc",
|
||||
"fh_div_2_1_rv32gc", "fh_div_2_1_rv64gc", "fh_div_2_2_rv32gc",
|
||||
"fh_div_2_2_rv64gc", "fh_div_2_4_rv32gc", "fh_div_2_4_rv64gc",
|
||||
"fh_div_4_1_rv32gc", "fh_div_4_1_rv64gc", "fh_div_4_2_rv32gc",
|
||||
"fh_div_4_2_rv64gc", "fh_div_4_4_rv32gc", "fh_div_4_4_rv64gc",
|
||||
"f_div_2_1_rv32gc", "f_div_2_1_rv64gc", "f_div_2_2_rv32gc",
|
||||
"f_div_2_2_rv64gc", "f_div_2_4_rv32gc", "f_div_2_4_rv64gc",
|
||||
"f_div_4_1_rv32gc", "f_div_4_1_rv64gc", "f_div_4_2_rv32gc",
|
||||
"f_div_4_2_rv64gc", "f_div_4_4_rv32gc", "f_div_4_4_rv64gc"
|
||||
]
|
||||
|
||||
# list of tests not supported by ImperasDV yet that should be waived during lockstep testing
|
||||
lockstepwaivers = [
|
||||
"WALLY-q-01.S_ref.elf", # Q extension is not supported by ImperasDV
|
||||
"WALLY-q-01.S_ref.elf", # Q extension is not supported by ImperasDV
|
||||
"coverage_tlbMisaligned.elf", # Issue 976: ImperasDV bug disagrees with Wally related to misaligned pages when PBMT makes page uncachable
|
||||
"WALLY-cbom-01.S_ref.elf" #, # cbom extension is not supported by ImperasDV because there is no cache model in ImperasDV
|
||||
"WALLY-cbom-01.S_ref.elf", # cbom extension is not supported by ImperasDV because there is no cache model in ImperasDV
|
||||
]
|
||||
|
||||
##################################
|
||||
@ -263,9 +297,9 @@ class bcolors:
|
||||
BOLD = '\033[1m'
|
||||
UNDERLINE = '\033[4m'
|
||||
|
||||
def addTests(tests, sim):
|
||||
def addTests(testList, sim, coverStr, configs):
|
||||
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
||||
for test in tests:
|
||||
for test in testList:
|
||||
config = test[0]
|
||||
suites = test[1]
|
||||
args = f" --args {test[2]}" if len(test) >= 3 else ""
|
||||
@ -283,56 +317,37 @@ def addTests(tests, sim):
|
||||
configs.append(tc)
|
||||
|
||||
|
||||
def addTestsByDir(testDir, config, sim, lockstepMode=0):
|
||||
if os.path.isdir(testDir):
|
||||
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
||||
if coverStr == "--fcov": # use --fcov in place of --lockstep
|
||||
cmdPrefix=f"wsim --sim {sim} {coverStr} {config}"
|
||||
gs="Mismatches : 0"
|
||||
if ("cvw-arch-verif/tests" in testDir and "priv" not in testDir):
|
||||
fileEnd = "ALL.elf"
|
||||
else:
|
||||
fileEnd = ".elf"
|
||||
elif coverStr == "--ccov":
|
||||
cmdPrefix=f"wsim --sim {sim} {coverStr} {config}"
|
||||
gs="Single Elf file tests are not signatured verified."
|
||||
if ("cvw-arch-verif/tests" in testDir and "priv" not in testDir):
|
||||
fileEnd = "ALL.elf"
|
||||
else:
|
||||
fileEnd = ".elf"
|
||||
elif lockstepMode:
|
||||
cmdPrefix=f"wsim --lockstep --sim {sim} {config}"
|
||||
gs="Mismatches : 0"
|
||||
fileEnd = ".elf"
|
||||
else:
|
||||
cmdPrefix=f"wsim --sim {sim} {config}"
|
||||
gs="Single Elf file tests are not signatured verified."
|
||||
fileEnd = ".elf"
|
||||
for dirpath, _, filenames in os.walk(os.path.abspath(testDir)):
|
||||
for file in filenames:
|
||||
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
||||
if file.endswith(fileEnd):
|
||||
fullfile = os.path.join(dirpath, file)
|
||||
fields = fullfile.rsplit('/', 3)
|
||||
if fields[2] == "ref":
|
||||
shortelf = f"{fields[1]}_{fields[3]}"
|
||||
else:
|
||||
shortelf = f"{fields[2]}_{fields[3]}"
|
||||
if shortelf in lockstepwaivers: # skip tests that itch bugs in ImperasDV
|
||||
print(f"{bcolors.WARNING}Skipping waived test {shortelf}{bcolors.ENDC}")
|
||||
continue
|
||||
sim_log = f"{sim_logdir}{config}_{shortelf}.log"
|
||||
tc = TestCase(
|
||||
name=file,
|
||||
variant=config,
|
||||
cmd=f"{cmdPrefix} {fullfile} > {sim_log}",
|
||||
grepstr=gs,
|
||||
grepfile = sim_log)
|
||||
configs.append(tc)
|
||||
else:
|
||||
def addTestsByDir(testDir, config, sim, coverStr, configs, lockstepMode=0):
|
||||
if not os.path.isdir(testDir):
|
||||
print(f"Error: Directory not found: {testDir}")
|
||||
sys.exit(1)
|
||||
|
||||
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
||||
cmdPrefix = f"wsim --sim {sim} {coverStr} {'--lockstep' if lockstepMode else ''} {config}"
|
||||
gs = "Mismatches : 0" if lockstepMode or coverStr == "--fcov" else "Single Elf file tests are not signatured verified."
|
||||
# fcov/ccov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
||||
fileEnd = "ALL.elf" if "cvw-arch-verif/tests" in testDir and "priv" not in testDir and (coverStr == "--fcov" or coverStr == "--ccov") else ".elf"
|
||||
for dirpath, _, filenames in os.walk(os.path.abspath(testDir)):
|
||||
for file in filenames:
|
||||
if file.endswith(fileEnd):
|
||||
fullfile = os.path.join(dirpath, file)
|
||||
fields = fullfile.rsplit('/', 3)
|
||||
if fields[2] == "ref":
|
||||
shortelf = f"{fields[1]}_{fields[3]}"
|
||||
else:
|
||||
shortelf = f"{fields[2]}_{fields[3]}"
|
||||
if shortelf in lockstepwaivers: # skip tests that itch bugs in ImperasDV
|
||||
print(f"{bcolors.WARNING}Skipping waived test {shortelf}{bcolors.ENDC}")
|
||||
continue
|
||||
sim_log = f"{sim_logdir}{config}_{shortelf}.log"
|
||||
tc = TestCase(
|
||||
name=file,
|
||||
variant=config,
|
||||
cmd=f"{cmdPrefix} {fullfile} > {sim_log}",
|
||||
grepstr=gs,
|
||||
grepfile = sim_log)
|
||||
configs.append(tc)
|
||||
|
||||
def search_log_for_text(text, grepfile):
|
||||
"""Search through the given log file for text, returning True if it is found or False if it is not"""
|
||||
grepwarn = f"grep -i -H Warning: {grepfile}"
|
||||
@ -365,154 +380,116 @@ def run_test_case(config, dryrun: bool = False):
|
||||
print(f" Check {grepfile}", flush=True)
|
||||
return 1
|
||||
|
||||
##################################
|
||||
# Main body
|
||||
##################################
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ccov", help="Code Coverage", action="store_true")
|
||||
parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
|
||||
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
||||
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
||||
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
|
||||
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
def process_args(args):
|
||||
if args.nightly:
|
||||
nightMode = "--nightly"
|
||||
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
||||
else:
|
||||
nightMode = ""
|
||||
sims = [defaultsim]
|
||||
|
||||
if args.ccov:
|
||||
coverStr = "--ccov"
|
||||
elif args.fcov:
|
||||
coverStr = "--fcov"
|
||||
else:
|
||||
coverStr = ""
|
||||
return nightMode, sims, coverStr
|
||||
|
||||
|
||||
WALLY = os.environ.get('WALLY')
|
||||
regressionDir = WALLY + '/sim'
|
||||
os.chdir(regressionDir)
|
||||
def selectTests(nightMode, args, sims, coverStr):
|
||||
# Run Lint
|
||||
configs = [
|
||||
TestCase(
|
||||
name="lints",
|
||||
variant="all",
|
||||
cmd=f"lint-wally {nightMode} | tee {WALLY}/sim/verilator/logs/all_lints.log",
|
||||
grepstr="lints run with no errors or warnings",
|
||||
grepfile = f"{WALLY}/sim/verilator/logs/all_lints.log")
|
||||
]
|
||||
|
||||
coveragesim = "questa" # Questa is required for code/functional coverage
|
||||
#defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready
|
||||
defaultsim = "verilator" # Default simulator for all other tests
|
||||
lockstepsim = "questa"
|
||||
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
|
||||
if args.buildroot:
|
||||
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
|
||||
addTests(tests_buildrootbootlockstep, lockstepsim, coverStr, configs) # lockstep with Questa and ImperasDV runs overnight
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ccov", help="Code Coverage", action="store_true")
|
||||
parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
|
||||
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
||||
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
||||
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
|
||||
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.ccov: # only run RV64GC tests on Questa in code coverage mode
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim, coverStr, configs)
|
||||
elif args.fcov: # run tests in lockstep in functional coverage mode
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim, coverStr, configs)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim, coverStr, configs)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||
else:
|
||||
for sim in sims:
|
||||
if not (args.buildroot and sim == lockstepsim): # skip short buildroot sim if running long one
|
||||
addTests(tests_buildrootshort, sim, coverStr, configs)
|
||||
addTests(tests, sim, coverStr, configs)
|
||||
addTests(tests64gc_nofp, sim, coverStr, configs)
|
||||
addTests(tests64gc_fp, sim, coverStr, configs)
|
||||
|
||||
if args.nightly:
|
||||
nightMode = "--nightly"
|
||||
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
||||
else:
|
||||
nightMode = ""
|
||||
sims = [defaultsim]
|
||||
# run derivative configurations and lockstep tests in nightly regression
|
||||
if args.nightly:
|
||||
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1, coverStr, configs)
|
||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, coverStr, configs, 1)
|
||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, coverStr, configs, 1)
|
||||
addTests(derivconfigtests, defaultsim, coverStr, configs)
|
||||
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
||||
|
||||
if args.ccov: # only run RV64GC tests in coverage mode
|
||||
coverStr = '--ccov'
|
||||
elif args.fcov: # only run RV64GC tests in lockstep in coverage mode
|
||||
coverStr = '--fcov'
|
||||
else:
|
||||
coverStr = ''
|
||||
# testfloat tests
|
||||
if args.testfloat: # for testfloat alone, just run testfloat tests
|
||||
configs = []
|
||||
if (args.testfloat or args.nightly): # for nightly, run testfloat along with others
|
||||
testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved
|
||||
testfloatconfigs = ["fdqh_rv64gc", "fdq_rv64gc", "fdh_rv64gc", "fd_rv64gc", "fh_rv64gc", "f_rv64gc", "fdqh_rv32gc", "f_rv32gc"]
|
||||
for config in testfloatconfigs:
|
||||
tests = ["div", "sqrt", "add", "sub", "mul", "cvtint", "cvtfp", "fma", "cmp"]
|
||||
if "f_" in config:
|
||||
tests.remove("cvtfp")
|
||||
for test in tests:
|
||||
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
||||
tc = TestCase(
|
||||
name=test,
|
||||
variant=config,
|
||||
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
||||
grepstr="All Tests completed with 0 errors",
|
||||
grepfile = sim_log)
|
||||
configs.append(tc)
|
||||
for config in testfloatdivconfigs:
|
||||
# div test case
|
||||
tests = ["div", "sqrt", "cvtint", "cvtfp"]
|
||||
if "f_" in config:
|
||||
tests.remove("cvtfp")
|
||||
for test in tests:
|
||||
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
||||
tc = TestCase(
|
||||
name=test,
|
||||
variant=config,
|
||||
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
||||
grepstr="All Tests completed with 0 errors",
|
||||
grepfile = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log")
|
||||
configs.append(tc)
|
||||
return configs
|
||||
|
||||
|
||||
# Run Lint
|
||||
configs = [
|
||||
TestCase(
|
||||
name="lints",
|
||||
variant="all",
|
||||
cmd=f"lint-wally {nightMode} | tee {WALLY}/sim/verilator/logs/all_lints.log",
|
||||
grepstr="lints run with no errors or warnings",
|
||||
grepfile = f"{WALLY}/sim/verilator/logs/all_lints.log")
|
||||
]
|
||||
|
||||
|
||||
|
||||
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
|
||||
if args.buildroot:
|
||||
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
|
||||
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
|
||||
|
||||
if args.ccov: # only run RV64GC tests on Questa in code coverage mode
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim)
|
||||
elif args.fcov: # run tests in lockstep in functional coverage mode
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim)
|
||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
||||
|
||||
else:
|
||||
for sim in sims:
|
||||
if not (args.buildroot and sim == lockstepsim): # skip short buildroot sim if running long one
|
||||
addTests(tests_buildrootshort, sim)
|
||||
addTests(tests, sim)
|
||||
addTests(tests64gc_nofp, sim)
|
||||
addTests(tests64gc_fp, sim)
|
||||
|
||||
# run derivative configurations and lockstep tests in nightly regression
|
||||
if args.nightly:
|
||||
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1)
|
||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1)
|
||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1)
|
||||
addTests(derivconfigtests, defaultsim)
|
||||
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
||||
|
||||
# testfloat tests
|
||||
if args.testfloat: # for testfloat alone, just run testfloat tests
|
||||
configs = []
|
||||
if (args.testfloat or args.nightly): # for nightly, run testfloat along with others
|
||||
testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved
|
||||
testfloatconfigs = ["fdqh_rv64gc", "fdq_rv64gc", "fdh_rv64gc", "fd_rv64gc", "fh_rv64gc", "f_rv64gc", "fdqh_rv32gc", "f_rv32gc"]
|
||||
for config in testfloatconfigs:
|
||||
tests = ["div", "sqrt", "add", "sub", "mul", "cvtint", "cvtfp", "fma", "cmp"]
|
||||
if "f_" in config:
|
||||
tests.remove("cvtfp")
|
||||
for test in tests:
|
||||
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
||||
tc = TestCase(
|
||||
name=test,
|
||||
variant=config,
|
||||
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
||||
grepstr="All Tests completed with 0 errors",
|
||||
grepfile = sim_log)
|
||||
configs.append(tc)
|
||||
|
||||
|
||||
testfloatdivconfigs = [
|
||||
"fdh_div_2_1_rv32gc", "fdh_div_2_1_rv64gc", "fdh_div_2_2_rv32gc",
|
||||
"fdh_div_2_2_rv64gc", "fdh_div_2_4_rv32gc", "fdh_div_2_4_rv64gc",
|
||||
"fdh_div_4_1_rv32gc", "fdh_div_4_1_rv64gc", "fdh_div_4_2_rv32gc",
|
||||
"fdh_div_4_2_rv64gc", "fdh_div_4_4_rv32gc", "fdh_div_4_4_rv64gc",
|
||||
"fd_div_2_1_rv32gc", "fd_div_2_1_rv64gc", "fd_div_2_2_rv32gc",
|
||||
"fd_div_2_2_rv64gc", "fd_div_2_4_rv32gc", "fd_div_2_4_rv64gc",
|
||||
"fd_div_4_1_rv32gc", "fd_div_4_1_rv64gc", "fd_div_4_2_rv32gc",
|
||||
"fd_div_4_2_rv64gc", "fd_div_4_4_rv32gc", "fd_div_4_4_rv64gc",
|
||||
"fdqh_div_2_1_rv32gc", "fdqh_div_2_1_rv64gc", "fdqh_div_2_2_rv32gc",
|
||||
"fdqh_div_2_2_rv64gc", "fdqh_div_2_4_rv32gc", "fdqh_div_2_4_rv64gc",
|
||||
"fdqh_div_4_1_rv32gc", "fdqh_div_4_1_rv64gc", "fdqh_div_4_2_rv32gc",
|
||||
"fdqh_div_4_2_rv64gc", "fdqh_div_4_4_rv32gc", "fdqh_div_4_4_rv64gc",
|
||||
"fdq_div_2_1_rv32gc", "fdq_div_2_1_rv64gc", "fdq_div_2_2_rv32gc",
|
||||
"fdq_div_2_2_rv64gc", "fdq_div_2_4_rv32gc", "fdq_div_2_4_rv64gc",
|
||||
"fdq_div_4_1_rv32gc", "fdq_div_4_1_rv64gc", "fdq_div_4_2_rv32gc",
|
||||
"fdq_div_4_2_rv64gc", "fdq_div_4_4_rv32gc", "fdq_div_4_4_rv64gc",
|
||||
"fh_div_2_1_rv32gc", "fh_div_2_1_rv64gc", "fh_div_2_2_rv32gc",
|
||||
"fh_div_2_2_rv64gc", "fh_div_2_4_rv32gc", "fh_div_2_4_rv64gc",
|
||||
"fh_div_4_1_rv32gc", "fh_div_4_1_rv64gc", "fh_div_4_2_rv32gc",
|
||||
"fh_div_4_2_rv64gc", "fh_div_4_4_rv32gc", "fh_div_4_4_rv64gc",
|
||||
"f_div_2_1_rv32gc", "f_div_2_1_rv64gc", "f_div_2_2_rv32gc",
|
||||
"f_div_2_2_rv64gc", "f_div_2_4_rv32gc", "f_div_2_4_rv64gc",
|
||||
"f_div_4_1_rv32gc", "f_div_4_1_rv64gc", "f_div_4_2_rv32gc",
|
||||
"f_div_4_2_rv64gc", "f_div_4_4_rv32gc", "f_div_4_4_rv64gc"
|
||||
]
|
||||
for config in testfloatdivconfigs:
|
||||
# div test case
|
||||
tests = ["div", "sqrt", "cvtint", "cvtfp"]
|
||||
if "f_" in config:
|
||||
tests.remove("cvtfp")
|
||||
for test in tests:
|
||||
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
||||
tc = TestCase(
|
||||
name=test,
|
||||
variant=config,
|
||||
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
||||
grepstr="All Tests completed with 0 errors",
|
||||
grepfile = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log")
|
||||
configs.append(tc)
|
||||
|
||||
|
||||
def main():
|
||||
def main(args):
|
||||
"""Run the tests and count the failures"""
|
||||
# global configs, args
|
||||
nightMode, sims, coverStr = process_args(args)
|
||||
configs = selectTests(nightMode, args, sims, coverStr)
|
||||
|
||||
os.chdir(regressionDir)
|
||||
dirs = ["questa/logs", "questa/wkdir", "verilator/logs", "verilator/wkdir", "vcs/logs", "vcs/wkdir"]
|
||||
for d in dirs:
|
||||
@ -561,4 +538,5 @@ def main():
|
||||
return num_fail
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
args = parse_args()
|
||||
sys.exit(main(args))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user