mirror of
https://github.com/openhwgroup/cvw
synced 2025-02-11 06:05:49 +00:00
Begin refactoring regression-wally
This commit is contained in:
Jordan Carlin
parent
32b62e9d0c
commit
1a2569e6f1
@ -4,6 +4,7 @@
|
|||||||
# regression-wally
|
# regression-wally
|
||||||
# David_Harris@Hmc.edu 25 January 2021
|
# David_Harris@Hmc.edu 25 January 2021
|
||||||
# Modified by Jarred Allen <jaallen@g.hmc.edu> and many others
|
# Modified by Jarred Allen <jaallen@g.hmc.edu> and many others
|
||||||
|
# jcarlin@hmc.edu December 2024
|
||||||
# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
|
# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
|
||||||
#
|
#
|
||||||
# Run a regression with multiple configurations in parallel and exit with
|
# Run a regression with multiple configurations in parallel and exit with
|
||||||
@ -18,6 +19,14 @@ import multiprocessing
|
|||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
from multiprocessing import Pool, TimeoutError as MPTimeoutError
|
from multiprocessing import Pool, TimeoutError as MPTimeoutError
|
||||||
|
|
||||||
|
# Globals
|
||||||
|
WALLY = os.environ.get('WALLY')
|
||||||
|
regressionDir = WALLY + '/sim'
|
||||||
|
coveragesim = "questa" # Questa is required for code/functional coverage
|
||||||
|
defaultsim = "verilator" # Default simulator for all other tests
|
||||||
|
lockstepsim = "questa"
|
||||||
|
|
||||||
|
|
||||||
##################################
|
##################################
|
||||||
# Define lists of configurations and tests to run on each configuration
|
# Define lists of configurations and tests to run on each configuration
|
||||||
##################################
|
##################################
|
||||||
@ -38,6 +47,21 @@ tests = [
|
|||||||
["rv64i", ["arch64i"]]
|
["rv64i", ["arch64i"]]
|
||||||
]
|
]
|
||||||
|
|
||||||
|
tests64gc_fp = [
|
||||||
|
["rv64gc", ["arch64f", "arch64d", "arch64zfh",
|
||||||
|
"arch64f_fma", "arch64d_fma", "arch64zfh_fma",
|
||||||
|
"arch64f_divsqrt", "arch64d_divsqrt", "arch64zfh_divsqrt",
|
||||||
|
"arch64zfaf", "arch64zfad"]]
|
||||||
|
]
|
||||||
|
|
||||||
|
# Separate out floating-point tests for RV64 to speed up coverage
|
||||||
|
tests64gc_nofp = [
|
||||||
|
["rv64gc", ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m", "arch64zcb",
|
||||||
|
"arch64zifencei", "arch64zicond", "arch64a_amo", "wally64a_lrsc", "wally64periph", "wally64priv",
|
||||||
|
"arch64zbkb", "arch64zbkc", "arch64zbkx", "arch64zknd", "arch64zkne", "arch64zknh",
|
||||||
|
"arch64zba", "arch64zbb", "arch64zbc", "arch64zbs"]] # add when working: "arch64zicboz"
|
||||||
|
]
|
||||||
|
|
||||||
# Separate test for short buildroot run through OpenSBI UART output
|
# Separate test for short buildroot run through OpenSBI UART output
|
||||||
tests_buildrootshort = [
|
tests_buildrootshort = [
|
||||||
["buildroot", ["buildroot"], ["+INSTR_LIMIT=1400000"], # Instruction limit gets to first OpenSBI UART output
|
["buildroot", ["buildroot"], ["+INSTR_LIMIT=1400000"], # Instruction limit gets to first OpenSBI UART output
|
||||||
@ -55,22 +79,6 @@ tests_buildrootbootlockstep = [
|
|||||||
"WallyHostname login: ", "buildroot_uart.out"]
|
"WallyHostname login: ", "buildroot_uart.out"]
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
# Separate out floating-point tests for RV64 to speed up coverage
|
|
||||||
tests64gc_nofp = [
|
|
||||||
["rv64gc", ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m", "arch64zcb",
|
|
||||||
"arch64zifencei", "arch64zicond", "arch64a_amo", "wally64a_lrsc", "wally64periph", "wally64priv",
|
|
||||||
"arch64zbkb", "arch64zbkc", "arch64zbkx", "arch64zknd", "arch64zkne", "arch64zknh",
|
|
||||||
"arch64zba", "arch64zbb", "arch64zbc", "arch64zbs"]] # add when working: "arch64zicboz"
|
|
||||||
]
|
|
||||||
|
|
||||||
tests64gc_fp = [
|
|
||||||
["rv64gc", ["arch64f", "arch64d", "arch64zfh",
|
|
||||||
"arch64f_fma", "arch64d_fma", "arch64zfh_fma",
|
|
||||||
"arch64f_divsqrt", "arch64d_divsqrt", "arch64zfh_divsqrt",
|
|
||||||
"arch64zfaf", "arch64zfad"]]
|
|
||||||
]
|
|
||||||
|
|
||||||
derivconfigtests = [
|
derivconfigtests = [
|
||||||
# memory system
|
# memory system
|
||||||
["tlb2_rv32gc", ["wally32priv"]],
|
["tlb2_rv32gc", ["wally32priv"]],
|
||||||
@ -93,21 +101,21 @@ derivconfigtests = [
|
|||||||
["ram_1_1_rv64gc", ["ahb64"]],
|
["ram_1_1_rv64gc", ["ahb64"]],
|
||||||
["ram_2_0_rv64gc", ["ahb64"]],
|
["ram_2_0_rv64gc", ["ahb64"]],
|
||||||
["ram_2_1_rv64gc", ["ahb64"]],
|
["ram_2_1_rv64gc", ["ahb64"]],
|
||||||
# RV32 cacheless designs will not work unless DTIM supports FLEN > XLEN. This support is not planned.
|
# RV32 cacheless designs will not work unless DTIM supports FLEN > XLEN. This support is not planned.
|
||||||
# ["nodcache_rv32gc", ["ahb32"]],
|
# ["nodcache_rv32gc", ["ahb32"]],
|
||||||
# ["nocache_rv32gc", ["ahb32"]],
|
# ["nocache_rv32gc", ["ahb32"]],
|
||||||
["noicache_rv32gc", ["ahb32"]],
|
["noicache_rv32gc", ["ahb32"]],
|
||||||
["noicache_rv64gc", ["ahb64"]],
|
["noicache_rv64gc", ["ahb64"]],
|
||||||
["nodcache_rv64gc", ["ahb64"]],
|
["nodcache_rv64gc", ["ahb64"]],
|
||||||
["nocache_rv64gc", ["ahb64"]],
|
["nocache_rv64gc", ["ahb64"]],
|
||||||
|
|
||||||
# Atomic variants
|
# Atomic variants
|
||||||
["zaamo_rv64gc", ["arch64i", "arch64a_amo"]],
|
["zaamo_rv64gc", ["arch64i", "arch64a_amo"]],
|
||||||
["zalrsc_rv64gc", ["arch64i", "wally64a_lrsc"]],
|
["zalrsc_rv64gc", ["arch64i", "wally64a_lrsc"]],
|
||||||
["zaamo_rv32gc", ["arch32i", "arch32a_amo"]],
|
["zaamo_rv32gc", ["arch32i", "arch32a_amo"]],
|
||||||
["zalrsc_rv32gc", ["arch32i", "wally32a_lrsc"]],
|
["zalrsc_rv32gc", ["arch32i", "wally32a_lrsc"]],
|
||||||
|
|
||||||
# Bit manipulation and crypto variants
|
# Bit manipulation and crypto variants
|
||||||
["zba_rv32gc", ["arch32i", "arch32zba"]],
|
["zba_rv32gc", ["arch32i", "arch32zba"]],
|
||||||
["zbb_rv32gc", ["arch32i", "arch32zbb"]],
|
["zbb_rv32gc", ["arch32i", "arch32zbb"]],
|
||||||
["zbc_rv32gc", ["arch32i", "arch32zbc"]],
|
["zbc_rv32gc", ["arch32i", "arch32zbc"]],
|
||||||
@ -130,7 +138,7 @@ derivconfigtests = [
|
|||||||
["zknd_rv64gc", ["arch64i", "arch64zknd"]],
|
["zknd_rv64gc", ["arch64i", "arch64zknd"]],
|
||||||
["zknh_rv64gc", ["arch64i", "arch64zknh"]],
|
["zknh_rv64gc", ["arch64i", "arch64zknh"]],
|
||||||
|
|
||||||
# No privilege modes variants
|
# No privilege modes variants
|
||||||
["noS_rv32gc", ["arch32i", "arch32f", "arch32priv", "arch32c", "arch32m", "arch32a_amo", "arch32zifencei", "arch32zicond",
|
["noS_rv32gc", ["arch32i", "arch32f", "arch32priv", "arch32c", "arch32m", "arch32a_amo", "arch32zifencei", "arch32zicond",
|
||||||
"arch32zba", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "arch32zcb", "arch32zbkx", "arch32zknd"]],
|
"arch32zba", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "arch32zcb", "arch32zbkx", "arch32zknd"]],
|
||||||
["noS_rv64gc", ["arch64i", "arch64f", "arch64priv", "arch64c", "arch64m", "arch64a_amo", "arch64zifencei", "arch64zicond",
|
["noS_rv64gc", ["arch64i", "arch64f", "arch64priv", "arch64c", "arch64m", "arch64a_amo", "arch64zifencei", "arch64zicond",
|
||||||
@ -182,7 +190,6 @@ derivconfigtests = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
bpredtests = [
|
bpredtests = [
|
||||||
|
|
||||||
["nobpred_rv32gc", ["rv32i"]],
|
["nobpred_rv32gc", ["rv32i"]],
|
||||||
["bpred_TWOBIT_6_16_10_0_rv32gc", ["embench"], "-GPrintHPMCounters=1"],
|
["bpred_TWOBIT_6_16_10_0_rv32gc", ["embench"], "-GPrintHPMCounters=1"],
|
||||||
["bpred_TWOBIT_8_16_10_0_rv32gc", ["embench"], "-GPrintHPMCounters=1"],
|
["bpred_TWOBIT_8_16_10_0_rv32gc", ["embench"], "-GPrintHPMCounters=1"],
|
||||||
@ -231,11 +238,38 @@ bpredtests = [
|
|||||||
["bpred_GSHARE_10_10_10_1_rv32gc", ["embench"], "-GPrintHPMCounters=1"]
|
["bpred_GSHARE_10_10_10_1_rv32gc", ["embench"], "-GPrintHPMCounters=1"]
|
||||||
]
|
]
|
||||||
|
|
||||||
|
testfloatdivconfigs = [
|
||||||
|
"fdh_div_2_1_rv32gc", "fdh_div_2_1_rv64gc", "fdh_div_2_2_rv32gc",
|
||||||
|
"fdh_div_2_2_rv64gc", "fdh_div_2_4_rv32gc", "fdh_div_2_4_rv64gc",
|
||||||
|
"fdh_div_4_1_rv32gc", "fdh_div_4_1_rv64gc", "fdh_div_4_2_rv32gc",
|
||||||
|
"fdh_div_4_2_rv64gc", "fdh_div_4_4_rv32gc", "fdh_div_4_4_rv64gc",
|
||||||
|
"fd_div_2_1_rv32gc", "fd_div_2_1_rv64gc", "fd_div_2_2_rv32gc",
|
||||||
|
"fd_div_2_2_rv64gc", "fd_div_2_4_rv32gc", "fd_div_2_4_rv64gc",
|
||||||
|
"fd_div_4_1_rv32gc", "fd_div_4_1_rv64gc", "fd_div_4_2_rv32gc",
|
||||||
|
"fd_div_4_2_rv64gc", "fd_div_4_4_rv32gc", "fd_div_4_4_rv64gc",
|
||||||
|
"fdqh_div_2_1_rv32gc", "fdqh_div_2_1_rv64gc", "fdqh_div_2_2_rv32gc",
|
||||||
|
"fdqh_div_2_2_rv64gc", "fdqh_div_2_4_rv32gc", "fdqh_div_2_4_rv64gc",
|
||||||
|
"fdqh_div_4_1_rv32gc", "fdqh_div_4_1_rv64gc", "fdqh_div_4_2_rv32gc",
|
||||||
|
"fdqh_div_4_2_rv64gc", "fdqh_div_4_4_rv32gc", "fdqh_div_4_4_rv64gc",
|
||||||
|
"fdq_div_2_1_rv32gc", "fdq_div_2_1_rv64gc", "fdq_div_2_2_rv32gc",
|
||||||
|
"fdq_div_2_2_rv64gc", "fdq_div_2_4_rv32gc", "fdq_div_2_4_rv64gc",
|
||||||
|
"fdq_div_4_1_rv32gc", "fdq_div_4_1_rv64gc", "fdq_div_4_2_rv32gc",
|
||||||
|
"fdq_div_4_2_rv64gc", "fdq_div_4_4_rv32gc", "fdq_div_4_4_rv64gc",
|
||||||
|
"fh_div_2_1_rv32gc", "fh_div_2_1_rv64gc", "fh_div_2_2_rv32gc",
|
||||||
|
"fh_div_2_2_rv64gc", "fh_div_2_4_rv32gc", "fh_div_2_4_rv64gc",
|
||||||
|
"fh_div_4_1_rv32gc", "fh_div_4_1_rv64gc", "fh_div_4_2_rv32gc",
|
||||||
|
"fh_div_4_2_rv64gc", "fh_div_4_4_rv32gc", "fh_div_4_4_rv64gc",
|
||||||
|
"f_div_2_1_rv32gc", "f_div_2_1_rv64gc", "f_div_2_2_rv32gc",
|
||||||
|
"f_div_2_2_rv64gc", "f_div_2_4_rv32gc", "f_div_2_4_rv64gc",
|
||||||
|
"f_div_4_1_rv32gc", "f_div_4_1_rv64gc", "f_div_4_2_rv32gc",
|
||||||
|
"f_div_4_2_rv64gc", "f_div_4_4_rv32gc", "f_div_4_4_rv64gc"
|
||||||
|
]
|
||||||
|
|
||||||
# list of tests not supported by ImperasDV yet that should be waived during lockstep testing
|
# list of tests not supported by ImperasDV yet that should be waived during lockstep testing
|
||||||
lockstepwaivers = [
|
lockstepwaivers = [
|
||||||
"WALLY-q-01.S_ref.elf", # Q extension is not supported by ImperasDV
|
"WALLY-q-01.S_ref.elf", # Q extension is not supported by ImperasDV
|
||||||
"coverage_tlbMisaligned.elf", # Issue 976: ImperasDV bug disagrees with Wally related to misaligned pages when PBMT makes page uncachable
|
"coverage_tlbMisaligned.elf", # Issue 976: ImperasDV bug disagrees with Wally related to misaligned pages when PBMT makes page uncachable
|
||||||
"WALLY-cbom-01.S_ref.elf" #, # cbom extension is not supported by ImperasDV because there is no cache model in ImperasDV
|
"WALLY-cbom-01.S_ref.elf", # cbom extension is not supported by ImperasDV because there is no cache model in ImperasDV
|
||||||
]
|
]
|
||||||
|
|
||||||
##################################
|
##################################
|
||||||
@ -263,9 +297,9 @@ class bcolors:
|
|||||||
BOLD = '\033[1m'
|
BOLD = '\033[1m'
|
||||||
UNDERLINE = '\033[4m'
|
UNDERLINE = '\033[4m'
|
||||||
|
|
||||||
def addTests(tests, sim):
|
def addTests(testList, sim, coverStr, configs):
|
||||||
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
||||||
for test in tests:
|
for test in testList:
|
||||||
config = test[0]
|
config = test[0]
|
||||||
suites = test[1]
|
suites = test[1]
|
||||||
args = f" --args {test[2]}" if len(test) >= 3 else ""
|
args = f" --args {test[2]}" if len(test) >= 3 else ""
|
||||||
@ -283,56 +317,37 @@ def addTests(tests, sim):
|
|||||||
configs.append(tc)
|
configs.append(tc)
|
||||||
|
|
||||||
|
|
||||||
def addTestsByDir(testDir, config, sim, lockstepMode=0):
|
def addTestsByDir(testDir, config, sim, coverStr, configs, lockstepMode=0):
|
||||||
if os.path.isdir(testDir):
|
if not os.path.isdir(testDir):
|
||||||
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
|
||||||
if coverStr == "--fcov": # use --fcov in place of --lockstep
|
|
||||||
cmdPrefix=f"wsim --sim {sim} {coverStr} {config}"
|
|
||||||
gs="Mismatches : 0"
|
|
||||||
if ("cvw-arch-verif/tests" in testDir and "priv" not in testDir):
|
|
||||||
fileEnd = "ALL.elf"
|
|
||||||
else:
|
|
||||||
fileEnd = ".elf"
|
|
||||||
elif coverStr == "--ccov":
|
|
||||||
cmdPrefix=f"wsim --sim {sim} {coverStr} {config}"
|
|
||||||
gs="Single Elf file tests are not signatured verified."
|
|
||||||
if ("cvw-arch-verif/tests" in testDir and "priv" not in testDir):
|
|
||||||
fileEnd = "ALL.elf"
|
|
||||||
else:
|
|
||||||
fileEnd = ".elf"
|
|
||||||
elif lockstepMode:
|
|
||||||
cmdPrefix=f"wsim --lockstep --sim {sim} {config}"
|
|
||||||
gs="Mismatches : 0"
|
|
||||||
fileEnd = ".elf"
|
|
||||||
else:
|
|
||||||
cmdPrefix=f"wsim --sim {sim} {config}"
|
|
||||||
gs="Single Elf file tests are not signatured verified."
|
|
||||||
fileEnd = ".elf"
|
|
||||||
for dirpath, _, filenames in os.walk(os.path.abspath(testDir)):
|
|
||||||
for file in filenames:
|
|
||||||
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
|
||||||
if file.endswith(fileEnd):
|
|
||||||
fullfile = os.path.join(dirpath, file)
|
|
||||||
fields = fullfile.rsplit('/', 3)
|
|
||||||
if fields[2] == "ref":
|
|
||||||
shortelf = f"{fields[1]}_{fields[3]}"
|
|
||||||
else:
|
|
||||||
shortelf = f"{fields[2]}_{fields[3]}"
|
|
||||||
if shortelf in lockstepwaivers: # skip tests that itch bugs in ImperasDV
|
|
||||||
print(f"{bcolors.WARNING}Skipping waived test {shortelf}{bcolors.ENDC}")
|
|
||||||
continue
|
|
||||||
sim_log = f"{sim_logdir}{config}_{shortelf}.log"
|
|
||||||
tc = TestCase(
|
|
||||||
name=file,
|
|
||||||
variant=config,
|
|
||||||
cmd=f"{cmdPrefix} {fullfile} > {sim_log}",
|
|
||||||
grepstr=gs,
|
|
||||||
grepfile = sim_log)
|
|
||||||
configs.append(tc)
|
|
||||||
else:
|
|
||||||
print(f"Error: Directory not found: {testDir}")
|
print(f"Error: Directory not found: {testDir}")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
sim_logdir = f"{WALLY}/sim/{sim}/logs/"
|
||||||
|
cmdPrefix = f"wsim --sim {sim} {coverStr} {'--lockstep' if lockstepMode else ''} {config}"
|
||||||
|
gs = "Mismatches : 0" if lockstepMode or coverStr == "--fcov" else "Single Elf file tests are not signatured verified."
|
||||||
|
# fcov/ccov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
||||||
|
fileEnd = "ALL.elf" if "cvw-arch-verif/tests" in testDir and "priv" not in testDir and (coverStr == "--fcov" or coverStr == "--ccov") else ".elf"
|
||||||
|
for dirpath, _, filenames in os.walk(os.path.abspath(testDir)):
|
||||||
|
for file in filenames:
|
||||||
|
if file.endswith(fileEnd):
|
||||||
|
fullfile = os.path.join(dirpath, file)
|
||||||
|
fields = fullfile.rsplit('/', 3)
|
||||||
|
if fields[2] == "ref":
|
||||||
|
shortelf = f"{fields[1]}_{fields[3]}"
|
||||||
|
else:
|
||||||
|
shortelf = f"{fields[2]}_{fields[3]}"
|
||||||
|
if shortelf in lockstepwaivers: # skip tests that itch bugs in ImperasDV
|
||||||
|
print(f"{bcolors.WARNING}Skipping waived test {shortelf}{bcolors.ENDC}")
|
||||||
|
continue
|
||||||
|
sim_log = f"{sim_logdir}{config}_{shortelf}.log"
|
||||||
|
tc = TestCase(
|
||||||
|
name=file,
|
||||||
|
variant=config,
|
||||||
|
cmd=f"{cmdPrefix} {fullfile} > {sim_log}",
|
||||||
|
grepstr=gs,
|
||||||
|
grepfile = sim_log)
|
||||||
|
configs.append(tc)
|
||||||
|
|
||||||
def search_log_for_text(text, grepfile):
|
def search_log_for_text(text, grepfile):
|
||||||
"""Search through the given log file for text, returning True if it is found or False if it is not"""
|
"""Search through the given log file for text, returning True if it is found or False if it is not"""
|
||||||
grepwarn = f"grep -i -H Warning: {grepfile}"
|
grepwarn = f"grep -i -H Warning: {grepfile}"
|
||||||
@ -365,154 +380,116 @@ def run_test_case(config, dryrun: bool = False):
|
|||||||
print(f" Check {grepfile}", flush=True)
|
print(f" Check {grepfile}", flush=True)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
##################################
|
def parse_args():
|
||||||
# Main body
|
parser = argparse.ArgumentParser()
|
||||||
##################################
|
parser.add_argument("--ccov", help="Code Coverage", action="store_true")
|
||||||
|
parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
|
||||||
|
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
||||||
|
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
||||||
|
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
||||||
|
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
|
||||||
|
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
def process_args(args):
|
||||||
|
if args.nightly:
|
||||||
|
nightMode = "--nightly"
|
||||||
|
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
||||||
|
else:
|
||||||
|
nightMode = ""
|
||||||
|
sims = [defaultsim]
|
||||||
|
|
||||||
|
if args.ccov:
|
||||||
|
coverStr = "--ccov"
|
||||||
|
elif args.fcov:
|
||||||
|
coverStr = "--fcov"
|
||||||
|
else:
|
||||||
|
coverStr = ""
|
||||||
|
return nightMode, sims, coverStr
|
||||||
|
|
||||||
|
|
||||||
WALLY = os.environ.get('WALLY')
|
def selectTests(nightMode, args, sims, coverStr):
|
||||||
regressionDir = WALLY + '/sim'
|
# Run Lint
|
||||||
os.chdir(regressionDir)
|
configs = [
|
||||||
|
TestCase(
|
||||||
|
name="lints",
|
||||||
|
variant="all",
|
||||||
|
cmd=f"lint-wally {nightMode} | tee {WALLY}/sim/verilator/logs/all_lints.log",
|
||||||
|
grepstr="lints run with no errors or warnings",
|
||||||
|
grepfile = f"{WALLY}/sim/verilator/logs/all_lints.log")
|
||||||
|
]
|
||||||
|
|
||||||
coveragesim = "questa" # Questa is required for code/functional coverage
|
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
|
||||||
#defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready
|
if args.buildroot:
|
||||||
defaultsim = "verilator" # Default simulator for all other tests
|
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
|
||||||
lockstepsim = "questa"
|
addTests(tests_buildrootbootlockstep, lockstepsim, coverStr, configs) # lockstep with Questa and ImperasDV runs overnight
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
if args.ccov: # only run RV64GC tests on Questa in code coverage mode
|
||||||
parser.add_argument("--ccov", help="Code Coverage", action="store_true")
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||||
parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||||
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim, coverStr, configs)
|
||||||
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
elif args.fcov: # run tests in lockstep in functional coverage mode
|
||||||
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim, coverStr, configs)
|
||||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||||
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim, coverStr, configs)
|
||||||
args = parser.parse_args()
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, coverStr, configs)
|
||||||
|
else:
|
||||||
|
for sim in sims:
|
||||||
|
if not (args.buildroot and sim == lockstepsim): # skip short buildroot sim if running long one
|
||||||
|
addTests(tests_buildrootshort, sim, coverStr, configs)
|
||||||
|
addTests(tests, sim, coverStr, configs)
|
||||||
|
addTests(tests64gc_nofp, sim, coverStr, configs)
|
||||||
|
addTests(tests64gc_fp, sim, coverStr, configs)
|
||||||
|
|
||||||
if args.nightly:
|
# run derivative configurations and lockstep tests in nightly regression
|
||||||
nightMode = "--nightly"
|
if args.nightly:
|
||||||
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1, coverStr, configs)
|
||||||
else:
|
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, coverStr, configs, 1)
|
||||||
nightMode = ""
|
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, coverStr, configs, 1)
|
||||||
sims = [defaultsim]
|
addTests(derivconfigtests, defaultsim, coverStr, configs)
|
||||||
|
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
||||||
|
|
||||||
if args.ccov: # only run RV64GC tests in coverage mode
|
# testfloat tests
|
||||||
coverStr = '--ccov'
|
if args.testfloat: # for testfloat alone, just run testfloat tests
|
||||||
elif args.fcov: # only run RV64GC tests in lockstep in coverage mode
|
configs = []
|
||||||
coverStr = '--fcov'
|
if (args.testfloat or args.nightly): # for nightly, run testfloat along with others
|
||||||
else:
|
testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved
|
||||||
coverStr = ''
|
testfloatconfigs = ["fdqh_rv64gc", "fdq_rv64gc", "fdh_rv64gc", "fd_rv64gc", "fh_rv64gc", "f_rv64gc", "fdqh_rv32gc", "f_rv32gc"]
|
||||||
|
for config in testfloatconfigs:
|
||||||
|
tests = ["div", "sqrt", "add", "sub", "mul", "cvtint", "cvtfp", "fma", "cmp"]
|
||||||
|
if "f_" in config:
|
||||||
|
tests.remove("cvtfp")
|
||||||
|
for test in tests:
|
||||||
|
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
||||||
|
tc = TestCase(
|
||||||
|
name=test,
|
||||||
|
variant=config,
|
||||||
|
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
||||||
|
grepstr="All Tests completed with 0 errors",
|
||||||
|
grepfile = sim_log)
|
||||||
|
configs.append(tc)
|
||||||
|
for config in testfloatdivconfigs:
|
||||||
|
# div test case
|
||||||
|
tests = ["div", "sqrt", "cvtint", "cvtfp"]
|
||||||
|
if "f_" in config:
|
||||||
|
tests.remove("cvtfp")
|
||||||
|
for test in tests:
|
||||||
|
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
||||||
|
tc = TestCase(
|
||||||
|
name=test,
|
||||||
|
variant=config,
|
||||||
|
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
||||||
|
grepstr="All Tests completed with 0 errors",
|
||||||
|
grepfile = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log")
|
||||||
|
configs.append(tc)
|
||||||
|
return configs
|
||||||
|
|
||||||
|
|
||||||
# Run Lint
|
def main(args):
|
||||||
configs = [
|
|
||||||
TestCase(
|
|
||||||
name="lints",
|
|
||||||
variant="all",
|
|
||||||
cmd=f"lint-wally {nightMode} | tee {WALLY}/sim/verilator/logs/all_lints.log",
|
|
||||||
grepstr="lints run with no errors or warnings",
|
|
||||||
grepfile = f"{WALLY}/sim/verilator/logs/all_lints.log")
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
|
|
||||||
if args.buildroot:
|
|
||||||
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
|
|
||||||
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
|
|
||||||
|
|
||||||
if args.ccov: # only run RV64GC tests on Questa in code coverage mode
|
|
||||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
|
||||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
|
||||||
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim)
|
|
||||||
elif args.fcov: # run tests in lockstep in functional coverage mode
|
|
||||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim)
|
|
||||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
|
||||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim)
|
|
||||||
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
|
||||||
|
|
||||||
else:
|
|
||||||
for sim in sims:
|
|
||||||
if not (args.buildroot and sim == lockstepsim): # skip short buildroot sim if running long one
|
|
||||||
addTests(tests_buildrootshort, sim)
|
|
||||||
addTests(tests, sim)
|
|
||||||
addTests(tests64gc_nofp, sim)
|
|
||||||
addTests(tests64gc_fp, sim)
|
|
||||||
|
|
||||||
# run derivative configurations and lockstep tests in nightly regression
|
|
||||||
if args.nightly:
|
|
||||||
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1)
|
|
||||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1)
|
|
||||||
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1)
|
|
||||||
addTests(derivconfigtests, defaultsim)
|
|
||||||
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
|
||||||
|
|
||||||
# testfloat tests
|
|
||||||
if args.testfloat: # for testfloat alone, just run testfloat tests
|
|
||||||
configs = []
|
|
||||||
if (args.testfloat or args.nightly): # for nightly, run testfloat along with others
|
|
||||||
testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved
|
|
||||||
testfloatconfigs = ["fdqh_rv64gc", "fdq_rv64gc", "fdh_rv64gc", "fd_rv64gc", "fh_rv64gc", "f_rv64gc", "fdqh_rv32gc", "f_rv32gc"]
|
|
||||||
for config in testfloatconfigs:
|
|
||||||
tests = ["div", "sqrt", "add", "sub", "mul", "cvtint", "cvtfp", "fma", "cmp"]
|
|
||||||
if "f_" in config:
|
|
||||||
tests.remove("cvtfp")
|
|
||||||
for test in tests:
|
|
||||||
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
|
||||||
tc = TestCase(
|
|
||||||
name=test,
|
|
||||||
variant=config,
|
|
||||||
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
|
||||||
grepstr="All Tests completed with 0 errors",
|
|
||||||
grepfile = sim_log)
|
|
||||||
configs.append(tc)
|
|
||||||
|
|
||||||
|
|
||||||
testfloatdivconfigs = [
|
|
||||||
"fdh_div_2_1_rv32gc", "fdh_div_2_1_rv64gc", "fdh_div_2_2_rv32gc",
|
|
||||||
"fdh_div_2_2_rv64gc", "fdh_div_2_4_rv32gc", "fdh_div_2_4_rv64gc",
|
|
||||||
"fdh_div_4_1_rv32gc", "fdh_div_4_1_rv64gc", "fdh_div_4_2_rv32gc",
|
|
||||||
"fdh_div_4_2_rv64gc", "fdh_div_4_4_rv32gc", "fdh_div_4_4_rv64gc",
|
|
||||||
"fd_div_2_1_rv32gc", "fd_div_2_1_rv64gc", "fd_div_2_2_rv32gc",
|
|
||||||
"fd_div_2_2_rv64gc", "fd_div_2_4_rv32gc", "fd_div_2_4_rv64gc",
|
|
||||||
"fd_div_4_1_rv32gc", "fd_div_4_1_rv64gc", "fd_div_4_2_rv32gc",
|
|
||||||
"fd_div_4_2_rv64gc", "fd_div_4_4_rv32gc", "fd_div_4_4_rv64gc",
|
|
||||||
"fdqh_div_2_1_rv32gc", "fdqh_div_2_1_rv64gc", "fdqh_div_2_2_rv32gc",
|
|
||||||
"fdqh_div_2_2_rv64gc", "fdqh_div_2_4_rv32gc", "fdqh_div_2_4_rv64gc",
|
|
||||||
"fdqh_div_4_1_rv32gc", "fdqh_div_4_1_rv64gc", "fdqh_div_4_2_rv32gc",
|
|
||||||
"fdqh_div_4_2_rv64gc", "fdqh_div_4_4_rv32gc", "fdqh_div_4_4_rv64gc",
|
|
||||||
"fdq_div_2_1_rv32gc", "fdq_div_2_1_rv64gc", "fdq_div_2_2_rv32gc",
|
|
||||||
"fdq_div_2_2_rv64gc", "fdq_div_2_4_rv32gc", "fdq_div_2_4_rv64gc",
|
|
||||||
"fdq_div_4_1_rv32gc", "fdq_div_4_1_rv64gc", "fdq_div_4_2_rv32gc",
|
|
||||||
"fdq_div_4_2_rv64gc", "fdq_div_4_4_rv32gc", "fdq_div_4_4_rv64gc",
|
|
||||||
"fh_div_2_1_rv32gc", "fh_div_2_1_rv64gc", "fh_div_2_2_rv32gc",
|
|
||||||
"fh_div_2_2_rv64gc", "fh_div_2_4_rv32gc", "fh_div_2_4_rv64gc",
|
|
||||||
"fh_div_4_1_rv32gc", "fh_div_4_1_rv64gc", "fh_div_4_2_rv32gc",
|
|
||||||
"fh_div_4_2_rv64gc", "fh_div_4_4_rv32gc", "fh_div_4_4_rv64gc",
|
|
||||||
"f_div_2_1_rv32gc", "f_div_2_1_rv64gc", "f_div_2_2_rv32gc",
|
|
||||||
"f_div_2_2_rv64gc", "f_div_2_4_rv32gc", "f_div_2_4_rv64gc",
|
|
||||||
"f_div_4_1_rv32gc", "f_div_4_1_rv64gc", "f_div_4_2_rv32gc",
|
|
||||||
"f_div_4_2_rv64gc", "f_div_4_4_rv32gc", "f_div_4_4_rv64gc"
|
|
||||||
]
|
|
||||||
for config in testfloatdivconfigs:
|
|
||||||
# div test case
|
|
||||||
tests = ["div", "sqrt", "cvtint", "cvtfp"]
|
|
||||||
if "f_" in config:
|
|
||||||
tests.remove("cvtfp")
|
|
||||||
for test in tests:
|
|
||||||
sim_log = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log"
|
|
||||||
tc = TestCase(
|
|
||||||
name=test,
|
|
||||||
variant=config,
|
|
||||||
cmd=f"wsim --tb testbench_fp --sim {testfloatsim} {config} {test} > {sim_log}",
|
|
||||||
grepstr="All Tests completed with 0 errors",
|
|
||||||
grepfile = f"{WALLY}/sim/{testfloatsim}/logs/{config}_{test}.log")
|
|
||||||
configs.append(tc)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""Run the tests and count the failures"""
|
"""Run the tests and count the failures"""
|
||||||
# global configs, args
|
nightMode, sims, coverStr = process_args(args)
|
||||||
|
configs = selectTests(nightMode, args, sims, coverStr)
|
||||||
|
|
||||||
os.chdir(regressionDir)
|
os.chdir(regressionDir)
|
||||||
dirs = ["questa/logs", "questa/wkdir", "verilator/logs", "verilator/wkdir", "vcs/logs", "vcs/wkdir"]
|
dirs = ["questa/logs", "questa/wkdir", "verilator/logs", "verilator/wkdir", "vcs/logs", "vcs/wkdir"]
|
||||||
for d in dirs:
|
for d in dirs:
|
||||||
@ -561,4 +538,5 @@ def main():
|
|||||||
return num_fail
|
return num_fail
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
sys.exit(main())
|
args = parse_args()
|
||||||
|
sys.exit(main(args))
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user