mirror of
https://github.com/openhwgroup/cvw
synced 2025-02-11 06:05:49 +00:00
Merge pull request #1156 from jordancarlin/update_ccov
Use cvw-arch-verif tests for code coverage
This commit is contained in:
commit
9b24938590
@ -34,7 +34,7 @@ tests = [
|
|||||||
"arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma",
|
"arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma",
|
||||||
"arch32zfh_divsqrt", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "wally32priv", "wally32periph", "arch32zcb",
|
"arch32zfh_divsqrt", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "wally32priv", "wally32periph", "arch32zcb",
|
||||||
"arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]],
|
"arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]],
|
||||||
["rv64i", ["arch64i"]]
|
["rv64i", ["arch64i"]]
|
||||||
]
|
]
|
||||||
|
|
||||||
# Separate test for short buildroot run through OpenSBI UART output
|
# Separate test for short buildroot run through OpenSBI UART output
|
||||||
@ -265,8 +265,8 @@ class bcolors:
|
|||||||
def addTests(tests, sim):
|
def addTests(tests, sim):
|
||||||
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
||||||
for test in tests:
|
for test in tests:
|
||||||
config = test[0];
|
config = test[0]
|
||||||
suites = test[1];
|
suites = test[1]
|
||||||
if (len(test) >= 3):
|
if (len(test) >= 3):
|
||||||
args = " --args " + " ".join(test[2])
|
args = " --args " + " ".join(test[2])
|
||||||
else:
|
else:
|
||||||
@ -291,18 +291,35 @@ def addTests(tests, sim):
|
|||||||
configs.append(tc)
|
configs.append(tc)
|
||||||
|
|
||||||
|
|
||||||
def addLockstepTestsByDir(dir, config, sim, fcovMode):
|
def addTestsByDir(dir, config, sim, lockstepMode=0):
|
||||||
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
if os.path.isdir(dir):
|
||||||
if (coverStr != ""): # use --fcov in place of --lockstep
|
sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
|
||||||
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
|
if coverStr == "--fcov": # use --fcov in place of --lockstep
|
||||||
else:
|
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
|
||||||
cmdPrefix="wsim --lockstep --sim " + sim + " " + config
|
gs="Mismatches : 0"
|
||||||
if (os.path.isdir(dir)):
|
if ("cvw-arch-verif/tests" in dir and not "priv" in dir):
|
||||||
|
fileEnd = "ALL.elf"
|
||||||
|
else:
|
||||||
|
fileEnd = ".elf"
|
||||||
|
elif coverStr == "--ccov":
|
||||||
|
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
|
||||||
|
gs="Single Elf file tests are not signatured verified."
|
||||||
|
if ("cvw-arch-verif/tests" in dir and not "priv" in dir):
|
||||||
|
fileEnd = "ALL.elf"
|
||||||
|
else:
|
||||||
|
fileEnd = ".elf"
|
||||||
|
elif lockstepMode:
|
||||||
|
cmdPrefix="wsim --lockstep --sim " + sim + " " + config
|
||||||
|
gs="Mismatches : 0"
|
||||||
|
fileEnd = ".elf"
|
||||||
|
else:
|
||||||
|
cmdPrefix="wsim --sim " + sim + " " + config
|
||||||
|
gs="Single Elf file tests are not signatured verified."
|
||||||
|
fileEnd = ".elf"
|
||||||
for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)):
|
for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)):
|
||||||
for file in filenames:
|
for file in filenames:
|
||||||
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
|
||||||
if ((file.endswith(".elf") and (fcovMode == 0 or "tests/priv" in dir)) or
|
if file.endswith(fileEnd):
|
||||||
(file.endswith("ALL.elf") and fcovMode == 1)):
|
|
||||||
fullfile = os.path.join(dirpath, file)
|
fullfile = os.path.join(dirpath, file)
|
||||||
fields = fullfile.rsplit('/', 3)
|
fields = fullfile.rsplit('/', 3)
|
||||||
if (fields[2] == "ref"):
|
if (fields[2] == "ref"):
|
||||||
@ -318,7 +335,7 @@ def addLockstepTestsByDir(dir, config, sim, fcovMode):
|
|||||||
name=file,
|
name=file,
|
||||||
variant=config,
|
variant=config,
|
||||||
cmd=cmdPrefix + " " + fullfile + " > " + sim_log,
|
cmd=cmdPrefix + " " + fullfile + " > " + sim_log,
|
||||||
grepstr="Mismatches : 0",
|
grepstr=gs,
|
||||||
grepfile = sim_log)
|
grepfile = sim_log)
|
||||||
configs.append(tc)
|
configs.append(tc)
|
||||||
else:
|
else:
|
||||||
@ -379,14 +396,13 @@ parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
|
|||||||
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
|
||||||
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
||||||
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
||||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true")
|
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
|
||||||
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if (args.nightly):
|
if (args.nightly):
|
||||||
nightMode = "--nightly";
|
nightMode = "--nightly"
|
||||||
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
|
||||||
# sims = ["questa", "verilator"] # exercise all simulators; can omit a sim if no license is available
|
|
||||||
else:
|
else:
|
||||||
nightMode = ""
|
nightMode = ""
|
||||||
sims = [defaultsim]
|
sims = [defaultsim]
|
||||||
@ -396,7 +412,7 @@ if (args.ccov): # only run RV64GC tests in coverage mode
|
|||||||
elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode
|
elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode
|
||||||
coverStr = '--fcov'
|
coverStr = '--fcov'
|
||||||
else:
|
else:
|
||||||
coverStr = ''
|
coverStr = ''
|
||||||
|
|
||||||
|
|
||||||
# Run Lint
|
# Run Lint
|
||||||
@ -417,15 +433,14 @@ if (args.buildroot):
|
|||||||
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
|
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
|
||||||
|
|
||||||
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode
|
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode
|
||||||
addTests(tests64gc_nofp, coveragesim)
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
||||||
if (args.fp):
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
||||||
addTests(tests64gc_fp, coveragesim)
|
addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim)
|
||||||
elif (args.fcov): # run tests in lockstep in functional coverage mode
|
elif (args.fcov): # run tests in lockstep in functional coverage mode
|
||||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim, 1)
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim)
|
||||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, 1)
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
|
||||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim, 1)
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim)
|
||||||
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, 1)
|
addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
|
||||||
#addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m/privilege/src/", "rv64gc", coveragesim, 0)
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
for sim in sims:
|
for sim in sims:
|
||||||
@ -437,9 +452,9 @@ else:
|
|||||||
|
|
||||||
# run derivative configurations and lockstep tests in nightly regression
|
# run derivative configurations and lockstep tests in nightly regression
|
||||||
if (args.nightly):
|
if (args.nightly):
|
||||||
addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 0)
|
addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1)
|
||||||
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 0)
|
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1)
|
||||||
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 0)
|
addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1)
|
||||||
addTests(derivconfigtests, defaultsim)
|
addTests(derivconfigtests, defaultsim)
|
||||||
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
||||||
|
|
||||||
@ -540,17 +555,17 @@ def main():
|
|||||||
else:
|
else:
|
||||||
ImperasDVLicenseCount = 10000 # effectively no license limit for non-lockstep tests
|
ImperasDVLicenseCount = 10000 # effectively no license limit for non-lockstep tests
|
||||||
with Pool(processes=min(len(configs),multiprocessing.cpu_count(), ImperasDVLicenseCount)) as pool:
|
with Pool(processes=min(len(configs),multiprocessing.cpu_count(), ImperasDVLicenseCount)) as pool:
|
||||||
num_fail = 0
|
num_fail = 0
|
||||||
results = {}
|
results = {}
|
||||||
for config in configs:
|
for config in configs:
|
||||||
results[config] = pool.apply_async(run_test_case,(config, args.dryrun))
|
results[config] = pool.apply_async(run_test_case,(config, args.dryrun))
|
||||||
for (config,result) in results.items():
|
for (config,result) in results.items():
|
||||||
try:
|
try:
|
||||||
num_fail+=result.get(timeout=TIMEOUT_DUR)
|
num_fail+=result.get(timeout=TIMEOUT_DUR)
|
||||||
except TimeoutError:
|
except TimeoutError:
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
num_fail+=1
|
num_fail+=1
|
||||||
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))
|
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))
|
||||||
|
|
||||||
# Coverage report
|
# Coverage report
|
||||||
if args.ccov:
|
if args.ccov:
|
||||||
|
4
bin/wsim
4
bin/wsim
@ -101,10 +101,10 @@ if (args.lockstep or args.lockstepverbose):
|
|||||||
if(args.locksteplog != 0): ImperasPlusArgs = " +IDV_TRACE2LOG=" + str(EnableLog) + " +IDV_TRACE2LOG_AFTER=" + str(args.locksteplog)
|
if(args.locksteplog != 0): ImperasPlusArgs = " +IDV_TRACE2LOG=" + str(EnableLog) + " +IDV_TRACE2LOG_AFTER=" + str(args.locksteplog)
|
||||||
else: ImperasPlusArgs = ""
|
else: ImperasPlusArgs = ""
|
||||||
if(args.fcov):
|
if(args.fcov):
|
||||||
CovEnableStr = "1" if int(args.covlog) > 0 else "0";
|
CovEnableStr = "1" if int(args.covlog) > 0 else "0"
|
||||||
if(args.covlog >= 1): EnableLog = 1
|
if(args.covlog >= 1): EnableLog = 1
|
||||||
else: EnableLog = 0
|
else: EnableLog = 0
|
||||||
ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr;
|
ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr
|
||||||
suffix = ""
|
suffix = ""
|
||||||
else:
|
else:
|
||||||
CovEnableStr = ""
|
CovEnableStr = ""
|
||||||
|
@ -29,8 +29,8 @@ deriv:
|
|||||||
|
|
||||||
.PHONY: QuestaCodeCoverage collect_functcov combine_functcov remove_functcov_artifacts riscvdv riscvdv_functcov
|
.PHONY: QuestaCodeCoverage collect_functcov combine_functcov remove_functcov_artifacts riscvdv riscvdv_functcov
|
||||||
|
|
||||||
QuestaCodeCoverage: questa/ucdb/rv64gc_arch64i.ucdb
|
QuestaCodeCoverage:
|
||||||
vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log
|
vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log
|
||||||
# vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log
|
# vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log
|
||||||
vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt
|
vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt
|
||||||
vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt
|
vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt
|
||||||
|
Loading…
Reference in New Issue
Block a user