Github_Repos/cvw
1
0
Fork 1
mirror of https://github.com/openhwgroup/cvw synced 2025-02-11 06:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1156 from jordancarlin/update_ccov

Browse Source 
Use cvw-arch-verif tests for code coverage
This commit is contained in:
David Harris 2024-11-29 16:15:11 -08:00 committed by GitHub
commit 9b24938590
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 58 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
bin/regression-wally
View File

@ -34,7 +34,7 @@ tests = [
"arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma", "arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma",
"arch32zfh_divsqrt", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "wally32priv", "wally32periph", "arch32zcb", "arch32zfh_divsqrt", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "wally32priv", "wally32periph", "arch32zcb",
"arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]], "arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]],
["rv64i", ["arch64i"]] ["rv64i", ["arch64i"]]
] ]
# Separate test for short buildroot run through OpenSBI UART output # Separate test for short buildroot run through OpenSBI UART output
@ -265,8 +265,8 @@ class bcolors:
def addTests(tests, sim): def addTests(tests, sim):
sim_logdir = WALLY+ "/sim/" + sim + "/logs/" sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
for test in tests: for test in tests:
config = test[0]; config = test[0]
suites = test[1]; suites = test[1]
if (len(test) >= 3): if (len(test) >= 3):
args = " --args " + " ".join(test[2]) args = " --args " + " ".join(test[2])
else: else:
@ -291,18 +291,35 @@ def addTests(tests, sim):
configs.append(tc) configs.append(tc)
def addLockstepTestsByDir(dir, config, sim, fcovMode): def addTestsByDir(dir, config, sim, lockstepMode=0):
sim_logdir = WALLY+ "/sim/" + sim + "/logs/" if os.path.isdir(dir):
if (coverStr != ""): # use --fcov in place of --lockstep sim_logdir = WALLY+ "/sim/" + sim + "/logs/"
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config if coverStr == "--fcov": # use --fcov in place of --lockstep
else: cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
cmdPrefix="wsim --lockstep --sim " + sim + " " + config gs="Mismatches : 0"
if (os.path.isdir(dir)): if ("cvw-arch-verif/tests" in dir and not "priv" in dir):
fileEnd = "ALL.elf"
else:
fileEnd = ".elf"
elif coverStr == "--ccov":
cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config
gs="Single Elf file tests are not signatured verified."
if ("cvw-arch-verif/tests" in dir and not "priv" in dir):
fileEnd = "ALL.elf"
else:
fileEnd = ".elf"
elif lockstepMode:
cmdPrefix="wsim --lockstep --sim " + sim + " " + config
gs="Mismatches : 0"
fileEnd = ".elf"
else:
cmdPrefix="wsim --sim " + sim + " " + config
gs="Single Elf file tests are not signatured verified."
fileEnd = ".elf"
for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)): for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)):
for file in filenames: for file in filenames:
# fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files # fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files
if ((file.endswith(".elf") and (fcovMode == 0 or "tests/priv" in dir)) or if file.endswith(fileEnd):
(file.endswith("ALL.elf") and fcovMode == 1)):
fullfile = os.path.join(dirpath, file) fullfile = os.path.join(dirpath, file)
fields = fullfile.rsplit('/', 3) fields = fullfile.rsplit('/', 3)
if (fields[2] == "ref"): if (fields[2] == "ref"):
@ -318,7 +335,7 @@ def addLockstepTestsByDir(dir, config, sim, fcovMode):
name=file, name=file,
variant=config, variant=config,
cmd=cmdPrefix + " " + fullfile + " > " + sim_log, cmd=cmdPrefix + " " + fullfile + " > " + sim_log,
grepstr="Mismatches : 0", grepstr=gs,
grepfile = sim_log) grepfile = sim_log)
configs.append(tc) configs.append(tc)
else: else:
@ -379,14 +396,13 @@ parser.add_argument("--fcov", help="Functional Coverage", action="store_true")
parser.add_argument("--nightly", help="Run large nightly regression", action="store_true") parser.add_argument("--nightly", help="Run large nightly regression", action="store_true")
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true") parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true") parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true") parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
args = parser.parse_args() args = parser.parse_args()
if (args.nightly): if (args.nightly):
nightMode = "--nightly"; nightMode = "--nightly"
sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available
# sims = ["questa", "verilator"] # exercise all simulators; can omit a sim if no license is available
else: else:
nightMode = "" nightMode = ""
sims = [defaultsim] sims = [defaultsim]
@ -396,7 +412,7 @@ if (args.ccov): # only run RV64GC tests in coverage mode
elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode
coverStr = '--fcov' coverStr = '--fcov'
else: else:
coverStr = '' coverStr = ''
# Run Lint # Run Lint
@ -417,15 +433,14 @@ if (args.buildroot):
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode if (args.ccov): # only run RV64GC tests on Questa in code coverage mode
addTests(tests64gc_nofp, coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
if (args.fp): addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
addTests(tests64gc_fp, coveragesim) addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim)
elif (args.fcov): # run tests in lockstep in functional coverage mode elif (args.fcov): # run tests in lockstep in functional coverage mode
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim, 1) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim)
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, 1) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim)
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim, 1) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim)
addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, 1) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim)
#addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m/privilege/src/", "rv64gc", coveragesim, 0)
else: else:
for sim in sims: for sim in sims:
@ -437,9 +452,9 @@ else:
# run derivative configurations and lockstep tests in nightly regression # run derivative configurations and lockstep tests in nightly regression
if (args.nightly): if (args.nightly):
addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 0) addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1)
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 0) addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1)
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 0) addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1)
addTests(derivconfigtests, defaultsim) addTests(derivconfigtests, defaultsim)
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script. # addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
@ -540,17 +555,17 @@ def main():
else: else:
ImperasDVLicenseCount = 10000 # effectively no license limit for non-lockstep tests ImperasDVLicenseCount = 10000 # effectively no license limit for non-lockstep tests
with Pool(processes=min(len(configs),multiprocessing.cpu_count(), ImperasDVLicenseCount)) as pool: with Pool(processes=min(len(configs),multiprocessing.cpu_count(), ImperasDVLicenseCount)) as pool:
num_fail = 0 num_fail = 0
results = {} results = {}
for config in configs: for config in configs:
results[config] = pool.apply_async(run_test_case,(config, args.dryrun)) results[config] = pool.apply_async(run_test_case,(config, args.dryrun))
for (config,result) in results.items(): for (config,result) in results.items():
try: try:
num_fail+=result.get(timeout=TIMEOUT_DUR) num_fail+=result.get(timeout=TIMEOUT_DUR)
except TimeoutError: except TimeoutError:
pool.terminate() pool.terminate()
num_fail+=1 num_fail+=1
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR)) print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))
# Coverage report # Coverage report
if args.ccov: if args.ccov:

4
bin/wsim
View File

@ -101,10 +101,10 @@ if (args.lockstep or args.lockstepverbose):
if(args.locksteplog != 0): ImperasPlusArgs = " +IDV_TRACE2LOG=" + str(EnableLog) + " +IDV_TRACE2LOG_AFTER=" + str(args.locksteplog) if(args.locksteplog != 0): ImperasPlusArgs = " +IDV_TRACE2LOG=" + str(EnableLog) + " +IDV_TRACE2LOG_AFTER=" + str(args.locksteplog)
else: ImperasPlusArgs = "" else: ImperasPlusArgs = ""
if(args.fcov): if(args.fcov):
CovEnableStr = "1" if int(args.covlog) > 0 else "0"; CovEnableStr = "1" if int(args.covlog) > 0 else "0"
if(args.covlog >= 1): EnableLog = 1 if(args.covlog >= 1): EnableLog = 1
else: EnableLog = 0 else: EnableLog = 0
ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr; ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr
suffix = "" suffix = ""
else: else:
CovEnableStr = "" CovEnableStr = ""

4
sim/Makefile
View File

@ -29,8 +29,8 @@ deriv:
.PHONY: QuestaCodeCoverage collect_functcov combine_functcov remove_functcov_artifacts riscvdv riscvdv_functcov .PHONY: QuestaCodeCoverage collect_functcov combine_functcov remove_functcov_artifacts riscvdv riscvdv_functcov
QuestaCodeCoverage: questa/ucdb/rv64gc_arch64i.ucdb QuestaCodeCoverage:
vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log
# vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log # vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log
vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt
vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt