diff --git a/Makefile b/Makefile index 740287dcc..050f066fa 100644 --- a/Makefile +++ b/Makefile @@ -91,9 +91,11 @@ combine_functcov: mkdir -p ${SIM}/questa/functcov mkdir -p ${SIM}/questa/functcov_logs cd ${SIM}/questa/functcov && rm -rf * - run-elf-cov.bash --seed ${SIM}/questa/seed0.txt --verbose --coverdb ${SIM}/questa/functcov/add.ucdb --elf ${WALLY}/tests/functcov/rv64/I/WALLY-COV-add.elf >> ${SIM}/questa/functcov_logs/add.log 2>&1 - run-elf-cov.bash --seed ${SIM}/questa/seed0.txt --verbose --coverdb ${SIM}/questa/functcov/and.ucdb --elf ${WALLY}/tests/functcov/rv64/I/WALLY-COV-and.elf >> ${SIM}/questa/functcov_logs/add.log 2>&1 - run-elf-cov.bash --seed ${SIM}/questa/seed0.txt --verbose --coverdb ${SIM}/questa/functcov/ori.ucdb --elf ${WALLY}/tests/functcov/rv64/I/WALLY-COV-ori.elf >> ${SIM}/questa/functcov_logs/add.log 2>&1 + wsim rv64gc ${WALLY}/tests/functcov/rv64/I/WALLY-COV-add.elf > ${SIM}/questa/functcov_logs/add.log 2>&1 + + #run-elf-cov.bash --seed ${SIM}/questa/seed0.txt --verbose --coverdb ${SIM}/questa/functcov/add.ucdb --elf ${WALLY}/tests/functcov/rv64/I/WALLY-COV-add.elf >> ${SIM}/questa/functcov_logs/add.log 2>&1 + #run-elf-cov.bash --seed ${SIM}/questa/seed0.txt --verbose --coverdb ${SIM}/questa/functcov/and.ucdb --elf ${WALLY}/tests/functcov/rv64/I/WALLY-COV-and.elf >> ${SIM}/questa/functcov_logs/add.log 2>&1 + #run-elf-cov.bash --seed ${SIM}/questa/seed0.txt --verbose --coverdb ${SIM}/questa/functcov/ori.ucdb --elf ${WALLY}/tests/functcov/rv64/I/WALLY-COV-ori.elf >> ${SIM}/questa/functcov_logs/add.log 2>&1 vcover merge ${SIM}/questa/functcov/functcov.ucdb ${SIM}/questa/functcov/*.ucdb ${SIM}/questa/functcov_ucdbs/* -suppress 6854 -64 # vcover merge ${SIM}/questa/functcov/functcov.ucdb ${SIM}/questa/functcov_ucdbs/* -suppress 6854 -64 diff --git a/bin/regression-wally b/bin/regression-wally index 8225968aa..a059bf648 100755 --- a/bin/regression-wally +++ b/bin/regression-wally @@ -320,7 +320,7 @@ coveragesim = "questa" # Questa is required for code/functional coverage #defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready defaultsim = "verilator" # Default simulator for all other tests -coverage = '--coverage' in sys.argv +ccov = '--ccov' in sys.argv fp = '--fp' in sys.argv nightly = '--nightly' in sys.argv testfloat = '--testfloat' in sys.argv @@ -334,8 +334,8 @@ else: nightMode = "" sims = [defaultsim] -if (coverage): # only run RV64GC tests in coverage mode - coverStr = '--coverage' +if (ccov): # only run RV64GC tests in coverage mode + coverStr = '--ccov' else: coverStr = '' @@ -357,7 +357,7 @@ if (buildroot): # addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours addTests(tests_buildrootbootlockstep, "questa") # lockstep with Questa and ImperasDV runs overnight -if (coverage): # only run RV64GC tests on Questa in coverage mode +if (ccov): # only run RV64GC tests on Questa in code coverage mode addTests(tests64gc_nofp, "questa") if (fp): addTests(tests64gc_fp, "questa") @@ -385,7 +385,7 @@ if (nightly): tc = TestCase( name="lockstep_wally-riscv-arch-test", variant="rv64gc", - cmd="iterelf " + WALLY + "/tests/riscof/work/wally-riscv-arch-test/rv64i_m/privilege > " + sim_log, + cmd="iterelf " + WALLY + "/tests/riscof/work/wally-riscv-arch-test/rv64i_m > " + sim_log, grepstr="SUCCESS! All tests ran without failures", grepfile = sim_log) configs.append(tc) @@ -393,7 +393,7 @@ if (nightly): # testfloat tests if (testfloat): # for testfloat alone, just run testfloat tests configs = [] -if (testfloat or nightly): # for nightly, run testfloat along with othres +if (testfloat or nightly): # for nightly, run testfloat along with others testfloatsim = "questa" # change to Verilator when Issue #707 about testfloat not running Verilator is resolved testfloatconfigs = ["fdqh_ieee_rv64gc", "fdq_ieee_rv64gc", "fdh_ieee_rv64gc", "fd_ieee_rv64gc", "fh_ieee_rv64gc", "f_ieee_rv64gc", "fdqh_ieee_rv32gc", "f_ieee_rv32gc"] for config in testfloatconfigs: @@ -458,7 +458,7 @@ if (testfloat or nightly): # for nightly, run testfloat along with othres def main(): """Run the tests and count the failures""" - global configs, coverage + global configs, ccov os.chdir(regressionDir) dirs = ["questa/logs", "questa/wkdir", "verilator/logs", "verilator/wkdir", "vcs/logs", "vcs/wkdir"] for d in dirs: @@ -472,7 +472,7 @@ def main(): os.chdir(regressionDir) os.system('./make-tests.sh | tee ./logs/make-tests.log') - elif '--coverage' in sys.argv: + elif '--ccov' in sys.argv: TIMEOUT_DUR = 20*60 # seconds os.system('rm -f questa/cov/*.ucdb') elif '--nightly' in sys.argv: @@ -497,8 +497,8 @@ def main(): print(f"{bcolors.FAIL}%s_%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.variant, config.name, TIMEOUT_DUR)) # Coverage report - if coverage: - os.system('make QuestaCoverage') + if ccov: + os.system('make QuestaCodeCoverage') # Count the number of failures if num_fail: print(f"{bcolors.FAIL}Regression failed with %s failed configurations{bcolors.ENDC}" % num_fail) diff --git a/bin/wsim b/bin/wsim index 68c6ce019..f5c297067 100755 --- a/bin/wsim +++ b/bin/wsim @@ -26,8 +26,8 @@ parser.add_argument("--elf", "-e", help="ELF File name; use if name does not end parser.add_argument("--sim", "-s", help="Simulator", choices=["questa", "verilator", "vcs"], default="questa") parser.add_argument("--tb", "-t", help="Testbench", choices=["testbench", "testbench_fp"], default="testbench") parser.add_argument("--gui", "-g", help="Simulate with GUI", action="store_true") -parser.add_argument("--coverage", "-c", help="Code & Functional Coverage", action="store_true") -parser.add_argument("--fcov", "-f", help="Code & Functional Coverage", action="store_true") +parser.add_argument("--ccov", "-c", help="Code Coverage", action="store_true") +parser.add_argument("--fcov", "-f", help="Functional Coverage", action="store_true") parser.add_argument("--args", "-a", help="Optional arguments passed to simulator via $value$plusargs", default="") parser.add_argument("--vcd", "-v", help="Generate testbench.vcd", action="store_true") parser.add_argument("--lockstep", "-l", help="Run ImperasDV lock, step, and compare.", action="store_true") @@ -55,7 +55,7 @@ if(args.testsuite.endswith('.elf') and args.elf == ""): # No --elf argument; che # Validate arguments -if (args.gui or args.coverage or args.fcov or args.lockstep): +if (args.gui or args.ccov or args.fcov or args.lockstep): if args.sim not in ["questa", "vcs"]: print("Option only supported for Questa and VCS") exit(1) @@ -86,14 +86,14 @@ else: flags = suffix + " " + ImperasPlusArgs # other flags -if (args.coverage): - flags += " --coverage" +if (args.ccov): + flags += " --ccov" if (args.fcov): flags += " --fcov" # create the output sub-directories. regressionDir = WALLY + '/sim/' -for d in ["logs", "wkdir", "cov"]: +for d in ["logs", "wkdir", "cov", "ucdb"]: try: os.mkdir(regressionDir+args.sim+"/"+d) except: diff --git a/sim/Makefile b/sim/Makefile index 09d417124..e8fe3c860 100644 --- a/sim/Makefile +++ b/sim/Makefile @@ -17,28 +17,27 @@ all: riscoftests memfiles coveragetests deriv wally-riscv-arch-test: wallyriscoftests memfiles -QuestaCoverage: questa/cov/rv64gc_arch64i.ucdb - #iter-elf.bash --cover --search ../tests/coverage - vcover merge -out questa/cov/cov.ucdb questa/cov/rv64gc_arch64i.ucdb questa/cov/rv64gc*.ucdb -logfile questa/cov/log -# vcover merge -out questa/cov/cov.ucdb questa/cov/rv64gc_arch64i.ucdb questa/cov/rv64gc*.ucdb questa/cov/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log - vcover report -details questa/cov/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt - vcover report questa/cov/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt - vcover report questa/cov/cov.ucdb -details -instance=/core/priv. > questa/cov/rv64gc_coverage_priv.rpt - vcover report questa/cov/cov.ucdb -details -instance=/core/ifu. > questa/cov/rv64gc_coverage_ifu.rpt - vcover report questa/cov/cov.ucdb -details -instance=/core/lsu. > questa/cov/rv64gc_coverage_lsu.rpt - vcover report questa/cov/cov.ucdb -details -instance=/core/fpu. > questa/cov/rv64gc_coverage_fpu.rpt - vcover report questa/cov/cov.ucdb -details -instance=/core/ieu. > questa/cov/rv64gc_coverage_ieu.rpt - vcover report questa/cov/cov.ucdb -below 100 -details -instance=/core/ebu. > questa/cov/rv64gc_uncovered_ebu.rpt - vcover report questa/cov/cov.ucdb -below 100 -details -instance=/core/priv. > questa/cov/rv64gc_uncovered_priv.rpt - vcover report questa/cov/cov.ucdb -below 100 -details -instance=/core/ifu. > questa/cov/rv64gc_uncovered_ifu.rpt - vcover report questa/cov/cov.ucdb -below 100 -details -instance=/core/lsu. > questa/cov/rv64gc_uncovered_lsu.rpt - vcover report questa/cov/cov.ucdb -below 100 -details -instance=/core/fpu. > questa/cov/rv64gc_uncovered_fpu.rpt - vcover report questa/cov/cov.ucdb -below 100 -details -instance=/core/ieu. > questa/cov/rv64gc_uncovered_ieu.rpt - vcover report -hierarchical questa/cov/cov.ucdb > questa/cov/rv64gc_coverage_hierarchical.rpt - vcover report -below 100 -hierarchical questa/cov/cov.ucdb > questa/cov/rv64gc_uncovered_hierarchical.rpt -# vcover report -below 100 questa/cov/cov.ucdb > questa/cov/rv64gc_coverage.rpt -# vcover report -recursive questa/cov/cov.ucdb > questa/cov/rv64gc_recursive.rpt - vcover report -details -threshH 100 -html questa/cov/cov.ucdb +QuestaCodeCoverage: questa/ucdb/rv64gc_arch64i.ucdb + vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log +# vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log + vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt + vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt + vcover report questa/ucdb/cov.ucdb -details -instance=/core/priv. > questa/cov/rv64gc_coverage_priv.rpt + vcover report questa/ucdb/cov.ucdb -details -instance=/core/ifu. > questa/cov/rv64gc_coverage_ifu.rpt + vcover report questa/ucdb/cov.ucdb -details -instance=/core/lsu. > questa/cov/rv64gc_coverage_lsu.rpt + vcover report questa/ucdb/cov.ucdb -details -instance=/core/fpu. > questa/cov/rv64gc_coverage_fpu.rpt + vcover report questa/ucdb/cov.ucdb -details -instance=/core/ieu. > questa/cov/rv64gc_coverage_ieu.rpt + vcover report questa/ucdb/cov.ucdb -below 100 -details -instance=/core/ebu. > questa/cov/rv64gc_uncovered_ebu.rpt + vcover report questa/ucdb/cov.ucdb -below 100 -details -instance=/core/priv. > questa/cov/rv64gc_uncovered_priv.rpt + vcover report questa/ucdb/cov.ucdb -below 100 -details -instance=/core/ifu. > questa/cov/rv64gc_uncovered_ifu.rpt + vcover report questa/ucdb/cov.ucdb -below 100 -details -instance=/core/lsu. > questa/cov/rv64gc_uncovered_lsu.rpt + vcover report questa/ucdb/cov.ucdb -below 100 -details -instance=/core/fpu. > questa/cov/rv64gc_uncovered_fpu.rpt + vcover report questa/ucdb/cov.ucdb -below 100 -details -instance=/core/ieu. > questa/cov/rv64gc_uncovered_ieu.rpt + vcover report -hierarchical questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_hierarchical.rpt + vcover report -below 100 -hierarchical questa/ucdb/cov.ucdb > questa/cov/rv64gc_uncovered_hierarchical.rpt +# vcover report -below 100 questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage.rpt +# vcover report -recursive questa/ucdb/cov.ucdb > questa/cov/rv64gc_recursive.rpt + vcover report -details -threshH 100 -html questa/ucdb/cov.ucdb allclean: clean all diff --git a/sim/questa/wally.do b/sim/questa/wally.do index a0e6fdbb6..c3dbc3b0f 100644 --- a/sim/questa/wally.do +++ b/sim/questa/wally.do @@ -8,7 +8,7 @@ # # Takes 1:10 to run RV64IC tests using gui -# Usage: do wally-batch.do [-coverage] [+acc] [any number of +value] [any number of -G VAR=VAL] +# Usage: do wally-batch.do [--ccov] [--fcov] [+acc] [any number of +value] [any number of -G VAR=VAL] # Example: do wally-batch.do rv64gc arch64i testbench # Use this wally-batch.do file to run this example. @@ -40,7 +40,7 @@ vlib ${WKDIR} # Create directory for coverage data mkdir -p cov -set coverage 0 +set ccov 0 set CoverageVoptArg "" set CoverageVsimArg "" @@ -104,9 +104,9 @@ if {$AccIndex >= 0} { } # if +coverage found set flag and remove from list -set CoverageIndex [lsearch -exact $lst "--coverage"] +set CoverageIndex [lsearch -exact $lst "--ccov"] if {$CoverageIndex >= 0} { - set coverage 1 + set ccov 1 set CoverageVoptArg "+cover=sbecf" set CoverageVsimArg "-coverage" set lst [lreplace $lst $CoverageIndex $CoverageIndex] @@ -166,7 +166,7 @@ foreach otherArg $lst { if {$DEBUG > 0} { echo "GUI = $GUI" - echo "coverage = $coverage" + echo "ccov = $ccov" echo "lockstep = $lockstep" echo "FunctCoverage = $FunctCoverage" echo "remaining list = $lst" @@ -215,14 +215,18 @@ if { ${GUI} } { } } +if {$ccov || $FunctCoverage} { + set UCDB ${WALLY}/sim/questa/ucdb/${CFG}_${TESTSUITE}.ucdb + echo "Saving coverage to ${UCDB}" + coverage save -instance /testbench/dut/core ${UCDB} +} + run -all # power off -r /dut/core/* -if {$coverage || $FunctCoverage} { - set UCDB ${WALLY}/sim/questa/cov/${CFG}_${TESTSUITE}.ucdb - echo "Saving coverage to ${UCDB}" +# Code coverage exclusions +if {$ccov} { do coverage-exclusions-rv64gc.do # beware: this assumes testing the rv64gc configuration - coverage save -instance /testbench/dut/core ${UCDB} } # These aren't doing anything helpful