From b27447f552f45fa4e451aed2bfaca8ef549c40f7 Mon Sep 17 00:00:00 2001 From: Jordan Carlin Date: Fri, 29 Nov 2024 15:41:36 -0800 Subject: [PATCH 1/4] Change addLockstepTestsByDir to addTestsByDir --- bin/regression-wally | 52 +++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/bin/regression-wally b/bin/regression-wally index f53940d89..58e87b6a2 100755 --- a/bin/regression-wally +++ b/bin/regression-wally @@ -291,18 +291,35 @@ def addTests(tests, sim): configs.append(tc) -def addLockstepTestsByDir(dir, config, sim, fcovMode): - sim_logdir = WALLY+ "/sim/" + sim + "/logs/" - if (coverStr != ""): # use --fcov in place of --lockstep - cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config - else: - cmdPrefix="wsim --lockstep --sim " + sim + " " + config - if (os.path.isdir(dir)): +def addTestsByDir(dir, config, sim, lockstepMode=0): + if os.path.isdir(dir): + sim_logdir = WALLY+ "/sim/" + sim + "/logs/" + if coverStr == "--fcov": # use --fcov in place of --lockstep + cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config + gs="Mismatches : 0" + if ("cvw-arch-verif/tests" in dir and not "priv" in dir): + fileEnd = "ALL.elf" + else: + fileEnd = ".elf" + elif coverStr == "--ccov": + cmdPrefix="wsim --sim " + sim + " " + coverStr + " " + config + gs="Single Elf file tests are not signatured verified." + if ("cvw-arch-verif/tests" in dir and not "priv" in dir): + fileEnd = "ALL.elf" + else: + fileEnd = ".elf" + elif lockstepMode: + cmdPrefix="wsim --lockstep --sim " + sim + " " + config + gs="Mismatches : 0" + fileEnd = ".elf" + else: + cmdPrefix="wsim --sim " + sim + " " + config + gs="Single Elf file tests are not signatured verified." + fileEnd = ".elf" for dirpath, dirnames, filenames in os.walk(os.path.abspath(dir)): for file in filenames: # fcov lockstep only runs on WALLY-COV-ALL.elf files; other lockstep runs on all files - if ((file.endswith(".elf") and (fcovMode == 0 or "tests/priv" in dir)) or - (file.endswith("ALL.elf") and fcovMode == 1)): + if file.endswith(fileEnd): fullfile = os.path.join(dirpath, file) fields = fullfile.rsplit('/', 3) if (fields[2] == "ref"): @@ -318,7 +335,7 @@ def addLockstepTestsByDir(dir, config, sim, fcovMode): name=file, variant=config, cmd=cmdPrefix + " " + fullfile + " > " + sim_log, - grepstr="Mismatches : 0", + grepstr=gs, grepfile = sim_log) configs.append(tc) else: @@ -421,11 +438,10 @@ if (args.ccov): # only run RV64GC tests on Questa in code coverage mode if (args.fp): addTests(tests64gc_fp, coveragesim) elif (args.fcov): # run tests in lockstep in functional coverage mode - addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim, 1) - addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim, 1) - addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim, 1) - addLockstepTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim, 1) - #addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m/privilege/src/", "rv64gc", coveragesim, 0) + addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim) + addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim) + addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv32/", "rv32gc", coveragesim) + addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim) else: for sim in sims: @@ -437,9 +453,9 @@ else: # run derivative configurations and lockstep tests in nightly regression if (args.nightly): - addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 0) - addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 0) - addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 0) + addTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 1) + addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 1) + addTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv32i_m", "rv32gc", lockstepsim, 1) addTests(derivconfigtests, defaultsim) # addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script. From 716656b71b275909791182c6201f31b86f5227fd Mon Sep 17 00:00:00 2001 From: Jordan Carlin Date: Fri, 29 Nov 2024 15:42:23 -0800 Subject: [PATCH 2/4] Code cleanup --- bin/regression-wally | 33 ++++++++++++++++----------------- bin/wsim | 4 ++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/bin/regression-wally b/bin/regression-wally index 58e87b6a2..51827bb76 100755 --- a/bin/regression-wally +++ b/bin/regression-wally @@ -34,7 +34,7 @@ tests = [ "arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma", "arch32zfh_divsqrt", "arch32zfaf", "arch32zfad", "wally32a_lrsc", "wally32priv", "wally32periph", "arch32zcb", "arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]], - ["rv64i", ["arch64i"]] + ["rv64i", ["arch64i"]] ] # Separate test for short buildroot run through OpenSBI UART output @@ -265,8 +265,8 @@ class bcolors: def addTests(tests, sim): sim_logdir = WALLY+ "/sim/" + sim + "/logs/" for test in tests: - config = test[0]; - suites = test[1]; + config = test[0] + suites = test[1] if (len(test) >= 3): args = " --args " + " ".join(test[2]) else: @@ -401,9 +401,8 @@ parser.add_argument("--dryrun", help="Print commands invoked to console without args = parser.parse_args() if (args.nightly): - nightMode = "--nightly"; + nightMode = "--nightly" sims = ["questa", "verilator", "vcs"] # exercise all simulators; can omit a sim if no license is available -# sims = ["questa", "verilator"] # exercise all simulators; can omit a sim if no license is available else: nightMode = "" sims = [defaultsim] @@ -413,7 +412,7 @@ if (args.ccov): # only run RV64GC tests in coverage mode elif (args.fcov): # only run RV64GC tests in lockstep in coverage mode coverStr = '--fcov' else: - coverStr = '' + coverStr = '' # Run Lint @@ -556,17 +555,17 @@ def main(): else: ImperasDVLicenseCount = 10000 # effectively no license limit for non-lockstep tests with Pool(processes=min(len(configs),multiprocessing.cpu_count(), ImperasDVLicenseCount)) as pool: - num_fail = 0 - results = {} - for config in configs: - results[config] = pool.apply_async(run_test_case,(config, args.dryrun)) - for (config,result) in results.items(): - try: - num_fail+=result.get(timeout=TIMEOUT_DUR) - except TimeoutError: - pool.terminate() - num_fail+=1 - print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR)) + num_fail = 0 + results = {} + for config in configs: + results[config] = pool.apply_async(run_test_case,(config, args.dryrun)) + for (config,result) in results.items(): + try: + num_fail+=result.get(timeout=TIMEOUT_DUR) + except TimeoutError: + pool.terminate() + num_fail+=1 + print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR)) # Coverage report if args.ccov: diff --git a/bin/wsim b/bin/wsim index 2b3849ecc..56abd9567 100755 --- a/bin/wsim +++ b/bin/wsim @@ -101,10 +101,10 @@ if (args.lockstep or args.lockstepverbose): if(args.locksteplog != 0): ImperasPlusArgs = " +IDV_TRACE2LOG=" + str(EnableLog) + " +IDV_TRACE2LOG_AFTER=" + str(args.locksteplog) else: ImperasPlusArgs = "" if(args.fcov): - CovEnableStr = "1" if int(args.covlog) > 0 else "0"; + CovEnableStr = "1" if int(args.covlog) > 0 else "0" if(args.covlog >= 1): EnableLog = 1 else: EnableLog = 0 - ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr; + ImperasPlusArgs = " +IDV_TRACE2COV=" + str(EnableLog) + " +TRACE2LOG_AFTER=" + str(args.covlog) + " +TRACE2COV_ENABLE=" + CovEnableStr suffix = "" else: CovEnableStr = "" From e61f66eacac8aa511784b513cc1446f1c5c5d110 Mon Sep 17 00:00:00 2001 From: Jordan Carlin Date: Fri, 29 Nov 2024 15:43:21 -0800 Subject: [PATCH 3/4] Use cvw-arch-verif tests for code coverage --- bin/regression-wally | 6 +++--- sim/Makefile | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/regression-wally b/bin/regression-wally index 51827bb76..0383a1807 100755 --- a/bin/regression-wally +++ b/bin/regression-wally @@ -433,9 +433,9 @@ if (args.buildroot): addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight if (args.ccov): # only run RV64GC tests on Questa in code coverage mode - addTests(tests64gc_nofp, coveragesim) - if (args.fp): - addTests(tests64gc_fp, coveragesim) + addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim) + addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/priv/rv64/", "rv64gc", coveragesim) + addTestsByDir(WALLY+"/tests/coverage/", "rv64gc", coveragesim) elif (args.fcov): # run tests in lockstep in functional coverage mode addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv32/", "rv32gc", coveragesim) addTestsByDir(WALLY+"/addins/cvw-arch-verif/tests/rv64/", "rv64gc", coveragesim) diff --git a/sim/Makefile b/sim/Makefile index d94936381..5feb25ae4 100644 --- a/sim/Makefile +++ b/sim/Makefile @@ -29,8 +29,8 @@ deriv: .PHONY: QuestaCodeCoverage collect_functcov combine_functcov remove_functcov_artifacts riscvdv riscvdv_functcov -QuestaCodeCoverage: questa/ucdb/rv64gc_arch64i.ucdb - vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log +QuestaCodeCoverage: + vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc*.ucdb -logfile questa/cov/log # vcover merge -out questa/ucdb/cov.ucdb questa/ucdb/rv64gc_arch64i.ucdb questa/ucdb/rv64gc*.ucdb questa/ucdb/buildroot_buildroot.ucdb riscv.ucdb -logfile questa/cov/log vcover report -details questa/ucdb/cov.ucdb > questa/cov/rv64gc_coverage_details.rpt vcover report questa/ucdb/cov.ucdb -details -instance=/core/ebu. > questa/cov/rv64gc_coverage_ebu.rpt From c5bb2b42a54d98ac86c571256bdfee1aaa712e78 Mon Sep 17 00:00:00 2001 From: Jordan Carlin Date: Fri, 29 Nov 2024 15:43:59 -0800 Subject: [PATCH 4/4] cleanup --- bin/regression-wally | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/regression-wally b/bin/regression-wally index 0383a1807..09bd1fa9c 100755 --- a/bin/regression-wally +++ b/bin/regression-wally @@ -396,7 +396,7 @@ parser.add_argument("--fcov", help="Functional Coverage", action="store_true") parser.add_argument("--nightly", help="Run large nightly regression", action="store_true") parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true") parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true") -parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") +parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") # Currently not used parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true") args = parser.parse_args()