#!/usr/bin/python3 ################################## # # regression-wally # David_Harris@Hmc.edu 25 January 2021 # Modified by Jarred Allen # # Run a regression with multiple configurations in parallel and exit with # non-zero status code if an error happened, as well as printing human-readable # output. # ################################## import sys,os,shutil import multiprocessing #import os from collections import namedtuple from multiprocessing import Pool, TimeoutError ################################## # Define lists of configurations and tests to run on each configuration ################################## tests = [ ["rv32e", ["arch32e"]], ["rv32i", ["arch32i"]], ["rv32imc", ["arch32i", "arch32c", "arch32m", "wally32periph"]], ["rv32gc", ["arch32f", "arch32d", "arch32f_fma", "arch32d_fma", "arch32f_divsqrt", "arch32d_divsqrt", "arch32i", "arch32priv", "arch32c", "arch32m", "arch32a", "arch32zifencei", "arch32zicond", "arch32zba", "arch32zbb", "arch32zbc", "arch32zbs", "arch32zfh", "arch32zfh_fma", "arch32zfh_divsqrt", "arch32zfaf", "wally32a", "wally32priv", "wally32periph", "arch32zbkb", "arch32zbkc", "arch32zbkx", "arch32zknd", "arch32zkne", "arch32zknh"]], # "arch32zcb", "arch32zfad", ["rv64e", ["arch64e"]], ["rv64i", ["arch64i"]], ["rv64imc", ["arch64i", "arch64c", "arch64m", "wally64periph"]], ] # Separate out floating-point tests for RV64 to speed up coverage tests64gc_nofp = [ ["rv64gc", ["coverage64gc", "arch64i", "arch64priv", "arch64c", "arch64m", "arch64zifencei", "arch64zicond", "arch64a", "wally64a", "wally64periph", "wally64priv", "arch64zbkb", "arch64zbkc", "arch64zbkx", "arch64zknd", "arch64zkne", "arch64zknh", "arch64zba", "arch64zbb", "arch64zbc", "arch64zbs"]] # add when working: "arch64zcb", "arch64zicboz" ] tests64gc_fp = [ ["rv64gc", ["arch64f", "arch64d", "arch64zfh", "arch64f_fma", "arch64d_fma", "arch64zfh_fma", "arch64f_divsqrt", "arch64d_divsqrt", "arch64zfh_divsqrt", "arch64zfaf", "arch64zfad"]] ] derivconfigtests = [ # memory system ["tlb2_rv32gc", ["wally32priv"]], ["tlb16_rv32gc", ["wally32priv"]], ["tlb2_rv64gc", ["wally64priv"]], ["tlb16_rv64gc", ["wally64priv"]], ["way_1_4096_512_rv32gc", ["arch32i"]], ["way_2_4096_512_rv32gc", ["arch32i"]], ["way_8_4096_512_rv32gc", ["arch32i"]], ["way_4_2048_512_rv32gc", ["arch32i"]], ["way_4_4096_256_rv32gc", ["arch32i"]], ["way_1_4096_512_rv64gc", ["arch64i"]], ["way_2_4096_512_rv64gc", ["arch64i"]], ["way_8_4096_512_rv64gc", ["arch64i"]], ["way_4_2048_512_rv64gc", ["arch64i"]], ["way_4_4096_256_rv64gc", ["arch64i"]], ["way_4_4096_1024_rv64gc", ["arch64i"]], ["ram_0_0_rv64gc", ["ahb64"]], ["ram_1_0_rv64gc", ["ahb64"]], ["ram_1_1_rv64gc", ["ahb64"]], ["ram_2_0_rv64gc", ["ahb64"]], ["ram_2_1_rv64gc", ["ahb64"]], # RV32 cacheless designs will not work unless DTIM supports FLEN > XLEN. This support is not planned. # ["nodcache_rv32gc", ["ahb32"]], # ["nocache_rv32gc", ["ahb32"]], ["noicache_rv32gc", ["ahb32"]], ["noicache_rv64gc", ["ahb64"]], ["nodcache_rv64gc", ["ahb64"]], ["nocache_rv64gc", ["ahb64"]], ### add misaligned tests # fp/int divider permutations ["div_2_1_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_2_1i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_2_2_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_2_2i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_2_4_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_2_4i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_4_1_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_4_1i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_4_2_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_4_2i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_4_4_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_4_4i_rv32gc", ["arch32f_divsqrt", "arch32d_divsqrt", "arch32m"]], ["div_2_1_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_2_1i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_2_2_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_2_2i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_2_4_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_2_4i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_4_1_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_4_1i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_4_2_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_4_2i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_4_4_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], ["div_4_4i_rv64gc", ["arch64f_divsqrt", "arch64d_divsqrt", "arch64m"]], # fpu permutations ["f_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma"]], ["fh_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32zfh", "arch32zfh_divsqrt"]], ["fdh_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32d", "arch32d_divsqrt", "arch32d_fma", "arch32zfh", "arch32zfh_divsqrt"]], ["fdq_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32d", "arch32d_divsqrt", "arch32d_fma", "arch32i"]], ["fdqh_rv32gc", ["arch32f", "arch32f_divsqrt", "arch32f_fma", "arch32d", "arch32d_divsqrt", "arch32d_fma", "arch32zfh", "arch32zfh_divsqrt", "arch32i"]], ["f_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma"]], ["fh_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64zfh", "arch64zfh_divsqrt"]], ["fdh_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64d", "arch64d_divsqrt", "arch64d_fma", "arch64zfh", "arch64zfh_divsqrt"]], ["fdq_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64d", "arch64d_divsqrt", "arch64d_fma", "arch64i"]], ["fdqh_rv64gc", ["arch64f", "arch64f_divsqrt", "arch64f_fma", "arch64d", "arch64d_divsqrt", "arch64d_fma", "arch64zfh", "arch64zfh_divsqrt", "arch64i", "wally64q"]], ] bpredtests = [ ["bpred_TWOBIT_6_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_8_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_10_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_12_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_14_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_16_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_6_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_8_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_10_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_12_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_14_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_TWOBIT_16_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_6_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_6_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_8_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_8_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_12_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_12_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_14_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_14_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_16_16_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_16_16_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], # btb ["bpred_GSHARE_10_16_6_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_6_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_8_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_8_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_12_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_16_12_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], # ras ["bpred_GSHARE_10_2_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_2_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_3_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_3_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_4_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_4_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_6_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_6_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_10_10_0_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"], ["bpred_GSHARE_10_10_10_1_rv32gc", ["embench"], "configOptions", "-GPrintHPMCounters=1"] ] TestCase = namedtuple("TestCase", ['name', 'variant', 'cmd', 'grepstr']) # name: the name of this test configuration (used in printing human-readable # output and picking logfile names) # cmd: the command to run to test (should include the logfile as '{}', and # the command needs to write to that file) # grepstr: the string to grep through the log file for. The test succeeds iff # grep finds that string in the logfile (is used by grep, so it may # be any pattern grep accepts, see `man 1 grep` for more info). class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' OKCYAN = '\033[96m' OKGREEN = '\033[92m' WARNING = '\033[93m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' def getBuildrootTC(boot): INSTR_LIMIT = 1000000 # multiple of 100000; 4M is interesting because it gets into the kernel and enabling VM MAX_EXPECTED = 246000000 # *** TODO: replace this with a search for the login prompt. if boot: name="buildrootboot" BRcmd="vsim > {} -c < {} -c < {} -c <= 4 and test[2] == "configOptions"): configOptions = test[3] cmdPrefix = "vsim > {} -c < {} -c < /dev/null" % (text, logfile) return os.system(grepcmd) == 0 def run_test_case(config): """Run the given test case, and return 0 if the test suceeds and 1 if it fails""" logname = "questa/logs/"+config.variant+"_"+config.name+".log" # cmd = config.cmd.format(logname) cmd = config.cmd + " > " + logname print(cmd) os.chdir(regressionDir) os.system(cmd) if search_log_for_text(config.grepstr, logname): print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name)) return 0 else: print(f"{bcolors.FAIL}%s_%s: Failures detected in output{bcolors.ENDC}" % (config.variant, config.name)) print(" Check %s" % logname) return 1 regressionDir = os.path.dirname(os.path.abspath(__file__)) os.chdir(regressionDir) coveragesim = "questa" # Questa is required for code/functional coverage defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready coverage = '-coverage' in sys.argv fp = '-fp' in sys.argv nightly = '-nightly' in sys.argv testfloat = '-testfloat' in sys.argv if (nightly): nightMode = "-nightly"; sims = ["questa", "verilator", "vcs"] else: nightMode = ""; sims = [defaultsim] if (coverage): # only run RV64GC tests in coverage mode coverStr = '-coverage' else: coverStr = '' # Run Lint configs = [ TestCase( name="lints", variant="all", cmd="./lint-wally " + nightMode + " | tee {}", grepstr="lints run with no errors or warnings" ) ] if (coverage): # only run RV64GC tests on Questa in coverage mode addTests(tests64gc_nofp, "questa") if (fp): addTests(tests64gc_fp, "questa") else: for sim in sims: addTests(tests, sim) addTests(tests64gc_nofp, sim) addTests(tests64gc_fp, sim) # run derivative configurations in nightly regression if (nightly): addTests(derivconfigtests, defaultsim) # testfloat tests if (testfloat): configs = [] testfloatconfigs = [ "fdh_ieee_div_2_1_rv32gc", "fdh_ieee_div_2_1_rv64gc", "fdh_ieee_div_2_2_rv32gc", "fdh_ieee_div_2_2_rv64gc", "fdh_ieee_div_2_4_rv32gc", "fdh_ieee_div_2_4_rv64gc", "fdh_ieee_div_4_1_rv32gc", "fdh_ieee_div_4_1_rv64gc", "fdh_ieee_div_4_2_rv32gc", "fdh_ieee_div_4_2_rv64gc", "fdh_ieee_div_4_4_rv32gc", "fdh_ieee_div_4_4_rv64gc", "fd_ieee_div_2_1_rv32gc", "fd_ieee_div_2_1_rv64gc", "fd_ieee_div_2_2_rv32gc", "fd_ieee_div_2_2_rv64gc", "fd_ieee_div_2_4_rv32gc", "fd_ieee_div_2_4_rv64gc", "fd_ieee_div_4_1_rv32gc", "fd_ieee_div_4_1_rv64gc", "fd_ieee_div_4_2_rv32gc", "fd_ieee_div_4_2_rv64gc", "fd_ieee_div_4_4_rv32gc", "fd_ieee_div_4_4_rv64gc", "fdqh_ieee_div_2_1_rv32gc", "fdqh_ieee_div_2_1_rv64gc", "fdqh_ieee_div_2_2_rv32gc", "fdqh_ieee_div_2_2_rv64gc", "fdqh_ieee_div_2_4_rv32gc", "fdqh_ieee_div_2_4_rv64gc", "fdqh_ieee_div_4_1_rv32gc", "fdqh_ieee_div_4_1_rv64gc", "fdqh_ieee_div_4_2_rv32gc", "fdqh_ieee_div_4_2_rv64gc", "fdqh_ieee_div_4_4_rv32gc", "fdqh_ieee_div_4_4_rv64gc", "fdq_ieee_div_2_1_rv32gc", "fdq_ieee_div_2_1_rv64gc", "fdq_ieee_div_2_2_rv32gc", "fdq_ieee_div_2_2_rv64gc", "fdq_ieee_div_2_4_rv32gc", "fdq_ieee_div_2_4_rv64gc", "fdq_ieee_div_4_1_rv32gc", "fdq_ieee_div_4_1_rv64gc", "fdq_ieee_div_4_2_rv32gc", "fdq_ieee_div_4_2_rv64gc", "fdq_ieee_div_4_4_rv32gc", "fdq_ieee_div_4_4_rv64gc", "fh_ieee_div_2_1_rv32gc", "fh_ieee_div_2_1_rv64gc", "fh_ieee_div_2_2_rv32gc", "fh_ieee_div_2_2_rv64gc", "fh_ieee_div_2_4_rv32gc", "fh_ieee_div_2_4_rv64gc", "fh_ieee_div_4_1_rv32gc", "fh_ieee_div_4_1_rv64gc", "fh_ieee_div_4_2_rv32gc", "fh_ieee_div_4_2_rv64gc", "fh_ieee_div_4_4_rv32gc", "fh_ieee_div_4_4_rv64gc", "f_ieee_div_2_1_rv32gc", "f_ieee_div_2_1_rv64gc", "f_ieee_div_2_2_rv32gc", "f_ieee_div_2_2_rv64gc", "f_ieee_div_2_4_rv32gc", "f_ieee_div_2_4_rv64gc", "f_ieee_div_4_1_rv32gc", "f_ieee_div_4_1_rv64gc", "f_ieee_div_4_2_rv32gc", "f_ieee_div_4_2_rv64gc", "f_ieee_div_4_4_rv32gc", "f_ieee_div_4_4_rv64gc" ] for config in testfloatconfigs: # div test case divtest = TestCase( name="div", variant=config, cmd="vsim > {} -c < {} -c < {} -c < {} -c <