mirror of
https://github.com/openhwgroup/cvw
synced 2025-02-11 06:05:49 +00:00
Merge pull request #1045 from slmnemo/nightly_regression
Regression fixes and nightly build changes
This commit is contained in:
Jordan Carlin
GitHub
commit
de6d219e72
@ -288,33 +288,34 @@ class TestRunner:
|
||||
makefile_location = self.cvw.joinpath(makefile_path)
|
||||
os.chdir(makefile_location)
|
||||
|
||||
output_file = self.log_dir.joinpath(f"make-{target}-output.log")
|
||||
if target:
|
||||
output_file = self.log_dir.joinpath(f"make-{target}-output.log")
|
||||
else: output_file = self.log_dir.joinpath(f"make-output.log")
|
||||
|
||||
command = ["make"]
|
||||
|
||||
# Add target to the command if specified
|
||||
if target:
|
||||
command.append(target)
|
||||
self.logger.info(f"Command used in directory {makefile_location}: {command[0]} {command[1]}")
|
||||
# Execute make with target and cores/2
|
||||
if target:
|
||||
command = ["make", target, "--jobs=$(($(nproc)/2))"]
|
||||
else:
|
||||
self.logger.info(f"Command used in directory {makefile_location}: {command[0]}")
|
||||
command = ["make", "--jobs=$(($(nproc)/2))"]
|
||||
|
||||
self.logger.info(f"Command used in directory {makefile_location}: {' '.join(command)}")
|
||||
|
||||
# Execute the command using subprocess and save the output into a file
|
||||
with open(output_file, "w") as f:
|
||||
formatted_datetime = self.current_datetime.strftime("%Y-%m-%d %H:%M:%S")
|
||||
f.write(formatted_datetime)
|
||||
f.write("\n\n")
|
||||
result = subprocess.run(command, stdout=f, stderr=subprocess.STDOUT, text=True)
|
||||
result = subprocess.run(command, stdout=f, stderr=subprocess.STDOUT, text=True, shell=True)
|
||||
|
||||
# Execute the command using a subprocess and not save the output
|
||||
#result = subprocess.run(command, text=True)
|
||||
|
||||
# Check the result
|
||||
if result.returncode == 0:
|
||||
self.logger.info(f"Tests have been made with target: {target}")
|
||||
self.logger.info(f"Tests have been made with target: {' '.join(command)}")
|
||||
return True
|
||||
else:
|
||||
self.logger.error(f"Error making the tests. Target: {target}")
|
||||
self.logger.error(f"Error making the tests. Command: {' '.join(command)}")
|
||||
return False
|
||||
|
||||
def run_tests(self, test_type=None, test_name=None, test_extensions=None):
|
||||
@ -422,7 +423,7 @@ class TestRunner:
|
||||
elif "Failures detected in output" in line: # Text explicitly fails
|
||||
try:
|
||||
config_name = line.split(':')[0].strip()
|
||||
log_file = os.path.abspath(os.path.join("logs", config_name, ".log"))
|
||||
log_file = os.path.abspath(os.path.join("logs", config_name))
|
||||
failed_configs.append((config_name, log_file))
|
||||
except:
|
||||
failed_configs.append((config_name, "Log file not found"))
|
||||
@ -672,7 +673,7 @@ def main():
|
||||
|
||||
parser.add_argument('--path',default = "nightly", help='specify the path for where the nightly repositories will be cloned ex: "nightly-runs')
|
||||
parser.add_argument('--repository',default = "https://github.com/openhwgroup/cvw", help='specify which github repository you want to clone')
|
||||
parser.add_argument('--target', default = "--jobs", help='types of tests you can make are: all, wally-riscv-arch-test, no')
|
||||
parser.add_argument('--target', default = "", help='types of tests you can make are: all, wally-riscv-arch-test, no')
|
||||
parser.add_argument('--tests', default = "nightly", help='types of tests you can run are: nightly, test, test_lint')
|
||||
parser.add_argument('--send_email',default = "", nargs="+", help='What emails to send test results to. Example: "[email1],[email2],..."')
|
||||
|
||||
|
||||
@ -334,22 +334,29 @@ def search_log_for_text(text, grepfile):
|
||||
# print(" search_log_for_text invoking %s" % grepcmd)
|
||||
return os.system(grepcmd) == 0
|
||||
|
||||
def run_test_case(config):
|
||||
"""Run the given test case, and return 0 if the test suceeds and 1 if it fails"""
|
||||
def run_test_case(config, dryrun: bool = False):
|
||||
"""
|
||||
Run the given test case, and return 0 if the test suceeds and 1 if it fails
|
||||
|
||||
Do not execute commands if dryrun
|
||||
"""
|
||||
grepfile = config.grepfile
|
||||
cmd = config.cmd
|
||||
os.chdir(regressionDir)
|
||||
# print(" run_test_case invoking %s" % cmd, flush=True)
|
||||
os.system(cmd)
|
||||
if search_log_for_text(config.grepstr, grepfile):
|
||||
# Flush is needed to flush output to stdout when running in multiprocessing Pool
|
||||
# print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name), flush=True)
|
||||
print(f"{bcolors.OKGREEN}%s: Success{bcolors.ENDC}" % (config.cmd), flush=True)
|
||||
if dryrun:
|
||||
print(f"Executing {cmd}", flush=True)
|
||||
return 0
|
||||
else:
|
||||
print(f"{bcolors.FAIL}%s: Failures detected in output{bcolors.ENDC}" % (config.cmd), flush=True)
|
||||
print(" Check %s" % grepfile)
|
||||
return 1
|
||||
os.system(cmd)
|
||||
if search_log_for_text(config.grepstr, grepfile):
|
||||
# Flush is needed to flush output to stdout when running in multiprocessing Pool
|
||||
# print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name), flush=True)
|
||||
print(f"{bcolors.OKGREEN}%s: Success{bcolors.ENDC}" % (config.cmd), flush=True)
|
||||
return 0
|
||||
else:
|
||||
print(f"{bcolors.FAIL}%s: Failures detected in output{bcolors.ENDC}" % (config.cmd), flush=True)
|
||||
print(" Check %s" % grepfile)
|
||||
return 1
|
||||
|
||||
##################################
|
||||
# Main body
|
||||
@ -363,6 +370,7 @@ os.chdir(regressionDir)
|
||||
coveragesim = "questa" # Questa is required for code/functional coverage
|
||||
#defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready
|
||||
defaultsim = "verilator" # Default simulator for all other tests
|
||||
lockstepsim = "questa"
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ccov", help="Code Coverage", action="store_true")
|
||||
@ -371,6 +379,7 @@ parser.add_argument("--nightly", help="Run large nightly regression", action="st
|
||||
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
|
||||
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
|
||||
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true")
|
||||
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if (args.nightly):
|
||||
@ -404,7 +413,7 @@ configs = [
|
||||
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
|
||||
if (args.buildroot):
|
||||
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
|
||||
addTests(tests_buildrootbootlockstep, "questa") # lockstep with Questa and ImperasDV runs overnight
|
||||
addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
|
||||
|
||||
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode
|
||||
addTests(tests64gc_nofp, coveragesim)
|
||||
@ -417,7 +426,7 @@ elif (args.fcov): # only run RV64GC tests on Questa in lockstep in functional c
|
||||
|
||||
else:
|
||||
for sim in sims:
|
||||
if (not (args.buildroot and sim == defaultsim)): # skip short buildroot sim if running long one
|
||||
if (not (args.buildroot and sim == lockstepsim)): # skip short buildroot sim if running long one
|
||||
addTests(tests_buildrootshort, sim)
|
||||
addTests(tests, sim)
|
||||
addTests(tests64gc_nofp, sim)
|
||||
@ -425,9 +434,10 @@ else:
|
||||
|
||||
# run derivative configurations and lockstep tests in nightly regression
|
||||
if (args.nightly):
|
||||
addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", "questa", 0)
|
||||
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", "questa", 0)
|
||||
addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 0)
|
||||
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 0)
|
||||
addTests(derivconfigtests, defaultsim)
|
||||
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
|
||||
|
||||
# testfloat tests
|
||||
if (args.testfloat): # for testfloat alone, just run testfloat tests
|
||||
@ -540,9 +550,9 @@ def main():
|
||||
|
||||
# Coverage report
|
||||
if args.ccov:
|
||||
os.system('make QuestaCodeCoverage')
|
||||
os.system('make QuestaCodeCoverage')
|
||||
if args.fcov:
|
||||
os.system('make -f '+WALLY+'/addins/cvw-arch-verif/Makefile merge')
|
||||
os.system('make -f '+WALLY+'/addins/cvw-arch-verif/Makefile merge')
|
||||
# Count the number of failures
|
||||
if num_fail:
|
||||
print(f"{bcolors.FAIL}Regression failed with %s failed configurations{bcolors.ENDC}" % num_fail)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user