Github_Repos/cvw
1
0
Fork 1
mirror of https://github.com/openhwgroup/cvw synced 2025-02-11 06:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1045 from slmnemo/nightly_regression

Browse Source 
Regression fixes and nightly build changes
This commit is contained in:
Jordan Carlin 2024-10-30 11:59:07 -07:00 committed by GitHub
commit de6d219e72
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 41 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
bin/nightly_build.py
View File

@ -288,33 +288,34 @@ class TestRunner:
makefile_location = self.cvw.joinpath(makefile_path) makefile_location = self.cvw.joinpath(makefile_path)
os.chdir(makefile_location) os.chdir(makefile_location)
output_file = self.log_dir.joinpath(f"make-{target}-output.log") if target:
output_file = self.log_dir.joinpath(f"make-{target}-output.log")
else: output_file = self.log_dir.joinpath(f"make-output.log")
command = ["make"] # Execute make with target and cores/2
if target:
# Add target to the command if specified command = ["make", target, "--jobs=$(($(nproc)/2))"]
if target:
command.append(target)
self.logger.info(f"Command used in directory {makefile_location}: {command[0]} {command[1]}")
else: else:
self.logger.info(f"Command used in directory {makefile_location}: {command[0]}") command = ["make", "--jobs=$(($(nproc)/2))"]
self.logger.info(f"Command used in directory {makefile_location}: {' '.join(command)}")
# Execute the command using subprocess and save the output into a file # Execute the command using subprocess and save the output into a file
with open(output_file, "w") as f: with open(output_file, "w") as f:
formatted_datetime = self.current_datetime.strftime("%Y-%m-%d %H:%M:%S") formatted_datetime = self.current_datetime.strftime("%Y-%m-%d %H:%M:%S")
f.write(formatted_datetime) f.write(formatted_datetime)
f.write("\n\n") f.write("\n\n")
result = subprocess.run(command, stdout=f, stderr=subprocess.STDOUT, text=True) result = subprocess.run(command, stdout=f, stderr=subprocess.STDOUT, text=True, shell=True)
# Execute the command using a subprocess and not save the output # Execute the command using a subprocess and not save the output
#result = subprocess.run(command, text=True) #result = subprocess.run(command, text=True)
# Check the result # Check the result
if result.returncode == 0: if result.returncode == 0:
self.logger.info(f"Tests have been made with target: {target}") self.logger.info(f"Tests have been made with target: {' '.join(command)}")
return True return True
else: else:
self.logger.error(f"Error making the tests. Target: {target}") self.logger.error(f"Error making the tests. Command: {' '.join(command)}")
return False return False
def run_tests(self, test_type=None, test_name=None, test_extensions=None): def run_tests(self, test_type=None, test_name=None, test_extensions=None):
@ -422,7 +423,7 @@ class TestRunner:
elif "Failures detected in output" in line: # Text explicitly fails elif "Failures detected in output" in line: # Text explicitly fails
try: try:
config_name = line.split(':')[0].strip() config_name = line.split(':')[0].strip()
log_file = os.path.abspath(os.path.join("logs", config_name, ".log")) log_file = os.path.abspath(os.path.join("logs", config_name))
failed_configs.append((config_name, log_file)) failed_configs.append((config_name, log_file))
except: except:
failed_configs.append((config_name, "Log file not found")) failed_configs.append((config_name, "Log file not found"))
@ -672,7 +673,7 @@ def main():
parser.add_argument('--path',default = "nightly", help='specify the path for where the nightly repositories will be cloned ex: "nightly-runs') parser.add_argument('--path',default = "nightly", help='specify the path for where the nightly repositories will be cloned ex: "nightly-runs')
parser.add_argument('--repository',default = "https://github.com/openhwgroup/cvw", help='specify which github repository you want to clone') parser.add_argument('--repository',default = "https://github.com/openhwgroup/cvw", help='specify which github repository you want to clone')
parser.add_argument('--target', default = "--jobs", help='types of tests you can make are: all, wally-riscv-arch-test, no') parser.add_argument('--target', default = "", help='types of tests you can make are: all, wally-riscv-arch-test, no')
parser.add_argument('--tests', default = "nightly", help='types of tests you can run are: nightly, test, test_lint') parser.add_argument('--tests', default = "nightly", help='types of tests you can run are: nightly, test, test_lint')
parser.add_argument('--send_email',default = "", nargs="+", help='What emails to send test results to. Example: "[email1],[email2],..."') parser.add_argument('--send_email',default = "", nargs="+", help='What emails to send test results to. Example: "[email1],[email2],..."')

44
bin/regression-wally
View File

@ -334,22 +334,29 @@ def search_log_for_text(text, grepfile):
# print(" search_log_for_text invoking %s" % grepcmd) # print(" search_log_for_text invoking %s" % grepcmd)
return os.system(grepcmd) == 0 return os.system(grepcmd) == 0
def run_test_case(config): def run_test_case(config, dryrun: bool = False):
"""Run the given test case, and return 0 if the test suceeds and 1 if it fails""" """
Run the given test case, and return 0 if the test suceeds and 1 if it fails
Do not execute commands if dryrun
"""
grepfile = config.grepfile grepfile = config.grepfile
cmd = config.cmd cmd = config.cmd
os.chdir(regressionDir) os.chdir(regressionDir)
# print(" run_test_case invoking %s" % cmd, flush=True) if dryrun:
os.system(cmd) print(f"Executing {cmd}", flush=True)
if search_log_for_text(config.grepstr, grepfile):
# Flush is needed to flush output to stdout when running in multiprocessing Pool
# print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name), flush=True)
print(f"{bcolors.OKGREEN}%s: Success{bcolors.ENDC}" % (config.cmd), flush=True)
return 0 return 0
else: else:
print(f"{bcolors.FAIL}%s: Failures detected in output{bcolors.ENDC}" % (config.cmd), flush=True) os.system(cmd)
print(" Check %s" % grepfile) if search_log_for_text(config.grepstr, grepfile):
return 1 # Flush is needed to flush output to stdout when running in multiprocessing Pool
# print(f"{bcolors.OKGREEN}%s_%s: Success{bcolors.ENDC}" % (config.variant, config.name), flush=True)
print(f"{bcolors.OKGREEN}%s: Success{bcolors.ENDC}" % (config.cmd), flush=True)
return 0
else:
print(f"{bcolors.FAIL}%s: Failures detected in output{bcolors.ENDC}" % (config.cmd), flush=True)
print(" Check %s" % grepfile)
return 1
################################## ##################################
# Main body # Main body
@ -363,6 +370,7 @@ os.chdir(regressionDir)
coveragesim = "questa" # Questa is required for code/functional coverage coveragesim = "questa" # Questa is required for code/functional coverage
#defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready #defaultsim = "questa" # Default simulator for all other tests; change to Verilator when flow is ready
defaultsim = "verilator" # Default simulator for all other tests defaultsim = "verilator" # Default simulator for all other tests
lockstepsim = "questa"
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--ccov", help="Code Coverage", action="store_true") parser.add_argument("--ccov", help="Code Coverage", action="store_true")
@ -371,6 +379,7 @@ parser.add_argument("--nightly", help="Run large nightly regression", action="st
parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true") parser.add_argument("--buildroot", help="Include Buildroot Linux boot test (takes many hours, done along with --nightly)", action="store_true")
parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true") parser.add_argument("--testfloat", help="Include Testfloat floating-point unit tests", action="store_true")
parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true") parser.add_argument("--fp", help="Include floating-point tests in coverage (slower runtime)", action="store_true")
parser.add_argument("--dryrun", help="Print commands invoked to console without running regression", action="store_true")
args = parser.parse_args() args = parser.parse_args()
if (args.nightly): if (args.nightly):
@ -404,7 +413,7 @@ configs = [
# run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests # run full buildroot boot simulation (slow) if buildroot flag is set. Start it early to overlap with other tests
if (args.buildroot): if (args.buildroot):
# addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours # addTests(tests_buildrootboot, defaultsim) # non-lockstep with Verilator runs in about 2 hours
addTests(tests_buildrootbootlockstep, "questa") # lockstep with Questa and ImperasDV runs overnight addTests(tests_buildrootbootlockstep, lockstepsim) # lockstep with Questa and ImperasDV runs overnight
if (args.ccov): # only run RV64GC tests on Questa in code coverage mode if (args.ccov): # only run RV64GC tests on Questa in code coverage mode
addTests(tests64gc_nofp, coveragesim) addTests(tests64gc_nofp, coveragesim)
@ -417,7 +426,7 @@ elif (args.fcov): # only run RV64GC tests on Questa in lockstep in functional c
else: else:
for sim in sims: for sim in sims:
if (not (args.buildroot and sim == defaultsim)): # skip short buildroot sim if running long one if (not (args.buildroot and sim == lockstepsim)): # skip short buildroot sim if running long one
addTests(tests_buildrootshort, sim) addTests(tests_buildrootshort, sim)
addTests(tests, sim) addTests(tests, sim)
addTests(tests64gc_nofp, sim) addTests(tests64gc_nofp, sim)
@ -425,9 +434,10 @@ else:
# run derivative configurations and lockstep tests in nightly regression # run derivative configurations and lockstep tests in nightly regression
if (args.nightly): if (args.nightly):
addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", "questa", 0) addLockstepTestsByDir(WALLY+"/tests/coverage", "rv64gc", lockstepsim, 0)
addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", "questa", 0) addLockstepTestsByDir(WALLY+"/tests/riscof/work/wally-riscv-arch-test/rv64i_m", "rv64gc", lockstepsim, 0)
addTests(derivconfigtests, defaultsim) addTests(derivconfigtests, defaultsim)
# addTests(bpredtests, defaultsim) # This is currently broken in regression due to something related to the new wsim script.
# testfloat tests # testfloat tests
if (args.testfloat): # for testfloat alone, just run testfloat tests if (args.testfloat): # for testfloat alone, just run testfloat tests
@ -540,9 +550,9 @@ def main():
# Coverage report # Coverage report
if args.ccov: if args.ccov:
os.system('make QuestaCodeCoverage') os.system('make QuestaCodeCoverage')
if args.fcov: if args.fcov:
os.system('make -f '+WALLY+'/addins/cvw-arch-verif/Makefile merge') os.system('make -f '+WALLY+'/addins/cvw-arch-verif/Makefile merge')
# Count the number of failures # Count the number of failures
if num_fail: if num_fail:
print(f"{bcolors.FAIL}Regression failed with %s failed configurations{bcolors.ENDC}" % num_fail) print(f"{bcolors.FAIL}Regression failed with %s failed configurations{bcolors.ENDC}" % num_fail)