From 055bcc5f97b85d8148aad14fb596370947f7a77c Mon Sep 17 00:00:00 2001 From: slmnemo Date: Mon, 9 Sep 2024 12:43:58 -0700 Subject: [PATCH 1/6] Changed make names to match new Linux makefile. Removed --buildroot option from regression call. --- bin/nightly_build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/nightly_build.py b/bin/nightly_build.py index 629da074e..0770de66f 100755 --- a/bin/nightly_build.py +++ b/bin/nightly_build.py @@ -751,8 +751,7 @@ def main(): # Compile Linux for local testing test_runner.set_env_var("RISCV",str(test_runner.cvw)) linux_path = test_runner.cvw / "linux" - test_runner.execute_makefile(target = "all_nosudo", makefile_path=linux_path) - test_runner.execute_makefile(target = "dumptvs_nosudo", makefile_path=linux_path) + test_runner.execute_makefile(target = "all", makefile_path=linux_path) ############################################# # RUN TESTS # From 25da694a2403c5c37cc8b32dffa0e7359e80aad9 Mon Sep 17 00:00:00 2001 From: slmnemo Date: Wed, 11 Sep 2024 12:18:57 -0700 Subject: [PATCH 2/6] Laid out new CSV daily tracker and emailsender classes. --- bin/nightly_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nightly_build.py b/bin/nightly_build.py index 0770de66f..7d0dcefbe 100755 --- a/bin/nightly_build.py +++ b/bin/nightly_build.py @@ -794,8 +794,8 @@ def main(): logger.info(f"The total sucesses for all tests ran are: {total_number_success}") logger.info(f"The total failures for all tests ran are: {total_number_failures}") - # Copy actual test logs from sim/questa, sim/verilator - test_runner.copy_sim_logs([test_runner.cvw / "sim/questa/logs", test_runner.cvw / "sim/verilator/logs"]) + # Copy actual test logs from sim/questa, sim/verilator, sim/vcs + test_runner.copy_sim_logs([test_runner.cvw / "sim/questa/logs", test_runner.cvw / "sim/verilator/logs", test_runner.cvw / "sim/vcs/logs"]) ############################################# # FORMAT TESTS # From 35fa3ff461b3335f2372c8b96addef850d9336cf Mon Sep 17 00:00:00 2001 From: slmnemo Date: Thu, 26 Sep 2024 19:58:24 -0700 Subject: [PATCH 3/6] Fixed ERROR versus error bug in logger, fixed error in failed lint tests. Labeled lint tests in regression output. Changed test run directory to call direct location rather than calling unknown regression script --- bin/nightly_build.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/nightly_build.py b/bin/nightly_build.py index 7d0dcefbe..b8bce5982 100755 --- a/bin/nightly_build.py +++ b/bin/nightly_build.py @@ -409,16 +409,16 @@ class TestRunner: if "Success" in line: passed_configs.append(line.split(':')[0].strip()) elif "passed lint" in line: - passed_configs.append(line.split(' ')[0].strip()) + passed_configs.append(f"Lint: {line.split(' ')[0].strip()}") #passed_configs.append(line) # potentially use a space elif "failed lint" in line: - failed_configs.append(line.split(' ')[0].strip(), "no log file") + failed_configs.append([f"Lint: {line.split(' ')[0].strip()}", "No Log File"]) #failed_configs.append(line) elif "Failures detected in output" in line: try: config_name = line.split(':')[0].strip() - log_file = os.path.abspath("logs/"+config_name+".log") + log_file = os.path.abspath(os.path.join("logs", config_name, ".log")) failed_configs.append((config_name, log_file)) except: failed_configs.append((config_name, "Log file not found")) @@ -619,7 +619,7 @@ class TestRunner: # check if there are any emails if not receiver_emails: - self.logger.ERROR("No receiver emails provided.") + self.logger.error("No receiver emails provided.") return # grab the html file @@ -777,7 +777,7 @@ def main(): passed, failed = test_runner.clean_format_output(input_file = output_location) logger.info(f"{test_name} has been formatted to markdown") except: - logger.ERROR(f"Error occured with formatting {test_name}") + logger.error(f"Error occured with formatting {test_name}") logger.info(f"The # of failures are for {test_name}: {len(failed)}") total_number_failures+= len(failed) @@ -787,15 +787,19 @@ def main(): total_number_success += len(passed) total_success.append(passed) test_runner.rewrite_to_markdown(test_name, passed, failed) + + newlinechar = "\n" + logger.info(f"Failed tests: \n{newlinechar.join([x[0] for x in failed])}") except Exception as e: - logger.error("There was an error in running the tests: {e}") + logger.error(f"There was an error in running the tests: {e}") logger.info(f"The total sucesses for all tests ran are: {total_number_success}") logger.info(f"The total failures for all tests ran are: {total_number_failures}") # Copy actual test logs from sim/questa, sim/verilator, sim/vcs - test_runner.copy_sim_logs([test_runner.cvw / "sim/questa/logs", test_runner.cvw / "sim/verilator/logs", test_runner.cvw / "sim/vcs/logs"]) + if not args.tests == "test_lint": + test_runner.copy_sim_logs([test_runner.cvw / "sim/questa/logs", test_runner.cvw / "sim/verilator/logs", test_runner.cvw / "sim/vcs/logs"]) ############################################# # FORMAT TESTS # From e4b4cff270eb95a09a3e18fb4ee347bbd6475f75 Mon Sep 17 00:00:00 2001 From: slmnemo Date: Wed, 16 Oct 2024 18:48:08 -0700 Subject: [PATCH 4/6] Changed nightly_build.py to allow for nightly and buildroot to be passed as arguments --- bin/nightly_build.py | 50 +++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/bin/nightly_build.py b/bin/nightly_build.py index b8bce5982..506a0e997 100755 --- a/bin/nightly_build.py +++ b/bin/nightly_build.py @@ -84,7 +84,7 @@ from pathlib import Path class FolderManager: """A class for managing folders and repository cloning.""" - def __init__(self): + def __init__(self, basedir): """ Initialize the FolderManager instance. @@ -92,8 +92,12 @@ class FolderManager: base_dir (str): The base directory where folders will be managed and repository will be cloned. """ env_extract_var = 'WALLY' - self.base_dir = os.environ.get(env_extract_var) - self.base_parent_dir = os.path.dirname(self.base_dir) + if os.environ.get(env_extract_var): + self.base_dir = os.environ.get(env_extract_var) + self.base_parent_dir = os.path.dirname(self.base_dir) + else: + self.base_dir = basedir + self.base_parent_dir = os.path.dirname(self.base_dir) # logger.info(f"Base directory: {self.base_dir}") # logger.info(f"Parent Base directory: {self.base_parent_dir}") @@ -313,7 +317,7 @@ class TestRunner: self.logger.error(f"Error making the tests. Target: {target}") return False - def run_tests(self, test_type=None, test_name=None, test_extension=None): + def run_tests(self, test_type=None, test_name=None, test_extensions=None): """ Run a script through the terminal and save the output to a file. @@ -329,12 +333,12 @@ class TestRunner: output_file = self.log_dir.joinpath(f"{test_name}-output.log") os.chdir(self.sim_dir) - if test_extension: - command = [test_type, test_name, test_extension] - self.logger.info(f"Command used to run tests: {test_type} {test_name} {test_extension}") + if test_extensions: + command = [test_type, test_name] + test_extensions + self.logger.info(f"Command used to run tests in directory {self.sim_dir}: {test_type} {test_name} {' '.join(test_extensions)}") else: command = [test_type, test_name] - self.logger.info(f"Command used to run tests: {test_type} {test_name}") + self.logger.info(f"Command used to run tests in directory {self.sim_dir}: {test_type} {test_name}") # Execute the command using subprocess and save the output into a file @@ -348,10 +352,10 @@ class TestRunner: self.logger.error("There was an error in running the tests in the run_tests function: {e}") # Check if the command executed successfuly if result.returncode or result.returncode == 0: - self.logger.info(f"Test ran successfuly. Test type: {test_type}, test name: {test_name}, test extention: {test_extension}") + self.logger.info(f"Test ran successfuly. Test type: {test_type}, test name: {test_name}, test extension: {' '.join(test_extensions)}") return True, output_file else: - self.logger.error(f"Error making test. Test type: {test_type}, test name: {test_name}, test extention: {test_extension}") + self.logger.error(f"Error making test. Test type: {test_type}, test name: {test_name}, test extension: {' '.join(test_extensions)}") return False, output_file @@ -535,7 +539,7 @@ class TestRunner: md_file.write(f"\n**Total failed tests: {total_number_failures}**") for (test_item, item) in zip(test_list, failed_tests): md_file.write(f"\n\n### {test_item[1]} test") - md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {test_item[2]}\n\n") + md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {' '.join(test_item[2])}\n\n") md_file.write(f"**Failed Tests:**\n") @@ -558,7 +562,7 @@ class TestRunner: md_file.write(f"\n**Total successful tests: {total_number_success}**") for (test_item, item) in zip(test_list, passed_tests): md_file.write(f"\n\n### {test_item[1]} test") - md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {test_item[2]}\n\n") + md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {' '.join(test_item[2])}\n\n") md_file.write(f"\n**Successful Tests:**\n") @@ -682,7 +686,7 @@ def main(): log_file_path = log_path.joinpath("nightly_build.log") previous_cvw_path = Path.home().joinpath(args.path,f"{yesterday}/cvw") # creates the object - folder_manager = FolderManager() + folder_manager = FolderManager(basedir=args.path) # setting the path on where to clone new repositories of cvw folder_manager.create_folders([cvw_path, results_path, log_path]) @@ -691,14 +695,18 @@ def main(): folder_manager.clone_repository(cvw_path, args.repository) # Define tests that we can run + # + # flags are a list + if (args.tests == "all"): + test_list = [["python", "./regression-wally", ["--nightly", "--buildroot"]]] if (args.tests == "nightly"): - test_list = [["python", "regression-wally", "--nightly --buildroot"]] - elif (args.tests == "test"): - test_list = [["python", "regression-wally", ""]] - elif (args.tests == "test_lint"): - test_list = [["bash", "lint-wally", "-nightly"]] + test_list = [["python", "./regression-wally", ["--nightly"]]] + elif (args.tests == "regression"): + test_list = [["python", "./regression-wally", []]] + elif (args.tests == "lint"): + test_list = [["bash", "./lint-wally", ["--nightly"]]] else: - print(f"Error: Invalid test '"+args.test+"' specified") + print(f"Error: Invalid test {args.tests} specified") raise SystemExit ############################################# @@ -765,9 +773,9 @@ def main(): total_failures = [] total_success = [] - for test_type, test_name, test_extension in test_list: + for test_type, test_name, test_extensions in test_list: - check, output_location = test_runner.run_tests(test_type=test_type, test_name=test_name, test_extension=test_extension) + check, output_location = test_runner.run_tests(test_type=test_type, test_name=test_name, test_extensions=test_extensions) try: if check: # this checks if the test actually ran successfuly output_log_list.append(output_location) From dfbe01084fd49cabdcce9fabfa82d415acc09688 Mon Sep 17 00:00:00 2001 From: slmnemo Date: Wed, 23 Oct 2024 08:47:56 -0700 Subject: [PATCH 5/6] Fixed error where an elif was written as if in test conditionals --- bin/nightly_build.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/bin/nightly_build.py b/bin/nightly_build.py index 506a0e997..6d89f777f 100755 --- a/bin/nightly_build.py +++ b/bin/nightly_build.py @@ -410,7 +410,7 @@ class TestRunner: # Remove ANSI escape codes line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index]) - if "Success" in line: + if "Success" in line: # test succeeds passed_configs.append(line.split(':')[0].strip()) elif "passed lint" in line: passed_configs.append(f"Lint: {line.split(' ')[0].strip()}") @@ -419,13 +419,21 @@ class TestRunner: failed_configs.append([f"Lint: {line.split(' ')[0].strip()}", "No Log File"]) #failed_configs.append(line) - elif "Failures detected in output" in line: + elif "Failures detected in output" in line: # Text explicitly fails try: config_name = line.split(':')[0].strip() log_file = os.path.abspath(os.path.join("logs", config_name, ".log")) failed_configs.append((config_name, log_file)) except: failed_configs.append((config_name, "Log file not found")) + + elif "Timeout" in line: # Test times out + try: + config_name = line.split(':')[0].strip() + log_file = os.path.abspath("logs/"+config_name+".log") + failed_configs.append((f"Timeout: {config_name}", log_file)) + except: + failed_configs.append((f"Timeout: {config_name}", "No Log File")) index += 1 @@ -664,7 +672,7 @@ def main(): parser.add_argument('--path',default = "nightly", help='specify the path for where the nightly repositories will be cloned ex: "nightly-runs') parser.add_argument('--repository',default = "https://github.com/openhwgroup/cvw", help='specify which github repository you want to clone') - parser.add_argument('--target', default = "all", help='types of tests you can make are: all, wally-riscv-arch-test, no') + parser.add_argument('--target', default = "--jobs", help='types of tests you can make are: all, wally-riscv-arch-test, no') parser.add_argument('--tests', default = "nightly", help='types of tests you can run are: nightly, test, test_lint') parser.add_argument('--send_email',default = "", nargs="+", help='What emails to send test results to. Example: "[email1],[email2],..."') @@ -699,7 +707,7 @@ def main(): # flags are a list if (args.tests == "all"): test_list = [["python", "./regression-wally", ["--nightly", "--buildroot"]]] - if (args.tests == "nightly"): + elif (args.tests == "nightly"): test_list = [["python", "./regression-wally", ["--nightly"]]] elif (args.tests == "regression"): test_list = [["python", "./regression-wally", []]] @@ -755,11 +763,12 @@ def main(): if args.target != "no": test_runner.execute_makefile(target = args.target, makefile_path=test_runner.cvw) - if args.target == "all": - # Compile Linux for local testing - test_runner.set_env_var("RISCV",str(test_runner.cvw)) - linux_path = test_runner.cvw / "linux" - test_runner.execute_makefile(target = "all", makefile_path=linux_path) + # TODO: remove vestigial code if no longer wanted + # if args.target == "all": + # # Compile Linux for local testing + # test_runner.set_env_var("RISCV",str(test_runner.cvw)) + # linux_path = test_runner.cvw / "linux" + # test_runner.execute_makefile(target = "all", makefile_path=linux_path) ############################################# # RUN TESTS # From 44e17efcf3520773b1004bc09ecadb58c766a731 Mon Sep 17 00:00:00 2001 From: Kaitlin Lucio Date: Thu, 24 Oct 2024 16:29:46 -0700 Subject: [PATCH 6/6] Removed unnecessary os.path.dirname in foldermanager init --- bin/nightly_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nightly_build.py b/bin/nightly_build.py index 6d89f777f..349bdb378 100755 --- a/bin/nightly_build.py +++ b/bin/nightly_build.py @@ -97,7 +97,7 @@ class FolderManager: self.base_parent_dir = os.path.dirname(self.base_dir) else: self.base_dir = basedir - self.base_parent_dir = os.path.dirname(self.base_dir) + self.base_parent_dir = self.base_dir # logger.info(f"Base directory: {self.base_dir}") # logger.info(f"Parent Base directory: {self.base_parent_dir}")