diff --git a/bin/nightly_build.py b/bin/nightly_build.py
index 38cfa74fc..c4cbc0e77 100755
--- a/bin/nightly_build.py
+++ b/bin/nightly_build.py
@@ -55,6 +55,7 @@ Dependencies:
- markdown
- subprocess
- argparse
+ - logging
Bash:
- mutt (email sender)
@@ -71,7 +72,30 @@ import re
import markdown
import subprocess
import argparse
+import logging
+# Logger
+
+# Set up the logger
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+# Create a file handler
+file_handler = logging.FileHandler('../../logs/nightly_build.log')
+file_handler.setLevel(logging.DEBUG)
+
+# Create a console handler
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# Create a formatter and add it to the handlers
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# Add the handlers to the logger
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)
@@ -86,14 +110,12 @@ class FolderManager:
base_dir (str): The base directory where folders will be managed and repository will be cloned.
"""
env_extract_var = 'WALLY'
- # print(f"The environemntal variable is {env_extract_var}")
self.base_dir = os.environ.get(env_extract_var)
- print(f"The base directory is: {self.base_dir}")
self.base_parent_dir = os.path.dirname(self.base_dir)
- # print(f"The new WALLY vairable is: {os.environ.get('WALLY')}")
- # print(f"The Base Directory is now : {self.base_dir}")
- # print(f"The Base Parent Directory is now : {self.base_parent_dir}")
+ logger.info(f"Base directory: {self.base_dir}")
+ logger.info(f"Parent Base directory: {self.base_parent_dir}")
+
def create_preliminary_folders(self, folders):
"""
@@ -115,6 +137,7 @@ class FolderManager:
folder_path = os.path.join(self.base_parent_dir, folder)
if not os.path.exists(folder_path):
os.makedirs(folder_path)
+ logger.info(f"Preliminary folders created: {folders}")
def create_new_folder(self, folders):
"""
@@ -136,7 +159,7 @@ class FolderManager:
return_folder_path.append(folder_path)
else:
return_folder_path.append(None) # Folder already exists
-
+ logger.info(f"New folder created. Path: {folder_path}")
return return_folder_path
def clone_repository(self, folder, repo_url):
@@ -157,7 +180,8 @@ class FolderManager:
os.makedirs(repo_folder)
os.system(f"git clone --recurse-submodules {repo_url} {repo_folder}")
os.makedirs(tmp_folder)
-
+
+ logger.info(f"Repository cloned: {repo_url}")
class TestRunner:
"""A class for making, running, and formatting test results."""
@@ -166,8 +190,9 @@ class TestRunner:
self.base_dir = os.environ.get('WALLY')
self.base_parent_dir = os.path.dirname(self.base_dir)
self.current_datetime = datetime.now()
- #self.temp_dir = self.base_parent_dir
- #print(f"Base Directory: {self.base_parent_dir}")
+
+ logger.info("Test runner object is initialized")
+
def copy_setup_script(self, folder):
"""
@@ -180,7 +205,7 @@ class TestRunner:
folder: the "nightly-runs/repos/"
Returns:
- bool: True if the script is copied successfully, False otherwise.
+ bool: True if the script is copied successfuly, False otherwise.
"""
# Get today's date in YYYY-MM-DD format
todays_date = datetime.now().strftime("%Y-%m-%d")
@@ -191,22 +216,22 @@ class TestRunner:
# Check if the source script exists
if not os.path.exists(source_script):
- print(f"Error: Source script '{source_script}' not found.")
+ logger.error(f"Error: Source script '{source_script}' not found.")
return False
# Check if the destination folder exists, create it if necessary
if not os.path.exists(destination_folder):
- print(f"Error: Destination folder '{destination_folder}' not found.")
+ logger.error(f"Error: Destination folder '{destination_folder}' not found.")
return False
# Copy the script to the destination folder
try:
shutil.copy(source_script, destination_folder)
- #print(f"Setup script copied to: {destination_folder}")
+ logger.info(f"Setup script copied to: {destination_folder}")
return True
except Exception as e:
- print(f"Error copying setup script: {e}")
+ logger.error(f"Error copying setup script: {e}")
return False
@@ -231,15 +256,19 @@ class TestRunner:
self.base_parent_dir = os.path.dirname(self.base_dir)
self.temp_dir = self.base_parent_dir
- # print(f"The new WALLY vairable is: {os.environ.get('WALLY')}")
- # print(f"The Base Directory is now : {self.base_dir}")
- # print(f"The Base Parent Directory is now : {self.base_parent_dir}")
+ logger.info(f"Tests are going to be ran from: {self.base_dir}")
+ logger.info(f"WALLY environmental variable is: {os.environ.get('WALLY')}")
+
def change_time_dur(self, time_duriation=1):
# Prepare the command to execute the Makefile
make_file_path = os.path.join(self.base_dir, "sim")
- os.chdir(make_file_path)
+ logger.info(f"Make file path is set to: {make_file_path}")
+ try:
+ os.chdir(make_file_path)
+ except Exception as e:
+ logger.error(f"Error nagivating to the make file path. Error: {e}")
file_path = "regression-wally"
line_number = 450 # TIMEOUT_DUR = 1 day at this line in regression-wally
new_line = f" TIMEOUT_DUR = {60*time_duriation}"
@@ -248,13 +277,14 @@ class TestRunner:
lines = file.readlines()
if line_number < 1 or line_number > len(lines):
- print("Error: Line number out of range.")
+ logger.error("Error: Line number out of range.")
return False
lines[line_number - 1] = new_line + '\n'
with open(file_path, 'w') as file:
file.writelines(lines)
+ logger.info(f"Timeduration in ./regression-wally has been changed to: {time_duriation*60} seconds")
return True
def execute_makefile(self, target=None):
@@ -280,7 +310,9 @@ class TestRunner:
# Add target to the command if specified
if target:
command.append(target)
- #print(f"The command is: {command}")
+ logger.info(f"Command used: {command[0]} {command[1]}")
+ else:
+ logger.info(f"Command used: {command[0]}")
# Execute the command using subprocess and save the output into a file
with open(output_file, "w") as f:
@@ -294,10 +326,10 @@ class TestRunner:
# Check the result
if result.returncode == 0:
- #print(f"Makefile executed successfully{' with target ' + target if target else ''}.")
+ logger.info(f"Tests have been made with tag target: {target}")
return True
else:
- #print("Error executing Makefile.")
+ logger.error(f"Error making the tests. Target: {target}")
return False
def run_tests(self, test_type=None, test_name=None, test_exctention=None):
@@ -319,8 +351,11 @@ class TestRunner:
if test_exctention:
command = [test_type, test_name, test_exctention]
+ logger.info(f"Command used to run tests: {test_type} {test_name} -{test_exctention}")
else:
command = [test_type, test_name]
+ logger.info(f"Command used to run tests: {test_type} {test_name}")
+
# Execute the command using subprocess and save the output into a file
with open(output_file, "w") as f:
@@ -329,11 +364,12 @@ class TestRunner:
f.write("\n\n")
result = subprocess.run(command, stdout=f, stderr=subprocess.STDOUT, text=True)
- # Check if the command executed successfully
+ # Check if the command executed successfuly
if result.returncode or result.returncode == 0:
+ logger.info(f"Test ran successfuly. Test type: {test_type}, test name: {test_name}, test extention: {test_exctention}")
return True, output_file
else:
- print("Error:", result.returncode)
+ logger.error(f"Error making test. Test type: {test_type}, test name: {test_name}, test extention: {test_exctention}")
return False, output_file
@@ -364,11 +400,10 @@ class TestRunner:
while index < len(lines):
# Remove ANSI escape codes
line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index])
- #print(line)
+
if "Success" in line:
passed_configs.append(line.split(':')[0].strip())
elif "passed lint" in line:
- #print(line)
passed_configs.append(line.split(' ')[0].strip())
#passed_configs.append(line) # potentially use a space
elif "failed lint" in line:
@@ -379,7 +414,6 @@ class TestRunner:
try:
config_name = line.split(':')[0].strip()
log_file = os.path.abspath("logs/"+config_name+".log")
- #print(f"The log file saving to: {log_file} in the current working directory: {os.getcwd()}")
failed_configs.append((config_name, log_file))
except:
failed_configs.append((config_name, "Log file not found"))
@@ -393,8 +427,7 @@ class TestRunner:
if len(failed_configs) != 0:
failed_configs.sort()
- #print(f"The passed configs are: {passed_configs}")
- #print(f"The failed configs are {failed_configs}")
+ logger.info(f"Cleaned test results. Passed configs {passed_configs}. Failed configs: {failed_configs}")
return passed_configs, failed_configs
def rewrite_to_markdown(self, test_name, passed_configs, failed_configs):
@@ -415,8 +448,7 @@ class TestRunner:
os.chdir(output_directory)
current_directory = os.getcwd()
output_file = os.path.join(current_directory, f"{test_name}.md")
- #print("Current directory:", current_directory)
- #print("Output File:", output_file)
+
with open(output_file, 'w') as md_file:
@@ -440,7 +472,9 @@ class TestRunner:
for config in passed_configs:
md_file.write(f"- {config}\n")
- def combine_markdown_files(self, passed_tests, failed_tests, test_list, total_number_failures, total_number_success, test_type="default", markdown_file=None):
+ logger.info("writing test outputs to markdown")
+
+ def combine_markdown_files(self, passed_tests, failed_tests, test_list, total_number_failures, total_number_success, test_type="default", markdown_file=None, args=None):
"""
First we want to display the server properties like:
- Server full name
@@ -479,6 +513,9 @@ class TestRunner:
os_info = subprocess.check_output(['uname', '-a']).strip().decode('utf-8')
md_file.write(f"\n**Operating System Information:** {os_info}")
md_file.write("\n")
+
+ md_file.write(f"\n**Command used to execute test:** python nightly_build.py --path {args.path} --repository {args.repository} --target {args.target} --send_email {args.send_email}")
+ md_file.write("\n")
except subprocess.CalledProcessError as e:
# Handle if the command fails
md_file.write(f"Failed to identify host and Operating System information: {str(e)}")
@@ -516,17 +553,17 @@ class TestRunner:
md_file.write("\n")
md_file.write(f"* {config} ({log_file})\n")
md_file.write("\n")
- # Successfull Tests
+ # Successful Tests
- md_file.write(f"\n\n## Successfull Tests")
- md_file.write(f"\n**Total successfull tests: {total_number_success}**")
+ md_file.write(f"\n\n## Successful Tests")
+ md_file.write(f"\n**Total successful tests: {total_number_success}**")
for (test_item, item) in zip(test_list, passed_tests):
md_file.write(f"\n\n### {test_item[1]} test")
md_file.write(f"\n**General Information**\n")
md_file.write(f"\n* Test type: {test_item[0]}")
md_file.write(f"\n* Test name: {test_item[1]}")
md_file.write(f"\n* Test extension: {test_item[2]}\n\n")
- md_file.write(f"\n**Successfull Tests:**\n")
+ md_file.write(f"\n**Successful Tests:**\n")
@@ -542,7 +579,8 @@ class TestRunner:
md_file.write(f"* {config}\n")
md_file.write("\n")
-
+ logger.info("Combining markdown files")
+
def convert_to_html(self, markdown_file="results.md", html_file="results.html"):
"""
@@ -567,7 +605,7 @@ class TestRunner:
with open(html_file, 'w') as html_file:
html_file.write(html_content)
-
+ logger.info("Converting markdown file to html file.")
def send_email(self, sender_email=None, receiver_emails=None, subject="Nightly Regression Test"):
"""
@@ -585,7 +623,7 @@ class TestRunner:
# check if there are any emails
if not receiver_emails:
- print("No receiver emails provided.")
+ logger.ERROR("No receiver emails provided.")
return
# grab thge html file
todays_date = self.current_datetime.strftime("%Y-%m-%d")
@@ -613,10 +651,11 @@ class TestRunner:
process = subprocess.Popen(command, stdin=subprocess.PIPE)
# Write the email body to the subprocess
process.communicate(body.encode('utf-8'))
+ logger.info("Sent email")
except expression as identifier:
- print(f"Error sending email: {identifier}")
+ logger.error(f"Error sending email with error: {identifier}")
except expression as identifier:
- print(f"Error sending email: {identifier}")
+ logger.error(f"Error sending email with error: {identifier}")
@@ -634,6 +673,11 @@ def main():
args = parser.parse_args()
+ logger.info(f"path: {args.path}")
+ logger.info(f"repository: {args.repository}")
+ logger.info(f"target: {args.target}")
+ logger.info(f"send_email: {args.send_email}")
+
# file paths for where the results and repos will be saved: repos and results can be changed to whatever
repos_path = f"{args.path}/repos/"
results_path = f"{args.path}/results/"
@@ -649,9 +693,7 @@ def main():
# clone the cvw repo
folder_manager.clone_repository(repos_path, args.repository)
- #############################################
- # SETUP #
- #############################################
+
test_runner = TestRunner() # creates the object
test_runner.set_env_var(repos_path) # ensures that the new WALLY environmental variable is set correctly
@@ -665,19 +707,18 @@ def main():
The goal of this section is to replace the TIMEOUT_DUR for regression tests.
"""
- # if test_runner.change_time_dur(time_duriation=2):
- # pass
- # else:
- # print("Error occured changing the TIMEOUT duration in './regression-wally'")
+ if test_runner.change_time_dur(time_duriation=2):
+ pass
+ else:
+ logger.error("Error occured changing the TIMEOUT duration in './regression-wally'")
#############################################
# MAKE TESTS #
#############################################
-
-
- # if test_runner.execute_makefile(target = args.target):
- # print(f"The {args.target} tests were made successfully")
+ if args.target != "no":
+ # test_runner.execute_makefile(target = "deriv")
+ test_runner.execute_makefile(target = args.target)
#############################################
# RUN TESTS #
@@ -693,34 +734,30 @@ def main():
total_success = []
for test_type, test_name, test_exctention in test_list:
- print("--------------------------------------------------------------")
- print(f"Test type: {test_type}")
- print(f"Test name: {test_name}")
- print(f"Test extention: {test_exctention}")
-
+
check, output_location = test_runner.run_tests(test_type=test_type, test_name=test_name, test_exctention=test_exctention)
- print(f"Did the tests run?: {check}")
- print(f"The tests log files are saved to: {output_location}")
- if check: # this checks if the test actually ran successfully
+
+ if check: # this checks if the test actually ran successfuly
output_log_list.append(output_location)
-
+ logger.info(f"{test_name} ran successfuly. Output location: {output_location}")
# format tests to markdown
try:
passed, failed = test_runner.clean_format_output(input_file = output_location)
+ logger.info(f"{test_name} has been formatted to markdown")
except:
- print("There was an error cleaning the data")
+ logger.ERROR(f"Error occured with formatting {test_name}")
- print(f"The # of failures are for {test_name}: {len(failed)}")
+ logger.info(f"The # of failures are for {test_name}: {len(failed)}")
total_number_failures+= len(failed)
total_failures.append(failed)
- print(f"The # of sucesses are for {test_name}: {len(passed)}")
+ logger.info(f"The # of sucesses are for {test_name}: {len(passed)}")
total_number_success += len(passed)
total_success.append(passed)
test_runner.rewrite_to_markdown(test_name, passed, failed)
- print(f"The total sucesses are: {total_number_success}")
- print(f"The total failures are: {total_number_failures}")
+ logger.info(f"The total sucesses for all tests ran are: {total_number_success}")
+ logger.info(f"The total failures for all tests ran are: {total_number_failures}")
@@ -733,7 +770,7 @@ def main():
# Combine multiple markdown files into one file
- test_runner.combine_markdown_files(passed_tests = total_success, failed_tests = total_failures, test_list = test_list, total_number_failures = total_number_failures, total_number_success = total_number_success, test_type=args.target, markdown_file=None)
+ test_runner.combine_markdown_files(passed_tests = total_success, failed_tests = total_failures, test_list = test_list, total_number_failures = total_number_failures, total_number_success = total_number_success, test_type=args.target, markdown_file=None, args=args)
#############################################
@@ -758,4 +795,4 @@ def main():
test_runner.send_email(sender_email=sender_email, receiver_emails=testing_emails)
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()