From 4adc38deb59a7b4cc8323b1fc6239f625944c4fb Mon Sep 17 00:00:00 2001
From: Thomas Kidd <89825065+Thomas-J-Kidd@users.noreply.github.com>
Date: Thu, 1 Feb 2024 10:44:36 -0600
Subject: [PATCH] Adding nightly regression into the bin directory (#607)
* updated install tool chain file to use verilator v5.016
* updated gitignore file
* adding nightly tests
* reverted the verilator checkout to checkout master
* Revert "reverted the verilator checkout to checkout master"
This reverts commit 942a2804c33a28cf31171f4a203144a91f50fb15.
* Revert "updated gitignore file"
This reverts commit 981c3ccf6b4775505c6356ff3e9bd60f7f0ba5f4.
* Revert "updated install tool chain file to use verilator v5.016"
This reverts commit 3168859268dfa125d79f35cabb9c1b86feaa363f.
* Revert "updated gitignore file"
This reverts commit 981c3ccf6b4775505c6356ff3e9bd60f7f0ba5f4.
---
bin/nightly_build/nightly_build_v13_beta.sh | 85 ++++++++++++++++
bin/nightly_build/src/error_detector.py | 37 +++++++
bin/nightly_build/src/parse_coverage.py | 95 ++++++++++++++++++
bin/nightly_build/src/parse_regression.py | 102 ++++++++++++++++++++
bin/nightly_build/src/send_mail_html.sh | 79 +++++++++++++++
5 files changed, 398 insertions(+)
create mode 100755 bin/nightly_build/nightly_build_v13_beta.sh
create mode 100644 bin/nightly_build/src/error_detector.py
create mode 100644 bin/nightly_build/src/parse_coverage.py
create mode 100644 bin/nightly_build/src/parse_regression.py
create mode 100755 bin/nightly_build/src/send_mail_html.sh
diff --git a/bin/nightly_build/nightly_build_v13_beta.sh b/bin/nightly_build/nightly_build_v13_beta.sh
new file mode 100755
index 000000000..2e0ca42d4
--- /dev/null
+++ b/bin/nightly_build/nightly_build_v13_beta.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# set WALLY path
+WALLY=$(dirname ${BASH_SOURCE[0]:-$0})
+export WALLY=$(cd "$WALLY/../../" && pwd)
+echo "WALLY is set to: $WALLY"
+
+# Going to nightly runs
+cd $WALLY/../
+
+# check if directories exist
+if [ ! -d "build-results" ]; then
+ echo "Directory does not exist, creating it..."
+ mkdir -p "build-results"
+ if [ $? -eq 0 ]; then
+ echo "Directory created successfully."
+ else
+ echo "Failed to create directory."
+ exit 1
+ fi
+else
+ echo "Directory already exists."
+fi
+
+if [ ! -d "logs" ]; then
+ echo "Directory does not exist, creating it..."
+ mkdir -p "logs"
+ if [ $? -eq 0 ]; then
+ echo "Directory created successfully."
+ else
+ echo "Failed to create directory."
+ exit 1
+ fi
+else
+ echo "Directory already exists."
+fi
+
+# setup source okstate file
+echo "Sourcing setup files"
+source $WALLY/setup_host.sh
+source $WALLY/../setup-files/setup_tools.sh
+
+# Navigate to the gir repo
+cd $WALLY
+
+# pull the repository
+echo "Pulling submodules"
+#git pull --recurse-submodules origin main
+
+# build the regression tests
+echo "Building the regression tests"
+cd sim
+#if make wally-riscv-arch-test; then
+#if make all; then
+# echo "Make successfull"
+#else
+# echo "Make failed"
+# cd $WALLY/..
+ # add the the regression result and the coverage result that there was an error in making the tests
+# python $WALLY/bin/nightly_build/src/error_detector.py --tag make -o $WALLY/../build-results/regression_results.md
+# python $WALLY/bin/nightly_build/src/error_detector.py --tag make -o $WALLY/../build-results/coverage_results.md
+
+ # exit the program
+ #exit 1
+#fi
+
+# execute the simulation / regression tests and save output to a file
+echo "running the regression test"
+#./regression-wally > $WALLY/../logs/regression_output.log 2>&1
+
+echo "running coverage tests"
+#./coverage > $WALLY/../logs/coverage_output.log 2>&1
+
+
+# run the Python script to parse the output and generate the log file
+echo "Parsing output data from the regression test"
+cd $WALLY/../
+
+python $WALLY/bin/nightly_build/src/parse_regression.py -i $WALLY/../logs/regression_output.log -o $WALLY/../build-results/regression_results.md
+
+python $WALLY/bin/nightly_build/src/parse_coverage.py -i $WALLY/../logs/coverage_output.log -o $WALLY/../build-results/coverage_results.md
+
+# email update
+cd $WALLY/bin/nightly_build/src/
+./send_mail_html.sh
diff --git a/bin/nightly_build/src/error_detector.py b/bin/nightly_build/src/error_detector.py
new file mode 100644
index 000000000..c239032e9
--- /dev/null
+++ b/bin/nightly_build/src/error_detector.py
@@ -0,0 +1,37 @@
+import argparse
+import datetime
+
+def add_failure_to_markdown(tag, output_file, input_file = None):
+ # Get the current date and time
+ current_datetime = datetime.datetime.now()
+ formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+ # Create the failure message based on the provided tag and input content
+ if tag == "make":
+ failure_message = f"# {tag.capitalize()} riscof comilation failure - {formatted_datetime}\n\n"
+ failure_message += f"The error was due to a problem in compiling the the riscof tests:\n\n"
+ if input_file != None:
+ failure_message += f"The particular error: {input_file}\n\n"
+ else:
+ failure_message = f"# {tag.capitalize()} Failure - {formatted_datetime}\n\n"
+ failure_message += f":\n\n"
+
+ # Append the failure message to the specified output file
+ with open(output_file, "a") as file:
+ file.write(failure_message)
+
+ print(f"Failure information added to {output_file}.")
+
+if __name__ == "__main__":
+ # Set up argparse
+ parser = argparse.ArgumentParser(description="Add failure information to Markdown file.")
+ parser.add_argument("--tag", required=True, help="Specify the tag for the failure type (e.g., 'make', 'custom').")
+ parser.add_argument("-i", required=False, help="Specify the input file containing failure details.")
+ parser.add_argument("-o", required=True, help="Specify the output file to write the failure information.")
+
+ # Parse command-line arguments
+ args = parser.parse_args()
+
+ # Call the function with the specified tag, input file, and output file
+ add_failure_to_markdown(args.tag, args.o)
+
diff --git a/bin/nightly_build/src/parse_coverage.py b/bin/nightly_build/src/parse_coverage.py
new file mode 100644
index 000000000..33811b0a1
--- /dev/null
+++ b/bin/nightly_build/src/parse_coverage.py
@@ -0,0 +1,95 @@
+import os
+import argparse
+from datetime import datetime
+import re
+from colorama import init, Fore, Style
+
+def parse_regression_output(output):
+ passed_configs = []
+ failed_configs = []
+
+ lines = output.split('\n')
+ index = 0
+
+ while index < len(lines):
+ # Remove ANSI escape codes
+ line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index])
+ if "Success" in line:
+ passed_configs.append(line.split(':')[0].strip())
+ elif "Failures detected in output" in line:
+ try:
+ config_name = line.split(':')[0].strip()
+ log_file = os.path.abspath(config_name+".log")
+ failed_configs.append((config_name, log_file))
+ except:
+ failed_configs.append((config_name, "Log file not found"))
+
+ index += 1
+
+ # alphabetically sort the configurations
+ passed_configs.sort()
+ failed_configs.sort()
+ return passed_configs, failed_configs
+
+def write_to_markdown(passed_configs, failed_configs, output_file):
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+ with open(output_file, 'a') as md_file:
+ md_file.write(f"\n\n# Coverage Test Results - {timestamp}\n\n")
+
+ md_file.write("\n## Failed Configurations\n")
+ for config, log_file in failed_configs:
+ md_file.write(f"- {config} ({log_file})\n")
+ if len(failed_configs) == 0:
+ md_file.write(" - no failures\n")
+
+ md_file.write("\n## Passed Configurations\n")
+ for config in passed_configs:
+ md_file.write(f"- {config}\n")
+
+
+def write_new_markdown(passed_configs, failed_configs):
+ timestamp = datetime.now().strftime("%Y-%m-%d")
+ output_file = f"/home/thkidd/nightly_runs/build-results/builds/coverage/wally_coverage_{timestamp}.md"
+
+ with open(output_file, 'w') as md_file:
+ # Title
+ md_file.write(f"\n\n# Coverage Test Results - {timestamp}\n\n")
+
+ ## File path
+ md_file.write(f"\n**File:** {output_file}\n")
+
+ md_file.write("\n## Failed Configurations\n")
+ # add in if there were no failures
+ if len(failed_configs) == 0:
+ md_file.write(f"No Failures\n")
+
+ for config, log_file in failed_configs:
+ md_file.write(f"- {config} ({log_file})\n")
+
+ md_file.write("\n## Passed Configurations\n")
+ for config in passed_configs:
+ md_file.write(f"- {config}\n")
+
+
+
+
+if __name__ == "__main__":
+ init(autoreset=True) # Initialize colorama
+ parser = argparse.ArgumentParser(description='Parse regression test output and append to a markdown file.')
+ parser.add_argument('-i', '--input', help='Input file containing regression test output', required=True)
+ parser.add_argument('-o', '--output', help='Output markdown file', default='regression_results.md')
+ args = parser.parse_args()
+
+ with open(args.input, 'r') as input_file:
+ regression_output = input_file.read()
+
+ passed_configs, failed_configs = parse_regression_output(regression_output)
+ write_to_markdown(passed_configs, failed_configs, args.output)
+
+ print(f"Markdown file updated: {args.output}")
+
+ write_new_markdown(passed_configs, failed_configs)
+
+ print("New markdown file created")
+
diff --git a/bin/nightly_build/src/parse_regression.py b/bin/nightly_build/src/parse_regression.py
new file mode 100644
index 000000000..0d8c9467d
--- /dev/null
+++ b/bin/nightly_build/src/parse_regression.py
@@ -0,0 +1,102 @@
+import argparse
+import os
+from datetime import datetime
+import re
+from colorama import init, Fore, Style
+
+def parse_regression_output(output):
+ passed_configs = []
+ failed_configs = []
+
+ lines = output.split('\n')
+ index = 0
+
+ while index < len(lines):
+ # Remove ANSI escape codes
+ line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index])
+ #print("The cleaned line: ", line)
+ if "Success" in line:
+ passed_configs.append(line.split(':')[0].strip())
+ elif "Failures detected in output" in line:
+ try:
+ config_name = line.split(':')[0].strip()
+ log_file = os.path.abspath(config_name+".log")
+ failed_configs.append((config_name, log_file))
+ except:
+ failed_configs.append((config_name, "Log file not found"))
+ elif "Timeout" in line:
+ try:
+ config_name = line.split(':')[0].strip()
+ log_file = os.path.abspath(config_name+".log")
+ failed_configs.append((config_name, log_file))
+ except:
+ failed_configs.append((config_name, "Log file not found"))
+ index += 1
+
+ # alphabetically sort the configurations
+ passed_configs.sort()
+ failed_configs.sort()
+ return passed_configs, failed_configs
+
+def write_to_markdown(passed_configs, failed_configs, output_file):
+ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ with open(output_file, 'a') as md_file:
+ md_file.write(f"\n\n\n# Regression Test Results - {timestamp}\n
\n\n")
+ #md_file.write(f"\n\n# Regression Test Results - {timestamp}\n\n")
+
+ if failed_configs:
+ md_file.write("## Failed Configurations\n")
+ for config, log_file in failed_configs:
+ md_file.write(f"- {config} ({log_file})\n")
+ md_file.write("\n")
+ else:
+ md_file.write("## No Failed Configurations\n")
+
+ md_file.write("\n## Passed Configurations\n")
+ for config in passed_configs:
+ md_file.write(f"- {config}\n")
+
+def write_new_markdown(passed_configs, failed_configs):
+ timestamp = datetime.now().strftime("%Y-%m-%d")
+ output_file = f"/home/thkidd/nightly_runs/build-results/builds/regression/wally_regression_{timestamp}.md"
+ with open(output_file, 'w') as md_file:
+
+ # Title
+ md_file.write(f"\n\n# Regression Test Results - {timestamp}\n\n")
+ #md_file.write(f"\n\n\n# Regression Test Results - {timestamp}\n
\n\n")
+
+ # File Path
+ md_file.write(f"\n**File:** {output_file}\n\n")
+
+ if failed_configs:
+ md_file.write("## Failed Configurations\n\n")
+ for config, log_file in failed_configs:
+ md_file.write(f"- {config} ({log_file})\n")
+ md_file.write("\n")
+ else:
+ md_file.write("## Failed Configurations\n")
+ md_file.write(f"No Failures\n")
+
+ md_file.write("\n## Passed Configurations\n")
+ for config in passed_configs:
+ md_file.write(f"- {config}\n")
+
+if __name__ == "__main__":
+ init(autoreset=True) # Initialize colorama
+ parser = argparse.ArgumentParser(description='Parse regression test output and append to a markdown file.')
+ parser.add_argument('-i', '--input', help='Input file containing regression test output', required=True)
+ parser.add_argument('-o', '--output', help='Output markdown file containing formatted file', default='regression_results.md')
+ args = parser.parse_args()
+
+ with open(args.input, 'r') as input_file:
+ regression_output = input_file.read()
+
+ passed_configs, failed_configs = parse_regression_output(regression_output)
+ write_to_markdown(passed_configs, failed_configs, args.output)
+
+ print(f"Markdown file updated: {args.output}")
+
+ write_new_markdown(passed_configs, failed_configs)
+
+ print("New markdown file created")
+
diff --git a/bin/nightly_build/src/send_mail_html.sh b/bin/nightly_build/src/send_mail_html.sh
new file mode 100755
index 000000000..5c3735a3a
--- /dev/null
+++ b/bin/nightly_build/src/send_mail_html.sh
@@ -0,0 +1,79 @@
+current_date=$(date "+%Y-%m-%d")
+email_address="thomas.kidd@okstate.edu"
+#email_address="WALLY-REGRESSION@LISTSERV.OKSTATE.EDU"
+subject="WALLY regression and coverage test report"
+attachments=""
+html_body=""
+host_name=$(hostname)
+os_info=$(lsb_release -a 2>/dev/null)
+script_location=$WALLY/bin/nightly_build/
+
+html_body="
+
System Information
+
Server Name: $host_name@okstate.edu
+
Operating System: $os_info
+
Script Origin: $script_location
+
Testing sending HTML content through mutt
"
+
+# Iterate through the files and concatenate their content to the body
+for file in $WALLY/../build-results/builds/*/wally_*_"$current_date"*.md; do
+ attachments+=" -a $file"
+
+ # Convert Markdown to HTML using pandoc
+ html_content=$(pandoc "$file")
+
+ # add the file full path
+ # html_body+="File: $file
"
+ # Append the HTML content to the body
+ html_body+="$html_content"
+done
+echo "Sending email"
+
+# Get server hostname and OS information
+host_name=$(hostname)
+os_info=$(uname -a)
+
+# Define HTML body content
+
+# Use mutt to send the email with HTML body
+#mutt -e "my_hdr From:James Stine " -s "$subject" $attachments \
+mutt -e "my_hdr From:Thomas Kidd " -s "$subject" \
+ -e "set content_type=text/html" -- $email_address <
+
+
+
+ Nightly Build Results - $current_date
+
+
+
+ Test Results - $current_date
+
+ $html_body
+
+