Adding nightly regression into the bin directory (#607)

* updated install tool chain file to use verilator v5.016

* updated gitignore file

* adding nightly tests

* reverted the verilator checkout to checkout master

* Revert "reverted the verilator checkout to checkout master"

This reverts commit 942a2804c3.

* Revert "updated gitignore file"

This reverts commit 981c3ccf6b.

* Revert "updated install tool chain file to use verilator v5.016"

This reverts commit 3168859268.

* Revert "updated gitignore file"

This reverts commit 981c3ccf6b.

bin/nightly_build/nightly_build_v13_beta.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# set WALLY path
+WALLY=$(dirname ${BASH_SOURCE[0]:-$0})
+export WALLY=$(cd "$WALLY/../../" && pwd)
+echo "WALLY is set to: $WALLY"
+
+# Going to nightly runs
+cd $WALLY/../
+
+# check if directories exist
+if [ ! -d "build-results" ]; then
+    echo "Directory does not exist, creating it..."
+    mkdir -p "build-results"
+    if [ $? -eq 0 ]; then
+        echo "Directory created successfully."
+    else
+        echo "Failed to create directory."
+        exit 1
+    fi
+else
+    echo "Directory already exists."
+fi
+
+if [ ! -d "logs" ]; then
+    echo "Directory does not exist, creating it..."
+    mkdir -p "logs"
+    if [ $? -eq 0 ]; then
+        echo "Directory created successfully."
+    else
+        echo "Failed to create directory."
+        exit 1
+    fi
+else
+    echo "Directory already exists."
+fi
+
+# setup source okstate file
+echo "Sourcing setup files"
+source $WALLY/setup_host.sh
+source $WALLY/../setup-files/setup_tools.sh
+
+# Navigate to the gir repo
+cd $WALLY
+
+# pull the repository
+echo "Pulling submodules"
+#git pull --recurse-submodules origin main
+
+# build the regression tests
+echo "Building the regression tests"
+cd sim
+#if make wally-riscv-arch-test; then
+#if make all; then
+#	echo "Make successfull"
+#else
+#	echo "Make failed"
+#	cd $WALLY/..
+	# add the the regression result and the coverage result that there was an error in making the tests
+#	python $WALLY/bin/nightly_build/src/error_detector.py --tag make -o $WALLY/../build-results/regression_results.md
+#	python $WALLY/bin/nightly_build/src/error_detector.py --tag make -o $WALLY/../build-results/coverage_results.md
+
+	# exit the program
+	#exit 1
+#fi
+
+# execute the simulation / regression tests and save output to a file
+echo "running the regression test"
+#./regression-wally > $WALLY/../logs/regression_output.log 2>&1
+
+echo "running coverage tests"
+#./coverage > $WALLY/../logs/coverage_output.log 2>&1
+
+
+# run the Python script to parse the output and generate the log file
+echo "Parsing output data from the regression test"
+cd $WALLY/../
+
+python $WALLY/bin/nightly_build/src/parse_regression.py -i $WALLY/../logs/regression_output.log -o $WALLY/../build-results/regression_results.md
+
+python $WALLY/bin/nightly_build/src/parse_coverage.py -i $WALLY/../logs/coverage_output.log -o $WALLY/../build-results/coverage_results.md
+
+# email update
+cd $WALLY/bin/nightly_build/src/
+./send_mail_html.sh

bin/nightly_build/src/error_detector.py

@@ -0,0 +1,37 @@
+import argparse
+import datetime
+
+def add_failure_to_markdown(tag, output_file, input_file = None):
+    # Get the current date and time
+    current_datetime = datetime.datetime.now()
+    formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+    # Create the failure message based on the provided tag and input content
+    if tag == "make":
+        failure_message = f"# {tag.capitalize()} riscof comilation failure - {formatted_datetime}\n\n"
+        failure_message += f"The error was due to a problem in compiling the the riscof tests:\n\n"
+        if input_file != None:
+            failure_message += f"The particular error: {input_file}\n\n"
+    else:
+        failure_message = f"# {tag.capitalize()} Failure - {formatted_datetime}\n\n"
+        failure_message += f":\n\n"
+
+    # Append the failure message to the specified output file
+    with open(output_file, "a") as file:
+        file.write(failure_message)
+
+    print(f"Failure information added to {output_file}.")
+
+if __name__ == "__main__":
+    # Set up argparse
+    parser = argparse.ArgumentParser(description="Add failure information to Markdown file.")
+    parser.add_argument("--tag", required=True, help="Specify the tag for the failure type (e.g., 'make', 'custom').")
+    parser.add_argument("-i", required=False, help="Specify the input file containing failure details.")
+    parser.add_argument("-o", required=True, help="Specify the output file to write the failure information.")
+
+    # Parse command-line arguments
+    args = parser.parse_args()
+
+    # Call the function with the specified tag, input file, and output file
+    add_failure_to_markdown(args.tag,  args.o)
+

bin/nightly_build/src/parse_coverage.py

@@ -0,0 +1,95 @@
+import os
+import argparse
+from datetime import datetime
+import re
+from colorama import init, Fore, Style
+
+def parse_regression_output(output):
+    passed_configs = []
+    failed_configs = []
+
+    lines = output.split('\n')
+    index = 0
+
+    while index < len(lines):
+        # Remove ANSI escape codes
+        line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index])  
+        if "Success" in line:
+            passed_configs.append(line.split(':')[0].strip())
+        elif "Failures detected in output" in line:
+            try:
+                config_name = line.split(':')[0].strip()
+                log_file = os.path.abspath(config_name+".log") 
+                failed_configs.append((config_name, log_file))
+            except:
+                failed_configs.append((config_name, "Log file not found"))
+
+        index += 1
+
+    # alphabetically sort the configurations
+    passed_configs.sort()
+    failed_configs.sort()
+    return passed_configs, failed_configs
+
+def write_to_markdown(passed_configs, failed_configs, output_file):
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    with open(output_file, 'a') as md_file:
+        md_file.write(f"\n\n# Coverage Test Results - {timestamp}\n\n")
+
+        md_file.write("\n## Failed Configurations\n")
+        for config, log_file in failed_configs:
+            md_file.write(f"- <span class=\"failure\" style=\"color: red;\">{config}</span> ({log_file})\n")
+            if len(failed_configs) == 0:
+                md_file.write(" - no failures\n")
+
+        md_file.write("\n## Passed Configurations\n")
+        for config in passed_configs:
+            md_file.write(f"- <span class=\"success\" style=\"color: green;\">{config}</span>\n")
+
+
+def write_new_markdown(passed_configs, failed_configs):
+    timestamp = datetime.now().strftime("%Y-%m-%d")
+    output_file = f"/home/thkidd/nightly_runs/build-results/builds/coverage/wally_coverage_{timestamp}.md"
+    
+    with open(output_file, 'w') as md_file:
+        # Title
+        md_file.write(f"\n\n# Coverage Test Results - {timestamp}\n\n")
+        
+        ## File path
+        md_file.write(f"\n**File:** {output_file}\n")
+
+        md_file.write("\n## Failed Configurations\n")
+        # add in if there were no failures
+        if len(failed_configs) == 0:
+            md_file.write(f"No Failures\n")
+        
+        for config, log_file in failed_configs:
+                md_file.write(f"- <span class=\"failure regression\" style=\"color: red;\">{config}</span> ({log_file})\n")
+
+        md_file.write("\n## Passed Configurations\n")
+        for config in passed_configs:
+            md_file.write(f"- <span class=\"success regression\" style=\"color: green;\">{config}</span>\n")
+
+
+
+
+if __name__ == "__main__":
+    init(autoreset=True)  # Initialize colorama
+    parser = argparse.ArgumentParser(description='Parse regression test output and append to a markdown file.')
+    parser.add_argument('-i', '--input', help='Input file containing regression test output', required=True)
+    parser.add_argument('-o', '--output', help='Output markdown file', default='regression_results.md')
+    args = parser.parse_args()
+
+    with open(args.input, 'r') as input_file:
+        regression_output = input_file.read()
+
+    passed_configs, failed_configs = parse_regression_output(regression_output)
+    write_to_markdown(passed_configs, failed_configs, args.output)
+
+    print(f"Markdown file updated: {args.output}")
+
+    write_new_markdown(passed_configs, failed_configs)
+
+    print("New markdown file created")
+

bin/nightly_build/src/parse_regression.py

@@ -0,0 +1,102 @@
+import argparse
+import os
+from datetime import datetime
+import re
+from colorama import init, Fore, Style
+
+def parse_regression_output(output):
+    passed_configs = []
+    failed_configs = []
+
+    lines = output.split('\n')
+    index = 0
+
+    while index < len(lines):
+        # Remove ANSI escape codes
+        line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index])  
+        #print("The cleaned line: ", line)
+        if "Success" in line:
+            passed_configs.append(line.split(':')[0].strip())
+        elif "Failures detected in output" in line:
+            try:
+                config_name = line.split(':')[0].strip()
+                log_file = os.path.abspath(config_name+".log") 
+                failed_configs.append((config_name, log_file))
+            except:
+                failed_configs.append((config_name, "Log file not found"))
+        elif "Timeout" in line:
+            try:
+                config_name = line.split(':')[0].strip()
+                log_file = os.path.abspath(config_name+".log")
+                failed_configs.append((config_name, log_file))
+            except:
+                failed_configs.append((config_name, "Log file not found"))
+        index += 1
+    
+    # alphabetically sort the configurations
+    passed_configs.sort()
+    failed_configs.sort()
+    return passed_configs, failed_configs
+
+def write_to_markdown(passed_configs, failed_configs, output_file):
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    with open(output_file, 'a') as md_file:
+        md_file.write(f"\n\n<div class=\"regression\">\n# Regression Test Results - {timestamp}\n</div>\n\n")
+        #md_file.write(f"\n\n# Regression Test Results - {timestamp}\n\n")
+
+        if failed_configs:
+            md_file.write("## Failed Configurations\n")
+            for config, log_file in failed_configs:
+                md_file.write(f"- <span class=\"failure regression\" style=\"color: red;\">{config}</span> ({log_file})\n")
+            md_file.write("\n")
+        else:
+            md_file.write("## No Failed Configurations\n")
+
+        md_file.write("\n## Passed Configurations\n")
+        for config in passed_configs:
+            md_file.write(f"- <span class=\"success regression\" style=\"color: green;\">{config}</span>\n")
+
+def write_new_markdown(passed_configs, failed_configs):
+    timestamp = datetime.now().strftime("%Y-%m-%d")
+    output_file = f"/home/thkidd/nightly_runs/build-results/builds/regression/wally_regression_{timestamp}.md"
+    with open(output_file, 'w') as md_file:
+       
+        # Title
+        md_file.write(f"\n\n# Regression Test Results - {timestamp}\n\n")
+        #md_file.write(f"\n\n<div class=\"regression\">\n# Regression Test Results - {timestamp}\n</div>\n\n")
+
+        # File Path
+        md_file.write(f"\n**File:** {output_file}\n\n")
+
+        if failed_configs:
+            md_file.write("## Failed Configurations\n\n")
+            for config, log_file in failed_configs:
+                md_file.write(f"- <span class=\"failure regression\" style=\"color: red;\">{config}</span> ({log_file})\n")
+            md_file.write("\n")
+        else:
+            md_file.write("## Failed Configurations\n")
+            md_file.write(f"No Failures\n")
+        
+        md_file.write("\n## Passed Configurations\n")
+        for config in passed_configs:
+            md_file.write(f"- <span class=\"success regression\" style=\"color: green;\">{config}</span>\n")
+
+if __name__ == "__main__":
+    init(autoreset=True)  # Initialize colorama
+    parser = argparse.ArgumentParser(description='Parse regression test output and append to a markdown file.')
+    parser.add_argument('-i', '--input', help='Input file containing regression test output', required=True)
+    parser.add_argument('-o', '--output', help='Output markdown file containing formatted file', default='regression_results.md')
+    args = parser.parse_args()
+
+    with open(args.input, 'r') as input_file:
+        regression_output = input_file.read()
+
+    passed_configs, failed_configs = parse_regression_output(regression_output)
+    write_to_markdown(passed_configs, failed_configs, args.output)
+
+    print(f"Markdown file updated: {args.output}")
+
+    write_new_markdown(passed_configs, failed_configs)
+
+    print("New markdown file created")
+

bin/nightly_build/src/send_mail_html.sh

@@ -0,0 +1,79 @@
+current_date=$(date "+%Y-%m-%d")
+email_address="thomas.kidd@okstate.edu"
+#email_address="WALLY-REGRESSION@LISTSERV.OKSTATE.EDU"
+subject="WALLY regression and coverage test report"
+attachments=""
+html_body=""
+host_name=$(hostname) 
+os_info=$(lsb_release -a 2>/dev/null)
+script_location=$WALLY/bin/nightly_build/
+
+html_body="<div id='system-info'>
+    <h3>System Information</h3>
+    <p><strong>Server Name:</strong> $host_name@okstate.edu</p>  
+    <p><strong>Operating System:</strong> $os_info</p>
+    <p><strong>Script Origin:</strong> $script_location</p>
+</div>
+
+<p>Testing sending HTML content through mutt</p>"
+
+# Iterate through the files and concatenate their content to the body
+for file in $WALLY/../build-results/builds/*/wally_*_"$current_date"*.md; do
+    attachments+=" -a $file"
+
+    # Convert Markdown to HTML using pandoc
+    html_content=$(pandoc "$file")
+
+    # add the file full path
+    # html_body+="<p>File: $file</p>"
+    # Append the HTML content to the body
+    html_body+="$html_content"
+done
+echo "Sending email"
+
+# Get server hostname and OS information
+host_name=$(hostname)
+os_info=$(uname -a)
+
+# Define HTML body content
+
+# Use mutt to send the email with HTML body
+#mutt -e "my_hdr From:James Stine <james.stine@okstate.edu>" -s "$subject" $attachments \
+mutt -e "my_hdr From:Thomas Kidd <thomas.kidd@okstate.edu>" -s "$subject"  \
+    -e "set content_type=text/html" -- $email_address <<EOF
+<html>
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Nightly Build Results - $current_date</title>
+    <style>
+        body {
+            font-family: 'Helvetica Neue', Arial, sans-serif;
+            margin: 20px;
+            padding: 20px;
+            background-color: #f4f4f4;
+        }
+        h1, h3, p {
+            color: #333;
+        }
+        #system-info {
+            border: 1px solid #ddd;
+            padding: 15px;
+            margin-bottom: 20px;
+            background-color: #fff;
+            border-radius: 5px;
+            box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
+        }
+<!--	.success { color: green; } -->
+<!--    .failure { color: red; }   -->
+
+    </style>
+</head>
+<body>
+    <h1 id="test-results---$current_date">Test Results - $current_date</h1>
+    
+    $html_body
+</body>
+</html>
+EOF
+