first attemtpt at checkpoint infrastructure

wally-pipelined/linux-testgen/testvector-generation/GenerateCheckpoint.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Oftentimes this script runs so long you'll go to sleep.
+# But you don't want the script to die when your computer goes to sleep.
+# So consider invoking this with nohup (i.e. "nohup ./logAllBuildroot.sh")
+# You can run "tail -f nohup.out" to see what would've
+# outputted to the terminal if you didn't use nohup
+
+# use on tera.
+customQemu="/courses/e190ax/qemu_sim/rv64_initrd/qemu_experimental/qemu/build/qemu-system-riscv64"
+# use on other systems
+#customQemu="qemu-system-riscv64"
+
+instrs=8500000
+
+imageDir="../buildroot-image-output"
+outDir="../linux-testvectors/checkpoint$instrs"
+intermedDir="$outDir/intermediate-outputs"
+
+
+read -p "This scripts is going to create a checkpoint at $instrs instrs.
+Is that what you wanted? (y/n) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    mkdir -p $outDir
+    mkdir -p $intermedDir
+    ($customQemu -M virt -nographic -bios $imageDir/fw_jump.elf -kernel $imageDir/Image -append "root=/dev/vda ro" -initrd $imageDir/rootfs.cpio -rtc clock=vm -icount shift=1 -d nochain,cpu,in_asm -serial /dev/null -singlestep -gdb tcp::1240 -S 2>&1 1>&2 | ./parse_qemu.py | ./parseNew.py | ./remove_dup.awk > $intermedDir/rawTrace.txt) & riscv64-unknown-elf-gdb -x ./checkpoint.gdb -ex "createCheckpoint $instrs \"$intermedDir\""
+    ./fix_mem.py "$intermedDir/ramGDB.txt" "$outDir/ram.txt"
+    ./parseState.py "$outDir"
+else
+    echo "You can change the number of instructions by editing the \"instrs\" variable in this script."
+    echo "Have a nice day!"
+fi

wally-pipelined/linux-testgen/testvector-generation/checkpoint.gdb

@@ -0,0 +1,53 @@
+define createCheckpoint 
+    # GDB config
+    set pagination off
+    set logging overwrite on
+    set logging redirect on
+    set confirm off
+
+    # QEMU must also use TCP port 1240
+    target extended-remote :1240
+
+    # Argument Parsing
+    set $statePath=$arg1
+    set $ramPath=$arg1
+    eval "set $statePath = \"%s/stateGDB.txt\"", $statePath
+    eval "set $ramPath = \"%s/ramGDB.txt\"", $ramPath
+
+    # Symbol file
+    file ../buildroot-image-output/vmlinux
+
+    # Step over reset vector into actual code
+    stepi 1000
+    # Set breakpoint for where to stop
+    b do_idle
+    # Proceed to checkpoint 
+    printf "GDB proceeding to checkpoint at %d instrs\n", $arg0
+    stepi $arg0-1000
+ 
+    printf "Reached checkpoint at %d instrs\n", $arg0
+
+    # Log all registers to a file
+    printf "GDB storing state to %s\n", $statePath
+    set logging file $statePath
+    set logging on
+    info all-registers
+    set logging off
+
+    # Log main memory to a file
+    printf "GDB storing RAM to %s\n", $ramPath
+    set logging file ../linux-testvectors/intermediate-outputs/ramGDB.txt
+    set logging on
+    x/134217728xb 0x80000000
+    set logging off
+    
+    # Continue to checkpoint; stop on the 3rd time
+    # Should reach login prompt by then
+    printf "GDB continuing execution to login prompt\n"
+    ignore 1 2
+    c
+    
+    printf "GDB reached login prompt!\n"
+    kill
+    q
+end

wally-pipelined/linux-testgen/testvector-generation/fix_mem.py

@@ -7,8 +7,9 @@ inputFile = sys.argv[1]
 outputFile = sys.argv[2]
 if not os.path.exists(inputFile):
     sys.exit('Error input file '+inputFile+' not found')
-print('Translating '+os.path.basename(inputFile)+' to '+os.path.basename(outputFile))
+print('Begin translating '+os.path.basename(inputFile)+' to '+os.path.basename(outputFile))
 with open(inputFile, 'r') as f:
     with open(outputFile, 'w') as w:
         for l in f:
             w.write(f'{"".join([x[2:] for x in l.split()[:0:-1]])}\n')
+print('Finished translating '+os.path.basename(inputFile)+' to '+os.path.basename(outputFile)+'!')

wally-pipelined/linux-testgen/testvector-generation/parseNew.py

@@ -140,6 +140,7 @@ CurrentInstr = ['0', '0', None, 'other', {'zero': 0, 'ra': 0, 'sp': 0, 'gp': 0,
 lineNum = 0
 StartLine = 0
 EndLine = 0
+numInstrs = 0
 #instructions = []
 MemAdr = 0
 lines = []
@@ -195,6 +196,10 @@ for line in fileinput.input('-'):
         lines.clear()
         #instructions.append(MoveInstrToRegWriteLst)
         PrintInstr(MoveInstrToRegWriteLst, sys.stdout)
+        numInstrs +=1
+        if (numInstrs % 1e4 == 0):
+            sys.stderr.write('Trace parser reached '+str(numInstrs/1.0e6)+' million instrs.\n')
+            sys.stderr.flush()
     lineNum += 1
 
 

wally-pipelined/linux-testgen/testvector-generation/parseState.py

@@ -0,0 +1,79 @@
+#! /usr/bin/python3
+import sys
+
+################
+# Helper Funcs #
+################
+
+def tokenize(string):
+    tokens = []
+    token = ''
+    whitespace = 0
+    prevWhitespace = 0
+    for char in string:
+        prevWhitespace = whitespace
+        whitespace = char in ' \t\n'
+        if (whitespace):
+            if ((not prevWhitespace) and (token != '')): 
+                tokens.append(token)
+            token = ''
+        else:
+            token = token + char
+    return tokens
+
+#############
+# Main Code #
+#############
+print("Begin parsing state.")
+
+# Parse Args
+if len(sys.argv) != 2:
+    sys.exit('Error parseState.py expects 1 arg:\n parseState.py <path_to_checkpoint_dir>')
+outDir = sys.argv[1]
+stateGDBpath = outDir+'/intermediate-outputs/stateGDB1K.txt'
+if not os.path.exists(stateGDBpath):
+    sys.exit('Error input file '+stateGDBpath+'not found')
+
+listCSRs = ['hpmcounter','pmpcfg','pmpaddr']
+singleCSRs = ['mip','mie','mscratch','mcause','mepc','mtvec','medeleg','mideleg','mcounteren','sscratch','scause','sepc','stvec','sedeleg','sideleg','scounteren','satp','mstatus']
+
+# Initialize List CSR files to empty
+# (because later we'll open them in append mode)
+for csr in listCSRs:
+    outFileName = 'checkpoint-'+csr.upper() 
+    outFile = open(outDir+outFileName, 'w')
+    outFile.close()
+
+# Initial State for Main Loop
+currState = 'regFile'
+regFileIndex = 0
+outFileName = 'checkpoint-regfile.txt'
+outFile = open(outDir+outFileName, 'w')
+
+# Main Loop
+with open(stateGDBpath, 'r') as stateGDB:
+    for line in stateGDB:
+        line = tokenize(line)
+        name = line[0]
+        val = line[1][2:]
+        if (currState == 'regFile'):
+            if (regFileIndex == 0 and name != 'zero'):
+                print('Whoops! Expected regFile registers to come first, starting with zero')
+                exit(1)
+            outFile.write(val+'\n')
+            regFileIndex += 1
+            if (regFileIndex == 32):
+                outFile.close()
+                currState = 'CSRs'
+        elif (currState == 'CSRs'):
+            if name in singleCSRs: 
+                outFileName = 'checkpoint-'+name.upper() 
+                outFile = open(outDir+outFileName, 'w')
+                outFile.write(val+'\n')
+                outFile.close()
+            elif name.strip('0123456789') in listCSRs:
+                outFileName = 'checkpoint-'+name.upper().strip('0123456789')
+                outFile = open(outDir+outFileName, 'a')
+                outFile.write(val+'\n')
+                outFile.close()
+print("Finished parsing state!")