buildroot progress -- able to mimic GDB output

wally-pipelined/linux-testgen/combineGDBs.py

@@ -0,0 +1,68 @@
+#! /usr/bin/python3
+
+instrs = 0
+def readBlock(f, start, end):
+  l = f.readline()
+  if not l:
+    quit()
+  while not (l.startswith(start) and 'in ' not in l):
+    l = f.readline()
+    if not l:
+      quit()
+  ret = l
+  while not l.startswith(end):
+    l = f.readline()
+    if not l:
+      quit()
+    ret += l
+  return ret.split('\n'), f.readline()
+
+with open('gdbcombined.txt', 'w') as out:
+  with open('/mnt/scratch/riscv_gp/riscv_gp.txt', 'r') as gp:
+    with open('/mnt/scratch/riscv_sp1/riscv_sp1.txt', 'r') as sp1:
+      with open('/mnt/scratch/riscv_sp2/riscv_sp2.txt', 'r') as sp2:
+        with open('/mnt/scratch/riscv_sp3/riscv_sp3.txt', 'r') as sp3:
+          with open('/mnt/scratch/riscv_decodepc_threads/riscv_decodepc.txt.disassembly', 'r') as inst:
+            inst.readline()
+            while(True):
+              instrs += 1
+              g, i1 = readBlock(gp, 'ra', 't6')
+              p1, i2 = readBlock(sp1, 'mie', 'scounteren')
+              p2, i3 = readBlock(sp2, '0x', 'mideleg')
+              p3, i4 = readBlock(sp3, 'mcause', 'stvec')
+              instr = inst.readline()
+              if not instr:
+                quit()
+              while '...' in instr:
+                instr = inst.readline()
+                if not instr:
+                  quit()
+              if i1 != i2 or i2 != i3 or i3 != i4 or int(p2[0].split()[0].split(':')[0], 16) != int(instr.split()[0].split(':')[0], 16):
+                print("error: PC was not the same")
+                print("instruction {}".format(instrs))
+                print(i1)
+                print(i2)
+                print(i3)
+                print(i4)
+                print(p2[0])
+                print(instr)
+                quit()
+              if "unimp" in instr:
+                instrs -= 1
+                continue
+              out.write('=> {}'.format(instr.split(':')[2][1:].replace(' ', ':\t', 1)))
+              out.write(p2[0] + '\n')
+              out.write("zero           0x0      0\n")
+              out.write("\n".join(g))
+              pc = p2[0].split()[0]
+              if pc.endswith(':'):
+                pc = pc[:-1]
+              out.write("pc             {}   {}\n".format(pc, pc))
+              out.write("\n".join(p1))
+              out.write("\n".join(p3))
+              out.write("\n".join(p2[2:]))
+              out.write("-----\n")
+              if instrs % 10000 == 0:
+                print(instrs)
+              #if instrs >= 1000010:
+              #  quit()

wally-pipelined/linux-testgen/find_csr.sh

@@ -0,0 +1,2 @@
+#grep '=>.*csr' $1 | rev | cut -d'	' -f1 | rev | tee >(cut -d',' -f1) | cut -d',' -f2 | grep -Ev 'a[0-7]|t[0-6]|zero|[0-8]' | sort | uniq | paste -s -d, -
+grep 'csr' /mnt/scratch/riscv_decodepc_threads/riscv_decodepc.txt.disassembly | rev | cut -d' ' -f1 | rev | tee >(cut -d',' -f1 | sort -u) >(cut -d',' -f2 | sort -u) | (cut -d',' -f3 | sort -u) | sort -u  | paste -s -d, -

wally-pipelined/linux-testgen/fix_mem.py

@@ -0,0 +1,9 @@
+#! /usr/bin/python3
+test_dir = '/courses/e190ax/buildroot_boot/'
+infiles = ['bootmemGDB.txt', 'ramGDB.txt']
+outfiles = ['bootmem.txt', 'ram.txt']
+for i in range(len(infiles)):
+    with open(f'{test_dir}{infiles[i]}', 'r') as f:
+        with open(f'{test_dir}{outfiles[i]}', 'w') as w:
+            for l in f:
+                w.write(f'{"".join([x[2:] for x in l.split()[:0:-1]])}\n')

wally-pipelined/linux-testgen/gdbinit

@@ -0,0 +1,15 @@
+set pagination off
+set logging overwrite on
+set logging redirect on
+set logging file /mnt/scratch/riscv_testbench/riscv_boot_regs.txt
+set logging on
+x/i $pc
+x/x $pc
+info all-registers
+while ($pc != 0xffffffe000018fa4)
+  si
+  x/i $pc
+  x/x $pc
+  info all-registers
+end
+set logging off

wally-pipelined/linux-testgen/gdbinit_mem

@@ -0,0 +1,23 @@
+set pagination off
+target extended-remote :1234
+set logging overwrite on
+set logging redirect on
+printf "Creating bootmemGDB.txt\n"
+set logging file /courses/e190ax/buildroot_boot/bootmemGDB.txt
+set logging on
+x/4096xb 0x1000
+set logging off
+printf "Creating bootmem_untrimmed_GDB.txt\n"
+printf "Warning - please verify that the second half of bootmem_untrimmed_GDB.txt is all 0s\n"
+set logging file /courses/e190ax/buildroot_boot/bootmem_untrimmed_GDB.txt
+set logging on
+x/8192xb 0x1000
+set logging off
+printf "Creating ramGDB.txt\n"
+set logging file /courses/e190ax/buildroot_boot/ramGDB.txt
+set logging on
+x/134217728xb 0x80000000
+set logging off
+set confirm off
+kill
+q

wally-pipelined/linux-testgen/gdbinit_qemulog

@@ -0,0 +1,10 @@
+set pagination off
+target extended-remote :1234
+b *0xffffffe00020144e
+c
+c
+c
+c
+set confirm off
+kill
+q

wally-pipelined/linux-testgen/gdbinit_qemulog_debug

@@ -0,0 +1,9 @@
+set pagination off
+target extended-remote :1234
+b *0x000000008020103c
+c
+del 1
+stepi 100
+set confirm off
+kill
+q

wally-pipelined/linux-testgen/logAllBuildroot.sh

@@ -0,0 +1,26 @@
+# =========== Debug the Process ========== 
+# Uncomment this version for GDB/QEMU debugging
+# - Opens up GDB interactively
+# - Logs raw QEMU output to qemu_output.txt
+#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2> /mnt/scratch/wally_linux_output/qemu_output.txt) & riscv64-unknown-elf-gdb
+
+# Uncomment this version to generate qemu_output.txt
+# - Uses GDB script
+# - Logs raw QEMU output to qemu_output.txt
+#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>/mnt/scratch/wally_linux_output/qemu_output.txt) & riscv64-unknown-elf-gdb -x gdbinit_qemulog
+
+# Uncomment this version for parse_qemu.py debugging
+# - Uses qemu_output.txt
+# - Makes qemu_in_gdb_format.txt
+# - Logs parse_qemu.py's simulated gdb output to qemu_in_gdb_format.txt
+#cat /mnt/scratch/wally_linux_output/qemu_output.txt | ./parse_qemu.py >/mnt/scratch/wally_linux_output/qemu_in_gdb_format.txt
+
+# Uncomment this version for parse_gdb_output.py debugging
+# - Uses qemu_in_gdb_format.txt
+# - Logs info needed by buildroot testbench
+cat /mnt/scratch/wally_linux_output/qemu_in_gdb_format.txt | ./parse_gdb_output.py "/courses/e190ax/buildroot_boot/"
+
+# =========== Just Do the Thing ========== 
+# Uncomment this version for the whole thing (if it works ha ha_
+# - Logs info needed by buildroot testbench
+#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>&1 >/dev/null | pv -l | ./parse_qemu.py | ./parse_gdb_output.py "/courses/e190ax/buildroot_boot/") & riscv64-unknown-elf-gdb -x gdbinit_qemulog

wally-pipelined/linux-testgen/logBuildrootMem.sh

@@ -0,0 +1,4 @@
+(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>/dev/null >/dev/null ) &
+riscv64-unknown-elf-gdb -x gdbinit_mem
+#sed -i '$d' $file
+echo "Done"

wally-pipelined/linux-testgen/parseAllBusybear.sh

@@ -0,0 +1 @@
+./combineGDBs.py && cat gdbcombined.txt | ./parse_gdb_output.py "/courses/e190ax/busybear_boot_new/"

wally-pipelined/linux-testgen/parse_gdb_output.py

@@ -0,0 +1,164 @@
+#! /usr/bin/python3
+import sys, fileinput
+
+sys.stderr.write("reminder: this script takes input from stdin\n")
+
+csrs = ['fcsr','mcause','mcounteren','medeleg','mepc','mhartid','mideleg','mie','mip','misa','mscratch','mstatus','mtval','mtvec','pmpaddr0','pmpcfg0','satp','scause','scounteren','sepc','sie','sscratch','sstatus','stval','stvec']
+
+# just for now, since these CSRs aren't yet ready to be checked in testbench-linux
+list(map(csrs.remove, ['fcsr','mhartid','pmpcfg0','pmpaddr0','mip']))
+#output_path = '/courses/e190ax/busybear_boot_new/'
+#output_path = '/courses/e190ax/buildroot_boot/'
+output_path = sys.argv[1]
+print(f'output dir: {output_path}')
+instrs = -1
+try:
+    with open('{}parsedPC.txt'.format(output_path), 'w') as wPC:
+      with open('{}parsedRegs.txt'.format(output_path), 'w') as wReg:
+        with open('{}parsedMemRead.txt'.format(output_path), 'w') as wMem:
+          with open('{}parsedMemWrite.txt'.format(output_path), 'w') as wMemW:
+            with open('{}parsedCSRs.txt'.format(output_path), 'w') as wCSRs:
+              firstCSR = True
+              curCSRs = {}
+              lastRead = ''
+              currentRead = ''
+              readOffset = ''
+              lastReadLoc = ''
+              readType = ''
+              lastReadType = ''
+              readLoc = ''
+              instrStart = -1
+              lastRegs = ''
+              curRegs = ''
+              storeReg = ''
+              storeOffset = ''
+              storeLoc = ''
+              storeAMO = ''
+              lastAMO = ''
+              lastStoreReg = ''
+              lastStoreLoc = ''
+              for l in fileinput.input('-'):
+                l = l.split("#")[0].rstrip()
+                if l.startswith('=>'):
+                  instrs += 1
+                  storeAMO = ''
+                  if instrs % 10000 == 0:
+                    print(instrs)
+                  wPC.write('{} ***\n'.format(' '.join(l.split(':')[1].split()[0:2])))
+                  if '\tld' in l or '\tlw' in l or '\tlh' in l or '\tlb' in l:
+                    currentRead = l.split()[-1].split(',')[0]
+                    if len(l.split()[-1].split(',')) < 2:
+                      print(l)
+                    readOffset = l.split()[-1].split(',')[1].split('(')[0]
+                    readLoc = l.split()[-1].split(',')[1].split('(')[1][:-1]
+                    readType = l.split()[-2]
+                  if 'amo' in l:
+                    #print(l)
+                    currentRead = l.split()[-1].split(',')[0]
+                    readOffset = "0"
+                    readLoc = l.split()[-1].split('(')[1][:-1]
+                    readType = l.split()[-2]
+                    storeOffset = "0"
+                    storeLoc = readLoc
+                    storeReg = l.split()[-1].split(',')[1]
+                    storeAMO = l.split()[-2]
+                  if '\tsd' in l or '\tsw' in l or '\tsh' in l or '\tsb' in l:
+                    #print(l)
+                    s = l.split('#')[0].split()[-1]
+                    storeReg = s.split(',')[0]
+                    if len(s.split(',')) < 2:
+                      print(s)
+                      print(l)
+                    if len(s.split(',')[1].split('(')) < 1:
+                      print(s)
+                      print(l)
+                    storeOffset = s.split(',')[1].split('(')[0]
+                    storeLoc = s.split(',')[1].split('(')[1][:-1]
+                  instrStart = 0
+                elif instrStart != -1:
+                  instrStart += 1
+                  if instrStart == 1:
+                    wPC.write('{}\n'.format(l.split()[-1][2:]))
+                  elif instrStart < 34:
+                    if lastRead == l.split()[0]:
+                      readData  = int(l.split()[1][2:], 16)
+                      readData <<= (8 * (lastReadLoc % 8))
+                      #if(lastReadLoc % 8 != 0 and ('lw' in lastReadType or 'lb' in lastReadType)):
+                      #  readData <<= 32
+                      wMem.write('{:x}\n'.format(readData))
+                    if readLoc == l.split()[0]:
+                      readLoc = l.split()[1][2:]
+                    if storeReg == l.split()[0]:
+                      storeReg = l.split()[1]
+                    if storeLoc == l.split()[0]:
+                      storeLoc = l.split()[1][2:]
+                    if instrStart > 2:
+                      #print(l)
+                      #print(instrStart)
+                      curRegs += '{}\n'.format(l.split()[1][2:])
+                  elif instrStart < 35:
+                    #print("----------")
+                    #print(l.split()[1][2:])
+                    wPC.write('{}\n'.format(l.split()[1][2:]))
+                    #print(l.split()[1][2:])
+                if any([c == l.split()[0] for c in csrs]):
+                  if l.split()[0] in curCSRs:
+                    if curCSRs[l.split()[0]] != l.split()[1]:
+                      if firstCSR:
+                        wCSRs.write('---\n')
+                        firstCSR = False
+                      wCSRs.write('{}\n{}\n'.format(l.split()[0], l.split()[1][2:]))
+                  else:
+                    wCSRs.write('{}\n{}\n'.format(l.split()[0], l.split()[1][2:]))
+                  curCSRs[l.split()[0]] = l.split()[1]
+                if '-----' in l: # end of each cycle
+                    if curRegs != lastRegs:
+                      if lastRegs == '':
+                        wReg.write(curRegs)
+                      else:
+                        for i in range(32):
+                          if curRegs.split('\n')[i] != lastRegs.split('\n')[i]:
+                            wReg.write('{}\n'.format(i+1))
+                            wReg.write('{}\n'.format(curRegs.split('\n')[i]))
+                            break
+                      lastRegs = curRegs
+                    if lastAMO != '':
+                      if 'amoadd' in lastAMO:
+                        lastStoreReg = hex(int(lastStoreReg[2:], 16) + readData)[2:]
+                      elif 'amoand' in lastAMO:
+                        lastStoreReg = hex(int(lastStoreReg[2:], 16) & readData)[2:]
+                      elif 'amoor' in lastAMO:
+                        lastStoreReg = hex(int(lastStoreReg[2:], 16) | readData)[2:]
+                      elif 'amoswap' in lastAMO:
+                        lastStoreReg = hex(int(lastStoreReg[2:], 16))[2:]
+                      else:
+                        print(lastAMO)
+                        exit()
+                      wMemW.write('{}\n'.format(lastStoreReg))
+                      wMemW.write('{:x}\n'.format(int(lastStoreLoc, 16)))
+                    if storeReg != '' and storeOffset != '' and storeLoc != '' and storeAMO == '':
+                      storeLocOffset = int(storeOffset,10) + int(storeLoc, 16)
+                      #wMemW.write('{:x}\n'.format(int(storeReg, 16) << (8 * (storeLocOffset % 8))))
+                      wMemW.write('{}\n'.format(storeReg[2:]))
+                      wMemW.write('{:x}\n'.format(storeLocOffset))
+                    if readOffset != '' and readLoc != '':
+                      wMem.write('{:x}\n'.format(int(readOffset,10) + int(readLoc, 16)))
+                      lastReadLoc = int(readOffset,10) + int(readLoc, 16)
+                    lastReadType = readType
+                    readOffset = ''
+                    readLoc = ''
+                    curRegs = ''
+                    instrStart = -1
+                    lastRead = currentRead
+                    currentRead = ''
+                    lastStoreReg = storeReg
+                    lastStoreLoc = storeLoc
+                    storeReg = ''
+                    storeOffset = ''
+                    storeLoc = ''
+                    lastAMO = storeAMO
+
+
+except (FileNotFoundError):
+  print('please give gdb output file as argument')
+

wally-pipelined/linux-testgen/parse_qemu.py

@@ -0,0 +1,107 @@
+#! /usr/bin/python3
+import fileinput, sys
+
+sys.stderr.write("reminder: this script takes input from stdin\n")
+parseState = "idle"
+inPageFault = 0
+CSRs = {}
+pageFaultCSRs = {}
+regs = {}
+pageFaultRegs = {}
+instrs = {}
+
+def printPC(l):
+    global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs
+    if not inPageFault:
+        inst = l.split()
+        if len(inst) > 3:
+            print(f'=> {inst[1]}:\t{inst[2]} {inst[3]}')
+        else:
+            print(f'=> {inst[1]}:\t{inst[2]}')
+        print(f'{inst[0]} 0x{inst[1]}')
+
+def printCSRs():
+    global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs
+    if not inPageFault:
+        for (csr,val) in CSRs.items():
+            print('{}{}{:#x}  {}'.format(csr, ' '*(15-len(csr)), val, val))
+        print('-----')
+
+def parseCSRs(l):
+    global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs
+    if l.strip() and (not l.startswith("Disassembler")) and (not l.startswith("Please")):
+        if l.startswith(' x0/zero'):
+            parseState = "regFile"
+            instr = instrs[CSRs["pc"]]
+            printPC(instr)
+            parseRegs(l)
+        else:
+            csr = l.split()[0]
+            val = int(l.split()[1],16)
+            if inPageFault:
+                if l.startswith("mstatus") or l.startswith("mepc") or l.startswith("mcause") or l.startswith("mtval") or l.startswith("sepc") or l.startswith("scause") or l.startswith("stval"):
+                    # We do update some CSRs
+                    CSRs[csr] = val
+                else:
+                    # Others we preserve until changed later
+                    pageFaultCSRs[csr] = val
+            elif pageFaultCSRs and (csr in pageFaultCSRs):
+                if (val != pageFaultCSRs[csr]):
+                    del pageFaultCSRs[csr]
+                    CSRs[csr] = val
+            else:
+                CSRs[csr] = val
+
+def parseRegs(l):
+    global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs
+    if "mcounteren" in l:
+        printCSRs()
+        # New non-disassembled instruction
+        parseState = "CSRs"
+        parseCSRs(l)
+    elif l.startswith('--------'):
+        # End of disassembled instruction
+        printCSRs()
+        parseState = "idle"
+    else:
+        s = l.split()
+        for i in range(0,len(s),2):
+            if '/' in s[i]:
+                reg = s[i].split('/')[1]
+                val = int(s[i+1], 16)
+                if inPageFault:
+                    pageFaultRegs[reg] = val
+                else:
+                    if pageFaultRegs and (reg in pageFaultRegs):
+                        if (val != pageFaultRegs[reg]):
+                            del pageFaultRegs[reg]
+                            regs[reg] = val
+                    else:
+                        regs[reg] = val
+                    val = regs[reg]
+                    print('{}{}{:#x}  {}'.format(reg, ' '*(15-len(reg)), val, val))
+            else:
+                sys.stderr.write("Whoops. Expected a list of reg file regs; got:\n"+l)
+
+#############
+# Main Code #
+#############
+for l in fileinput.input():
+    if l.startswith('qemu-system-riscv64: QEMU: Terminated via GDBstub'):
+        break
+    elif l.startswith('IN:'):
+        # New disassembled instr
+        parseState = "instr"
+    elif (parseState == "instr") and l.startswith('0x'):
+        if "out of bounds" in l:
+            sys.stderr.write("Detected QEMU page fault error\n")
+            inPageFault = 1
+        else: 
+            inPageFault = 0
+            adr = int(l.split()[0][2:-1], 16)
+            instrs[adr] = l
+        parseState = "CSRs"
+    elif parseState == "CSRs":
+        parseCSRs(l)
+    elif parseState == "regFile":
+        parseRegs(l)

wally-pipelined/linux-testgen/setup_OVP.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+source /cad/riscv/OVP/Imperas.20200630/bin/setup.sh
+setupImperas /cad/riscv/OVP/Imperas.20200630 -m32
+source /cad/riscv/OVP/Imperas.20200630/bin/switchRuntime.sh 2>/dev/null
+echo 1 | switchRuntimeImperas
+source /cad/riscv/OVP/Imperas.20200630/bin/switchISS.sh 2>/dev/null
+echo 1 | switchISSImperas

wally-pipelined/linux-testgen/start_OVP.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+sh /cad/riscv/OVP/Imperas.20200630/Demo/Platforms/riscv_RV64GC_Virtio_Linux/harness/RUN_Virtio_Linux.sh --gdbconsole --gdbinit /mnt/scratch/riscv_testbench/gdbinit