From 1f52a2f93827f209c4a8802fd109975be0f6f3c8 Mon Sep 17 00:00:00 2001 From: bbracker Date: Fri, 9 Jul 2021 17:03:47 -0400 Subject: [PATCH] organize/update buildroot scripts for new image --- .gitignore | 1 + .../linux-testgen/WALLY-README.txt | 7 +++- .../buildroot-config-src/main.config | 23 ++++++----- .../parseAllBusybear.sh | 0 .../{ => deprecated-scripts}/setup_OVP.sh | 0 .../{ => deprecated-scripts}/start_OVP.sh | 0 .../testvector-generation/fix_mem.py | 8 ++-- .../testvector-generation/gdbinit_mem | 6 +-- .../testvector-generation/logAllBuildroot.sh | 30 +++++++-------- .../testvector-generation/logBuildrootMem.sh | 7 +++- .../testvector-generation/parse_qemu.py | 38 +++++++++++++------ 11 files changed, 73 insertions(+), 47 deletions(-) rename wally-pipelined/linux-testgen/testvector-generation/{ => deprecated-scripts}/parseAllBusybear.sh (100%) rename wally-pipelined/linux-testgen/testvector-generation/{ => deprecated-scripts}/setup_OVP.sh (100%) rename wally-pipelined/linux-testgen/testvector-generation/{ => deprecated-scripts}/start_OVP.sh (100%) diff --git a/.gitignore b/.gitignore index 47d77f15..e5fdb399 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,6 @@ wally-pipelined/linux-testgen/x* !wally-pipelined/linux-testgen/linux-testvectors/tvCopier.py !wally-pipelined/linux-testgen/linux-testvectors/tvLinker.sh wally-pipelined/linux-testgen/buildroot/ +wally-pipelined/linux-testgen/buildroot-image-output wally-pipelined/regression/slack-notifier/slack-webhook-url.txt diff --git a/wally-pipelined/linux-testgen/WALLY-README.txt b/wally-pipelined/linux-testgen/WALLY-README.txt index 40d7f260..d657733a 100644 --- a/wally-pipelined/linux-testgen/WALLY-README.txt +++ b/wally-pipelined/linux-testgen/WALLY-README.txt @@ -13,7 +13,7 @@ as described below. commit 4047e10ed6e20492bae572d4929eaa5d67eed746 Author: Gwenhael Goavec-Merou Date: Wed Jun 30 06:27:10 2021 +0200 - Then link ./buildroot-config-src/main.config to ./buildroot/.config. + Then hard link ./buildroot-config-src/main.config to ./buildroot/.config. That config file will in turn point to the other config files in ./buildroot-config-src. If you wish to modify the configs, then 1. Copy ./buildroot-config-src/linux.config to ./buildroot/output/build/linux-5.10.7/.config @@ -29,7 +29,10 @@ as described below. "vmlinux" output. * To generate new RAMs and testvectors from a Linux image, - link ./buildroot-image-output to either your new image in ./buildroot/output/image + sym link ./buildroot-image-output to either your new image in ./buildroot/output/image or the existing image at /courses/e190ax/buildroot/output/image on Tera. + (This might require first deleting the empty buildroot-image-output directory). Then run ./testvector-generation/logBuildrootMem.sh to generate RAMs. Then run ./testvector-generation/logAllBuildroot.sh to generate testvectors. + Note that you can only have one instance of QEMU open at a time! Check "ps -ef" to see if + anybody else is running QEMU. diff --git a/wally-pipelined/linux-testgen/buildroot-config-src/main.config b/wally-pipelined/linux-testgen/buildroot-config-src/main.config index 8cb86a49..15442991 100644 --- a/wally-pipelined/linux-testgen/buildroot-config-src/main.config +++ b/wally-pipelined/linux-testgen/buildroot-config-src/main.config @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Buildroot 2021.05-338-g4047e10-dirty Configuration +# Buildroot -g6622c9f-dirty Configuration # BR2_HAVE_DOT_CONFIG=y @@ -82,7 +82,7 @@ BR2_BZCAT="bzcat" BR2_XZCAT="xzcat" BR2_LZCAT="lzip -d -c" BR2_TAR_OPTIONS="" -BR2_DEFCONFIG="/courses/e190ax/buildroot/configs/qemu_riscv64_virt_defconfig" +BR2_DEFCONFIG="./configs/qemu_riscv64_virt_defconfig" BR2_DL_DIR="$(TOPDIR)/dl" BR2_HOST_DIR="$(BASE_DIR)/host" @@ -431,12 +431,12 @@ BR2_LINUX_KERNEL_PATCH="" # BR2_LINUX_KERNEL_USE_DEFCONFIG is not set # BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG is not set BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y -BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="/courses/e190ax/buildroot/output/build/linux-5.10.7/linux.config" +BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="../buildroot-config-src/linux.config" BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="" BR2_LINUX_KERNEL_CUSTOM_LOGO_PATH="" -BR2_LINUX_KERNEL_IMAGE=y +# BR2_LINUX_KERNEL_IMAGE is not set # BR2_LINUX_KERNEL_IMAGEGZ is not set -# BR2_LINUX_KERNEL_VMLINUX is not set +BR2_LINUX_KERNEL_VMLINUX=y # BR2_LINUX_KERNEL_IMAGE_TARGET_CUSTOM is not set BR2_LINUX_KERNEL_GZIP=y # BR2_LINUX_KERNEL_LZ4 is not set @@ -471,7 +471,7 @@ BR2_LINUX_KERNEL_GZIP=y # Target packages # BR2_PACKAGE_BUSYBOX=y -BR2_PACKAGE_BUSYBOX_CONFIG="/courses/e190ax/buildroot/output/build/busybox-1.33.1/busybox.config" +BR2_PACKAGE_BUSYBOX_CONFIG="../buildroot-config-src/busybox.config" BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="" BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y # BR2_PACKAGE_BUSYBOX_INDIVIDUAL_BINARIES is not set @@ -689,6 +689,7 @@ BR2_PACKAGE_LTP_TESTSUITE_ARCH_SUPPORTS=y # # Development tools # +# BR2_PACKAGE_BATS_CORE is not set # BR2_PACKAGE_BINUTILS is not set # BR2_PACKAGE_BITWISE is not set # BR2_PACKAGE_BSDIFF is not set @@ -2449,7 +2450,8 @@ BR2_PACKAGE_NCURSES_ADDITIONAL_TERMINFO="" # # re2 needs a toolchain w/ C++, threads, gcc >= 4.8 # -# BR2_PACKAGE_READLINE is not set +BR2_PACKAGE_READLINE=y +# BR2_PACKAGE_READLINE_BRACKETED_PASTE is not set # BR2_PACKAGE_SLANG is not set # @@ -2973,7 +2975,8 @@ BR2_PACKAGE_MONGREL2_LIBC_SUPPORTS=y # # Shells # -# BR2_PACKAGE_BASH is not set +BR2_PACKAGE_BASH=y +# BR2_PACKAGE_BASH_LOADABLE_EXAMPLES is not set # BR2_PACKAGE_DASH is not set # BR2_PACKAGE_MKSH is not set # BR2_PACKAGE_ZSH is not set @@ -2982,6 +2985,7 @@ BR2_PACKAGE_MONGREL2_LIBC_SUPPORTS=y # Utilities # # BR2_PACKAGE_AT is not set +# BR2_PACKAGE_BASH_COMPLETION is not set # BR2_PACKAGE_CCRYPT is not set # BR2_PACKAGE_DIALOG is not set # BR2_PACKAGE_DTACH is not set @@ -3175,10 +3179,9 @@ BR2_TARGET_OPENSBI_LATEST_VERSION=y # BR2_TARGET_OPENSBI_CUSTOM_GIT is not set BR2_TARGET_OPENSBI_VERSION="0.9" BR2_TARGET_OPENSBI_PLAT="generic" -BR2_TARGET_OPENSBI_INSTALL_DYNAMIC_IMG=y +# BR2_TARGET_OPENSBI_INSTALL_DYNAMIC_IMG is not set BR2_TARGET_OPENSBI_INSTALL_JUMP_IMG=y # BR2_TARGET_OPENSBI_INSTALL_PAYLOAD_IMG is not set -# BR2_TARGET_OPENSBI_LINUX_PAYLOAD is not set # BR2_TARGET_UBOOT is not set # diff --git a/wally-pipelined/linux-testgen/testvector-generation/parseAllBusybear.sh b/wally-pipelined/linux-testgen/testvector-generation/deprecated-scripts/parseAllBusybear.sh similarity index 100% rename from wally-pipelined/linux-testgen/testvector-generation/parseAllBusybear.sh rename to wally-pipelined/linux-testgen/testvector-generation/deprecated-scripts/parseAllBusybear.sh diff --git a/wally-pipelined/linux-testgen/testvector-generation/setup_OVP.sh b/wally-pipelined/linux-testgen/testvector-generation/deprecated-scripts/setup_OVP.sh similarity index 100% rename from wally-pipelined/linux-testgen/testvector-generation/setup_OVP.sh rename to wally-pipelined/linux-testgen/testvector-generation/deprecated-scripts/setup_OVP.sh diff --git a/wally-pipelined/linux-testgen/testvector-generation/start_OVP.sh b/wally-pipelined/linux-testgen/testvector-generation/deprecated-scripts/start_OVP.sh similarity index 100% rename from wally-pipelined/linux-testgen/testvector-generation/start_OVP.sh rename to wally-pipelined/linux-testgen/testvector-generation/deprecated-scripts/start_OVP.sh diff --git a/wally-pipelined/linux-testgen/testvector-generation/fix_mem.py b/wally-pipelined/linux-testgen/testvector-generation/fix_mem.py index ca5fda08..86b25b93 100755 --- a/wally-pipelined/linux-testgen/testvector-generation/fix_mem.py +++ b/wally-pipelined/linux-testgen/testvector-generation/fix_mem.py @@ -1,9 +1,11 @@ #! /usr/bin/python3 -test_dir = '/courses/e190ax/buildroot_boot/' +test_dir = '../' +gdbMemfileDir = '../linux-testvectors/intermediate-outputs' +fixedMemfileDir = '../linux-testvectors' infiles = ['bootmemGDB.txt', 'ramGDB.txt'] outfiles = ['bootmem.txt', 'ram.txt'] for i in range(len(infiles)): - with open(f'{test_dir}{infiles[i]}', 'r') as f: - with open(f'{test_dir}{outfiles[i]}', 'w') as w: + with open(f'{gdbMemfileDir}{infiles[i]}', 'r') as f: + with open(f'{fixedMemfileDir}{outfiles[i]}', 'w') as w: for l in f: w.write(f'{"".join([x[2:] for x in l.split()[:0:-1]])}\n') diff --git a/wally-pipelined/linux-testgen/testvector-generation/gdbinit_mem b/wally-pipelined/linux-testgen/testvector-generation/gdbinit_mem index 1a05f4af..5193a732 100755 --- a/wally-pipelined/linux-testgen/testvector-generation/gdbinit_mem +++ b/wally-pipelined/linux-testgen/testvector-generation/gdbinit_mem @@ -3,18 +3,18 @@ target extended-remote :1234 set logging overwrite on set logging redirect on printf "Creating bootmemGDB.txt\n" -set logging file /courses/e190ax/buildroot_boot/bootmemGDB.txt +set logging file ../linux-testvectors/intermediate-outputs/bootmemGDB.txt set logging on x/4096xb 0x1000 set logging off printf "Creating bootmem_untrimmed_GDB.txt\n" printf "Warning - please verify that the second half of bootmem_untrimmed_GDB.txt is all 0s\n" -set logging file /courses/e190ax/buildroot_boot/bootmem_untrimmed_GDB.txt +set logging file ../linux-testvectors/intermediate-outputs/bootmem_untrimmed_GDB.txt set logging on x/8192xb 0x1000 set logging off printf "Creating ramGDB.txt\n" -set logging file /courses/e190ax/buildroot_boot/ramGDB.txt +set logging file ../linux-testvectors/intermediate-outputs/ramGDB.txt set logging on x/134217728xb 0x80000000 set logging off diff --git a/wally-pipelined/linux-testgen/testvector-generation/logAllBuildroot.sh b/wally-pipelined/linux-testgen/testvector-generation/logAllBuildroot.sh index cb64b8bb..8f9d0c13 100755 --- a/wally-pipelined/linux-testgen/testvector-generation/logAllBuildroot.sh +++ b/wally-pipelined/linux-testgen/testvector-generation/logAllBuildroot.sh @@ -4,37 +4,37 @@ # You can run "tail -f nohup.out" to see what would've # outputted to the terminal if you didn't use nohup +customQemu="/courses/e190ax/qemu_sim/rv64_initrd/qemu_experimental/qemu/build/qemu-system-riscv64" +imageDir="../buildroot-image-output" +intermedDir="../linux-testvectors/intermediate-outputs" +outDir="../linux-testvectors" + # =========== Debug the Process ========== # Uncomment this version for GDB/QEMU debugging # - Opens up GDB interactively # - Logs raw QEMU output to qemu_output.txt -#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2> qemu_output.txt) & riscv64-unknown-elf-gdb +#($customQemu -M virt -nographic -bios $imageDir/fw_jump.elf -kernel $imageDir/Image -append "root=/dev/vda ro" -initrd $imageDir/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2> $intermedDir/qemu_output.txt) & riscv64-unknown-elf-gdb # Uncomment this version to generate qemu_output.txt # - Uses GDB script # - Logs raw QEMU output to qemu_output.txt -#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>qemu_output.txt) & riscv64-unknown-elf-gdb -x gdbinit_qemulog_debug +#($customQemu -M virt -nographic -bios $imageDir/fw_jump.elf -kernel $imageDir/Image -append "root=/dev/vda ro" -initrd $imageDir/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2> $intermedDir/qemu_output.txt) & riscv64-unknown-elf-gdb -x gdbinit_qemulog_debug # Uncomment this version for parse_qemu.py debugging # - Uses qemu_output.txt # - Makes qemu_in_gdb_format.txt -# - Logs parse_qemu.py's simulated gdb output to qemu_in_gdb_format.txt -#cat qemu_output.txt | ./parse_qemu.py >qemu_in_gdb_format.txt -#cat qemu_output.txt | ./parse_qemu.py | ./parse_gdb_output.py "/courses/e190ax/buildroot_boot/" - -# Uncomment this version in case you just want to have qemu_in_gdb_format.txt around -# It is often helpful for general debugging -#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>&1 >/dev/null | ./parse_qemu.py >/courses/e190ax/buildroot_boot/qemu_in_gdb_format.txt) & riscv64-unknown-elf-gdb -x gdbinit_qemulog -# Split qemu_in_gdb_format.txt into chunks of 100,000 instructions for easier inspection -#cd /courses/e190ax/buildroot_boot -#split -d -l 5600000 qemu_in_gdb_format.txt --verbose +# - Splits qemu_in_gdb_format.txt into chunks of 100,000 instrs +cat $intermedDir/qemu_output.txt | ./parse_qemu.py >$intermedDir/qemu_in_gdb_format.txt +cd $intermedDir +split -d -l 5600000 ./qemu_in_gdb_format.txt --verbose +cd ../../testvector-generation # Uncomment this version for parse_gdb_output.py debugging # - Uses qemu_in_gdb_format.txt -# - Logs info needed by buildroot testbench -#cat qemu_in_gdb_format.txt | ./parse_gdb_output.py "/courses/e190ax/buildroot_boot/" +# - Makes testvectors +#cat qemu_in_gdb_format.txt | ./parse_gdb_output.py "$outdir" # =========== Just Do the Thing ========== # Uncomment this version for the whole thing # - Logs info needed by buildroot testbench -#(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>&1 >/dev/null | ./parse_qemu.py | ./parse_gdb_output.py "/courses/e190ax/buildroot_boot_new/") & riscv64-unknown-elf-gdb -x gdbinit_qemulog +#($customQemu -M virt -nographic -bios $imageDir/fw_jump.elf -kernel $imageDir/Image -append "root=/dev/vda ro" -initrd $imageDir/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>&1 >/dev/null | ./parse_qemu.py | ./parse_gdb_output.py "$outdir") & riscv64-unknown-elf-gdb -x gdbinit_qemulog diff --git a/wally-pipelined/linux-testgen/testvector-generation/logBuildrootMem.sh b/wally-pipelined/linux-testgen/testvector-generation/logBuildrootMem.sh index d41d85d0..400dd885 100755 --- a/wally-pipelined/linux-testgen/testvector-generation/logBuildrootMem.sh +++ b/wally-pipelined/linux-testgen/testvector-generation/logBuildrootMem.sh @@ -1,4 +1,7 @@ -(qemu-system-riscv64 -M virt -nographic -bios /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/fw_jump.elf -kernel /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/Image -append "root=/dev/vda ro" -initrd /courses/e190ax/qemu_sim/rv64_initrd/buildroot_experimental/output/images/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>/dev/null >/dev/null ) & +customQemu="/courses/e190ax/qemu_sim/rv64_initrd/qemu_experimental/qemu/build/qemu-system-riscv64" +imageDir="../buildroot-image-output" +($customQemu -M virt -nographic -bios $imageDir/fw_jump.elf -kernel $imageDir/Image -append "root=/dev/vda ro" -initrd $imageDir/rootfs.cpio -d nochain,cpu,in_asm -serial /dev/null -singlestep -s -S 2>/dev/null >/dev/null) & riscv64-unknown-elf-gdb -x gdbinit_mem -#sed -i '$d' $file +echo "Translating Mem from GDB to Questa format" +./fix_mem.py echo "Done" diff --git a/wally-pipelined/linux-testgen/testvector-generation/parse_qemu.py b/wally-pipelined/linux-testgen/testvector-generation/parse_qemu.py index 4da168ab..2e9f477a 100755 --- a/wally-pipelined/linux-testgen/testvector-generation/parse_qemu.py +++ b/wally-pipelined/linux-testgen/testvector-generation/parse_qemu.py @@ -3,13 +3,16 @@ import fileinput, sys sys.stderr.write("reminder: this script takes input from stdin\n") parseState = "idle" +beginPageFault = 0 inPageFault = 0 +endPageFault = 0 CSRs = {} pageFaultCSRs = {} regs = {} pageFaultRegs = {} instrs = {} instrCount = 0 +returnAdr = 0 def printPC(l): global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs, instrCount @@ -33,8 +36,8 @@ def printCSRs(): def parseCSRs(l): global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs - if l.strip() and (not l.startswith("Disassembler")) and (not l.startswith("Please")): - if l.startswith(' x0/zero'): + if l.strip() and (not l.startswith("Disassembler")) and (not l.startswith("Please")) and not inPageFault: + if l.startswith(' x0/zero'): parseState = "regFile" instr = instrs[CSRs["pc"]] printPC(instr) @@ -42,24 +45,31 @@ def parseCSRs(l): else: csr = l.split()[0] val = int(l.split()[1],16) - if inPageFault: + # Commented out this conditional because the pageFault instrs don't corrupt CSRs + #if inPageFault: # Not sure if these CSRs should be updated or not during page fault. - if l.startswith("mstatus") or l.startswith("mepc") or l.startswith("mcause") or l.startswith("mtval") or l.startswith("sepc") or l.startswith("scause") or l.startswith("stval"): + #if l.startswith("mstatus") or l.startswith("mepc") or l.startswith("mcause") or l.startswith("mtval") or l.startswith("sepc") or l.startswith("scause") or l.startswith("stval"): # We do update some CSRs - CSRs[csr] = val - else: + # CSRs[csr] = val + #else: # Others we preserve until changed later - pageFaultCSRs[csr] = val - elif pageFaultCSRs and (csr in pageFaultCSRs): - if (val != pageFaultCSRs[csr]): - del pageFaultCSRs[csr] - CSRs[csr] = val + # pageFaultCSRs[csr] = val + #elif pageFaultCSRs and (csr in pageFaultCSRs): + # if (val != pageFaultCSRs[csr]): + # del pageFaultCSRs[csr] + # CSRs[csr] = val + #else: + # CSRs[csr] = val + # + # However SEPC and STVAL do get corrupted upon exiting + if endPageFault and ((csr == 'sepc') or (csr == 'stval')): + CSRs[csr] = returnAdr else: CSRs[csr] = val def parseRegs(l): global parseState, inPageFault, CSRs, pageFaultCSRs, regs, pageFaultCSRs, instrs - if "mcounteren" in l: + if "pc" in l: printCSRs() # New non-disassembled instruction parseState = "CSRs" @@ -100,8 +110,12 @@ for l in fileinput.input(): elif (parseState == "instr") and l.startswith('0x'): if "out of bounds" in l: sys.stderr.write("Detected QEMU page fault error\n") + beginPageFault = ~(inPageFault) + if beginPageFault: + returnAdr = int(l.split()[0][2:-1], 16) inPageFault = 1 else: + endPageFault = inPageFault inPageFault = 0 adr = int(l.split()[0][2:-1], 16) instrs[adr] = l