Merge pull request #1244 from jordancarlin/linux-updates

Overhaul Linux build Makefile and add qemuBoot script

bin/regression-wally

@@ -57,7 +57,7 @@ standard_tests = [
 
 # Separate test for short buildroot run through OpenSBI UART output
 tests_buildrootshort = [
-                    ["buildroot", ["buildroot"], "--args +INSTR_LIMIT=1400000", # Instruction limit gets to first OpenSBI UART output
+                    ["buildroot", ["buildroot"], "--args +INSTR_LIMIT=1600000", # Instruction limit gets to first OpenSBI UART output
                         "OpenSBI v", "buildroot_uart.out"]
     ]
 

linux/Makefile

@@ -1,34 +1,43 @@
 BUILDROOT         := buildroot
-IMAGES := ${BUILDROOT}/output/images
+IMAGE_DIR         := ${BUILDROOT}/output/images
+DISASSEMBLY_DIR   := ${IMAGE_DIR}/disassembly
 WALLYLINUX        := $(WALLY)/linux
-DIS := ${IMAGES}/disassembly
 BR2_EXTERNAL_TREE := $(WALLYLINUX)/br2-external-tree
+LINUX_TESTVECTORS := $(RISCV)/linux-testvectors
 
-# set sudo if needed depending on $RISCV
+BUILDROOT_OUTPUTS := Image fw_jump.bin fw_jump.elf rootfs.cpio vmlinux busybox
-SUDO := $(shell mkdir -p $(RISCV)/.test > /dev/null 2>&1 || echo sudo)
+BUILDROOT_OUTPUTS := $(foreach name, $(BUILDROOT_OUTPUTS), $(IMAGE_DIR)/$(name))
 
 # Device tree files
-DTS ?= $(shell find devicetree -type f -regex ".*\.dts" | sort)
+DTS ?= $(wildcard devicetree/*.dts)
-DTB := $(DTS:%.dts=%.dtb)
+DTB := $(foreach name, $(DTS:%.dts=%.dtb), $(IMAGE_DIR)/$(notdir $(name)))
-DTB := $(foreach name, $(DTB), $(IMAGES)/$(shell basename $(name)))
 
-# Disassembly stuff
+# Disassembly files
 BINARIES := fw_jump.elf vmlinux busybox
-OBJDUMPS := $(foreach name, $(BINARIES), $(basename $(name) .elf))
+OBJDUMPS := $(foreach name, $(basename $(BINARIES) .elf), $(DISASSEMBLY_DIR)/$(name).objdump)
-OBJDUMPS := $(foreach name, $(OBJDUMPS), $(DIS)/$(name).objdump)
 
-.PHONY: all generate disassemble install clean cleanDTB check_write_permissions check_environment
+# Testvector files
+RAW_RAM_FILE     := ${LINUX_TESTVECTORS}/ramGDB.bin
+RAM_FILE         := ${LINUX_TESTVECTORS}/ram.bin
+RAW_BOOTMEM_FILE := ${LINUX_TESTVECTORS}/bootmemGDB.bin
+BOOTMEM_FILE     := ${LINUX_TESTVECTORS}/bootmem.bin
 
-all: check_environment check_write_permissions clean download Image disassemble install dumptvs
+.PHONY: all check_environment check_write_permissions config build disassemble devicetrees install dumptvs clean cleanDTB
 
-check_environment: $(RISCV)
+# Default target
+all: check_write_permissions clean config build disassemble install dumptvs
+
+# Check if the environment variables are set correctly
+check_environment: $(RISCV) $(WALLY)
 ifeq ($(findstring :$(RISCV)/lib:,:$(LD_LIBRARY_PATH):),)
 	@(echo "ERROR: Your environment variables are not set correctly." >&2 \
 	&& echo "Make sure to source setup.sh or install buildroot using the wally-tool-chain-install.sh script." >&2 \
 	&& exit 1)
 endif
 
-check_write_permissions:
+# Check if the user has write permissions to the RISCV directory, potentially using sudo
+SUDO := $(shell mkdir -p $(RISCV)/.test > /dev/null 2>&1 || echo sudo)
+check_write_permissions: check_environment
 ifeq ($(SUDO), sudo)
 	@echo "Cannot write to '$(RISCV)'." \
 		"Using sudo (you may be prompted for your password several times throughout the install)"
@@ -38,75 +47,92 @@ endif
 		&& exit 1)
 	@$(SUDO) rm -r $(RISCV)/.test
 
-Image: check_environment
+# Build buildroot and device tree binaries
-	bash -c "unset LD_LIBRARY_PATH; $(MAKE) -C $(BUILDROOT)"
+build: $(BUILDROOT_OUTPUTS) devicetrees
-	$(MAKE) generate
-	@echo "Buildroot Image successfully generated."
 
-install: check_write_permissions check_environment
+# Build buildroot itself
+# LD_LIBRARY_PATH must be unset to avoid conflicts between the host and cross compiler
+$(BUILDROOT_OUTPUTS) $(IMAGE_DIR): check_environment $(BUILDROOT)
+	bash -c "unset LD_LIBRARY_PATH; $(MAKE) -C $(BUILDROOT)"
+
+# Install buildroot to $RISCV
+install: check_write_permissions
 	$(SUDO) rm -rf $(RISCV)/$(BUILDROOT)
 	$(SUDO) mv $(BUILDROOT) $(RISCV)/$(BUILDROOT)
-	@echo "Buildroot successfully installed."
 
-dumptvs: check_write_permissions check_environment
+# Generate linux boot testvectors
-	$(SUDO) mkdir -p $(RISCV)/linux-testvectors
+dumptvs: ${RAM_FILE} ${BOOTMEM_FILE}
-	./genInitMem.sh
-	@echo "Testvectors successfully generated."
 
-generate: $(DTB) $(IMAGES)
+# Format QEMU memory dumps for use as testvectors
+${LINUX_TESTVECTORS}/%.bin: ${LINUX_TESTVECTORS}/%GDB.bin
+	truncate -s %8 $^ # Extend file to 8 byte multiple
+	objcopy --reverse-bytes=8 -F binary $^ $@ # Reverse bytes
 
-$(IMAGES)/%.dtb: ./devicetree/%.dts
+# Generate memory dumps from QEMU buildroot boot
+TCP_PORT := 1235
+${LINUX_TESTVECTORS}/%GDB.bin: | $(LINUX_TESTVECTORS)
+	${WALLYLINUX}/qemuBoot.sh --gdb ${TCP_PORT} &
+	riscv64-unknown-elf-gdb -batch \
+		-ex "target remote :${TCP_PORT}" \
+		-ex "maintenance packet Qqemu.PhyMemMode:1" \
+		-ex "printf \"Creating ${RAW_BOOTMEM_FILE}\n\"" \
+		-ex "dump binary memory ${RAW_BOOTMEM_FILE} 0x1000 0x1fff" \
+		-ex "printf \"Creating ${RAW_RAM_FILE}\n\"" \
+		-ex "dump binary memory ${RAW_RAM_FILE} 0x80000000 0x8fffffff" \
+		-ex "kill"
+
+# Generate device tree binaries
+devicetrees: $(DTB)
+$(IMAGE_DIR)/%.dtb: ${WALLYLINUX}/devicetree/%.dts | $(IMAGE_DIR)
 	dtc -I dts -O dtb $< > $@
 
-$(IMAGES):
+# Create disassembly files
-	@ echo "No output/images directory in buildroot."
+disassemble: check_environment $(OBJDUMPS) $(DISASSEMBLY_DIR)/rootfs
-	@ echo "Run make --jobs in buildroot directory before generating device tree binaries."; exit 1
 
-$(RISCV):
+# Extract rootfs
-	@ echo "ERROR: No $(RISCV) directory. Make sure you have installed the Wally Toolchain."
+$(DISASSEMBLY_DIR)/rootfs: $(IMAGE_DIR)/rootfs.cpio
-	@ echo "and sourced setup.sh"
-
-# Disassembly rules ---------------------------------------------------
-disassemble: check_environment
-	rm -rf $(BUILDROOT)/output/images/disassembly
-	find  $(BUILDROOT)/output/build/linux-* -maxdepth 1 -name "vmlinux" | xargs  cp -t $(BUILDROOT)/output/images/
-	mkdir -p $(DIS)
-	$(MAKE) $(OBJDUMPS)
-	# extract rootfs
-	mkdir -p $(BUILDROOT)/output/images/disassembly/rootfs
 	@echo "Ignore error about dev/console when extracting rootfs from rootfs.cpio"
-	-cpio -i -D $(BUILDROOT)/output/images/disassembly/rootfs < $(BUILDROOT)/output/images/rootfs.cpio
+	-cpio -id -D $(DISASSEMBLY_DIR)/rootfs -F $(IMAGE_DIR)/rootfs.cpio
-	@echo "Disassembly successfully completed."
 
-$(DIS)/%.objdump: $(IMAGES)/%.elf
+# Disassemble binaries
-	riscv64-unknown-elf-objdump -DS $< >> $@
+$(DISASSEMBLY_DIR)/%.objdump: $(IMAGE_DIR)/% | $(DISASSEMBLY_DIR)
-	$(WALLY)/bin/extractFunctionRadix.sh $@
-
-$(DIS)/%.objdump: $(IMAGES)/%
 	riscv64-unknown-elf-objdump -S $< >> $@
 	$(WALLY)/bin/extractFunctionRadix.sh $@
 
-$(IMAGES)/vmlinux:
+# Disassemble binaries ending in .elf
-	linuxDir=$$(find $(BUILDROOT)/output/build -maxdepth 2 -type d -regex ".*/linux-[0-9]+\.[0-9]+\.[0-9]+$$") ;\
+$(DISASSEMBLY_DIR)/%.objdump: $(IMAGE_DIR)/%.elf | $(DISASSEMBLY_DIR)
-	cp $$linuxDir/vmlinux $@ ;\
+	riscv64-unknown-elf-objdump -SD $< >> $@
+	$(WALLY)/bin/extractFunctionRadix.sh $@
 
-$(IMAGES)/busybox:
+# Load wally buildroot configuration
-	busyboxDir=$$(find $(BUILDROOT)/output/build -maxdepth 2 -type d -regex ".*/busybox-[0-9]+\.[0-9]+\.[0-9]+$$") ;\
+config: $(BUILDROOT) $(BR2_EXTERNAL_TREE)/configs/wally_defconfig
-	cp $$busyboxDir/busybox $@ ;\
-
-# Generating new Buildroot directories --------------------------------
-download: $(BUILDROOT)
 	$(MAKE) -C $(BUILDROOT) wally_defconfig BR2_EXTERNAL=$(BR2_EXTERNAL_TREE)
-	@echo "Buildroot successfully download."
 
+# Clone buildroot and checkout the correct version
 $(BUILDROOT):
 	git clone https://github.com/buildroot/buildroot.git $@
 	cd $@; git checkout 2024.11.x
 
-# ---------------------------------------------------------------------
+# Create directories
+$(LINUX_TESTVECTORS): check_write_permissions
+	$(SUDO) mkdir -p $@
 
+$(DISASSEMBLY_DIR):
+	mkdir -p $@
+
+# Remove device tree binaries
 cleanDTB:
-	rm -f $(IMAGES)/*.dtb
+	rm -f $(IMAGE_DIR)/*.dtb
 
+# Remove buildroot directory
 clean:
 	rm -rf $(BUILDROOT)
+
+# Check if the RISCV environment variable is set
+$(RISCV):
+	@ echo "ERROR: No $(RISCV) directory. Make sure you have installed the Wally Toolchain."
+	@ echo "and sourced setup.sh"
+
+# Check if the WALLY environment variable is set
+$(WALLY):
+	@ echo "ERROR: $$WALLY is not set. Make sure you have sourced setup.sh"

linux/README.MD

@@ -35,7 +35,7 @@ The device tree files for the various FPGAs Wally supports, as well as QEMU's de
 They are built automatically using the main `make` command. To build the device tree binaries (.dtb) from the device tree sources (.dts) separately, we can build all of them at once using:
 
 ```bash
-$ make generate # optionally override BUILDROOT
+$ make devicetrees # optionally override BUILDROOT
 ```
 
 The .dts files will end up in the `<BUILDROOT>/output/images` folder of your chosen buildroot directory.

linux/br2-external-tree/board/wally/post_image.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Copy linux and busybox binaries (with symbol info) to images directory
+cp "$BUILD_DIR"/linux-*/vmlinux "$BINARIES_DIR"/vmlinux
+cp "$BUILD_DIR"/busybox-*/busybox "$BINARIES_DIR"/busybox

linux/br2-external-tree/configs/wally_defconfig

@@ -10,6 +10,7 @@ BR2_GNU_MIRROR="http://ftpmirror.gnu.org"
 BR2_ENABLE_DEBUG=y
 BR2_DEBUG_3=y
 # BR2_STRIP_strip is not set
+BR2_PER_PACKAGE_DIRECTORIES=y
 # BR2_PIC_PIE is not set
 BR2_SSP_NONE=y
 BR2_RELRO_NONE=y
@@ -19,6 +20,7 @@ BR2_TARGET_GENERIC_ISSUE="Greetings! This RISC-V Linux image was built for Wally
 BR2_ROOTFS_DEVICE_TABLE_SUPPORTS_EXTENDED_ATTRIBUTES=y
 BR2_SYSTEM_DHCP="eth0"
 BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_WALLY_PATH)/board/wally/rootfs_overlay"
+BR2_ROOTFS_POST_IMAGE_SCRIPT="$(BR2_EXTERNAL_WALLY_PATH)/board/wally/post_image.sh"
 BR2_LINUX_KERNEL=y
 BR2_LINUX_KERNEL_CUSTOM_VERSION=y
 BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE="6.12.8"

linux/devicetree/wally-artya7.dts

@@ -31,7 +31,7 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa-base = "rv64i";
-			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svade", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
 			riscv,cboz-block-size = <64>;
 			riscv,cbom-block-size = <64>;
 			mmu-type = "riscv,sv48";

linux/devicetree/wally-vcu108.dts

@@ -31,7 +31,7 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa-base = "rv64i";
-			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svade", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
 			riscv,cboz-block-size = <64>;
 			riscv,cbom-block-size = <64>;
 			mmu-type = "riscv,sv48";

linux/devicetree/wally-vcu118.dts

@@ -31,7 +31,7 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa-base = "rv64i";
-			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svade", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
 			riscv,cboz-block-size = <64>;
 			riscv,cbom-block-size = <64>;
 			mmu-type = "riscv,sv48";

linux/devicetree/wally-virt.dts

@@ -31,7 +31,7 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa-base = "rv64i";
-			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "sstc", "svade", "svadu", "svinval", "svnapot", "svpbmt", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zfa", "zfh", "zkn", "zkt", "zicbom", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihpm";
 			riscv,cboz-block-size = <64>;
 			riscv,cbom-block-size = <64>;
 			mmu-type = "riscv,sv48";

linux/genInitMem.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-set -e
-tcpPort=1235
-imageDir=$RISCV/buildroot/output/images
-tvDir=$RISCV/linux-testvectors
-rawRamFile="$tvDir/ramGDB.bin"
-ramFile="$tvDir/ram.bin"
-rawBootmemFile="$tvDir/bootmemGDB.bin"
-bootmemFile="$tvDir/bootmem.bin"
-rawUntrimmedBootmemFile="$tvDir/untrimmedBootmemFileGDB.bin"
-DEVICE_TREE=${imageDir}/wally-virt.dtb
-
-if ! mkdir -p "$tvDir"; then
-    echo "Error: unable to create linux testvector directory $tvDir!">&2
-    echo "Please try running as sudo.">&2
-    exit 1
-fi
-if ! test -w "$tvDir"; then
-    echo "Using sudo to gain access to $tvDir"
-    if ! sudo chmod -R a+rw "$tvDir"; then
-        echo "Error: insuffcient write privileges for linux testvector directory $tvDir !">&2
-        echo "Please chmod it. For example:">&2
-        echo "    sudo chmod -R a+rw $tvDir">&2
-        exit 1
-    fi
-fi
-
-echo "Launching QEMU in replay mode!"
-(qemu-system-riscv64 \
--M virt -m 256M -dtb "$DEVICE_TREE" \
--nographic \
--bios "$imageDir"/fw_jump.bin -kernel "$imageDir"/Image -append "root=/dev/vda ro" -initrd "$imageDir"/rootfs.cpio \
--gdb tcp::$tcpPort -S) \
-& riscv64-unknown-elf-gdb --quiet \
--ex "set pagination off" \
--ex "set logging overwrite on" \
--ex "set logging redirect on" \
--ex "set confirm off" \
--ex "target extended-remote :$tcpPort" \
--ex "maintenance packet Qqemu.PhyMemMode:1" \
--ex "printf \"Creating $rawBootmemFile\n\"" \
--ex "dump binary memory $rawBootmemFile 0x1000 0x1fff" \
--ex "printf \"Creating $rawRamFile\n\"" \
--ex "dump binary memory $rawRamFile 0x80000000 0x8fffffff" \
--ex "kill" \
--ex "q"
-
-echo "Changing Endianness"
-# Extend files to 8 byte multiple
-truncate -s %8 "$rawRamFile"
-truncate -s %8 "$rawBootmemFile"
-# Reverse bytes
-objcopy --reverse-bytes=8 -F binary "$rawRamFile" "$ramFile"
-objcopy --reverse-bytes=8 -F binary "$rawBootmemFile" "$bootmemFile"
-rm -f "$rawRamFile" "$rawBootmemFile" "$rawUntrimmedBootmemFile"
-
-echo "genInitMem.sh completed!"
-echo "You may want to restrict write access to $tvDir now and give cad ownership of it."
-echo "Run the following:"
-echo "    sudo chown -R cad:cad $tvDir"
-echo "    sudo chmod -R go-w $tvDir"

linux/qemuBoot.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+###########################################
+## Boot linux on QEMU configured to match Wally
+##
+## Written: Jordan Carlin, jcarlin@hmc.edu
+## Created: 20 January 2025
+## Modified:
+##
+## A component of the CORE-V-WALLY configurable RISC-V project.
+## https://github.com/openhwgroup/cvw
+##
+## Copyright (C) 2021-25 Harvey Mudd College & Oklahoma State University
+##
+## SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1
+##
+## Licensed under the Solderpad Hardware License v 2.1 (the “License”); you may not use this file
+## except in compliance with the License, or, at your option, the Apache License version 2.0. You
+## may obtain a copy of the License at
+##
+## https:##solderpad.org/licenses/SHL-2.1/
+##
+## Unless required by applicable law or agreed to in writing, any work distributed under the
+## License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+## either express or implied. See the License for the specific language governing permissions
+## and limitations under the License.
+################################################################################################
+
+BUILDROOT="${BUILDROOT:-$RISCV/buildroot}"
+IMAGES="$BUILDROOT"/output/images
+
+if [[ "$1" == "--gdb" && -n "$2" ]]; then
+    GDB_FLAG="-gdb tcp::$2 -S"
+fi
+
+qemu-system-riscv64 \
+	-M virt -m 256M -nographic \
+	-bios "$IMAGES"/fw_jump.bin \
+	-kernel "$IMAGES"/Image \
+	-initrd "$IMAGES"/rootfs.cpio \
+	-dtb "$IMAGES"/wally-virt.dtb \
+	-cpu rva22s64,zicond=true,zfa=true,zfh=true,zcb=true,zbc=true,zkn=true,sstc=true,svadu=true,svnapot=true \
+	$GDB_FLAG