Merge pull request #718 from Karl-Han/docker

Putting Toolchains and Regression inside Docker with Scripts
2025-02-02 17:55:19 +00:00 · 2024-04-16 09:10:21 -05:00 · 2024-04-16 09:10:21 -05:00 · f612581c1d
commit f612581c1d
parent 6d464157f7 a85f55d3c7
10 changed files with 650 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -189,6 +189,8 @@ sim/branch/*
 sim/obj_dir
 examples/verilog/fulladder/obj_dir
 config/deriv
+docs/docker/buildroot-config-src
+docs/docker/testvector-generation
 sim/questa/cov
 sim/questa/covhtmlreport/
 sim/questa/logs
--- a/docs/docker/Dockerfile.builds
+++ b/docs/docker/Dockerfile.builds
@ -0,0 +1,104 @@
+FROM wallysoc/ubuntu_wally
+
+# SET ENVIRONMENT VARIABLES
+# assume 4 threads are available to speed up
+ARG NUM_THREADS=4
+ENV RISCV=/opt/riscv
+ENV PATH="$PATH:${RISCV}/bin"
+ENV MANPATH="$MANPATH:${RISCV}/share/man"
+
+# this is required by podman
+USER root
+WORKDIR /opt/riscv
+
+# TOOLCHAIN
+RUN git clone https://github.com/riscv/riscv-gnu-toolchain && \
+    cd riscv-gnu-toolchain && \
+    sed -i 's/https/git/' .gitmodules && git submodule sync && \
+    ./configure --prefix=${RISCV} --enable-multilib \
+        --with-multilib-generator="rv32e-ilp32e--;rv32i-ilp32--;rv32im-ilp32--;rv32iac-ilp32--;rv32imac-ilp32--;rv32imafc-ilp32f--;rv32imafdc-ilp32d--;rv64i-lp64--;rv64ic-lp64--;rv64iac-lp64--;rv64imac-lp64--;rv64imafdc-lp64d--;rv64im-lp64--;" && \
+    make --jobs ${NUM_THREADS} && \
+    make install && cd ${RISCV} && \
+    rm -rf ${RISCV}/riscv-gnu-toolchain
+
+# elf2hex
+RUN git clone https://github.com/sifive/elf2hex.git && \
+    cd elf2hex && \
+    autoreconf -i && \
+    ./configure --target=riscv64-unknown-elf --prefix=${RISCV} && \
+    make && \
+    make install && cd ${RISCV} && \
+    rm -rf ${RISCV}/elf2hex
+
+# QEMU
+RUN git clone --recursive https://github.com/qemu/qemu && \
+    cd qemu && \
+    ./configure --target-list=riscv64-softmmu --prefix=${RISCV}  && \
+    make --jobs ${NUM_THREADS} && \
+    make install && cd ${RISCV} && \
+    rm -rf ${RISCV}/qemu
+
+# Spike
+RUN git clone https://github.com/riscv-software-src/riscv-isa-sim && \
+    mkdir riscv-isa-sim/build && \
+    cd riscv-isa-sim/build && \
+    ../configure --prefix=$RISCV --enable-commitlog && \
+    make --jobs ${NUM_THREADS} && \
+    make install && cd ${RISCV} && \
+    rm -rf ${RISCV}/riscv-isa-sim
+
+# SAIL
+RUN opam init -y --disable-sandboxing && \
+    opam switch create 5.1.0 && \
+    opam install sail -y && \
+    eval $(opam config env) && \
+    git clone https://github.com/riscv/sail-riscv.git && \
+    cd sail-riscv && \
+    ARCH=RV32 make c_emulator/riscv_sim_RV32 && \
+    ARCH=RV64 make c_emulator/riscv_sim_RV64 && \
+    cp ${RISCV}/sail-riscv/c_emulator/riscv_sim_RV64 ${RISCV}/bin/riscv_sim_RV64 && \
+    cp ${RISCV}/sail-riscv/c_emulator/riscv_sim_RV32 ${RISCV}/bin/riscv_sim_RV32 && \
+    rm -rf ${RISCV}/sail-riscv
+
+COPY ./buildroot-config-src /opt/riscv/buildroot-config-src
+COPY ./testvector-generation /opt/riscv/testvector-generation
+
+# # Buildroot
+RUN git clone https://github.com/buildroot/buildroot.git && \
+    cd buildroot && \
+    git checkout 2021.05 && \
+    cp -r /opt/riscv/buildroot-config-src ./board/wally && \
+    cp ./board/wally/main.config .config && \
+    make --jobs ${NUM_THREADS} && \
+    # generate files for buildroot regression
+    mkdir -p ${RISCV}/linux-testvectors && \
+    cd /opt/riscv/testvector-generation && \
+    curl https://raw.githubusercontent.com/openhwgroup/cvw/main/linux/devicetree/wally-virt.dts --output ${RISCV}/buildroot/output/images/wally-virt.dts && \
+    dtc -I dts -O dtb ${RISCV}/buildroot/output/images/wally-virt.dts > ${RISCV}/buildroot/output/images/wally-virt.dtb && \
+    make && ./genInitMem.sh && \
+    chmod -R a+rw ${RISCV}/linux-testvectors && \
+    rm -rf ${RISCV}/buildroot
+
+RUN pip3 install --no-cache-dir \
+    testresources riscv_config \
+    git+https://github.com/riscv/riscof.git
+
+# Wally needs Verilator 5.021 or later.
+# Verilator needs to be built from scratch to get the latest version
+# apt-get install verilator installs version 4.028 as of 6/8/23
+RUN apt-get install -y \
+    perl g++ ccache help2man libgoogle-perftools-dev numactl perl-doc zlib1g \
+    libfl2  libfl-dev && \
+    apt-get clean
+RUN git clone https://github.com/verilator/verilator && \
+    cd verilator && \
+    git pull && \
+    git checkout v5.022 && \
+    autoconf && \
+    ./configure --prefix=${RISCV} && \
+    make -j ${NUM_THREADS} && \
+    make install && \
+    cd ${RISCV} && rm -rf ${RISCV}/verilator
+
+USER ${USERNAME}
+WORKDIR /home/${USERNAME}/cvw
--- a/docs/docker/Dockerfile.regression
+++ b/docs/docker/Dockerfile.regression
@ -0,0 +1,16 @@
+FROM wallysoc/toolchains_wally
+
+ENV QUESTA=/cad/mentor/questa_sim-xxxx.x_x
+ENV RUN_QUESTA=false
+ENV USERNAME=cad
+
+VOLUME [ "/home/${USERNAME}/cvw" ]
+USER root
+
+COPY --chown=${USERNAME}:${USERNAME} . /home/${USERNAME}
+RUN chown -R ${USERNAME}:${USERNAME} /home/${USERNAME}
+
+USER ${USERNAME}
+WORKDIR /home/${USERNAME}/cvw
+
+CMD [ "/bin/sh", "-c", "/home/${USERNAME}/run_regression.sh" ]
--- a/docs/docker/Dockerfile.ubuntu
+++ b/docs/docker/Dockerfile.ubuntu
@ -0,0 +1,33 @@
+FROM ubuntu:22.04@sha256:aa772c98400ef833586d1d517d3e8de670f7e712bf581ce6053165081773259d
+
+RUN apt update && \
+    apt install -y \
+        git gawk make texinfo bison flex build-essential python3 libz-dev libexpat-dev autoconf device-tree-compiler ninja-build libpixman-1-dev build-essential ncurses-base ncurses-bin libncurses5-dev dialog curl wget ftp libgmp-dev python3-pip pkg-config libglib2.0-dev opam z3 zlib1g-dev automake autotools-dev libmpc-dev libmpfr-dev gperf libtool patchutils verilator cpio bc vim emacs gedit nano && \
+    apt-get clean
+
+# COPY requirements.txt /root
+RUN pip3 install --no-cache-dir \
+    sphinx sphinx_rtd_theme matplotlib scipy scikit-learn adjustText lief
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Create a user group 'xyzgroup'
+ENV USERNAME=cad
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Create the user
+RUN groupadd --gid $USER_GID ${USERNAME} \
+    && useradd --uid $USER_UID --gid $USER_GID -m ${USERNAME} \
+    # [Optional] Add sudo support. Omit if you don't need to install software after connecting.
+    && apt-get update \
+    && apt-get install -y sudo \
+    && echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} \
+    && chmod 0440 /etc/sudoers.d/${USERNAME}
+
+# Change RISCV user
+RUN chown -Rf cad:cad /opt
+
+# Add cad user
+USER ${USERNAME}
+WORKDIR /home/${USERNAME}
--- a/docs/docker/README.md
+++ b/docs/docker/README.md
@ -0,0 +1,334 @@
+# Consistant Build of Toolchain for Wally
+
+`Dockerfile.*` contains a ~~multi-stage~~ build for all the toolchains that are required for Wally's open-source features.
+
+Hazards:
+
+- If there is any change in `${CVW_HOME}/linux/buildroot-config-src` folder with main.config, you have to copy it to the current folder to `buildroot-config-src`
+- If there is any change in `${CVW_HOME}/linux/testvector-generation` folder with main.config, you have to copy it to the current folder to `testvector-generation`
+
+If you have any other questions, please read the [troubleshooting]() first.
+
+## TODOs
+
+- [ ] Pinning the tools version
+    - As for the consistent tool build, should we use specific versions of tools in order to avoid bugs at the master branch?
+    - And we will upgrade the versions of tool after a certain period of time to get the latest features and patches while verifying it won’t cause any problem.
+- [x] Mount the ~~EDA Tools~~QuestaSIM and Path
+    - USE_QUESTA and QUESTA for path
+- [x] Enable X11 forwarding for docker
+    - `--network=host` for docker run
+    - `xhost +localhost:${USER}` for host
+- [x] Regression Script
+- [x] Configure the license for Questa
+- [ ] Change the condition from empty string to 1
+- [x] Add linux testvector-generation
+    - [x] Estimate the useless building intermediate files
+
+## TL;DR
+
+Steps:
+
+1. Install either Docker Engine or Podman for container support
+2. Run start-up script `docs/docker/start.sh` to start a stateless container to run the toolchains and EDA tool
+
+### Docker Engine or Podman
+
+First and foremost, install either Docker Engine or Podman:
+
+- Docker Engine (More Popular, default): https://docs.docker.com/engine/install/
+- Podman: https://podman.io/docs/installation
+
+Here are some common installation commands (not guarantee to be up to date)
+
+```shell
+# For Ubuntu
+# Add Docker's official GPG key:
+sudo apt-get update
+sudo apt-get install ca-certificates curl
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update
+# Installation
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+# Hello-World Example
+docker run hello-world
+```
+
+### Use of Start-up Script
+
+Files at this folder can help you to build/fetch environment you need to run wally with the help of Docker.
+
+Here are some common use cases, it will **provides you an environment with RISC-V toolchains** that required by this project:
+
+```shell
+# By default, we assume that you have cloned the cvw respository and running the script at relative path `docs/docker`
+
+# For HMC students, /opt/riscv is available and nothing needs to be built
+TOOLCHAINS_MOUNT=/opt/riscv QUESTA=/cad/mentor/questa_sim-2023.4 ./start.sh
+
+# For those with all the toolchains installed, simply mount the toolchains
+TOOLCHAINS_MOUNT=<path-to-toolchains> ./start.sh
+
+# For those have nothing, fetching the builds are easiest thing
+CVW_MOUNT=<path-to-cvw> ./start.sh
+# if you want to use Podman instead of Docker Engine
+USE_PODMAN=1 ./start.sh
+
+# For other cases, checkout start-up script for building option
+```
+For further usage, please consult the following configuration.
+
+### Regression on Master Branch
+
+The regression script is used by image `wallysoc/regression_wally` to run regressions on the master branch of a specific repository.
+
+If you are using docker, then replace the following podman with docker.
+
+```shell
+# create volume for permanent storage
+podman volume create cvw_temp
+
+# run regression on the OpenHW/cvw
+podman run \
+    -e CLEAN_CVW= -e BUILD_RISCOF= -e RUN_QUESTA= \
+    -v cvw_temp:/home/cad/cvw \
+    -v /cad/mentor/questa_sim-2023.4:/cad/mentor/questa_sim-xxxx.x_x \
+    --privileged --network=host \
+    --rm wallysoc/regression_wally
+
+# run regression on the Karl-Han/cvw
+podman run \
+    -e CLEAN_CVW= -e BUILD_RISCOF= -e RUN_QUESTA= \
+    -e CVW_GIT=https://github.com/Karl-Han/cvw \
+    -v cvw_temp:/home/cad/cvw \
+    -v /cad/mentor/questa_sim-2023.4:/cad/mentor/questa_sim-xxxx.x_x \
+    --privileged --network=host \
+    --rm wallysoc/regression_wally
+
+# get into the container command line to debug or reading files
+podman run -it \
+    -e RUN_QUESTA= \
+    -v cvw_temp:/home/cad/cvw \
+    -v /cad/mentor/questa_sim-2023.4:/cad/mentor/questa_sim-xxxx.x_x \
+    --privileged --network=host \
+    wallysoc/regression_wally /bin/bash
+```
+
+## Conventions
+
+- In the container
+    - default user is `cad`
+    - RISCV is defined as `/opt/riscv`
+    - QUESTA is defined as `/cad/mentor/questa_sim-xxxx.x_x`
+        - bin location is in `$QUESTA/questasim/bin`
+    - cvw folder should be mounted on `/home/${USERNAME}/cvw`
+        - as for `cad`, it is `/home/cad/cvw`
+- In the current shell environment: checkout the constants in the following script section
+
+## New Dockerfile Design
+
+There are two parts of the new docker-related design:
+
+- build proper image(s)
+- scripts for different purposes
+
+### Problem Statement
+
+The following 3 problems are to be resolved:
+
+- remove storage of useless files with `Dockerfile`
+    - git build: clean up
+    - packages info
+        - apt: `apt-get clean`
+        - pip: `--no-cache-dir`
+- reuse storage
+    - read-only $RISCV volume across different users with `docker image`
+    - local-built RISCV with `start-up script`
+- use commercial EDA tools: optional environment variable configuration with `start-up script`
+
+### Dockerfiles
+
+There are two Dockerfiles:
+
+- `Dockerfile.ubuntu`: basic ubuntu system setup and produce `ubuntu_wally` image in docker
+    - corresponds to `wallysoc/ubuntu_wally`
+- `Dockerfile.builds`: all the required toolchains are built
+    - corresponds to `wallysoc/toolchains_wally`
+
+Because we are going to use the whole environment of ubuntu to get both executables and python packages, we are not going to use multi-stage builds, which runs only executables.
+
+### Scripts
+
+There are four scripts:
+
+- `start.sh` (most often used): start running the container
+    - if you don't care about toolchains and running regression automatically, this script is only thing you need to know
+- `get_images.sh`: get docker image `wallysoc/ubuntu_wally` or `wallysoc/toolchains_wally`
+- `get_buildroot_testvector.py`: copy buildroot and testvector configuration required by building
+- `run_regression.sh`: run regressions with Verilator (and QuestaSIM) on specific CVW
+    - this script is not intended to be run directly, but inside the container
+    - However, it is a good resource to look into to know what is happening
+- `test.sh`: a test script to facilitate the above three
+
+All the following options in the corresponding scripts should be set by either:
+
+- define it right before the command: `USE_PODMAN=1 ./start.sh` with USE_PODMAN on start-up script
+    - the variable is only effective for the current command, not the following environment
+- declare it globally and use it any time afterwards:
+
+```shell
+# declare it globally in the environment
+export DOCKER_EXEC=$(which docker)
+export UBUNTU_BUILD=1
+
+# run the script with all the above variables
+./get_images.sh
+```
+
+#### Start-up Script: start.sh
+
+There are two settings:
+
+- build/fetch both ubuntu_wally and toolchains in docker image and use it
+- build/fetch only ubuntu_wally and use local toolchains folder
+
+Options:
+
+- USE_PODMAN:
+    - by default, docker is used
+    - set USE_PODMAN=1 otherwise
+- UBUNTU_BUILD:
+    - fetch by default
+    - set UBUNTU_BUILD=1 if you want to build with Dockerfile.ubuntu
+- TOOLCHAINS_BUILD:
+    - fetch by default
+    - set TOOLCHAINS_BUILD=1 if you want to build with Dockerfile.build
+
+#### Image Building Script: get_images.sh
+
+Options (if you want to build the images):
+
+- DOCKER_EXEC:
+    - docker by default
+    - if you want to use podman, then set it to $(which podman)
+- UBUNTU_BUILD:
+    - fetch by default
+    - set it to 1 if you want to build it instead of fetching it
+- TOOLCHAINS_BUILD:
+    - fetch by default
+    - set it to 1 if you want to build it instead of fetching it
+
+#### Regression Script: run_regression.sh
+
+There are two parts for regression:
+
+- Verilator: must be able to run as it is open-sourced
+- Questa: OPTIONAL as it is commercial EDA Tool
+
+
+There are three main knobs:
+
+1. CLEAN_CVW: remove the `/home/${USERNAME}/cvw` inside the container (it can be a volume) and clone the `${CVW_GIT}`.
+2. BUILD_RISCOF: build RISCOF in the `/home/${USERNAME}/cvw`, sometimes you don't want to rebuild if there is no change in the test suite.
+3. RUN_QUESTA: enable the QuestaSIM in regression
+
+Options:
+
+- CVW_GIT: git clone address, only main branch supported
+- CLEAN_CVW: declared with empty string to clone
+- BUILD_RISCOF: declared with empty string to rebuild RISCOF
+- RUN_QUESTA: declared with empty string to run vsim to check
+    - QUESTA: home folder for mounted QuestaSIM `/cad/mentor/questa_sim-xxxx.x_x` if enabled
+    - for example, if your vsim is in `/cad/mentor/questa_sim-2023.4/questasim/bin/vsim` then your local QuestaSIM folder is `/cad/mentor/questa_sim-2023.4`, so you have to add `-v /cad/mentor/questa_sim-2023.4:/cad/mentor/questa_sim-xxxx.x_x -e RUN_QUESTA=1`
+
+### Commercial EDA Tools
+
+This is kind of tricky, because Docker network is a different network from the host. Therefore, it should be carefully handled in order to use the host's license server while minimizing the access of docker container.
+
+There are at least two ways to solve this problem:
+
+- use `--network=host` in docker-run to use the same network
+- (NOT WORKING) use bridge in docker-run while forwarding docker network's traffic to localhost without setting `X11UseLocalhost no`
+    - this idea is from https://stackoverflow.com/a/64284364/10765798
+
+## Old Dockerfile Analysis
+
+> Refer to https://github.com/openhwgroup/cvw/blob/91919150a94ccf8e750cf7c9eec1c400efaef7f5/docs/Dockerfile
+
+There are stages in the old Dockerfile:
+
+- debian-based package installtion
+    - apt package
+    - python3 package
+- user and its group configuration
+- clone and build toolchain with prefix=$RISCV
+    - riscv-gnu-toolchain: https://github.com/riscv-collab/riscv-gnu-toolchain
+    - elf2hex: https://github.com/sifive/elf2hex
+    - qemu: https://github.com/qemu/qemu
+    - spike: https://github.com/riscv-software-src/riscv-isa-sim
+    - sail: https://github.com/riscv/sail-riscv/commits/master/
+    - buildroot: https://github.com/buildroot/buildroot
+    - verilator: https://github.com/verilator/verilator
+
+### Tool Versions till 20240331
+
+- riscv-gnu-toolchain: `2024.03.01`
+- elf2hex: `f28a3103c06131ed3895052b1341daf4ca0b1c9c`
+- qemu: not tested
+- spike: `3427b459f88d2334368a1abbdf5a3000957f08e8`
+- sail: `f601c866153c79a7ae8404f939dc2d66aa2e41f9`
+- buildroot: `2021.05`
+- verilator: `v5.022`
+
+## Troubleshooting
+
+### Permission Denied for .git
+
+Description: permission problem in `/home/$USERNAME/cvw`.
+
+```text
+$ podman run -v cvw_temp:/home/cad/cvw -e CLEAN_CVW= -e BUILD_RISCOF= -e RUN_QUESTA= -v /cad/mentor/que
+sta_sim-2023.4:/cad/mentor/questa_sim-xxxx.x_x --rm wallysoc/regression_wally                          
+No CVW_GIT is provided                                                                                 
+rm: cannot remove '/home/cad/cvw': Device or resource busy                                             
+Cloning into '/home/cad/cvw'...                                                                        
+/home/cad/cvw/.git: Permission denied                                                                  
+chmod: cannot access '/home/cad/cvw/setup.sh': No such file or directory                               
+chmod: cannot access '/home/cad/cvw/site-setup.sh': No such file or directory                          
+make: *** No rule to make target 'install'.  Stop.                                                     
+make: *** No rule to make target 'verify'.  Stop.                                                      
+make: *** No rule to make target 'coverage'.  Stop.                                                    
+make: *** No rule to make target 'benchmarks'.  Stop.                                                  
+/home/cad/run_regression.sh: line 64: cd: /home/cad/cvw/sim: No such file or directory                 
+/home/cad/run_regression.sh: line 65: /home/cad/cvw/sim/regression_verilator.out: No such file or direc
+tory       
+```
+
+It may be caused by podman and I am not sure why it happens.
+
+Solution: get into the container with interaction as root and change the permission of the folder/volume `/home/$USERNAME/cvw`
+
+```shell
+podman run -it --user root -v cvw_temp:/home/cad/cvw -e RUN_QUESTA= -v /cad/mentor/questa_sim-2023.4:/cad/mentor/questa_sim-xxxx.x_x --net=host --rm --privileged wallysoc/regression_wally /bin/bash
+
+chown -R $USERNAME:$USERNAME /home/$USERNAME
+```
+
+## References
+
+- Dockerfile Docs: https://docs.docker.com/reference/dockerfile/
+- Multi-stage builds: https://docs.docker.com/build/building/multi-stage/
+- Best Practices: https://docs.docker.com/develop/develop-images/guidelines/
+- Chinese Reference: https://yeasy.gitbook.io/docker_practice/
+- Clean Cache
+    - apt cache: https://gist.github.com/marvell/7c812736565928e602c4
+    - pip cache: https://stackoverflow.com/questions/50333650/install-python-package-in-docker-file
+- Docker Network: https://docs.docker.com/network/
--- a/docs/docker/get_buildroot_testvector.py
+++ b/docs/docker/get_buildroot_testvector.py
@ -0,0 +1,13 @@
+import shutil, os
+
+# if WALLY is defined, then get it
+WALLY_HOME = os.getenv("WALLY")
+if WALLY_HOME is None or WALLY_HOME == "":
+    # otherwise, it is assumed as ../../
+    WALLY_HOME = "../../"
+
+BUILDROOT_SRC = "linux/buildroot-config-src/wally"
+TESTVECTOR_SRC = "linux/testvector-generation"
+
+shutil.copytree(os.path.join(WALLY_HOME, BUILDROOT_SRC), "./buildroot-config-src")
+shutil.copytree(os.path.join(WALLY_HOME, TESTVECTOR_SRC), "./testvector-generation")
--- a/docs/docker/get_images.sh
+++ b/docs/docker/get_images.sh
@ -0,0 +1,23 @@
+UBUNTU_BUILD=${UBUNTU_BUILD:-0}
+TOOLCHAINS_BUILD=${TOOLCHAINS_BUILD:-0}
+
+if [ -n "$USE_PODMAN" ]; then
+    DOCKER_EXEC=$(which podman)
+else
+    DOCKER_EXEC=$(which docker)
+fi
+
+if [ $UBUNTU_BUILD -eq 1 ]; then
+    ${DOCKER_EXEC} build -t ubuntu_wally -f Dockerfile.ubuntu .
+    ${DOCKER_EXEC} tag ubuntu_wally:latest wallysoc/ubuntu_wally:latest
+else
+    ${DOCKER_EXEC} pull wallysoc/ubuntu_wally
+fi
+
+if [ $TOOLCHAINS_BUILD -eq 1 ]; then
+    `which python` get_buildroot_testvector.py
+    ${DOCKER_EXEC} build -t toolchains_wally -f Dockerfile.builds .
+    ${DOCKER_EXEC} tag toolchains_wally:latest wallysoc/toolchains_wally:latest
+else
+    ${DOCKER_EXEC} pull wallysoc/toolchains_wally
+fi
--- a/docs/docker/run_regression.sh
+++ b/docs/docker/run_regression.sh
@ -0,0 +1,69 @@
+#!/bin/bash
+# this script is used to run regression inside the Dockerfile.regression
+# of course, you can run it in the current environment as soon as
+#   - RISCV is defined
+#   - QUESTA is defined
+# declare with empty string: export ABC=
+
+# Options:
+# - CVW_GIT: git clone address, only main branch supported
+# - CLEAN_CVW: declared with empty string to clone
+# - BUILD_RISCOF: declared with empty string to rebuild RISCOF
+# - RUN_QUESTA: declared with empty string to run vsim to check
+
+# now only main branch is supported
+if [ -z "${CVW_GIT}" ]; then
+    echo "No CVW_GIT is provided"
+    export CVW_GIT="https://github.com/openhwgroup/cvw"
+else
+    echo "Using customized CVW_GIT: ${CVW_GIT}"
+fi
+
+git config --global http.version HTTP/1.1
+
+# if cvw is not available or CLEAN_CVW(empty string) is defined
+if [[ ! -f "/home/${USERNAME}/cvw/setup.sh" ]] || [[ -z "${CLEAN_CVW-x}" ]]; then
+    cd /home/${USERNAME} && rm -rf /home/${USERNAME}/cvw
+    git clone --recurse-submodules  ${CVW_GIT} /home/${USERNAME}/cvw
+    # if failed to clone submodules for some reason, please run `git submodule update`
+fi
+
+# Preset Environment Variable
+export PATH="${RISCV}/bin:${PATH}"
+export CVW_HOME="/home/${USERNAME}/cvw"
+export QUESTA="/cad/mentor/questa_sim-xxxx.x_x"
+export PATH="${QUESTA}/questasim/bin:${PATH}"
+
+# cd /home/${USERNAME}/cvw && chmod +x ./setup.sh && ./setup.sh
+chmod +x ${CVW_HOME}/setup.sh && source ${CVW_HOME}/setup.sh
+chmod +x ${CVW_HOME}/site-setup.sh && source ${CVW_HOME}/site-setup.sh
+
+# Overwriting
+export QUESTAPATH=/cad/mentor/questa_sim-xxxx.x_x/questasim/bin
+
+# # if you are making it alone, it works
+# make
+
+cd ${CVW_HOME}
+# build it only if BUILD_RISCOF is defined with empty string
+if [[ -z "${BUILD_RISCOF-x}" ]]; then
+    make install && make riscof && make testfloat
+fi
+
+if [[ -z "${RUN_QUESTA-x}" ]] ; then
+    if [ ! -f "${QUESTA}/questasim/bin/vsim" ]; then
+        echo "Cannot find vsim with ${QUESTA}/questasim/bin/vsim"
+    else
+        # cd sim && ./regression-wally 2>&1 > ./regression_questa.out && cd ..
+        # make verify
+        regression-wally
+    fi
+fi
+
+make coverage
+make benchmarks
+
+if [[ ! NO_VERILATOR -eq 1 ]]; then
+    cd ${CVW_HOME}/sim/verilator && verilator -GTEST="\"arch64i\"" -DVERILATOR=1 --timescale "1ns/1ns" --timing --binary --top-module testbench -I${CVW_HOME}/config/shared -I${CVW_HOME}/config/rv64gc ${CVW_HOME}/src/cvw.sv ${CVW_HOME}/testbench/testbench.sv ${CVW_HOME}/testbench/common/*.sv ${CVW_HOME}/src/*/*.sv ${CVW_HOME}/src/*/*/*.sv --relative-includes
+    ${CVW_HOME}/sim/verilator/obj_dir/Vtestbench
+fi
--- a/docs/docker/start.sh
+++ b/docs/docker/start.sh
@ -0,0 +1,34 @@
+if [ -n "$USE_PODMAN" ]; then
+    DOCKER_EXEC=$(which podman)
+else
+    DOCKER_EXEC=$(which docker)
+fi
+CVW_MOUNT=${CVW_MOUNT:$(pwd)/../../}
+USERNAME="cad"
+
+UBUNTU_WALLY_HASH=$(${DOCKER_EXEC} images --quiet wallysoc/ubuntu_wally)
+TOOLCHAINS_HASH=$(${DOCKER_EXEC} images --quiet wallysoc/toolchains_wally)
+TOOLCHAINS_MOUNT=${TOOLCHAINS_MOUNT}
+
+if [ -z $UBUNTU_WALLY_HASH ]; then
+    echo "CANNOT FIND wallysoc/ubuntu_wally, please get the image first with \`get_image.sh\`";
+    exit 1
+else
+    echo "Get ${UBUNTU_WALLY_HASH} for ubuntu_wally"
+fi
+
+if [ ! -z $TOOLCHAINS_MOUNT ]; then
+    if [ -n "$QUESTA" ]; then
+        ${DOCKER_EXEC} run -it --rm -v ${TOOLCHAINS_MOUNT}:/opt/riscv -v ${CVW_MOUNT}:/home/${USERNAME}/cvw -v ${QUESTA}:/cad/mentor/questa_sim-xxxx.x_x wallysoc/ubuntu_wally
+    else
+        ${DOCKER_EXEC} run -it --rm -v ${TOOLCHAINS_MOUNT}:/opt/riscv -v ${CVW_MOUNT}:/home/${USERNAME}/cvw wallysoc/ubuntu_wally
+    fi
+elif [ -z $TOOLCHAINS_HASH ]; then
+    echo "CANNOT FIND wallysoc/toolchains_wally, please get the image first with \`get_image.sh\`";
+    exit 1
+else
+    echo "Get ${TOOLCHAINS_HASH} for toolchains_wally"
+    ${DOCKER_EXEC} run -it --rm -v ${CVW_MOUNT}:/home/${USERNAME}/cvw wallysoc/toolchains_wally
+fi
+
+echo "Successfully reach the end"
--- a/docs/docker/test.sh
+++ b/docs/docker/test.sh
@ -0,0 +1,22 @@
+# There are three parts in testing as there are three scripts
+# It is fine to use either docker or podman as they are equivalent in the context
+export USE_PODMAN=1
+
+## build either one of them
+UBUNTU_BUILD=1 ./get_image.sh
+echo "get images is $?"
+
+## mount toolchian and questa
+TOOLCHAINS_MOUNT=/opt/riscv QUESTA=/cad/mentor/questa_sim-2023.4 ./start.sh
+# then run 
+# - `file ${QUESTA}/questasim/vsim` to check if it is properly mount
+# - `file $RISCV/bin/riscv64-unknown-elf-gcc` to check the toolchain
+
+## use internal toolchain
+./start.sh
+# then run 
+# - `file $RISCV/bin/riscv64-unknown-elf-gcc` to check the toolchain
+# - `which verilator | grep riscv` to check the verilator
+
+## mount questa
+RUN_QUESTA=true QUESTA=/cad/mentor/questa_sim-2023.4 ./run_regression.sh