[Do Not Merge] CMake and Upstream Bump: linux, QEMU, buildroot (#132)

* CMake and Upstream Bump: linux, QEMU, buildroot

A large change has been made for preparing RV32 support.
- Linux (5.1)
  A few relavant patches have been added: for CMA support and
  buildroot compatibility
- QEMU (4.1)
  The upstream QEMU had a PMP bug, so this includes a patch for the bug.
  The patch has been submitted to upstream, and will be included in 4.2.
  BootROM has not been integrated, and will be in the following commits
- Buildroot (2019.08)
  Buildroot has been bumped, so configure files have been also changed
  Current config is GCC 7, toolchain header 4.13 but kernel header 5.1.

CMakeLists.txt is introduced for better flexibility in the future
  The project will be supporting multiple platforms with different ABIs,
  so we inevitably moved to our own build system using CMake instead of
  relying on hifive's old Makefile.

  Hifive FU540 is not included yet, but will be added in the following
  commits.

Documentation is not updated, and will be in the following commits

* Travis Test Working
* Travis keeps path after cd
* Travis ran out of time; trying -j3
* Omit unnecessary build from buildroot
* Small Fix
* Add image as a dependency

.gitmodules

@@ -1,15 +1,9 @@
 [submodule "riscv-gnu-toolchain"]
 	path = riscv-gnu-toolchain
 	url = https://github.com/riscv/riscv-gnu-toolchain
-[submodule "riscv-linux"]
-	path = riscv-linux
-	url = https://github.com/riscv/riscv-linux
 [submodule "riscv-pk"]
 	path = riscv-pk
 	url = https://github.com/keystone-enclave/riscv-pk
-[submodule "riscv-qemu"]
-	path = riscv-qemu
-	url = https://github.com/riscv/riscv-qemu
 [submodule "sdk"]
 	path = sdk
 	url = https://github.com/keystone-enclave/keystone-sdk
@@ -18,4 +12,16 @@
 	url = https://github.com/keystone-enclave/linux-keystone-driver
 [submodule "buildroot"]
 	path = buildroot
-	url = https://github.com/keystone-enclave/buildroot.git
+	url = https://github.com/buildroot/buildroot.git
+	shallow = true
+[submodule "linux"]
+	path = linux
+	url = https://github.com/torvalds/linux
+	shallow = true
+[submodule "qemu"]
+	path = qemu
+	url = https://github.com/qemu/qemu
+[submodule "riscv-linux"]
+	shallow = true
+[submodule "riscv-qemu"]
+	shallow = true

.travis.yml

@@ -6,8 +6,8 @@ cache:
   directories:
   # - $TRAVIS_BUILD_DIR/firesim-riscv-tools-prebuilt
   - $TRAVIS_BUILD_DIR/riscv
-  - $TRAVIS_BUILD_DIR/riscv-linux
-  - $TRAVIS_BUILD_DIR/riscv-qemu
+  - $TRAVIS_BUILD_DIR/linux
+  - $TRAVIS_BUILD_DIR/qemu
 
 git:
   submodules: false
@@ -62,15 +62,15 @@ branches:
 
 
 before_install:
-  - git submodule init -- riscv-linux
-  - git submodule init -- riscv-qemu
-  - if [ -d "riscv-linux/.git" ]; then ls; else git clone --shallow-since=2018-05-01 https://github.com/riscv/riscv-linux riscv-linux; fi
-  - if [ -d "riscv-qemu/.git" ]; then ls; else git clone --shallow-since=2018-05-01 https://github.com/riscv/riscv-qemu riscv-qemu; fi
-  - git submodule update --depth=1 -- riscv-linux
-  - git submodule update --depth=1 -- riscv-qemu
+  - git submodule init -- linux
+  - git submodule init -- qemu
+  - if [ -d "linux/.git" ]; then ls; else git clone --shallow-since=2018-05-01 https://github.com/torvalds/linux linux; fi
+  - if [ -d "qemu/.git" ]; then ls; else git clone --shallow-since=2018-05-01 https://github.com/qemu/qemu qemu; fi
+  - git submodule update --depth=1 -- linux
+  - git submodule update --depth=1 -- qemu
   - ./fast-setup.sh
-  - cp -r riscv-linux/.git riscv-linux-git
-  - cp -r riscv-qemu/.git riscv-qemu-git
+  - cp -r linux/.git linux-git
+  - cp -r qemu/.git qemu-git
 
 jobs:
   include:
@@ -79,10 +79,14 @@ jobs:
     - stage: build everything & test
       script:
         - source ./source.sh
-        - travis_wait 120 make -j2
+        - mkdir build
+        - cd build
+        - cmake ../
+        - travis_wait 120 make -j3
         - travis_wait 10 make run-tests
-        - cd riscv-qemu; git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../riscv-qemu-git  .git;cd ..
-        - cd riscv-linux;git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../riscv-linux-git .git;cd ..
+        - cd ..
+        - cd qemu; git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../qemu-git  .git;cd ..
+        - cd linux;git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../linux-git .git;cd ..
       after_failure:
         - cat screenlog.0
         - cat output.log

CMakeLists.txt

@@ -0,0 +1,161 @@
+cmake_minimum_required(VERSION 3.10)
+project(keystone C)
+include(ProcessorCount)
+
+if(NOT DEFINED ENV{RISCV})
+  message(FATAL_ERROR "set RISCV environment variable")
+endif()
+
+if(Debug)
+  message(STATUS "debug")
+  set(CFLAGS -g)
+endif()
+
+if(RISCV32)
+  message(STATUS "riscv32")
+  set(BITS 32)
+  set(ABI ilp32d)
+else()
+  message(STATUS "riscv64")
+  set(BITS 64)
+  set(ABI lp64d)
+endif()
+
+set(ISA rv${BITS}imafdc)
+set(confdir ${CMAKE_SOURCE_DIR}/conf)
+set(patchdir ${CMAKE_SOURCE_DIR}/patches)
+set(cross_compile riscv${BITS}-unknown-linux-gnu-)
+
+set(bootrom_wrkdir ${CMAKE_BINARY_DIR}/bootrom${BITS})
+set(bootrom_srcdir ${CMAKE_SOURCE_DIR}/bootrom)
+set(qemu_wrkdir ${CMAKE_SOURCE_DIR}/qemu)
+set(qemu_srcdir ${CMAKE_SOURCE_DIR}/qemu)
+set(sm_srcdir ${CMAKE_SOURCE_DIR}/riscv-pk)
+set(sm_wrkdir ${CMAKE_BINARY_DIR}/riscv-pk${BITS})
+set(buildroot_srcdir ${CMAKE_SOURCE_DIR}/buildroot)
+set(buildroot_wrkdir ${CMAKE_BINARY_DIR}/buildroot${BITS})
+set(buildroot_config ${confdir}/qemu_riscv${BITS}_virt_defconfig)
+set(overlay_dir ${CMAKE_BINARY_DIR}/overlay)
+set(overlay_root ${overlay_dir}/root)
+set(linux_wrkdir ${CMAKE_BINARY_DIR}/linux${BITS})
+set(linux_srcdir ${CMAKE_SOURCE_DIR}/linux)
+set(linux_defconfig ${confdir}/linux-v5.0-defconfig-rv${BITS})
+set(linux_vmlinux ${linux_wrkdir}/vmlinux)
+set(linux_vmlinux_stripped ${linux_wrkdir}/vmlinux-stripped)
+set(driver_srcdir ${CMAKE_SOURCE_DIR}/linux-keystone-driver)
+set(driver_wrkdir ${CMAKE_BINARY_DIR}/linux-keystone-driver${BITS})
+set(tests_srcdir ${CMAKE_SOURCE_DIR}/tests)
+
+
+# QEMU
+add_custom_target("qemu" ALL COMMAND $(MAKE) -C ${qemu_srcdir} DEPENDS "qemu-config")
+add_custom_target("qemu-config" DEPENDS ${qemu_srcdir}/ ${CMAKE_SOURCE_DIR}/patches/qemu/
+  WORKING_DIRECTORY ${qemu_srcdir}
+  COMMAND patch --forward -p0 < ${patchdir}/qemu/qemu-pmp-bug.patch || true
+  COMMAND patch --forward -p0 < ${patchdir}/qemu/qemu-secure-boot.patch || true
+  COMMAND ./configure --target-list=riscv${BITS}-softmmu,riscv${BITS}-linux-user
+)
+
+# buildroot
+add_custom_command(OUTPUT ${overlay_root} COMMAND mkdir -p ${overlay_root})
+add_custom_command(OUTPUT ${overlay_dir} COMMAND mkdir -p ${overlay_dir})
+add_custom_command(OUTPUT ${buildroot_wrkdir} COMMAND mkdir -p ${buildroot_wrkdir})
+add_custom_target("buildroot" ALL DEPENDS ${buildroot_srcdir} ${buildroot_wrkdir}/.config ${overlay_root} ${buildroot_wrkdir}
+  COMMAND $(MAKE) -s -C ${buildroot_srcdir} RISCV=$ENV{RISCV} PATH=$ENV{PATH} O=${buildroot_wrkdir}
+)
+
+string(REPLACE "/" "\\/" overlay_dir_stripped ${overlay_dir})
+add_custom_command(DEPENDS ${buildroot_config} OUTPUT ${buildroot_wrkdir}/.config
+  COMMAND mkdir -p ${buildroot_wrkdir}
+  COMMAND cp ${buildroot_config} ${buildroot_wrkdir}/.config
+  COMMAND sed \"s/^BR2_ROOTFS_OVERLAY=.*/BR2_ROOTFS_OVERLAY=\\\"${overlay_dir_stripped}\\\"/g\" -i ${buildroot_wrkdir}/.config
+  COMMAND $(MAKE) -s -C ${buildroot_srcdir} RISCV=$ENV{RISCV} PATH=$ENV{PATH}
+    O=${buildroot_wrkdir} CROSS_COMPILE=${cross_compile} olddefconfig
+)
+
+# boot ROM
+add_custom_command(OUTPUT ${bootrom_wrkdir} COMMAND mkdir -p ${bootrom_wrkdir})
+add_custom_target("bootrom" ALL
+  COMMAND $(MAKE) -C ${bootrom_srcdir} O=${bootrom_wrkdir}
+  DEPENDS ${bootrom_wrkdir} ${bootrom_srcdir})
+
+# linux
+add_custom_command(OUTPUT ${linux_wrkdir} COMMAND mkdir -p ${linux_wrkdir})
+add_custom_target("linux" ALL DEPENDS ${linux_srcdir} ${linux_wrkdir}/.config
+  COMMAND cd ${linux_srcdir} \; patch --forward -p0 < ${patchdir}/linux/linux.patch || true
+  COMMAND $(MAKE) -C ${linux_srcdir} O=${linux_wrkdir} CROSS_COMPILE=${cross_compile} ARCH=riscv vmlinux
+  COMMAND ${cross_compile}strip -o ${linux_vmlinux_stripped} ${linux_vmlinux}
+)
+
+add_custom_command(OUTPUT ${linux_wrkdir}/.config DEPENDS ${linux_defconfig}
+  COMMAND mkdir -p ${linux_wrkdir}
+  COMMAND cp ${linux_defconfig} ${linux_wrkdir}/.config
+  COMMAND $(MAKE) -C ${linux_srcdir} O=${linux_wrkdir} ARCH=riscv olddefconfig
+)
+
+# linux module
+add_custom_target("driver" ALL DEPENDS ${driver_srcdir} ${linux_srcdir} ${linux_wrkdir}
+  COMMAND mkdir -p ${driver_wrkdir}
+  COMMAND $(MAKE) -C ${linux_srcdir} O=${linux_wrkdir} CROSS_COMPILE=${cross_compile} ARCH=riscv
+    M=${driver_srcdir} modules
+  COMMAND $(MAKE) -C ${linux_srcdir} O=${linux_wrkdir} CROSS_COMPILE=${cross_compile} ARCH=riscv
+    M=${driver_srcdir} modules_install INSTALL_MOD_PATH=${driver_wrkdir}
+)
+
+# bbl + sm
+add_custom_command(OUTPUT ${sm_wrkdir} COMMAND mkdir -p ${sm_wrkdir})
+add_custom_target("sm" ALL DEPENDS ${sm_srcdir} ${sm_wrkdir} ${linux_vmlinux_stripped} WORKING_DIRECTORY ${sm_wrkdir}
+  COMMAND ${sm_srcdir}/configure --enable-sm --host=riscv${BITS}-unknown-linux-gnu --with-payload=${linux_vmlinux_stripped}
+    --enable-logo --with-logo=${confdir}/sifive_logo.txt --with-target-platform=default
+    CFLAGS=${CFLAGS}
+  COMMAND env CFLAGS='-g -mabi=${ABI} -march=${ISA}' $(MAKE) -C ${sm_wrkdir}
+)
+add_dependencies("sm" "linux")
+add_dependencies("driver" "linux")
+
+add_custom_target("tests" DEPENDS "driver" ${overlay_root} ${tests_srcdir}
+  COMMAND OUTPUT_DIR=${overlay_root} ${tests_srcdir}/tests/vault.sh
+  COMMAND find ${driver_wrkdir} -name "*.ko" -exec cp {} ${overlay_root} \\\\;
+)
+
+add_custom_target("image" DEPENDS "sm" "tests" ${buildroot_srcdir} ${buildroot_wrkdir}/.config ${overlay_root}
+  COMMAND $(MAKE) -s -C ${buildroot_srcdir} RISCV=$ENV{RISCV} PATH=$ENV{PATH} O=${buildroot_wrkdir}
+)
+
+# scripts
+set(scripts ${CMAKE_BINARY_DIR}/scripts)
+add_custom_command(OUTPUT ${scripts} COMMAND mkdir -p ${scripts})
+add_custom_command(OUTPUT ${scripts}/run-qemu.sh
+  WORKING_DIRECTORY ${scripts}
+  COMMAND echo "\
+    ${qemu_wrkdir}/riscv${BITS}-softmmu/qemu-system-riscv${BITS}\
+    -m 2G \
+    -nographic \
+    -machine virt \
+    -bios ${bootrom_wrkdir}/bootrom.bin \
+    -kernel ${sm_wrkdir}/bbl \
+    -append \"console=ttyS0 ro root=/dev/vda\" \
+    -drive file=${buildroot_wrkdir}/images/rootfs.ext2,format=raw,id=hd0 \
+    -device virtio-blk-device,drive=hd0 \
+    -netdev user,id=net0,net=192.168.100.1/24,dhcpstart=192.168.100.128,hostfwd=tcp::\$\{HOST_PORT\}-:22 \
+    -device virtio-net-device,netdev=net0" > run-qemu.sh
+    VERBATIM
+  COMMAND
+    chmod +x run-qemu.sh
+  )
+add_custom_command(OUTPUT ${scripts}/test-qemu.sh DEPENDS ${CMAKE_SOURCE_DIR}/scripts
+  COMMAND cp ${CMAKE_SOURCE_DIR}/scripts/test-qemu.sh ${scripts})
+add_custom_command(OUTPUT ${scripts}/travis.sh DEPENDS ${CMAKE_SOURCE_DIR}/scripts
+  COMMAND cp ${CMAKE_SOURCE_DIR}/scripts/travis.sh ${scripts})
+
+add_custom_target(
+  "all-scripts" ALL
+  DEPENDS ${scripts} ${scripts}/run-qemu.sh ${scripts}/test-qemu.sh ${scripts}/travis.sh
+)
+
+add_custom_target(
+  "run-tests"
+  DEPENDS "all-scripts" "image"
+  COMMAND
+    ./scripts/travis.sh ${CMAKE_SOURCE_DIR}/tests
+)

bootrom/Makefile

@@ -5,19 +5,21 @@ CFLAGS = \
 	-nostdlib -nostartfiles -fno-common -std=gnu11 \
 	-static \
 	-fPIC \
-	-O2 -Wall\
+	-O2 -Wall
+O ?=.
 
 # ^ consider taking out -g -Og and putting in -O2
 
 bootloaders=\
-	bootrom.elf
+	$(O)/bootrom.elf \
+	$(O)/bootrom.bin
 
 .PHONY: all
 all: $(bootloaders)
 
 .PHONY: clean
 clean:
-	rm -f bootrom.elf
+	rm -f $(bootloaders)
 
 bootrom_sources = \
 	./bootloader.S \
@@ -27,3 +29,7 @@ bootrom_sources = \
 
 %.elf: $(bootrom_sources) bootloader.lds
 	$(CC) $(CFLAGS) -I./ -L . -T bootloader.lds -o $@ $(bootrom_sources)
+
+%.bin: %.elf
+	$(OBJCOPY) -O binary --only-section=.text $< $@;
+

bootrom/bootloader.S

@@ -13,7 +13,7 @@ reset:
   csrr t0, mhartid
   bnez t0, loop
 
-  sd a1, DTB(sp)
+  # sd a1, DTB(sp)
 
   /* secure boot */
   call bootloader
@@ -35,7 +35,8 @@ boot:
 
   # boot payload
 	csrr a0, mhartid
-  ld a1, DTB(sp)
+  la a1, _dtb
+	# ld a1, DTB(sp)
   li t0, DRAM_BASE
   jr t0
 

bootrom/bootloader.lds

@@ -4,14 +4,15 @@ ENTRY( _entry )
 
 SECTIONS
 {
-  . = 0x2000; /* boot loader lives in boot ROM after the device tree */
+  . = 0x1000; /* boot loader lives in boot ROM after the device tree */
   PROVIDE( reset_vector = . );
   .text :
   {
     PROVIDE( _entry = . );
     *(.reset)
     *(*)
   }
+  PROVIDE( _dtb = . );
 }
 
 INCLUDE sanctum_params.lds