Moving to OpenSBI, deprecate BBL (riscv-pk), and more (#220)

* Moving to OpenSBI, deprecate BBL (riscv-pk) * Bump latest riscv-gnu-toolchain (2021.01) * Bump buildroot (2020.11) * Remove all newly-introduced compiler errors * Test and fix RV32 (ilp32d/rv32gc) dtb was not aligned, which results in system hang at boot. align dtb in both bootrom and QEMU. * Better scripting for RV32 (always use fast-setup.sh) * Buildroot configs for the new toolchain (glibc) * Linux config mcmodel to medany (to be consistent) * CI changes for the latest toolchain * CI changes for RV32 test * Remove unused patches * Switch CI from Travis to CircleCI (Travis is no longer free for OSS) Issues resolved: - #212 - #195 - #127 - #147 - #148 - keystone-enclave/keystone-sdk#48 - keystone-enclave/keystone-runtime#46
keystone-enclave · Feb 10, 2021 · b4b6cc5 · b4b6cc5
1 parent 7e3e348
commit b4b6cc5
Show file tree

Hide file tree

Showing 30 changed files with 323 additions and 348 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,75 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+# prebuilt docker images with toolchain
+executors:
+  setup-rv64gc:
+    docker:
+      - image: keystoneenclaveorg/keystone:init-rv64gc
+  setup-rv32gc:
+    docker:
+      - image: keystoneenclaveorg/keystone:init-rv32gc
+
+
+commands:
+  update-riscv-toolchain-path:
+    steps:
+      - run: echo 'export PATH=/keystone/riscv64/bin:/keystone/riscv32/bin:$PATH' >> $BASH_ENV
+
+jobs:
+  build-and-test-rv64-qemu:
+    executor: setup-rv64gc
+    working_directory: /keystone
+    steps:
+      - checkout
+      - update-riscv-toolchain-path
+      - run:
+          name: "Build rv64 and run tests"
+          command: |
+              ./fast-setup.sh
+              source ./source.sh
+              mkdir build64
+              cd build64
+              cmake /keystone
+              make -j$(nproc)
+              make run-tests
+          no_output_timeout: 120m
+  build-rv64-fu540:
+    executor: setup-rv64gc
+    working_directory: /keystone
+    steps:
+      - checkout
+      - update-riscv-toolchain-path
+      - run:
+          name: "Build rv64 sifive FU540"
+          command: |
+              ./fast-setup.sh
+              source ./source.sh
+              mkdir build64-fu540
+              cd build64-fu540
+              cmake /keystone -DLINUX_SIFIVE=y -DSM_PLATFORM=sifive/fu540
+              make -j$(nproc)
+  build-and-test-rv32-qemu:
+    executor: setup-rv32gc
+    working_directory: /keystone
+    steps:
+      - checkout
+      - update-riscv-toolchain-path
+      - run:
+          name: "Build rv32 and run tests"
+          command: |
+              BITS=32 ./fast-setup.sh
+              source ./source.sh
+              mkdir build32
+              cd build32
+              cmake /keystone -DRISCV32=y
+              make -j$(nproc)
+              make run-tests
+          no_output_timeout: 120m
+
+workflows:
+  build-and-test:
+    jobs:
+      - build-and-test-rv64-qemu
+      - build-rv64-fu540
+      - build-and-test-rv32-qemu
diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +1,6 @@
 [submodule "riscv-gnu-toolchain"]
 	path = riscv-gnu-toolchain
 	url = https://github.com/riscv/riscv-gnu-toolchain
-[submodule "riscv-pk"]
-	path = riscv-pk
-	url = https://github.com/keystone-enclave/riscv-pk
 [submodule "sdk"]
 	path = sdk
 	url = https://github.com/keystone-enclave/keystone-sdk
@@ -19,3 +16,6 @@
 [submodule "qemu"]
 	path = qemu
 	url = https://github.com/qemu/qemu
+[submodule "sm"]
+	path = sm
+	url = https://github.com/keystone-enclave/sm
diff --git a/.prebuilt_tools_shasums b/.prebuilt_tools_shasums
@@ -1,3 +1,4 @@
-05211edea5a47ebaf906ef4bed2c9609a93e8c6ad5e45f8c87678eabbfd424e7  1.0.tar.gz
-bdc9e3ec47ac461ecc7865609fda6b820439c36130e9da9275af010d9f4fe4bc  2.0.tar.gz
-e73fa1191ce97be2531401d4628384c321c45ca5e4c83756e8415f2cc31a0b18  rv32gc.tar.gz
+ad8c0e3357579ae8b9c6f8adb76aae5ac9155134304e11e429b81111283a4718  riscv-toolchain-ilp32d-rv32gc-2021.01.xenial.7z
+ed41a452ba736d40de73b847777ad9cedd54e528f5d8930c44a6eb937e4d36d9  riscv-toolchain-lp64d-rv64gc-2021.01.xenial.7z
+7e232c2030cc8ab8dab169a4e9dac1cc75c41ea1da8cbe1a70ba416a6d6b971f  riscv-toolchain-ilp32d-rv32gc-2021.01.bionic.7z
+2c37fa700f5ecf7ab8c1e5aeddd018fc15e0c187721b769a1d1318ed58228685  riscv-toolchain-lp64d-rv64gc-2021.01.bionic.7z
diff --git a/.travis.yml b/.travis.yml
@@ -1,20 +1,19 @@
 language: minimal
-dist: xenial
+dist: bionic
 cache:
   apt: true
   timeout: 1000 #In seconds
   directories:
-  # - $TRAVIS_BUILD_DIR/firesim-riscv-tools-prebuilt
-  - $TRAVIS_BUILD_DIR/riscv
-  - $TRAVIS_BUILD_DIR/linux
-  - $TRAVIS_BUILD_DIR/qemu
+  - $TRAVIS_BUILD_DIR/riscv64
+  - $TRAVIS_BUILD_DIR/linux/.git
+  - $TRAVIS_BUILD_DIR/qemu/.git
 
 git:
   submodules: false
   depth: 1
 
 env:
-  - RISCV=$TRAVIS_BUILD_DIR/riscv PATH=$PATH:$RISCV/bin
+  - RISCV=$TRAVIS_BUILD_DIR/riscv64 PATH=$PATH:$RISCV/bin
 
 addons:
   apt:
@@ -54,6 +53,7 @@ addons:
     - libssl-dev
     - makeself
     - unzip
+    - p7zip-full
 
 branches:
   only:
@@ -62,24 +62,22 @@ branches:
 
 
 before_install:
+  - if [ -d "linux/.git" ]; then ls; else git clone --shallow-since=2020-04-01 https://github.com/torvalds/linux linux; fi
+  - if [ -d "qemu/.git" ]; then ls; else git clone --shallow-since=2020-04-01 https://github.com/qemu/qemu qemu; fi
   - git submodule init -- linux
   - git submodule init -- qemu
-  - if [ -d "linux/.git" ]; then ls; else git clone --shallow-since=2018-05-01 https://github.com/torvalds/linux linux; fi
-  - if [ -d "qemu/.git" ]; then ls; else git clone --shallow-since=2018-05-01 https://github.com/qemu/qemu qemu; fi
   - git submodule update --depth=1 -- linux
   - git submodule update --depth=1 -- qemu
   - ./fast-setup.sh
-  - cp -r linux/.git linux-git
-  - cp -r qemu/.git qemu-git
 
 jobs:
   include:
     - stage: cache warmup
       script: true
       after_failure:
         - ls -al
-        - cat firesim-riscv-tools-prebuilt-*/riscv-tools-install.log
-    - stage: build everything & test
+        - git submodule status
+    - stage: build rv64 & test
       script:
         - source ./source.sh
         - mkdir build
@@ -88,8 +86,6 @@ jobs:
         - travis_wait 120 make -j3
         - travis_wait 10 make run-tests
         - cd ..
-        - cd qemu; git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../qemu-git  .git;cd ..
-        - cd linux;git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../linux-git .git;cd ..
       after_failure:
         - cat screenlog.0
         - cat output.log
@@ -98,8 +94,19 @@ jobs:
         - source ./source.sh
         - mkdir build
         - cd build
-        - cmake -DLINUX_SIFIVE=y -DSM_PLATFORM=fu540 ../
+        - cmake -DLINUX_SIFIVE=y -DSM_PLATFORM=sifive/fu540 ../
         - travis_wait 120 make -j3
         - cd ..
-        - cd qemu; git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../qemu-git  .git;cd ..
-        - cd linux;git clean -fxd > /dev/null;git reset --hard > /dev/null;rm -rf .git;mv ../linux-git .git;cd ..
+    - stage: build rv32 & test
+      script:
+        - BITS=32 ./fast-setup.sh
+        - source source.sh
+    - stage: build rv32 & test
+      script:
+        - BITS=32 ./fast-setup.sh
+        - source source.sh
+        - mkdir build32
+        - cmake ../ -DRISCV32=y
+        - travis_wait 120 make -j3
+        - travis_wait 10 make run-tests
+        - cd ..
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,7 +3,6 @@ project(keystone C)
 include(ProcessorCount)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
-
 ###############################################################################
 ## MACROS
 ###############################################################################
@@ -38,7 +37,7 @@ endif()
 
 set(USE_RUST_SM FALSE CACHE BOOL "Use Rust version of the security monitor.")
 set(SM_CONFIGURE_ARGS --enable-opt=2 CACHE STRING "Security Monitor configure script arguments")
-set(SM_PLATFORM "default" CACHE STRING "Board name for SM hardware-specific functions")
+set(SM_PLATFORM "generic" CACHE STRING "Board name for SM hardware-specific functions")
 set(platform ${SM_PLATFORM})
 message(STATUS "platform=${platform}")
 
@@ -71,8 +70,8 @@ set(bootrom_srcdir ${CMAKE_SOURCE_DIR}/bootrom)
 set(bootrom_wrkdir ${CMAKE_BINARY_DIR}/bootrom.build)
 set(qemu_wrkdir ${CMAKE_SOURCE_DIR}/qemu)
 set(qemu_srcdir ${CMAKE_SOURCE_DIR}/qemu)
-set(sm_srcdir ${CMAKE_SOURCE_DIR}/riscv-pk)
-mkdir(sm_wrkdir ${CMAKE_BINARY_DIR}/riscv-pk.build)
+set(sm_srcdir ${CMAKE_SOURCE_DIR}/sm)
+mkdir(sm_wrkdir ${CMAKE_BINARY_DIR}/sm.build)
 set(buildroot_srcdir ${CMAKE_SOURCE_DIR}/buildroot)
 set(buildroot_wrkdir ${CMAKE_BINARY_DIR}/buildroot.build)
 set(buildroot_config ${confdir}/qemu_riscv${BITS}_virt_defconfig)
@@ -98,11 +97,12 @@ endif()
 set(linux_srcdir ${CMAKE_SOURCE_DIR}/linux)
 mkdir(linux_wrkdir ${CMAKE_BINARY_DIR}/linux.build)
 set(linux_symvers ${linux_wrkdir}/Modules.symvers)
-set(linux_vmlinux ${linux_wrkdir}/vmlinux)
-set(linux_vmlinux_stripped ${linux_wrkdir}/vmlinux-stripped)
+set(linux_image ${linux_wrkdir}/arch/riscv/boot/Image)
 set(driver_srcdir ${CMAKE_SOURCE_DIR}/linux-keystone-driver)
 set(driver_wrkdir ${CMAKE_BINARY_DIR}/linux-keystone-driver.build)
-set(final_image ${CMAKE_BINARY_DIR}/bbl.bin)
+set(fw_elf ${sm_wrkdir}/platform/${platform}/firmware/fw_payload.elf)
+set(fw_bin ${sm_wrkdir}/platform/${platform}/firmware/fw_payload.bin)
+set(final_image ${CMAKE_BINARY_DIR}/firmware.bin)
 set(initramfs_sysroot ${CMAKE_BINARY_DIR}/initramfs-sysroot)
 
 
@@ -111,7 +111,6 @@ set(initramfs_sysroot ${CMAKE_BINARY_DIR}/initramfs-sysroot)
 ###############################################################################
 
 set(qemu_system ${qemu_wrkdir}/riscv${BITS}-softmmu/qemu-system-riscv${BITS})
-add_patch("qemu" "qemu-pmp-bug.patch" ${qemu_srcdir} qemu_patches)
 add_patch("qemu" "qemu-secure-boot.patch" ${qemu_srcdir} qemu_patches)
 
 add_custom_target("qemu" ALL DEPENDS ${qemu_system})
@@ -190,21 +189,19 @@ if(initramfs)
   execute_process(COMMAND id -g OUTPUT_VARIABLE gid)
   string(STRIP ${gid} gid)
   add_custom_command(OUTPUT ${initramfs_sysroot} COMMAND mkdir -p ${initramfs_sysroot})
-  add_custom_command(OUTPUT ${linux_vmlinux_stripped} ${linux_vmlinux} DEPENDS ${initramfs_sysroot} ${linux_srcdir} "linux-symvers" "buildroot" ${buildroot_wrkdir}/images/rootfs.tar
+  add_custom_command(OUTPUT ${linux_image} DEPENDS ${initramfs_sysroot} ${linux_srcdir} "linux-symvers" "buildroot" ${buildroot_wrkdir}/images/rootfs.tar
     COMMAND tar -xpf ${buildroot_wrkdir}/images/rootfs.tar -C ${initramfs_sysroot} --exclude ./dev --exclude ./usr/share/locale
     COMMAND echo "::sysinit:/bin/mount -t devtmpfs devtmpfs /dev" >> ${initramfs_sysroot}/etc/inittab
     COMMAND $(MAKE) -C ${linux_srcdir}
       O=${linux_wrkdir} CONFIG_INITRAMFS_SOURCE="${confdir}/initramfs.txt ${initramfs_sysroot}"
       CONFIG_INITRAMFS_ROOT_UID=${uid} CONFIG_INITRAMFS_ROOT_GID=${gid}
       CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y
-      CROSS_COMPILE=${cross_compile} ARCH=riscv vmlinux
-    COMMAND ${cross_compile}strip -o ${linux_vmlinux_stripped} ${linux_vmlinux}
+      CROSS_COMPILE=${cross_compile} ARCH=riscv
     COMMENT "Building linux (initramfs)"
   )
 else()
-  add_custom_command(OUTPUT ${linux_vmlinux_stripped} ${linux_vmlinux} DEPENDS ${linux_srcdir} "linux-symvers"
-    COMMAND $(MAKE) -C ${linux_srcdir} O=${linux_wrkdir} CROSS_COMPILE=${cross_compile} ARCH=riscv vmlinux
-    COMMAND ${cross_compile}strip -o ${linux_vmlinux_stripped} ${linux_vmlinux}
+  add_custom_command(OUTPUT ${linux_image} DEPENDS ${linux_srcdir} "linux-symvers"
+    COMMAND $(MAKE) -C ${linux_srcdir} O=${linux_wrkdir} CROSS_COMPILE=${cross_compile} ARCH=riscv
     COMMENT "Building linux"
   )
 endif()
@@ -214,7 +211,7 @@ add_custom_command(OUTPUT ${linux_symvers} DEPENDS ${linux_srcdir} "linux-config
   COMMENT "Building linux symvers"
 )
 add_custom_target("linux-symvers" DEPENDS ${linux_symvers})
-add_custom_target("linux" ALL DEPENDS ${linux_vmlinux_stripped} ${linux_vmlinux})
+add_custom_target("linux" ALL DEPENDS ${linux_image})
 
 
 ###############################################################################
@@ -232,38 +229,17 @@ add_custom_target("driver" ALL DEPENDS ${driver_srcdir} ${linux_srcdir} "linux-s
 
 
 ###############################################################################
-## COMPONENT: bbl + sm
+## COMPONENT: security monitor (sm)
 ###############################################################################
 
-if (${USE_RUST_SM})
-  set(enabled_sm --enable-sm_rs)
-else()
-  set(enabled_sm --enable-sm)
-endif()
-
-if(firesim)
-  add_patch("riscv-pk" "riscv-pk.firesim.patch" ${sm_srcdir} sm_patches)
-elseif(sifive)
-  add_patch("riscv-pk" "riscv-pk.sifive.patch" ${sm_srcdir} sm_patches)
-endif()
-
-add_custom_command(OUTPUT ${sm_wrkdir}/Makefile WORKING_DIRECTORY ${sm_wrkdir}
-  DEPENDS ${sm_wrkdir_exists}
-  COMMAND ${sm_srcdir}/configure --host=riscv${BITS}-unknown-linux-gnu --with-payload=${linux_vmlinux_stripped}
-    --enable-logo --with-logo=${confdir}/sifive_logo.txt ${enabled_sm} ${SM_CONFIGURE_ARGS} --with-target-platform=${platform}
-  COMMENT "Configuring sm"
-)
-add_custom_target("sm" ALL DEPENDS ${sm_wrkdir}/Makefile "linux" ${sm_patches} WORKING_DIRECTORY ${sm_wrkdir}
-  COMMAND env CFLAGS='${CFLAGS} -mabi=${ABI} -march=${ISA}' $(MAKE) -C ${sm_wrkdir}
+add_patch("sm/opensbi" "opensbi-firmware-secure-boot.patch" ${sm_srcdir}/opensbi sm_patches)
+add_custom_target("sm" ALL DEPENDS "linux" ${sm_wrkdir_exists} ${sm_patches} WORKING_DIRECTORY ${sm_wrkdir}
+  COMMAND $(MAKE) -C ${sm_srcdir}/opensbi O=${sm_wrkdir} PLATFORM_DIR=${sm_srcdir}/plat/${platform}
+    CROSS_COMPILE=riscv${BITS}-unknown-elf- FW_PAYLOAD_PATH=${linux_image} FW_PAYLOAD=y PLATFORM_RISCV_XLEN=${BITS}
+    PLATFORM_RISCV_ISA=${ISA} PLATFORM_RISCV_ABI=${ABI}
   COMMENT "Building sm"
 )
 
-if(firesim)
-  add_patch("riscv-pk" "riscv-pk.firesim.patch" ${sm_srcdir} "sm")
-elseif(sifive)
-  add_patch("riscv-pk" "riscv-pk.sifive.patch" ${sm_srcdir} "sm")
-endif()
-
 ###############################################################################
 ## COMPONENT: tests
 ###############################################################################
@@ -284,9 +260,9 @@ add_custom_target("image-deps" DEPENDS "tests" "driver" ${overlay_root}
   COMMAND find ${driver_wrkdir} -name "*.ko" -exec cp {} ${overlay_root} \\\\;
 )
 add_custom_target("image" DEPENDS "buildroot" "sm"
-  COMMAND ${cross_compile}objcopy -S -O binary --change-addresses -0x80000000 ${sm_wrkdir}/bbl ${final_image}
   COMMENT "Generating image"
 )
+
 add_dependencies("buildroot" "image-deps")
 
 ###############################################################################
@@ -326,7 +302,7 @@ add_custom_command(OUTPUT ${scripts}/run-qemu.sh DEPENDS ${scripts}
       -nographic \
       -machine virt \
       -bios ${bootrom_wrkdir}/bootrom.bin \
-      -kernel ${sm_wrkdir}/bbl \
+      -kernel ${fw_elf} \
       ${extra_qemu_options} \
       -netdev user,id=net0,net=192.168.100.1/24,dhcpstart=192.168.100.128,hostfwd=tcp::\$\{HOST_PORT\}-:22 \
       -device virtio-net-device,netdev=net0 \

diff --git a/bootrom/bootloader.lds b/bootrom/bootloader.lds
@@ -12,6 +12,7 @@ SECTIONS
     *(.reset)
     *(*)
   }
+  . = ALIGN(4);
   PROVIDE( _dtb = . );
 }
 

diff --git a/buildroot b/buildroot
diff --git a/conf/linux32-defconfig b/conf/linux32-defconfig
@@ -1,5 +1,5 @@
 CONFIG_ARCH_RV32I=y
-CONFIG_CMODEL_MEDLOW=y
+CONFIG_CMODEL_MEDANY=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_IKCONFIG=y

diff --git a/conf/linux64-defconfig b/conf/linux64-defconfig
@@ -3126,3 +3126,23 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_UBSAN is not set
 CONFIG_UBSAN_ALIGNMENT=y
 # end of Kernel hacking
+CONFIG_DEBUG_FS=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_PER_CPU_MAPS=y
+CONFIG_DEBUG_TIMEKEEPING=y
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_SPINLOCK=y
+CONFIG_DEBUG_MUTEXES=y
+CONFIG_DEBUG_RWSEMS=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_STACKTRACE=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+CONFIG_RCU_EQS_DEBUG=y
+CONFIG_DEBUG_BLOCK_EXT_DEVT=y
+CONFIG_DEBUG_INFO=y
+#