support rk3588 smp

Cargo.toml

@@ -15,6 +15,7 @@ fs = ["axstd/fs"]
 [dependencies]
 log = "=0.4.21"
 bitflags = "2.2"
+cfg-if = "1.0"
 spin = "0.9"
 kspin = "0.1"
 lazyinit = "0.2"

Makefile

@@ -8,6 +8,7 @@
 #     - `MODE`: Build mode: release, debug
 #     - `LOG:` Logging level: warn, error, info, debug, trace
 #     - `V`: Verbose level: (empty), 1, 2
+#     - `GICV3`: Enable GICv3 (default is n)
 #     - `TARGET_DIR`: Artifact output directory (cargo target directory)
 #     - `EXTRA_CONFIG`: Extra config specification file
 #     - `OUT_CONFIG`: Final config file that takes effect
@@ -38,6 +39,7 @@ PLATFORM ?=
 SMP ?= 1
 MODE ?= release
 LOG ?= warn
+GICV3 ?= n
 V ?=
 EXTRA_CONFIG ?=
 OUT_CONFIG ?= $(PWD)/.axconfig.toml
@@ -108,6 +110,7 @@ export AX_LOG=$(LOG)
 export AX_TARGET=$(TARGET)
 export AX_IP=$(IP)
 export AX_GW=$(GW)
+export AX_GICV3=$(GICV3)
 
 ifneq ($(filter $(MAKECMDGOALS),unittest unittest_no_fail_fast),)
   # When running unit tests, set `AX_CONFIG_PATH` to empty for dummy config
@@ -152,7 +155,7 @@ ifeq ($(PLAT_NAME), aarch64-raspi4)
   include scripts/make/raspi4.mk
 else ifeq ($(PLAT_NAME), aarch64-bsta1000b-virt-hv)
   include scripts/make/bsta1000b-fada.mk
-else ifeq ($(PLAT_NAME), aarch64-rk3588j)
+else ifeq ($(PLAT_NAME), aarch64-rk3588j-hv)
   include scripts/make/rk3588.mk
 endif
 

configs/platforms/aarch64-qemu-virt-hv.toml

@@ -39,6 +39,7 @@ mmio-regions = [
     [0x0904_0000, 0x1000],      # PL011 UART2
     [0x0910_0000, 0x1000],      # PL031 RTC
     [0x0800_0000, 0x2_0000],    # GICv2
+    [0x080a_0000, 0xf6_0000],   # GICv3 gicr
     [0x0a00_0000, 0x4000],      # VirtIO
     [0x1000_0000, 0x2eff_0000],     # PCI memory ranges (ranges 1: 32-bit MMIO space)
     [0x40_1000_0000, 0x1000_0000],  # PCI config space
@@ -98,6 +99,9 @@ gicc-paddr = 0x0801_0000        # uint
 # GIC Distributor base address
 gicd-paddr = 0x0800_0000        # uint
 
+# GIC Redistributor base address
+gicr-paddr = 0x080a_0000
+
 # PSCI
 psci-method = "smc"             # str
 

configs/platforms/aarch64-rk3588j-hv.toml

@@ -62,6 +62,7 @@ uart-irq = 0x14d # uint
 gicd-paddr = 0xfe600000 # uint
 # GICR Address
 gicc-paddr = 0xfe680000 # uint
+# GICR Address
 gicr-paddr = 0xfe680000 # uint
 
 # PSCI

configs/vms/linux-rk3588-aarch64-smp.toml

@@ -0,0 +1,161 @@
+# Vm base info configs
+#
+[base]
+# Guest vm id.
+id = 1
+# Guest vm name.
+name = "linux"
+# Virtualization type.
+vm_type = 1
+# The number of virtual CPUs.
+cpu_num = 4
+# Guest vm physical cpu sets.
+# phys_cpu_sets = [1, 2, 4, 8, 16, 32, 64, 128]
+phys_cpu_ids = [0x00, 0x100, 0x200, 0x300]
+phys_cpu_sets = [1, 2, 4, 8]
+
+#
+# Vm kernel configs
+#
+[kernel]
+# The entry point of the kernel image.
+entry_point = 0x1008_0000
+# The load address of the kernel image.
+kernel_load_addr = 0x1008_0000
+# The load address of the device tree blob (DTB).
+dtb_load_addr = 0x1000_0000
+# The location of image: "memory" | "fs".
+# load from memory
+image_location = "memory"
+# The file path of the kernel image.
+kernel_path = "/path/to/linux-aarch64.bin"
+# The file path of the device tree blob (DTB).
+dtb_path = "/path/to/dtb"
+
+# load from file system.
+# image_location = "fs".
+## The file path of the kernel image.
+# kernel_path = "linux-arceos-aarch64.bin"
+## The file path of the device tree blob (DTB).
+# dtb_path = "linux-rk3588.dtb"
+
+## The file path of the ramdisk image.
+# ramdisk_path = ""
+## The load address of the ramdisk image.
+# ramdisk_load_addr = 0
+## The path of the disk image.
+# disk_path = "disk.img"
+
+# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`).
+# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`.
+memory_regions = [
+    # [0x0, 0x10_f000, 0x7, 1],        # passthrough uncahed MAP_IDENTICAL
+    [0x940_0000, 0xe6c00000, 0x7, 1], # ram 3G MAP_IDENTICAL
+    # [0x4000_0000, 0x4000_0000, 0x7, 1], # ram 1G MAP_IDENTICAL
+]
+
+#
+# Device specifications
+#
+[devices]
+# Emu_devices.
+# Name Base-Ipa Ipa_len Alloc-Irq Emu-Type EmuConfig.
+emu_devices = []
+
+# Pass-through devices.
+# Name Base-Ipa Base-Pa Length Alloc-Irq.
+passthrough_devices = [
+    [
+        "ramoops",
+        0x11_0000,
+        0x11_0000,
+        0xf_0000,
+        0x17,
+    ],
+    [
+        "sram",
+        0x10_f000,
+        0x10_f000,
+        0x1000,
+        0x17,
+    ],
+    [
+        "gpu",
+        0xfb00_0000,
+        0xfb00_0000,
+        0x20_0000,
+        0x17,
+    ],
+    [
+        "uart8250 UART",
+        0xfd00_0000,
+        0xfd00_0000,
+        0x200_0000,
+        0x17,
+    ],
+    [
+        "usb",
+        0xfc00_0000,
+        0xfc00_0000,
+        0x100_0000,
+        0x17,
+    ],
+    [
+        "uncached",
+        0x0,
+        0x0,
+        0x10_f000,
+        0x17,
+    ],
+    # [
+    #     "gicr",
+    #     0xfe68_0000,
+    #     0xfe68_0000,
+    #     0x10_0000,
+    #     0x1,
+    # ],
+    # [
+    #     "uncached",
+    #     0xf300_0000,
+    #     0xf300_0000,
+    #     0x100_0000,
+    #     0x17,
+    # ],
+    # [
+    #     "uncached",
+    #     0xf400_0000,
+    #     0xf400_0000,
+    #     0x100_0000,
+    #     0x17,
+    # ],
+    # [
+    #     "uncached",
+    #     0xa_4100_0000,
+    #     0xa_4100_0000,
+    #     0x40_0000,
+    #     0x17,
+    # ],
+    # [
+    #     "uncached",
+    #     0xa_40c0_0000,
+    #     0xa_40c0_0000,
+    #     0x40_0000,
+    #     0x17,
+    # ],
+    #     [
+    #     "uncached",
+    #     0x920_0000,
+    #     0x920_0000,
+    #     0x20_0000,
+    #     0x17,
+    # ],
+]
+
+
+#  [0xfe600000, 0x10000], # gic-v3 gicd
+#     [0xfe680000, 0x10_0000], # gic-v3 gicr
+
+    # [0xa41000000, 0x400000],
+    # [0xa40c00000, 0x400000],
+    # [0xf4000000,0x1000000],
+    # [0xf3000000,0x1000000],

scripts/make/features.mk

@@ -48,6 +48,10 @@ ifeq ($(BUS),pci)
   ax_feat += bus-pci
 endif
 
+ifeq ($(GICV3),y)
+   ax_feat += gicv3
+endif
+
 ifeq ($(shell test $(SMP) -gt 1; echo $$?),0)
   lib_feat += smp
 endif

scripts/make/qemu.mk

@@ -24,7 +24,6 @@ qemu_args-riscv64 := \
 
 qemu_args-aarch64 := \
   -cpu cortex-a72 \
-  -machine virt,virtualization=on,gic-version=2 \
   -kernel $(OUT_BIN)
 
 qemu_args-y := -m $(MEM) -smp $(SMP) $(qemu_args-$(ARCH))
@@ -66,7 +65,11 @@ ifeq ($(GRAPHIC), n)
 endif 
 
 ifeq ($(ARCH), aarch64)
-  qemu_args-y += -machine virtualization=on,gic-version=2
+   ifeq ($(GICV3),y)
+     qemu_args-y += -machine virt,virtualization=on,gic-version=3
+   else
+     qemu_args-y += -machine virt,virtualization=on,gic-version=2
+   endif
 endif
 
 ifeq ($(QEMU_LOG), y)

src/main.rs

@@ -11,6 +11,7 @@ extern crate axstd as std;
 mod hal;
 mod logo;
 mod task;
+mod utils;
 mod vmm;
 
 #[unsafe(no_mangle)]

src/utils/arch/aarch64/cache.S

@@ -0,0 +1,27 @@
+// void cache_invalidate_d(u64 start, u64 length);
+.global cache_invalidate_d
+cache_invalidate_d:
+	add x2, x0, x1 /* calculate the end address */
+	bic x0, x0, #(64 - 1) /* align the start with a cache line */
+1:
+	dc ivac, x0 /* invalidate cache to PoC by VA */
+	add x0, x0, #64
+	cmp x0, x2
+	blt 1b
+	mov x0, xzr
+	dsb sy
+	ret
+
+// void cache_clean_invalidate_d(u64 start, u64 length);
+.global cache_clean_invalidate_d
+cache_clean_invalidate_d:
+	add x2, x0, x1 /* calculate the end address */
+	bic x0, x0, #(64 - 1) /* align the start with a cache line */
+1:
+	dc civac, x0 /* invalidate cache to PoC by VA */
+	add x0, x0, #64
+	cmp x0, x2
+	blt 1b
+	mov x0, xzr
+	dsb sy
+	ret

src/utils/arch/aarch64/cache.rs

@@ -0,0 +1,10 @@
+use core::arch::global_asm;
+
+global_asm!(include_str!("cache.S"));
+
+unsafe extern "C" {
+    /// Invalidate the data cache.
+    pub unsafe fn cache_invalidate_d(start: usize, len: usize);
+    /// Clean and invalidate the data cache.
+    pub unsafe fn cache_clean_invalidate_d(start: usize, len: usize);
+}

src/utils/arch/aarch64/mod.rs

@@ -0,0 +1,2 @@
+/// Cache-related operations
+pub mod cache;

src/utils/arch/mod.rs

@@ -0,0 +1,14 @@
+//! Architecture-specific types and operations.
+
+cfg_if::cfg_if! {
+    if #[cfg(target_arch = "x86_64")] {
+        mod x86_64;
+        pub use self::x86_64::*;
+    } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] {
+        mod riscv;
+        pub use self::riscv::*;
+    } else if #[cfg(target_arch = "aarch64")]{
+        mod aarch64;
+        pub use self::aarch64::*;
+    }
+}

src/utils/arch/riscv/mod.rs

@@ -0,0 +1,11 @@
+/// Cache-related operations
+pub mod cache {
+    /// Invalidate the data cache.
+    pub unsafe fn cache_invalidate_d(start: usize, len: usize) {
+        // Dummy implementation
+    }
+    /// Clean and invalidate the data cache.
+    pub unsafe fn cache_clean_invalidate_d(start: usize, len: usize) {
+        // Dummy implementation
+    }
+}

src/utils/arch/x86_64/mod.rs

@@ -0,0 +1,11 @@
+/// Cache-related operations
+pub mod cache {
+    /// Invalidate the data cache.
+    pub unsafe fn cache_invalidate_d(start: usize, len: usize) {
+        // Dummy implementation
+    }
+    /// Clean and invalidate the data cache.
+    pub unsafe fn cache_clean_invalidate_d(start: usize, len: usize) {
+        // Dummy implementation
+    }
+}

src/utils/mod.rs

@@ -0,0 +1,12 @@
+mod arch;
+
+pub mod cache {
+    /// Invalidate the data cache.
+    pub unsafe fn cache_invalidate_d(start: usize, len: usize) {
+        super::arch::cache::cache_invalidate_d(start, len);
+    }
+    /// Clean and invalidate the data cache.
+    pub unsafe fn cache_clean_invalidate_d(start: usize, len: usize) {
+        super::arch::cache::cache_clean_invalidate_d(start, len);
+    }
+}