Kernel/x86: Bake the Prekernel and the Kernel into one image

Kernel/Arch/init.cpp

@@ -99,9 +99,6 @@ extern "C" USB::DriverInitFunction driver_init_table_end[];
 
 extern "C" u8 end_of_kernel_image[];
 
-multiboot_module_entry_t multiboot_copy_boot_modules_array[16];
-size_t multiboot_copy_boot_modules_count;
-
 READONLY_AFTER_INIT SetOnce g_not_in_early_boot;
 
 namespace Kernel {
@@ -152,14 +149,14 @@ READONLY_AFTER_INIT StringView kernel_cmdline;
 READONLY_AFTER_INIT u32 multiboot_flags;
 READONLY_AFTER_INIT multiboot_memory_map_t* multiboot_memory_map;
 READONLY_AFTER_INIT size_t multiboot_memory_map_count;
-READONLY_AFTER_INIT multiboot_module_entry_t* multiboot_modules;
-READONLY_AFTER_INIT size_t multiboot_modules_count;
 READONLY_AFTER_INIT PhysicalAddress multiboot_framebuffer_addr;
 READONLY_AFTER_INIT u32 multiboot_framebuffer_pitch;
 READONLY_AFTER_INIT u32 multiboot_framebuffer_width;
 READONLY_AFTER_INIT u32 multiboot_framebuffer_height;
 READONLY_AFTER_INIT u8 multiboot_framebuffer_bpp;
 READONLY_AFTER_INIT u8 multiboot_framebuffer_type;
+READONLY_AFTER_INIT PhysicalAddress multiboot_module_physical_ptr;
+READONLY_AFTER_INIT size_t multiboot_module_length;
 }
 
 Atomic<Graphics::Console*> g_boot_console;
@@ -188,8 +185,8 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT NO_SANITIZE_COVERAGE void init([[maybe_
     multiboot_flags = boot_info.multiboot_flags;
     multiboot_memory_map = (multiboot_memory_map_t*)boot_info.multiboot_memory_map;
     multiboot_memory_map_count = boot_info.multiboot_memory_map_count;
-    multiboot_modules = (multiboot_module_entry_t*)boot_info.multiboot_modules;
-    multiboot_modules_count = boot_info.multiboot_modules_count;
+    multiboot_module_physical_ptr = PhysicalAddress { boot_info.multiboot_module_physical_ptr };
+    multiboot_module_length = boot_info.multiboot_module_length;
     multiboot_framebuffer_addr = PhysicalAddress { boot_info.multiboot_framebuffer_addr };
     multiboot_framebuffer_pitch = boot_info.multiboot_framebuffer_pitch;
     multiboot_framebuffer_width = boot_info.multiboot_framebuffer_width;
@@ -218,8 +215,7 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT NO_SANITIZE_COVERAGE void init([[maybe_
     multiboot_memory_map = mmap;
     multiboot_memory_map_count = 2;
 
-    multiboot_modules = nullptr;
-    multiboot_modules_count = 0;
+    multiboot_module_length = 0;
     // FIXME: Read the /chosen/bootargs property.
     kernel_cmdline = RPi::Mailbox::the().query_kernel_command_line(s_command_line_buffer);
 #elif ARCH(RISCV64)
@@ -235,11 +231,6 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT NO_SANITIZE_COVERAGE void init([[maybe_
     // We need to copy the command line before kmalloc is initialized,
     // as it may overwrite parts of multiboot!
     CommandLine::early_initialize(kernel_cmdline);
-    if (multiboot_modules_count > 0) {
-        VERIFY(multiboot_modules);
-        memcpy(multiboot_copy_boot_modules_array, multiboot_modules, multiboot_modules_count * sizeof(multiboot_module_entry_t));
-    }
-    multiboot_copy_boot_modules_count = multiboot_modules_count;
 
     new (&bsp_processor()) Processor();
     bsp_processor().early_initialize(0);

Kernel/Boot/BootInfo.h

@@ -33,8 +33,8 @@ extern "C" StringView kernel_cmdline;
 extern "C" u32 multiboot_flags;
 extern "C" multiboot_memory_map_t* multiboot_memory_map;
 extern "C" size_t multiboot_memory_map_count;
-extern "C" multiboot_module_entry_t* multiboot_modules;
-extern "C" size_t multiboot_modules_count;
+extern "C" PhysicalAddress multiboot_module_physical_ptr;
+extern "C" size_t multiboot_module_length;
 extern "C" PhysicalAddress multiboot_framebuffer_addr;
 extern "C" u32 multiboot_framebuffer_pitch;
 extern "C" u32 multiboot_framebuffer_width;

Kernel/CMakeLists.txt

@@ -848,17 +848,40 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang$")
     target_link_libraries(Kernel PRIVATE kernel_heap clang_rt.builtins)
 endif()
 
-add_custom_command(
+if ("${SERENITY_ARCH}" STREQUAL "x86_64")
+    set(KERNEL_ELF_OBJCOPY_TARGET "elf64-x86-64")
+endif()
+
+# In x86_64 the Kernel is linked to the Pre-kernel binary.
+if ("${SERENITY_ARCH}" STREQUAL "x86_64")
+    add_custom_command(
+        TARGET Kernel POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E env NM=${CMAKE_NM} sh ${CMAKE_CURRENT_SOURCE_DIR}/mkmap.sh
+        COMMAND ${CMAKE_COMMAND} -E env OBJCOPY=${CMAKE_OBJCOPY} sh ${CMAKE_CURRENT_SOURCE_DIR}/embedmap.sh
+        COMMAND ${CMAKE_OBJCOPY} --only-keep-debug Kernel Kernel.debug
+        COMMAND ${CMAKE_OBJCOPY} --strip-debug Kernel
+        COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=Kernel.debug Kernel
+        COMMAND ${CMAKE_OBJCOPY} --set-section-flags .heap=load Kernel Kernel_shared_object
+        COMMAND ${CMAKE_OBJCOPY} -I binary -O ${KERNEL_ELF_OBJCOPY_TARGET} --rename-section .data=.Kernel_image --set-section-alignment .Kernel_image=4096 Kernel_shared_object Kernel.o
+        BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/Kernel.o ${CMAKE_CURRENT_BINARY_DIR}/kernel.map
+    )
+elseif ("${SERENITY_ARCH}" STREQUAL "aarch64" OR "${SERENITY_ARCH}" STREQUAL "riscv64")
+    add_custom_command(
         TARGET Kernel POST_BUILD
         COMMAND "${CMAKE_COMMAND}" -E env NM=${CMAKE_NM} sh ${CMAKE_CURRENT_SOURCE_DIR}/mkmap.sh
         COMMAND "${CMAKE_COMMAND}" -E env OBJCOPY=${CMAKE_OBJCOPY} sh ${CMAKE_CURRENT_SOURCE_DIR}/embedmap.sh
         COMMAND ${CMAKE_OBJCOPY} --only-keep-debug Kernel Kernel.debug
         COMMAND ${CMAKE_OBJCOPY} --strip-debug Kernel
         COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=Kernel.debug Kernel
         BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/kernel.map
-)
+    )
+endif()
 
+# Architectures (x86_64, aarch64, riscv64) share the same location in their respective architecture build folder.
+# In x86_64 the Kernel is linked to the Pre-kernel binary and then generates the result Kernel binary.
+# In aarch64 and riscv64 there's no Pre-kernel stage yet, so we immediately get a Kernel binary.
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/Kernel" DESTINATION boot)
+
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/Kernel.debug" DESTINATION boot)
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/kernel.map" DESTINATION res)
 

Kernel/Memory/MemoryManager.cpp

@@ -570,13 +570,10 @@ UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_fdt(MemoryManager::GlobalD
 UNMAP_AFTER_INIT void MemoryManager::parse_memory_map_multiboot(MemoryManager::GlobalData& global_data)
 {
     // Register used memory regions that we know of.
-    if (multiboot_flags & 0x4) {
-        auto* bootmods_start = multiboot_copy_boot_modules_array;
-        auto* bootmods_end = bootmods_start + multiboot_copy_boot_modules_count;
-
-        for (auto* bootmod = bootmods_start; bootmod < bootmods_end; bootmod++) {
-            global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress(bootmod->start), PhysicalAddress(bootmod->end) });
-        }
+    if (multiboot_flags & 0x4 && !multiboot_module_physical_ptr.is_null()) {
+        dmesgln("MM: Multiboot module @ {}, length={}", multiboot_module_physical_ptr, multiboot_module_length);
+        VERIFY(multiboot_module_length != 0);
+        global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, multiboot_module_physical_ptr, multiboot_module_physical_ptr.offset(multiboot_module_length) });
     }
 
     auto* mmap_begin = multiboot_memory_map;

Kernel/Prekernel/CMakeLists.txt

@@ -8,14 +8,21 @@ set(SOURCES
     )
 
 if ("${SERENITY_ARCH}" STREQUAL "x86_64")
-    set(PREKERNEL_TARGET Prekernel64)
+    set(PREKERNEL_TARGET kernel_x86-64)
 elseif("${SERENITY_ARCH}" STREQUAL "aarch64")
     message(SEND_ERROR "Prekernel is not needed on aarch64 and should not be compiled!")
 endif()
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static")
 
-add_executable(${PREKERNEL_TARGET} ${SOURCES})
+add_library(KernelObject OBJECT IMPORTED)
+
+set_property(TARGET KernelObject PROPERTY
+    IMPORTED_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/../Kernel.o
+)
+
+add_executable(${PREKERNEL_TARGET} ${SOURCES} $<TARGET_OBJECTS:KernelObject>)
+add_dependencies(${PREKERNEL_TARGET} Kernel)
 add_dependencies(${PREKERNEL_TARGET} install_libc_headers)
 target_compile_options(${PREKERNEL_TARGET} PRIVATE -no-pie -fno-pic -fno-threadsafe-statics)
 
@@ -31,11 +38,11 @@ endif()
 
 add_custom_command(
     TARGET ${PREKERNEL_TARGET} POST_BUILD
-    COMMAND ${CMAKE_OBJCOPY} -O elf32-i386 ${CMAKE_CURRENT_BINARY_DIR}/${PREKERNEL_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/Prekernel
+    COMMAND ${CMAKE_OBJCOPY} -O elf32-i386 ${CMAKE_CURRENT_BINARY_DIR}/${PREKERNEL_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/../Kernel
     BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/Prekernel
 )
 
-install(FILES "${CMAKE_CURRENT_BINARY_DIR}/Prekernel" DESTINATION boot)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/../Kernel" DESTINATION boot)
 
 # Remove options which the Prekernel environment doesn't support.
 get_target_property(PREKERNEL_TARGET_OPTIONS ${PREKERNEL_TARGET} COMPILE_OPTIONS)

Kernel/Prekernel/Prekernel.h

@@ -40,8 +40,8 @@ struct [[gnu::packed]] BootInfo {
     u32 multiboot_flags;
     u64 multiboot_memory_map;
     u32 multiboot_memory_map_count;
-    u64 multiboot_modules;
-    u32 multiboot_modules_count;
+    u64 multiboot_module_physical_ptr;
+    u32 multiboot_module_length;
     u64 multiboot_framebuffer_addr;
     u32 multiboot_framebuffer_pitch;
     u32 multiboot_framebuffer_width;

Kernel/Prekernel/init.cpp

@@ -27,6 +27,9 @@ extern "C" [[noreturn]] void __stack_chk_fail();
 extern "C" u8 start_of_prekernel_image[];
 extern "C" u8 end_of_prekernel_image[];
 
+extern "C" u8 _binary_Kernel_standalone_start[];
+extern "C" u8 end_of_prekernel_image_after_kernel_image[];
+
 extern "C" u8 gdt64ptr[];
 extern "C" u16 code64_sel;
 extern "C" u64 boot_pml4t[512];
@@ -92,21 +95,29 @@ static void memmove_virt(void* dest_virt, FlatPtr dest_phys, void* src, size_t n
 
 extern "C" [[noreturn]] void init()
 {
-    if (multiboot_info_ptr->mods_count < 1)
-        halt();
+    u32 initrd_module_start = 0;
+    u32 initrd_module_end = 0;
+    if (multiboot_info_ptr->mods_count > 0) {
+        // We only consider the first specified multiboot module, and ignore
+        // the rest of the modules.
+        multiboot_module_entry_t* initrd_module = (multiboot_module_entry_t*)(FlatPtr)multiboot_info_ptr->mods_addr;
+        if (initrd_module->start > initrd_module->end)
+            halt();
 
-    multiboot_module_entry_t* kernel_module = (multiboot_module_entry_t*)(FlatPtr)multiboot_info_ptr->mods_addr;
+        initrd_module_start = initrd_module->start;
+        initrd_module_end = initrd_module->end;
+    }
 
-    u8* kernel_image = (u8*)(FlatPtr)kernel_module->start;
+    u8* kernel_image = _binary_Kernel_standalone_start;
     // copy the ELF header and program headers because we might end up overwriting them
     Elf_Ehdr kernel_elf_header = *(Elf_Ehdr*)kernel_image;
     Elf_Phdr kernel_program_headers[16];
     if (kernel_elf_header.e_phnum > array_size(kernel_program_headers))
         halt();
     __builtin_memcpy(kernel_program_headers, kernel_image + kernel_elf_header.e_phoff, sizeof(Elf_Phdr) * kernel_elf_header.e_phnum);
 
-    FlatPtr kernel_physical_base = 0x200000;
-    FlatPtr default_kernel_load_base = KERNEL_MAPPING_BASE + 0x200000;
+    FlatPtr kernel_physical_base = (FlatPtr)kernel_image;
+    FlatPtr default_kernel_load_base = KERNEL_MAPPING_BASE + kernel_physical_base;
 
     FlatPtr kernel_load_base = default_kernel_load_base;
 
@@ -179,18 +190,19 @@ extern "C" [[noreturn]] void init()
     // overwriting mbi end as to avoid to check whether it's mapped after reloading page tables.
     BootInfo info {};
 
-    multiboot_info_ptr->mods_count--;
-    multiboot_info_ptr->mods_addr += sizeof(multiboot_module_entry_t);
-
     auto adjust_by_mapping_base = [kernel_mapping_base](auto ptr) {
         return (decltype(ptr))((FlatPtr)ptr + kernel_mapping_base);
     };
 
     info.multiboot_flags = multiboot_info_ptr->flags;
     info.multiboot_memory_map = adjust_by_mapping_base((FlatPtr)multiboot_info_ptr->mmap_addr);
     info.multiboot_memory_map_count = multiboot_info_ptr->mmap_length / sizeof(multiboot_memory_map_t);
-    info.multiboot_modules = adjust_by_mapping_base((FlatPtr)multiboot_info_ptr->mods_addr);
-    info.multiboot_modules_count = multiboot_info_ptr->mods_count;
+
+    if (initrd_module_start != 0 && initrd_module_end != 0) {
+        info.multiboot_module_physical_ptr = initrd_module_start;
+        info.multiboot_module_length = initrd_module_end - initrd_module_start;
+    }
+
     if ((multiboot_info_ptr->flags & MULTIBOOT_INFO_FRAMEBUFFER_INFO) != 0) {
         info.multiboot_framebuffer_addr = multiboot_info_ptr->framebuffer_addr;
         info.multiboot_framebuffer_pitch = multiboot_info_ptr->framebuffer_pitch;

Kernel/Prekernel/linker.ld

@@ -44,4 +44,13 @@ SECTIONS
     } :bss
 
     end_of_prekernel_image = .;
+
+    .Kernel_image ALIGN(4K) : AT (ADDR(.Kernel_image))
+    {
+        _binary_Kernel_standalone_start = .;
+        KEEP(*(.Kernel_image))
+    }
+
+    end_of_prekernel_image_after_kernel_image = .;
+
 }

Meta/debug-kernel.sh

@@ -64,7 +64,7 @@ exec $SERENITY_KERNEL_DEBUGGER \
     -ex "file $SCRIPT_DIR/../Build/${SERENITY_ARCH:-x86_64}$toolchain_suffix/Kernel/Prekernel/$prekernel_image" \
     -ex "set confirm off" \
     -ex "directory $SCRIPT_DIR/../Build/${SERENITY_ARCH:-x86_64}$toolchain_suffix/" \
-    -ex "add-symbol-file $SCRIPT_DIR/../Build/${SERENITY_ARCH:-x86_64}$toolchain_suffix/Kernel/Kernel -o $kernel_base" \
+    -ex "add-symbol-file $SCRIPT_DIR/../Build/${SERENITY_ARCH:-x86_64}$toolchain_suffix/Kernel/Kernel_shared_object -o $kernel_base" \
     -ex "set confirm on" \
     -ex "set arch $gdb_arch" \
     -ex "set print frame-arguments none" \

Meta/run.py

@@ -692,7 +692,7 @@ def set_up_kernel(config: Configuration):
     elif config.architecture == Arch.RISCV64:
         config.kernel_and_initrd_arguments = ["-kernel", "Kernel/Kernel.bin"]
     elif config.architecture == Arch.x86_64:
-        config.kernel_and_initrd_arguments = ["-kernel", "Kernel/Prekernel/Prekernel", "-initrd", "Kernel/Kernel"]
+        config.kernel_and_initrd_arguments = ["-kernel", "Kernel/Kernel"]
 
 
 def set_up_machine_devices(config: Configuration):