0856167 Lab 07 08

Makefile

@@ -2,7 +2,7 @@
 # https://github.com/s-matyukevich/raspberry-pi-os/blob/master/src/lesson01/Makefile
 ARMGNU ?= aarch64-linux-gnu
 
-COPS = -Wall -nostdlib -nostartfiles -ffreestanding -Iinclude #-ggdb #-D__DEBUG
+COPS = -Wall -nostdlib -nostartfiles -ffreestanding -Iinclude #-ggdb  #-D__FS_DEBUG #-D__DEBUG #-D__DEBUG_MM #-D__DEBUG_MM_ALLOC #-D__DEBUG_MM_SCHED
 ASMOPS = -Iinclude 
 
 BUILD_DIR = build
@@ -36,7 +36,15 @@ run:
 	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -display none -serial null -serial stdio
 
 debug:
-	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -initrd initramfs.cpio -serial null -display none -S -s 
+	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -initrd initramfs.cpio -serial null -drive if=sd,file=./sdcard/sfn_nctuos.img,format=raw -display none -S -s 
 
 run_cpio:
-	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -display none -serial null -serial stdio -initrd initramfs.cpio
+	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -display none -serial null -serial stdio -initrd initramfs.cpio
+
+# run with sd card (Lab7)
+run_sd:
+	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -display none -serial null -serial stdio -drive if=sd,file=./sdcard/sfn_nctuos.img,format=raw
+
+# run with cpio and sd card
+run_all:
+	qemu-system-aarch64 -M raspi3 -kernel kernel8.img -display none -serial null -serial stdio -initrd initramfs.cpio -drive if=sd,file=./sdcard/sfn_nctuos.img,format=raw

README.md

@@ -1,4 +1,4 @@
-# OSDI 2020 - LAB 06 Virtual File System
+# OSDI 2021 - Lab 08 Virtual Memory
 
 ## Author
 
@@ -7,30 +7,26 @@
 | 0856167    | Yunyung        | 許振揚| yungyung7654321@gmail.com  |
 
 ### Introduction
-A file system manages data in storage mediums. Each file system has a specific way to store and retrieve the data. Hence, a virtual file system(VFS) is common in general-purpose OS to provide a unified interface for all file systems.
+Virtual memory provides isolated address spaces, so each user process can run in its address space without interfering with others.
 
-In this lab, we’ll implement a memory-based file system(tmpfs) to get familiar with the concept of VFS. In the next lab, we’ll implement the FAT32 file system to access files from an SD card. It’s recommended to do both together.
+In this lab, we need to initialize the memory management unit(MMU) and set up the address spaces for the kernel and user processes to achieve process isolation
 
 ### Goals of this lab
-- Understand how to set up a root file system.
+- Understand ARMv8-A virtual memory system architecture.
 
-- Implement temporary file system (tmpfs) as root file system.
+- Understand how to design multitasking with virtual memory.
 
-- Understand how to create, open, close, read, and write files.
+- Understand how to prevent invalid memory access.
 
-- Implement VFS interface, including create, open, close, read, write, mkdir, chdir, ls <directory>, mount/unmount interface.
+- Understand how the kernel manages memory for user processes.
 
-- Implement multi-level VFS and absoute/relative pathname lookup.
+- Implement simple mmap
 
-- Understand how a user process access files through the virtual file system.
+- Understand how demand paging works.
 
-- Implement a user process to access files through VFS.
+- Implement demand paging mechanism.
 
-- Understand how to mount a file system and look up a file across file systems.
-
-- Implement mount/unmount a file system and look up a file across file systems.
-
-- Understand how to design the procfs.
+- Understand how copy-on-write works.
 
 ## Directory structure
 ```
@@ -62,7 +58,12 @@ In this lab, we’ll implement a memory-based file system(tmpfs) to get familiar
 │   ├── wait.h
 │   ├── vfs.h
 │   ├── fs.h
-│   └── tmpfs.h
+│   ├── tmpfs.h
+│   ├── mbr.h           # header file of Master boot Record
+│   ├── sdhost.h    
+│   ├── fat32.h  
+│   ├── base.h  
+│   └── mmu.h
 │
 ├── src                 # source files
 │   ├── command.c       # source file to process command
@@ -76,6 +77,7 @@ In this lab, we’ll implement a memory-based file system(tmpfs) to get familiar
 │   ├── uart.c          # source file to process uart interface and implement uart asynchronous read/write cooperate with shell in shell.c
 │   ├── mm.c            # Implementation of buudy system and object allocator(slab) for memory allocation
 │   ├── mm.S   
+│   ├── utils.c 
 │   ├── utils.S        
 │   ├── entry.S 
 │   ├── exception.c  
@@ -89,8 +91,14 @@ In this lab, we’ll implement a memory-based file system(tmpfs) to get familiar
 │   ├── sched.c
 │   ├── vfs.c 
 │   ├── fs.c
-│   └── tmpfs.c
+│   ├── tmpfs.c
+│   ├── sdhost.c        # Implmentation of sd care device driver
+│   └── fat32.c
 │ 
+├── sdcard              # Files for sd card device (Lab7)
+│   ├── sfn_nctuos.img  # Flash Bootable Image for SD Card with FAT32 file system. FAT32 is Short Filenames(SFN) version.
+│   └── ...             # Other simple files to test Lab7
+│
 ├── rootfs              # files and user programs will be made as cpio archive
 │   └── ...             # any file 
 │
@@ -133,12 +141,13 @@ make clean
 
 ## Run on QEMU
 ```
-make run
-```
-
-## Run on QEMU with cpio
-```
+# Run QEMU with initramfs.cpio and sd card
+make run_all
+# Run QEMU without initramfs.cpio
 make run_cpio
+# Run QEMU without initramfs.cpio and sd card
+make run 
+
 ```
 
 ## How to interact with Rpi3

include/base.h

@@ -1,6 +1,9 @@
 #ifndef	_BASE_H
 #define	_BASE_H
 
-#define MMIO_BASE       0x3F000000
+#include "mm.h"
+
+#define DEVICE_BASE     0x3F000000
+#define MMIO_BASE       (VA_START + DEVICE_BASE)
 
 #endif  /*_P_BASE_H */

include/cpio.h

@@ -12,12 +12,14 @@
 #ifndef _LIB_CPIO_H_
 #define _LIB_CPIO_H_
 
+#include "base.h"
+
 /* Magic identifiers for the "cpio" file format. */
 #define CPIO_HEADER_MAGIC "070701"
 #define CPIO_FOOTER_MAGIC "TRAILER!!!"
 #define CPIO_ALIGNMENT 4
 
-#define INITRAMFS_ADDR 0x8000000
+#define INITRAMFS_ADDR (void *)0x8000000 + VA_START
 
 #ifndef NULL
 #define NULL ((void *)0)
@@ -71,9 +73,9 @@ struct cpio_info {
  *
  * @param[in] archive     The location of the CPIO archive
  * @param[out] filename   ###The name of the file in question.
- * @param[out] _filesize  ###The name of the file in question.
- * @param[out] data       ###The name of the file in question.
- * @param[out] next       ####The name of the file in question.
+ * @param[out] filesize  ###The filesize of the file in question.
+ * @param[out] data       ###The data of the file in question.
+ * @param[out] next       ####The next file in question.
  */
 int cpio_parse_header(struct cpio_header *archive,
         const char **filename, unsigned long *_filesize, void **data,

include/entry.h

@@ -26,7 +26,8 @@
 #define ERROR_INVALID_EL0_32    15 
 
 #define SYNC_ERROR              16 
-#define SYSCALL_ERROR           17 
+#define SYSCALL_ERROR           17
+#define DATA_ABORT_ERROR		18
 
 #ifndef __ASSEMBLER__
 extern void enable_irq_persist();

include/exception.h

@@ -3,7 +3,7 @@
 
 #include <base.h>
 
-#define CORE0_IRQ_SRC       ((volatile unsigned int*)0x40000060)
+#define CORE0_IRQ_SRC       ((volatile unsigned int*)(MMIO_BASE+0x40000060))
 // To identify IRQ interrupt source is from AUX
 // // GPU pending 1 register, see p.115 in BCM2835 ARM Peripherals docs
 #define IRQ_PENDING_1       ((volatile unsigned int*)(MMIO_BASE+0x0000B204))
@@ -12,7 +12,7 @@
 // See p.116 in BCM2835 ARM Peripherals docs
 #define ENABLE_IRQS_1       ((volatile unsigned int*)(MMIO_BASE+0x0000B210))
 // To Disable second level interrupt controller’s IRQs1(0x3f00b210)’s bit29. 
-#define DISABLE_IRQS_1		((volatile unsigned int*)(PBASE+0x0000B21C))
+#define DISABLE_IRQS_1		((volatile unsigned int*)(MMIO_BASE+0x0000B21C))
 // enable/disable/check pending miniUART interrupt bit 29
 // See p.113 ARM peripherals interrupts table in BCM2835 docs
 #define AUX_IRQ (1 << 29) 

include/fat32.h

@@ -0,0 +1,133 @@
+#ifndef _FAT32_H
+#define _FAT32_H
+
+#include "types.h"
+#include "vfs.h"
+#include "printf.h"
+
+#define BLOCK_SIZE                          512
+#define FAT32_ENTRY_SIZE                    4      // FAT table entry size in bytes
+#define FAT32_ENTRY_PER_BLOCK               (BLOCK_SIZE / FAT32_ENTRY_SIZE)
+
+#define FAT32_DIRECTORY_ENTRY_SIZE          4      // Regular Directory Entry Size in bytes
+#define FAT32_DIRECTORY_ENTRY_PER_BLOCK     (BLOCK_SIZE / FAT32_DIRECTORY_ENTRY_SIZE)
+
+#define DIRECTORY_ENTRY_ATTR_DIRECTORY      0x10
+
+#define INVALID_CLUSTER_INDEX               0       // 0 and 1 is not a valid value, We also use it as a error code here. 
+#define FAT32_ERROR -1
+
+// end-of-cluster-chain marker (typically 0x0FFFFFFF or 0x0FFFFFF8 on FAT32)
+// For spec, EOC can be 0x?FFFFFF8 -0x?FFFFFFF. In our lab, EOC is 0x0FFFFFFF 
+#define EOC_FILTER 0x0FFFFFFF 
+
+struct fat32_boot_sector 
+{
+    uint8_t  bootjmp[3];                        // 0x0
+    uint8_t  oem_name[8];                       // 0x3
+
+    // BIOS Parameter Block
+    uint16_t bytes_per_logical_sector;          // 0xB-0xC
+    uint8_t  sector_per_cluster;                // 0xD
+    uint16_t nr_reserved_sectors;               // 0xE-0xF
+    uint8_t  nr_fat_table;                      // 0x10
+    uint16_t nr_max_root_dir_entries_16;        // 0x11-0x12
+    uint16_t nr_logical_sectors_16;             // 0x13-0x14
+    uint8_t  media_descriptor;                  // 0x15
+    uint16_t logical_sector_per_fat_16;         // 0x16-0x17
+    uint16_t physical_sector_per_track;         // 0x18-0x19
+    uint16_t nr_heads;                          // 0x1A-0x1B
+    uint32_t nr_hidden_sectors;                 // 0x1C-0x1F
+    uint32_t nr_sectors_32;                     // 0x20-0x23
+
+    // FAT32 Extended BIOS Parameter Block (EBPB)
+    uint32_t nr_sectors_per_fat_32;             // 0x24-0x27
+    uint16_t mirror_flag;                       // 0x28-0x29
+    uint16_t version;                           // 0x2A-0x2B
+    uint32_t root_dir_start_cluster_num;        // 0x2C-0x2F
+    uint16_t fs_info_sector_num;                // 0x30-0x31
+    uint16_t boot_sector_bak_first_sector_num;  // 0x32-0x33
+    uint32_t reserved[3];                       // 0x34-0x3F
+    uint8_t  physical_drive_num;                // 0x40
+    uint8_t  unused;                            // 0x41
+    uint8_t  extended_boot_signature;           // 0x42
+    uint32_t volume_id;                         // 0x43-0x46
+    uint8_t  volume_label[11];                  // 0x47-0x51
+    uint8_t  fat_system_type[8];                // 0x52-0x59
+} __attribute__((packed));
+
+/**
+ * Important metadata for fat32
+ * In our lab, block idx is equal to sector idx (Because block size equal to sector size here)
+ */
+struct fat32_metadata
+{
+    uint32_t fat_region_blk_idx;    // Fat region block idx
+    uint32_t data_region_blk_idx;   // data region block idx
+    uint32_t rootDir_first_cluster; // block idx of first cluster of root directory, Often this field is set to 2. (Often first cluster number of all cluster)
+    uint8_t  sector_per_cluster;
+    uint32_t nr_fat;
+    uint32_t sector_per_fat;
+};
+
+// Struct of fat32 directory entry. Short Filenames(SFN) version
+struct fat32_dirEnt {
+    uint8_t name[8];            // 0x0-0x7.   File name: 8 ASCII characters, padded with spaces. If the file name starts with 0x00, the previous entry was the last entry.
+    uint8_t ext[3];             // 0x8-0xA.   File extension
+    uint8_t attr;               // 0xB.       Attributes of the file
+    uint8_t reserved;           // 0xC.       Reserved
+    uint8_t create_time[3];     // 0xD-0xF.
+    uint16_t create_date;       // 0x10-0x11.
+    uint16_t last_access_date;  // 0x12-0x13.
+    uint16_t cluster_high;      // 0x14-0x15.
+    uint32_t ext_attr;          // 0x16-0x19. 0x16 time last write,  0x18 date last write                                    
+    uint16_t cluster_low;       // 0x1A-0x1B.
+    uint32_t size;              // 0x1C-0x1F. The size of the file in bytes.
+} __attribute__((packed));
+
+/**
+ * Internal struct for each fat32 vnode (vnode type either directory or regular file)
+ */
+struct fat32_internal
+{
+    uint32_t first_cluster;                    // Idx of first block(sector) of cluster 
+    uint32_t dirEntry_blk_idx;                 // Idx of block(sector) of direcotry entry 
+    uint32_t size;                             // The size of the file in bytes.
+};
+
+
+// For sd card fat32 metadata
+extern struct fat32_metadata fat32_metadata; 
+
+
+int fat32_setup_mount(struct filesystem* fs, struct mount* mount, const char *component_name);
+int fat32_register();
+struct dentry* fat32_create_dentry(struct dentry *parent, const char *name, int type);
+struct vnode* fat32_create_vnode(struct dentry *dentry, int type);
+
+int fat32_lookup(struct vnode *dir_node, struct vnode **target, const char *component_name);
+int fat32_write(struct file *file, const void *buf, size_t len);
+int fat32_read(struct file *file, void *buf, size_t len);
+int fat32_create(struct vnode *dir_node, struct vnode **target, const char *component_name);
+int fat32_mkdir(struct vnode *parent, const char *component_name);
+
+static inline void _dump_fat32_metadata(struct fat32_metadata *fat32_meta)
+{
+    printf("============dump fat32_metadata============\n");
+    printf("fat_region_blk_idx : %u\n",    fat32_meta->fat_region_blk_idx);   // Fat region block idx
+    printf("data_region_blk_idx : %u\n",   fat32_meta->data_region_blk_idx);   // data region block idx
+    printf("rootDir_first_cluster : %u\n",   fat32_meta->rootDir_first_cluster); // block idx of first cluster of root directory
+    printf("sector_per_cluster : %u\n",      fat32_meta->sector_per_cluster);
+    printf("nr_fat : %u\n",                  fat32_meta->nr_fat);
+    printf("sector_per_fat : %u\n",          fat32_meta->sector_per_fat);
+}
+
+static inline void _dump_fat32_internal(struct fat32_internal *internal)
+{
+    printf("============dump fat32_internal============\n");
+    printf("first_cluster : %u\n",                  internal->first_cluster);
+    printf("DirEntry_first_cluster_blk_idx : 0x%x\n", internal->dirEntry_blk_idx);
+    printf("size : %u\n",                           internal->size);
+}
+
+#endif

include/fork.h

@@ -23,6 +23,8 @@ struct pt_regs {
 
 int copy_process(unsigned long clone_flags, unsigned long fn, unsigned long arg, unsigned long stack);
 int move_to_user_mode(unsigned long pc);
+int copy_process_virt(unsigned long clone_flags, unsigned long fn, unsigned long arg); // virtual address
+int move_to_user_mode_virt(unsigned long pc, unsigned long user_start_address);
 struct pt_regs *task_pt_regs(struct task_struct *tsk);
 void _init_files_struct(struct task_struct *tsk);
 long assignPID();

include/fs.h

@@ -1 +1 @@
-extern struct filesystem tmpfs;
+extern struct filesystem tmpfs, fat32;

include/mbr.h

@@ -0,0 +1,31 @@
+#ifndef _MBR_H
+#define _MBR_H
+
+#include "printf.h"
+
+// The struct of partition of Master Boot Record (MBR) for disk
+struct mbr_partition
+{
+    unsigned char status_flag;              //0x0
+    unsigned char partition_begin_head;     //0x1
+    unsigned short partition_begin_sector;  //0x2-0x3
+    unsigned char partition_type;           //0x4
+    unsigned char partition_end_head;       //0x5
+    unsigned short partition_end_sector;    //0x6-0x7
+    unsigned int starting_sector;           //0x8-0xB
+    unsigned int nr_sector;                 //0xC-0xF
+} __attribute__ ((packed));
+
+static inline void dump_mbr_partition(struct mbr_partition *mbr_part)
+{
+    // You can check mbr spec here: 
+    // https://tc.gts3.org/cs3210/2020/spring/r/fat-structs.pdf
+    printf("============Dump MBR Partition============\n");
+    printf("status_flag : 0x%x\n", mbr_part->status_flag);
+    printf("Partition Type : 0x%x\n", mbr_part->partition_type);
+    printf("starting_sector : 0x%x\n", mbr_part->starting_sector);
+    printf("nr_sector : 0x%x\n", mbr_part->nr_sector);
+
+}
+
+#endif /* _MBR_H */