Horizon patch 0.2

- Some reorg and general cleanup
- Improved config setup
- My crude attempt at mimicking the HOS scheduler
- Runs Super Mario Odyssey !!

Co-authored-by: Uclydde <julianbraha@gmail.com>

arch/arm64/configs/horizon.config

@@ -0,0 +1,2 @@
+# CONFIG_TRANSPARENT_HUGEPAGE is not set
+CONFIG_HORIZON=y

arch/arm64/include/asm/horizon/unistd.h

@@ -19,7 +19,9 @@
 #define __HNR_exit_thread			0x0A
 #define __HNR_sleep_thread			0x0B
 #define __HNR_get_thread_priority		0x0C
+#define __HNR_set_thread_priority		0x0D
 #define __HNR_set_thread_core_mask		0x0F
+#define __HNR_get_current_processor_number	0x10
 #define __HNR_clear_event			0x12
 #define __HNR_map_shared_memory			0x13
 #define __HNR_unmap_shared_memory		0x14
@@ -54,7 +56,9 @@ __SYSCALL(__HNR_start_thread, hsys_start_thread)
 __SYSCALL(__HNR_exit_thread, hsys_exit_thread)
 __SYSCALL(__HNR_sleep_thread, hsys_sleep_thread)
 __SYSCALL(__HNR_get_thread_priority, hsys_get_thread_priority)
+__SYSCALL(__HNR_set_thread_priority, hsys_set_thread_priority)
 __SYSCALL(__HNR_set_thread_core_mask, hsys_set_thread_core_mask)
+__SYSCALL(__HNR_get_current_processor_number, hsys_get_current_processor_number)
 __SYSCALL(__HNR_clear_event, hsys_clear_event)
 __SYSCALL(__HNR_map_shared_memory, hsys_map_shared_memory)
 __SYSCALL(__HNR_unmap_shared_memory, hsys_unmap_shared_memory)

fs/binfmt_horizon.c

@@ -66,9 +66,13 @@ static int load_horizon_binary(struct linux_binprm * bprm)
 	}
 
 	current->hzn_title_id = hdr->title_id;
+	current->hzn_ideal_core = hdr->ideal_core;
 	current->hzn_address_space_type = hdr->address_space_type;
 	current->hzn_system_resource_size = hdr->system_resource_size;
 
+	if (!set_hzn_priority(current, hdr->main_thread_priority))
+		return -EINVAL;
+
 	/*
 	 * Requires a mmap handler.
 	 */
@@ -125,7 +129,8 @@ static int load_horizon_binary(struct linux_binprm * bprm)
 		(current->mm->start_data =
 		 HORIZON_TXTADDR(hdr) + hdr->codesets[0].segments[0].size);
 	current->mm->brk = 0 +
-		(current->mm->start_brk = HORIZON_TXTADDR(hdr) + total_size);
+		(current->mm->start_brk = HORIZON_TXTADDR(hdr) + total_size +
+		 HZN_ALIAS_REGION_SIZE(current));
 
 	retval = setup_arg_pages(bprm, STACK_TOP, EXSTACK_DEFAULT);
 	if (retval < 0)

fs/exec.c

@@ -68,6 +68,8 @@
 #include <linux/mman.h>
 #include <linux/horizon/handle_table.h>
 
+#include <uapi/linux/sched/types.h>
+
 #include <linux/uaccess.h>
 #include <asm/mmu_context.h>
 #include <asm/tlb.h>
@@ -2168,14 +2170,18 @@ static inline void horizon_pre_exec(void)
 
 	// default values, may be updated by loader
 	current->hzn_title_id = 0;
+	current->hzn_ideal_core = 0;
 	current->hzn_address_space_type = HZN_IS_39_BIT;
 	current->hzn_system_resource_size = 0x1FE00000;
+	current->hzn.priority = 0;
 }
 
 static inline int horizon_post_exec(void)
 {
+	static const struct sched_param sp = { 0 };
 	unsigned long tls_addr;
 	unsigned long start_brk, start_code;
+	int ret;
 
 	current->hzn_thread_handle =
 		hzn_handle_table_add(current->files, get_task_pid(current, PIDTYPE_PID), &hzn_thread_fops);
@@ -2187,6 +2193,13 @@ static inline int horizon_post_exec(void)
 	task_pt_regs(current)->regs[0] = 0;
 	task_pt_regs(current)->regs[1] = current->hzn_thread_handle;
 
+	if (current->hzn_ideal_core >= (int)num_online_cpus())
+		return -EINVAL;
+	if ((ret = set_cpus_allowed_ptr(current, cpumask_of(current->hzn_ideal_core))) < 0)
+		return ret;
+
+	sched_setscheduler_nocheck(current, SCHED_HORIZON, &sp);
+
 	BUG_ON(!current->mm);
 
 	spin_lock(&current->mm->arg_lock);
@@ -2197,11 +2210,11 @@ static inline int horizon_post_exec(void)
 	// alias code at code start
 	current->mm->hzn_alias_code_start = start_code;
 
-	// alias just past heap
-	current->mm->hzn_alias_start = start_brk + HZN_HEAP_REGION_SIZE(current) + PAGE_SIZE;
+	// alias sits before the heap
+	current->mm->hzn_alias_start = start_brk - HZN_ALIAS_REGION_SIZE(current);
 
-	// TLS just past alias
-	tls_addr = vm_mmap(NULL, current->mm->hzn_alias_start + HZN_ALIAS_REGION_SIZE(current) + PAGE_SIZE,
+	// TLS allocated within alias
+	tls_addr = vm_mmap(NULL, current->mm->hzn_alias_start + 0x18c000,
 			   PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0);
 	if (unlikely(IS_ERR_VALUE(tls_addr))) {
 		pr_err("horizon post exec vm_mmap: %ld\n", tls_addr);

include/asm-generic/vmlinux.lds.h

@@ -131,6 +131,7 @@
 	*(__dl_sched_class)			\
 	*(__rt_sched_class)			\
 	*(__fair_sched_class)			\
+	*(__hzn_sched_class)			\
 	*(__idle_sched_class)			\
 	__sched_class_lowest = .;
 

include/linux/horizon.h

@@ -32,6 +32,19 @@ enum hzn_address_space_type {
 #define HZN_STACK_REGION_SIZE(tsk)					\
 	((tsk)->hzn_address_space_type == HZN_IS_39_BIT ? SZ_2G : 0)	\
 
+#define HZN_TLS_AREA_START(tsk) \
+	((tsk)->thread.uw.tp_value)
+#define HZN_TLS_AREA_END(tsk) \
+	(HZN_TLS_AREA_START(tsk) + PAGE_SIZE)
+#define HZN_ALIAS_CODE_REGION_START(tsk) \
+	((tsk)->mm->hzn_alias_code_start)
+#define HZN_ALIAS_CODE_REGION_END(tsk) \
+	(HZN_ALIAS_CODE_REGION_START(tsk) + HZN_ALIAS_CODE_REGION_SIZE(tsk))
+#define HZN_ALIAS_REGION_START(tsk) \
+	((tsk)->mm->hzn_alias_start)
+#define HZN_ALIAS_REGION_END(tsk) \
+	(HZN_ALIAS_REGION_START(tsk) + HZN_ALIAS_REGION_SIZE(tsk))
+
 #define HZN_ADDRESS_SPACE_START 0
 #define HZN_ADDRESS_SPACE_END(tsk) TASK_SIZE_OF(tsk)
 

include/linux/horizon/handle_table.h

@@ -81,6 +81,6 @@ static inline u32 hzn_handle_table_add(struct files_struct *files,
 }
 
 extern const struct file_operations hzn_thread_fops;
-extern const struct file_operations hzn_hzn_session_fops;
+extern const struct file_operations hzn_session_fops;
 
 #endif

include/linux/horizon/types.h

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022
+ * Kent Hall <kjh2166@columbia.edu>
+ */
+
+#ifndef _LINUX_HORIZON_TYPES_H
+#define _LINUX_HORIZON_TYPES_H
+
+enum memory_attribute {
+	HZN_MEMORY_ATTRIBUTE_LOCKED = (1 << 0),
+	HZN_MEMORY_ATTRIBUTE_IPC_LOCKED = (1 << 1),
+	HZN_MEMORY_ATTRIBUTE_DEVICE_SHARED = (1 << 2),
+	HZN_MEMORY_ATTRIBUTE_UNCACHED = (1 << 3),
+};
+
+enum memory_state {
+	HZN_MEMORY_STATE_FREE = 0x00,
+	HZN_MEMORY_STATE_IO = 0x01,
+	HZN_MEMORY_STATE_STATIC = 0x02,
+	HZN_MEMORY_STATE_CODE = 0x03,
+	HZN_MEMORY_STATE_CODE_DATA = 0x04,
+	HZN_MEMORY_STATE_NORMAL = 0x05,
+	HZN_MEMORY_STATE_SHARED = 0x06,
+	HZN_MEMORY_STATE_ALIAS = 0x07,
+	HZN_MEMORY_STATE_ALIAS_CODE = 0x08,
+	HZN_MEMORY_STATE_ALIAS_CODE_DATA = 0x09,
+	HZN_MEMORY_STATE_IPC = 0x0A,
+	HZN_MEMORY_STATE_STACK = 0x0B,
+	HZN_MEMORY_STATE_THREAD_LOCAL = 0x0C,
+	HZN_MEMORY_STATE_TRANSFERRED = 0x0D,
+	HZN_MEMORY_STATE_SHARED_TRANSFERRED = 0x0E,
+	HZN_MEMORY_STATE_SHARED_CODE = 0x0F,
+	HZN_MEMORY_STATE_INACCESSIBLE = 0x10,
+	HZN_MEMORY_STATE_NON_SECURE_IPC = 0x11,
+	HZN_MEMORY_STATE_NON_DEVICE_IPC = 0x12,
+	HZN_MEMORY_STATE_KERNEL = 0x13,
+	HZN_MEMORY_STATE_GENERATED_CODE = 0x14,
+	HZN_MEMORY_STATE_CODE_OUT = 0x15,
+};
+
+enum memory_permission {
+	HZN_MEMORY_PERMISSION_NONE = (0 << 0),
+	HZN_MEMORY_PERMISSION_READ = (1 << 0),
+	HZN_MEMORY_PERMISSION_WRITE = (1 << 1),
+	HZN_MEMORY_PERMISSION_EXECUTE = (1 << 2),
+	HZN_MEMORY_PERMISSION_READ_WRITE = HZN_MEMORY_PERMISSION_READ | HZN_MEMORY_PERMISSION_WRITE,
+	HZN_MEMORY_PERMISSION_READ_EXECUTE = HZN_MEMORY_PERMISSION_READ | HZN_MEMORY_PERMISSION_EXECUTE,
+	HZN_MEMORY_PERMISSION_DONT_CARE = (1 << 28),
+};
+
+struct memory_info {
+	u64 addr;
+	u64 size;
+	enum memory_state state;
+	enum memory_attribute attr;
+	enum memory_permission perm;
+	u32 ipc_refcount;
+	u32 device_refcount;
+	u32 padding;
+};
+
+#endif

include/linux/sched.h

@@ -34,6 +34,7 @@
 #include <linux/rseq.h>
 #include <linux/seqlock.h>
 #include <linux/kcsan.h>
+#include <linux/sched/horizon.h>
 #include <linux/horizon.h>
 #include <asm/kmap_size.h>
 
@@ -661,6 +662,25 @@ struct sched_dl_entity {
 #endif
 };
 
+#ifdef CONFIG_HORIZON
+struct sched_hzn_entity {
+	struct list_head list;
+	int priority;
+	enum hzn_yield_type yield_type;
+	struct rq *rq;
+	volatile long state;
+};
+
+/* Used in hzn_entity->state: */
+#define HZN_FIXED	0L
+#define HZN_SWITCHABLE	1L
+
+#define set_hzn_state(tsk, state_value)				\
+	smp_store_mb((tsk)->hzn.state, (state_value))
+#define set_current_hzn_state(state_value)			\
+	set_hzn_state(current, state_value)
+#endif
+
 #ifdef CONFIG_UCLAMP_TASK
 /* Number of utilization clamp buckets (shorter alias) */
 #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
@@ -778,6 +798,9 @@ struct task_struct {
 	struct sched_entity		se;
 	struct sched_rt_entity		rt;
 	struct sched_dl_entity		dl;
+#ifdef CONFIG_HORIZON
+	struct sched_hzn_entity		hzn;
+#endif
 	const struct sched_class	*sched_class;
 
 #ifdef CONFIG_SCHED_CORE
@@ -972,6 +995,7 @@ struct task_struct {
 		/* used by horizon process */
 		struct {
 			u64				hzn_title_id;
+			u8				hzn_ideal_core;
 			u32				hzn_system_resource_size;
 			enum hzn_address_space_type	hzn_address_space_type;
 			atomic_t			hzn_request_state;
@@ -1915,6 +1939,9 @@ static __always_inline bool is_idle_task(const struct task_struct *p)
 extern struct task_struct *curr_task(int cpu);
 extern void ia64_set_curr_task(int cpu, struct task_struct *p);
 
+#ifdef CONFIG_HORIZON
+void __yield(enum hzn_yield_type type);
+#endif
 void yield(void);
 
 union thread_union {
@@ -2266,10 +2293,6 @@ static inline bool vcpu_is_preempted(int cpu)
 extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
 extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
 
-#ifdef CONFIG_HORIZON
-extern void do_sched_yield(void);
-#endif
-
 #ifndef TASK_SIZE_OF
 #define TASK_SIZE_OF(tsk)	TASK_SIZE
 #endif

include/linux/sched/horizon.h

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022
+ * Kent Hall <kjh2166@columbia.edu>
+ */
+
+#ifndef _LINUX_SCHED_HORIZON_H
+#define _LINUX_SCHED_HORIZON_H
+
+#include <linux/types.h>
+
+#define HZN_LOWEST_THREAD_PRIORITY 63
+#define HZN_HIGHEST_THREAD_PRIORITY 0
+
+enum hzn_yield_type {
+	HZN_YIELD_TYPE_WITHOUT_CORE_MIGRATION = -2,
+	HZN_YIELD_TYPE_WITH_CORE_MIGRATION = -1,
+	HZN_YIELD_TYPE_TO_ANY_THREAD = 0,
+
+	HZN_YIELD_NONE = 1,
+};
+
+struct task_struct;
+
+extern int get_hzn_priority(struct task_struct *p);
+
+extern bool set_hzn_priority(struct task_struct *p, int priority);
+
+#endif

include/uapi/linux/horizon.h

@@ -51,9 +51,11 @@ struct horizon_codeset_hdr {
 struct horizon_hdr {
 	__u32 magic;
 	__u64 title_id;
+	__u8 ideal_core;
 	__u8 is_64bit;
 	__u8 address_space_type;
 	__u32 system_resource_size;
+	__s32 main_thread_priority;
 	__u32 num_codesets;
 	struct horizon_codeset_hdr codesets[];
 };

include/uapi/linux/sched.h

@@ -118,6 +118,7 @@ struct clone_args {
 /* SCHED_ISO: reserved but not implemented yet */
 #define SCHED_IDLE		5
 #define SCHED_DEADLINE		6
+#define SCHED_HORIZON		7
 
 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
 #define SCHED_RESET_ON_FORK     0x40000000

init/init_task.c

@@ -95,6 +95,15 @@ struct task_struct init_task
 		.run_list	= LIST_HEAD_INIT(init_task.rt.run_list),
 		.time_slice	= RR_TIMESLICE,
 	},
+#ifdef CONFIG_HORIZON
+	.hzn		= {
+		.list		= LIST_HEAD_INIT(init_task.hzn.list),
+		.priority	= 0,
+		.yield_type	= HZN_YIELD_NONE,
+		.rq		= NULL,
+		.state		= HZN_FIXED,
+	},
+#endif
 	.tasks		= LIST_HEAD_INIT(init_task.tasks),
 #ifdef CONFIG_SMP
 	.pushable_tasks	= PLIST_NODE_INIT(init_task.pushable_tasks, MAX_PRIO),

kernel/exit.c

@@ -716,7 +716,7 @@ static void exit_notify(struct task_struct *tsk, int group_dead)
 }
 
 #ifdef CONFIG_HORIZON
-static void horizon_exit(void)
+static void horizon_service_exit(void)
 {
 	struct hzn_named_service *iter, *tmp;
 	struct hzn_session_request *req;
@@ -788,7 +788,7 @@ void __noreturn do_exit(long code)
 	WARN_ON(tsk->plug);
 
 #ifdef CONFIG_HORIZON
-	horizon_exit();
+	horizon_service_exit();
 #endif
 
 	kcov_task_exit(tsk);

kernel/fork.c

@@ -1367,7 +1367,12 @@ struct file *get_task_exe_file(struct task_struct *task)
 	struct mm_struct *mm;
 
 	task_lock(task);
+#ifdef CONFIG_HORIZON
+	// so that horizon_servctl may safely switch out the mm
+	mm = task->mm ? task->active_mm : NULL;
+#else
 	mm = task->mm;
+#endif
 	if (mm) {
 		if (!(task->flags & PF_KTHREAD))
 			exe_file = get_mm_exe_file(mm);
@@ -1390,7 +1395,12 @@ struct mm_struct *get_task_mm(struct task_struct *task)
 	struct mm_struct *mm;
 
 	task_lock(task);
+#ifdef CONFIG_HORIZON
+	// so that horizon_servctl may safely switch out the mm
+	mm = task->mm ? task->active_mm : NULL;
+#else
 	mm = task->mm;
+#endif
 	if (mm) {
 		if (task->flags & PF_KTHREAD)
 			mm = NULL;