Move lthread changes out

arch/x86_64/pthread_arch.h

@@ -1,7 +1,7 @@
-static inline struct schedctx *__scheduler_self()
+static inline struct pthread *__pthread_self()
 {
-	struct schedctx *self;
-	__asm__ __volatile__ ("mov %%fs:48,%0" : "=r" (self) );
+	struct pthread *self;
+	__asm__ ("mov %%fs:0,%0" : "=r" (self) );
 	return self;
 }
 

include/alltypes.h.in

@@ -47,7 +47,7 @@ TYPEDEF unsigned useconds_t;
 #ifdef __cplusplus
 TYPEDEF unsigned long pthread_t;
 #else
-TYPEDEF struct lthread * pthread_t;
+TYPEDEF struct __pthread * pthread_t;
 #endif
 TYPEDEF int pthread_once_t;
 TYPEDEF unsigned pthread_key_t;

include/elf.h

@@ -1038,6 +1038,8 @@ typedef struct {
 
 #define AT_MINSIGSTKSZ		51
 
+// SGX-LKL specific entries
+#define AT_HW_MODE 100 // a_val == 1 if sgx-lkl running in hw mode
 
 typedef struct {
   Elf32_Word n_namesz;

include/threads.h

@@ -8,7 +8,7 @@
 extern "C" {
 typedef unsigned long thrd_t;
 #else
-typedef struct lthread *thrd_t;
+typedef struct __pthread *thrd_t;
 #define thread_local _Thread_local
 #endif
 

ldso/dynlink.c

@@ -106,7 +106,7 @@ struct symdef {
 
 static struct builtin_tls {
 	char c;
-        struct schedctx pt; /* was: pthread */
+	struct pthread pt;
 	void *space[16];
 } builtin_tls[1];
 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
@@ -1621,6 +1621,10 @@ static void dl_debug_state(void)
 
 weak_alias(dl_debug_state, _dl_debug_state);
 
+void __init_tls(size_t *auxv)
+{
+}
+
 hidden void *__tls_get_new(tls_mod_off_t *v)
 {
 	pthread_t self = __pthread_self();
@@ -1666,23 +1670,23 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
 static void update_tls_size()
 {
 
-if (!sgxlkl_in_sw_debug_mode()) {
-	static int fsgsbase_warn = 0;
-	if (!libc.user_tls_enabled && tls_cnt > 0 && !fsgsbase_warn) {
-		fprintf(stderr, "[    SGX-LKL   ] Warning: The application requires thread-local storage (TLS), but the current system configuration does not allow SGX-LKL to provide full TLS support in hardware mode. See sgx-lkl-run-oe --help-tls for more information.\n");
-		fsgsbase_warn = 1;
+	if (!sgxlkl_in_sw_debug_mode()) {
+		static int fsgsbase_warn = 0;
+		if (!libc.user_tls_enabled && tls_cnt > 0 && !fsgsbase_warn) {
+			fprintf(stderr, "[    SGX-LKL   ] Warning: The application requires thread-local storage (TLS), but the current system configuration does not allow SGX-LKL to provide full TLS support in hardware mode. See sgx-lkl-run-oe --help-tls for more information.\n");
+			fsgsbase_warn = 1;
+		}
 	}
-}
+	
 	libc.tls_cnt = tls_cnt;
 	libc.tls_align = tls_align;
 	libc.tls_size = ALIGN(
 		(1+tls_cnt) * sizeof(void *) +
 		tls_offset +
-		sizeof(struct lthread_tcb_base) +
-		tls_align * 2,
-	tls_align);
+		sizeof(struct pthread) +
+		libc.tls_align * 2,
+	libc.tls_align);
 }
-
 /* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
  * following stage 2 and stage 3 functions via primitive symbolic lookup
  * since it does not have access to their addresses to begin with. */
@@ -1764,14 +1768,17 @@ hidden void *__dls2(unsigned char *base, size_t *sp)
 
 void *__dls2b(size_t *sp)
 {
-//	/* Setup early thread pointer in builtin_tls for ldso/libc itself to
-//	 * use during dynamic linking. If possible it will also serve as the
-//	 * thread pointer at runtime. */
-//	libc.tls_size = sizeof builtin_tls;
-//	libc.tls_align = tls_align;
-//	if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) {
-//		a_crash();
-//	}
+	/* Setup early thread pointer in builtin_tls for ldso/libc itself to
+	 * use during dynamic linking. If possible it will also serve as the
+	 * thread pointer at runtime. */
+	libc.tls_size = sizeof builtin_tls;
+	libc.tls_align = tls_align;
+	// if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) {
+	// 	a_crash();
+	// }
+	/* This is set here so that userspace components of LKL setup can
+	 * create threads */
+	libc.can_do_threads = 1;
 
 	// We don't call __dls3 here. Once we've got here, we're safe enough to
 	// init LKL, after which we can then run stage 3.
@@ -1808,9 +1815,15 @@ prepare_stack_and_jmp_to_exec(void *at_entry, elf64_stack_t *stack, void *tos) {
 
 	tosptr = (char**)tos;
 
-	// provide empty auxv
-	*(tosptr--) = NULL;
-	*(tosptr--) = AT_NULL;
+	// copy auxv
+	base = t = (char **)stack->auxv;
+	while (*t) { t+=2; } // find AT_NULL entry
+	t++;
+	while (t >= base) {
+		*(tosptr--) = *(t--);
+	}
+	*(tosptr--) = (char*) app_entry;
+	*(tosptr--) = (char*) AT_ENTRY;
 
 	// copy envp
 	base = t = stack->envp;
@@ -1840,7 +1853,7 @@ prepare_stack_and_jmp_to_exec(void *at_entry, elf64_stack_t *stack, void *tos) {
 void __dls3(elf64_stack_t *stack, void *tos)
 {
 	static struct dso app, vdso;
-	size_t aux[AUX_CNT], *auxv;
+	void *at_entry;
 	size_t i;
 	char *env_preload=0;
 	char *replace_argv0=0;
@@ -1849,9 +1862,6 @@ void __dls3(elf64_stack_t *stack, void *tos)
 	char **argv = stack->argv;
 	char **envp = stack->envp;
 
-	auxv = libc.auxv;
-	decode_vec(auxv, aux, AUX_CNT);
-
 	/* Only trust user/env if kernel says we're not suid/sgid */
 	if (!libc.secure) {
 		env_path = getenv("LD_LIBRARY_PATH");
@@ -1874,7 +1884,7 @@ void __dls3(elf64_stack_t *stack, void *tos)
 	}
 	close(fd);
 	app.name = argv[0];
-	aux[AT_ENTRY] = (size_t)laddr(&app, ehdr->e_entry);
+	at_entry = laddr(&app, ehdr->e_entry);
 
 	if (app.tls.size) {
 		libc.tls_head = tls_tail = &app.tls;
@@ -1934,32 +1944,29 @@ void __dls3(elf64_stack_t *stack, void *tos)
 	reloc_all(app.next);
 	reloc_all(&app);
 
-	__init_utls(&app.tls);
-
 	update_tls_size();
-	void *initial_tls = calloc(libc.tls_size, 1);
-	if (!initial_tls) {
-		dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
-			argv[0], libc.tls_size);
-		_exit(127);
-	}
-
-    struct lthread *lt = lthread_self();
-    lt->itls = initial_tls;
-    lt->itlssz = libc.tls_size;
-	if (__init_utp(__copy_utls(lt, lt->itls, lt->itlssz), 1) < 0) {
-		a_crash();
-	}
-//	else {
-//		size_t tmp_tls_size = libc.tls_size;
-//		pthread_t self = __pthread_self();
-//		/* Temporarily set the tls size to the full size of
-//		 * builtin_tls so that __copy_tls will use the same layout
-//		 * as it did for before. Then check, just to be safe. */
-//		libc.tls_size = sizeof builtin_tls;
-//		if (__copy_tls((void*)builtin_tls) != self) a_crash();
-//		libc.tls_size = tmp_tls_size;
-//	}
+	// The else logic is commented because it makes assumptions that the
+	// execution of the earlier stages of the linker happened in the same thread.
+	//if (libc.tls_size > sizeof builtin_tls || tls_align > MIN_TLS_ALIGN) {
+		void *initial_tls = calloc(libc.tls_size, 1);
+		if (!initial_tls) {
+			dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
+				argv[0], libc.tls_size);
+			_exit(127);
+		}
+		if (__init_tp(__copy_tls(initial_tls)) < 0) {
+			a_crash();
+		}
+	// } else {
+	// 	size_t tmp_tls_size = libc.tls_size;
+	// 	pthread_t self = __pthread_self();
+	// 	/* Temporarily set the tls size to the full size of
+	// 	 * builtin_tls so that __copy_tls will use the same layout
+	// 	 * as it did for before. Then check, just to be safe. */
+	// 	libc.tls_size = sizeof builtin_tls;
+	// 	if (__copy_tls((void*)builtin_tls) != self) a_crash();
+	// 	libc.tls_size = tmp_tls_size;
+	// }
 	static_tls_cnt = tls_cnt;
 
 	do_init_fini(&app);
@@ -1995,11 +2002,11 @@ void __dls3(elf64_stack_t *stack, void *tos)
 	// Set thread name
 	char * app_name = strrchr(app.name, '/');
 	if (app_name) app_name++;
-	lthread_set_funcname(lthread_self(), app_name ? app_name : app.name);
+	pthread_setname_np(__pthread_self(), app_name ? app_name : app.name);
 
 	errno = 0;
 
-	prepare_stack_and_jmp_to_exec((void *)aux[AT_ENTRY], stack, tos);
+	prepare_stack_and_jmp_to_exec(at_entry, stack, tos);
 }
 
 static void prepare_lazy(struct dso *p)

src/aio/aio.c

@@ -183,7 +183,7 @@ static void cleanup(void *ctx)
 		__syscall(SYS_rt_sigqueueinfo, si.si_pid, si.si_signo, &si);
 	}
 	if (sev.sigev_notify == SIGEV_THREAD) {
-		//a_store(&__pthread_self()->cancel, 0);
+		a_store(&__pthread_self()->cancel, 0);
 		sev.sigev_notify_function(sev.sigev_value);
 	}
 }