Make embedding more useful

Two major changes:
- Errors (syntax, etc.) longjmp and return to the caller fairly cleanly.
  There seems to be some private bytes growth in normal usage over a
  30min test, but no huge leaks, and no file handle leaks on "normal"
  failure paths.
- Support taking updates directly from the embedder. By accepting the
  contents of a file, the embedder (https://github.com/sgraham/rdy is a
  test) can directly pass from the editor to the compiler rather than
  having to write to and re-read from disk. This is less for efficiency
  than to avoid the messy state that files can be in while the text
  editor is in the middle of (re)writing them, e.g. not there, locked,
  zero byte long, etc. "rdy" hooks directly to rpc change events in
  nvim and passes the updated contents directly.

Author: sgraham

alloc.c

@@ -26,6 +26,7 @@ void __asan_unpoison_memory_region(void const volatile* addr, size_t size);
 #define NUM_BUMP_HEAPS (AL_Link + 1)
 
 UserContext* user_context;
+jmp_buf toplevel_update_jmpbuf;
 CompilerState compiler_state;
 LinkerState linker_state;
 
@@ -41,7 +42,7 @@ static HeapData heap[NUM_BUMP_HEAPS] = {
     {NULL, NULL, 128 << 20},   // AL_Link
 };
 
-// Reports an error and exit.
+// Reports an error and exit update.
 void error(char* fmt, ...) {
   va_list ap;
   va_start(ap, fmt);
@@ -51,7 +52,7 @@ void error(char* fmt, ...) {
     user_context->output_function(2, fmt, ap);
   }
   logerr("\n");
-  exit(1);
+  longjmp(toplevel_update_jmpbuf, 1);
 }
 
 void alloc_init(AllocLifetime lifetime) {
@@ -70,9 +71,14 @@ void alloc_init(AllocLifetime lifetime) {
 void alloc_reset(AllocLifetime lifetime) {
   assert(lifetime < NUM_BUMP_HEAPS);
   HeapData* hd = &heap[lifetime];
-  ASAN_POISON_MEMORY_REGION(hd->base, hd->size);
-  free_executable_memory(hd->base, hd->size);
-  hd->alloc_pointer = NULL;
+  // We allow double resets because we may longjmp out during error handling,
+  // and don't know which heaps are initialized at that point.
+  if (hd->base) {
+    ASAN_POISON_MEMORY_REGION(hd->base, hd->size);
+    free_executable_memory(hd->base, hd->size);
+    hd->alloc_pointer = NULL;
+    hd->base = NULL;
+  }
 }
 
 void* bumpcalloc(size_t num, size_t size, AllocLifetime lifetime) {
@@ -129,7 +135,7 @@ void* allocate_writable_memory(size_t size) {
 #if X64WIN
   void* p = VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
   if (!p) {
-    error("VirtualAlloc failed: 0x%x\n", GetLastError());
+    error("VirtualAlloc of %zu failed: 0x%x\n", size, GetLastError());
   }
   ASAN_UNPOISON_MEMORY_REGION(p, size);
   return p;
@@ -150,7 +156,7 @@ bool make_memory_executable(void* m, size_t size) {
 #if X64WIN
   DWORD old_protect;
   if (!VirtualProtect(m, size, PAGE_EXECUTE_READ, &old_protect)) {
-    error("VirtualProtect failed: 0x%x\n", GetLastError());
+    error("VirtualProtect %p %zu failed: 0x%x\n", m, size, GetLastError());
   }
   return true;
 #else
@@ -166,7 +172,7 @@ void free_executable_memory(void* p, size_t size) {
 #if X64WIN
   (void)size;  // If |size| is passed, free will fail.
   if (!VirtualFree(p, 0, MEM_RELEASE)) {
-    error("VirtualFree failed: 0x%x\n", GetLastError());
+    error("VirtualFree %p %zu failed: 0x%x\n", p, size, GetLastError());
   }
   ASAN_POISON_MEMORY_REGION(p, size);
 #else

dyibicc.h

@@ -4,6 +4,7 @@
 #include <ctype.h>
 #include <errno.h>
 #include <limits.h>
+#include <setjmp.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdint.h>
@@ -173,6 +174,7 @@ File* new_file(char* name, char* contents);
 Token* tokenize_string_literal(Token* tok, Type* basety);
 Token* tokenize(File* file);
 Token* tokenize_file(char* filename);
+Token* tokenize_filecontents(char* path, char* contents);
 
 #define unreachable() error("internal error at %s:%d", __FILE__, __LINE__)
 
@@ -610,7 +612,6 @@ struct Scope {
 typedef struct DyoLinkData {
   char* source_name;
   char* output_dyo_name;
-  int64_t last_compiled_timestamp;
   char* codeseg_base_address;  // Just the address, not a string.
   size_t codeseg_size;
 } DyoLinkData;
@@ -692,10 +693,9 @@ typedef struct CompilerState {
 typedef struct LinkerState {
   // link.c
   HashMap link__runtime_function_map;
-
-  // main.c
 } LinkerState;
 
 extern UserContext* user_context;
+extern jmp_buf toplevel_update_jmpbuf;
 extern CompilerState compiler_state;
 extern LinkerState linker_state;

entry.c

@@ -100,7 +100,7 @@ int main(int argc, char** argv) {
 
   int result = 0;
 
-  if (dyibicc_update(ctx)) {
+  if (dyibicc_update(ctx, NULL, NULL)) {
     if (ctx->entry_point) {
       int myargc = 1;
       char* myargv[] = {"prog", NULL};

libdyibicc.h

@@ -30,6 +30,6 @@ typedef struct DyibiccContext {
 
 DyibiccContext* dyibicc_set_environment(DyibiccEnviromentData* env_data);
 
-bool dyibicc_update(DyibiccContext* context);
+bool dyibicc_update(DyibiccContext* context, char* file, char* contents);
 
 void dyibicc_free(DyibiccContext* context);

main.c

@@ -152,13 +152,6 @@
 #define C(x) compiler_state.main__##x
 #define L(x) linker_state.main__##x
 
-static Token* must_tokenize_file(char* path) {
-  Token* tok = tokenize_file(path);
-  if (!tok)
-    error("%s: %s", path, strerror(errno));
-  return tok;
-}
-
 #if 0  // for -E call after preprocess().
 static void print_tokens(Token* tok) {
   int line = 1;
@@ -384,7 +377,18 @@ static FILE* open_file(char* path) {
   return out;
 }
 
-bool dyibicc_update(DyibiccContext* context) {
+bool dyibicc_update(DyibiccContext* context, char* filename, char* contents) {
+  if (setjmp(toplevel_update_jmpbuf) != 0) {
+    alloc_reset(AL_Compile);
+    alloc_reset(AL_Temp);
+    alloc_reset(AL_Link);
+    memset(&compiler_state, 0, sizeof(compiler_state));
+    memset(&linker_state, 0, sizeof(linker_state));
+    // TODO: files are being left open
+    // TODO: AL_Manual memory or other mallocs may be leaked
+    return false;
+  }
+
   UserContext* ctx = (UserContext*)context;
   bool link_result = true;
 
@@ -395,19 +399,24 @@ bool dyibicc_update(DyibiccContext* context) {
     for (size_t i = 0; i < ctx->num_files; ++i) {
       DyoLinkData* dld = &ctx->files[i];
 
-      {
-        int64_t cur_mtime = stat_single_file(dld->source_name);
-        if (cur_mtime == dld->last_compiled_timestamp) {
-          continue;
-        }
+      if (filename && strcmp(dld->source_name, filename) != 0) {
+        // If a specific update is provided, we only compile that one.
+        continue;
+      }
 
+      {
         alloc_init(AL_Compile);
 
-        // logdbg("%s => %s\n", dld->source_name, dld->output_dyo_name);
-
         init_macros();
         C(base_file) = dld->source_name;
-        Token* tok = must_tokenize_file(C(base_file));
+        Token* tok;
+        if (filename) {
+          tok = tokenize_filecontents(filename, contents);
+        } else {
+          tok = tokenize_file(C(base_file));
+        }
+        if (!tok)
+          error("%s: %s", filename, strerror(errno));
         tok = preprocess(tok);
 
         codegen_init();  // Initializes dynasm so that parse() can assign labels.
@@ -420,7 +429,6 @@ bool dyibicc_update(DyibiccContext* context) {
         codegen(prog, dyo_out);
         fclose(dyo_out);
 
-        dld->last_compiled_timestamp = cur_mtime;
         compiled_any = true;
 
         alloc_reset(AL_Compile);

makefile.win

@@ -6,6 +6,7 @@ CC=@cl
 #CFLAGS=/nologo /Od /Zi /D_CRT_SECURE_NO_DEPRECATE /DX64WIN=1 /W4 /WX
 CFLAGS=/nologo /Ox /GL /Zi /D_CRT_SECURE_NO_DEPRECATE /DX64WIN=1 /W4 /WX
 LDFLAGS=gdi32.lib user32.lib onecore.lib /LTCG /OPT:REF /OPT:ICF
+#LDFLAGS=gdi32.lib user32.lib onecore.lib
 
 #CC=@"C:\Program Files\LLVM\bin\clang-cl.exe"
 #CFLAGS=/nologo /Od /fsanitize=address /Zi /D_CRT_SECURE_NO_DEPRECATE /DX64WIN=1 /W4 /WX -Wno-missing-field-initializers -Wno-switch -Wno-unused-function

tokenize.c

@@ -42,14 +42,14 @@ void error_at(char* loc, char* fmt, ...) {
   va_list ap;
   va_start(ap, fmt);
   verror_at(C(current_file)->name, C(current_file)->contents, line_no, loc, fmt, ap);
-  exit(1);
+  longjmp(toplevel_update_jmpbuf, 1);
 }
 
 void error_tok(Token* tok, char* fmt, ...) {
   va_list ap;
   va_start(ap, fmt);
   verror_at(tok->file->name, tok->file->contents, tok->line_no, tok->loc, fmt, ap);
-  exit(1);
+  longjmp(toplevel_update_jmpbuf, 1);
 }
 
 void warn_tok(Token* tok, char* fmt, ...) {
@@ -763,11 +763,7 @@ static void convert_universal_chars(char* p) {
   *q = '\0';
 }
 
-Token* tokenize_file(char* path) {
-  char* p = read_file(path, AL_Compile);
-  if (!p)
-    return NULL;
-
+Token* tokenize_filecontents(char* path, char* p) {
   // UTF-8 texts may start with a 3-byte "BOM" marker sequence.
   // If exists, just skip them because they are useless bytes.
   // (It is actually not recommended to add BOM markers to UTF-8
@@ -782,3 +778,10 @@ Token* tokenize_file(char* path) {
   File* file = new_file(path, p);
   return tokenize(file);
 }
+
+Token* tokenize_file(char* path) {
+  char* p = read_file(path, AL_Compile);
+  if (!p)
+    return NULL;
+  return tokenize_filecontents(path, p);
+}