Merge branch 'fixes'

* fixes:
  Foreign exceptions: basic support
  Added support for `.so' libraries, fixed segfault, small bugs
  OptionParser: optional options and arguments shifting
  XML: workaround for bug in libxml2 2.9.9 (crystal-lang#7477)
  Implement resource owner credentials (crystal-lang#7424)
  Implement #annotations (crystal-lang#7326)
  Handle signals in a separate fiber
  Compiler: reactively compute a union's type, and check for missing types
  Compiler: fix as? casting when target doesn't have a type yet
  Compiler: fix as casting when target doesn't have a type yet
  Compiler: give pare error when assigning a constant inside a multiassign
  Format: fix indent of nested array elements (crystal-lang#7450)
  Disable double write buffering in OpenSSL sockets (crystal-lang#7460)

src/callstack.cr

@@ -33,6 +33,9 @@ struct CallStack
     dir
   end
 
+  # _Unwind_Exception class (UInt64) generated from name of main "library" ("Crystal")
+  UNWIND_EXCEPTION_CLASS = 7165923497316606834_u64 # "Cr}90.cr"
+
   @@skip = [] of String
 
   def self.skip(filename)
@@ -457,4 +460,13 @@ struct CallStack
       end
     end
   end
+
+  def self.decode_exception_class(except_class : UInt64) : String
+    buf = uninitialized UInt8[8]
+    8.times do |i|
+      buf[7 - i] = except_class.to_u8
+      except_class = except_class >> 8
+    end
+    String.new(buf.to_unsafe, buf.size)
+  end
 end

src/callstack/lib_unwind.cr

@@ -111,6 +111,7 @@ lib LibUnwind
 
     fun backtrace = _Unwind_Backtrace((Context, Void*) -> ReasonCode, Void*) : Int32
     fun raise_exception = _Unwind_RaiseException(ucb : ControlBlock*) : ReasonCode
+    fun delete_exception = _Unwind_DeleteException(ucb : ControlBlock*) : Void
     fun vrs_get = _Unwind_VRS_Get(context : Context, regclass : UVRSC, regno : UInt32, representation : UVRSD, valuep : Void*) : UVRSR
     fun vrs_set = _Unwind_VRS_Set(context : Context, regclass : UVRSC, regno : UInt32, representation : UVRSD, valuep : Void*) : UVRSR
     fun __gnu_unwind_frame(ucb : ControlBlock*, context : Context) : ReasonCode
@@ -131,6 +132,7 @@ lib LibUnwind
     fun set_ip = _Unwind_SetIP(context : Context, ip : LibC::SizeT) : LibC::SizeT
     fun set_gr = _Unwind_SetGR(context : Context, index : Int32, value : LibC::SizeT)
     fun raise_exception = _Unwind_RaiseException(ex : Exception*) : ReasonCode
+    fun delete_exception = _Unwind_DeleteException(ex : Exception*) : Void
   {% end %}
 
   {% if flag?(:x86_64) || flag?(:arm) || flag?(:aarch64) %}

src/compiler/crystal/codegen/codegen.cr

@@ -6,15 +6,16 @@ require "../program"
 require "./llvm_builder_helper"
 
 module Crystal
-  MAIN_NAME           = "__crystal_main"
-  RAISE_NAME          = "__crystal_raise"
-  RAISE_OVERFLOW_NAME = "__crystal_raise_overflow"
-  MALLOC_NAME         = "__crystal_malloc64"
-  MALLOC_ATOMIC_NAME  = "__crystal_malloc_atomic64"
-  REALLOC_NAME        = "__crystal_realloc64"
-  GET_EXCEPTION_NAME  = "__crystal_get_exception"
-
   class Program
+    getter(main_name)             { "__#{library_name}_main" }
+    getter(malloc_name)           { "__#{library_name}_malloc64" }
+    getter(malloc_atomic_name)    { "__#{library_name}_malloc_atomic64" }
+    getter(realloc_name)          { "__#{library_name}_realloc64" }
+    getter(raise_name)            { "__#{library_name}_raise" }
+    getter(raise_overflow_name)   { "__#{library_name}_raise_overflow" }
+    getter(get_exception_name)    { "__#{library_name}_get_exception" }
+    getter(delete_exception_name) { "__#{library_name}_delete_exception" }
+
     def run(code, filename = nil, debug = Debug::Default)
       parser = Parser.new(code)
       parser.filename = filename
@@ -26,7 +27,7 @@ module Crystal
 
     def evaluate(node, debug = Debug::Default)
       llvm_mod = codegen(node, single_module: true, debug: debug)[""].mod
-      main = llvm_mod.functions[MAIN_NAME]
+      main = llvm_mod.functions[main_name]
 
       main_return_type = main.return_type
 
@@ -89,6 +90,7 @@ module Crystal
 
     include LLVMBuilderHelper
 
+    getter program : Crystal::Program
     getter llvm_mod : LLVM::Module
     getter builder : CrystalLLVMBuilder
     getter main : LLVM::Function
@@ -158,7 +160,7 @@ module Crystal
       @llvm_id = LLVMId.new(@program)
       @main_ret_type = node.type? || @program.nil_type
       ret_type = @llvm_typer.llvm_return_type(@main_ret_type)
-      @main = @llvm_mod.functions.add(MAIN_NAME, [llvm_context.int32, llvm_context.void_pointer.pointer], ret_type)
+      @main = @llvm_mod.functions.add(program.main_name, [llvm_context.int32, llvm_context.void_pointer.pointer], ret_type)
 
       if @program.has_flag? "windows"
         @personality_name = "__CxxFrameHandler3"
@@ -294,8 +296,11 @@ module Crystal
       end
 
       def visit(node : FunDef)
+        program = @codegen.program
         case node.name
-        when MALLOC_NAME, MALLOC_ATOMIC_NAME, REALLOC_NAME, RAISE_NAME, @codegen.personality_name, GET_EXCEPTION_NAME, RAISE_OVERFLOW_NAME
+        when program.malloc_name, program.malloc_atomic_name, program.realloc_name,
+            program.raise_name, @codegen.personality_name, program.get_exception_name,
+            program.delete_exception_name, program.raise_overflow_name
           @codegen.accept node
         end
         false
@@ -1903,36 +1908,36 @@ module Crystal
     end
 
     def crystal_malloc_fun
-      @malloc_fun ||= @main_mod.functions[MALLOC_NAME]?
+      @malloc_fun ||= @main_mod.functions[@program.malloc_name]?
       if malloc_fun = @malloc_fun
-        check_main_fun MALLOC_NAME, malloc_fun
+        check_main_fun @program.malloc_name, malloc_fun
       else
         nil
       end
     end
 
     def crystal_malloc_atomic_fun
-      @malloc_atomic_fun ||= @main_mod.functions[MALLOC_ATOMIC_NAME]?
+      @malloc_atomic_fun ||= @main_mod.functions[@program.malloc_atomic_name]?
       if malloc_fun = @malloc_atomic_fun
-        check_main_fun MALLOC_ATOMIC_NAME, malloc_fun
+        check_main_fun @program.malloc_atomic_name, malloc_fun
       else
         nil
       end
     end
 
     def crystal_realloc_fun
-      @realloc_fun ||= @main_mod.functions[REALLOC_NAME]?
+      @realloc_fun ||= @main_mod.functions[@program.realloc_name]?
       if realloc_fun = @realloc_fun
-        check_main_fun REALLOC_NAME, realloc_fun
+        check_main_fun @program.realloc_name, realloc_fun
       else
         nil
       end
     end
 
     def crystal_raise_overflow_fun
-      @raise_overflow_fun ||= @main_mod.functions[RAISE_OVERFLOW_NAME]?
+      @raise_overflow_fun ||= @main_mod.functions[@program.raise_overflow_name]?
       if raise_overflow_fun = @raise_overflow_fun
-        check_main_fun RAISE_OVERFLOW_NAME, raise_overflow_fun
+        check_main_fun @program.raise_overflow_name, raise_overflow_fun
       else
         raise "BUG: __crystal_raise_overflow is not defined"
       end

src/compiler/crystal/codegen/debug.cr

@@ -231,7 +231,7 @@ module Crystal
     end
 
     def get_current_debug_scope(location)
-      if context.fun.name == MAIN_NAME
+      if context.fun.name == @program.main_name
         main_scopes = (@main_scopes ||= {} of {String, String} => LibLLVMExt::Metadata)
         file, dir = file_and_dir(location.filename)
         main_scopes[{file, dir}] ||= begin
@@ -268,7 +268,7 @@ module Crystal
     def emit_main_def_debug_metadata(main_fun, filename)
       file, dir = file_and_dir(filename)
       scope = di_builder.create_file(file, dir)
-      fn_metadata = di_builder.create_function(scope, MAIN_NAME, MAIN_NAME, scope,
+      fn_metadata = di_builder.create_function(scope, @program.main_name, @program.main_name, scope,
         0, fun_metadata_type, true, true, 0, LLVM::DIFlags::Zero, false, main_fun)
       fun_metadatas[main_fun] = fn_metadata
     end

src/compiler/crystal/codegen/exception.cr

@@ -145,11 +145,11 @@ class Crystal::CodeGenVisitor
         unwind_ex_obj = extract_value lp, 0
         exception_type_id = extract_value lp, 1
 
-        # We call __crystal_get_exception to get the actual crystal `Exception` object.
-        get_exception_fun = main_fun(GET_EXCEPTION_NAME)
+        get_exception_fun    = main_fun(@program.get_exception_name)
+        delete_exception_fun = main_fun(@program.delete_exception_name)
+        get_exception_fun_args    = [bit_cast(unwind_ex_obj, get_exception_fun.params.first.type)]
+        delete_exception_fun_args = [bit_cast(unwind_ex_obj, delete_exception_fun.params.first.type)]
         set_current_debug_location node if @debug.line_numbers?
-        caught_exception_ptr = call get_exception_fun, [bit_cast(unwind_ex_obj, get_exception_fun.params.first.type)]
-        caught_exception = int2ptr caught_exception_ptr, llvm_typer.type_id_pointer
       end
 
       if node_rescues
@@ -202,10 +202,16 @@ class Crystal::CodeGenVisitor
             if a_rescue_name = a_rescue.name
               context.vars = context.vars.dup
 
+              # We call __crystal_get_exception to get the actual crystal `Exception` object,
+              # or new `ForeignException` object.
+              caught_exception = call(get_exception_fun.not_nil!, get_exception_fun_args.not_nil!) unless windows
               # Cast the caught exception to the type restriction, then assign it
-              cast_caught_exception = cast_to caught_exception, a_rescue.type
+              cast_caught_exception = cast_to caught_exception.not_nil!, a_rescue.type
               var = context.vars[a_rescue_name]
               assign var.pointer, var.type, a_rescue.type, cast_caught_exception
+            else
+              # Delete exception by calling __crystal_delete_exception. Necessary for foreign exceptions.
+              call(delete_exception_fun.not_nil!, delete_exception_fun_args.not_nil!) unless windows
             end
 
             accept a_rescue.body
@@ -283,7 +289,7 @@ class Crystal::CodeGenVisitor
       call windows_throw_fun, [llvm_context.void_pointer.null, llvm_context.void_pointer.null]
       unreachable
     else
-      raise_fun = main_fun(RAISE_NAME)
+      raise_fun = main_fun(@program.raise_name)
       codegen_call_or_invoke(node, nil, nil, raise_fun, [bit_cast(unwind_ex_obj.not_nil!, raise_fun.params.first.type)], true, @program.no_return)
     end
   end

src/compiler/crystal/program.cr

@@ -243,6 +243,8 @@ module Crystal
     # in one branch of an `if` expression.
     getter(nil_var) { Var.new("<nil_var>", nil_type) }
 
+    getter(library_name) { "crystal" }
+
     # Defines a predefined constant in the Crystal module, such as BUILD_DATE and VERSION.
     private def define_crystal_constants
       if build_commit = Crystal::Config.build_commit

src/exception.cr

@@ -69,6 +69,34 @@ class Exception
   end
 end
 
+class ForeignException < Exception
+  def initialize(@unwind_exception : LibUnwind::Exception*, message = "Foreign exception")
+    super(message)
+  end
+
+  def unwind_exception
+    raise "Already reraised" if @unwind_exception.null?
+    @unwind_exception
+  end
+
+  def to_unsafe
+    unwind_exception
+  end
+
+  def reraise : NoReturn
+    exception, @unwind_exception = unwind_exception, Pointer(LibUnwind::Exception).null
+    __crystal_raise(exception)
+  end
+
+  def finalize
+    __crystal_delete_exception(@unwind_exception) unless @unwind_exception.null?
+  end
+
+  def exception_class
+    CallStack.decode_exception_class(unwind_exception.value.exception_class)
+  end
+end
+
 # Raised when the given index is invalid.
 #
 # ```

src/raise.cr

@@ -133,7 +133,8 @@ end
 
           if actions.includes? LibUnwind::Action::HANDLER_FRAME
             __crystal_unwind_set_gr(context, LibUnwind::EH_REGISTER_0, ucb.address.to_u32)
-            __crystal_unwind_set_gr(context, LibUnwind::EH_REGISTER_1, ucb.value.exception_type_id.to_u32)
+            __crystal_unwind_set_gr(context, LibUnwind::EH_REGISTER_1, ucb.value.exception_class == CallStack::UNWIND_EXCEPTION_CLASS ?
+                                    ucb.value.exception_type_id.to_u32 : ForeignException.crystal_instance_type_id)
             __crystal_unwind_set_ip(context, start + cs_addr)
             #puts "install"
             return LibUnwind::ReasonCode::INSTALL_CONTEXT
@@ -151,7 +152,8 @@ end
     ip = LibUnwind.get_ip(context)
     throw_offset = ip - 1 - start
     lsd = LibUnwind.get_language_specific_data(context)
-    #puts "Personality - actions : #{actions}, start: #{start}, ip: #{ip}, throw_offset: #{throw_offset}"
+    return LibUnwind::ReasonCode::CONTINUE_UNWIND if lsd.null? # if no LSDA, then there are no handlers or cleanups
+    #puts "Personality - actions : #{actions}, start: #{start}, ip: #{ip}, throw_offset: #{throw_offset}, exception_class: #{CallStack.decode_exception_class(exception_class)}"
 
     leb = LEBReader.new(lsd)
     leb.read_uint8               # @LPStart encoding
@@ -169,21 +171,24 @@ end
       action = leb.read_uleb128
       #puts "cs_offset: #{cs_offset}, cs_length: #{cs_length}, cs_addr: #{cs_addr}, action: #{action}"
 
-      if cs_addr != 0
-        if cs_offset <= throw_offset && throw_offset <= cs_offset + cs_length
+      break if cs_offset > throw_offset # the table is sorted, so if we've passed the ip, stop
+      if throw_offset < cs_offset + cs_length
+        if cs_addr != 0 # if ip is present, and has a null landing pad, there are no cleanups or handlers to be run
           if actions.includes? LibUnwind::Action::SEARCH_PHASE
             #puts "found"
             return LibUnwind::ReasonCode::HANDLER_FOUND
           end
 
           if actions.includes? LibUnwind::Action::HANDLER_FRAME
             LibUnwind.set_gr(context, LibUnwind::EH_REGISTER_0, exception_object.address)
-            LibUnwind.set_gr(context, LibUnwind::EH_REGISTER_1, exception_object.value.exception_type_id)
+            LibUnwind.set_gr(context, LibUnwind::EH_REGISTER_1, exception_class == CallStack::UNWIND_EXCEPTION_CLASS ?
+                             exception_object.value.exception_type_id : ForeignException.crystal_instance_type_id)
             LibUnwind.set_ip(context, start + cs_addr)
             #puts "install"
             return LibUnwind::ReasonCode::INSTALL_CONTEXT
           end
         end
+        break
       end
     end
 
@@ -203,8 +208,25 @@ end
   end
 
   # :nodoc:
-  fun __crystal_get_exception(unwind_ex : LibUnwind::Exception*) : UInt64
-    unwind_ex.value.exception_object
+  fun __crystal_get_exception(unwind_ex : LibUnwind::Exception*) : Int32*
+    if unwind_ex.value.exception_class == CallStack::UNWIND_EXCEPTION_CLASS
+      Pointer(Int32).new(unwind_ex.value.exception_object)
+    else
+      __get_foreign_exception(unwind_ex)
+    end
+  end
+
+  # :nodoc:
+  fun __crystal_delete_exception(unwind_ex : LibUnwind::Exception*) : Void
+    LibUnwind.delete_exception(unwind_ex)
+  end
+
+  # :nodoc:
+  @[NoInline]
+  private def __get_foreign_exception(unwind_ex : LibUnwind::Exception*)
+    exception = ForeignException.new(unwind_ex)
+    exception.callstack ||= CallStack.new
+    Pointer(Int32).new(exception.object_id)
   end
 
   # Raises the *exception*.
@@ -220,7 +242,7 @@ end
 
     exception.callstack ||= CallStack.new
     unwind_ex = Pointer(LibUnwind::Exception).malloc
-    unwind_ex.value.exception_class = LibC::SizeT.zero
+    unwind_ex.value.exception_class = CallStack::UNWIND_EXCEPTION_CLASS
     unwind_ex.value.exception_cleanup = LibC::SizeT.zero
     unwind_ex.value.exception_object = exception.object_id
     unwind_ex.value.exception_type_id = exception.crystal_type_id