Umbrella commit switching to delayed reference counting (still

experimental work in progress).

This is a larger overhaul of HILTI's runtime memory management. Rather
than generally keeping all reference counts up to date all the time
(which involves a lot of pretty much uncessary operations), we now
enforce correct reference counts only during explicit "safepoints". In
between safepoints, HILTI is free to skip counting. For now, we
trigger safepoints upon entry/exit into HILTI (either from/to C, or
from/to the thread scheduler), which means that we can generally omit
all reference counting operations for objects that don't persist
across these boundaries, including for everything on the stack.

Internally this is achieved by maintaining "nullbuffers" that record
objects that need to have their reference counts checked for being
zero at the next safepoint; and by adding extra code to adjust all
live variables when safepoints may trigger. (Currently, that's still a
bit inefficient, but should work well once we can leverage LLVM's
stackmaps in the future).

With this change comes a bit more baked in:

    - Calling conventations have changed to return values at +0
    (instead of +1). With the delayed reference counting, even freshly
    created heap objects can be returned without increased reference
    counts (but are put into the nullbuffer immediately). An attribute
    &ref can be used for HILTI-C function that do want to return at
    +1.

    - Handling of attributes have been unified inside the HILTI
    compiler. Also moving regexp's attributes from type information to
    the values.

    - Heap values can now be "hoisted" to the stack if they don't need
    to persist longer that the stack's lifetime. That avoids having to
    allocate them dynamically. Currently that's triggered with a
    &hoist attribute and works for bytes only; however there's no
    correctness checks in place and must be used only if it satisfies
    all the constraints. Eventually HILTI will learn to identify the
    right situations and apply &hoist itself as suitable.

    - hilti-build now disables debugging when optimizions turned on.
    Tests turn on debugging by default now.

    - Starting to port HILTI to LLVM 3.5-to-be (current master).

This is all still experimental (but passes the test-suite).

Makefile

@@ -16,5 +16,9 @@ clean:
 	rm -rf build
 	( cd doc && $(MAKE) clean )
 
+test:
+	( cd tests && btest -j -f diag.log )
+	( cd bro/tests && btest -j -f diag.log )
+
 tags:
 	update-tags

binpac/codegen/codegen.cc

@@ -318,19 +318,21 @@ shared_ptr<hilti::declaration::Function> CodeGen::hiltiDefineFunction(shared_ptr
         internalError("unexpected calling convention in hiltiDefineFunction()");
     }
 
+    hilti::builder::attribute_set attrs;
+
     // TODO: Do we always want to export these?
     if ( ! declare_only )
         moduleBuilder()->exportID(name);
 
     if ( func->body() && ! declare_only ) {
-        auto decl = moduleBuilder()->pushFunction(name, result, params, cc, nullptr, hilti::builder::function::attributes(), false, func->location());
+        auto decl = moduleBuilder()->pushFunction(name, result, params, cc, attrs, false, func->location());
         hiltiStatement(func->body());
         moduleBuilder()->popFunction();
         return decl;
     }
 
     else
-        return moduleBuilder()->declareFunction(name, result, params, cc, func->location());
+        return moduleBuilder()->declareFunction(name, result, params, cc, attrs, func->location());
 }
 
 shared_ptr<hilti::ID> CodeGen::hiltiFunctionName(shared_ptr<binpac::Function> func, const string& scope)

binpac/codegen/operators/regexp.cc

@@ -10,8 +10,8 @@ void CodeBuilder::visit(ctor::RegExp* r)
     hilti::builder::regexp::re_pattern_list patterns;
 
     for ( auto p : r->patterns() )
-        patterns.push_back(std::make_tuple(p.first, p.second));
+        patterns.push_back(p);
 
-    auto result = hilti::builder::regexp::create(patterns, r->location());
+    auto result = hilti::builder::regexp::create(patterns, hilti::AttributeSet(), r->location());
     setResult(result);
 }

binpac/codegen/parser-builder.cc

@@ -835,7 +835,7 @@ shared_ptr<hilti::Type> ParserBuilder::hiltiTypeParseObject(shared_ptr<type::Uni
         auto sfield = hilti::builder::struct_::field(name, type, nullptr, false, f->location());
 
         if ( f->anonymous() )
-            sfield->setCanRemove();
+            sfield->type()->attributes().add(hilti::attribute::CANREMOVE);
 
         have.insert(name);
         fields.push_back(sfield);
@@ -909,7 +909,7 @@ shared_ptr<hilti::Type> ParserBuilder::hiltiTypeParseObject(shared_ptr<type::Uni
 
     for ( auto f : fields ) {
         if ( util::startsWith(f->id()->name(), "__") )
-            f->setCanRemove();
+            f->type()->attributes().add(hilti::attribute::CANREMOVE);
     }
 
     auto s = hilti::builder::struct_::type(fields, u->location());
@@ -1346,7 +1346,7 @@ void ParserBuilder::_hiltiDefineHook(shared_ptr<ID> id, shared_ptr<type::unit::I
     else
         rtype = hilti::builder::void_::type();
 
-    cg()->moduleBuilder()->pushHook(name, hilti::builder::function::result(rtype), p, nullptr, priority, 0, false);
+    cg()->moduleBuilder()->pushHook(name, hilti::builder::function::result(rtype), p, hilti::AttributeSet(), priority, 0, false);
 
     shared_ptr<hilti::builder::BlockBuilder> hook = nullptr;
     shared_ptr<hilti::builder::BlockBuilder> cont = nullptr;
@@ -1708,19 +1708,20 @@ shared_ptr<hilti::Expression> ParserBuilder::_hiltiMatchTokenInit(const string&
     auto glob = cg()->moduleBuilder()->lookupNode("match-token", name);
 
     if ( ! glob ) {
-        std::list<std::pair<string, string>> tokens;
+        std::list<string> tokens;
 
         for ( auto t : terms ) {
             // FIXME: Really shouldn't have to do this cast here. Should we
             // pass in a list of Literals instead?
             auto l = ast::checkedCast<production::Literal>(t);
             for ( auto p : l->patterns() )
-                tokens.push_back(std::make_pair(util::fmt("%s{#%d}", p.first, l->tokenID()), ""));
+                tokens.push_back(util::fmt("%s{#%d}", p, l->tokenID()));
         }
 
         auto op = hilti::builder::regexp::create(tokens);
-        auto rty = hilti::builder::reference::type(hilti::builder::regexp::type({"&nosub"}));
-        glob = cg()->moduleBuilder()->addGlobal(hilti::builder::id::node(name), rty, op);
+        auto re = ast::checkedCast<hilti::ctor::RegExp>(op->ctor());
+        re->attributes().add(attribute::NOSUB);
+        glob = cg()->moduleBuilder()->addGlobal(hilti::builder::id::node(name), op->type(), op);
 
         cg()->moduleBuilder()->cacheNode("match-token", name, glob);
     }

binpac/codegen/type-builder.cc

@@ -219,7 +219,7 @@ void TypeBuilder::visit(type::RegExp* r)
 {
     TypeInfo ti;
 
-    auto t = hilti::builder::regexp::type(hilti::builder::regexp::attribute_list(), r->location());
+    auto t = hilti::builder::regexp::type(r->location());
     ti.hilti_type = hilti::builder::reference::type(t, r->location());
 
     setResult(ti);

binpac/ctor.cc

@@ -172,16 +172,18 @@ Map::element_list Map::elements() const
     return elems;
 }
 
-RegExp::RegExp(const string& regexp, const string& flags, const attribute_list& attrs, const Location& l) : Ctor(l)
+RegExp::RegExp(const string& regexp, const attribute_list& attrs, const Location& l) : Ctor(l)
 {
-    _patterns = { pattern(regexp, flags) };
-    _type = std::make_shared<type::RegExp>(attrs, l);
+    _patterns = { regexp };
+    _attributes = attrs;
+    _type = std::make_shared<type::RegExp>(l);
 }
 
 RegExp::RegExp(const pattern_list& patterns, const attribute_list& attrs, const Location& l)
 {
     _patterns = patterns;
-    _type = std::make_shared<type::RegExp>(attrs, l);
+    _attributes = attrs;
+    _type = std::make_shared<type::RegExp>(l);
     addChild(_type);
 }
 
@@ -195,6 +197,11 @@ Ctor::pattern_list RegExp::patterns() const
     return _patterns;
 }
 
+const attribute_list& RegExp::attributes() const
+{
+    return _attributes;
+}
+
 Unit::Unit(const item_list& items, const Location& l) : Ctor(l)
 {
     for ( auto i : items ) {

binpac/ctor.h

@@ -16,10 +16,7 @@ namespace binpac {
 class Ctor : public ast::Ctor<AstInfo>
 {
 public:
-    /// A pattern is a tuple of two strings. The first element is the regexp
-    /// itself, and the second a string with optional patterns flags.
-    /// Currently, no flags are supported though.
-    typedef std::pair<string, string> pattern;
+    typedef std::string pattern;
 
     /// A list of patterns.
     typedef std::list<pattern> pattern_list;
@@ -201,23 +198,26 @@ class RegExp : public Ctor
     ///
     /// flags: The string with flags.
     ///
-    /// attrs: Type attributes that will become part of the returned type::RegExp.
+    /// attrs: Optional attributes.
     ///
     /// l: An associated location.
-    RegExp(const string& regexp, const string& flags, const attribute_list& attrs = attribute_list(), const Location& l=Location::None);
+    RegExp(const string& regexp, const attribute_list& attrs = attribute_list(), const Location& l=Location::None);
 
     /// Constructor.
     ///
     /// patterns: List of patterns.
     ///
-    /// attrs: Type attributes that will become part of the returned type::RegExp.
+    /// attrs: Optional attributes.
     ///
     /// l: An associated location.
     RegExp(const pattern_list& patterns, const attribute_list& attrs = attribute_list(), const Location& l=Location::None);
 
     /// Returns the pattern.
     pattern_list patterns() const override;
 
+    /// Returns the attributes.
+    const attribute_list& attributes() const;
+
     /// Returns the type of the constructed object. Pattern constants are
     /// always of type \c regexp<>. To add further type attributes, they need
     /// to be coerced to a regexp type that has them.
@@ -228,6 +228,7 @@ class RegExp : public Ctor
 private:
     node_ptr<Type> _type;
     pattern_list _patterns;
+    attribute_list _attributes;
 };
 
 /// AST node for a unit constructor.

binpac/parser/driver.h

@@ -72,7 +72,6 @@ struct yystype_binpac {
     binpac::type::unit::item::field::Switch::case_list switch_cases;
     binpac::type::Integer::bits_list bits;
     binpac::ctor::RegExp::pattern_list re_patterns;
-    binpac::ctor::RegExp::pattern re_pattern;
     binpac::type::function::CallingConvention cc;
 
     std::list<string> strings;

binpac/parser/parser.yy

@@ -207,7 +207,6 @@ inline shared_ptr<type::unit::item::Field> makeVectorField(shared_ptr<type::unit
 %type <switch_case>      unit_switch_case
 %type <switch_cases>     unit_switch_cases
 %type <re_patterns>      re_patterns
-%type <re_pattern>       re_pattern
 %type <sval>             re_pattern_constant
 %type <types>            types
 %type <map_element>      map_elem
@@ -682,11 +681,8 @@ unit_ctor_item
               : dollar_id '=' expr               { $$ = ctor::Unit::item($1, $3); }
               | expr                             { $$ = ctor::Unit::item(nullptr, $1); }
 
-re_patterns   : re_patterns '|' re_pattern       { $$ = $1; $$.push_back($3); }
-              | re_pattern                       { $$ = ctor::RegExp::pattern_list(); $$.push_back($1); }
-
-re_pattern    : re_pattern_constant              { $$ = std::make_pair($1, ""); }
-              | re_pattern_constant IDENT        { $$ = std::make_pair($1, $2); }
+re_patterns   : re_patterns '|' re_pattern_constant { $$ = $1; $$.push_back($3); }
+              | re_pattern_constant                 { $$ = ctor::RegExp::pattern_list(); $$.push_back($1); }
 
 re_pattern_constant
               : '/' { driver.enablePatternMode(); } CREGEXP { driver.disablePatternMode(); } '/' { $$ = $3; }

binpac/passes/printer.cc

@@ -324,7 +324,7 @@ void Printer::visit(ctor::RegExp* r)
         if ( ! first )
             p << " | ";
 
-        p << "/" << pat.first << "/";
+        p << "/" << pat << "/";
         first = false;
     }
 }

bro/pac2/libbro.hlt

@@ -114,7 +114,7 @@ declare "C-HILTI" void    object_mapping_unregister_hilti(any obj)
 declare "C-HILTI" BroVal  object_mapping_lookup_bro(any obj)
 declare "C-HILTI" any     object_mapping_lookup_hilti(BroVal val)
 
-declare "C-HILTI" ref<BroAny> any_from_hilti(const any obj, BroType btype, caddr to_val_func)
+declare "C-HILTI" ref<BroAny> any_from_hilti_ref(const any obj, BroType btype, caddr to_val_func)
 declare "C-HILTI" any         any_to_hilti(const ref<BroAny> a)
 
 declare "C-HILTI" ref<BroAny> any_from_val(BroVal v)

bro/src/Manager.cc

@@ -2657,14 +2657,9 @@ void Manager::ExtractParsers(hlt_list* parsers)
 		parser_map.insert(std::make_tuple(name, p));
 		hlt_free(name);
 
-		hlt_iterator_list j = i;
 		i = hlt_iterator_list_incr(i, &excpt, ctx);
-		GC_DTOR(j, hlt_iterator_list);
 		}
 
-	GC_DTOR(i, hlt_iterator_list);
-	GC_DTOR(end, hlt_iterator_list);
-
 	for ( auto a : pimpl->pac2_analyzers )
 		{
 		if ( a->unit_name_orig.size() )
@@ -2674,7 +2669,7 @@ void Manager::ExtractParsers(hlt_list* parsers)
 				if ( i != parser_map.end() )
 						{
 						a->parser_orig = i->second;
-						GC_CCTOR(a->parser_orig, hlt_BinPACHilti_Parser);
+						GC_CCTOR(a->parser_orig, hlt_BinPACHilti_Parser, ctx);
 						}
 				}
 
@@ -2685,7 +2680,7 @@ void Manager::ExtractParsers(hlt_list* parsers)
 				if ( i != parser_map.end() )
 						{
 						a->parser_resp = i->second;
-						GC_CCTOR(a->parser_resp, hlt_BinPACHilti_Parser);
+						GC_CCTOR(a->parser_resp, hlt_BinPACHilti_Parser, ctx);
 						}
 				}
 		}
@@ -2700,13 +2695,13 @@ void Manager::ExtractParsers(hlt_list* parsers)
 		if ( i != parser_map.end() )
 			{
 			a->parser = i->second;
-			GC_CCTOR(a->parser, hlt_BinPACHilti_Parser);
+			GC_CCTOR(a->parser, hlt_BinPACHilti_Parser, ctx);
 			}
 		}
 
 	for ( auto p : parser_map )
 		{
-		GC_DTOR(p.second, hlt_BinPACHilti_Parser);
+		GC_DTOR(p.second, hlt_BinPACHilti_Parser, ctx);
 		}
 	}
 
@@ -2874,7 +2869,7 @@ ::Val* Manager::RuntimeCallFunctionInternal(const ::Func* func, val_list* args)
 		char* e = hlt_exception_to_asciiz(excpt, &etmp, ctx);
 		reporter::error(::util::fmt("event/function %s raised exception: %s", func->Name(), e));
 		hlt_free(e);
-		GC_DTOR(excpt, hlt_exception);
+		GC_DTOR(excpt, hlt_exception, ctx);
 		}
 
 	return result ? result : new ::Val(0, ::TYPE_VOID);

bro/src/Pac2Analyzer.cc

@@ -52,13 +52,15 @@ void Pac2_Analyzer::Init()
 
 void Pac2_Analyzer::Done()
 	{
-	GC_DTOR(orig.parser, hlt_BinPACHilti_Parser);
-	GC_DTOR(orig.data, hlt_bytes);
-	GC_DTOR(orig.resume, hlt_exception);
+	hlt_execution_context* ctx = hlt_global_execution_context();
+
+	GC_DTOR(orig.parser, hlt_BinPACHilti_Parser, ctx);
+	GC_DTOR(orig.data, hlt_bytes, ctx);
+	GC_DTOR(orig.resume, hlt_exception, ctx);
 
-	GC_DTOR(resp.parser, hlt_BinPACHilti_Parser);
-	GC_DTOR(resp.data, hlt_bytes);
-	GC_DTOR(resp.resume, hlt_exception);
+	GC_DTOR(resp.parser, hlt_BinPACHilti_Parser, ctx);
+	GC_DTOR(resp.data, hlt_bytes, ctx);
+	GC_DTOR(resp.resume, hlt_exception, ctx);
 
 	Init();
 	}
@@ -110,7 +112,7 @@ int Pac2_Analyzer::FeedChunk(int len, const u_char* data, bool is_orig, bool eod
 			return 1;
 			}
 
-		GC_CCTOR(endp->parser, hlt_BinPACHilti_Parser);
+		GC_CCTOR(endp->parser, hlt_BinPACHilti_Parser, ctx);
 		}
 
 	int result = 0;
@@ -123,6 +125,7 @@ int Pac2_Analyzer::FeedChunk(int len, const u_char* data, bool is_orig, bool eod
 		debug_msg(endp->cookie.protocol_cookie.analyzer, "initial chunk", len, data, is_orig);
 
 		endp->data = hlt_bytes_new_from_data_copy((const int8_t*)data, len, &excpt, ctx);
+        GC_CCTOR(endp->data, hlt_bytes, ctx);
 
 		if ( eod )
 			hlt_bytes_freeze(endp->data, 1, &excpt, ctx);
@@ -137,7 +140,7 @@ int Pac2_Analyzer::FeedChunk(int len, const u_char* data, bool is_orig, bool eod
 		profile_update(PROFILE_HILTI_LAND, PROFILE_STOP);
 #endif
 
-		GC_DTOR_GENERIC(&pobj, endp->parser->type_info);
+		GC_DTOR_GENERIC(&pobj, endp->parser->type_info, ctx);
 		}
 
 	else
@@ -163,7 +166,7 @@ int Pac2_Analyzer::FeedChunk(int len, const u_char* data, bool is_orig, bool eod
 		profile_update(PROFILE_HILTI_LAND, PROFILE_STOP);
 #endif
 
-		GC_DTOR_GENERIC(&pobj, endp->parser->type_info);
+		GC_DTOR_GENERIC(&pobj, endp->parser->type_info, ctx);
 		endp->resume = 0;
 		}
 
@@ -181,13 +184,11 @@ int Pac2_Analyzer::FeedChunk(int len, const u_char* data, bool is_orig, bool eod
 			{
 			// A parse error.
 			hlt_exception* excpt2 = 0;
-			hlt_string s = hlt_exception_to_string(&hlt_type_info_hlt_exception, &excpt, 0, &excpt2, ctx);
-			char* e = hlt_string_to_native(s, &excpt2, ctx);
+			char* e = hlt_exception_to_asciiz(excpt, &excpt2, ctx);
 			assert(! excpt2);
 			ParseError(e, is_orig);
 			hlt_free(e);
-			GC_DTOR(s, hlt_string);
-			GC_DTOR(excpt, hlt_exception);
+			GC_DTOR(excpt, hlt_exception, ctx);
 			excpt = 0;
 			error = true;
 			result = 0;
@@ -203,10 +204,7 @@ int Pac2_Analyzer::FeedChunk(int len, const u_char* data, bool is_orig, bool eod
 	// TODO: For now we just stop on error, later we might attempt to
 	// restart parsing.
 	if ( eod || done || error )
-		{
-		GC_DTOR(endp->data, hlt_bytes);
-		endp->data = 0; // Marker that we're done parsing.
-		}
+        GC_CLEAR(endp->data, hlt_bytes, ctx);  // Marker that we're done parsing.
 
 	return result;
 	}