feat: introduce special immutables address space (#2689)

This commit introduces a new address space for manipulating immutables.
It also modifies the existing "code" address space which resolves
pointers as starting from the start of the "data" section instead of the
start of the "text" section of the code (air quotes because these
sections are conceptual rather than part of an object format). Lastly it
slightly changes the semantics of the `lll` opcode (and for clarity,
renames it to `deploy`) in order to save a few instructions in the
constructor.

This is important for two reasons. One, other VMs like zksync model
immutables differently by mapping them onto storage. In order to capture
different handling of the address space (immutables -> storage in zksync
vs immutables -> runtime code in EVM), the new pseudo opcodes `iload`
and `istore` can be compiled differently depending on the backend. For
EVM, they compile to `mload` and `mstore` at magic locations calculated
at assembly time. Note that `iload` and `istore` are not valid outside
of constructor code (that is, code which does not have a child `deploy`
section), and assembly will panic if they are used in non-constructor
code.

The second reason has to do with runtime code layout. Currently, the IR
codegen has no idea what the codesize will be (as that can only be known
after assembly), so we have runtime logic to calculate offsets of data
that is stored in code (e.g. currently to get data at position `x` in
the runtime code, we need to issue pointer arithmetic `(add ~codelen x)`
to get the runtime location of the data). By introducing new pseudo
opcodes, we can resolve these locations at assembly time instead of at
runtime.

This commit also renames the "code" address space to "data", to make it
clearer that it is used to access the data section of the code. The
pseudo-opcode `codeload` has been renamed to `dload`. For the same
efficiency reason described above, `data` is considered to start at the
end of the runtime code, so `dload` usage is `dload x` instead of
`codeload (add ~codelen x)`.

To accomplish all this, a couple magic locations have been introduced
into the assembly: `_mem_deploy_start` and `_mem_deploy_end`. These are
resolved at assembly time to the end of the runtime code just prior to
deploy - immutables are stored starting from `_mem_deploy_end`. To
support the assembly-time calculation of these offsets, two magic
opcodes have been added to the assembly: `_DEPLOY_MEM_OFST_<N>` and
`_OFST`. `_DEPLOY_MEM_OFST_<N>` is used to pass information from the
memory allocator to the assembler (which is required to calculate the
location of where immutables will be stored in memory), and `_OFST` is
used to resolve offsets from `_sym_code_end` and `_mem_deploy_end` at
compile time.

Author: charles-cooper

tests/compiler/LLL/test_compile_lll.py

@@ -38,16 +38,14 @@ def test_compile_lll_good(good_lll, get_contract_from_lll):
 def test_lll_from_s_expression(get_contract_from_lll):
     code = """
 (seq
-  (return
+  (deploy
     0
-    (lll ; just return 32 byte of calldata back
-      0
-      (seq
-          (calldatacopy 0 4 32)
-          (return 0 32)
-          stop
-        )
-      )))
+    (seq ; just return 32 byte of calldata back
+      (calldatacopy 0 4 32)
+      (return 0 32)
+      stop
+     )
+    0))
     """
     abi = [
         {

vyper/builtin_functions/functions.py

@@ -564,7 +564,7 @@ def build_LLL(self, expr, context):
                 if arg.typ.maxlen == 0:
                     continue
                 # Get the length of the current argument
-                if arg.location in ("memory", "calldata", "code"):
+                if arg.location in ("memory", "calldata", "data", "immutables"):
                     length = LLLnode.from_list(
                         [load_op(arg.location), "_arg"], typ=BaseType("int128")
                     )

vyper/codegen/context.py

@@ -152,6 +152,8 @@ def _new_variable(
         self, name: str, typ: NodeType, var_size: int, is_internal: bool, is_mutable: bool = True
     ) -> int:
         if is_internal:
+            # TODO CMC 2022-03-02 change this to `.allocate_memory()`
+            # and make `expand_memory()` private.
             var_pos = self.memory_allocator.expand_memory(var_size)
         else:
             var_pos = self.memory_allocator.allocate_memory(var_size)

vyper/codegen/core.py

@@ -115,7 +115,7 @@ def make_byte_array_copier(dst, src, pos=None):
 
 # TODO maybe move me to types.py
 def wordsize(location):
-    if location in ("memory", "calldata", "code"):
+    if location in ("memory", "calldata", "data", "immutables"):
         return 32
     if location == "storage":
         return 1
@@ -232,21 +232,28 @@ def copy_bytes(dst, src, length, length_bound, pos=None):
             ret = LLLnode.from_list(copy_op, annotation=annotation)
             return b1.resolve(b2.resolve(b3.resolve(ret)))
 
-        if dst.location == "memory" and src.location in ("memory", "calldata", "code"):
+        if dst.location == "memory" and src.location in ("memory", "calldata", "data"):
             # special cases: batch copy to memory
+            # TODO: iloadbytes
             if src.location == "memory":
                 copy_op = ["staticcall", "gas", 4, src, length, dst, length]
                 gas_bound = _identity_gas_bound(length_bound)
             elif src.location == "calldata":
                 copy_op = ["calldatacopy", dst, src, length]
                 gas_bound = _calldatacopy_gas_bound(length_bound)
-            elif src.location == "code":
-                copy_op = ["codecopy", dst, src, length]
+            elif src.location == "data":
+                copy_op = ["dloadbytes", dst, src, length]
+                # note: dloadbytes compiles to CODECOPY
                 gas_bound = _codecopy_gas_bound(length_bound)
 
             ret = LLLnode.from_list(copy_op, annotation=annotation, add_gas_estimate=gas_bound)
             return b1.resolve(b2.resolve(b3.resolve(ret)))
 
+        if dst.location == "immutables" and src.location in ("memory", "data"):
+            # TODO istorebytes-from-mem, istorebytes-from-calldata(?)
+            # compile to identity, CODECOPY respectively.
+            pass
+
         # general case, copy word-for-word
         # pseudocode for our approach (memory-storage as example):
         # for i in range(len, bound=MAX_LEN):
@@ -259,15 +266,15 @@ def copy_bytes(dst, src, length, length_bound, pos=None):
 
         i = LLLnode.from_list(_freshname("copy_bytes_ix"), typ="uint256")
 
-        if src.location in ("memory", "calldata", "code"):
+        if src.location in ("memory", "calldata", "data", "immutables"):
             loader = [load_op(src.location), ["add", src, _mul(32, i)]]
         elif src.location == "storage":
             loader = [load_op(src.location), ["add", src, i]]
         else:
             raise CompilerPanic(f"Unsupported location: {src.location}")  # pragma: notest
 
-        if dst.location == "memory":
-            setter = ["mstore", ["add", dst, _mul(32, i)], loader]
+        if dst.location in ("memory", "immutables"):
+            setter = [store_op(dst.location), ["add", dst, _mul(32, i)], loader]
         elif dst.location == "storage":
             setter = ["sstore", ["add", dst, i], loader]
         else:
@@ -454,7 +461,7 @@ def _get_element_ptr_tuplelike(parent, key, pos):
     if parent.location == "storage":
         for i in range(index):
             ofst += typ.members[attrs[i]].storage_size_in_words
-    elif parent.location in ("calldata", "memory", "code"):
+    elif parent.location in ("calldata", "memory", "data", "immutables"):
         for i in range(index):
             ofst += typ.members[attrs[i]].memory_bytes_required
     else:
@@ -522,7 +529,7 @@ def _get_element_ptr_array(parent, key, pos, array_bounds_check):
 
     if parent.location == "storage":
         element_size = subtype.storage_size_in_words
-    elif parent.location in ("calldata", "memory", "code"):
+    elif parent.location in ("calldata", "memory", "data", "immutables"):
         element_size = subtype.memory_bytes_required
 
     ofst = _mul(ix, element_size)
@@ -578,8 +585,13 @@ def load_op(location):
         return "sload"
     if location == "calldata":
         return "calldataload"
-    if location == "code":
-        return "codeload"
+    if location == "data":
+        # refers to data section of currently executing code
+        return "dload"
+    if location == "immutables":
+        # special address space for manipulating immutables before deploy
+        # only makes sense in a constructor
+        return "iload"
     raise CompilerPanic(f"unreachable {location}")  # pragma: notest
 
 
@@ -588,12 +600,14 @@ def store_op(location):
         return "mstore"
     if location == "storage":
         return "sstore"
+    if location == "immutables":
+        return "istore"
     raise CompilerPanic(f"unreachable {location}")  # pragma: notest
 
 
 # Unwrap location
 def unwrap_location(orig):
-    if orig.location in ("memory", "storage", "calldata", "code"):
+    if orig.location in ("memory", "storage", "calldata", "data", "immutables"):
         return LLLnode.from_list([load_op(orig.location), orig], typ=orig.typ)
     else:
         # CMC 20210909 TODO double check if this branch can be removed

vyper/codegen/expr.py

@@ -315,39 +315,26 @@ def parse_Name(self):
             return LLLnode.from_list(
                 [obj], typ=BaseType(typ, is_literal=True), pos=getpos(self.expr)
             )
+
         elif self.expr._metadata["type"].is_immutable:
-            # immutable variable
-            # need to handle constructor and outside constructor
             var = self.context.globals[self.expr.id]
-            is_constructor = self.expr.get_ancestor(vy_ast.FunctionDef).get("name") == "__init__"
-            if is_constructor:
-                # store memory position for later access in module.py in the variable record
-                memory_loc = self.context.new_variable(self.expr.id, var.typ)
-                self.context.global_ctx._globals[self.expr.id].pos = memory_loc
-                # store the data offset in the variable record as well for accessing
-                data_offset = self.expr._metadata["type"].position.offset
-                self.context.global_ctx._globals[self.expr.id].data_offset = data_offset
+            ofst = self.expr._metadata["type"].position.offset
 
-                return LLLnode.from_list(
-                    memory_loc,
-                    typ=var.typ,
-                    location="memory",
-                    pos=getpos(self.expr),
-                    annotation=self.expr.id,
-                    mutable=True,
-                )
+            if self.context.sig.is_init_func:
+                mutable = True
+                location = "immutables"
             else:
-                immutable_section_size = self.context.global_ctx.immutable_section_size
-                offset = self.expr._metadata["type"].position.offset
-                # TODO: resolve code offsets for immutables at compile time
-                return LLLnode.from_list(
-                    ["sub", "codesize", immutable_section_size - offset],
-                    typ=var.typ,
-                    location="code",
-                    pos=getpos(self.expr),
-                    annotation=self.expr.id,
-                    mutable=False,
-                )
+                mutable = False
+                location = "data"
+
+            return LLLnode.from_list(
+                ofst,
+                typ=var.typ,
+                location=location,
+                pos=getpos(self.expr),
+                annotation=self.expr.id,
+                mutable=mutable,
+            )
 
     # x.y or x[5]
     def parse_Attribute(self):
@@ -481,7 +468,7 @@ def parse_Attribute(self):
                 return LLLnode.from_list(["chainid"], typ="uint256", pos=getpos(self.expr))
         # Other variables
         else:
-            sub = Expr.parse_variable_location(self.expr.value, self.context)
+            sub = Expr(self.expr.value, self.context).lll_node
             # contract type
             if isinstance(sub.typ, InterfaceType):
                 return sub

vyper/codegen/function_definitions/external_function.py

@@ -46,9 +46,7 @@ def _register_function_args(context: Context, sig: FunctionSignature) -> List[LL
 
     # tuple with the abi_encoded args
     if sig.is_init_func:
-        base_args_ofst = LLLnode(
-            "~codelen", location="code", typ=base_args_t, encoding=Encoding.ABI
-        )
+        base_args_ofst = LLLnode(0, location="data", typ=base_args_t, encoding=Encoding.ABI)
     else:
         base_args_ofst = LLLnode(4, location="calldata", typ=base_args_t, encoding=Encoding.ABI)
 

vyper/codegen/global_context.py

@@ -236,6 +236,10 @@ def parse_type(self, ast_node):
             custom_structs=self._structs,
         )
 
+    @property
+    def immutables(self):
+        return [t for t in self._globals.values() if t.is_immutable]
+
     @cached_property
-    def immutable_section_size(self):
-        return sum([imm.size * 32 for imm in self._globals.values() if imm.is_immutable])
+    def immutable_section_bytes(self):
+        return sum([imm.size * 32 for imm in self.immutables])

vyper/codegen/lll_node.py

@@ -242,10 +242,9 @@ def _check(condition, err):
                     )
                 self.valency = sum([arg.valency for arg in self.args])
                 self.gas = sum([arg.gas for arg in self.args])
-            # LLL brackets (don't bother gas counting)
-            elif self.value == "lll":
-                self.valency = 1
-                self.gas = NullAttractor()
+            elif self.value == "deploy":
+                self.valency = 0
+                self.gas = NullAttractor()  # unknown
             # Stack variables
             else:
                 self.valency = 1

vyper/codegen/memory_allocator.py

@@ -100,6 +100,7 @@ def allocate_memory(self, size: int) -> int:
         # if no deallocated slots are available, expand memory
         return self.expand_memory(size)
 
+    # TODO this should be an internal function
     def expand_memory(self, size: int) -> int:
         """
         Allocate `size` bytes in memory, starting from the free memory pointer.

vyper/codegen/module.py

@@ -4,7 +4,6 @@
 
 from vyper import ast as vy_ast
 from vyper.ast.signatures.function_signature import FunctionSignature, FunctionSignatures
-from vyper.codegen.core import make_setter
 from vyper.codegen.function_definitions import (
     generate_lll_for_function,
     is_default_func,
@@ -90,7 +89,7 @@ def parse_external_interfaces(external_interfaces, global_ctx):
 
 
 def parse_regular_functions(
-    o, regular_functions, sigs, external_interfaces, global_ctx, default_function
+    regular_functions, sigs, external_interfaces, global_ctx, default_function, init_function
 ):
     # check for payable/nonpayable external functions to optimize nonpayable assertions
     func_types = [i._metadata["type"] for i in global_ctx._defs]
@@ -179,50 +178,7 @@ def parse_regular_functions(
     ]
     runtime.extend(internal_funcs)
 
-    immutables = [_global for _global in global_ctx._globals.values() if _global.is_immutable]
-
-    # TODO: enable usage of the data section beyond just user defined immutables
-    # https://github.com/vyperlang/vyper/pull/2466#discussion_r722816358
-    if len(immutables) > 0:
-        # find position of the last immutable so we do not overwrite it in memory
-        # when we codecopy the runtime code to memory
-        immutables = sorted(immutables, key=lambda imm: imm.pos)
-        start_pos = immutables[-1].pos + immutables[-1].size * 32
-
-        # create sequence of actions to copy immutables to the end of the runtime code in memory
-        # TODO: if possible, just use identity precompile
-        data_section = []
-        for immutable in immutables:
-            # store each immutable at the end of the runtime code
-            memory_loc, offset = (
-                immutable.pos,
-                immutable.data_offset,
-            )
-            lhs = LLLnode.from_list(
-                ["add", start_pos + offset, "_lllsz"], typ=immutable.typ, location="memory"
-            )
-            rhs = LLLnode.from_list(memory_loc, typ=immutable.typ, location="memory")
-            data_section.append(make_setter(lhs, rhs, pos=None))
-
-        # TODO: use GlobalContext.immutable_section_size
-        data_section_size = sum([immutable.size * 32 for immutable in immutables])
-        o.append(
-            [
-                "with",
-                "_lllsz",  # keep size of runtime bytecode in sz var
-                ["lll", start_pos, runtime],  # store runtime code at `start_pos`
-                # sequence of copying immutables, with final action of returning the runtime code
-                ["seq", *data_section, ["return", start_pos, ["add", data_section_size, "_lllsz"]]],
-            ]
-        )
-
-    else:
-        # NOTE: lll macro first argument is the location in memory to store
-        # the compiled bytecode
-        # https://lll-docs.readthedocs.io/en/latest/lll_reference.html#code-lll
-        o.append(["return", 0, ["lll", 0, runtime]])
-
-    return o, runtime
+    return runtime
 
 
 # Main python parse tree => LLL method
@@ -257,10 +213,10 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode, Func
     if global_ctx._contracts or global_ctx._interfaces:
         external_interfaces = parse_external_interfaces(external_interfaces, global_ctx)
 
-    # TODO: fix for #2251 is to move this after parse_regular_functions
+    init_func_lll = None
     if init_function:
         o.append(init_func_init_lll())
-        init_func_lll, _frame_start, _frame_size = generate_lll_for_function(
+        init_func_lll, _frame_start, init_frame_size = generate_lll_for_function(
             init_function,
             {**{"self": sigs}, **external_interfaces},
             global_ctx,
@@ -269,15 +225,27 @@ def parse_tree_to_lll(global_ctx: GlobalContext) -> Tuple[LLLnode, LLLnode, Func
         o.append(init_func_lll)
 
     if regular_functions or default_function:
-        o, runtime = parse_regular_functions(
-            o,
+        runtime = parse_regular_functions(
             regular_functions,
             sigs,
             external_interfaces,
             global_ctx,
             default_function,
+            init_func_lll,
         )
     else:
-        runtime = o.copy()
+        # for some reason, somebody may want to deploy a contract with no code,
+        # or more likely, a "pure data" contract which contains immutables
+        runtime = LLLnode.from_list(["seq"])
+
+    immutables_len = global_ctx.immutable_section_bytes
+
+    if init_function:
+        memsize = init_func_lll.context.memory_allocator.size_of_mem  # type: ignore
+    else:
+        memsize = 0
+
+    # note: (deploy mem_ofst, code, extra_padding)
+    o.append(["deploy", memsize, runtime, immutables_len])  # type: ignore
 
     return LLLnode.from_list(o), LLLnode.from_list(runtime), sigs

vyper/compiler/output.py

@@ -174,11 +174,9 @@ def _build_asm(asm_list):
                 output_string += " "
             in_push -= 1
         else:
-            assert isinstance(node, str), node
+            output_string += str(node) + " "
 
-            output_string += node + " "
-
-            if node.startswith("PUSH"):
+            if isinstance(node, str) and node.startswith("PUSH"):
                 assert in_push == 0
                 in_push = int(node[4:])
                 output_string += "0x"