[DO NOT MERGE] Adding LLD linker support to EVM arch.

.github/workflows/compiler-integration-tests.yml

@@ -1,16 +1,9 @@
 name: Integration tests
 
 on:
-  push:
-    branches:
-      - main
   pull_request:
   workflow_dispatch:
     inputs:
-      llvm_build_type:
-        description: "LLVM build type: debug | release"
-        required: true
-        default: "release"
       compiler_tester_branch:
         description: "compiler-tester branch"
         required: true
@@ -21,57 +14,24 @@ on:
         default: "main"
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.repository_id }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
 
-  test:
-    name: Integration tests
-    runs-on: [ self-hosted, ci-runner-compiler ]
-    container:
-      image: matterlabs/llvm_runner:ubuntu22-llvm17-latest
-      options: -m 110g
-    steps:
-
-      - name: Checkout era-compiler-tester
-        uses: actions/checkout@v4
-        with:
-          repository: matter-labs/era-compiler-tester
-          ref: ${{ github.event.inputs.compiler_tester_branch || 'main' }}
-          submodules: recursive
-
-      - name: Checkout LLVM
-        uses: actions/checkout@v4
-        with:
-          path: "llvm"
-          ref: ${{ github.event.inputs.compiler_llvm_branch || github.head_ref }}
-
-      - name: Build LLVM
-        uses: ./llvm/.github/actions/build
-        with:
-          extra-args: "\\-DLLVM_ENABLE_WERROR=On \\-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
-          enable-tests: true
-          enable-assertions: true
-          debug: ${{ github.event.inputs.llvm_build_type || 'release' }}
-
-      - name: Building and running the compiler tester
-        id: compiler_tester_run
-        env:
-          RUST_BACKTRACE: "full"
-        run: |
-          cargo build --verbose --release --bin 'compiler-tester'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
-          ./target/release/compiler-tester \
-            --zksolc './target-zksolc/release/zksolc' \
-            --zkvyper './target-zkvyper/release/zkvyper'
-
-      - uses: 8398a7/action-slack@v3
-        if: always()
-        with:
-          status: ${{ job.status }}
-          job_name: 'Integration tests' # required if job name is explicitly set
-          fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+  # Check for secrets leak in the repository
+  secrets-scanner:
+    uses: matter-labs/era-compiler-ci/.github/workflows/secrets-scanner.yaml@v1
+    secrets: inherit
+
+  # Integration tests workflow call from the era-compiler-ci repository
+  # This is a common part of the integration tests workflow for all repositories
+  # If you would like to make a change to the integration tests workflow, please do it in the era-compiler-ci repository
+  integration-tests:
+    uses: matter-labs/era-compiler-ci/.github/workflows/integration-tests.yaml@v1
+    secrets: inherit
+    with:
+      compiler-tester-ref: ${{ github.event.inputs.compiler_tester_branch || 'main' }}
+      llvm-ref: ${{ github.event.inputs.compiler_llvm_branch || github.head_ref || github.event.repository.default_branch }}
+      ccache-key-type: 'static' # rotate ccache key every month
+      compiler-llvm-repo: ${{ github.event.pull_request.head.repo.full_name }} # required to properly test forks

lld/CMakeLists.txt

@@ -187,6 +187,7 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
     )
 endif()
 
+add_subdirectory(lld-c)
 add_subdirectory(Common)
 add_subdirectory(tools/lld)
 

lld/Common/DriverDispatcher.cpp

@@ -150,6 +150,22 @@ static Driver whichDriver(llvm::SmallVectorImpl<const char *> &argsV,
   return it->d;
 }
 
+// EVM local begin
+static DriverMemBuf whichDriver(llvm::SmallVectorImpl<const char *> &argsV,
+                                llvm::ArrayRef<DriverDefMemBuf> drivers) {
+  Flavor f = parseFlavor(argsV);
+  const auto *const it =
+      llvm::find_if(drivers, [=](auto &driverdef) { return driverdef.f == f; });
+  if (it == drivers.end()) {
+    // Driver is invalid or not available in this build.
+    return [](llvm::ArrayRef<llvm::MemoryBufferRef>, llvm::raw_pwrite_stream *,
+              llvm::ArrayRef<const char *>, llvm::raw_ostream &,
+              llvm::raw_ostream &, bool, bool) { return false; };
+  }
+  return it->d;
+}
+// EVM local end
+
 namespace lld {
 bool inTestOutputDisabled = false;
 
@@ -174,6 +190,35 @@ int unsafeLldMain(llvm::ArrayRef<const char *> args,
 
   return r;
 }
+
+// EVM local begin
+/// Universal linker main(). This linker emulates the gnu, darwin, or
+/// windows linker based on the argv[0] or -flavor option.
+int unsafeLldMainMemBuf(llvm::ArrayRef<llvm::MemoryBufferRef> inData,
+                        llvm::raw_pwrite_stream *outData,
+                        llvm::ArrayRef<const char *> args,
+                        llvm::raw_ostream &stdoutOS,
+                        llvm::raw_ostream &stderrOS,
+                        llvm::ArrayRef<DriverDefMemBuf> drivers,
+                        bool exitEarly) {
+  SmallVector<const char *, 256> argsV(make_range(args.begin(), args.end()));
+  DriverMemBuf d = whichDriver(argsV, drivers);
+  // Run the driver. If an error occurs, false will be returned.
+  const int r = !d(inData, outData, argsV, stdoutOS, stderrOS, exitEarly,
+                   inTestOutputDisabled);
+  // At this point 'r' is either 1 for error, and 0 for no error.
+
+  // Call exit() if we can to avoid calling destructors.
+  if (exitEarly)
+    exitLld(r);
+
+  // Delete the global context and clear the global context pointer, so that it
+  // cannot be accessed anymore.
+  CommonLinkerContext::destroy();
+
+  return r;
+}
+// EVM local end
 } // namespace lld
 
 Result lld::lldMain(llvm::ArrayRef<const char *> args,
@@ -201,3 +246,35 @@ Result lld::lldMain(llvm::ArrayRef<const char *> args,
   }
   return {r, /*canRunAgain=*/true};
 }
+
+// EVM local begin
+Result lld::lldMainMemBuf(llvm::ArrayRef<llvm::MemoryBufferRef> inData,
+                          llvm::raw_pwrite_stream *outData,
+                          llvm::ArrayRef<const char *> args,
+                          llvm::raw_ostream &stdoutOS,
+                          llvm::raw_ostream &stderrOS,
+                          llvm::ArrayRef<DriverDefMemBuf> drivers) {
+  int r = 0;
+  {
+    // The crash recovery is here only to be able to recover from arbitrary
+    // control flow when fatal() is called (through setjmp/longjmp or
+    // __try/__except).
+    llvm::CrashRecoveryContext crc;
+    if (!crc.RunSafely([&]() {
+          r = unsafeLldMainMemBuf(inData, outData, args, stdoutOS, stderrOS,
+                                  drivers,
+                                  /*exitEarly=*/false);
+        }))
+      return {crc.RetCode, /*canRunAgain=*/false};
+  }
+
+  // Cleanup memory and reset everything back in pristine condition. This path
+  // is only taken when LLD is in test, or when it is used as a library.
+  llvm::CrashRecoveryContext crc;
+  if (!crc.RunSafely([&]() { CommonLinkerContext::destroy(); })) {
+    // The memory is corrupted beyond any possible recovery.
+    return {r, /*canRunAgain=*/false};
+  }
+  return {r, /*canRunAgain=*/true};
+}
+// EVM local end

lld/ELF/Arch/EVM.cpp

@@ -0,0 +1,94 @@
+//===- EVM.cpp ------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// EVM is a stack-based virtual machine with a word size of 256 bits intendent
+// for execution of smart contracts in Ethereum blockchain environment.
+//
+// Since it is a baremetal programming, there's usually no loader to load
+// ELF files on EVMs. You are expected to link your program against address
+// 0 and pull out a .text section from the result using objcopy, so that you
+// can write the linked code to on-chip flush memory. You can do that with
+// the following commands:
+//
+//   ld.lld -Ttext=0 -o foo foo.o
+//   objcopy -O binary --only-section=.text foo output.bin
+//
+//===----------------------------------------------------------------------===//
+
+#include "InputFiles.h"
+#include "Symbols.h"
+#include "Target.h"
+#include "lld/Common/ErrorHandler.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/Support/Endian.h"
+
+using namespace llvm;
+using namespace llvm::object;
+using namespace llvm::support::endian;
+using namespace llvm::ELF;
+using namespace lld;
+using namespace lld::elf;
+
+namespace {
+class EVM final : public TargetInfo {
+public:
+  uint32_t calcEFlags() const override;
+  RelExpr getRelExpr(RelType type, const Symbol &s,
+                     const uint8_t *loc) const override;
+  void relocate(uint8_t *loc, const Relocation &rel,
+                uint64_t val) const override;
+};
+} // namespace
+
+RelExpr EVM::getRelExpr(RelType type, const Symbol &s,
+                        const uint8_t *loc) const {
+  switch (type) {
+  case R_EVM_DATA:
+    return R_ABS;
+  default:
+    error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) +
+          ") against symbol " + toString(s));
+    return R_NONE;
+  }
+}
+
+void EVM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
+  switch (rel.type) {
+  case R_EVM_DATA: {
+    if (val > std::numeric_limits<uint32_t>::max())
+      llvm_unreachable("R_EVM_DATA: to big relocation value");
+    write32be(loc, val);
+    break;
+  }
+  default:
+    llvm_unreachable("unknown relocation");
+  }
+}
+
+TargetInfo *elf::getEVMTargetInfo() {
+  static EVM target;
+  return &target;
+}
+
+static uint32_t getEFlags(InputFile *file) {
+  return cast<ObjFile<ELF32LE>>(file)->getObj().getHeader().e_flags;
+}
+
+uint32_t EVM::calcEFlags() const {
+  assert(!ctx.objectFiles.empty());
+
+  const uint32_t flags = getEFlags(ctx.objectFiles[0]);
+  for (InputFile *f : ArrayRef(ctx.objectFiles).slice(1)) {
+    const uint32_t objFlags = getEFlags(f);
+    if ((objFlags /* & EF_EVM_ARCH_MASK*/) != (flags /* & EF_EVM_ARCH_MASK*/))
+      error(toString(f) +
+            ": cannot link object files with incompatible target ISA");
+  }
+
+  return flags;
+}

lld/ELF/CMakeLists.txt

@@ -24,6 +24,7 @@ add_lld_library(lldELF
   Arch/AMDGPU.cpp
   Arch/ARM.cpp
   Arch/AVR.cpp
+  Arch/EVM.cpp
   Arch/EraVM.cpp
   Arch/Hexagon.cpp
   Arch/LoongArch.cpp

lld/ELF/Config.h

@@ -150,6 +150,11 @@ struct Config {
   llvm::CachePruningPolicy thinLTOCachePolicy;
   llvm::SetVector<llvm::CachedHashString> dependencyFiles; // for --dependency-file
   llvm::StringMap<uint64_t> sectionStartMap;
+  // EVM local begin
+  llvm::ArrayRef<MemoryBufferRef> inData;
+  llvm::raw_pwrite_stream *outData;
+  bool useMemBuf = false;
+  // EVM local end
   llvm::StringRef bfdname;
   llvm::StringRef chroot;
   llvm::StringRef dependencyFile;