Synced with the internal version of the code:

- Added support for new versions of the Intel SDM. - Removed support for older versions of the SDM (2018 and older). These versions do not parse correctly with the latest version of the code; they can still be parsed by reverting to an earlier snapshot. - Removed the rule that downloaded and parsed the SDM on build to avoid broken builds when the SDM is updated at the same URL. - More stable character clustering algorithm. - Added definitions of recent x86-64 microarchitectures. - Updated dependencies as needed. - Refactoring, reformatting, code cleanups.
google · Jan 21, 2022 · 5509312 · 5509312
1 parent ca6ab91
commit 5509312
Show file tree

Hide file tree

Showing 129 changed files with 4,870 additions and 8,832 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -1,4 +1,4 @@
 build --copt=-Wno-parentheses --copt=-Wno-sign-compare  --copt=-D_XOPEN_SOURCE \
   --copt=-Wno-misleading-indentation --copt=-Wno-narrowing --define USE_GLPK= \
-  --strict_proto_deps=off --cxxopt=-std=c++17 --host_cxxopt=-std=c++17
+  --strict_proto_deps=off --cxxopt=-std=c++20 --host_cxxopt=-std=c++20
 
diff --git a/BUILD b/BUILD
@@ -10,6 +10,20 @@ exports_files([
     "llvm.bzl",
 ])
 
+# Generates a text file with the version of LLVM used in the build. The version
+# is required to generate some of the LLVM headers, but it is not possible to
+# get the version from inside the @llvm_git repository.
+# We use the git hash or tag as a version string; this hash or tag is extracted
+# from the llvm_git rule in the WORKSPACE file, to avoid duplication of this
+# information.
+genrule(
+    name = "llvm_git_revision_gen",
+    srcs = ["WORKSPACE"],
+    outs = ["llvm_git_revision"],
+    cmd = r"""perl -ne 'if (/strip_prefix = "llvm-project-(.*)\/llvm"/) { print "$$1\n"; exit } ' < $< > $@""",
+    visibility = ["//visibility:public"],
+)
+
 package_group(
     name = "internal_users",
     packages = [

diff --git a/WORKSPACE b/WORKSPACE
@@ -1,14 +1,14 @@
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 # ===== Abseil =====
 
 http_archive(
     name = "com_google_absl",
-    sha256 = "38f5113dff685ddfd3bf53d1473a18dfcfe8de89ee7c9b3708e3a070274c6b38",
-    strip_prefix = "abseil-cpp-e63a5a61045e516b7b3dbca090e2b9ff1d057e46",
+    sha256 = "a4567ff02faca671b95e31d315bab18b42b6c6f1a60e91c6ea84e5a2142112c2",
+    strip_prefix = "abseil-cpp-20211102.0",
     urls = [
-        "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/e63a5a61045e516b7b3dbca090e2b9ff1d057e46.tar.gz",
-        "https://github.com/abseil/abseil-cpp/archive/e63a5a61045e516b7b3dbca090e2b9ff1d057e46.tar.gz",
+        "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.zip",
+        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.zip",
     ],
 )
 
@@ -21,13 +21,15 @@ http_archive(
 
 # ===== re2 =====
 
+# NOTE(ondrasej): The commit for the RE2 library must be taken from the Abseil
+# branch so that RE2 works correctly with absl::string_view.
 http_archive(
     name = "com_googlesource_code_re2",
-    sha256 = "5306526bcdf35ff34c67913bef8f7b15a3960f4f0ab3a2b6a260af4f766902d4",
-    strip_prefix = "re2-c4f65071cc07eb34d264b25f7b9bbb679c4d5a5a",
+    sha256 = "fb002599c6620611368933083b4cff031a32c8a98638877fe097f56142646ba9",
+    strip_prefix = "re2-e8cb5ecb8ee1066611aa937a42fa10514edf30fb",
     urls = [
-        "https://mirror.bazel.build/github.com/google/re2/archive/c4f65071cc07eb34d264b25f7b9bbb679c4d5a5a.tar.gz",
-        "https://github.com/google/re2/archive/c4f65071cc07eb34d264b25f7b9bbb679c4d5a5a.tar.gz",
+        "https://mirror.bazel.build/github.com/google/re2/archive/e8cb5ecb8ee1066611aa937a42fa10514edf30fb.zip",
+        "https://github.com/google/re2/archive/e8cb5ecb8ee1066611aa937a42fa10514edf30fb.zip",
     ],
 )
 
@@ -36,35 +38,63 @@ http_archive(
 
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "2ddbc2f0c30e8f9ee7cc8281707859827c143a1dc088f74d38aca66fe6b664a2",
-    strip_prefix = "protobuf-93a18cf2be6d590d23f34757a32133306e290ca9",
+    strip_prefix = "protobuf-3.19.3",
+    sha256 = "6b6bf5cd8d0cca442745c4c3c9f527c83ad6ef35a405f64db5215889ac779b42",
     urls = [
-        "https://mirror.bazel.build/github.com/google/protobuf/archive/93a18cf2be6d590d23f34757a32133306e290ca9.tar.gz",
-        "https://github.com/google/protobuf/archive/93a18cf2be6d590d23f34757a32133306e290ca9.tar.gz",
+        "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/refs/tags/v3.19.3.zip",
+        "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.19.3.zip",
     ],
 )
 
-load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-protobuf_deps()
-
 http_archive(
     name = "com_google_protobuf_cc",
-    sha256 = "2ddbc2f0c30e8f9ee7cc8281707859827c143a1dc088f74d38aca66fe6b664a2",
-    strip_prefix = "protobuf-93a18cf2be6d590d23f34757a32133306e290ca9",
+    strip_prefix = "protobuf-3.19.3",
+    sha256 = "6b6bf5cd8d0cca442745c4c3c9f527c83ad6ef35a405f64db5215889ac779b42",
+    urls = [
+        "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/refs/tags/v3.19.3.zip",
+        "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.19.3.zip",
+    ],
+)
+
+http_archive(
+    name = "rules_cc",
+    sha256 = "08f3b5c25eba2f7fb51db25692c2b9df6c557e0d8084bd8dc5e936d239f7641d",
+    strip_prefix = "rules_cc-22532c537959149599e79df29808176345784cc1",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/22532c537959149599e79df29808176345784cc1.tar.gz",
+        "https://github.com/bazelbuild/rules_cc/archive/22532c537959149599e79df29808176345784cc1.tar.gz",
+    ],
+)
+
+# rules_proto defines abstract rules for building Protocol Buffers.
+http_archive(
+    name = "rules_proto",
+    sha256 = "20b240eba17a36be4b0b22635aca63053913d5c1ee36e16be36499d167a2f533",
+    strip_prefix = "rules_proto-11bf7c25e666dd7ddacbcd4d4c4a9de7a25175f8",
     urls = [
-        "https://mirror.bazel.build/github.com/google/protobuf/archive/93a18cf2be6d590d23f34757a32133306e290ca9.tar.gz",
-        "https://github.com/google/protobuf/archive/93a18cf2be6d590d23f34757a32133306e290ca9.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/11bf7c25e666dd7ddacbcd4d4c4a9de7a25175f8.tar.gz",
+        "https://github.com/bazelbuild/rules_proto/archive/11bf7c25e666dd7ddacbcd4d4c4a9de7a25175f8.tar.gz",
     ],
 )
 
-# Required by protobuf.
 http_archive(
     name = "bazel_skylib",
-    sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d",
-    strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
+    sha256 = "fc64d71583f383157e3e5317d24e789f942bc83c76fde7e5981cadc097a3c3cc",
+    strip_prefix = "bazel-skylib-1.1.1",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/archive/refs/tags/1.1.1.zip",
+        "https://github.com/bazelbuild/bazel-skylib/archive/refs/tags/1.1.1.zip",
+    ],
 )
 
+load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies", "rules_cc_toolchains")
+rules_cc_dependencies()
+rules_cc_toolchains()
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+rules_proto_dependencies()
+rules_proto_toolchains()
+
 # ===== googletest =====
 
 http_archive(
@@ -179,23 +209,28 @@ http_archive(
 http_archive(
     name = "llvm_git",
     build_file = "@//:llvm.BUILD",
-    sha256 = "95b3a609cc0db719ea19daed2689a770f97aea974a3d2cf70468d3b067e8b068",
-    strip_prefix = "llvm-project-c10248829357fd90030ba091e01b6c253e5848f1/llvm",
+    sha256 = "a84450dd5c585c69ccf82e6e1e877d689bc9366221b955e2b2c8f02a47a4639a",
+    strip_prefix = "llvm-project-579c4921c0105698617cc1e1b86a9ecf3b4717ce/llvm",
     urls = [
         # Please don't refactor out the SHA; if URLs aren't greppable,
         # we can't offer you an asynchronous mirroring service that
         # allows this huge archive to download in one second.
-        "https://mirror.bazel.build/github.com/llvm/llvm-project/archive/c10248829357fd90030ba091e01b6c253e5848f1.tar.gz",
-        "https://github.com/llvm/llvm-project/archive/c10248829357fd90030ba091e01b6c253e5848f1.tar.gz",
+        "https://mirror.bazel.build/github.com/llvm/llvm-project/archive/579c4921c0105698617cc1e1b86a9ecf3b4717ce.tar.gz",
+        "https://github.com/llvm/llvm-project/archive/579c4921c0105698617cc1e1b86a9ecf3b4717ce.tar.gz",
     ],
 )
 
-# ===== Intel SDM =====
-
-http_file(
-    name = "intel_sdm_pdf",
-    sha256 = "2b16c29d13aca1b98ea61f72ca0123649334564f90472db10067957b229e0dae",
-    urls = ["https://software.intel.com/content/dam/develop/public/us/en/documents/325462-sdm-vol-1-2abcd-3abcd.pdf"],
+http_archive(
+    name = "llvm_git_bazel_overlay",
+    sha256 = "a84450dd5c585c69ccf82e6e1e877d689bc9366221b955e2b2c8f02a47a4639a",
+    strip_prefix = "llvm-project-579c4921c0105698617cc1e1b86a9ecf3b4717ce/utils/bazel/llvm-project-overlay/llvm",
+    urls = [
+        # Please don't refactor out the SHA; if URLs aren't greppable,
+        # we can't offer you an asynchronous mirroring service that
+        # allows this huge archive to download in one second.
+        "https://mirror.bazel.build/github.com/llvm/llvm-project/archive/579c4921c0105698617cc1e1b86a9ecf3b4717ce.tar.gz",
+        "https://github.com/llvm/llvm-project/archive/579c4921c0105698617cc1e1b86a9ecf3b4717ce.tar.gz",
+    ],
 )
 
 # ==============================================================================

diff --git a/exegesis/arm/xml/BUILD b/exegesis/arm/xml/BUILD
@@ -144,6 +144,7 @@ cc_test(
     deps = [
         ":parser",
         "//exegesis/testing:test_util",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
         "@com_google_googletest//:gtest_main",

diff --git a/exegesis/arm/xml/converter_test.cc b/exegesis/arm/xml/converter_test.cc
@@ -30,7 +30,7 @@ using ::exegesis::testing::EqualsProto;
 using ::google::protobuf::TextFormat;
 
 TEST(ConverterTest, ConvertToArchitectureProto) {
-  static constexpr char kXmlDatabase[] = R"proto(
+  static constexpr char kXmlDatabase[] = R"pb(
     base_index {
       files {
         filename: "instruction_1.xml"
@@ -198,11 +198,11 @@ TEST(ConverterTest, ConvertToArchitectureProto) {
           docvars { mnemonic: "I3" cond_setting: S isa: A32 }
         }
       }
-    })proto";
+    })pb";
   XmlDatabase xml_database;
   ASSERT_TRUE(TextFormat::ParseFromString(kXmlDatabase, &xml_database));
 
-  static constexpr char kExpectedArchitecture[] = R"proto(
+  static constexpr char kExpectedArchitecture[] = R"pb(
     name: 'ARMv8'
     instruction_set {
       source_infos { source_name: "ARM XML Database" }
@@ -301,7 +301,7 @@ TEST(ConverterTest, ConvertToArchitectureProto) {
         flags_affected { content: "S" }
         short_description: "Third instruction group."
       }
-    })proto";
+    })pb";
   EXPECT_THAT(ConvertToArchitectureProto(xml_database),
               EqualsProto(kExpectedArchitecture));
 }

diff --git a/exegesis/arm/xml/parser_test.cc b/exegesis/arm/xml/parser_test.cc
@@ -16,6 +16,7 @@
 
 #include <string>
 
+#include "absl/flags/flag.h"
 #include "absl/status/status.h"
 #include "absl/strings/str_cat.h"
 #include "exegesis/testing/test_util.h"
@@ -39,7 +40,7 @@ std::string GetFilePath(const std::string& filename) {
 }
 
 TEST(ParserTest, ParseXmlIndex) {
-  static constexpr char kExpectedIndex[] = R"proto(
+  static constexpr char kExpectedIndex[] = R"pb(
     isa: A64
     files {
       filename: "instruction_1.xml"
@@ -52,7 +53,7 @@ TEST(ParserTest, ParseXmlIndex) {
       heading: "I2"
       xml_id: "id_i_2"
       description: "Second instruction."
-    })proto";
+    })pb";
   EXPECT_THAT(ParseXmlIndex(GetFilePath("index.xml")),
               IsOkAndHolds(EqualsProto(kExpectedIndex)));
 
@@ -68,7 +69,7 @@ TEST(ParserTest, ParseXmlIndex_Missing) {
 }
 
 TEST(ParserTest, ParseXmlInstruction) {
-  static constexpr char kExpectedInstruction[] = R"proto(
+  static constexpr char kExpectedInstruction[] = R"pb(
     xml_id: "id_i_1"
     heading: "I1 Title"
     brief_description: "First instruction"
@@ -192,13 +193,13 @@ TEST(ParserTest, ParseXmlInstruction) {
           pieces { text: ", 1" }
         }
       }
-    })proto";
+    })pb";
   EXPECT_THAT(ParseXmlInstruction(GetFilePath("instruction_1.xml")),
               IsOkAndHolds(EqualsProto(kExpectedInstruction)));
 }
 
 TEST(ParserTest, ParseXmlInstructionWithConstraints) {
-  static constexpr char kExpectedInstruction[] = R"proto(
+  static constexpr char kExpectedInstruction[] = R"pb(
     xml_id: "id_i_2"
     heading: "I2 Title"
     brief_description: "Second instruction"
@@ -234,7 +235,7 @@ TEST(ParserTest, ParseXmlInstructionWithConstraints) {
         }
         asm_template { pieces { text: "I2 " } }
       }
-    })proto";
+    })pb";
   EXPECT_THAT(ParseXmlInstruction(GetFilePath("instruction_2.xml")),
               IsOkAndHolds(EqualsProto(kExpectedInstruction)));
 }
@@ -246,7 +247,7 @@ TEST(ParserTest, ParseXmlInstruction_Missing) {
 }
 
 TEST(ParserTest, ParseXmlDatabase) {
-  static constexpr char kExpectedXmlDatabase[] = R"proto(
+  static constexpr char kExpectedXmlDatabase[] = R"pb(
     base_index {
       isa: A64
       files {
@@ -431,7 +432,7 @@ TEST(ParserTest, ParseXmlDatabase) {
           asm_template { pieces { text: "I2 " } }
         }
       }
-    })proto";
+    })pb";
   EXPECT_THAT(ParseXmlDatabase(GetFilePath("")),
               IsOkAndHolds(EqualsProto(kExpectedXmlDatabase)));
 }

diff --git a/exegesis/base/BUILD b/exegesis/base/BUILD
@@ -77,6 +77,7 @@ cc_test(
         ":architecture_provider",
         "//exegesis/testing:test_util",
         "//exegesis/util:proto_util",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
@@ -95,7 +96,6 @@ cc_library(
         "//exegesis/proto:instructions_cc_proto",
         "//exegesis/util:instruction_syntax",
         "//exegesis/util:status_util",
-        "//util/gtl:map_util",
         "@com_github_glog_glog//:glog",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/flags:flag",
@@ -195,7 +195,6 @@ cc_library(
     deps = [
         "//base",
         "//exegesis/proto:microarchitecture_cc_proto",
-        "//util/gtl:map_util",
         "@com_github_glog_glog//:glog",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",

diff --git a/exegesis/base/architecture_provider_test.cc b/exegesis/base/architecture_provider_test.cc
@@ -14,6 +14,7 @@
 
 #include "exegesis/base/architecture_provider.h"
 
+#include "absl/flags/flag.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"

diff --git a/exegesis/base/architecture_test.cc b/exegesis/base/architecture_test.cc
@@ -35,7 +35,7 @@ using ::exegesis::testing::EqualsProto;
 using ::testing::Contains;
 using ::testing::HasSubstr;
 
-const char kArchitectureProto[] = R"proto(
+const char kArchitectureProto[] = R"pb(
   instruction_set {
     instructions {
       llvm_mnemonic: 'ADD8rm'
@@ -77,7 +77,7 @@ const char kArchitectureProto[] = R"proto(
       encoding_scheme: 'MR'
       description: 'Move r16 to r16/m16.'
     }
-  })proto";
+  })pb";
 
 bool CompareProtosByLLVMMnemonic(const InstructionProto& left,
                                  const InstructionProto& right) {
@@ -163,7 +163,7 @@ TEST_F(ArchitectureTest, GetInstructionIndicesByRawEncodingSpecification) {
 }
 
 TEST(PrintInstructionSetProtoProtoTest, PrintZeroIndexedGroup) {
-  constexpr char kArchitectureProto[] = R"proto(
+  constexpr char kArchitectureProto[] = R"pb(
     instructions {
       description: "Enter VMX root operation."
       vendor_syntax {
@@ -172,7 +172,7 @@ TEST(PrintInstructionSetProtoProtoTest, PrintZeroIndexedGroup) {
       }
       raw_encoding_specification: "F3 0F C7 /6"
       instruction_group_index: 0
-    })proto";
+    })pb";
   const InstructionSetProto proto =
       ParseProtoFromStringOrDie<InstructionSetProto>(kArchitectureProto);
 

diff --git a/exegesis/base/cleanup_instruction_set.cc b/exegesis/base/cleanup_instruction_set.cc
@@ -29,7 +29,6 @@
 #include "src/google/protobuf/io/zero_copy_stream_impl_lite.h"
 #include "src/google/protobuf/repeated_field.h"
 #include "src/google/protobuf/util/message_differencer.h"
-#include "util/gtl/map_util.h"
 
 ABSL_FLAG(bool, exegesis_print_transform_names_to_log, true,
           "Print the names of the transforms executed by the transform "
@@ -103,7 +102,7 @@ RegisterInstructionSetTransform::RegisterInstructionSetTransform(
     InstructionSetTransformRawFunction transform) {
   InstructionSetTransformsByName& transforms_by_name =
       *GetMutableTransformsByName();
-  CHECK(!gtl::ContainsKey(transforms_by_name, transform_name))
+  CHECK(!transforms_by_name.contains(transform_name))
       << "Transform name '" << transform_name << "' is already used!";
   InstructionSetTransform transform_wrapper =
       [transform_name, transform](InstructionSetProto* instruction_set) {