Merge pull request #10 in ANDROID/buck from no/upstream to master

* commit 'a36df7f27840a50c81a4417507ba80550d492dc0': (59 commits) Revert: Fix typo in WatchedFileHashCache Revert: Inverted order of FHC gets to assess impact of loading a file from memory. use regex for field_name in diag tool no compilation flags for sanitizing for VS toolchain Fix for duplicate log messages from ClientExecutor's scheduler thread. Added oncall and excludes for buck targets Remove MapDB and RocksDB Write cross-cell used classes paths pre-pended by cell name Add CellPathResolver as a second param to SuppportsDependencyFileRuleKey#getInputsAfterBuildingLocally() Make CompareAbis play nice with caching Verify SHA1 after materializing source files. Fix typo in WatchedFileHashCache Generate package-info class when needed fix build Teach InnerClassesTable to be able to examine PackageElements Detect inner class references in default values of annotation parameters Don't list inner classes of inner classes (unless they are referenced) Don't list inner classes more than once Include references to inner classes from type parameter annotations InnerClassesTable and tests ...
cwoodwar6 · May 26, 2017 · 7e25c5e · 7e25c5e
2 parents 18b5de7 + a36df7f
commit 7e25c5e
Show file tree

Hide file tree

Showing 255 changed files with 4,390 additions and 2,824 deletions.
diff --git a/buck.iml b/buck.iml
@@ -37,6 +37,7 @@
       <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/haskell/testdata" />
       <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/ide/intellij/deprecated/testdata" />
       <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/ide/intellij/testdata" />
+      <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/ide/intellij/projectview/testdata" />
       <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/io/testdata" />
       <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/js/testdata" />
       <excludeFolder url="file://$MODULE_DIR$/test/com/facebook/buck/jvm/groovy/testdata" />
@@ -91,14 +92,8 @@
     <orderEntry type="library" name="xz-java-1.5" level="project" />
     <orderEntry type="library" name="jna-4.2.0" level="project" />
     <orderEntry type="library" name="jna-platform-4.2.0" level="project" />
-    <orderEntry type="library" name="rocksdbjni-5.1.2" level="project" />
     <orderEntry type="library" name="diffutils-1.3.0" level="project" />
     <orderEntry type="library" name="jdom" level="project" />
-    <orderEntry type="library" name="eclipse-collections" level="project" />
-    <orderEntry type="library" name="jcip-annotations" level="project" />
-    <orderEntry type="library" name="kotlin" level="project" />
-    <orderEntry type="library" name="lz4" level="project" />
-    <orderEntry type="library" name="mapdb" level="project" />
     <orderEntry type="library" name="okhttp-3.6.0" level="project" />
     <orderEntry type="library" name="sqlite" level="project" />
   </component>

diff --git a/build.xml b/build.xml
@@ -87,16 +87,6 @@
     <include name="nailgun/nailgun-server-0.9.2-SNAPSHOT.jar" />
     <include name="jna/jna-4.2.0.jar" />
     <include name="jna/jna-platform-4.2.0.jar" />
-    <include name="rocksdbjni/rocksdbjni-5.1.2.jar" />
-    <include name="eclipse-collections/eclipse-collections-7.1.1.jar" />
-    <include name="eclipse-collections/eclipse-collections-api-7.1.1.jar" />
-    <include name="eclipse-collections/eclipse-collections-forkjoin-7.1.1.jar" />
-    <include name="jcip-annotations/jcip-annotations-1.0.jar" />
-    <include name="kotlin/kotlin-runtime-1.0.6.jar" />
-    <include name="kotlin/kotlin-stdlib-1.0.6.jar" />
-    <include name="lz4/lz4-1.3.0.jar" />
-    <include name="mapdb/elsa-3.0.0-M5.jar" />
-    <include name="mapdb/mapdb-3.0.3.jar" />
     <include name="sqlite/sqlite-jdbc-3.16.1.jar" />
   </fileset>
 

diff --git a/programs/buck_repo.py b/programs/buck_repo.py
@@ -33,9 +33,6 @@
     "third-party/java/dd-plist/dd-plist.jar",
     "third-party/java/ddmlib/ddmlib-25.2.0.jar",
     "third-party/java/diffutils/diffutils-1.3.0.jar",
-    "third-party/java/eclipse-collections/eclipse-collections-7.1.1.jar",
-    "third-party/java/eclipse-collections/eclipse-collections-api-7.1.1.jar",
-    "third-party/java/eclipse-collections/eclipse-collections-forkjoin-7.1.1.jar",
     "third-party/java/eclipse/org.eclipse.core.contenttype_3.5.100.v20160418-1621.jar",
     "third-party/java/eclipse/org.eclipse.core.jobs_3.8.0.v20160319-0610.jar",
     "third-party/java/eclipse/org.eclipse.core.resources_3.11.0.v20160422-0304.jar",
@@ -61,26 +58,19 @@
     "third-party/java/jackson/jackson-databind-2.7.8.jar",
     "third-party/java/jackson/jackson-datatype-guava-2.7.8.jar",
     "third-party/java/jackson/jackson-datatype-jdk8-2.7.8.jar",
-    "third-party/java/jcip-annotations/jcip-annotations-1.0.jar",
     "third-party/java/jdom/jdom-2.0.6.jar",
     "third-party/java/jetty/jetty-all-9.2.10.v20150310.jar",
     "third-party/java/jna/jna-4.2.0.jar",
     "third-party/java/jna/jna-platform-4.2.0.jar",
     "third-party/java/jsr/javax.inject-1.jar",
     "third-party/java/jsr/jsr305.jar",
-    "third-party/java/kotlin/kotlin-runtime-1.0.6.jar",
-    "third-party/java/kotlin/kotlin-stdlib-1.0.6.jar",
     "third-party/java/kxml2/kxml2-2.3.0.jar",
-    "third-party/java/lz4/lz4-1.3.0.jar",
-    "third-party/java/mapdb/elsa-3.0.0-M5.jar",
-    "third-party/java/mapdb/mapdb-3.0.3.jar",
     "third-party/java/nailgun/nailgun-server-0.9.2-SNAPSHOT.jar",
     "third-party/java/nuprocess/nuprocess-1.1.0.jar",
     "third-party/java/ObjCBridge/ObjCBridge.jar",
     "third-party/java/okhttp/okhttp-3.6.0.jar",
     "third-party/java/okio/okio-1.11.0.jar",
     "third-party/java/oshi/oshi-core-3.3-SNAPSHOT.jar",
-    "third-party/java/rocksdbjni/rocksdbjni-5.1.2.jar",
     "third-party/java/servlet-api/javax.servlet-api-3.1.0.jar",
     "third-party/java/slf4j/slf4j-jdk14-1.7.5.jar",
     "third-party/java/sqlite/sqlite-jdbc-3.16.1.jar",

diff --git a/scripts/diff_rulekeys.py b/scripts/diff_rulekeys.py
@@ -443,13 +443,15 @@ def diffInternal(
 
 
 def diffAndReturnSeen(starting_refs, left_info, right_info, verbose,
-                      format_tuple, check_paths, seen_keys):
+                      format_tuple, check_paths, seen_keys, excludes):
     queue = collections.deque(starting_refs)
     result = []
     visited_keys = []
     while len(queue) > 0:
         p = queue.popleft()
         label, ref_pair = p
+        if label in excludes:
+            continue
         (left_key, right_key) = ref_pair
         visited_keys.append(ref_pair)
         report, changed_key_pairs_with_labels = diffInternal(
@@ -482,15 +484,15 @@ def diff(name, left_info, right_info, verbose, format_tuple=None,
         raise KeyError('Right log does not contain ' + name)
     result, _ = diffAndReturnSeen([(name, (left_key, right_key))], left_info,
                                   right_info, verbose, format_tuple,
-                                  check_paths, set())
+                                  check_paths, set(), set())
     if not result and left_key != right_key:
         result.append("I don't know why RuleKeys for {} do not match.".format(
             name))
     return result
 
 
 def diffAll(left_info, right_info, verbose, format_tuple=None,
-            check_paths=False):
+            check_paths=False, excludes=set()):
     # Ghetto ordered set implementation.
     seen_left_names = collections.OrderedDict(
         [(k, True) for k in left_info.getAllNames()])
@@ -519,8 +521,8 @@ def diffAll(left_info, right_info, verbose, format_tuple=None,
         all_seen_before = len(all_seen)
         single_result, visited_keys = diffAndReturnSeen(
                 [(name, (left_key, right_key))], left_info, right_info,
-                verbose, format_tuple, check_paths, all_seen)
-        if not single_result and left_key != right_key:
+                verbose, format_tuple, check_paths, all_seen, excludes)
+        if name not in excludes and not single_result and left_key != right_key:
             single_result.append(
                 "I don't know why RuleKeys for {} do not match.".format(name))
         all_results.extend(single_result)

diff --git a/scripts/rulekey_diff2.py b/scripts/rulekey_diff2.py
@@ -98,6 +98,9 @@ def read_rulekeys_autodetect(path):
 TOKEN_VALUE_REGEX = re.compile(r'[a-zA-Z]+\((.*)\)')
 CONTAINER_LENGTH_REGEX = re.compile(r'container\(.*,len=(\d+)\)')
 
+TARGET_NAME_TOKEN_PATTERN = r'key\(\.name\)'
+TARGET_NAME_FIELD_PATTERN = r'\.name'
+RULEKEY_TYPE_FIELD_PATTERN = r'\.key_type'
 
 def token_type(token):
     match = TOKEN_TYPE_REGEX.match(token)
@@ -334,7 +337,7 @@ def find_keys(keys, criteria):
             field_found = field_name is None
             for token in tokens:
                 if field_name is not None and token_type(token) == 'key':
-                    field_found = token_value(token) == field_name
+                    field_found = re.search(field_name, token_value(token)) is not None
                 if field_found and re.search(pattern, token) is not None:
                     break
             else:
@@ -350,7 +353,7 @@ def extract_target(struct):
     """
     Extracts the target name from a structured rulekey
     """
-    for s in find_children(struct, '.name', 1):
+    for s in find_children(struct, TARGET_NAME_TOKEN_PATTERN, 1):
         if len(s) == 1 and token_type(s[0].token) == 'string':
             return token_value(s[0].token)[1:-1]  # strip quotes
     return None
@@ -381,7 +384,10 @@ def find_keys_for_target(keys_left, keys_right, target):
     print('Searching for the specified build target...')
     print(target)
     target_pattern = '"' + target + '"'
-    criteria = [(target_pattern, '.name'), (r'DEFAULT', '.key_type')]
+    criteria = [
+        (target_pattern, TARGET_NAME_FIELD_PATTERN),
+        (r'DEFAULT', RULEKEY_TYPE_FIELD_PATTERN),
+    ]
     rulekeys_left = find_keys(keys_left, criteria)
     rulekeys_right = find_keys(keys_right, criteria)
     if not check_rulekeys_count(rulekeys_left, 'left', target):
@@ -406,39 +412,45 @@ def diff_keys_recursively(
     reported_keys = set()
     reported_differences = set()
 
+    def print_path():
+        for (rk1, rk2) in keys_path_stack:
+            if rk1 in reported_keys and rk2 in reported_keys:
+                continue
+            reported_keys.add(rk1)
+            reported_keys.add(rk2)
+            target = 'unknown'
+            if rk1 in keys_left:
+                target = extract_target(reconstruct_rulekey(keys_left[rk1]))
+            elif rk2 in keys_right:
+                target = extract_target(reconstruct_rulekey(keys_right[rk2]))
+            print('%s vs %s (%s) ...' % (rk1, rk2, target))
+
     def rec(rulekey1, rulekey2):
         if rulekey1 in processed_keys and rulekey2 in processed_keys:
             return
-        if rulekey1 not in keys_left or rulekey2 not in keys_right:
-            # no more rulekey diagnostics data, probably due to a cache hit so not built locally
-            return
-
         processed_keys.add(rulekey1)
         processed_keys.add(rulekey2)
         keys_path_stack.append((rulekey1, rulekey2))
-
-        struct1 = reconstruct_rulekey(keys_left[rulekey1])
-        struct2 = reconstruct_rulekey(keys_right[rulekey2])
-        for (v1, s1, v2, s2) in diff_rulekeys_to_list(struct1, struct2):
-            t1 = get_token(s1)
-            t2 = get_token(s2)
-            if len(reported_differences) >= max_differences_to_report:
-                print('stopped after reporting %d differences' % len(reported_differences))
-                break
-            if get_type(s1) == 'ruleKey' and get_type(s2) == 'ruleKey':
-                # value looks like 'sha1=8960bfa79841fa1482f7883f28ea54c78a5ed26a', strip 'sha1='
-                rec(token_value(t1)[5:], token_value(t2)[5:])
-            elif not report_only_unique_causes or (t1, t2) not in reported_differences:
-                reported_differences.add((t1, t2))
-                for (rk1, rk2) in keys_path_stack:
-                    if rk1 in reported_keys and rk2 in reported_keys:
-                        continue
-                    reported_keys.add(rk1)
-                    reported_keys.add(rk2)
-                    target = extract_target(reconstruct_rulekey(keys_left[rk1]))
-                    print('%s vs %s (%s) ...' % (rk1, rk2, target))
-                print('  L: ' + v1)
-                print('  R: ' + v2)
+        if rulekey1 in keys_left and rulekey2 in keys_right:
+            struct1 = reconstruct_rulekey(keys_left[rulekey1])
+            struct2 = reconstruct_rulekey(keys_right[rulekey2])
+            for (v1, s1, v2, s2) in diff_rulekeys_to_list(struct1, struct2):
+                t1 = get_token(s1)
+                t2 = get_token(s2)
+                if len(reported_differences) >= max_differences_to_report:
+                    print('stopped after reporting %d differences' % len(reported_differences))
+                    break
+                if get_type(s1) == 'ruleKey' and get_type(s2) == 'ruleKey':
+                    # value looks like 'sha1=8960bfa79841fa1482f7...', strip 'sha1='
+                    rec(token_value(t1)[5:], token_value(t2)[5:])
+                elif not report_only_unique_causes or (t1, t2) not in reported_differences:
+                    reported_differences.add((t1, t2))
+                    print_path()
+                    print('  L:', v1)
+                    print('  R:', v2)
+        else:
+            # no more rulekey diagnostics data, probably due to a cache hit so not built locally
+            pass
         keys_path_stack.pop()
 
     rec(keys_tuple_to_diff[0], keys_tuple_to_diff[1])

diff --git a/src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java b/src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java
@@ -104,7 +104,8 @@ public BuildRule createBuildRule(
           installableApk.getFullyQualifiedName(),
           installableApk.getType());
     }
-    AndroidBinary apkUnderTest = getUnderlyingApk((HasInstallableApk) installableApk);
+    AndroidBinary apkUnderTest =
+        ApkGenruleDescription.getUnderlyingApk((HasInstallableApk) installableApk);
 
     ImmutableSortedSet<JavaLibrary> rulesToExcludeFromDex =
         new ImmutableSortedSet.Builder<>(Ordering.<JavaLibrary>natural())
@@ -190,18 +191,6 @@ public BuildRule createBuildRule(
         dxExecutorService);
   }
 
-  private static AndroidBinary getUnderlyingApk(HasInstallableApk installable) {
-    if (installable instanceof AndroidBinary) {
-      return (AndroidBinary) installable;
-    } else if (installable instanceof ApkGenrule) {
-      return getUnderlyingApk(((ApkGenrule) installable).getInstallableApk());
-    } else {
-      throw new IllegalStateException(
-          installable.getBuildTarget().getFullyQualifiedName()
-              + " must be backed by either an android_binary() or an apk_genrule()");
-    }
-  }
-
   @BuckStyleImmutable
   @Value.Immutable
   interface AbstractAndroidInstrumentationApkDescriptionArg

diff --git a/src/com/facebook/buck/android/AndroidInstrumentationTest.java b/src/com/facebook/buck/android/AndroidInstrumentationTest.java
@@ -115,7 +115,7 @@ public boolean supportsStreamingTests() {
   public ImmutableList<Step> runTests(
       ExecutionContext executionContext,
       TestRunningOptions options,
-      SourcePathResolver pathResolver,
+      BuildContext buildContext,
       TestReportingCallback testReportingCallback) {
     Preconditions.checkArgument(executionContext.getAdbOptions().isPresent());
 
@@ -128,10 +128,12 @@ public ImmutableList<Step> runTests(
 
     Path pathToTestOutput = getPathToTestOutputDirectory();
     steps.addAll(MakeCleanDirectoryStep.of(getProjectFilesystem(), pathToTestOutput));
-    steps.add(new ApkInstallStep(pathResolver, apk));
+    steps.add(new ApkInstallStep(buildContext.getSourcePathResolver(), apk));
     if (apk instanceof AndroidInstrumentationApk) {
       steps.add(
-          new ApkInstallStep(pathResolver, ((AndroidInstrumentationApk) apk).getApkUnderTest()));
+          new ApkInstallStep(
+              buildContext.getSourcePathResolver(),
+              ((AndroidInstrumentationApk) apk).getApkUnderTest()));
     }
 
     AdbHelper adb = AdbHelper.get(executionContext, true);
@@ -144,7 +146,7 @@ public ImmutableList<Step> runTests(
 
     steps.add(
         getInstrumentationStep(
-            pathResolver,
+            buildContext.getSourcePathResolver(),
             executionContext.getPathToAdbExecutable(),
             Optional.of(getProjectFilesystem().resolve(pathToTestOutput)),
             Optional.of(device.getSerialNumber()),
@@ -292,21 +294,29 @@ public boolean runTestSeparately() {
   public ExternalTestRunnerTestSpec getExternalTestRunnerSpec(
       ExecutionContext executionContext,
       TestRunningOptions testRunningOptions,
-      SourcePathResolver pathResolver) {
+      BuildContext buildContext) {
     Optional<Path> apkUnderTestPath = Optional.empty();
     if (apk instanceof AndroidInstrumentationApk) {
       apkUnderTestPath =
           Optional.of(
-              pathResolver.getAbsolutePath(
-                  ((AndroidInstrumentationApk) apk).getApkUnderTest().getApkInfo().getApkPath()));
+              buildContext
+                  .getSourcePathResolver()
+                  .getAbsolutePath(
+                      ((AndroidInstrumentationApk) apk)
+                          .getApkUnderTest()
+                          .getApkInfo()
+                          .getApkPath()));
     }
     InstrumentationStep step =
         getInstrumentationStep(
-            pathResolver,
+            buildContext.getSourcePathResolver(),
             executionContext.getPathToAdbExecutable(),
             Optional.empty(),
             Optional.empty(),
-            Optional.of(pathResolver.getAbsolutePath(apk.getApkInfo().getApkPath())),
+            Optional.of(
+                buildContext
+                    .getSourcePathResolver()
+                    .getAbsolutePath(apk.getApkInfo().getApkPath())),
             Optional.empty(),
             apkUnderTestPath);
 
@@ -325,7 +335,7 @@ public Stream<BuildTarget> getRuntimeDeps() {
     builder.add(apk.getBuildTarget());
 
     if (apk instanceof ApkGenrule) {
-      builder.add(((ApkGenrule) apk).getInstallableApk().getBuildTarget());
+      builder.add(ApkGenruleDescription.getUnderlyingApk(apk).getBuildTarget());
     }
 
     return builder.build();

diff --git a/src/com/facebook/buck/android/ApkGenruleDescription.java b/src/com/facebook/buck/android/ApkGenruleDescription.java
@@ -85,4 +85,16 @@ default Optional<String> getType() {
       return Optional.of("apk");
     }
   }
+
+  static AndroidBinary getUnderlyingApk(HasInstallableApk installable) {
+    if (installable instanceof AndroidBinary) {
+      return (AndroidBinary) installable;
+    } else if (installable instanceof ApkGenrule) {
+      return getUnderlyingApk(((ApkGenrule) installable).getInstallableApk());
+    } else {
+      throw new IllegalStateException(
+          installable.getBuildTarget().getFullyQualifiedName()
+              + " must be backed by either an android_binary() or an apk_genrule()");
+    }
+  }
 }
diff --git a/src/com/facebook/buck/apple/AppleBundle.java b/src/com/facebook/buck/apple/AppleBundle.java
@@ -23,7 +23,6 @@
 import com.facebook.buck.cxx.BuildRuleWithBinary;
 import com.facebook.buck.cxx.CxxPlatform;
 import com.facebook.buck.cxx.CxxPreprocessorInput;
-import com.facebook.buck.cxx.HeaderVisibility;
 import com.facebook.buck.cxx.NativeTestable;
 import com.facebook.buck.cxx.ProvidesLinkedBinaryDeps;
 import com.facebook.buck.file.WriteFile;
@@ -887,13 +886,12 @@ public boolean isTestedBy(BuildTarget testRule) {
   }
 
   @Override
-  public CxxPreprocessorInput getCxxPreprocessorInput(
-      CxxPlatform cxxPlatform, HeaderVisibility headerVisibility)
+  public CxxPreprocessorInput getPrivateCxxPreprocessorInput(CxxPlatform cxxPlatform)
       throws NoSuchBuildTargetException {
     if (binary.isPresent()) {
       BuildRule binaryRule = binary.get();
       if (binaryRule instanceof NativeTestable) {
-        return ((NativeTestable) binaryRule).getCxxPreprocessorInput(cxxPlatform, headerVisibility);
+        return ((NativeTestable) binaryRule).getPrivateCxxPreprocessorInput(cxxPlatform);
       }
     }
     return CxxPreprocessorInput.EMPTY;