Merge directories with identical files

src/rust/engine/fs/src/snapshot.rs

@@ -192,7 +192,10 @@ impl Snapshot {
 
   ///
   /// Given Digest(s) representing Directory instances, merge them recursively into a single
-  /// output Directory Digest. Fails for collisions.
+  /// output Directory Digest.
+  ///
+  /// If a file is present with the same name and contents multiple times, it will appear once.
+  /// If a file is present with the same name, but different contents, an error will be returned.
   ///
   pub fn merge_directories(store: Store, dir_digests: Vec<Digest>) -> BoxFuture<Digest, String> {
     if dir_digests.is_empty() {
@@ -218,16 +221,17 @@ impl Snapshot {
         let mut out_dir = bazel_protos::remote_execution::Directory::new();
 
         // Merge FileNodes.
+        let file_nodes = Itertools::flatten(
+          directories
+            .iter_mut()
+            .map(|directory| directory.take_files().into_iter()),
+        ).sorted_by(|a, b| a.name.cmp(&b.name));
+
         out_dir.set_files(protobuf::RepeatedField::from_vec(
-          Itertools::flatten(
-            directories
-              .iter_mut()
-              .map(|directory| directory.take_files().into_iter()),
-          ).collect(),
+          file_nodes.into_iter().dedup().collect(),
         ));
-        out_dir.mut_files().sort_by(|a, b| a.name.cmp(&b.name));
         let unique_count = out_dir
-          .mut_files()
+          .get_files()
           .iter()
           .map(|v| v.get_name())
           .dedup()
@@ -540,6 +544,36 @@ mod tests {
     );
   }
 
+  #[test]
+  fn merge_directories_same_files() {
+    let (store, _, _, _) = setup();
+
+    let containing_roland = TestDirectory::containing_roland();
+    let containing_roland_and_treats = TestDirectory::containing_roland_and_treats();
+
+    store
+      .record_directory(&containing_roland.directory(), false)
+      .wait()
+      .expect("Storing roland directory");
+    store
+      .record_directory(&containing_roland_and_treats.directory(), false)
+      .wait()
+      .expect("Storing treats directory");
+
+    let result = Snapshot::merge_directories(
+      store,
+      vec![
+        containing_roland.digest(),
+        containing_roland_and_treats.digest(),
+      ],
+    ).wait();
+
+    assert_eq!(
+      result,
+      Ok(TestDirectory::containing_roland_and_treats().digest())
+    );
+  }
+
   #[test]
   fn snapshot_merge_two_files() {
     let (store, tempdir, _, digester) = setup();