move existing changeset derivation logic to mercurial_derived_data

Summary:
This change move logic associated with mercurial changeset derivation to `mercurial_derived_data` crate.

NOTE: it is not converted to derived data infrastructure at this point, it is a preparation step to actually do this

Reviewed By: farnz

Differential Revision: D23573610

fbshipit-source-id: 6e8cbf7d53ab5dbd39d5bf5e06c3f0fc5a8305c8

eden/mononoke/blobrepo/Cargo.toml

@@ -47,6 +47,7 @@ derived_data_filenodes = { path = "../derived_data/filenodes" }
 fixtures = { path = "../tests/fixtures" }
 manifest = { path = "../manifest" }
 memblob = { path = "../blobstore/memblob" }
+mercurial_derived_data = { path = "../derived_data/mercurial_derived_data" }
 mercurial_types = { path = "../mercurial/types" }
 mercurial_types-mocks = { path = "../mercurial/types/mocks" }
 scuba_ext = { path = "../common/scuba_ext" }

eden/mononoke/blobrepo/blobrepo_hg/Cargo.toml

@@ -18,24 +18,19 @@ changesets = { path = "../../changesets" }
 context = { path = "../../server/context" }
 filenodes = { path = "../../filenodes" }
 manifest = { path = "../../manifest" }
+mercurial_derived_data = { path = "../../derived_data/mercurial_derived_data" }
 mercurial_mutation = { path = "../../mercurial/mutation" }
 mercurial_types = { path = "../../mercurial/types" }
 mononoke_types = { path = "../../mononoke_types" }
 repo_blobstore = { path = "../repo_blobstore" }
 scuba_ext = { path = "../../common/scuba_ext" }
-topo_sort = { path = "../../common/topo_sort" }
 cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 failure_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 futures_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 futures_stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
-time_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 tracing = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 anyhow = "1.0"
 futures = { version = "0.3.5", features = ["async-await", "compat"] }
 futures-old = { package = "futures", version = "0.1" }
-maplit = "1.0"
-rand = { version = "0.7", features = ["small_rng"] }
-slog = { version = "2.5", features = ["max_level_debug"] }
-tokio = { version = "=0.2.13", features = ["full"] }
 uuid = { version = "0.8.1", features = ["v4"] }

eden/mononoke/blobrepo/blobrepo_hg/src/lib.rs

@@ -9,13 +9,13 @@
 
 mod bonsai_generation;
 mod create_changeset;
-pub mod derive_hg_changeset;
-pub mod derive_hg_manifest;
 pub mod repo_commit;
 pub use crate::repo_commit::ChangesetHandle;
 pub use changeset_fetcher::ChangesetFetcher;
 // TODO: This is exported for testing - is this the right place for it?
-pub use crate::repo_commit::{check_case_conflicts, compute_changed_files, UploadEntries};
+pub use crate::repo_commit::{
+    check_case_conflict_in_manifest, check_case_conflicts, compute_changed_files, UploadEntries,
+};
 pub mod errors {
     pub use blobrepo_errors::*;
 }
@@ -453,7 +453,7 @@ impl BlobRepoHg for BlobRepo {
         ctx: CoreContext,
         bcs_id: ChangesetId,
     ) -> BoxFuture<HgChangesetId, Error> {
-        derive_hg_changeset::get_hg_from_bonsai_changeset(self, ctx, bcs_id).boxify()
+        mercurial_derived_data::get_hg_from_bonsai_changeset(self, ctx, bcs_id).boxify()
     }
 }
 

eden/mononoke/blobrepo/blobrepo_hg/src/repo_commit.rs

@@ -5,7 +5,6 @@
  * GNU General Public License version 2.
  */
 
-use crate::derive_hg_changeset::check_case_conflict_in_manifest;
 use crate::BlobRepoHg;
 use anyhow::{format_err, Error, Result};
 use cloned::cloned;
@@ -19,7 +18,7 @@ use futures_ext::{
     BoxFuture as OldBoxFuture, BoxStream as OldBoxStream, FutureExt as OldFutureExt,
 };
 use futures_old::future::{
-    self as old_future, result, Future as OldFuture, Shared, SharedError, SharedItem,
+    self as old_future, loop_fn, result, Future as OldFuture, Loop, Shared, SharedError, SharedItem,
 };
 use futures_old::stream::Stream as OldStream;
 use futures_old::sync::oneshot;
@@ -31,7 +30,7 @@ use std::collections::{HashMap, HashSet};
 use std::mem;
 use std::sync::{Arc, Mutex};
 
-use ::manifest::{find_intersection_of_diffs, Diff, Entry, ManifestOps};
+use ::manifest::{find_intersection_of_diffs, Diff, Entry, Manifest, ManifestOps};
 pub use blobrepo_common::changed_files::compute_changed_files;
 use blobstore::{Blobstore, Loadable};
 use context::CoreContext;
@@ -669,3 +668,86 @@ pub fn make_new_changeset(
     let changeset = HgChangesetContent::new_from_parts(parents, root_hash, cs_metadata, files);
     HgBlobChangeset::new(changeset)
 }
+
+/// Check if adding a single path to manifest would cause case-conflict
+///
+/// Implementation traverses manifest and checks if correspoinding path element is present,
+/// if path element is not present, it lowercases current path element and checks if it
+/// collides with any existing elements inside manifest. if so it also needs to check that
+/// child manifest contains this entry, because it might have been removed.
+pub fn check_case_conflict_in_manifest(
+    repo: BlobRepo,
+    ctx: CoreContext,
+    parent_mf_id: HgManifestId,
+    child_mf_id: HgManifestId,
+    path: MPath,
+) -> impl OldFuture<Item = Option<MPath>, Error = Error> {
+    let child_mf_id = child_mf_id.clone();
+    parent_mf_id
+        .load(ctx.clone(), &repo.get_blobstore())
+        .compat()
+        .from_err()
+        .and_then(move |mf| {
+            loop_fn(
+                (None, mf, path.into_iter()),
+                move |(cur_path, mf, mut elements): (Option<MPath>, _, _)| {
+                    let element = match elements.next() {
+                        None => return old_future::ok(Loop::Break(None)).boxify(),
+                        Some(element) => element,
+                    };
+
+                    match mf.lookup(&element) {
+                        Some(entry) => {
+                            let cur_path = MPath::join_opt_element(cur_path.as_ref(), &element);
+                            match entry {
+                                Entry::Leaf(..) => old_future::ok(Loop::Break(None)).boxify(),
+                                Entry::Tree(manifest_id) => manifest_id
+                                    .load(ctx.clone(), repo.blobstore())
+                                    .compat()
+                                    .from_err()
+                                    .map(move |mf| Loop::Continue((Some(cur_path), mf, elements)))
+                                    .boxify(),
+                            }
+                        }
+                        None => {
+                            let element_utf8 = String::from_utf8(Vec::from(element.as_ref()));
+                            let mut potential_conflicts = vec![];
+                            // Find all entries in the manifests that can potentially be a conflict.
+                            // Entry can potentially be a conflict if its lowercased version
+                            // is the same as lowercased version of the current element
+
+                            for (basename, _) in mf.list() {
+                                let path =
+                                    MPath::join_element_opt(cur_path.as_ref(), Some(&basename));
+                                match (&element_utf8, std::str::from_utf8(basename.as_ref())) {
+                                    (Ok(ref element), Ok(ref basename)) => {
+                                        if basename.to_lowercase() == element.to_lowercase() {
+                                            potential_conflicts.extend(path);
+                                        }
+                                    }
+                                    _ => {}
+                                }
+                            }
+
+                            // For each potential conflict we need to check if it's present in
+                            // child manifest. If it is, then we've got a conflict, otherwise
+                            // this has been deleted and it's no longer a conflict.
+                            child_mf_id
+                                .find_entries(
+                                    ctx.clone(),
+                                    repo.get_blobstore(),
+                                    potential_conflicts,
+                                )
+                                .collect()
+                                .map(|entries| {
+                                    // NOTE: We flatten here because we cannot have a conflict
+                                    // at the root.
+                                    Loop::Break(entries.into_iter().next().and_then(|x| x.0))
+                                })
+                                .boxify()
+                        }
+                    }
+                },
+            )
+        })
+}

eden/mononoke/blobrepo/test/main.rs

@@ -17,10 +17,7 @@ use assert_matches::assert_matches;
 use blobrepo::BlobRepo;
 use blobrepo_errors::ErrorKind;
 use blobrepo_hg::{
-    derive_hg_changeset::{
-        check_case_conflict_in_manifest, get_hg_from_bonsai_changeset_with_impl,
-        get_manifest_from_bonsai,
-    },
+    check_case_conflict_in_manifest,
     repo_commit::{compute_changed_files, UploadEntries},
     BlobRepoHg,
 };
@@ -36,6 +33,7 @@ use futures_ext::{BoxFuture, FutureExt};
 use futures_old::{Future, Stream};
 use maplit::btreemap;
 use memblob::LazyMemblob;
+use mercurial_derived_data::{get_hg_from_bonsai_changeset_with_impl, get_manifest_from_bonsai};
 use mercurial_types::{
     blobs::{
         BlobManifest, ContentBlobMeta, File, HgBlobChangeset, UploadHgFileContents,

eden/mononoke/blobrepo_utils/Cargo.toml

@@ -21,6 +21,7 @@ blobstore = { path = "../blobstore" }
 cacheblob = { path = "../blobstore/cacheblob" }
 context = { path = "../server/context" }
 manifest = { path = "../manifest" }
+mercurial_derived_data = { path = "../derived_data/mercurial_derived_data" }
 mercurial_types = { path = "../mercurial/types" }
 mononoke_types = { path = "../mononoke_types" }
 cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }

eden/mononoke/blobrepo_utils/src/bonsai/manifest.rs

@@ -8,7 +8,6 @@
 use crate::changeset::{visit_changesets, ChangesetVisitMeta, ChangesetVisitor};
 use anyhow::{bail, Error};
 use blobrepo::BlobRepo;
-use blobrepo_hg::derive_hg_manifest::derive_hg_manifest;
 use blobrepo_override::DangerousOverride;
 use blobstore::{Blobstore, Loadable};
 use cacheblob::MemWritesBlobstore;
@@ -21,6 +20,7 @@ use futures_old::{
     Future, Stream,
 };
 use manifest::{bonsai_diff, BonsaiDiffFileChange, Diff, Entry, ManifestOps};
+use mercurial_derived_data::derive_hg_manifest;
 use mercurial_types::{
     blobs::{BlobManifest, HgBlobChangeset, HgBlobEntry},
     HgChangesetId, HgFileNodeId, HgManifestId, HgNodeHash, Type,

eden/mononoke/cmds/bonsai_verify/main.rs

@@ -10,7 +10,7 @@
 mod config;
 
 use anyhow::{format_err, Error, Result};
-use blobrepo_hg::{derive_hg_changeset::get_manifest_from_bonsai, BlobRepoHg};
+use blobrepo_hg::BlobRepoHg;
 use blobrepo_utils::{BonsaiMFVerify, BonsaiMFVerifyResult};
 use blobstore::Loadable;
 use clap::{App, Arg, ArgMatches, SubCommand};
@@ -30,6 +30,7 @@ use futures_old::{
     Future, Stream,
 };
 use lock_ext::LockExt;
+use mercurial_derived_data::get_manifest_from_bonsai;
 use mercurial_types::HgChangesetId;
 use revset::AncestorsNodeStream;
 use slog::{debug, error, info, warn, Logger};

eden/mononoke/derived_data/mercurial_derived_data/Cargo.toml

@@ -11,12 +11,27 @@ path = "lib.rs"
 
 [dependencies]
 blobrepo = { path = "../../blobrepo" }
-blobrepo_hg = { path = "../../blobrepo/blobrepo_hg" }
+blobrepo_common = { path = "../../blobrepo/common" }
+blobrepo_errors = { path = "../../blobrepo/errors" }
+blobstore = { path = "../../blobstore" }
 bonsai_hg_mapping = { path = "../../bonsai_hg_mapping" }
 context = { path = "../../server/context" }
 derived_data = { path = ".." }
+manifest = { path = "../../manifest" }
 mercurial_types = { path = "../../mercurial/types" }
 mononoke_types = { path = "../../mononoke_types" }
+scuba_ext = { path = "../../common/scuba_ext" }
+topo_sort = { path = "../../common/topo_sort" }
+cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 futures_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
+futures_stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
+stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
+time_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
+tracing = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 anyhow = "1.0"
-futures = "0.1"
+futures = { version = "0.3.5", features = ["async-await", "compat"] }
+futures-old = { package = "futures", version = "0.1" }
+maplit = "1.0"
+rand = { version = "0.7", features = ["small_rng"] }
+slog = { version = "2.5", features = ["max_level_debug"] }
+tokio = { version = "=0.2.13", features = ["full"] }