Merge without resolving conflicts

core/server/master/src/main/java/alluxio/master/file/meta/InodeTree.java

@@ -601,11 +601,17 @@ public CreatePathResult createPath(LockedInodePath inodePath, CreatePathOptions<
             continue;
           }
         } else {
-          // Successfully added the child, while holding the write lock.
-          dir.setPinned(currentInodeDirectory.isPinned());
-          if (options.isPersisted()) {
-            // Do not journal the persist entry, since a creation entry will be journaled instead.
-            syncPersistDirectory(dir, NoopJournalContext.INSTANCE);
+          try {
+            // Successfully added the child, while holding the write lock.
+            dir.setPinned(currentInodeDirectory.isPinned());
+            if (options.isPersisted()) {
+              // Do not journal the persist entry, since a creation entry will be journaled instead.
+              syncPersistDirectory(dir, NoopJournalContext.INSTANCE);
+            }
+          } catch (Exception e) {
+            // Failed to persist the directory, so remove it from the parent.
+            currentInodeDirectory.removeChild(dir);
+            throw e;
           }
           // Journal the new inode.
           journalContext.append(dir.toJournalEntry());

dev/scripts/src/alluxio.org/build-distribution/cmd/generate-tarball.go

@@ -90,6 +90,9 @@ func getCommonMvnArgs(hadoopDistribution string) []string {
 	if hadoopDistribution != "" {
 		hadoopVersion := hadoopDistributions[hadoopDistribution]
 		args = append(args, fmt.Sprintf("-Dhadoop.version=%v", hadoopVersion), fmt.Sprintf("-P%v", hadoopVersion.hadoopProfile()))
+		if hadoopVersion.major >= 2 && hadoopVersion.minor >= 4 {
+			args = append(args, "-Pyarn")
+		}
 	}
 	return args
 }
@@ -131,6 +134,7 @@ func addAdditionalFiles(srcPath, dstPath, version string) {
 		"integration/kubernetes/alluxio-master.yaml.template",
 		"integration/kubernetes/alluxio-worker.yaml.template",
 		"integration/kubernetes/conf/alluxio.properties.template",
+<<<<<<< HEAD
 		fmt.Sprintf("lib/alluxio-underfs-gcs-%v.jar", version),
 		fmt.Sprintf("lib/alluxio-underfs-hdfs-%v.jar", version),
 		fmt.Sprintf("lib/alluxio-underfs-local-%v.jar", version),
@@ -139,6 +143,20 @@ func addAdditionalFiles(srcPath, dstPath, version string) {
 		fmt.Sprintf("lib/alluxio-underfs-swift-%v.jar", version),
 		fmt.Sprintf("lib/alluxio-underfs-wasb-%v.jar", version),
 		"libexec/alluxio-config.sh",
+||||||| merged common ancestors
+		// FUSE
+		"integration/fuse/bin/alluxio-fuse",
+=======
+		// FUSE
+		"integration/fuse/bin/alluxio-fuse",
+		// MESOS
+		"integration/mesos/bin/alluxio-env-mesos.sh",
+		"integration/mesos/bin/alluxio-mesos-start.sh",
+		"integration/mesos/bin/alluxio-master-mesos.sh",
+		"integration/mesos/bin/alluxio-mesos-stop.sh",
+		"integration/mesos/bin/alluxio-worker-mesos.sh",
+		"integration/mesos/bin/common.sh",
+>>>>>>> upstream/branch-1.7
 	}
 	for _, path := range pathsToCopy {
 		mkdir(filepath.Join(dstPath, filepath.Dir(path)))

tests/src/test/java/alluxio/master/file/FileSystemMasterIntegrationTest.java

@@ -39,10 +39,19 @@
 import alluxio.master.file.options.DeleteOptions;
 import alluxio.master.file.options.FreeOptions;
 import alluxio.master.file.options.ListStatusOptions;
+import alluxio.master.file.options.MountOptions;
 import alluxio.master.file.options.RenameOptions;
 import alluxio.master.file.options.SetAttributeOptions;
 import alluxio.security.authentication.AuthenticatedClientUser;
+<<<<<<< HEAD
 import alluxio.underfs.UnderFileSystem;
+||||||| merged common ancestors
+=======
+import alluxio.underfs.UfsDirectoryStatus;
+import alluxio.underfs.UnderFileSystem;
+import alluxio.underfs.UnderFileSystemFactory;
+import alluxio.underfs.UnderFileSystemFactoryRegistry;
+>>>>>>> upstream/branch-1.7
 import alluxio.util.CommonUtils;
 import alluxio.util.IdUtils;
 import alluxio.wire.CommonOptions;
@@ -58,6 +67,8 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.Timeout;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -769,6 +780,77 @@ public void syncReplay() throws Exception {
     Assert.assertTrue(filenames.contains("ufs_file"));
   }
 
+  @Test
+  public void unavailableUfsRecursiveCreate() throws Exception {
+    String ufsBase = "test://test";
+
+    UnderFileSystemFactory mockUfsFactory = Mockito.mock(UnderFileSystemFactory.class);
+    Mockito.when(mockUfsFactory.supportsPath(Matchers.anyString(), Matchers.any()))
+        .thenReturn(Boolean.FALSE);
+    Mockito.when(mockUfsFactory.supportsPath(Matchers.eq(ufsBase), Matchers.any()))
+        .thenReturn(Boolean.TRUE);
+
+    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
+    Mockito.when(mockUfsFactory.create(Matchers.eq(ufsBase), Matchers.any())).thenReturn(mockUfs);
+    Mockito.when(mockUfs.isDirectory(ufsBase)).thenReturn(true);
+    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), ""))
+        .thenReturn(new AlluxioURI(ufsBase));
+    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "/dir1"))
+        .thenReturn(new AlluxioURI(ufsBase + "/dir1"));
+    Mockito.when(mockUfs.getDirectoryStatus(ufsBase))
+        .thenReturn(new UfsDirectoryStatus("test", "owner", "group", (short) 511));
+    Mockito.when(mockUfs.mkdirs(Matchers.eq(ufsBase + "/dir1"), Matchers.any()))
+        .thenThrow(new IOException("ufs unavailable"));
+
+    UnderFileSystemFactoryRegistry.register(mockUfsFactory);
+
+    mFsMaster.mount(new AlluxioURI("/mnt"), new AlluxioURI(ufsBase), MountOptions.defaults());
+
+    AlluxioURI root = new AlluxioURI("/mnt/");
+    AlluxioURI alluxioFile = new AlluxioURI("/mnt/dir1/dir2/file");
+
+    // Create a persisted Alluxio file (but no ufs file).
+    try {
+      mFsMaster.createFile(alluxioFile,
+          CreateFileOptions.defaults().setPersisted(true).setRecursive(true));
+      Assert.fail("persisted create should fail, when UFS is unavailable");
+    } catch (Exception e) {
+      // expected, ignore
+    }
+
+    List<FileInfo> files = mFsMaster.listStatus(root, ListStatusOptions.defaults());
+    Assert.assertTrue(files.isEmpty());
+
+    try {
+      // should not exist
+      files = mFsMaster.listStatus(new AlluxioURI("/mnt/dir1/"), ListStatusOptions.defaults());
+      Assert.fail("dir should not exist, when UFS is unavailable");
+    } catch (Exception e) {
+      // expected, ignore
+    }
+
+    try {
+      // should not exist
+      mFsMaster.delete(new AlluxioURI("/mnt/dir1/"), DeleteOptions.defaults().setRecursive(true));
+      Assert.fail("cannot delete non-existing directory, when UFS is unavailable");
+    } catch (Exception e) {
+      // expected, ignore
+      files = null;
+    }
+
+    files = mFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusOptions.defaults());
+    Assert.assertTrue(files.isEmpty());
+
+    // Stop Alluxio.
+    mLocalAlluxioClusterResource.get().stopFS();
+    // Create the master using the existing journal.
+    MasterRegistry registry = MasterTestUtils.createLeaderFileSystemMasterFromJournal();
+    FileSystemMaster newFsMaster = registry.get(FileSystemMaster.class);
+
+    files = newFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusOptions.defaults());
+    Assert.assertTrue(files.isEmpty());
+  }
+
   // TODO(gene): Journal format has changed, maybe add Version to the format and add this test back
   // or remove this test when we have better tests against journal checkpoint.
   // @Test