Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Closes OOZIE-133 fix oozie move fs action to include target as hdfs p…

…ath, and allowing target directory to exist
  • Loading branch information...
commit ec352102e97158aa527c1522eb751b1bd72c7b13 1 parent 1d19a90
Mona Chitnis authored bansalmayank committed
View
24 core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
@@ -56,11 +56,11 @@ void validatePath(Path path, boolean withScheme) throws ActionExecutorException
else {
if (!scheme.equals("hdfs")) {
throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS002",
- "Scheme [{0}] not support in path [{1}]", scheme, path);
+ "Scheme [{0}] not supported in path [{1}]", scheme, path);
}
}
}
- else {
+ else {
if (scheme != null) {
throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS003",
"Scheme [{0}] not allowed in path [{1}]", scheme, path);
@@ -68,6 +68,18 @@ void validatePath(Path path, boolean withScheme) throws ActionExecutorException
}
}
+ void validateSameNN(Path source, Path dest) throws ActionExecutorException {
+ Path destPath = new Path(source, dest);
+ String t = destPath.toUri().getScheme() + destPath.toUri().getAuthority();
+ String s = source.toUri().getScheme() + source.toUri().getAuthority();
+
+ //checking whether NN prefix of source and target is same. can modify this to adjust for a set of multiple whitelisted NN
+ if(!t.equals(s)) {
+ throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS007",
+ "move, target NN URI different from that of source", dest);
+ }
+ }
+
@SuppressWarnings("unchecked")
void doOperations(Context context, Element element) throws ActionExecutorException {
try {
@@ -214,7 +226,7 @@ public void delete(String user, String group, Path path) throws ActionExecutorEx
public void move(Context context, Path source, Path target, boolean recovery) throws ActionExecutorException {
try {
validatePath(source, true);
- validatePath(target, false);
+ validateSameNN(source, target);
FileSystem fs = getFileSystemFor(source, context);
if (!fs.exists(source) && !recovery) {
@@ -222,12 +234,6 @@ public void move(Context context, Path source, Path target, boolean recovery) th
"move, source path [{0}] does not exist", source);
}
- Path path = new Path(source, target);
- if (fs.exists(path) && !recovery) {
- throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS007",
- "move, target path [{0}] already exists", target);
- }
-
if (!fs.rename(source, target) && !recovery) {
throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS008",
"move, could not move [{0}] to [{1}]", source, target);
View
57 core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
@@ -14,6 +14,7 @@
*/
package org.apache.oozie.action.hadoop;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -25,7 +26,6 @@
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
-
import java.text.MessageFormat;
public class TestFsActionExecutor extends ActionExecutorTestCase {
@@ -55,7 +55,7 @@ private Context createContext(String actionXml) throws Exception {
return new Context(wf, action);
}
-
+
public void testValidatePath() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
ae.validatePath(new Path("hdfs://x/bla"), true);
@@ -65,21 +65,40 @@ public void testValidatePath() throws Exception {
ae.validatePath(new Path("hdfs://x/bla"), false);
}
catch (ActionExecutorException ex) {
- assertEquals("FS003", ex.getErrorCode());
+ assertEquals("FS003", ex.getErrorCode());
}
try {
ae.validatePath(new Path("bla"), true);
}
catch (ActionExecutorException ex) {
- assertEquals("FS001", ex.getErrorCode());
+ assertEquals("FS001", ex.getErrorCode());
}
try {
ae.validatePath(new Path("file://bla"), true);
}
catch (ActionExecutorException ex) {
- assertEquals("FS002", ex.getErrorCode());
+ assertEquals("FS002", ex.getErrorCode());
+ }
+ }
+
+ public void testvalidateSameNN() throws Exception {
+ FsActionExecutor ae = new FsActionExecutor();
+ ae.validateSameNN(new Path("hdfs://x/bla"), new Path("hdfs://x/foo"));
+
+ try {
+ ae.validateSameNN(new Path("hdfs://x/bla"), new Path("viefs://x/bla"));
+ }
+ catch (ActionExecutorException ex) {
+ assertEquals("FS007", ex.getErrorCode());
+ }
+
+ try {
+ ae.validateSameNN(new Path("hdfs://x/bla"), new Path("hdfs://y/bla"));
+ }
+ catch (ActionExecutorException ex) {
+ assertEquals("FS007", ex.getErrorCode());
}
}
@@ -120,35 +139,45 @@ public void testMove() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
FileSystem fs = getFileSystem();
- Path source = new Path(getFsTestCaseDir(), "source");
+ Path source = new Path(getFsTestCaseDir(), "source");
Path target = new Path(getFsTestCaseDir(), "target");
-
Context context = createContext("<fs/>");
fs.mkdirs(source);
+ fs.createNewFile(new Path(source+"/newfile1"));
+ fs.mkdirs(target);
- ae.move(context, source, new Path(target.toUri().getPath()), false);
+ String dest = target.toUri().getPath();
+ Path destPath = new Path(dest);
+ ae.move(context, new Path(source+"/newfile1"), destPath, false);
- assertTrue(!fs.exists(source));
+ assertTrue(!fs.exists(new Path(source+"/newfile1")));
assertTrue(fs.exists(target));
try {
- ae.move(context, source, new Path(target.toUri().getPath()), false);
- fail();
+ ae.move(context, new Path(source+"/newfile1"), destPath, false);
+ fail();
}
catch (ActionExecutorException ex) {
assertEquals("FS006", ex.getErrorCode());
}
+ fs.mkdirs(source);
+ fs.createNewFile(new Path(source+"/newfile"));
+ Path complexTarget = new Path(target+"/a/b");
+ fs.mkdirs(complexTarget);
+
+ ae.move(context, source, complexTarget, false);
+ assertTrue(fs.exists(new Path(complexTarget+"/"+source.getName())));
+
fs.mkdirs(source);
try {
- ae.move(context, source, new Path(target.toUri().getPath()), false);
- fail();
+ ae.move(context, source, new Path(target.toUri().getScheme()+"://foo/"+destPath), false);
+ fail();
}
catch (ActionExecutorException ex) {
assertEquals("FS007", ex.getErrorCode());
}
-
fs.delete(source, true);
ae.move(context, source, new Path(target.toUri().getPath()), true);
View
1  release-log.txt
@@ -16,6 +16,7 @@ OOZIE-100 escape characters for xml when create dag evaluator
OOZIE-98 parametrization of 'name' attribute in workflow/coordinator/bundle
OOZIE-106 add new queue class to iterate next callable when current callable type has reached max concorrency
OOZIE-108 Upgrade pom version to 3.1.0.
+OOZIE-133 Fs 'move' action made consistent and able to have existing target dir
-- Oozie 3.0.2 release
Please sign in to comment.
Something went wrong with that request. Please try again.