Skip to content

Commit

Permalink
use class loader and reflection
Browse files Browse the repository at this point in the history
  • Loading branch information
@LuQQiu
LuQQiu committed May 15, 2019
1 parent 2c7835a commit c4497ef
Show file tree
Hide file tree
Showing 4 changed files with 341 additions and 415 deletions.
26 changes: 0 additions & 26 deletions examples/src/main/java/alluxio/cli/CliUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,9 @@

import alluxio.Constants;

import alluxio.underfs.UfsFileStatus;
import alluxio.underfs.UfsStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.DeleteOptions;
import alluxio.util.io.PathUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.concurrent.Callable;

/**
Expand Down Expand Up @@ -63,23 +56,4 @@ public static boolean runExample(final Callable<Boolean> example) {
CliUtils.printPassInfo(result);
return result;
}

/**
* Cleans all the files or sub directories inside the given directory
* in the under filesystem.
*
* @param ufs the under filesystem
* @param directory the directory to clean
*/
public static void cleanupUfs(UnderFileSystem ufs, String directory) throws IOException {
UfsStatus[] statuses = ufs.listStatus(directory);
for (UfsStatus status : statuses) {
if (status instanceof UfsFileStatus) {
ufs.deleteFile(PathUtils.concatPath(directory, status.getName()));
} else {
ufs.deleteDirectory(PathUtils.concatPath(directory, status.getName()),
DeleteOptions.defaults().setRecursive(true));
}
}
}
}
230 changes: 135 additions & 95 deletions examples/src/main/java/alluxio/cli/UnderFileSystemContractTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
import alluxio.conf.PropertyKey;
import alluxio.examples.S3ASpecificOperations;
import alluxio.examples.UnderFileSystemCommonOperations;
import alluxio.underfs.UfsFileStatus;
import alluxio.underfs.UfsStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.UnderFileSystemConfiguration;
import alluxio.underfs.UnderFileSystemFactory;
Expand All @@ -31,6 +33,7 @@
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
Expand Down Expand Up @@ -107,31 +110,37 @@ private void runS3AOperations() throws Exception {
loadAndRunTests(ops, testDir);
}

public void loadAndRunTests(Object object, String testDir) throws Exception {
private void loadAndRunTests(Object operationsToTest, String testDir) throws Exception {
try {
Class classToRun = object.getClass();
Method[] methods = classToRun.getMethods();
for(Method method : methods){
String methodName = method.getName();
if (methodName.endsWith("Test")) {
mFailedTest = methodName;
LOG.info("Running test: " + methodName);
method.invoke(object);
LOG.info("Test Passed!");
}
Class classToRun = operationsToTest.getClass();
Field[] fields = classToRun.getDeclaredFields();
for (Field field : fields) {
field.setAccessible(true);
}
mFailedTest = "";
} catch (InvocationTargetException e) {
if (mUfsType.equals("S3A")) {
List<String> operations = getRelatedS3AOperations();
if (operations.size() > 0) {
LOG.info("Related S3 operations: "
+ StringUtils.join(operations, ","));
Method[] tests = classToRun.getDeclaredMethods();
for (Method test : tests) {
String testName = test.getName();
if (testName.endsWith("Test")) {
LOG.info("Running test: " + testName);
try {
test.invoke(operationsToTest);
} catch (InvocationTargetException e) {
if (mUfsType.equals("S3A")) {
List<String> operations = getRelatedS3AOperations(testName);
if (operations.size() > 0) {
LOG.info("Related S3 operations: "
+ StringUtils.join(operations, ","));
}
}
throw new IOException(e.getTargetException());
}
LOG.info("Test Passed!");
cleanupUfs(testDir);
}
}
throw new IOException(e.getTargetException());
} finally {
cleanupUfs(mUfs, testDir);
mUfs.deleteDirectory(testDir, DeleteOptions.defaults().setRecursive(true));
mUfs.close();
}
}

Expand All @@ -145,128 +154,123 @@ private UnderFileSystem createUnderFileSystem() {
return ufs;
}

private void cleanupUfs(UnderFileSystem ufs, String testDir) throws IOException {
ufs.deleteDirectory(testDir, DeleteOptions.defaults().setRecursive(true));
ufs.close();
}

/**
* @param args the input arguments
* Cleans all the files or sub directories inside the given directory
* in the under filesystem.
*
* @param directory the directory to clean
*/
public static void main(String[] args) throws Exception {
UnderFileSystemContractTest test = new UnderFileSystemContractTest();
JCommander jc = new JCommander(test);
jc.setProgramName(UnderFileSystemContractTest.class.getName());
try {
jc.parse(args);
} catch (Exception e) {
LOG.error(e.getMessage());
jc.usage();
LOG.info(getHelpMessage());
System.exit(1);
}
if (test.mHelp) {
jc.usage();
LOG.info(getHelpMessage());
} else {
test.run();
private void cleanupUfs(String directory) throws IOException {
UfsStatus[] statuses = mUfs.listStatus(directory);
for (UfsStatus status : statuses) {
if (status instanceof UfsFileStatus) {
mUfs.deleteFile(PathUtils.concatPath(directory, status.getName()));
} else {
mUfs.deleteDirectory(PathUtils.concatPath(directory, status.getName()),
DeleteOptions.defaults().setRecursive(true));
}
}
}

private static String getHelpMessage() {
return "Test description:\n"
+ "Test the integration between Alluxio and the under filesystem. "
+ "If the given under filesystem name is S3A, this test can also be used as "
+ "a S3A compatibility test to test if the target under filesystem can "
+ "fulfill the minimum S3A compatibility requirements in order to "
+ "work well with Alluxio through Alluxio's integration with S3A. \n"
+ "Command line example: 'bin/alluxio runUnderFileSystemTest --path=s3a://testPath "
+ "-Daws.accessKeyId=<accessKeyId> -Daws.secretKeyId=<secretKeyId>"
+ "-Dalluxio.underfs.s3.endpoint=<endpoint_url> "
+ "-Dalluxio.underfs.s3.disable.dns.buckets=true'";
}

/**
* Gets the S3A operations related to the failed test. This method
* should only be called when the ufs is S3A.
*
* @param testName the name of the failed test
* @return the related S3A operations
*/
public List<String> getRelatedS3AOperations() {
private List<String> getRelatedS3AOperations(String testName) {
List<String> operations = new ArrayList<>();
switch (mFailedTest) {
case "createAtomic":
case "createEmpty":
case "createParent":
case "createThenGetExistingFileStatus":
case "createThenGetExistingStatus":
case "getFileSize":
case "getFileStatus":
case "getModTime":
switch (testName) {
case "createFileLessThanOnePartTest":
operations.add("AmazonS3Client.initiateMultipartUpload()");
operations.add("AmazonS3Client.uploadPart()");
operations.add("AmazonS3Client.completeMultipartUpload()");
break;
case "createMultipartFileTest":
operations.add("AmazonS3Client.initiateMultipartUpload()");
operations.add("AmazonS3Client.uploadPart()");
operations.add("AmazonS3Client.completeMultipartUpload()");
operations.add("AmazonS3Client.listObjects()");
break;
case "createAndAbortMultipartFileTest":
operations.add("AmazonS3Client.initiateMultipartUpload()");
operations.add("AmazonS3Client.uploadPart()");
operations.add("AmazonS3Client.listMultipartUploads()");
operations.add("TransferManager.abortMultipartUploads()");
break;
case "createAtomicTest":
case "createEmptyTest":
case "createParentTest":
case "createThenGetExistingFileStatusTest":
case "createThenGetExistingStatusTest":
case "getFileSizeTest":
case "getFileStatusTest":
case "getModTimeTest":
operations.add("TransferManager.upload()");
operations.add("AmazonS3Client.getObjectMetadata()");
break;
case "createOpen":
case "createOpenAtPosition":
case "createOpenEmpty":
case "createOpenExistingLargeFile":
case "createOpenLarge":
case "createOpenTest":
case "createOpenAtPositionTest":
case "createOpenEmptyTest":
case "createOpenExistingLargeFileTest":
case "createOpenLargeTest":
operations.add("TransferManager.upload()");
operations.add("AmazonS3Client.getObject()");
break;
case "deleteFile":
case "deleteFileTest":
operations.add("TransferManager.upload()");
operations.add("AmazonS3Client.deleteObject()");
operations.add("AmazonS3Client.getObjectMetadata()");
break;
case "deleteDir":
case "deleteLargeDirectory":
case "createThenDeleteExistingDirectory":
case "deleteDirTest":
case "deleteLargeDirectoryTest":
case "createThenDeleteExistingDirectoryTest":
operations.add("AmazonS3Client.putObject()");
operations.add("AmazonS3Client.deleteObjects()");
operations.add("AmazonS3Client.listObjectsV2()");
operations.add("AmazonS3Client.getObjectMetadata()");
break;
case "createDeleteFileConjuction":
case "createDeleteFileConjuctionTest":
operations.add("TransferManager.upload()");
operations.add("AmazonS3Client.getObjectMetadata()");
operations.add("AmazonS3Client.deleteObject()");
break;
case "exists":
case "getDirectoryStatus":
case "existsTest":
case "getDirectoryStatusTest":
case "createThenGetExistingDirectoryStatus":
operations.add("AmazonS3Client.putObject()");
operations.add("AmazonS3Client.getObjectMetadata()");
break;
case "isFile":
case "isFileTest":
operations.add("AmazonS3Client.putObject()");
operations.add("AmazonS3Client.deleteObject()");
break;
case "listStatus":
case "listStatusEmpty":
case "listStatusFile":
case "listLargeDirectory":
case "listStatusRecursive":
case "mkdirs":
case "objectCommonPrefixesIsDirectory":
case "objectCommonPrefixesListStatusNonRecursive":
case "objectCommonPrefixesListStatusRecursive":
case "objectNestedDirsListStatusRecursive":
case "listStatusTest":
case "listStatusEmptyTest":
case "listStatusFileTest":
case "listLargeDirectoryTest":
case "listStatusRecursiveTest":
case "mkdirsTest":
case "objectCommonPrefixesIsDirectoryTest":
case "objectCommonPrefixesListStatusNonRecursiveTest":
case "objectCommonPrefixesListStatusRecursiveTest":
case "objectNestedDirsListStatusRecursiveTest":
operations.add("AmazonS3Client.putObject()");
operations.add("AmazonS3Client.listObjectsV2()");
operations.add("AmazonS3Client.getObjectMetadata()");
break;
case "renameFile":
case "renameRenamableFile":
case "renameFileTest":
case "renameRenamableFileTest":
operations.add("TransferManager.upload()");
operations.add("TransferManager.copyObject()");
operations.add("AmazonS3Client.deleteObject()");
operations.add("AmazonS3Client.getObjectMetadata()");
break;
case "renameDirectory":
case "renameDirectoryDeep":
case "renameLargeDirectory":
case "renameRenameableDirectory":
case "renameDirectoryTest":
case "renameDirectoryDeepTest":
case "renameLargeDirectoryTest":
case "renameRenameableDirectoryTest":
operations.add("AmazonS3Client.putObject()");
operations.add("TransferManager.upload()");
operations.add("TransferManager.copyObject()");
Expand All @@ -278,4 +282,40 @@ public List<String> getRelatedS3AOperations() {
}
return operations;
}

private static String getHelpMessage() {
return "Test description:\n"
+ "Test the integration between Alluxio and the under filesystem. "
+ "If the given under filesystem name is S3A, this test can also be used as "
+ "a S3A compatibility test to test if the target under filesystem can "
+ "fulfill the minimum S3A compatibility requirements in order to "
+ "work well with Alluxio through Alluxio's integration with S3A. \n"
+ "Command line example: 'bin/alluxio runUnderFileSystemTest --path=s3a://testPath "
+ "-Daws.accessKeyId=<accessKeyId> -Daws.secretKeyId=<secretKeyId>"
+ "-Dalluxio.underfs.s3.endpoint=<endpoint_url> "
+ "-Dalluxio.underfs.s3.disable.dns.buckets=true'";
}

/**
* @param args the input arguments
*/
public static void main(String[] args) throws Exception {
UnderFileSystemContractTest test = new UnderFileSystemContractTest();
JCommander jc = new JCommander(test);
jc.setProgramName(UnderFileSystemContractTest.class.getName());
try {
jc.parse(args);
} catch (Exception e) {
LOG.error(e.getMessage());
jc.usage();
LOG.info(getHelpMessage());
System.exit(1);
}
if (test.mHelp) {
jc.usage();
LOG.info(getHelpMessage());
} else {
test.run();
}
}
}
Loading

0 comments on commit c4497ef

Please sign in to comment.