Skip to content

Java CLI with all tests passing #193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rpmoore merged 5 commits into from
May 1, 2018
Merged

Java CLI with all tests passing #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cc5dbdb
All tests pass
scribe Apr 5, 2018
62916b7
Re-testing
scribe Apr 9, 2018
4fe4b2e
Fixing error in certification
scribe Apr 13, 2018
404f278
Fix for change in BP
scribe Apr 16, 2018
72d66e1
Bump
scribe May 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ subprojects {
compile 'ch.qos.logback:logback-core:1.2.3'
compile 'ch.qos.logback:logback-classic:1.2.3'
compile 'com.google.guava:guava:23.0-jre'
compile 'com.spectralogic.ds3:ds3-sdk:3.5.4'
compile 'com.spectralogic.ds3:ds3-sdk:5.0.2'
compile 'com.google.inject:guice:4.2.0'

testCompile ('org.mockito:mockito-core:1.10.19') {
Expand Down
67 changes: 37 additions & 30 deletions ...certification/src/test/java/com/spectralogic/ds3cli/certification/Certification_Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import com.google.common.collect.Iterables;
import com.spectralogic.ds3cli.CommandResponse;
import com.spectralogic.ds3cli.exceptions.*;
import com.spectralogic.ds3cli.helpers.TempStorageIds;
import com.spectralogic.ds3cli.helpers.TempStorageUtil;
import com.spectralogic.ds3cli.helpers.Util;
import com.spectralogic.ds3client.Ds3Client;
import com.spectralogic.ds3client.Ds3ClientBuilder;
Expand All @@ -30,10 +32,7 @@
import com.spectralogic.ds3client.models.common.Credentials;
import com.spectralogic.ds3client.networking.FailedRequestException;
import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.*;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -45,6 +44,7 @@
import java.util.*;

import static com.spectralogic.ds3cli.certification.CertificationUtil.*;
import static com.spectralogic.ds3cli.helpers.TempStorageUtil.setupDataPolicy;
import static com.spectralogic.ds3cli.helpers.TempStorageUtil.verifyAvailableTapePartition;
import static junit.framework.TestCase.assertFalse;
import static org.hamcrest.CoreMatchers.is;
Expand All @@ -56,10 +56,10 @@

/**
* Implement tests to automate the BlackPearl Certification process for the JavaCLI.
*
* <p>
* For details, refer to
* https://developer.spectralogic.com/certification/
* https://developer.spectralogic.com/test-plan/
* https://developer.spectralogic.com/certification/
* https://developer.spectralogic.com/test-plan/
*/
@FixMethodOrder(MethodSorters.NAME_ASCENDING) // Order only matters for manually verifying the results

Expand All @@ -74,6 +74,7 @@ public class Certification_Test {
private static UUID envDataPolicyId;

private final static Ds3ExceptionHandlerMapper EXCEPTION_MAPPER = Ds3ExceptionHandlerMapper.getInstance();

static {
EXCEPTION_MAPPER.addHandler(FailedRequestException.class, new FailedRequestExceptionHandler());
EXCEPTION_MAPPER.addHandler(RuntimeException.class, new RuntimeExceptionHandler());
Expand Down Expand Up @@ -133,7 +134,7 @@ public void test_7_2_1_invalid_credentials() throws Exception {

try {
Util.command(invalid_client, "--http -c get_service");
} catch(final FailedRequestException e) {
} catch (final FailedRequestException e) {
final String formattedException = FailedRequestExceptionHandler.format(e);
OUT.insertPreformat(formattedException);
final String expectedError = "permissions / authorization error";
Expand All @@ -155,7 +156,7 @@ public void test_7_2_2_invalid_endpoint() throws Exception {
boolean success = false;
try {
Util.command(invalid_client, "--http -c get_service");
} catch(final UnknownHostException uhe) {
} catch (final UnknownHostException uhe) {
final String formattedException = ExceptionFormatter.format(uhe);
final String expectedError = "UnknownHost";
assertThat(formattedException, containsString(expectedError));
Expand All @@ -177,7 +178,7 @@ public void test_7_3_1_list_nonexistent_bucket() throws Exception {

try {
Util.command(client, "--http -c get_bucket -b " + bucketName);
} catch(final CommandException ce) {
} catch (final CommandException ce) {
final String formattedException = ExceptionFormatter.format(ce);
assertThat(formattedException, containsString("Error: Unknown bucket."));
OUT.insertLog("CommandResponse for 7.3.1 failed attempt to list nonexistent bucket:");
Expand Down Expand Up @@ -212,7 +213,7 @@ public void test_7_3_2_access_bucket_wrong_user() throws Exception {
// Attempt to access the bucket with the new user, which should fail
final String listBucketCmd = "--http -c get_bucket -b " + bucketName;
Util.command(invalid_client, listBucketCmd);
} catch(final FailedRequestException e) {
} catch (final FailedRequestException e) {
final String formattedException = FailedRequestExceptionHandler.format(e);
final String expectedError = "permissions / authorization error";
assertThat(formattedException, containsString(expectedError));
Expand Down Expand Up @@ -250,7 +251,7 @@ public void test_7_4_get_nonexistent_object() throws Exception {
final CommandResponse getNonExtandObjectResponse = Util.command(client, getNonExtantObject);
OUT.insertCommand(getNonExtantObject, getNonExtandObjectResponse.getMessage());

} catch(final FailedRequestException fre) {
} catch (final FailedRequestException fre) {
final String formattedException = FailedRequestExceptionHandler.format(fre);
final String expectedError = "not found";
assertThat(formattedException, containsString(expectedError));
Expand All @@ -266,6 +267,7 @@ public void test_7_4_get_nonexistent_object() throws Exception {

/**
* 7.5-7: Archive and Restore 3 objects larger than the chunk size to BP simultaneously.
*
* @throws Exception
*/
@Test
Expand All @@ -282,6 +284,7 @@ public void test_7_5_and_6_bulk_performance_3x110GB() throws Exception {

/**
* 7.7-8: Archive 250 objects of approximately 1GB size to BP.
*
* @throws Exception
*/
@Test
Expand Down Expand Up @@ -323,7 +326,7 @@ private static boolean testBulkPutAndBulkGetPerformance(

OUT.insertLog("Bulk PUT from bucket " + bucketName);
final long startPutTime = getCurrentTime();
final String putBulkCmd = "--http -c put_bulk -b " + bucketName + " -d " + bulkPutLocalTempDir.toString();
final String putBulkCmd = "--http -c put_bulk -b " + bucketName + " -d " + bulkPutLocalTempDir.toString();
final CommandResponse putBulkResponse = Util.command(client, putBulkCmd);
OUT.insertCommand(putBulkCmd, putBulkResponse.getMessage());
final long endPutTime = getCurrentTime();
Expand Down Expand Up @@ -368,14 +371,20 @@ public void test_8_1_versioning() throws Exception {
final Long fileSize = 1024L;
final String bucketName = CertificationUtil.getBucketName(testDescription);
boolean success = false;
final UUID dataPolicy = setupDataPolicy(testDescription, false, ChecksumType.Type.MD5, client);
final TempStorageIds tempStorageIds = TempStorageUtil.setup(testDescription, dataPolicy, client);


OUT.startNewTest(testDescription);
try {

OUT.insertLog("Set data policy to use versioning");
final String enableDataPolicyVersioningCmd = "--http -c modify_data_policy --modify-params versioning:KEEP_LATEST -i " + envDataPolicyId;
final String enableDataPolicyVersioningCmd = "--http -c modify_data_policy --modify-params versioning:KEEP_LATEST -i " + dataPolicy;
final CommandResponse modifyDataPolicyResponse = Util.command(client, enableDataPolicyVersioningCmd);
OUT.insertCommand(enableDataPolicyVersioningCmd, modifyDataPolicyResponse.getMessage());
assertThat(modifyDataPolicyResponse.getReturnCode(), is(0));
client.modifyUserSpectraS3(new ModifyUserSpectraS3Request("Administrator")
.withDefaultDataPolicyId(dataPolicy));

// create and store one file
Path bulkPutLocalTempDir = CertificationUtil.createTempFiles(bucketName, numFiles, fileSize);
Expand Down Expand Up @@ -407,11 +416,9 @@ public void test_8_1_versioning() throws Exception {
} catch (final Exception e) {
LOG.error("Exception: {}", e.getMessage(), e);
} finally {
Util.deleteBucket(client, bucketName);

// undo versioning
Util.command(client, "--http -c modify_data_policy --modify-params versioning:NONE -i " + envDataPolicyId);

client.modifyUserSpectraS3(new ModifyUserSpectraS3Request("Administrator")
.withDefaultDataPolicyId(envDataPolicyId));
TempStorageUtil.teardown(testDescription, tempStorageIds, client);
OUT.finishTest(testDescription, success);
assertTrue(testDescription + " did not complete", success);
}
Expand All @@ -432,7 +439,7 @@ public void test_8_2_partial_restore() throws Exception {
try {
final Path bulkPutLocalTempDir = CertificationUtil.createTempFiles(bucketName, numFiles, fileSize);

final String putBulkCmd = "--http -c put_bulk -b " + bucketName + " -d " + bulkPutLocalTempDir.toString();
final String putBulkCmd = "--http -c put_bulk -b " + bucketName + " -d " + bulkPutLocalTempDir.toString();
final CommandResponse putBulkResponse = Util.command(client, putBulkCmd);
OUT.insertCommand(putBulkCmd, putBulkResponse.getMessage());
assertThat(putBulkResponse.getReturnCode(), is(0));
Expand All @@ -445,7 +452,7 @@ public void test_8_2_partial_restore() throws Exception {
OUT.insertPreformat(objectName);

// restore first 100 bytes
final String getPartialObjectCmd = "--http -c get_object --range-offset 0 --range-length 100 -b " + bucketName + " -o " + objectName + " -d " + bulkPutLocalTempDir.toString();
final String getPartialObjectCmd = "--http -c get_object --range-offset 0 --range-length 100 -b " + bucketName + " -o " + objectName + " -d " + bulkPutLocalTempDir.toString();
final CommandResponse getPartialResponse = Util.command(client, getPartialObjectCmd);
OUT.insertCommand(getPartialObjectCmd, getPartialResponse.getMessage());
assertThat(getPartialResponse.getReturnCode(), is(0));
Expand All @@ -455,11 +462,11 @@ public void test_8_2_partial_restore() throws Exception {
final com.spectralogic.ds3cli.util.FileUtils.ObjectsToPut objectsToPut = com.spectralogic.ds3cli.util.FileUtils.getObjectsToPut(filesToPut, bulkPutLocalTempDir, "", true);
final Ds3Object obj = objectsToPut.getDs3Objects().get(0);
assertTrue(obj.getSize() < 150);
OUT.insertLog(obj.getName() + " size: " + Long.toString(obj.getSize()));
OUT.insertLog(obj.getName() + " size: " + Long.toString(obj.getSize()));
success = true;

} catch (final Exception e) {
LOG.error("Exception in " + testDescription, e );
LOG.error("Exception in " + testDescription, e);
} finally {
OUT.finishTest(testDescription, success);
Util.deleteBucket(client, bucketName);
Expand Down Expand Up @@ -501,7 +508,7 @@ public void test_8_4_change_priorities() throws Exception {
success = true;

} catch (final Exception e) {
LOG.error("Exception in " + testDescription, e );
LOG.error("Exception in " + testDescription, e);
} finally {
OUT.finishTest(testDescription, success);
CertificationUtil.deleteJob(client, jobId);
Expand Down Expand Up @@ -551,7 +558,7 @@ public boolean apply(@Nullable final Tape tape) {
assertTrue(postEjectResponse.getMessage().contains(ejectLocation));
success = true;
} catch (final Exception e) {
LOG.error("Exception in " + testDescription, e );
LOG.error("Exception in " + testDescription, e);
} finally {
CertificationUtil.cancelTapeEject(client, barcode);
OUT.finishTest(testDescription, success);
Expand Down Expand Up @@ -587,7 +594,7 @@ public void test_8_6_fully_persisted() throws Exception {
// Create temp files for BULK_PUT
final Path bulkPutLocalTempDir = CertificationUtil.createTempFiles(bucketName, numFiles, fileSize);

final String putBulkCmd = "--http -c put_bulk -b " + bucketName + " -d " + bulkPutLocalTempDir.toString();
final String putBulkCmd = "--http -c put_bulk -b " + bucketName + " -d " + bulkPutLocalTempDir.toString();
final CommandResponse putBulkResponse = Util.command(client, putBulkCmd);
assertThat(putBulkResponse.getReturnCode(), is(0));

Expand Down Expand Up @@ -620,7 +627,7 @@ public void test_8_6_fully_persisted() throws Exception {
final CommandResponse getPhysicalPlacementAfterResponse = Util.command(client, getPhysicalPlacementCmd);
OUT.insertCommand(getPhysicalPlacementCmd, getPhysicalPlacementAfterResponse.getMessage());
} catch (final Exception e) {
LOG.error("Exception in " + testDescription, e );
LOG.error("Exception in " + testDescription, e);
} finally {
OUT.finishTest(testDescription, success);
Util.deleteBucket(client, bucketName);
Expand All @@ -642,16 +649,16 @@ public void test_8_7_large_list() throws Exception {
boolean success = false;
try {
// Put 500 files into bucket
final CommandResponse performanceResponse = Util.putPerformanceFiles(client, bucketName, numFiles, fileSize);
final CommandResponse performanceResponse = Util.putPerformanceFiles(client, bucketName, numFiles, fileSize);
assertThat(performanceResponse.getReturnCode(), is(0));

final String listBucketArgs = "--http -c get_bucket -b " + bucketName;
final CommandResponse getBucketResponseAfterBulkPut = Util.command(client, listBucketArgs);
final CommandResponse getBucketResponseAfterBulkPut = Util.command(client, listBucketArgs);
OUT.insertCommand(listBucketArgs, getBucketResponseAfterBulkPut.getMessage());
assertThat(getBucketResponseAfterBulkPut.getReturnCode(), is(0));
success = true;
} catch (final Exception e) {
LOG.error("Exception in " + testDescription, e );
LOG.error("Exception in " + testDescription, e);
} finally {
OUT.finishTest(testDescription, success);
Util.deleteBucket(client, bucketName);
Expand Down
6 changes: 4 additions & 2 deletions ...ntegration/src/test/java/com/spectralogic/ds3cli/integration/FeatureIntegration_Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.spectralogic.ds3client.commands.GetBucketResponse;
import com.spectralogic.ds3client.commands.GetObjectRequest;
import com.spectralogic.ds3client.commands.GetObjectResponse;
import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
import com.spectralogic.ds3client.helpers.FileObjectPutter;
import com.spectralogic.ds3client.helpers.FolderNameFilter;
Expand Down Expand Up @@ -833,9 +834,9 @@ public void getPhysicalPlacement() throws Exception {

final Arguments args = new Arguments(new String[]{"--http", "-c", "get_physical_placement", "-b", bucketName, "-o", "beowulf.txt" });
final CommandResponse response = Util.command(client, args);
assertTrue(response.getMessage().contains("| Object Name | ID | In Cache | Length | Offset | Latest | Version |"));
assertTrue(response.getMessage().contains("| Object Name | ID | In Cache | Length | Offset | Latest | Version |"));
assertTrue(response.getMessage().contains("| beowulf.txt |"));
assertTrue(response.getMessage().contains("| true | 294059 | 0 | true | 1 |"));
assertTrue(response.getMessage().contains("| true | 294059 | 0 | true |"));

} finally {
Util.deleteBucket(client, bucketName);
Expand Down Expand Up @@ -1254,6 +1255,7 @@ public void testResettingDeadJobTimerWithModifyJob() throws Exception {
assertNotNull(response);
assertNotNull(response.getMessage());
assertFalse(response.getMessage().isEmpty());
client.cancelJobSpectraS3(new CancelJobSpectraS3Request(job.getJobId()));
} finally {
Util.deleteBucket(client, bucketName);
}
Expand Down
2 changes: 1 addition & 1 deletion ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/EjectStorageDomain.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public CliCommand init(final Arguments args) throws Exception {
public DefaultResult call() throws Exception {

final EjectStorageDomainSpectraS3Request request
= new EjectStorageDomainSpectraS3Request(id)
= new EjectStorageDomainSpectraS3Request(id.toString())
.withBucketId(bucket).withEjectLabel(ejectLabel).withEjectLocation(ejectLocation);

this.getClient().ejectStorageDomainSpectraS3(request);
Expand Down
2 changes: 1 addition & 1 deletion ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/GetDetailedObjects.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ private void checkFilterParams() throws BadArgumentException {
public GetDetailedObjectsResult call() throws Exception {

final FluentIterable<DetailedS3Object> detailedObjects = FluentIterable.from(new LazyIterable<>(
new GetObjectsFullDetailsLoaderFactory(getClient(), this.bucketName, this.prefix, 100, 5, true)))
new GetObjectsFullDetailsLoaderFactory(getClient(), this.bucketName, 100, 5, true)))
.filter(Predicates.and(getDatePredicate(), getSizePredicate(), getNamePredicate(), getOwnerPredicate()));

return new GetDetailedObjectsResult(detailedObjects);
Expand Down
2 changes: 1 addition & 1 deletion ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/GetObjectsOnTape.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public GetObjectsOnTapeResult call() throws CommandException, IOException {
try {

final GetBlobsOnTapeSpectraS3Response response
= getClient().getBlobsOnTapeSpectraS3(new GetBlobsOnTapeSpectraS3Request(null, this.tapeId));
= getClient().getBlobsOnTapeSpectraS3(new GetBlobsOnTapeSpectraS3Request(this.tapeId));

return new GetObjectsOnTapeResult(this.tapeId, response.getBulkObjectListResult().getObjects());
} catch (final FailedRequestException e) {
Expand Down
13 changes: 6 additions & 7 deletions ds3_java_cli/src/main/java/com/spectralogic/ds3cli/views/cli/GetDataPathBackendView.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public String render(final GetDataPathBackendResult obj) {
return "No valid Data Path Backend on remote appliance";
}

initTable(ImmutableList.of("Activated", "Auto Timeout", "Auto Inspect", "Conflict Resolution", "ID",
initTable(ImmutableList.of("Activated", "Auto Timeout", "Auto Inspect", "ID",
"Last Heartbeat", "Unavailable Media Policy", "Unavailable Pool Retry Mins", "Unavailable Partition Retry Mins"));

return ASCIITable.getInstance().getTable(getHeaders(), formatTableContents());
Expand All @@ -51,12 +51,11 @@ protected String[][] formatTableContents() {
attributesArray[0] = nullGuardToString(dataPathBackend.getActivated());
attributesArray[1] = nullGuardToString(dataPathBackend.getAutoActivateTimeoutInMins());
attributesArray[2] = nullGuardToString(dataPathBackend.getAutoInspect());
attributesArray[3] = nullGuardToString(dataPathBackend.getDefaultImportConflictResolutionMode());
attributesArray[4] = nullGuardToString(dataPathBackend.getId());
attributesArray[5] = nullGuardFromDate(dataPathBackend.getLastHeartbeat(), DATE_FORMAT);
attributesArray[6] = nullGuardToString(dataPathBackend.getUnavailableMediaPolicy());
attributesArray[7] = nullGuardToString(dataPathBackend.getUnavailablePoolMaxJobRetryInMins());
attributesArray[8] = nullGuardToString(dataPathBackend.getUnavailableTapePartitionMaxJobRetryInMins());
attributesArray[3] = nullGuardToString(dataPathBackend.getId());
attributesArray[4] = nullGuardFromDate(dataPathBackend.getLastHeartbeat(), DATE_FORMAT);
attributesArray[5] = nullGuardToString(dataPathBackend.getUnavailableMediaPolicy());
attributesArray[6] = nullGuardToString(dataPathBackend.getUnavailablePoolMaxJobRetryInMins());
attributesArray[7] = nullGuardToString(dataPathBackend.getUnavailableTapePartitionMaxJobRetryInMins());
contents.add(attributesArray);
return contents.toArray(new String[contents.size()][]);
}
Expand Down
Loading