Skip to content
This repository has been archived by the owner on Apr 16, 2021. It is now read-only.

Commit

Permalink
fix(s3): added integration tests with apache wagon provider test (#26)
Browse files Browse the repository at this point in the history 
* fix(s3): added integration tests with apache wagon provider test

* Reverted to private. Added variable to use.

The constant was not used. 
Then a value was hardcoded in the class.
Changed it in order to be used. 
Remained private due to being constant only in one class.

* Update KeyResolver.java

* Update EndpointProperty.java

* Update PathStyleEnabledProperty.java

* fix(s3): includes review change requested
  • Loading branch information
@sgandon @gkatzioura
sgandon authored and gkatzioura committed Jan 28, 2019
1 parent abb5d15 commit 4afb7b6
Show file tree
Hide file tree
Showing 12 changed files with 316 additions and 48 deletions.
4 changes: 2 additions & 2 deletions AzureStorageWagon/src/main/java/com/gkatzioura/maven/cloud/abs/AzureStorageWagon.java
View file
Expand Up @@ -49,7 +49,7 @@ public void get(String resourceName, File destination) throws TransferFailedExce

Resource resource = new Resource(resourceName);
transferListenerContainer.fireTransferInitiated(resource, TransferEvent.REQUEST_GET);
transferListenerContainer.fireTransferStarted(resource, TransferEvent.REQUEST_GET);
transferListenerContainer.fireTransferStarted(resource, TransferEvent.REQUEST_GET, destination);

final TransferProgress transferProgress = new TransferProgressImpl(resource, TransferEvent.REQUEST_GET, transferListenerContainer);

Expand Down Expand Up @@ -88,7 +88,7 @@ public void put(File file, String resourceName) throws TransferFailedException,
LOGGER.log(Level.FINER, String.format("Uploading file %s to %s", file.getAbsolutePath(), resourceName));

transferListenerContainer.fireTransferInitiated(resource,TransferEvent.REQUEST_PUT);
transferListenerContainer.fireTransferStarted(resource,TransferEvent.REQUEST_PUT);
transferListenerContainer.fireTransferStarted(resource,TransferEvent.REQUEST_PUT, file);
final TransferProgress transferProgress = new TransferProgressImpl(resource, TransferEvent.REQUEST_PUT, transferListenerContainer);

try {
Expand Down
5 changes: 4 additions & 1 deletion ...rageCore/src/main/java/com/gkatzioura/maven/cloud/listener/TransferListenerContainer.java
View file
Expand Up @@ -19,6 +19,8 @@
import org.apache.maven.wagon.events.TransferListener;
import org.apache.maven.wagon.resource.Resource;

import java.io.File;

public interface TransferListenerContainer {

/**
Expand Down Expand Up @@ -59,9 +61,10 @@ public interface TransferListenerContainer {
*
* @param resource The resource being transfered
* @param requestType The type of request being executed
* @param localFile local file used
* @see org.apache.maven.wagon.events.TransferEvent#TRANSFER_STARTED
*/
void fireTransferStarted(Resource resource, int requestType);
void fireTransferStarted(Resource resource, int requestType, File localFile);

/**
* Notify {@link TransferListener}s about the progress of a transfer
Expand Down
6 changes: 5 additions & 1 deletion ...Core/src/main/java/com/gkatzioura/maven/cloud/listener/TransferListenerContainerImpl.java
View file
Expand Up @@ -16,6 +16,7 @@

package com.gkatzioura.maven.cloud.listener;

import java.io.File;
import java.util.Vector;

import org.apache.maven.wagon.Wagon;
Expand Down Expand Up @@ -60,8 +61,11 @@ public void fireTransferInitiated(Resource resource, int requestType) {
}

@Override
public void fireTransferStarted(Resource resource, int requestType) {
public void fireTransferStarted(Resource resource, int requestType, File localFile) {
resource.setContentLength(localFile.length());
resource.setLastModified(localFile.lastModified());
TransferEvent transferEvent = new TransferEvent(this.wagon,resource,TransferEvent.TRANSFER_STARTED,requestType);
transferEvent.setLocalFile(localFile);
transferListeners.forEach(tl->tl.transferStarted(transferEvent));
}

Expand Down
6 changes: 4 additions & 2 deletions CloudStorageCore/src/main/java/com/gkatzioura/maven/cloud/resolver/KeyResolver.java
View file
Expand Up @@ -33,10 +33,12 @@ public String resolve(String... paths) {
}

private String replaceLast(StringBuilder stringBuilder) {
if(stringBuilder.length() == 0) {
return "";
}

stringBuilder.replace(stringBuilder.lastIndexOf("/"), stringBuilder.lastIndexOf("/") + 1, "" );
return stringBuilder.toString();
}



}
49 changes: 42 additions & 7 deletions ...re/src/main/java/com/gkatzioura/maven/cloud/transfer/TransferProgressFileInputStream.java
View file
Expand Up @@ -24,35 +24,70 @@
public final class TransferProgressFileInputStream extends FileInputStream {

private final TransferProgress transferProgress;
private long byteLeft;

public TransferProgressFileInputStream(File file, TransferProgress transferProgress) throws FileNotFoundException {



public TransferProgressFileInputStream(File file, TransferProgress transferProgress) throws IOException{
super(file);
this.transferProgress = transferProgress;
resetByteLeft();
}

private void resetByteLeft() throws IOException {
byteLeft = this.getChannel().size();
}

@Override
public synchronized void reset() throws IOException {
super.reset();
resetByteLeft();
}

@Override
public int read() throws IOException {
int b = super.read();
this.transferProgress.progress(new byte[]{(byte) b}, 1);
if(b != -1){
this.transferProgress.progress(new byte[]{(byte) b}, 1);
byteLeft--;
}//else we try to read but it was the end of the stream so nothing to report
return b;
}

@Override
public int read(byte b[]) throws IOException {
int count = super.read(b);
this.transferProgress.progress(b, b.length);
if (count != -1) {
this.transferProgress.progress(b, b.length);
byteLeft -= b.length;
}else{//end of the stream
this.transferProgress.progress(b, Math.toIntExact(byteLeft));
}
return count;
}

@Override
public int read(byte b[], int off, int len) throws IOException {
int count = super.read(b, off, len);
if (off == 0) {
this.transferProgress.progress(b, len);
if (count != -1) {
this.transferProgress.progress(b, count);
byteLeft -= count;
}else{//end of the stream
this.transferProgress.progress(b, Math.toIntExact(byteLeft));
}
} else {
byte[] bytes = new byte[len];
System.arraycopy(b, off, bytes, 0, len);
this.transferProgress.progress(bytes, len);
if (count != -1) {
byte[] bytes = new byte[count];
System.arraycopy(b, off, bytes, 0, count);
this.transferProgress.progress(bytes, len);
byteLeft -= count;
}else{//end of the stream
byte[] bytes = new byte[Math.toIntExact(byteLeft)];
System.arraycopy(b, off, bytes, 0, Math.toIntExact(byteLeft));
this.transferProgress.progress(b, Math.toIntExact(byteLeft));
}
}
return count;
}
Expand Down
4 changes: 2 additions & 2 deletions GoogleStorageWagon/src/main/java/com/gkatzioura/maven/cloud/gcs/GoogleStorageWagon.java
View file
Expand Up @@ -51,7 +51,7 @@ public void get(String resourceName, File destination) throws TransferFailedExce

Resource resource = new Resource(resourceName);
transferListenerContainer.fireTransferInitiated(resource, TransferEvent.REQUEST_GET);
transferListenerContainer.fireTransferStarted(resource, TransferEvent.REQUEST_GET);
transferListenerContainer.fireTransferStarted(resource, TransferEvent.REQUEST_GET, destination);

try {
googleStorageRepository.copy(resourceName, destination);
Expand Down Expand Up @@ -80,7 +80,7 @@ public void put(File file, String resourceName) throws TransferFailedException,
LOGGER.log(Level.FINER, String.format("Uploading file %s to %s", file.getAbsolutePath(), resourceName));

transferListenerContainer.fireTransferInitiated(resource,TransferEvent.REQUEST_PUT);
transferListenerContainer.fireTransferStarted(resource,TransferEvent.REQUEST_PUT);
transferListenerContainer.fireTransferStarted(resource,TransferEvent.REQUEST_PUT, file);
final TransferProgress transferProgress = new TransferProgressImpl(resource, TransferEvent.REQUEST_PUT, transferListenerContainer);

try(InputStream inputStream = new TransferProgressFileInputStream(file, transferProgress)) {
Expand Down
29 changes: 29 additions & 0 deletions S3StorageWagon/pom.xml
View file
Expand Up @@ -71,6 +71,12 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-test</artifactId>
<version>${wagon.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand All @@ -88,5 +94,28 @@
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>no-aws-s3-tests</id>
<activation>
<property>
<name>!real-s3-tests</name>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<!-- Tests that require an S3 setup with creadentials-->
<excludes>
<exclude>**/S3StorageWagonTest.*</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>

</project>
2 changes: 1 addition & 1 deletion S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/RegionProperty.java
View file
Expand Up @@ -41,7 +41,7 @@ public String get() {
if (region != null){
return region;
}
String regionEnv = System.getProperty("AWS_DEFAULT_REGION");
String regionEnv = System.getProperty(AWS_DEFAULT_REGION_TAG);
if(regionEnv != null) {
return regionEnv;
}
Expand Down
56 changes: 34 additions & 22 deletions S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageRepository.java
View file
Expand Up @@ -21,13 +21,15 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;

import com.amazonaws.SdkClientException;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.commons.io.IOUtils;
import org.apache.maven.wagon.ResourceDoesNotExistException;
import org.apache.maven.wagon.TransferFailedException;
Expand All @@ -52,7 +54,6 @@ public class S3StorageRepository {
private final String bucket;
private final String baseDirectory;

private final CredentialsFactory credentialsFactory = new CredentialsFactory();
private final KeyResolver keyResolver = new KeyResolver();

private AmazonS3 amazonS3;
Expand All @@ -74,16 +75,7 @@ public void connect(AuthenticationInfo authenticationInfo, RegionProperty region
try {
final Optional<String> regionOpt;

builder = AmazonS3ClientBuilder.standard().withCredentials(credentialsFactory.create(authenticationInfo));

if (endpoint.get() != null){
builder.setEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration(endpoint.get(), region.get()));
}else {
builder.withRegion(region.get());
}


builder.setPathStyleAccessEnabled(pathStyle.get());
builder = createAmazonS3ClientBuilder(authenticationInfo, region, endpoint, pathStyle);

amazonS3 = builder.build();
amazonS3.listBuckets();
Expand All @@ -106,6 +98,21 @@ public void connect(AuthenticationInfo authenticationInfo, RegionProperty region
}
}

public static AmazonS3ClientBuilder createAmazonS3ClientBuilder(AuthenticationInfo authenticationInfo, RegionProperty region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle) {
AmazonS3ClientBuilder builder;
builder = AmazonS3ClientBuilder.standard().withCredentials(new CredentialsFactory().create(authenticationInfo));

if (endpoint.get() != null){
builder.setEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration(endpoint.get(), region.get()));
}else {
builder.withRegion(region.get());
}


builder.setPathStyleAccessEnabled(pathStyle.get());
return builder;
}

public void copy(String resourceName, File destination, TransferProgress transferProgress) throws TransferFailedException, ResourceDoesNotExistException {

final String key = resolveKey(resourceName);
Expand All @@ -118,7 +125,7 @@ public void copy(String resourceName, File destination, TransferProgress transfe
} catch (AmazonS3Exception e) {
throw new ResourceDoesNotExistException("Resource does not exist");
}

destination.getParentFile().mkdirs();//make sure the folder exists or the outputStream will fail.
try(OutputStream outputStream = new TransferProgressFileOutputStream(destination,transferProgress);
InputStream inputStream = s3Object.getObjectContent()) {
IOUtils.copy(inputStream,outputStream);
Expand Down Expand Up @@ -169,23 +176,19 @@ public List<String> list(String path) {
ObjectListing objectListing = amazonS3.listObjects(new ListObjectsRequest()
.withBucketName(bucket)
.withPrefix(key));

return totalObjects(objectListing);
List<String> objects = new ArrayList<>();
retrieveAllObjects(objectListing, objects);
return objects;
}

private List<String> totalObjects(ObjectListing objectListing) {

List<String> objects = new ArrayList<>();
private void retrieveAllObjects(ObjectListing objectListing, List<String> objects) {

objectListing.getObjectSummaries().forEach(os->objects.add(os.getKey()));
objectListing.getObjectSummaries().forEach( os-> objects.add(os.getKey()));

if(objectListing.isTruncated()) {

ObjectListing nextObjectListing = amazonS3.listNextBatchOfObjects(objectListing);
objects.addAll(totalObjects(nextObjectListing));
retrieveAllObjects(nextObjectListing, objects);
}

return objects;
}

public boolean exists(String resourceName) {
Expand All @@ -208,4 +211,13 @@ private String resolveKey(String path) {
return keyResolver.resolve(baseDirectory,path);
}

public String getBucket() {
return bucket;
}

public String getBaseDirectory() {
return baseDirectory;
}


}

0 comments on commit 4afb7b6

Please sign in to comment.