Browse files

Fixes numerous bugs

  • Loading branch information...
1 parent 57f6a6a commit 3588ccc1df18102aef4dab14d65a58acf909d41b @jamessanders committed Jan 13, 2013
View
3 java/src/main/java/com/jsync/FsResource.java
@@ -2,6 +2,7 @@
import java.io.*;
import java.net.URLConnection;
import java.util.ArrayList;
+import java.util.Date;
public class FsResource implements IResource {
@@ -21,7 +22,7 @@ public FileProperties() {
}
File file;
- FileProperties props;
+ private FileProperties props;
public FsResource(File file) {
this.file = file;
View
12 java/src/main/java/com/jsync/FsResourceUnix.java
@@ -1,8 +1,18 @@
package com.jsync;
+import java.io.File;
+
/**
* User: sanders
* Date: 9/23/12
*/
-public class FsResourceUnix {
+public class FsResourceUnix extends FsResource {
+
+ public FsResourceUnix(File file) {
+ super(file);
+ }
+ public FsResourceUnix(String fn) {
+ super(new File(fn));
+ }
+
}
View
2 java/src/main/java/com/jsync/JSync.java
@@ -9,7 +9,7 @@
}
}
- public static void exitWithHelp(JSyncOptions options, int code) {
+ private static void exitWithHelp(JSyncOptions options, int code) {
System.err.println("");
System.err.println("Usage: jsync [options] <input_path> <output_path>");
System.err.println("");
View
3 java/src/main/java/com/jsync/ResourceCopier.java
@@ -9,7 +9,7 @@
public static void copy(S3Resource in, S3Resource out) throws IOException {
System.out.println(" + remote copy: " + in.getKey() + " -> " + out.getKey());
- in.getClient().copyObject(in.getBucket(), in.getKey(), out.getBucket(), out.getKey());
+ in.copy(out);
}
public static void copy(IResource in, IResource out) throws IOException {
@@ -19,7 +19,6 @@ public static void copy(IResource in, IResource out) throws IOException {
out.mkdirs();
-
int uid = in.getUid();
int gid = in.getGid();
int mode = in.getMode();
View
26 java/src/main/java/com/jsync/ResourceGenerator.java
@@ -40,23 +40,33 @@ public IResource generateResource(String path) {
try {
uri = new URI(path);
} catch (URISyntaxException e) {
- if (SystemUtils.IS_OS_WINDOWS) {
- return new FsResourceWindows(path);
- } else {
- return new FsResource(path);
- }
+ return generateFileResource(path);
}
String scheme = uri.getScheme();
if (scheme == null || scheme.equals("file")) {
- return new FsResource(uri.getPath());
+ return generateFileResource(path);
} else if (scheme.equals("s3")) {
- S3ResourceGenerator s3generator = new S3ResourceGenerator(options.getAwsCredentials(), path, options.useReducedRedundancy(), options.makePublic());
+ S3Client s3Client =
+ new S3Client(
+ uri,
+ options.getAwsCredentials(),
+ options.useReducedRedundancy(),
+ options.makePublic());
try {
- return s3generator.getResource(path);
+ return s3Client.getResource(path);
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
return null;
}
+
+ public IResource generateFileResource(String path) {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ return new FsResourceWindows(path);
+ } else {
+ return new FsResourceUnix(path);
+ }
+ }
+
}
View
151 ...n/java/com/jsync/S3ResourceGenerator.java → java/src/main/java/com/jsync/S3Client.java
@@ -1,61 +1,60 @@
package com.jsync;
+import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.*;
import org.apache.http.client.utils.URIBuilder;
+import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
-public class S3ResourceGenerator {
-
- String bucket;
- AmazonS3Client client;
- List<S3ObjectSummary> objects;
- URI uri;
- int threadsOpen;
- boolean useReducedRedundancy;
- boolean makePublic;
-
- public S3ResourceGenerator(AWSCredentials creds, String base, boolean useReducedRedundancy, boolean makePublic) {
- URI uri = null;
- try {
- uri = new URI(base);
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
- this.uri = uri;
- bucket = uri.getHost();
- client = new AmazonS3Client(creds);
- threadsOpen = 0;
-
+/**
+ * User: sanders
+ * Date: 1/12/13
+ */
+public class S3Client {
+
+ private AmazonS3Client client;
+ private int threadsOpen;
+ private List<S3ObjectSummary> objects;
+ private URI base;
+ private String bucket;
+ private boolean useReducedRedundancy;
+ private boolean makePublic;
+
+
+ public S3Client(URI base, AmazonS3Client client, boolean useReducedRedundancy, boolean makePublic) {
+ this.bucket = base.getHost();
+ this.base = base;
+ this.client = client;
+ this.threadsOpen = 0;
this.useReducedRedundancy = useReducedRedundancy;
this.makePublic = makePublic;
}
- public void incrementThreadCount() {
+ public S3Client(URI base, AWSCredentials creds, boolean useReducedRedundancy, boolean makePublic) {
+ this(base, new AmazonS3Client(creds), useReducedRedundancy, makePublic);
+ }
+
+ private void incrementThreadCount() {
threadsOpen += 1;
}
- public void decrementThreadCount() {
+ private void decrementThreadCount() {
threadsOpen -= 1;
}
- public int getThreadsOpen() {
+ private int getThreadsOpen() {
return threadsOpen;
}
- public AmazonS3Client getClient() {
- return client;
- }
-
private List<S3ObjectSummary> getObjectList() {
if (this.objects == null) {
this.objects = new ArrayList<S3ObjectSummary>();
- ObjectListing listing = client.listObjects(bucket, uri.getPath().substring(1));
+ ObjectListing listing = client.listObjects(getBucket(), this.base.getPath().substring(1));
while (true) {
for (S3ObjectSummary obj : listing.getObjectSummaries()) {
this.objects.add(obj);
@@ -73,8 +72,7 @@ public AmazonS3Client getClient() {
public List<S3Resource> getChildren(String path, int point) throws URISyntaxException {
- //System.out.println("GETTING CHILDREN: " + path);
-
+ String bucket = base.getHost();
URI uri;
uri = new URI(path);
@@ -87,10 +85,9 @@ public AmazonS3Client getClient() {
int c = point;
for (S3ObjectSummary object : getObjectList().subList(point, getObjectList().size())) {
+
String key = "/" + object.getKey();
- // System.out.println(" -> " + key);
String repl = key.replaceFirst("^" + Pattern.quote(uri.getPath()) + "/", "");
-
Boolean matches = key.matches("^" + Pattern.quote(uri.getPath()) + "/.*$");
if (key.equals(uri.getPath())) {
@@ -116,7 +113,7 @@ else if (!repl.contains("/") && !key.equals(uri.getPath()) && matches) {
builder.setScheme("s3");
builder.setHost(bucket);
builder.setPath(key);
- files.add(new S3Resource(object, bucket, builder.build().toString(), null, false, this, client, useReducedRedundancy, makePublic, c));
+ files.add(new S3Resource(object, builder.build(), false, this, c));
} else if ((files.size() != 0 || subdirs.size() != 0) || theFile != null){
break;
}
@@ -138,7 +135,7 @@ public S3Resource getResource(String path) throws URISyntaxException {
}
public S3Resource getResource(String path, int point) throws URISyntaxException {
- //System.out.println("GETTING RESOURCE: " + path);
+
URI uri;
uri = new URI(path);
@@ -175,10 +172,80 @@ else if (!repl.contains("/") && !key.equals(uri.getPath()) && matches) {
c += 1;
}
- if (numberOfChildren > 0) {
- return new S3Resource(theFile, bucket, path, null, true, this, client, useReducedRedundancy, makePublic, off);
- } else {
- return new S3Resource(theFile, bucket, path, null, false, this, client, useReducedRedundancy, makePublic, off);
+ return new S3Resource(theFile, uri, numberOfChildren > 0, this, off);
+
+ }
+
+ public OutputStream getWriter(final URI path, final ObjectMetadata metadata) {
+ final PipedInputStream in = new PipedInputStream();
+ final PipedOutputStream out;
+ try {
+ out = new PipedOutputStream(in);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return null;
+ }
+ incrementThreadCount();
+ new Thread(
+ new Runnable(){
+ public void run(){
+ try {
+ PutObjectRequest request = new PutObjectRequest(getBucket(), path.getPath().substring(1), in, metadata);
+ if (useReducedRedundancy) {
+ request.setStorageClass(StorageClass.ReducedRedundancy.toString());
+ }
+ if (makePublic) {
+ request = request.withCannedAcl(CannedAccessControlList.PublicRead);
+ }
+ client.putObject(request);
+ } catch (AmazonClientException e) {
+ e.printStackTrace();
+ }
+ try {
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ Thread.yield();
+ decrementThreadCount();
+ }
+ }
+ ).start();
+ return out;
+ }
+
+ public Boolean cleanupWrites () {
+ while (getThreadsOpen() >= 1) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
}
+ return true;
+ }
+
+ public S3Object getObject(S3Resource resource) {
+
+ return client.getObject(this.getBucket(), resource.getKey());
+
+ }
+
+ public void deleteObject(S3Resource resource) {
+
+ client.deleteObject(this.getBucket(), resource.getKey());
+
+ }
+
+ public ObjectMetadata getObjectMetadata(S3Resource resource) {
+ return client.getObjectMetadata(this.getBucket(), resource.getKey());
+ }
+
+ public void copyObject(S3Resource in, S3Resource out) {
+ client.copyObject(this.getBucket(), in.getKey(), out.getBucket(), out.getKey());
+ }
+
+ public String getBucket() {
+ return bucket;
}
}
View
119 java/src/main/java/com/jsync/S3Resource.java
@@ -20,48 +20,35 @@
public class S3Resource implements IResource {
- String bucket;
-
URI path;
List<S3Resource> children;
Boolean isDirectory;
- AmazonS3Client client;
- S3ResourceGenerator s3ResourceGenerator;
+ S3Client client;
S3ObjectSummary s3ObjectSummary;
ObjectMetadata s3MetaData;
- boolean useReducedRedundancy;
- boolean makePublic;
int offset;
public S3Resource(S3ObjectSummary s3ObjectSummary,
- String bucket,
- String path,
- List<S3Resource> children,
+ URI path,
Boolean isDirectory,
- S3ResourceGenerator s3ResourceGenerator,
- AmazonS3Client client,
- boolean useReducedRedundacy,
- boolean makePublic,
+ S3Client client,
int offset) {
- try {
- this.path = new URI(path);
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
+
+ this.path = path;
+ this.client = client;
this.s3ObjectSummary = s3ObjectSummary;
- this.bucket = bucket;
this.isDirectory = isDirectory;
- this.client = client;
- this.children = children;
- this.s3ResourceGenerator = s3ResourceGenerator;
+ this.children = null;
this.s3MetaData = null;
- this.useReducedRedundancy = useReducedRedundacy;
- this.makePublic = makePublic;
this.offset = offset;
}
- public AmazonS3Client getClient() { return client; }
- public String getBucket() { return bucket; }
+ private S3Client getClient() { return client; }
+
+ public String getBucket() {
+ return this.getClient().getBucket();
+ }
+
public String getKey() {
if (s3ObjectSummary != null) {
return s3ObjectSummary.getKey();
@@ -70,16 +57,22 @@ public String getKey() {
}
}
+ public void copy(S3Resource out) {
+
+ this.getClient().copyObject(this, out);
+
+ }
+
public String toString () {
return path.toString();
}
- public List<S3Resource> getChildren() {
+ private List<S3Resource> getChildren() {
if (this.children != null) {
return this.children;
}
try {
- this.children = this.s3ResourceGenerator.getChildren(this.path.getPath(),this.offset);
+ this.children = this.getClient().getChildren(this.path.getPath(), this.offset);
} catch (URISyntaxException e) {
e.printStackTrace();
}
@@ -131,13 +124,10 @@ public IResource join(String name) {
}
try {
return new S3Resource(null,
- bucket,
- builder.build().toString(),
- null,
+ builder.build(),
false,
- s3ResourceGenerator,
client,
- useReducedRedundancy, makePublic, this.offset);
+ this.offset);
} catch (URISyntaxException e) {
e.printStackTrace();
}
@@ -147,67 +137,32 @@ public IResource join(String name) {
public void mkdirs() {}
public OutputStream getWriter() {
- final PipedInputStream in = new PipedInputStream();
- final PipedOutputStream out;
- try {
- out = new PipedOutputStream(in);
- } catch (IOException e) {
- e.printStackTrace();
- return null;
- }
- s3ResourceGenerator.incrementThreadCount();
- new Thread(
- new Runnable(){
- public void run(){
- try {
- PutObjectRequest request = new PutObjectRequest(bucket, path.getPath().substring(1), in, getS3Metadata());
- if (useReducedRedundancy) {
- request.setStorageClass(StorageClass.ReducedRedundancy.toString());
- }
- if (makePublic) {
- request = request.withCannedAcl(CannedAccessControlList.PublicRead);
- }
- client.putObject(request);
- } catch (AmazonClientException e) {
- e.printStackTrace();
- }
- try {
- in.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- Thread.yield();
- s3ResourceGenerator.decrementThreadCount();
- }
- }
- ).start();
- return out;
+ return this.getClient().getWriter(this.path, getS3Metadata());
}
public InputStream getReader() {
- S3Object obj = this.client.getObject(this.bucket, this.path.getPath().substring(1));
+ S3Object obj = this.getClient().getObject(this);
return obj.getObjectContent();
}
-
- public Boolean finishWrite () {
- while (s3ResourceGenerator.getThreadsOpen() >= 1) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- return true;
+
+ public Boolean finishWrite() {
+ return this.getClient().cleanupWrites();
}
+
public void delete() {
- client.deleteObject(this.getBucket(), this.path.getPath().substring(1));
+ this.getClient().deleteObject(this);
}
public long lastModified() {
- if (s3ObjectSummary == null) {
+ ObjectMetadata metadata = this.getS3Metadata();
+ String mtime = metadata.getUserMetadata().get("mtime");
+ if (mtime != null) {
+ return (new Date(Long.parseLong(mtime))).getTime()*1000;
+ } else if (s3ObjectSummary == null) {
Date lastModifed = getS3Metadata().getLastModified();
if (lastModifed != null) {
+ System.err.println("Using current time as last modifed");
return lastModifed.getTime();
} else {
return 0;
@@ -220,7 +175,7 @@ public long lastModified() {
public ObjectMetadata getS3Metadata() {
if (this.s3MetaData == null) {
try {
- this.s3MetaData = client.getObjectMetadata(bucket, this.getKey());
+ this.s3MetaData = client.getObjectMetadata(this);
} catch (Throwable e) {
this.s3MetaData = new ObjectMetadata();
}

0 comments on commit 3588ccc

Please sign in to comment.