-
Notifications
You must be signed in to change notification settings - Fork 444
/
UploadFileExample.java
275 lines (241 loc) · 11.7 KB
/
UploadFileExample.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
package com.dropbox.core.examples.upload_file;
import com.dropbox.core.*;
import com.dropbox.core.json.JsonReader;
import com.dropbox.core.util.IOUtil.ProgressListener;
import com.dropbox.core.v2.DbxClientV2;
import com.dropbox.core.v2.DbxPathV2;
import com.dropbox.core.v2.files.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* An example command-line application that runs through the web-based OAuth
* flow (using {@link DbxWebAuth}).
*/
public class UploadFileExample {
// Adjust the chunk size based on your network speed and reliability. Larger chunk sizes will
// result in fewer network requests, which will be faster. But if an error occurs, the entire
// chunk will be lost and have to be re-uploaded. Use a multiple of 4MiB for your chunk size.
private static final long CHUNKED_UPLOAD_CHUNK_SIZE = 8L << 20; // 8MiB
private static final int CHUNKED_UPLOAD_MAX_ATTEMPTS = 5;
/**
* Uploads a file in a single request. This approach is preferred for small files since it
* eliminates unnecessary round-trips to the servers.
*
* @param dbxClient Dropbox user authenticated client
* @param localFile local file to upload
* @param dropboxPath Where to upload the file to within Dropbox
*/
private static void uploadFile(DbxClientV2 dbxClient, File localFile, String dropboxPath) {
try (InputStream in = new FileInputStream(localFile)) {
ProgressListener progressListener = l -> printProgress(l, localFile.length());
FileMetadata metadata = dbxClient.files().uploadBuilder(dropboxPath)
.withMode(WriteMode.ADD)
.withClientModified(new Date(localFile.lastModified()))
.uploadAndFinish(in, progressListener);
System.out.println(metadata.toStringMultiline());
} catch (DbxException ex) {
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
System.exit(1);
} catch (IOException ex) {
System.err.println("Error reading from file \"" + localFile + "\": " + ex.getMessage());
System.exit(1);
}
}
/**
* Uploads a file in chunks using multiple requests. This approach is preferred for larger files
* since it allows for more efficient processing of the file contents on the server side and
* also allows partial uploads to be retried (e.g. network connection problem will not cause you
* to re-upload all the bytes).
*
* @param dbxClient Dropbox user authenticated client
* @param localFile local file to upload
* @param dropboxPath Where to upload the file to within Dropbox
*/
private static void chunkedUploadFile(DbxClientV2 dbxClient, File localFile, String dropboxPath) {
long size = localFile.length();
// assert our file is at least the chunk upload size. We make this assumption in the code
// below to simplify the logic.
if (size < CHUNKED_UPLOAD_CHUNK_SIZE) {
System.err.println("File too small, use upload() instead.");
System.exit(1);
return;
}
long uploaded = 0L;
DbxException thrown = null;
ProgressListener progressListener = new ProgressListener() {
long uploadedBytes = 0;
@Override
public void onProgress(long l) {
printProgress(l + uploadedBytes, size);
if (l == CHUNKED_UPLOAD_CHUNK_SIZE) uploadedBytes += CHUNKED_UPLOAD_CHUNK_SIZE;
}
};
// Chunked uploads have 3 phases, each of which can accept uploaded bytes:
//
// (1) Start: initiate the upload and get an upload session ID
// (2) Append: upload chunks of the file to append to our session
// (3) Finish: commit the upload and close the session
//
// We track how many bytes we uploaded to determine which phase we should be in.
String sessionId = null;
for (int i = 0; i < CHUNKED_UPLOAD_MAX_ATTEMPTS; ++i) {
if (i > 0) {
System.out.printf("Retrying chunked upload (%d / %d attempts)\n", i + 1, CHUNKED_UPLOAD_MAX_ATTEMPTS);
}
try (InputStream in = new FileInputStream(localFile)) {
// if this is a retry, make sure seek to the correct offset
in.skip(uploaded);
// (1) Start
if (sessionId == null) {
sessionId = dbxClient.files().uploadSessionStart()
.uploadAndFinish(in, CHUNKED_UPLOAD_CHUNK_SIZE, progressListener)
.getSessionId();
uploaded += CHUNKED_UPLOAD_CHUNK_SIZE;
printProgress(uploaded, size);
}
UploadSessionCursor cursor = new UploadSessionCursor(sessionId, uploaded);
// (2) Append
while ((size - uploaded) > CHUNKED_UPLOAD_CHUNK_SIZE) {
dbxClient.files().uploadSessionAppendV2(cursor)
.uploadAndFinish(in, CHUNKED_UPLOAD_CHUNK_SIZE, progressListener);
uploaded += CHUNKED_UPLOAD_CHUNK_SIZE;
printProgress(uploaded, size);
cursor = new UploadSessionCursor(sessionId, uploaded);
}
// (3) Finish
long remaining = size - uploaded;
CommitInfo commitInfo = CommitInfo.newBuilder(dropboxPath)
.withMode(WriteMode.ADD)
.withClientModified(new Date(localFile.lastModified()))
.build();
FileMetadata metadata = dbxClient.files().uploadSessionFinish(cursor, commitInfo)
.uploadAndFinish(in, remaining, progressListener);
System.out.println(metadata.toStringMultiline());
return;
} catch (RetryException ex) {
thrown = ex;
// RetryExceptions are never automatically retried by the client for uploads. Must
// catch this exception even if DbxRequestConfig.getMaxRetries() > 0.
sleepQuietly(ex.getBackoffMillis());
continue;
} catch (NetworkIOException ex) {
thrown = ex;
// network issue with Dropbox (maybe a timeout?) try again
continue;
} catch (UploadSessionFinishErrorException ex) {
if (ex.errorValue.isLookupFailed() && ex.errorValue.getLookupFailedValue().isIncorrectOffset()) {
thrown = ex;
// server offset into the stream doesn't match our offset (uploaded). Seek to
// the expected offset according to the server and try again.
uploaded = ex.errorValue
.getLookupFailedValue()
.getIncorrectOffsetValue()
.getCorrectOffset();
continue;
} else {
// some other error occurred, give up.
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
System.exit(1);
return;
}
} catch (DbxException ex) {
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
System.exit(1);
return;
} catch (IOException ex) {
System.err.println("Error reading from file \"" + localFile + "\": " + ex.getMessage());
System.exit(1);
return;
}
}
// if we made it here, then we must have run out of attempts
System.err.println("Maxed out upload attempts to Dropbox. Most recent error: " + thrown.getMessage());
System.exit(1);
}
private static void printProgress(long uploaded, long size) {
System.out.printf("Uploaded %12d / %12d bytes (%5.2f%%)\n", uploaded, size, 100 * (uploaded / (double) size));
}
private static void sleepQuietly(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException ex) {
// just exit
System.err.println("Error uploading to Dropbox: interrupted during backoff.");
System.exit(1);
}
}
public static void main(String[] args) throws IOException {
// Only display important log messages.
Logger.getLogger("").setLevel(Level.WARNING);
if (args.length != 3) {
System.out.println("");
System.out.println("Usage: COMMAND <auth-file> <local-path> <dropbox-path>");
System.out.println("");
System.out.println(" <auth-file>: An \"auth file\" that contains the information necessary to make");
System.out.println(" an authorized Dropbox API request. Generate this file using the \"authorize\"");
System.out.println(" example program.");
System.out.println("");
System.out.println(" <local-path>: The path to a local file whose contents you want to upload.");
System.out.println("");
System.out.println(" <dropbox-path>: The path on Dropbox to save the file to.");
System.out.println("");
System.exit(1);
return;
}
String argAuthFile = args[0];
String localPath = args[1];
String dropboxPath = args[2];
// Read auth info file.
DbxAuthInfo authInfo;
try {
authInfo = DbxAuthInfo.Reader.readFromFile(new File(argAuthFile).getAbsolutePath());
} catch (JsonReader.FileLoadException ex) {
System.err.println("Error loading <auth-file>: " + ex.getMessage());
System.exit(1);
return;
}
runExample(authInfo, localPath, dropboxPath);
System.exit(0);
}
public static void runExample(DbxAuthInfo authInfo, String localPath, String dropboxPath) {
String pathError = DbxPathV2.findError(dropboxPath);
if (pathError != null) {
System.err.println("Invalid <dropbox-path>: " + pathError);
System.exit(1);
return;
}
File localFile = new File(localPath);
if (!localFile.exists()) {
System.err.println("Invalid <local-path>: file does not exist.");
System.exit(1);
return;
}
if (!localFile.isFile()) {
System.err.println("Invalid <local-path>: not a file.");
System.exit(1);
return;
}
// Create a DbxClientV2, which is what you use to make API calls.
DbxRequestConfig requestConfig = new DbxRequestConfig("examples-upload-file");
DbxClientV2 dbxClient = new DbxClientV2(requestConfig, authInfo.getAccessToken(), authInfo.getHost());
// upload the file with simple upload API if it is small enough, otherwise use chunked
// upload API for better performance. Arbitrarily chose 2 times our chunk size as the
// deciding factor. This should really depend on your network.
if (localFile.length() <= (2 * CHUNKED_UPLOAD_CHUNK_SIZE)) {
uploadFile(dbxClient, localFile, dropboxPath);
} else {
chunkedUploadFile(dbxClient, localFile, dropboxPath);
}
try {
// Delete the file we uploaded so we don't run out of space on the integration test account
dbxClient.files().deleteV2(dropboxPath);
} catch (DbxException e) {
throw new RuntimeException("Could not cleanup the test file we just uploaded at " + dropboxPath, e);
}
}
}