-
Notifications
You must be signed in to change notification settings - Fork 2.9k
/
UnderFileSystem.java
570 lines (503 loc) · 18.4 KB
/
UnderFileSystem.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.underfs;
import alluxio.AlluxioURI;
import alluxio.underfs.options.CreateOptions;
import alluxio.underfs.options.DeleteOptions;
import alluxio.underfs.options.FileLocationOptions;
import alluxio.underfs.options.MkdirsOptions;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.concurrent.ThreadSafe;
/**
* Alluxio stores data into an under layer file system. Any file system implementing this interface
* can be a valid under layer file system
*/
@ThreadSafe
public interface UnderFileSystem {
/**
* The factory for the {@link UnderFileSystem}.
*/
class Factory {
private static final Cache UFS_CACHE = new Cache();
private Factory() {} // prevent instantiation
/**
* A class used to cache UnderFileSystems.
*/
@ThreadSafe
private static final class Cache {
/**
* Maps from {@link Key} to {@link UnderFileSystem} instances.
*/
private final ConcurrentHashMap<Key, UnderFileSystem> mUnderFileSystemMap =
new ConcurrentHashMap<>();
private Cache() {}
/**
* Gets a UFS instance from the cache if exists. Otherwise, creates a new instance and adds
* that to the cache.
*
* @param path the ufs path
* @param ufsConf the ufs configuration
* @return the UFS instance
*/
UnderFileSystem get(String path, Object ufsConf) {
Key key = new Key(new AlluxioURI(path));
UnderFileSystem cachedFs = mUnderFileSystemMap.get(key);
if (cachedFs != null) {
return cachedFs;
}
UnderFileSystem fs = UnderFileSystemRegistry.create(path, ufsConf);
cachedFs = mUnderFileSystemMap.putIfAbsent(key, fs);
if (cachedFs == null) {
return fs;
}
try {
fs.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
return cachedFs;
}
void clear() {
mUnderFileSystemMap.clear();
}
}
/**
* The key of the UFS cache.
*/
private static class Key {
private final String mScheme;
private final String mAuthority;
Key(AlluxioURI uri) {
mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase();
mAuthority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase();
}
@Override
public int hashCode() {
return Objects.hashCode(mScheme, mAuthority);
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (!(object instanceof Key)) {
return false;
}
Key that = (Key) object;
return Objects.equal(mScheme, that.mScheme)
&& Objects.equal(mAuthority, that.mAuthority);
}
@Override
public String toString() {
return mScheme + "://" + mAuthority;
}
}
/**
* Clears the under file system cache.
*/
public static void clearCache() {
UFS_CACHE.clear();
}
/**
* Gets the UnderFileSystem instance according to its schema.
*
* @param path the file path storing over the ufs
* @return instance of the under layer file system
*/
public static UnderFileSystem get(String path) {
return get(path, null);
}
/**
* Gets the UnderFileSystem instance according to its scheme and configuration.
*
* @param path the file path storing over the ufs
* @param ufsConf the configuration object for ufs only
* @return instance of the under layer file system
*/
public static UnderFileSystem get(String path, Object ufsConf) {
Preconditions.checkArgument(path != null, "path may not be null");
return UFS_CACHE.get(path, ufsConf);
}
}
/**
* The different types of space indicate the total space, the free space and the space used in the
* under file system.
*/
enum SpaceType {
/**
* Indicates the storage capacity of the under file system.
*/
SPACE_TOTAL(0),
/**
* Indicates the amount of free space available in the under file system.
*/
SPACE_FREE(1),
/**
* Indicates the amount of space used in the under file system.
*/
SPACE_USED(2),
;
private final int mValue;
SpaceType(int value) {
mValue = value;
}
/**
* @return the integer value of this enum value
*/
public int getValue() {
return mValue;
}
}
/**
* Closes this under file system.
*
* @throws IOException if a non-Alluxio error occurs
*/
void close() throws IOException;
/**
* Configures and updates the properties. For instance, this method can add new properties or
* modify existing properties specified through {@link #setProperties(Map)}.
*
* The default implementation is a no-op. This should be overridden if a subclass needs
* additional functionality.
* @throws IOException if an error occurs during configuration
*/
void configureProperties() throws IOException;
/**
* Takes any necessary actions required to establish a connection to the under file system from
* the given master host e.g. logging in
* <p>
* Depending on the implementation this may be a no-op
* </p>
*
* @param hostname the host that wants to connect to the under file system
* @throws IOException if a non-Alluxio error occurs
*/
void connectFromMaster(String hostname) throws IOException;
/**
* Takes any necessary actions required to establish a connection to the under file system from
* the given worker host e.g. logging in
* <p>
* Depending on the implementation this may be a no-op
* </p>
*
* @param hostname the host that wants to connect to the under file system
* @throws IOException if a non-Alluxio error occurs
*/
void connectFromWorker(String hostname) throws IOException;
/**
* Creates a file in the under file system with the indicated name.
*
* @param path the file name
* @return A {@code OutputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
OutputStream create(String path) throws IOException;
/**
* Creates a file in the under file system with the specified {@link CreateOptions}.
* Implementations should make sure that the path under creation appears in listings only
* after a successful close and that contents are written in its entirety or not at all.
*
* @param path the file name
* @param options the options for create
* @return A {@code OutputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
OutputStream create(String path, CreateOptions options) throws IOException;
/**
* Creates a file in the under file system with the specified {@link CreateOptions}. This stream
* writes directly to the underlying storage without any atomicity guarantees.
*
* @param path the file name
* @param options the options for create
* @return A {@code OutputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
OutputStream createDirect(String path, CreateOptions options) throws IOException;
/**
* Deletes a directory from the under file system with the indicated name non-recursively.
*
* @param path of the directory to delete
* @return true if directory was found and deleted, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean deleteDirectory(String path) throws IOException;
/**
* Deletes a directory from the under file system with the indicated name.
*
* @param path of the directory to delete
* @param options for directory delete semantics
* @return true if directory was found and deleted, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean deleteDirectory(String path, DeleteOptions options) throws IOException;
/**
* Deletes a file from the under file system with the indicated name.
*
* @param path of the file to delete
* @return true if file was found and deleted, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean deleteFile(String path) throws IOException;
/**
* Gets the block size of a file in under file system, in bytes.
*
* @param path the file name
* @return the block size in bytes
* @throws IOException if a non-Alluxio error occurs
*/
long getBlockSizeByte(String path) throws IOException;
/**
* Gets the configuration object for UnderFileSystem.
*
* @return configuration object used for concrete ufs instance
*/
Object getConf();
/**
* Gets the list of locations of the indicated path.
*
* @param path the file name
* @return The list of locations
* @throws IOException if a non-Alluxio error occurs
*/
List<String> getFileLocations(String path) throws IOException;
/**
* Gets the list of locations of the indicated path given options.
*
* @param path the file name
* @param options method options
* @return The list of locations
* @throws IOException if a non-Alluxio error occurs
*/
List<String> getFileLocations(String path, FileLocationOptions options) throws IOException;
/**
* Gets the file size in bytes.
*
* @param path the file name
* @return the file size in bytes
* @throws IOException if a non-Alluxio error occurs
*/
long getFileSize(String path) throws IOException;
/**
* Gets the group of the given path. An empty implementation should be provided if not supported.
*
* @param path the path of the file
* @return the group of the file
* @throws IOException if a non-Alluxio error occurs
*/
String getGroup(String path) throws IOException;
/**
* Gets the mode of the given path in short format, e.g 0700. An empty implementation should
* be provided if not supported.
*
* @param path the path of the file
* @return the mode of the file
* @throws IOException if a non-Alluxio error occurs
*/
short getMode(String path) throws IOException;
/**
* Gets the UTC time of when the indicated path was modified recently in ms.
*
* @param path the file name
* @return modification time in milliseconds
* @throws IOException if a non-Alluxio error occurs
*/
long getModificationTimeMs(String path) throws IOException;
/**
* Gets the owner of the given path. An empty implementation should be provided if not supported.
*
* @param path the path of the file
* @return the owner of the file
* @throws IOException if a non-Alluxio error occurs
*/
String getOwner(String path) throws IOException;
/**
* @return the property map for this {@link UnderFileSystem}
*/
Map<String, String> getProperties();
/**
* Queries the under file system about the space of the indicated path (e.g., space left, space
* used and etc).
*
* @param path the path to query
* @param type the type of queries
* @return The space in bytes
* @throws IOException if a non-Alluxio error occurs
*/
long getSpace(String path, SpaceType type) throws IOException;
/**
* Returns the name of the under filesystem implementation.
*
* The name should be lowercase and not include any spaces, e.g. "hdfs", "s3".
*
* @return name of the under filesystem implementation
*/
String getUnderFSType();
/**
* Checks if a directory exists in under file system.
*
* @param path the absolute directory path
* @return true if the path exists and is a directory, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean isDirectory(String path) throws IOException;
/**
* Checks if a file exists in under file system.
*
* @param path the absolute file path
* @return true if the path exists and is a file, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean isFile(String path) throws IOException;
/**
* Returns an array of strings naming the files and directories in the directory denoted by this
* abstract pathname.
*
* <p>
* If this abstract pathname does not denote a directory, then this method returns {@code null}.
* Otherwise an array of strings is returned, one for each file or directory in the directory.
* Names denoting the directory itself and the directory's parent directory are not included in
* the result. Each string is a file name rather than a complete path.
*
* <p>
* There is no guarantee that the name strings in the resulting array will appear in any specific
* order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array naming the files and directories in the directory denoted by this
* abstract pathname. The array will be empty if the directory is empty. Returns
* {@code null} if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
UnderFileStatus[] list(String path) throws IOException;
/**
* Returns an array of strings naming the files and directories in the directory denoted by this
* abstract pathname, and all of its subdirectories.
*
* <p>
* If this abstract pathname does not denote a directory, then this method returns {@code null}.
* Otherwise an array of strings is returned, one for each file or directory in the directory and
* its subdirectories. Names denoting the directory itself and the directory's parent directory
* are not included in the result. Each string is a path relative to the given directory.
*
* <p>
* There is no guarantee that the name strings in the resulting array will appear in any specific
* order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array of strings naming the files and directories in the directory denoted by this
* abstract pathname and its subdirectories. The array will be empty if the directory is
* empty. Returns {@code null} if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
String[] listRecursive(String path) throws IOException;
/**
* Creates the directory named by this abstract pathname. If the folder already exists, the method
* returns false. The method creates any necessary but nonexistent parent directories.
*
* @param path the folder to create
* @return {@code true} if and only if the directory was created; {@code false} otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean mkdirs(String path) throws IOException;
/**
* Creates the directory named by this abstract pathname, with specified
* {@link MkdirsOptions}. If the folder already exists, the method returns false.
*
* @param path the folder to create
* @param options the options for mkdirs
* @return {@code true} if and only if the directory was created; {@code false} otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean mkdirs(String path, MkdirsOptions options) throws IOException;
/**
* Opens an {@link InputStream} at the indicated path.
*
* @param path the file name
* @return The {@code InputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
InputStream open(String path) throws IOException;
/**
* Renames a directory from {@code src} to {@code dst} in under file system.
*
* @param src the source directory path
* @param dst the destination directory path
* @return true if succeed, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean renameDirectory(String src, String dst) throws IOException;
/**
* Renames a file from {@code src} to {@code dst} in under file system.
*
* @param src the source file path
* @param dst the destination file path
* @return true if succeed, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
boolean renameFile(String src, String dst) throws IOException;
/**
* Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base
* UFS URI, and the Alluxio path from the base.
*
* The default implementation simply concatenates the path to the base URI. This should be
* overridden if a subclass needs alternate functionality.
*
* @param ufsBaseUri the base {@link AlluxioURI} in the ufs
* @param alluxioPath the path in Alluxio from the given base
* @return the UFS {@link AlluxioURI} representing the Alluxio path
*/
AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath);
/**
* Sets the configuration object for UnderFileSystem. The conf object is understood by the
* concrete underfs's implementation.
*
* @param conf the configuration object accepted by ufs
*/
void setConf(Object conf);
/**
* Sets the user and group of the given path. An empty implementation should be provided if
* unsupported.
*
* @param path the path of the file
* @param owner the new owner to set, unchanged if null
* @param group the new group to set, unchanged if null
* @throws IOException if a non-Alluxio error occurs
*/
void setOwner(String path, String owner, String group) throws IOException;
/**
* Sets the properties for this {@link UnderFileSystem}.
*
* @param properties a {@link Map} of property names to values
*/
void setProperties(Map<String, String> properties);
/**
* Changes posix file mode.
*
* @param path the path of the file
* @param mode the mode to set in short format, e.g. 0777
* @throws IOException if a non-Alluxio error occurs
*/
void setMode(String path, short mode) throws IOException;
/**
* Whether this type of UFS supports flush.
*
* @return true if this type of UFS supports flush, false otherwise
*/
boolean supportsFlush();
}