-
Notifications
You must be signed in to change notification settings - Fork 8.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
HADOOP-16005: Add XAttr support to WASB and ABFS #452
Changes from all commits
081c8c2
3cf5168
e37cdd9
20e5dde
ae5e947
939c142
b66d6c9
a167c87
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -64,6 +64,7 @@ | |
import org.apache.hadoop.fs.Seekable; | ||
import org.apache.hadoop.fs.StreamCapabilities; | ||
import org.apache.hadoop.fs.Syncable; | ||
import org.apache.hadoop.fs.XAttrSetFlag; | ||
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; | ||
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; | ||
import org.apache.hadoop.fs.azure.security.Constants; | ||
|
@@ -3563,6 +3564,76 @@ public void setOwner(Path p, String username, String groupname) | |
} | ||
} | ||
|
||
/** | ||
* Set the value of an attribute for a path. | ||
* | ||
* @param path The path on which to set the attribute | ||
* @param xAttrName The attribute to set | ||
* @param value The byte value of the attribute to set (encoded in utf-8) | ||
* @param flag The mode in which to set the attribute | ||
* @throws IOException If there was an issue setting the attribute on Azure | ||
c-w marked this conversation as resolved.
Show resolved
Hide resolved
|
||
*/ | ||
@Override | ||
public void setXAttr(Path path, String xAttrName, byte[] value, EnumSet<XAttrSetFlag> flag) throws IOException { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. too many line breaks in this method. Please remove the extra ones. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done in 54dcb1d. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Re-activating this comment as i don't see this fixed. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I removed some more newlines in 6ecbdc0. I'm finding it a bit difficult to discern what style to follow since the file seems to include multiple styles for newlines (e.g. my original implementation was close to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sorry about nit-picking but it would be good to just follow the formatting in other methods in this file. For example, you don't need new lines in the catch block and also remove 3588. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the pointers. I was previously following the style of setOwner. I removed the newlines you mentioned in f7c0d26. |
||
Path absolutePath = makeAbsolute(path); | ||
performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "setXAttr", absolutePath); | ||
|
||
String key = pathToKey(absolutePath); | ||
FileMetadata metadata; | ||
try { | ||
metadata = store.retrieveMetadata(key); | ||
} catch (IOException ex) { | ||
Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex); | ||
if (innerException instanceof StorageException | ||
&& NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) { | ||
throw new FileNotFoundException("File " + path + " doesn't exists."); | ||
} | ||
throw ex; | ||
} | ||
|
||
if (metadata == null) { | ||
throw new FileNotFoundException("File doesn't exist: " + path); | ||
} | ||
|
||
boolean xAttrExists = store.retrieveAttribute(key, xAttrName) != null; | ||
XAttrSetFlag.validate(xAttrName, xAttrExists, flag); | ||
store.storeAttribute(key, xAttrName, value); | ||
} | ||
|
||
/** | ||
* Get the value of an attribute for a path. | ||
* | ||
* @param path The path on which to get the attribute | ||
* @param xAttrName The attribute to get | ||
* @return The bytes of the attribute's value (encoded in utf-8) | ||
* or null if the attribute does not exist | ||
* @throws IOException If there was an issue getting the attribute from Azure | ||
*/ | ||
@Override | ||
public byte[] getXAttr(Path path, String xAttrName) throws IOException { | ||
Path absolutePath = makeAbsolute(path); | ||
performAuthCheck(absolutePath, WasbAuthorizationOperations.READ, "getXAttr", absolutePath); | ||
|
||
String key = pathToKey(absolutePath); | ||
FileMetadata metadata; | ||
try { | ||
metadata = store.retrieveMetadata(key); | ||
} catch (IOException ex) { | ||
Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(ex); | ||
if (innerException instanceof StorageException | ||
&& NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) { | ||
throw new FileNotFoundException("File " + path + " doesn't exists."); | ||
} | ||
throw ex; | ||
} | ||
|
||
if (metadata == null) { | ||
throw new FileNotFoundException("File doesn't exist: " + path); | ||
} | ||
|
||
return store.retrieveAttribute(key, xAttrName); | ||
} | ||
|
||
/** | ||
* Is the user allowed? | ||
* <ol> | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is there a reason for using a different encoding when compared to the
AzureBlobFileSystemStore
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The documentation for Azure Data Lake Storage Gen2 (backing
AzureBlobFileSystemStore
) states that x-ms-properties must be encoded in ISO-8859-1 (see path/update#request-headers). The documentation for Azure Blob Storage (backingAzureNativeFileSystemStore
) however only states that x-ms-meta must follow the conventions for C# identifiers (see set-blob-metadata#request-headers) which may contain Unicode letters and characters (see identifier-names). As such I believe that the two classes should use different encodings. What am I missing?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I was mostly curious about this one. I assume @DadanielZ can confirm this one?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This change is mostly looking good. If @DadanielZ can confirm the encoding issue, i can happy to commit this one.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sorry for being away so long, yes, this is fine