HIVE-24187: Handle _files creation for HA config with same nameservic… #1515
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
…e name on source and destination
aasha
reviewed
Sep 22, 2020
| @@ -522,6 +522,14 @@ private static void populateLlapDaemonVarsSet(Set<String> llapDaemonVarsSetLocal | |||
| REPLCMINTERVAL("hive.repl.cm.interval","3600s", | |||
| new TimeValidator(TimeUnit.SECONDS), | |||
| "Inteval for cmroot cleanup thread."), | |||
| REPL_HA_DATAPATH_REPLACE_REMOTE_NAMESERVICE("hive.repl.ha.datapath.replace.remote.nameservice", false, | |||
| "When HDFS is HA enabled and both source and target clusters are configured with same nameservice names," + | |||
| "enable this flag and provide a "), | |||
| /** | ||
| * Given a ReplChangeManger's encoded uri, replaces the namespace and returns the modified encoded uri. | ||
| */ | ||
| public static String replaceNameSpaceInEncodedURI(String cmEncodedURI, HiveConf hiveConf) throws SemanticException { |
| if ((fileChecksum != null) && (cmroot != null)) { | ||
| encodedUri = encodedUri + URI_FRAGMENT_SEPARATOR + fileChecksum + URI_FRAGMENT_SEPARATOR + cmroot; | ||
| } else { | ||
| encodedUri = encodedUri + URI_FRAGMENT_SEPARATOR + URI_FRAGMENT_SEPARATOR; |
There was a problem hiding this comment.
why do we have 2 URI_FRAGMENT_SEPARATOR
| encodedUri = encodedUri + URI_FRAGMENT_SEPARATOR + URI_FRAGMENT_SEPARATOR; | ||
| } | ||
| encodedUri = encodedUri + URI_FRAGMENT_SEPARATOR + ((encodedSubDir != null) ? encodedSubDir : ""); | ||
| if (LOG.isDebugEnabled()) { |
| @@ -424,6 +424,20 @@ public String encodeFileUri(String fileUriStr, String fileChecksum, String encod | |||
| return encodedUri; | |||
| } | |||
|
|
|||
| public static String encodeFileUri(String fileUriStr, String fileChecksum, String cmroot, String encodedSubDir) { | |||
| String encodedUri = fileUriStr; | |||
| if ((fileChecksum != null) && (cmroot != null)) { | |||
aasha
reviewed
Sep 22, 2020
| @@ -1963,4 +2062,12 @@ private void setupUDFJarOnHDFS(Path identityUdfLocalPath, Path identityUdfHdfsPa | |||
| FileSystem fs = primary.miniDFSCluster.getFileSystem(); | |||
| fs.copyFromLocalFile(identityUdfLocalPath, identityUdfHdfsPath); | |||
| } | |||
|
|
|||
| private List<String> getHdfsNamespaceClause() { | |||
…e name on source and destination
aasha
reviewed
Sep 22, 2020
| } | ||
|
|
||
| @Test | ||
| public void testHdfsNamespaceWithDataCopy() throws Throwable { |
…e name on source and destination
kgyrtkirk
added
tests failed
tests pending
tests unstable
and removed
tests pending
tests failed
labels
…e name on source and destination
kgyrtkirk
added
tests pending
tests passed
and removed
tests unstable
tests pending
labels
|
This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
…e name on source and destination
What changes were proposed in this pull request?
Why are the changes needed?
Does this PR introduce any user-facing change?
How was this patch tested?