Skip to content
This repository has been archived by the owner on Apr 4, 2021. It is now read-only.

FALCON-2047 Use /apps/falcon/extensions/mirroring as base dir for HiveDR instead of /apps/data-mirroring/ #252

Closed
wants to merge 1 commit into from
Closed

FALCON-2047 Use /apps/falcon/extensions/mirroring as base dir for HiveDR instead of /apps/data-mirroring/ #252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,13 +136,6 @@ private void init(String[] args) throws Exception {
inputOptions = parseOptions(args);
LOG.info("Input Options: {}", inputOptions);

// Update the source staging path
inputOptions.setSourceStagingPath();
inputOptions.setTargetStagingPath();

LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());

Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(),
inputOptions.getSourceNNKerberosPrincipal());
sourceClusterFS = FileSystem.get(sourceConf);
Expand All @@ -155,6 +148,14 @@ private void init(String[] args) throws Exception {

// init DR status store
drStore = new HiveDRStatusStore(targetClusterFS);

// Update the source staging path after initing DR status store
inputOptions.setSourceStagingPath();
inputOptions.setTargetStagingPath();

LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());

eventSoucerUtil = new EventSourcerUtils(jobConf, inputOptions.shouldKeepHistory(), inputOptions.getJobName());
}

Expand Down Expand Up @@ -310,7 +311,7 @@ private String getLastEvents(Configuration conf) throws Exception {
}

private Map<String, Long> getLastDBTableEvents(Path lastEventIdFile) throws Exception {
Map<String, Long> lastEventsIdMap = new HashMap<String, Long>();
Map<String, Long> lastEventsIdMap = new HashMap<>();
BufferedReader in = new BufferedReader(new InputStreamReader(jobFS.open(lastEventIdFile)));
try {
String line;
Expand Down
2 changes: 1 addition & 1 deletion addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
*/
public abstract class DRStatusStore {

public static final String BASE_DEFAULT_STORE_PATH = "/apps/data-mirroring/";
public static final String BASE_DEFAULT_STORE_PATH = "/apps/falcon/extensions/mirroring";
public static final FsPermission DEFAULT_STORE_PERMISSION =
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.NONE);

Expand Down
2 changes: 1 addition & 1 deletion addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
import java.util.Properties;

/**
* Utility class to handle Hive events for data-mirroring.
* Utility class to handle Hive events for hive-mirroring.
*/
public class EventUtils {
private static final String DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver";
Expand Down
4 changes: 1 addition & 3 deletions extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,16 +151,14 @@ public String getExtensionLibPath(final String extensionName) throws StoreAccess

for (FileStatus fileStatus : files) {
if (fileStatus.getPath().getName().equalsIgnoreCase(LIBS_DIR)) {
if (fileStatus.getLen() != 0) {
libsPath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath());
}
break;
}
}

if (libsPath == null) {
LOG.info("For extension " + extensionName + " there is no "
+ LIBS_DIR + "at the extension store path " + storePath);
+ LIBS_DIR + "at the extension store path " + extensionPath);
return null;
} else {
return libsPath.toString();
Expand Down
4 changes: 2 additions & 2 deletions extensions/src/test/resources/hive-mirroring-template.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@
<property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
</properties>

<workflow name="##workflow.name##" engine="oozie"
path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
<workflow name="##workflow.name##" engine="oozie" path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml"
lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##notification.type##" to="##notification.receivers##"/>
<ACL/>
Expand Down
2 changes: 1 addition & 1 deletion falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
<property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/>
</properties>

<workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/>
<workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/>
<notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
Expand Down
2 changes: 1 addition & 1 deletion falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ falcon.recipe.name=hdfs-replication-monthly
##### Workflow properties
falcon.recipe.workflow.name=hdfs-dr-workflow
# Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS
falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml
falcon.recipe.workflow.path=/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml

##### Cluster properties

Expand Down
2 changes: 1 addition & 1 deletion falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
</properties>

<workflow name="##workflow.name##" engine="oozie"
path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
Expand Down
2 changes: 1 addition & 1 deletion ...n/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
</properties>

<workflow name="##workflow.name##" engine="oozie"
path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/>
<ACL/>
Expand Down
4 changes: 2 additions & 2 deletions falcon-ui/app/js/services/entity/entity-model.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@
workflow: {
_name: "hdfs-dr-workflow",
_engine: "oozie",
_path: "/apps/data-mirroring/workflows/hdfs-replication-workflow.xml",
_path: "/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml",
_lib: ""
},
retry: {
Expand Down Expand Up @@ -434,7 +434,7 @@
workflow: {
_name: "falcon-dr-hive-workflow",
_engine: "oozie",
_path: "/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml",
_path: "/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml",
_lib: ""
},
retry: {
Expand Down
10 changes: 10 additions & 0 deletions src/main/assemblies/distributed-package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,16 @@
<directoryMode>0755</directoryMode>
</fileSet>

<fileSet>
<directory>../</directory>
<outputDirectory>extensions/mirroring</outputDirectory>
<excludes>
<exclude>*/**</exclude>
</excludes>
<fileMode>0770</fileMode>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please help me understand the reason for extra write permission here..

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@PraveenAdlakha : This has been the behavior since HiveDR feature was implemented.

Here is the reason why we falcon expects 770 perm for /apps/data-mirroring now /apps/extensions/mirroring directory :
Directory is used by multiple mirroring jobs to store replication specific data. Deleting this data, even accidentally, can cause mirror jobs to redo already replicated events and this can be expensive. Hence we decided that the directory has to be read+writable by Falcon and multiple users who belong to specific group, which in this case is “users" This way, rest of the world does not have permission to read/delete the data.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for the explanation @sowmyaramesh

<directoryMode>0770</directoryMode>
</fileSet>

<fileSet>
<directory>../examples/app</directory>
<outputDirectory>examples/app</outputDirectory>
Expand Down
10 changes: 10 additions & 0 deletions src/main/assemblies/standalone-package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,16 @@
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>

<fileSet>
<directory>../</directory>
<outputDirectory>extensions/mirroring</outputDirectory>
<excludes>
<exclude>*/**</exclude>
</excludes>
<fileMode>0770</fileMode>
<directoryMode>0770</directoryMode>
</fileSet>
</fileSets>

<files>
Expand Down
2 changes: 1 addition & 1 deletion webapp/src/test/resources/hive-mirroring-template.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
</properties>

<workflow name="##workflow.name##" engine="oozie"
path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/>
path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/>
<retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/>
<notification type="##notification.type##" to="##notification.receivers##"/>
<ACL/>
Expand Down