From 622cae425e924db4bc5abeef09d8a184ca3a54c7 Mon Sep 17 00:00:00 2001 From: Pracheer Agarwal Date: Tue, 20 Dec 2016 10:19:20 +0530 Subject: [PATCH 1/7] FALCON-2225 extension owner added for trusted extensions --- .../java/org/apache/falcon/extensions/store/ExtensionStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index df63779f6..b59741dab 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -104,7 +104,7 @@ private void initializeDbTable() { String description = getShortDescription(extension); String recipeName = extension; String location = storePath.toString() + '/' + extension; - String extensionOwner = CurrentUser.getUser(); + String extensionOwner = System.getProperty("user.name"); metaStore.storeExtensionBean(recipeName, location, extensionType, description, extensionOwner); } } catch (FalconException e) { From daa3ffc625e15dcc8a1243ab26f99b6c730fe9e2 Mon Sep 17 00:00:00 2001 From: Pracheer Agarwal Date: Tue, 20 Dec 2016 10:31:51 +0530 Subject: [PATCH 2/7] FALCON-2225 extension owner added for trusted extensions --- .../java/org/apache/falcon/extensions/store/ExtensionStore.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index b59741dab..72d449339 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -26,7 +26,6 @@ import org.apache.falcon.extensions.ExtensionType; import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.hadoop.HadoopClientFactory; -import org.apache.falcon.security.CurrentUser; import org.apache.falcon.util.StartupProperties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; From 5d01f9cd2a422ac99de7e3417ba6c6b7daed29f1 Mon Sep 17 00:00:00 2001 From: Pracheer Agarwal Date: Mon, 23 Jan 2017 12:03:08 +0530 Subject: [PATCH 3/7] FALCON-2247 include status of entities in getDetailExtensionJobs --- .../resource/AbstractExtensionManager.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java index 8ada57652..6ed2d69b5 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java @@ -21,6 +21,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.FalconWebException; +import org.apache.falcon.entity.EntityNotRegisteredException; +import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.parser.ValidationException; import org.apache.falcon.extensions.ExtensionStatus; import org.apache.falcon.entity.v0.EntityType; @@ -58,6 +60,8 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager { private static final String EXTENSION_TYPE = "type"; private static final String EXTENSION_DESC = "description"; private static final String EXTENSION_LOCATION = "location"; + private static final String ENTITY_EXISTS_STATUS = "EXISTS"; + private static final String ENTITY_NOT_EXISTS_STATUS = "NOT_EXISTS"; protected static void validateExtensionName(final String extensionName) { if (StringUtils.isBlank(extensionName)) { @@ -142,8 +146,29 @@ private JSONObject buildExtensionJobDetailResult(final String jobName) throws Fa try { detailsObject.put(JOB_NAME, jobsBean.getJobName()); detailsObject.put(EXTENSION_NAME, jobsBean.getExtensionName()); - detailsObject.put(FEEDS, StringUtils.join(jobsBean.getFeeds(), ",")); - detailsObject.put(PROCESSES, StringUtils.join(jobsBean.getProcesses(), ",")); + + JSONObject feedsObject = new JSONObject(); + for(String feed : jobsBean.getFeeds()){ + try { + EntityUtil.getEntity(EntityType.FEED, feed); + } catch (EntityNotRegisteredException e){ + feedsObject.put(feed, ENTITY_NOT_EXISTS_STATUS); + } + feedsObject.put(feed, ENTITY_EXISTS_STATUS); + } + detailsObject.put(FEEDS, feedsObject); + + JSONObject processObject = new JSONObject(); + for(String process : jobsBean.getProcesses()){ + try { + EntityUtil.getEntity(EntityType.PROCESS, process); + } catch (EntityNotRegisteredException e){ + processObject.put(process, ENTITY_NOT_EXISTS_STATUS); + } + processObject.put(process, ENTITY_EXISTS_STATUS); + } + detailsObject.put(PROCESSES, processObject); + detailsObject.put(CONFIG, jobsBean.getConfig()); detailsObject.put(CREATION_TIME, jobsBean.getCreationTime()); detailsObject.put(LAST_UPDATE_TIME, jobsBean.getLastUpdatedTime()); From 2ad4ce7d99bf9f3079451f9a716870a3eb909faf Mon Sep 17 00:00:00 2001 From: Pracheer Agarwal Date: Mon, 23 Jan 2017 12:29:13 +0530 Subject: [PATCH 4/7] bug fixes --- .../src/test/java/org/apache/falcon/unit/TestFalconUnit.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java b/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java index 72280f713..06e547c7b 100644 --- a/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java +++ b/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java @@ -482,7 +482,10 @@ public void testExtensionJobOperations() throws Exception { assertStatus(apiResult); String processes = new JSONObject(getExtensionJobDetails(TEST_JOB).getMessage()).get("processes").toString(); - Assert.assertEquals(processes, "sample"); + JSONObject processObject = new JSONObject(); + processObject.put("sample", "EXISTS"); + + Assert.assertEquals(processes, processObject.toString()); process = (Process) getClient().getDefinition(EntityType.PROCESS.toString(), "sample", null); Assert.assertEquals(process.getPipelines(), "testSample"); From c38c51b7f23d0cd074c8d98e76599d6ba447e642 Mon Sep 17 00:00:00 2001 From: Pracheer Agarwal Date: Mon, 23 Jan 2017 13:43:27 +0530 Subject: [PATCH 5/7] indentation changes --- .../apache/falcon/resource/AbstractExtensionManager.java | 8 ++++---- .../test/java/org/apache/falcon/unit/TestFalconUnit.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java index 6ed2d69b5..16c7163c7 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java @@ -148,10 +148,10 @@ private JSONObject buildExtensionJobDetailResult(final String jobName) throws Fa detailsObject.put(EXTENSION_NAME, jobsBean.getExtensionName()); JSONObject feedsObject = new JSONObject(); - for(String feed : jobsBean.getFeeds()){ + for (String feed : jobsBean.getFeeds()) { try { EntityUtil.getEntity(EntityType.FEED, feed); - } catch (EntityNotRegisteredException e){ + } catch (EntityNotRegisteredException e) { feedsObject.put(feed, ENTITY_NOT_EXISTS_STATUS); } feedsObject.put(feed, ENTITY_EXISTS_STATUS); @@ -159,10 +159,10 @@ private JSONObject buildExtensionJobDetailResult(final String jobName) throws Fa detailsObject.put(FEEDS, feedsObject); JSONObject processObject = new JSONObject(); - for(String process : jobsBean.getProcesses()){ + for (String process : jobsBean.getProcesses()) { try { EntityUtil.getEntity(EntityType.PROCESS, process); - } catch (EntityNotRegisteredException e){ + } catch (EntityNotRegisteredException e) { processObject.put(process, ENTITY_NOT_EXISTS_STATUS); } processObject.put(process, ENTITY_EXISTS_STATUS); diff --git a/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java b/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java index 06e547c7b..43f2c52b8 100644 --- a/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java +++ b/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java @@ -484,7 +484,7 @@ public void testExtensionJobOperations() throws Exception { String processes = new JSONObject(getExtensionJobDetails(TEST_JOB).getMessage()).get("processes").toString(); JSONObject processObject = new JSONObject(); processObject.put("sample", "EXISTS"); - + Assert.assertEquals(processes, processObject.toString()); process = (Process) getClient().getDefinition(EntityType.PROCESS.toString(), "sample", null); Assert.assertEquals(process.getPipelines(), "testSample"); From 76215d782852913d3fd11475a762318fca1399b7 Mon Sep 17 00:00:00 2001 From: Pracheer Agarwal Date: Mon, 23 Jan 2017 17:42:10 +0530 Subject: [PATCH 6/7] changes for review comments --- .../resource/AbstractExtensionManager.java | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java index 16c7163c7..e224ed1a2 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java @@ -146,29 +146,8 @@ private JSONObject buildExtensionJobDetailResult(final String jobName) throws Fa try { detailsObject.put(JOB_NAME, jobsBean.getJobName()); detailsObject.put(EXTENSION_NAME, jobsBean.getExtensionName()); - - JSONObject feedsObject = new JSONObject(); - for (String feed : jobsBean.getFeeds()) { - try { - EntityUtil.getEntity(EntityType.FEED, feed); - } catch (EntityNotRegisteredException e) { - feedsObject.put(feed, ENTITY_NOT_EXISTS_STATUS); - } - feedsObject.put(feed, ENTITY_EXISTS_STATUS); - } - detailsObject.put(FEEDS, feedsObject); - - JSONObject processObject = new JSONObject(); - for (String process : jobsBean.getProcesses()) { - try { - EntityUtil.getEntity(EntityType.PROCESS, process); - } catch (EntityNotRegisteredException e) { - processObject.put(process, ENTITY_NOT_EXISTS_STATUS); - } - processObject.put(process, ENTITY_EXISTS_STATUS); - } - detailsObject.put(PROCESSES, processObject); - + detailsObject.put(FEEDS, getEntitiesStatus(jobsBean.getFeeds(), EntityType.FEED)); + detailsObject.put(PROCESSES, getEntitiesStatus(jobsBean.getProcesses(), EntityType.PROCESS)); detailsObject.put(CONFIG, jobsBean.getConfig()); detailsObject.put(CREATION_TIME, jobsBean.getCreationTime()); detailsObject.put(LAST_UPDATE_TIME, jobsBean.getLastUpdatedTime()); @@ -281,4 +260,17 @@ protected static void checkIfExtensionJobNameExists(String jobName, String exten Response.Status.INTERNAL_SERVER_ERROR); } } + + private JSONObject getEntitiesStatus(List entities, EntityType type) throws JSONException, FalconException { + JSONObject entityObject = new JSONObject(); + for (String entity : entities) { + try { + EntityUtil.getEntity(type, entity); + } catch (EntityNotRegisteredException e) { + entityObject.put(entity, ENTITY_NOT_EXISTS_STATUS); + } + entityObject.put(entity, ENTITY_EXISTS_STATUS); + } + return entityObject; + } } From c57efa2b4cdb7b9d4a12682fcac4e122afcdd32f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 24 Jan 2017 15:57:13 +0530 Subject: [PATCH 7/7] FALCON-2247 review commits changes --- .../apache/falcon/resource/AbstractExtensionManager.java | 6 ++++-- .../test/java/org/apache/falcon/unit/TestFalconUnit.java | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java index d551883d8..ab08e000c 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java @@ -57,6 +57,7 @@ public class AbstractExtensionManager extends AbstractSchedulableEntityManager { private static final String LAST_UPDATE_TIME = "lastUpdatedTime"; public static final String NAME = "name"; + public static final String STATUS = "status"; private static final String EXTENSION_TYPE = "type"; private static final String EXTENSION_DESC = "description"; private static final String EXTENSION_LOCATION = "location"; @@ -275,11 +276,12 @@ private JSONObject getEntitiesStatus(List entities, EntityType type) thr JSONObject entityObject = new JSONObject(); for (String entity : entities) { try { + entityObject.put(NAME, entity); EntityUtil.getEntity(type, entity); + entityObject.put(STATUS, ENTITY_EXISTS_STATUS); } catch (EntityNotRegisteredException e) { - entityObject.put(entity, ENTITY_NOT_EXISTS_STATUS); + entityObject.put(STATUS, ENTITY_NOT_EXISTS_STATUS); } - entityObject.put(entity, ENTITY_EXISTS_STATUS); } return entityObject; } diff --git a/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java b/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java index 43f2c52b8..8949c4194 100644 --- a/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java +++ b/unit/src/test/java/org/apache/falcon/unit/TestFalconUnit.java @@ -483,7 +483,8 @@ public void testExtensionJobOperations() throws Exception { String processes = new JSONObject(getExtensionJobDetails(TEST_JOB).getMessage()).get("processes").toString(); JSONObject processObject = new JSONObject(); - processObject.put("sample", "EXISTS"); + processObject.put("name", "sample"); + processObject.put("status", "EXISTS"); Assert.assertEquals(processes, processObject.toString()); process = (Process) getClient().getDefinition(EntityType.PROCESS.toString(), "sample", null);