From f2860ce8bd744df8db100fe96719163362c07294 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 24 Aug 2016 10:14:28 -0400 Subject: [PATCH 1/6] METRON-390: Stellar functions should initialize upon use, rather than all at once --- .../profiler/bolt/ProfileBuilderBolt.java | 1 - .../profiler/bolt/ProfileSplitterBolt.java | 1 - .../common/dsl/BaseStellarFunction.java | 5 ++ .../metron/common/dsl/FunctionResolver.java | 1 - .../metron/common/dsl/StellarFunction.java | 1 + .../metron/common/dsl/StellarFunctions.java | 11 ++-- .../common/dsl/functions/MaaSFunctions.java | 63 +++++++++++++------ .../IPProtocolTransformation.java | 5 ++ .../transformation/RemoveTransformation.java | 2 +- .../common/stellar/BaseStellarProcessor.java | 10 ++- .../common/stellar/StellarCompiler.java | 3 + .../metron/common/stellar/StellarTest.java | 2 +- .../maas/StellarMaaSIntegrationTest.java | 7 ++- .../bolt/GenericEnrichmentBolt.java | 1 - .../enrichment/bolt/ThreatIntelJoinBolt.java | 1 - .../metron/parsers/bolt/ParserBolt.java | 1 - 16 files changed, 77 insertions(+), 38 deletions(-) diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index f5cfce718e..6cf505398f 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -114,7 +114,6 @@ protected void initializeStellar() { stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .build(); - StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(stellarContext); } @Override diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java index ad54f14143..c9af519705 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java @@ -90,7 +90,6 @@ protected void initializeStellar() { stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .build(); - StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(stellarContext); } @Override diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java index 0d8a207c7d..fa0183c64e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/BaseStellarFunction.java @@ -32,4 +32,9 @@ public Object apply(List args, Context context) throws ParseException { public void initialize(Context context) { } + + @Override + public boolean isInitialized() { + return true; + } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java index 917944999a..4535b3b758 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/FunctionResolver.java @@ -20,5 +20,4 @@ import java.util.function.Function; public interface FunctionResolver extends Function { - void initializeFunctions(Context context); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java index 82b5fee262..2f52f47380 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunction.java @@ -22,4 +22,5 @@ public interface StellarFunction { Object apply(List args, Context context) throws ParseException; void initialize(Context context); + boolean isInitialized(); } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java index 36c361ab7a..00712f7ba1 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/StellarFunctions.java @@ -132,14 +132,13 @@ public void initialize(Context context) { func.initialize(context); } + @Override + public boolean isInitialized() { + return func.isInitialized(); + } + public static FunctionResolver FUNCTION_RESOLVER() { return new FunctionResolver() { - @Override - public void initializeFunctions(Context context) { - for(StellarFunctions s : StellarFunctions.values()) { - s.initialize(context); - } - } @Override public StellarFunction apply(String s) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java index 6fee251cfc..a5266382a0 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java @@ -80,6 +80,7 @@ public int hashCode() { } } public static class ModelApply implements StellarFunction { + private boolean isInitialized = false; private ServiceDiscoverer discoverer; private Cache > resultCache; public ModelApply() { @@ -178,7 +179,7 @@ public Object apply(List args, Context context) throws ParseException { } @Override - public void initialize(Context context) { + public synchronized void initialize(Context context) { try { Optional discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); if (discovererOpt.isPresent()) { @@ -188,15 +189,27 @@ public void initialize(Context context) { catch(Exception ex) { LOG.error(ex.getMessage(), ex); } + finally { + isInitialized = true; + } + } + + @Override + public boolean isInitialized() { + return isInitialized; } } public static class GetEndpoint implements StellarFunction { ServiceDiscoverer discoverer; + private boolean isInitialized = false; + private boolean isValidState = false; + @Override public Object apply(List args, Context context) throws ParseException { - if(discoverer == null) { - throw new ParseException("Unable to find ServiceDiscoverer service..."); + if(!isValidState) { + LOG.error("Invalid state: Unable to find ServiceDiscoverer service."); + return null; } String modelName = null; String modelVersion = null; @@ -224,6 +237,7 @@ public Object apply(List args, Context context) throws ParseException { return null; } } + public static Map endpointToMap(String name, String version, Endpoint ep) { Map ret = new HashMap<>(); ret.put("url", ep.getUrl()); @@ -234,25 +248,36 @@ public static Map endpointToMap(String name, String version, End } return ret; } + @Override - public void initialize(Context context) { - Optional clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); - CuratorFramework client = null; - if(clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { - client = (CuratorFramework)clientOptional.get(); - } - else { - return; - } + public synchronized void initialize(Context context) { try { - MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); - discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); - discoverer.start(); - context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - return; + Optional clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework) clientOptional.get(); + } else { + return; + } + try { + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); + context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); + isValidState = true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + return; + } + } + finally { + isInitialized = true; } } + + @Override + public boolean isInitialized() { + return isInitialized; + } } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java index 0a4137a7ce..4cd22014f0 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/IPProtocolTransformation.java @@ -206,4 +206,9 @@ public Object apply(List objects, Context context) throws ParseException public void initialize(Context context) { } + + @Override + public boolean isInitialized() { + return true; + } } diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java index be8b409226..3b443a7666 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java @@ -40,7 +40,7 @@ public boolean validate(String rule) throws ParseException { } @Override - public boolean validate(String rule, boolean throwException) throws ParseException { + public boolean validate(String rule, boolean throwException, Context context) throws ParseException { return true; } }; diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java index e0b92ec691..e4cdcfae50 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java @@ -94,12 +94,16 @@ public T parse( String rule } public boolean validate(String rule) throws ParseException { - return validate(rule, true); + return validate(rule, true, Context.EMPTY_CONTEXT()); } - public boolean validate(String rule, boolean throwException) throws ParseException { + public boolean validate(String rule, Context context) throws ParseException { + return validate(rule, true, context); + } + + public boolean validate(String rule, boolean throwException, Context context) throws ParseException { try { - parse(rule, x -> null, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); + parse(rule, x -> null, StellarFunctions.FUNCTION_RESOLVER(), context); return true; } catch(Throwable t) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java index 8a91992b28..b818834cdd 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java @@ -240,6 +240,9 @@ public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) StellarFunction func = null; try { func = functionResolver.apply(funcName); + if(!func.isInitialized()) { + func.initialize(context); + } } catch(Exception iae) { throw new ParseException("Unable to find string function " + funcName + ". Valid functions are " diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java index 1ee2cebeaa..efd7d18733 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java @@ -359,7 +359,7 @@ public static Object run(String rule, Map variables) { } public static Object run(String rule, Map variables, Context context) { StellarProcessor processor = new StellarProcessor(); - Assert.assertTrue(rule + " not valid.", processor.validate(rule)); + Assert.assertTrue(rule + " not valid.", processor.validate(rule, context)); return processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context); } diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java index 82114f7a55..f5618f594c 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java @@ -31,8 +31,10 @@ import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.common.stellar.StellarTest; import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.config.MaaSConfig; import org.apache.metron.maas.config.ModelEndpoint; import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.util.RESTUtil; import org.junit.*; @@ -62,8 +64,9 @@ public static void setup() throws Exception { context = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .build(); - StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(context); - discoverer = (ServiceDiscoverer) context.getCapability(Context.Capabilities.SERVICE_DISCOVERER).get(); + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); endpointUrl = new URL("http://localhost:8282"); ModelEndpoint endpoint = new ModelEndpoint(); { diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java index 5039f4bf95..5d4943d82a 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java @@ -159,7 +159,6 @@ protected void initializeStellar() { stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .build(); - StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(stellarContext); } @Override diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java index 0029fa2a3f..00d0498b7a 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java @@ -73,7 +73,6 @@ protected void initializeStellar() { this.stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .build(); - StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(stellarContext); } @Override diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index 2b457a725f..bdb462b790 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -100,7 +100,6 @@ protected void initializeStellar() { this.stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .build(); - StellarFunctions.FUNCTION_RESOLVER().initializeFunctions(stellarContext); } @SuppressWarnings("unchecked") From 93cda005d60db3a045b91961668e72c5ec582085 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 24 Aug 2016 10:18:45 -0400 Subject: [PATCH 2/6] ServiceDiscoverer should be more resilient here. --- .../org/apache/metron/maas/discovery/ServiceDiscoverer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java index 051e50d484..a4d89c74af 100644 --- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java +++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/discovery/ServiceDiscoverer.java @@ -77,7 +77,7 @@ public ServiceDiscoverer(CuratorFramework client, String root, int blacklistExpi try { Stat exists = client.checkExists().forPath(root); if(exists == null) { - client.create().forPath(root); + client.create().creatingParentsIfNeeded().forPath(root); } } catch (Exception e) { LOG.error("Unable to create path: " + e.getMessage(), e); From b182565520b32c2ec4083ba74ec83dfdbb68f6ba Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 24 Aug 2016 10:46:06 -0400 Subject: [PATCH 3/6] QueryFilter needs the proper context. --- .../java/org/apache/metron/parsers/bolt/ParserBolt.java | 3 ++- .../org/apache/metron/parsers/filters/QueryFilter.java | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index bdb462b790..3ee04246af 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -73,10 +73,12 @@ public ParserBolt withMessageFilter(MessageFilter filter) { public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { super.prepare(stormConf, context, collector); this.collector = collector; + initializeStellar(); if(getSensorParserConfig() == null) { filter = new GenericMessageFilter(); } else if(filter == null) { + getSensorParserConfig().getParserConfig().putIfAbsent("stellarContext", stellarContext); filter = Filters.get(getSensorParserConfig().getFilterClassName() , getSensorParserConfig().getParserConfig() ); @@ -93,7 +95,6 @@ else if(filter == null) { throw new IllegalStateException("Unable to retrieve a parser config for " + getSensorType()); } parser.configure(config.getParserConfig()); - initializeStellar(); } protected void initializeStellar() { diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java index 0ec22e2f6b..16f2143208 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/QueryFilter.java @@ -42,8 +42,11 @@ public void configure(Map config) { if(o instanceof String) { query= o.toString(); } - - processor.validate(query, true); + Context stellarContext = (Context) config.get("stellarContext"); + if(stellarContext == null) { + stellarContext = Context.EMPTY_CONTEXT(); + } + processor.validate(query, true, stellarContext); } @Override From c621c534fa917ca252995127282c954ac1ecf6e7 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 24 Aug 2016 13:12:22 -0400 Subject: [PATCH 4/6] Updating functions to throw exceptions appropriately if we can't initialize them. --- .../apache/metron/common/dsl/functions/MaaSFunctions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java index a5266382a0..00700f3d7b 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java @@ -100,6 +100,9 @@ public Object apply(List args, Context context) throws ParseException { " [endpoint method:string], model_args:map" ); } + if(!isInitialized) { + return null; + } int i = 0; if(args.size() == 0) { return null; @@ -267,7 +270,7 @@ public synchronized void initialize(Context context) { isValidState = true; } catch (Exception e) { LOG.error(e.getMessage(), e); - return; + throw new IllegalStateException("Unable to initialize MAAS_GET_ENDPOINT", e); } } finally { From 22725a741c2737fdf77865714aedde3116d3e323 Mon Sep 17 00:00:00 2001 From: cstella Date: Wed, 24 Aug 2016 13:16:08 -0400 Subject: [PATCH 5/6] Better commenting. --- .../org/apache/metron/common/dsl/functions/MaaSFunctions.java | 1 + 1 file changed, 1 insertion(+) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java index 00700f3d7b..f80a42d363 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java @@ -193,6 +193,7 @@ public synchronized void initialize(Context context) { LOG.error(ex.getMessage(), ex); } finally { + //We always want to set initialize to true because we don't want to keep trying to initialize over and over isInitialized = true; } } From c9f91fc19cd8e4dc39bdbea94cd5d242a5458294 Mon Sep 17 00:00:00 2001 From: cstella Date: Thu, 25 Aug 2016 08:58:08 -0400 Subject: [PATCH 6/6] Added more robust initialization for MaaS function. --- .../common/dsl/functions/MaaSFunctions.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java index f80a42d363..8dfac6871a 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/MaaSFunctions.java @@ -21,6 +21,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.curator.framework.CuratorFramework; +import org.apache.hadoop.security.authorize.Service; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.ParseException; import org.apache.metron.common.dsl.StellarFunction; @@ -183,11 +184,22 @@ public Object apply(List args, Context context) throws ParseException { @Override public synchronized void initialize(Context context) { + try { Optional discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); if (discovererOpt.isPresent()) { discoverer = discovererOpt.get(); } + else { + Optional clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework) clientOptional.get(); + } else { + throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); + } + discoverer = createDiscoverer(client); + } } catch(Exception ex) { LOG.error(ex.getMessage(), ex); @@ -204,6 +216,13 @@ public boolean isInitialized() { } } + private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception { + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); + return discoverer; + } + public static class GetEndpoint implements StellarFunction { ServiceDiscoverer discoverer; private boolean isInitialized = false; @@ -261,12 +280,10 @@ public synchronized void initialize(Context context) { if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { client = (CuratorFramework) clientOptional.get(); } else { - return; + throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); } try { - MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); - discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); - discoverer.start(); + discoverer = createDiscoverer(client); context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); isValidState = true; } catch (Exception e) {