Skip to content

Commit a740901

Browse files
kulkabhaymneethiraj
kulkabhay
authored and
mneethiraj
committed
RANGER-4905:Reduce memory needed to create Ranger policy engine
(cherry picked from commit c0480ed)
1 parent b052cde commit a740901

File tree

11 files changed

+146
-42
lines changed

11 files changed

+146
-42
lines changed

‎agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,7 @@
2929
import org.apache.ranger.plugin.model.RangerServiceResource;
3030
import org.apache.ranger.plugin.model.RangerTag;
3131
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
32-
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
33-
import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
34-
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
35-
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
36-
import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
32+
import org.apache.ranger.plugin.policyengine.*;
3733
import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
3834
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
3935
import org.apache.ranger.plugin.util.DownloadTrigger;
@@ -437,7 +433,7 @@ private void processServiceTags(ServiceTags serviceTags) {
437433

438434
for (ListIterator<RangerServiceResource> iter = serviceResources.listIterator(); iter.hasNext(); ) {
439435
RangerServiceResource serviceResource = iter.next();
440-
RangerServiceResourceMatcher serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies);
436+
RangerServiceResourceMatcher serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies, getPluginContext());
441437

442438
if (serviceResourceMatcher != null) {
443439
resourceMatchers.add(serviceResourceMatcher);
@@ -484,7 +480,7 @@ private void processServiceTagDeltas(ServiceTags deltas, ServiceTags allServiceT
484480

485481
if (removedOldServiceResource) {
486482
if (!StringUtils.isEmpty(serviceResource.getResourceSignature())) {
487-
RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies);
483+
RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies, getPluginContext());
488484

489485
if (resourceMatcher != null) {
490486
for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
@@ -613,7 +609,7 @@ private boolean removeOldServiceResource(RangerServiceResource serviceResource,
613609
return ret;
614610
}
615611

616-
static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies) {
612+
static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies, RangerPluginContext pluginContext) {
617613

618614
if (LOG.isDebugEnabled()) {
619615
LOG.debug("==> createRangerServiceResourceMatcher(serviceResource=" + serviceResource + ")");
@@ -644,6 +640,7 @@ static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(Ra
644640

645641
matcher.setServiceDef(serviceDefHelper.getServiceDef());
646642
matcher.setPolicyResources(serviceResource.getResourceElements(), policyType);
643+
matcher.setPluginContext(pluginContext);
647644

648645
if (LOG.isDebugEnabled()) {
649646
LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource

‎agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ private boolean validateZoneServiceInAllZones(List<RangerSecurityZone> zones, St
354354
policyResources.put(resourceDefName, policyResource);
355355
}
356356

357-
RangerZoneResourceMatcher matcher = new RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDefHelper);
357+
RangerZoneResourceMatcher matcher = new RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDefHelper, null);
358358

359359
matchers.add(matcher);
360360
resourceNames.addAll(policyResources.keySet());

‎agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.ranger.plugin.model.RangerPolicy;
2323
import org.apache.ranger.plugin.model.RangerServiceDef;
24+
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
2425
import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
2526
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
2627
import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator;
@@ -41,17 +42,18 @@ public class RangerZoneResourceMatcher implements RangerResourceEvaluator {
4142
private final RangerPolicyResourceMatcher policyResourceMatcher;
4243
private RangerServiceDef.RangerResourceDef leafResourceDef;
4344

44-
public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDef serviceDef) {
45-
this(securityZoneName, policyResource, new RangerServiceDefHelper(serviceDef));
45+
public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDef serviceDef, RangerPluginContext pluginContext) {
46+
this(securityZoneName, policyResource, new RangerServiceDefHelper(serviceDef), pluginContext);
4647
}
4748

48-
public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDefHelper serviceDefHelper) {
49+
public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) {
4950
final RangerServiceDef serviceDef = serviceDefHelper.getServiceDef();
5051
final Collection<String> resourceKeys = policyResource.keySet();
5152
final RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher();
5253

5354
matcher.setServiceDef(serviceDef);
5455
matcher.setServiceDefHelper(serviceDefHelper);
56+
matcher.setPluginContext(pluginContext);
5557

5658
boolean found = false;
5759

‎agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ public PolicyEngine(ServicePolicies servicePolicies, RangerPluginContext pluginC
197197
}
198198

199199
normalizeServiceDefs(servicePolicies);
200+
pluginContext.cleanResourceMatchers();
200201

201202
this.pluginContext = pluginContext;
202203
this.lock = new RangerReadWriteLock(isUseReadWriteLock);

‎agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,26 @@
2323
import org.apache.ranger.admin.client.RangerAdminClient;
2424
import org.apache.ranger.admin.client.RangerAdminRESTClient;
2525
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
26+
import org.apache.ranger.plugin.model.RangerPolicy;
27+
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
2628
import org.apache.ranger.plugin.service.RangerAuthContext;
2729
import org.apache.ranger.plugin.service.RangerAuthContextListener;
2830
import org.slf4j.Logger;
2931
import org.slf4j.LoggerFactory;
3032

33+
import java.util.HashMap;
34+
import java.util.Map;
35+
import java.util.concurrent.locks.ReentrantReadWriteLock;
36+
3137
public class RangerPluginContext {
3238
private static final Logger LOG = LoggerFactory.getLogger(RangerPluginContext.class);
3339

34-
private final RangerPluginConfig config;
35-
private RangerAuthContext authContext;
36-
private RangerAuthContextListener authContextListener;
37-
private RangerAdminClient adminClient;
40+
private final RangerPluginConfig config;
41+
private RangerAuthContext authContext;
42+
private RangerAuthContextListener authContextListener;
43+
private RangerAdminClient adminClient;
44+
private final Map<String, Map<RangerPolicy.RangerPolicyResource, RangerResourceMatcher>> resourceMatchers = new HashMap<>();
45+
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); // fair lock
3846

3947

4048
public RangerPluginContext(RangerPluginConfig config) {
@@ -53,6 +61,65 @@ public String getClusterType() {
5361

5462
public RangerAuthContext getAuthContext() { return authContext; }
5563

64+
public RangerResourceMatcher getResourceMatcher(String resourceDefName, RangerPolicy.RangerPolicyResource resource) {
65+
if (LOG.isDebugEnabled()) {
66+
LOG.debug("==> getResourceMatcher(resourceDefName={}, resource={})", resourceDefName, resource);
67+
}
68+
RangerResourceMatcher ret = null;
69+
70+
try {
71+
lock.readLock().lock();
72+
73+
Map<RangerPolicy.RangerPolicyResource, RangerResourceMatcher> matchersForResource = resourceMatchers.get(resourceDefName);
74+
75+
if (matchersForResource != null) {
76+
ret = matchersForResource.get(resource);
77+
}
78+
} finally {
79+
lock.readLock().unlock();
80+
}
81+
if (LOG.isDebugEnabled()) {
82+
LOG.debug("<== getResourceMatcher(resourceDefName={}, resource={}) : ret={}", resourceDefName, resource, ret);
83+
}
84+
85+
return ret;
86+
}
87+
88+
public void setResourceMatcher(String resourceDefName, RangerPolicy.RangerPolicyResource resource, RangerResourceMatcher matcher) {
89+
if (LOG.isDebugEnabled()) {
90+
LOG.debug("==> setResourceMatcher(resourceDefName={}, resource={}, matcher={})", resourceDefName, resource, matcher);
91+
}
92+
if (config != null && config.getPolicyEngineOptions().enableResourceMatcherReuse) {
93+
try {
94+
lock.writeLock().lock();
95+
96+
Map<RangerPolicy.RangerPolicyResource, RangerResourceMatcher> matchersForResource = resourceMatchers.computeIfAbsent(resourceDefName, k -> new HashMap<>());
97+
matchersForResource.put(resource, matcher);
98+
} finally {
99+
lock.writeLock().unlock();
100+
}
101+
}
102+
if (LOG.isDebugEnabled()) {
103+
LOG.debug("<== setResourceMatcher(resourceDefName={}, resource={}, matcher={})", resourceDefName, resource, matcher);
104+
}
105+
}
106+
107+
void cleanResourceMatchers() {
108+
if (LOG.isDebugEnabled()) {
109+
LOG.debug("==> cleanResourceMatchers()");
110+
}
111+
try {
112+
lock.writeLock().lock();
113+
114+
resourceMatchers.clear();
115+
} finally {
116+
lock.writeLock().unlock();
117+
}
118+
if (LOG.isDebugEnabled()) {
119+
LOG.debug("<== cleanResourceMatchers()");
120+
}
121+
}
122+
56123
public void setAuthContext(RangerAuthContext authContext) { this.authContext = authContext; }
57124

58125
public void setAuthContextListener(RangerAuthContextListener authContextListener) { this.authContextListener = authContextListener; }

‎agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public class RangerPolicyEngineOptions {
3636
public boolean evaluateDelegateAdminOnly = false;
3737
public boolean enableTagEnricherWithLocalRefresher = false;
3838
public boolean enableUserStoreEnricherWithLocalRefresher = false;
39+
public boolean enableResourceMatcherReuse = true;
3940
@Deprecated
4041
public boolean disableAccessEvaluationWithPolicyACLSummary = true;
4142
public boolean optimizeTrieForRetrieval = false;
@@ -61,6 +62,7 @@ public RangerPolicyEngineOptions(final RangerPolicyEngineOptions other) {
6162
this.evaluateDelegateAdminOnly = other.evaluateDelegateAdminOnly;
6263
this.enableTagEnricherWithLocalRefresher = other.enableTagEnricherWithLocalRefresher;
6364
this.enableUserStoreEnricherWithLocalRefresher = other.enableUserStoreEnricherWithLocalRefresher;
65+
this.enableResourceMatcherReuse = other.enableResourceMatcherReuse;
6466
this.optimizeTrieForRetrieval = other.optimizeTrieForRetrieval;
6567
this.disableRoleResolution = other.disableRoleResolution;
6668
this.serviceDefHelper = null;
@@ -79,6 +81,7 @@ public void configureForPlugin(Configuration conf, String propertyPrefix) {
7981
disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", false);
8082

8183
cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true);
84+
enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true);
8285

8386
if (!disableTrieLookupPrefilter) {
8487
cacheAuditResults = false;
@@ -109,6 +112,7 @@ public void configureDefaultRangerAdmin(Configuration conf, String propertyPrefi
109112
enableUserStoreEnricherWithLocalRefresher = false;
110113
optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
111114
disableRoleResolution = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.role.resolution", true);
115+
enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true);
112116
}
113117

114118
public void configureDelegateAdmin(Configuration conf, String propertyPrefix) {
@@ -120,6 +124,7 @@ public void configureDelegateAdmin(Configuration conf, String propertyPrefix) {
120124
disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", true);
121125
disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", true);
122126
optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
127+
enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true);
123128

124129
cacheAuditResults = false;
125130
evaluateDelegateAdminOnly = true;
@@ -145,6 +150,7 @@ public void configureRangerAdminForPolicySearch(Configuration conf, String prope
145150
optimizeTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.space", false);
146151
optimizeTagTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.retrieval", false);
147152
optimizeTagTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.space", true);
153+
enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true);
148154
}
149155

150156
public RangerServiceDefHelper getServiceDefHelper() {
@@ -181,6 +187,7 @@ public boolean equals(Object other) {
181187
&& this.optimizeTrieForSpace == that.optimizeTrieForSpace
182188
&& this.optimizeTagTrieForRetrieval == that.optimizeTagTrieForRetrieval
183189
&& this.optimizeTagTrieForSpace == that.optimizeTagTrieForSpace
190+
&& this.enableResourceMatcherReuse == that.enableResourceMatcherReuse
184191
;
185192
}
186193
return ret;
@@ -221,6 +228,8 @@ public int hashCode() {
221228
ret *= 2;
222229
ret += optimizeTagTrieForSpace ? 1 : 0;
223230
ret *= 2;
231+
ret += enableResourceMatcherReuse ? 1 : 0;
232+
ret *= 2;
224233
return ret;
225234
}
226235

@@ -244,6 +253,7 @@ public String toString() {
244253
", optimizeTrieForSpace: " + optimizeTrieForSpace +
245254
", optimizeTagTrieForRetrieval: " + optimizeTagTrieForRetrieval +
246255
", optimizeTagTrieForSpace: " + optimizeTagTrieForSpace +
256+
", enableResourceMatcherReuse: " + enableResourceMatcherReuse +
247257
" }";
248258

249259
}

‎agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ private void buildZoneTrie(Map<String, SecurityZoneInfo> securityZones, RangerSe
171171
policyResources.put(resourceDefName, new RangerPolicyResource(resourceValues, false, isRecursive));
172172
}
173173

174-
matchers.add(new RangerZoneResourceMatcher(zoneName, policyResources, serviceDef));
174+
matchers.add(new RangerZoneResourceMatcher(zoneName, policyResources, serviceDef, pluginContext));
175175

176176
if (LOG.isDebugEnabled()) {
177177
LOG.debug("Built matcher for resource:[{}] in zone:[{}]", resource, zoneName);

‎agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,7 @@ public RangerDefaultPolicyResourceEvaluator(long id, Map<String, RangerPolicyRes
542542
this.resourceMatcher.setPolicyResources(resource, policyType);
543543
this.resourceMatcher.setServiceDef(serviceDef);
544544
this.resourceMatcher.setServiceDefHelper(serviceDefHelper);
545+
this.resourceMatcher.setPluginContext(pluginContext);
545546
this.resourceMatcher.init();
546547
}
547548

‎agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
3838
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
3939
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
40+
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
4041
import org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher;
4142
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
4243
import org.apache.ranger.plugin.util.RangerPerfTracer;
@@ -60,6 +61,7 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM
6061
private List<RangerResourceDef> validResourceHierarchy;
6162
private boolean isInitialized = false;
6263
private RangerServiceDefHelper serviceDefHelper;
64+
private RangerPluginContext pluginContext = null;
6365

6466
private final boolean forceEnableWildcardMatch;
6567

@@ -113,6 +115,9 @@ public void setServiceDefHelper(RangerServiceDefHelper serviceDefHelper) {
113115
this.serviceDefHelper = serviceDefHelper;
114116
}
115117

118+
@Override
119+
public void setPluginContext(RangerPluginContext pluginContext) { this.pluginContext = pluginContext; }
120+
116121
public int getPolicyType() { return policyType; }
117122

118123
public RangerServiceDefHelper getServiceDefHelper() {
@@ -812,29 +817,41 @@ private RangerResourceMatcher createResourceMatcher(RangerResourceDef resourceDe
812817
String resName = resourceDef.getName();
813818
String clsName = resourceDef.getMatcher();
814819

815-
if (!StringUtils.isEmpty(clsName)) {
816-
try {
817-
@SuppressWarnings("unchecked")
818-
Class<RangerResourceMatcher> matcherClass = (Class<RangerResourceMatcher>) Class.forName(clsName);
820+
if (pluginContext != null) {
821+
ret = pluginContext.getResourceMatcher(resName, resource);
822+
}
823+
824+
if (ret == null) {
825+
if (!StringUtils.isEmpty(clsName)) {
826+
try {
827+
@SuppressWarnings("unchecked") Class<RangerResourceMatcher> matcherClass = (Class<RangerResourceMatcher>) Class.forName(clsName);
819828

820-
ret = matcherClass.newInstance();
821-
} catch (Exception excp) {
822-
LOG.error("failed to instantiate resource matcher '" + clsName + "' for '" + resName + "'. Default resource matcher will be used", excp);
829+
ret = matcherClass.newInstance();
830+
} catch (Exception excp) {
831+
LOG.error("failed to instantiate resource matcher '" + clsName + "' for '" + resName + "'. Default resource matcher will be used", excp);
832+
}
823833
}
824-
}
825834

826835

827-
if (ret == null) {
828-
ret = new RangerDefaultResourceMatcher();
829-
}
836+
if (ret == null) {
837+
ret = new RangerDefaultResourceMatcher();
838+
}
830839

831-
if (forceEnableWildcardMatch && !Boolean.parseBoolean(resourceDef.getMatcherOptions().get(OPTION_WILD_CARD))) {
832-
resourceDef = serviceDefHelper.getWildcardEnabledResourceDef(resourceDef.getName(), policyType);
833-
}
840+
if (forceEnableWildcardMatch && !Boolean.parseBoolean(resourceDef.getMatcherOptions().get(OPTION_WILD_CARD))) {
841+
resourceDef = serviceDefHelper.getWildcardEnabledResourceDef(resourceDef.getName(), policyType);
842+
}
834843

835-
ret.setResourceDef(resourceDef);
836-
ret.setPolicyResource(resource);
837-
ret.init();
844+
ret.setResourceDef(resourceDef);
845+
ret.setPolicyResource(resource);
846+
ret.init();
847+
if (pluginContext != null) {
848+
pluginContext.setResourceMatcher(resName, resource, ret);
849+
}
850+
} else {
851+
if (LOG.isDebugEnabled()) {
852+
LOG.debug("Did not create a fresh matcher - used matcher from pluginContext");
853+
}
854+
}
838855
} else {
839856
LOG.error("RangerDefaultPolicyResourceMatcher: RangerResourceDef is null");
840857
}

0 commit comments

Comments
 (0)