diff --git a/DISCLAIMER.txt b/DISCLAIMER.txt index 4d8e175ece..b3bc2048ed 100644 --- a/DISCLAIMER.txt +++ b/DISCLAIMER.txt @@ -1,15 +1,6 @@ -Apache Ranger is an effort undergoing incubation at the Apache Software -Foundation (ASF), sponsored by the Apache Incubator PMC. +Apache Ranger is a Top Level Project (TLP) at the Apache Software Foundation (ASF). -Incubation is required of all newly accepted projects until a further review -indicates that the infrastructure, communications, and decision making process -have stabilized in a manner consistent with other successful ASF projects. +This product includes software developed at The Apache Software +Foundation (http://www.apache.org/). -While incubation status is not necessarily a reflection of the completeness -or stability of the code, it does indicate that the project has yet to be -fully endorsed by the ASF. - -For more information about the incubation status of the Apache Ranger project you -can go to the following page: - -http://ranger.incubator.apache.org +http://ranger.apache.org diff --git a/LICENSE.txt b/LICENSE.txt index 2877b1ff56..7aa4bf4e63 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -214,38 +214,39 @@ licenses. The Apache Ranger project bundles the following files under the MIT License: -This product includes jQuery (http://jquery.org - MIT license), Copyright © 2014, John Resig. -This product includes jQuery UI (http://jqueryui.com - MIT license), Copyright © 2013 jQuery Foundation. -This product includes Backbone (http://backbonejs.org - MIT license), Copyright © 2010-2014 Jeremy Ashkenas, DocumentCloud. -This product includes underscore (http:underscorejs.org - MIT license), Copyright © 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors. -This product includes Backbone.Marionette (http://marionettejs.com/ - MIT license). -This product includes Backbone.Wreqr (http://marionettejs.com/ - MIT license) Copyright ©,2012 Derick Bailey, Muted Solutions, LLC. -This product includes Backbone.BabySitter (http://marionettejs.com/ - MIT license), Copyright ©2013 Derick Bailey, Muted Solutions, LLC. -This product includes Backbone fetch cache (https://github.com/mrappleton/backbone-fetch-cache - MIT license), Copyright © 2012-2013 Andrew Appleton. -This product includes Backbone-forms (https://github.com/powmedia/backbone-forms - MIT license), Copyright © 2013 Charles Davison. -This product includes Backbone localStorage Adapter v1.1.7 (http://documentup.com/jeromegn/backbone.localStorage - MIT license), Copyright © 2010 Jerome Gravel-Niquet. -This product includes Backbone.BootstrapModal (https://github.com/powmedia/backbone.bootstrap-modal - MIT license), Copyright © 2013 Charles Davison. -This product includes X-editable - v1.5.0 (http://vitalets.github.io/x-editable/ - MIT license), Copyright © 2013 Vitaliy Potapov; -This product includes backgrid 0.3.5 (http://backgridjs.com/ - MIT license), Copyright © 2014 Jimmy Yuen Ho Wong and contributors. -This product includes backgrid-paginator (http://github.com/wyuenho/backgrid - MIT license), Copyright © 2013 Jimmy Yuen Ho Wong and contributors. This product includes backgrid-filter (http://github.com/wyuenho/backgrid - MIT license), Copyright © 2013 Jimmy Yuen Ho Wong and contributors. -This product includes backbone-pageable 1.3.2 (http://github.com/wyuenho/backbone-pageable - MIT license), Copyright © 2013 Jimmy Yuen Ho Wong. -This product includes jQuery Cookie Plugin v1.4.0 (https://github.com/carhartl/jquery-cookie - MIT license), Copyright 2013 Klaus Hartl -This product includes jQuery Toggles v2.0.4 (http://simontabor.com/labs/toggles - MIT license), Copyright 2013 Simon Tabor. -This product includes jQuery UI Tag-it! (http://aehlke.github.com/tag-it/ - MIT license), Copyright 2011, Levy Carneiro Jr. -This product includes bootbox.js v3.3.0 (http://bootboxjs.com/ - MIT license), Copyright © 2011-2014 by Nick Payne. -This product includes moment.js (http://moment.js - MIT license), Copyright © 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors. -This product includes Globalize (http://github.com/jquery/globalize - MIT license), Copyright Software Freedom Conservancy, Inc. Globalize license applies to all files under globalize including globalize/generator. -This product includes RequireJS 2.1.8 (https://github.com/requirejs/requirejs - MIT licensed), Copyright jQuery Foundation and other contributors. -This product includes Handlebars.js (http://handlebarsjs.com/ - MIT license), Copyright © 2011-2014 by Yehuda Katz. -This product includes Font Awesome 3.2.1 (http://fontawesome.io/ - MIT license) by Dave Gandy -This product includes Qunit(https://qunitjs.com/ - MIT license) Copyright (c) 2011 John Resig, Jörn Zaefferer -This product includes jQuery Migrate-v1.1.1 (https://plugins.jquery.com/migrate/ - MIT license) Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors +This product includes jQuery (http://jquery.org ), Copyright © 2014, John Resig. +This product includes jQuery UI (http://jqueryui.com ), Copyright © 2013 jQuery Foundation. +This product includes Backbone (http://backbonejs.org ), Copyright © 2010-2014 Jeremy Ashkenas, DocumentCloud. +This product includes underscore (http:underscorejs.org ), Copyright © 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors. +This product includes Backbone.Marionette (http://marionettejs.com/ ) Copyright (c)2013 Derick Bailey, Muted Solutions, LLC. +This product includes Backbone.Wreqr (http://marionettejs.com/ ) Copyright ©,2012 Derick Bailey, Muted Solutions, LLC. +This product includes Backbone.BabySitter (http://marionettejs.com/ ), Copyright ©2013 Derick Bailey, Muted Solutions, LLC. +This product includes Backbone fetch cache (https://github.com/mrappleton/backbone-fetch-cache ), Copyright © 2012-2013 Andrew Appleton. +This product includes Backbone-forms (https://github.com/powmedia/backbone-forms ), Copyright © 2013 Charles Davison. +This product includes Backbone localStorage Adapter v1.1.7 (http://documentup.com/jeromegn/backbone.localStorage ), Copyright © 2010 Jerome Gravel-Niquet. +This product includes Backbone.BootstrapModal (https://github.com/powmedia/backbone.bootstrap-modal ), Copyright © 2013 Charles Davison. +This product includes X-editable - v1.5.0 (http://vitalets.github.io/x-editable/ ), Copyright © 2013 Vitaliy Potapov; +This product includes backgrid 0.3.5 (http://backgridjs.com/ ), Copyright © 2014 Jimmy Yuen Ho Wong and contributors. +This product includes backgrid-paginator (http://github.com/wyuenho/backgrid ), Copyright © 2013 Jimmy Yuen Ho Wong and contributors. This product includes backgrid-filter (http://github.com/wyuenho/backgrid ), Copyright © 2013 Jimmy Yuen Ho Wong and contributors. +This product includes backbone-pageable 1.3.2 (http://github.com/wyuenho/backbone-pageable ), Copyright © 2013 Jimmy Yuen Ho Wong. +This product includes jQuery Cookie Plugin v1.4.0 (https://github.com/carhartl/jquery-cookie ), Copyright 2013 Klaus Hartl +This product includes jQuery Toggles v2.0.4 (http://simontabor.com/labs/toggles ), Copyright 2013 Simon Tabor. +This product includes jQuery UI Tag-it! (http://aehlke.github.com/tag-it/ ), Copyright 2011, Levy Carneiro Jr. +This product includes bootbox.js v3.3.0 (http://bootboxjs.com/ ), Copyright © 2011-2014 by Nick Payne. +This product includes moment.js (http://moment.js ), Copyright © 2011-2014 Tim Wood, Iskren Chernev, Moment.js contributors. +This product includes Globalize (http://github.com/jquery/globalize ), Copyright Software Freedom Conservancy, Inc. Globalize license applies to all files under globalize including globalize/generator. +This product includes RequireJS 2.1.8 (https://github.com/requirejs/requirejs ), Copyright (c) 2010-2012, The Dojo Foundation All Rights Reserved. +This product includes Handlebars.js (http://handlebarsjs.com/ ), Copyright © 2011-2016 by Yehuda Katz. +This product includes Font Awesome 3.2.1 (http://fontawesome.io/ ) by Dave Gandy +This product includes Qunit(https://qunitjs.com/ ) Copyright (c) 2011 John Resig, Jörn Zaefferer +This product includes jQuery Migrate-v1.1.1 (https://plugins.jquery.com/migrate/ ) Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors This product includes bootstrap v2.3.2 (http://getbootstrap.com/ - MIT License) Copyright 2013 Twitter, Inc. This product includes Block UI v1.1.1 (https://github.com/dreamerslab/jquery.msg/ - MIT License) Copyright 2011, Ben Lin. -This product includes Sizzle (http://sizzlejs.com/ - MIT license), Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors -This product includes ES5 15.5.4.20 (https://github.com/kriskowal/es5-shim - MIT license), Copyright 2009, 2010 Kristopher Michael Kowal. +This product includes Sizzle (http://sizzlejs.com/ ), Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors +This product includes ES5 15.5.4.20 (https://github.com/kriskowal/es5-shim ), Copyright 2009, 2010 Kristopher Michael Kowal. This product includes PURE CSS GUI ICONS 1.0.1(http://nicolasgallagher.com/pure-css-gui-icons/ - Dual licensed under MIT and GNU GPLv2), by Nicolas Gallagher -This product includes libpam4j 1.8 (https://github.com/kohsuke/libpam4j - MIT license), Copyright Kohsuke Kawaguchi. +This product includes libpam4j 1.8 (https://github.com/kohsuke/libpam4j ), Copyright Kohsuke Kawaguchi. +This product bundles VisualSearch.js 0.4.0 (http://documentcloud.github.io/visualsearch/docs/visualsearch.html ), © 2011 Samuel Clay, @samuelclay, DocumentCloud Inc. @@ -292,49 +293,11 @@ This product includes Require.js Handlebars Plugin (https://github.com/SlexAxton 0. You just DO WHAT THE FUCK YOU WANT TO. ------------------------------------------------------------------------ - VisualSearch License ------------------------------------------------------------------------ -This product includes VisualSearch.js 0.4.0 (http://documentcloud.github.com/visualsearch), © 2011 Samuel Clay, @samuelclay, DocumentCloud Inc. - -Copyright (c) 2011 Samuel Clay, @samuelclay, DocumentCloud -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - ------------------------------------------------------------------------ - The Apache License 2.0 ------------------------------------------------------------------------ -This product includes Bootstrap Datepicker (http://www.eyecon.ro/bootstrap-datepicker - Apache License - 2.0), Copyright 2012 Stefan Petre. -This product includes bootstrap-transition.js v2.3.2 (http://twbs.github.com/bootstrap/javascript.html#transitions - Apache License - 2.0), Copyright 2013 Twitter, Inc. -This product includes Select2 3.4.3 (http://ivaynberg.github.io/select2/ - Apache License - 2.0), Copyright 2012 Igor Vaynberg. -This product includes bootstrap-notify v1.0 (http://goodybag.github.io/bootstrap-notify/ - Apache License - 2.0), Copyright 2012 Goodybag, Inc. -This product includes Open Sans font (http://www.opensans.com/ - Apache License - 2.0), © 2016 OpenSans.com -This product includes Jisql (https://bitbucket.org/stdunbar/jisql - Apache License - 2.0), © 2004-2011 Scott Dunbar -This product includes PasswordComparisonAuthenticator.java in Spring Security (https://github.com/spring-projects/spring-security - Apache License - 2.0), Copyright 2004, 2005, 2006 Acegi Technology Pty Limited - ----------------------------------------------------------------------- BSD license ----------------------------------------------------------------------- This product includes Easing Functions in jQuery UI v1.10.3 (http://www.robertpenner.com/easing - BSD License), Copyright © 2001 by Robert Penner. - -Copyright © 2001 Robert Penner -All rights reserved. +This product includes Esprima (https://github.com/jquery/esprima - BSD License), Copyright (c) jQuery Foundation, Inc. and Contributors, All Rights Reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -354,101 +317,6 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ------------------------------------------------------------------------ - SIL Open Font License (OFL) ------------------------------------------------------------------------ -This product includes Font Awesome 3.2.1 (http://fontawesome.io/ - SIL Open Font License (OFL) licensee) by Dave Gandy - - -PREAMBLE -The goals of the Open Font License (OFL) are to stimulate worldwide -development of collaborative font projects, to support the font creation -efforts of academic and linguistic communities, and to provide a free and -open framework in which fonts may be shared and improved in partnership -with others. - -The OFL allows the licensed fonts to be used, studied, modified and -redistributed freely as long as they are not sold by themselves. The -fonts, including any derivative works, can be bundled, embedded, -redistributed and/or sold with any software provided that any reserved -names are not used by derivative works. The fonts and derivatives, -however, cannot be released under any other type of license. The -requirement for fonts to remain under this license does not apply -to any document created using the fonts or their derivatives. - -DEFINITIONS -"Font Software" refers to the set of files released by the Copyright -Holder(s) under this license and clearly marked as such. This may -include source files, build scripts and documentation. - -"Reserved Font Name" refers to any names specified as such after the -copyright statement(s). - -"Original Version" refers to the collection of Font Software components as -distributed by the Copyright Holder(s). - -"Modified Version" refers to any derivative made by adding to, deleting, -or substituting -- in part or in whole -- any of the components of the -Original Version, by changing formats or by porting the Font Software to a -new environment. - -"Author" refers to any designer, engineer, programmer, technical -writer or other person who contributed to the Font Software. - -PERMISSION & CONDITIONS -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Font Software, to use, study, copy, merge, embed, modify, -redistribute, and sell modified and unmodified copies of the Font -Software, subject to the following conditions: - -1) Neither the Font Software nor any of its individual components, -in Original or Modified Versions, may be sold by itself. - -2) Original or Modified Versions of the Font Software may be bundled, -redistributed and/or sold with any software, provided that each copy -contains the above copyright notice and this license. These can be -included either as stand-alone text files, human-readable headers or -in the appropriate machine-readable metadata fields within text or -binary files as long as those fields can be easily viewed by the user. - -3) No Modified Version of the Font Software may use the Reserved Font -Name(s) unless explicit written permission is granted by the corresponding -Copyright Holder. This restriction only applies to the primary font name as -presented to the users. - -4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font -Software shall not be used to promote, endorse or advertise any -Modified Version, except to acknowledge the contribution(s) of the -Copyright Holder(s) and the Author(s) or with their explicit written -permission. - -5) The Font Software, modified or unmodified, in part or in whole, -must be distributed entirely under this license, and must not be -distributed under any other license. The requirement for fonts to -remain under this license does not apply to any document created -using the Font Software. - -TERMINATION -This license becomes null and void if any of the above conditions are -not met. - -DISCLAIMER -THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE -COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL -DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM -OTHER DEALINGS IN THE FONT SOFTWARE. - ------------------------------------------------------------------------ - Public Domain License ------------------------------------------------------------------------ -This product includes json2.js (https://github.com/douglascrockford/JSON-js - Public Domain license) by Douglas Crockford - ------------------------------------------------------------------------------------------------------------------------------------------ diff --git a/NOTICE.txt b/NOTICE.txt index b5841f2f7a..7f6238eae1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,7 +1,7 @@ Apache Ranger -Copyright 2014-2016 The Apache Software Foundation - -This product includes software developed at The Apache Software -Foundation (http://www.apache.org/). - +Copyright 2014-2017 The Apache Software Foundation +This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +his product includes json2.js (https://github.com/douglascrockford/JSON-js - Public Domain license) by Douglas Crockford +This product includes Font Awesome 3.2.1 (http://fontawesome.io/ - SIL Open Font License (OFL) licensee) by Dave Gandy +This product includes software developed by Spring Security Project (http://www.springframework.org/security) diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index 40ec34d3b8..13d809d342 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java index 46f33a53b6..738c0918a8 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java @@ -54,7 +54,7 @@ public class SolrAuditDestination extends AuditDestination { public static final String DEFAULT_COLLECTION_NAME = "ranger_audits"; public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config"; - SolrClient solrClient = null; + private volatile SolrClient solrClient = null; public SolrAuditDestination() { } @@ -74,77 +74,80 @@ public void stop() { } synchronized void connect() { - if (solrClient == null) { - if (solrClient == null) { - String urls = MiscUtil.getStringProperty(props, propPrefix - + "." + PROP_SOLR_URLS); - if (urls != null) { - urls = urls.trim(); - } - if (urls != null && urls.equalsIgnoreCase("NONE")) { - urls = null; - } - - List solrURLs = new ArrayList(); - String zkHosts = null; - solrURLs = MiscUtil.toArray(urls, ","); - zkHosts = MiscUtil.getStringProperty(props, propPrefix + "." - + PROP_SOLR_ZK); - if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) { - zkHosts = null; - } - - String collectionName = MiscUtil.getStringProperty(props, - propPrefix + "." + PROP_SOLR_COLLECTION); - if (collectionName == null - || collectionName.equalsIgnoreCase("none")) { - collectionName = DEFAULT_COLLECTION_NAME; - } - - LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls - + ", collectionName=" + collectionName); - - if (zkHosts != null && !zkHosts.isEmpty()) { - LOG.info("Connecting to solr cloud using zkHosts=" - + zkHosts); - try { - // Instantiate - HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); - CloudSolrClient solrCloudClient = new CloudSolrClient( - zkHosts); - solrCloudClient.setDefaultCollection(collectionName); - solrClient = solrCloudClient; - } catch (Throwable t) { - LOG.fatal("Can't connect to Solr server. ZooKeepers=" - + zkHosts, t); + SolrClient me = solrClient; + if (me == null) { + synchronized(SolrAuditDestination.class) { + me = solrClient; + if (solrClient == null) { + String urls = MiscUtil.getStringProperty(props, propPrefix + + "." + PROP_SOLR_URLS); + if (urls != null) { + urls = urls.trim(); } - finally { - resetInitializerInSOLR() ; + if (urls != null && urls.equalsIgnoreCase("NONE")) { + urls = null; } - } else if (solrURLs != null && !solrURLs.isEmpty()) { - try { - LOG.info("Connecting to Solr using URLs=" + solrURLs); - HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); - LBHttpSolrClient lbSolrClient = new LBHttpSolrClient( - solrURLs.get(0)); - lbSolrClient.setConnectionTimeout(1000); + List solrURLs = new ArrayList(); + String zkHosts = null; + solrURLs = MiscUtil.toArray(urls, ","); + zkHosts = MiscUtil.getStringProperty(props, propPrefix + "." + + PROP_SOLR_ZK); + if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) { + zkHosts = null; + } + String collectionName = MiscUtil.getStringProperty(props, + propPrefix + "." + PROP_SOLR_COLLECTION); + if (collectionName == null + || collectionName.equalsIgnoreCase("none")) { + collectionName = DEFAULT_COLLECTION_NAME; + } + + LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls + + ", collectionName=" + collectionName); - for (int i = 1; i < solrURLs.size(); i++) { - lbSolrClient.addSolrServer(solrURLs.get(i)); + if (zkHosts != null && !zkHosts.isEmpty()) { + LOG.info("Connecting to solr cloud using zkHosts=" + + zkHosts); + try { + // Instantiate + HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); + CloudSolrClient solrCloudClient = new CloudSolrClient( + zkHosts); + solrCloudClient.setDefaultCollection(collectionName); + me = solrClient = solrCloudClient; + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. ZooKeepers=" + + zkHosts, t); + } + finally { + resetInitializerInSOLR() ; + } + } else if (solrURLs != null && !solrURLs.isEmpty()) { + try { + LOG.info("Connecting to Solr using URLs=" + solrURLs); + HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); + LBHttpSolrClient lbSolrClient = new LBHttpSolrClient( + solrURLs.get(0)); + lbSolrClient.setConnectionTimeout(1000); + + for (int i = 1; i < solrURLs.size(); i++) { + lbSolrClient.addSolrServer(solrURLs.get(i)); + } + me = solrClient = lbSolrClient; + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. URL=" + + solrURLs, t); + } + finally { + resetInitializerInSOLR() ; } - solrClient = lbSolrClient; - } catch (Throwable t) { - LOG.fatal("Can't connect to Solr server. URL=" - + solrURLs, t); - } - finally { - resetInitializerInSOLR() ; } } } } } + private void resetInitializerInSOLR() { javax.security.auth.login.Configuration solrConfig = javax.security.auth.login.Configuration.getConfiguration(); String solrConfigClassName = solrConfig.getClass().getName() ; diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index e3e818c003..e3c3508241 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -67,7 +67,7 @@ public class AuditProviderFactory { private static final int RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY = 30; - private static AuditProviderFactory sFactory; + private volatile static AuditProviderFactory sFactory = null; private AuditHandler mProvider = null; private String componentAppType = ""; @@ -80,15 +80,17 @@ private AuditProviderFactory() { } public static AuditProviderFactory getInstance() { - if (sFactory == null) { - synchronized (AuditProviderFactory.class) { - if (sFactory == null) { - sFactory = new AuditProviderFactory(); + AuditProviderFactory ret = sFactory; + if(ret == null) { + synchronized(AuditProviderFactory.class) { + ret = sFactory; + if(ret == null) { + ret = sFactory = new AuditProviderFactory(); } } } - return sFactory; + return ret; } public static AuditHandler getAuditProvider() { diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java index 535894c854..f5b2795aad 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java @@ -184,7 +184,9 @@ public static String getApplicationType() { } public static String getJvmInstanceId() { - String ret = Integer.toString(Math.abs(sJvmID.toString().hashCode())); + Integer val = Integer.valueOf(sJvmID.toString().hashCode()); + long longVal = val.longValue(); + String ret = Long.toString(Math.abs(longVal)); return ret; } @@ -565,6 +567,10 @@ static public boolean logErrorMessageByInterval(Log useLogger, */ static public boolean logErrorMessageByInterval(Log useLogger, String message, Throwable e) { + if (message == null) { + return false; + } + LogHistory log = logHistoryList.get(message); if (log == null) { log = new LogHistory(); @@ -636,7 +642,7 @@ public static void authWithConfig(String appName, Configuration config) { } } catch (Throwable t) { logger.fatal("Error logging as appName=" + appName + ", config=" - + config.toString()); + + config.toString() + ", error=" + t.getMessage()); } } @@ -715,7 +721,7 @@ public static void authWithKerberos(String keytab, String principal, } } catch (Throwable t) { - logger.error("Failed to login as [" + spnegoPrincipals + "]", t); + logger.error("Failed to login with given keytab and principal", t); } } diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/hdfs/HdfsLogDestination.java index 0951c6431b..f15b57a76b 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/hdfs/HdfsLogDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/hdfs/HdfsLogDestination.java @@ -471,7 +471,7 @@ private void logException(String msg, IOException excp) { return; } - String excpMsgToExclude = EXCP_MSG_FILESYSTEM_CLOSED;; + String excpMsgToExclude = EXCP_MSG_FILESYSTEM_CLOSED; String excpMsg = excp != null ? excp.getMessage() : null; boolean excpExcludeLogging = (excpMsg != null && excpMsg.contains(excpMsgToExclude)); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java index 9388914b49..8b42be0fc6 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java @@ -42,7 +42,7 @@ public class SolrAuditProvider extends AuditDestination { public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms"; static final Object lock = new Object(); - SolrClient solrClient = null; + volatile SolrClient solrClient = null; Date lastConnectTime = null; long lastFailTime = 0; @@ -61,9 +61,11 @@ public void init(Properties props) { } void connect() { - if (solrClient == null) { + SolrClient me = solrClient; + if (me == null) { synchronized (lock) { - if (solrClient == null) { + me = solrClient; + if (me == null) { String solrURL = MiscUtil.getStringProperty(props, "xasecure.audit.solr.solr_url"); @@ -89,7 +91,7 @@ void connect() { try { // TODO: Need to support SolrCloud also - solrClient = new HttpSolrClient(solrURL); + me = solrClient = new HttpSolrClient(solrURL); if (solrClient instanceof HttpSolrClient) { HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient; httpSolrClient.setAllowCompression(true); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java index 17ddab9e7b..8e3c99209d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java @@ -663,7 +663,7 @@ public boolean accept(File pathname) { } }); - if (logFiles.length > maxArchiveFiles) { + if (logFiles != null && logFiles.length > maxArchiveFiles) { int filesToDelete = logFiles.length - maxArchiveFiles; BufferedReader br = new BufferedReader(new FileReader( indexDoneFile)); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java index be9cdd39d3..2e8b7689df 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java @@ -148,14 +148,15 @@ public static void init(String propFile) throws Exception { properties.load(in); init(properties); } catch (IOException e) { - if (in != null) { - try { - in.close(); - } catch (Exception exception) { - // Ignore - } - } throw new Exception("Failed to load JAAS application properties", e); + } finally { + if ( in != null) { + try { + in.close(); + } catch ( Exception e) { + //Ignore + } + } } LOG.debug("<== InMemoryJAASConfiguration.init( {} ) ", propFile); } diff --git a/agents-common/pom.xml b/agents-common/pom.xml index 9b378bf20f..32faf616da 100644 --- a/agents-common/pom.xml +++ b/agents-common/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index 3975b18eb8..5f8acfe7be 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -117,9 +117,7 @@ public ClientResponse run() { if(response != null && response.getStatus() == 200) { ret = response.getEntity(ServicePolicies.class); - } else if(response != null && response.getStatus() == 304) { - // no change - } else { + } else if(!(response != null && response.getStatus() == 304)) { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.error("Error getting policies. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp.toString() + ", serviceName=" + serviceName); @@ -278,9 +276,7 @@ public ClientResponse run() { if(response != null && response.getStatus() == 200) { ret = response.getEntity(ServiceTags.class); - } else if(response != null && response.getStatus() == 304) { - // no change - } else { + } else if(!(response != null && response.getStatus() == 304)) { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.error("Error getting taggedResources. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp.toString() + ", serviceName=" + serviceName diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index 66cee76eb7..60d3b0ca8f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -67,7 +67,9 @@ public void processResults(Collection results) { Collection events = getAuthzEvents(results); - logAuthzAudits(events); + if (events != null) { + logAuthzAudits(events); + } if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultAuditHandler.processResults(" + results + ")"); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java index fc9842edc3..b18e7cd660 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesAccessedTogetherCondition.java @@ -72,7 +72,7 @@ public boolean isMatched(final RangerAccessRequest request) { if (isInitialized && CollectionUtils.isNotEmpty(matchers)) { RangerRequestedResources resources = RangerAccessRequestUtil.getRequestedResourcesFromContext(request.getContext()); - ret = resources == null ? false : !resources.isMutuallyExcluded(matchers); + ret = resources == null ? false : !resources.isMutuallyExcluded(matchers, request.getContext()); } else { LOG.error("RangerHiveResourcesAccessedTogetherCondition.isMatched() - condition is not initialized correctly and will NOT be enforced"); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java index 3b8e0094ef..5036a06d82 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerHiveResourcesNotAccessedTogetherCondition.java @@ -72,7 +72,7 @@ public boolean isMatched(final RangerAccessRequest request) { if (isInitialized && CollectionUtils.isNotEmpty(matchers)) { RangerRequestedResources resources = RangerAccessRequestUtil.getRequestedResourcesFromContext(request.getContext()); - ret = resources == null ? true : resources.isMutuallyExcluded(matchers); + ret = resources == null ? true : resources.isMutuallyExcluded(matchers, request.getContext()); } else { LOG.error("RangerHiveResourcesNotAccessedTogetherCondition.isMatched() - Enforcer is not initialized correctly, Mutual Exclusion will NOT be enforced"); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerTimeOfDayMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerTimeOfDayMatcher.java index f65f29ebcb..a1ea326d6b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerTimeOfDayMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerTimeOfDayMatcher.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.conditionevaluator; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; @@ -126,7 +127,7 @@ int[] extractDuration(String value) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerTimeOfDayMatcher.extractDuration(" + value + "): duration[" + result + "]"); + LOG.debug("<== RangerTimeOfDayMatcher.extractDuration(" + value + "): duration:" + Arrays.toString(result)); } return result; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java index 5c1ae64190..637423e309 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java @@ -19,24 +19,63 @@ package org.apache.ranger.plugin.contextenricher; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.ranger.plugin.util.ServiceDefUtil; -public class RangerServiceResourceMatcher { - private final RangerServiceResource serviceResource; +import java.util.Map; + +public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluator { + private final RangerServiceResource serviceResource; private final RangerPolicyResourceMatcher policyResourceMatcher; + private final Integer leafResourceLevel; public RangerServiceResourceMatcher(final RangerServiceResource serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) { - this.serviceResource = serviceResource; + this.serviceResource = serviceResource; this.policyResourceMatcher = policyResourceMatcher; + this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(getServiceDef(), getPolicyResource()); } public RangerServiceResource getServiceResource() { return serviceResource; } + @Override + public long getId() { + return serviceResource != null ? serviceResource.getId() :-1; + } + + @Override public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return policyResourceMatcher; } - public boolean isMatch(RangerAccessResource requestedResource) { - return this.policyResourceMatcher.isExactHeadMatch(requestedResource); + @Override + public Map getPolicyResource() { + return serviceResource != null ? serviceResource.getResourceElements() : null; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return policyResourceMatcher != null ? policyResourceMatcher.getResourceMatcher(resourceName) : null; + } + + @Override + public Integer getLeafResourceLevel() { + return leafResourceLevel; + } + + @Override + public int compareTo(RangerPolicyResourceEvaluator other) { + return Long.compare(getId(), other.getId()); + } + + public boolean isMatch(RangerAccessResource requestedResource, Map evalContext) { + return policyResourceMatcher != null ? policyResourceMatcher.isExactHeadMatch(requestedResource, evalContext) : false; + } + + RangerServiceDef getServiceDef() { + return policyResourceMatcher != null ? policyResourceMatcher.getServiceDef() : null; } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index c397ca123c..1a6e1b2ad4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; @@ -34,12 +35,16 @@ import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.RangerResourceTrie; import org.apache.ranger.plugin.util.ServiceTags; import java.io.*; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class); @@ -47,16 +52,15 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; + public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; + public static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter"; - public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; - - private RangerTagRefresher tagRefresher = null; - - private RangerTagRetriever tagRetriever = null; - - ServiceTags serviceTags = null; - - List serviceResourceMatchers; + private RangerTagRefresher tagRefresher = null; + private RangerTagRetriever tagRetriever = null; + private ServiceTags serviceTags = null; + private List serviceResourceMatchers = null; + private Map serviceResourceTrie = null; + private boolean disableTrieLookupPrefilter = false; @Override public void init() { @@ -70,6 +74,8 @@ public void init() { long pollingIntervalMs = getLongOption(TAG_REFRESHER_POLLINGINTERVAL_OPTION, 60 * 1000); + disableTrieLookupPrefilter = getBooleanOption(TAG_DISABLE_TRIE_PREFILTER_OPTION, false); + if (StringUtils.isNotBlank(tagRetrieverClassName)) { try { @@ -126,7 +132,7 @@ public void enrich(RangerAccessRequest request) { LOG.debug("==> RangerTagEnricher.enrich(" + request + ")"); } - List matchedTags = findMatchingTags(request.getResource()); + List matchedTags = findMatchingTags(request.getResource(), request.getContext()); RangerAccessRequestUtil.setRequestTagsInContext(request.getContext(), matchedTags); @@ -162,8 +168,19 @@ public void setServiceTags(final ServiceTags serviceTags) { } + Map serviceResourceTrie = null; + + if(!disableTrieLookupPrefilter) { + serviceResourceTrie = new HashMap(); + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers)); + } + } + this.serviceResourceMatchers = resourceMatchers; - this.serviceTags = serviceTags; + this.serviceResourceTrie = serviceResourceTrie; + this.serviceTags = serviceTags; } @Override @@ -185,13 +202,13 @@ public boolean preCleanup() { return ret; } - private List findMatchingTags(final RangerAccessResource resource) { + private List findMatchingTags(final RangerAccessResource resource, final Map evalContext) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerTagEnricher.findMatchingTags(" + resource + ")"); + LOG.debug("==> RangerTagEnricher.findMatchingTags(" + resource + ", " + evalContext + ")"); } List ret = null; - final List serviceResourceMatchers = this.serviceResourceMatchers; + final List serviceResourceMatchers = getEvaluators(resource); if (CollectionUtils.isNotEmpty(serviceResourceMatchers)) { @@ -199,7 +216,7 @@ private List findMatchingTags(final RangerAccessResource resource) { for (RangerServiceResourceMatcher resourceMatcher : serviceResourceMatchers) { - boolean matchResult = resourceMatcher.isMatch(resource); + boolean matchResult = resourceMatcher.isMatch(resource, evalContext); if (matchResult) { if (ret == null) { @@ -220,7 +237,71 @@ private List findMatchingTags(final RangerAccessResource resource) { } if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerTagEnricher.findMatchingTags(" + resource + ")"); + LOG.debug("<== RangerTagEnricher.findMatchingTags(" + resource + ", " + evalContext + ")"); + } + + return ret; + } + + private List getEvaluators(RangerAccessResource resource) { + List ret = null; + + if(serviceResourceTrie == null) { + ret = serviceResourceMatchers; + } else { + Set resourceKeys = resource == null ? null : resource.getKeys(); + + if (CollectionUtils.isNotEmpty(resourceKeys)) { + boolean isRetModifiable = false; + + for (String resourceName : resourceKeys) { + RangerResourceTrie trie = serviceResourceTrie.get(resourceName); + + if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level + continue; + } + + List resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); + + if (CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out + ret = null; + } else if (ret == null) { // initialize ret with policies found for this resource + ret = resourceEvaluators; + } else { // remove policies from ret that are not in resourceEvaluators + if (isRetModifiable) { + ret.retainAll(resourceEvaluators); + } else { + final List shorterList; + final List longerList; + + if (ret.size() < resourceEvaluators.size()) { + shorterList = ret; + longerList = resourceEvaluators; + } else { + shorterList = resourceEvaluators; + longerList = ret; + } + + ret = new ArrayList<>(shorterList); + ret.retainAll(longerList); + isRetModifiable = true; + } + } + + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + } + + if(ret == null) { + ret = Collections.emptyList(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.getEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size()); } return ret; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java index d8e19b7793..5e94bc722e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java @@ -50,6 +50,10 @@ public class RangerPolicy extends RangerBaseModelObject implements java.io.Seria POLICY_TYPE_ROWFILTER }; + public static final String MASK_TYPE_NULL = "MASK_NULL"; + public static final String MASK_TYPE_NONE = "MASK_NONE"; + public static final String MASK_TYPE_CUSTOM = "CUSTOM"; + // For future use private static final long serialVersionUID = 1L; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java index f6931b375a..2d27961a29 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java @@ -1260,7 +1260,6 @@ public static class RangerResourceDef implements java.io.Serializable { private String rbKeyDescription = null; private String rbKeyValidationMessage = null; - public RangerResourceDef() { this(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); } @@ -1931,6 +1930,7 @@ public int hashCode() { result = prime * result + ((name == null) ? 0 : name.hashCode()); result = prime * result + ((rbKeyLabel == null) ? 0 : rbKeyLabel.hashCode()); + return result; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java index a89ec19b3d..62bd100188 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java @@ -776,7 +776,7 @@ boolean isValidItemAccesses(List accesses, List resources, String user, Set userGroups, String accessType); - List getExactMatchPolicies(RangerAccessResource resource); + List getExactMatchPolicies(RangerAccessResource resource, Map evalContext); - List getExactMatchPolicies(Map resources); + List getExactMatchPolicies(Map resources, Map evalContext); List getAllowedPolicies(String user, Set userGroups, String accessType); RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request); + void reorderPolicyEvaluators(); + boolean preCleanup(); void setUseForwardedIPAddress(boolean useForwardedIPAddress); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index f20923cf9e..346453e98f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.policyengine; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,7 +35,13 @@ import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServicePolicies; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class); @@ -43,6 +50,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log PERF_POLICYENGINE_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policyengine.request"); private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit"); private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request"); + private static final Log PERF_POLICYENGINE_REBALANCE_LOG = RangerPerfTracer.getPerfLogger("policyengine.rebalance"); + private static final Log PERF_POLICYENGINE_USAGE_LOG = RangerPerfTracer.getPerfLogger("policyengine.usage"); private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 100; @@ -51,6 +60,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private List allContextEnrichers; + private final Map policyEvaluatorsMap; + private boolean useForwardedIPAddress = false; private String[] trustedProxyAddresses = null; @@ -63,6 +74,9 @@ public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, Ran if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerPolicyEngine.init(appId=" + appId + ",hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")"); + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); } if (options == null) { @@ -129,8 +143,16 @@ public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, Ran this.allContextEnrichers = tmpList; + policyEvaluatorsMap = createPolicyEvaluatorsMap(); + RangerPerfTracer.log(perf); + if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory - freeMemory) + ", Free memory:" + freeMemory); + } + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl()"); } @@ -151,9 +173,29 @@ public long getPolicyVersion() { return policyRepository.getPolicyVersion(); } + public RangerPolicyEvaluator getPolicyEvaluator(Long id) { + return policyEvaluatorsMap.get(id); + } + + public RangerPolicy getPolicy(Long id) { + RangerPolicyEvaluator evaluator = getPolicyEvaluator(id); + return evaluator != null ? evaluator.getPolicy() : null; + } + @Override public RangerAccessResult createAccessResult(RangerAccessRequest request) { - return new RangerAccessResult(this.getServiceName(), policyRepository.getServiceDef(), request); + RangerAccessResult ret = new RangerAccessResult(this.getServiceName(), policyRepository.getServiceDef(), request); + switch (policyRepository.getAuditModeEnum()) { + case AUDIT_ALL: + ret.setIsAudited(true); + break; + case AUDIT_NONE: + ret.setIsAudited(false); + break; + default: + break; + } + return ret; } @Override @@ -167,6 +209,8 @@ public void preProcess(RangerAccessRequest request) { ((RangerAccessRequestImpl) request).extractAndSetClientIPAddress(useForwardedIPAddress, trustedProxyAddresses); } + RangerAccessRequestUtil.setCurrentUserInContext(request.getContext(), request.getUser()); + List enrichers = allContextEnrichers; if(!CollectionUtils.isEmpty(enrichers)) { @@ -222,6 +266,8 @@ public RangerAccessResult isAccessAllowed(RangerAccessRequest request, RangerAcc RangerAccessResult ret = isAccessAllowedNoAudit(request); + updatePolicyUsageCounts(request, ret); + if (resultProcessor != null) { RangerPerfTracer perfAuditTracer = null; @@ -280,12 +326,17 @@ public RangerDataMaskResult evalDataMaskPolicies(RangerAccessRequest request, Ra RangerDataMaskResult ret = new RangerDataMaskResult(getServiceName(), getServiceDef(), request); if(request != null) { - List evaluators = policyRepository.getDataMaskPolicyEvaluators(); + List evaluators = policyRepository.getDataMaskPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { evaluator.evaluate(request, ret); if (ret.getIsAccessDetermined() && ret.getIsAuditedDetermined()) { - break; + if(!StringUtils.equalsIgnoreCase(ret.getMaskType(), RangerPolicy.MASK_TYPE_NONE)) { + break; + } else { + ret.setMaskType(null); + ret.setIsAccessDetermined(false); + } } } } @@ -295,6 +346,8 @@ public RangerDataMaskResult evalDataMaskPolicies(RangerAccessRequest request, Ra ret.setIsAudited(false); } + updatePolicyUsageCounts(request, ret); + if (resultProcessor != null) { resultProcessor.processResult(ret); } @@ -315,12 +368,16 @@ public RangerRowFilterResult evalRowFilterPolicies(RangerAccessRequest request, RangerRowFilterResult ret = new RangerRowFilterResult(getServiceName(), getServiceDef(), request); if(request != null) { - List evaluators = policyRepository.getRowFilterPolicyEvaluators(); + List evaluators = policyRepository.getRowFilterPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { evaluator.evaluate(request, ret); if (ret.getIsAccessDetermined() && ret.getIsAuditedDetermined()) { - break; + if(StringUtils.isNotEmpty(ret.getFilterExpr())) { + break; + } else { + ret.setIsAccessDetermined(false); + } } } } @@ -330,6 +387,8 @@ public RangerRowFilterResult evalRowFilterPolicies(RangerAccessRequest request, ret.setIsAudited(false); } + updatePolicyUsageCounts(request, ret); + if (resultProcessor != null) { resultProcessor.processResult(ret); } @@ -354,7 +413,7 @@ public boolean isAccessAllowed(RangerAccessResource resource, String user, Set resources, Stri } @Override - public List getExactMatchPolicies(RangerAccessResource resource) { + public List getExactMatchPolicies(RangerAccessResource resource, Map evalContext) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyEngineImpl.getExactMatchPolicies(" + resource + ")"); + LOG.debug("==> RangerPolicyEngineImpl.getExactMatchPolicies(" + resource + ", " + evalContext + ")"); } List ret = null; for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { - if (evaluator.isCompleteMatch(resource)) { + if (evaluator.isCompleteMatch(resource, evalContext)) { if(ret == null) { ret = new ArrayList(); } @@ -422,22 +481,22 @@ public List getExactMatchPolicies(RangerAccessResource resource) { } if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyEngineImpl.getExactMatchPolicies(" + resource + "): " + ret); + LOG.debug("<== RangerPolicyEngineImpl.getExactMatchPolicies(" + resource + ", " + evalContext + "): " + ret); } return ret; } @Override - public List getExactMatchPolicies(Map resources) { + public List getExactMatchPolicies(Map resources, Map evalContext) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyEngineImpl.getExactMatchPolicies(" + resources + ")"); + LOG.debug("==> RangerPolicyEngineImpl.getExactMatchPolicies(" + resources + ", " + evalContext + ")"); } List ret = null; for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { - if (evaluator.isCompleteMatch(resources)) { + if (evaluator.isCompleteMatch(resources, evalContext)) { if(ret == null) { ret = new ArrayList(); } @@ -447,7 +506,7 @@ public List getExactMatchPolicies(Map tagPolicyEvaluators = tagPolicyRepository == null ? null : tagPolicyRepository.getPolicyEvaluators(); - List resPolicyEvaluators = policyRepository.getPolicyEvaluators(); if (CollectionUtils.isNotEmpty(tagPolicyEvaluators)) { List tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); @@ -497,13 +555,17 @@ public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest reques for (RangerTag tag : tags) { RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(tag, tagPolicyRepository.getServiceDef(), request); - for (RangerPolicyEvaluator evaluator : tagPolicyEvaluators) { + List evaluators = tagPolicyRepository.getPolicyEvaluators(tagEvalRequest.getResource()); + + for (RangerPolicyEvaluator evaluator : evaluators) { evaluator.getResourceAccessInfo(tagEvalRequest, ret); } } } } + List resPolicyEvaluators = policyRepository.getPolicyEvaluators(request.getResource()); + if(CollectionUtils.isNotEmpty(resPolicyEvaluators)) { for (RangerPolicyEvaluator evaluator : resPolicyEvaluators) { evaluator.getResourceAccessInfo(request, ret); @@ -550,8 +612,9 @@ protected RangerAccessResult isAccessAllowedNoAudit(RangerAccessRequest request) ret.setIsAccessDetermined(false); // discard allowed result by tag-policies, to evaluate resource policies for possible deny } - List evaluators = policyRepository.getPolicyEvaluators(); + List evaluators = policyRepository.getPolicyEvaluators(request.getResource()); for (RangerPolicyEvaluator evaluator : evaluators) { + ret.incrementEvaluatedPoliciesCount(); evaluator.evaluate(request, ret); if(ret.getIsAllowed() && !evaluator.hasDeny()) { // all policies having deny have been evaluated @@ -585,9 +648,9 @@ protected void isAccessAllowedForTagPolicies(final RangerAccessRequest request, LOG.debug("==> RangerPolicyEngineImpl.isAccessAllowedForTagPolicies(" + request + ", " + result + ")"); } - List evaluators = tagPolicyRepository == null ? null : tagPolicyRepository.getPolicyEvaluators(); + List tagEvaluators = tagPolicyRepository == null ? null : tagPolicyRepository.getPolicyEvaluators(); - if (CollectionUtils.isNotEmpty(evaluators)) { + if (CollectionUtils.isNotEmpty(tagEvaluators)) { List tags = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); if(CollectionUtils.isNotEmpty(tags)) { @@ -607,9 +670,10 @@ protected void isAccessAllowedForTagPolicies(final RangerAccessRequest request, } tagEvalResult.setAuditResultFrom(result); + List evaluators = tagPolicyRepository.getPolicyEvaluators(tagEvalRequest.getResource()); + for (RangerPolicyEvaluator evaluator : evaluators) { - if(! evaluator.isMatch(tagEvalRequest.getResource())) - continue; + tagEvalResult.incrementEvaluatedPoliciesCount(); evaluator.evaluate(tagEvalRequest, tagEvalResult); @@ -632,6 +696,7 @@ protected void isAccessAllowedForTagPolicies(final RangerAccessRequest request, if (tagEvalResult.getIsAudited()) { result.setIsAudited(true); + result.setAuditPolicyId(tagEvalResult.getAuditPolicyId()); } if(!result.getIsAccessDetermined() && tagEvalResult.getIsAccessDetermined()) { @@ -658,25 +723,40 @@ protected void isAccessAllowedForTagPolicies(final RangerAccessRequest request, } } - private void setResourceServiceDef(RangerAccessRequest request) { - RangerAccessResource resource = request.getResource(); + @Override + public void reorderPolicyEvaluators() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> reorderEvaluators()"); + } + RangerPerfTracer perf = null; - if (resource.getServiceDef() == null) { - if (resource instanceof RangerMutableResource) { - RangerMutableResource mutable = (RangerMutableResource) resource; - mutable.setServiceDef(getServiceDef()); - } else { - LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap."); + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderEvaluators()"); + } + if (MapUtils.isNotEmpty(policyEvaluatorsMap)) { + for (Map.Entry entry : policyEvaluatorsMap.entrySet()) { + entry.getValue().setUsageCountImmutable(); } } - } - private boolean hasTagPolicies() { - return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies()); - } + if (tagPolicyRepository != null) { + tagPolicyRepository.reorderPolicyEvaluators(); + } + if (policyRepository != null) { + policyRepository.reorderPolicyEvaluators(); + } - private boolean hasResourcePolicies() { - return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies()); + if (MapUtils.isNotEmpty(policyEvaluatorsMap)) { + for (Map.Entry entry : policyEvaluatorsMap.entrySet()) { + entry.getValue().resetUsageCount(); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== reorderEvaluators()"); + } } @Override @@ -774,6 +854,100 @@ public StringBuilder toString(StringBuilder sb) { return sb; } + + + private void setResourceServiceDef(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + + if (resource.getServiceDef() == null) { + if (resource instanceof RangerMutableResource) { + RangerMutableResource mutable = (RangerMutableResource) resource; + mutable.setServiceDef(getServiceDef()); + } else { + LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap."); + } + } + } + + private boolean hasTagPolicies() { + return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies()); + } + + private boolean hasResourcePolicies() { + return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies()); + } + + private Map createPolicyEvaluatorsMap() { + Map tmpPolicyEvaluatorMap = new HashMap(); + + if (tagPolicyRepository != null) { + for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getDataMaskPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getRowFilterPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + } + for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : policyRepository.getDataMaskPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : policyRepository.getRowFilterPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + + return Collections.unmodifiableMap(tmpPolicyEvaluatorMap); + } + + private void updatePolicyUsageCounts(RangerAccessRequest accessRequest, RangerAccessResult accessResult) { + + boolean auditCountUpdated = false; + + if (accessResult.getIsAccessDetermined()) { + RangerPolicyEvaluator accessPolicy = getPolicyEvaluator(accessResult.getPolicyId()); + + if (accessPolicy != null) { + + if (accessPolicy.getPolicy().getIsAuditEnabled()) { + updateUsageCount(accessPolicy, 2); + accessResult.setAuditPolicyId(accessResult.getPolicyId()); + + auditCountUpdated = true; + } else { + updateUsageCount(accessPolicy, 1); + } + + } + } + + if (!auditCountUpdated && accessResult.getIsAuditedDetermined()) { + long auditPolicyId = accessResult.getAuditPolicyId(); + RangerPolicyEvaluator auditPolicy = auditPolicyId == -1 ? null : getPolicyEvaluator(auditPolicyId); + + updateUsageCount(auditPolicy, 1); + } + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_USAGE_LOG)) { + RangerAccessRequestImpl rangerAccessRequest = (RangerAccessRequestImpl) accessRequest; + RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_USAGE_LOG, + "RangerPolicyEngine.usage(accessingUser=" + rangerAccessRequest.getUser() + + ",accessedResource=" + rangerAccessRequest.getResource().getAsString() + + ",accessType=" + rangerAccessRequest.getAccessType() + + ",evaluatedPoliciesCount=" + accessResult.getEvaluatedPoliciesCount() + ")"); + RangerPerfTracer.logAlways(perf); + } + } + + private void updateUsageCount(RangerPolicyEvaluator evaluator, int number) { + if (evaluator != null) { + evaluator.incrementUsageCount(number); + } + } } class RangerTagResource extends RangerAccessResourceImpl { private static final String KEY_TAG = "tag"; @@ -799,6 +973,7 @@ public RangerTagAccessRequest(RangerTag resourceTag, RangerServiceDef tagService RangerAccessRequestUtil.setCurrentTagInContext(request.getContext(), resourceTag); RangerAccessRequestUtil.setCurrentResourceInContext(request.getContext(), request.getResource()); + RangerAccessRequestUtil.setCurrentUserInContext(request.getContext(), request.getUser()); super.setContext(requestContext); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index 805f5a5060..2b2cf9b375 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -29,4 +29,5 @@ public class RangerPolicyEngineOptions { public boolean disableCustomConditions = false; public boolean disableTagPolicyEvaluation = true; public boolean evaluateDelegateAdminOnly = false; + public boolean disableTrieLookupPrefilter = false; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index be98f3b96c..ad9b23dcfe 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -32,16 +32,47 @@ import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.store.AbstractServiceStore; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.RangerResourceTrie; import org.apache.ranger.plugin.util.ServiceDefUtil; import org.apache.ranger.plugin.util.ServicePolicies; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; -public class RangerPolicyRepository { +class RangerPolicyRepository { private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); + enum AuditModeEnum { + AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT + } + + static private final class AuditInfo { + final boolean isAudited; + final long auditPolicyId; + + AuditInfo() { + this(false, -1); + } + AuditInfo(boolean isAudited, long auditPolicyId) { + this.isAudited = isAudited; + this.auditPolicyId = auditPolicyId; + } + long getAuditPolicyId() { + return this.auditPolicyId; + } + boolean getIsAudited() { + return isAudited; + } + } + private final String serviceName; private final String appId; private final RangerServiceDef serviceDef; @@ -51,10 +82,14 @@ public class RangerPolicyRepository { private List policyEvaluators; private List dataMaskPolicyEvaluators; private List rowFilterPolicyEvaluators; - private final Map accessAuditCache; + private final AuditModeEnum auditModeEnum; + private final Map accessAuditCache; private final String componentServiceName; private final RangerServiceDef componentServiceDef; + private final Map policyResourceTrie; + private final Map dataMaskResourceTrie; + private final Map rowFilterResourceTrie; RangerPolicyRepository(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) { super(); @@ -71,19 +106,47 @@ public class RangerPolicyRepository { LOG.debug("RangerPolicyRepository : building resource-policy-repository for service " + serviceName); } - String propertyName = "ranger.plugin." + serviceName + ".policyengine.auditcachesize"; + String auditMode = servicePolicies.getAuditMode(); - if (options.cacheAuditResults) { - final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64*1024; - - int auditResultCacheSize = RangerConfiguration.getInstance().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE); - accessAuditCache = Collections.synchronizedMap(new CacheMap(auditResultCacheSize)); + if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_ALL)) { + auditModeEnum = AuditModeEnum.AUDIT_ALL; + } else if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_NONE)) { + auditModeEnum = AuditModeEnum.AUDIT_NONE; } else { + auditModeEnum = AuditModeEnum.AUDIT_DEFAULT; + } + + if (auditModeEnum == AuditModeEnum.AUDIT_DEFAULT) { + String propertyName = "ranger.plugin." + serviceName + ".policyengine.auditcachesize"; + + if (options.cacheAuditResults) { + final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024; + + int auditResultCacheSize = RangerConfiguration.getInstance().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE); + accessAuditCache = Collections.synchronizedMap(new CacheMap(auditResultCacheSize)); + } else { accessAuditCache = null; } + } else { + this.accessAuditCache = null; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyRepository : building policy-repository for service[" + serviceName + + "] with auditMode[" + auditModeEnum + "]"); + } init(options); + if(options.disableTrieLookupPrefilter) { + policyResourceTrie = null; + dataMaskResourceTrie = null; + rowFilterResourceTrie = null; + } else { + policyResourceTrie = createResourceTrieMap(policyEvaluators); + dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators); + rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators); + } } RangerPolicyRepository(String appId, ServicePolicies.TagPolicies tagPolicies, RangerPolicyEngineOptions options, @@ -100,14 +163,35 @@ public class RangerPolicyRepository { this.policies = Collections.unmodifiableList(normalizeAndPrunePolicies(tagPolicies.getPolicies(), componentServiceDef.getName())); this.policyVersion = tagPolicies.getPolicyVersion() != null ? tagPolicies.getPolicyVersion() : -1; + + String auditMode = tagPolicies.getAuditMode(); + + if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_ALL)) { + auditModeEnum = AuditModeEnum.AUDIT_ALL; + } else if (StringUtils.equals(auditMode, RangerPolicyEngine.AUDIT_NONE)) { + auditModeEnum = AuditModeEnum.AUDIT_NONE; + } else { + auditModeEnum = AuditModeEnum.AUDIT_DEFAULT; + } + this.accessAuditCache = null; if(LOG.isDebugEnabled()) { - LOG.debug("RangerPolicyRepository : building tag-policy-repository for tag service " + serviceName); + LOG.debug("RangerPolicyRepository : building tag-policy-repository for tag service[" + serviceName + + "] with auditMode[" + auditModeEnum +"]"); } init(options); + if(options.disableTrieLookupPrefilter) { + policyResourceTrie = null; + dataMaskResourceTrie = null; + rowFilterResourceTrie = null; + } else { + policyResourceTrie = createResourceTrieMap(policyEvaluators); + dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators); + rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators); + } } public String getServiceName() { return serviceName; } @@ -126,18 +210,96 @@ public long getPolicyVersion() { public List getContextEnrichers() { return contextEnrichers; } - public List getPolicyEvaluators() { + List getPolicyEvaluators() { return policyEvaluators; } - public List getDataMaskPolicyEvaluators() { + List getPolicyEvaluators(RangerAccessResource resource) { + String resourceStr = resource == null ? null : resource.getAsString(); + + return policyResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getPolicyEvaluators() : getPolicyEvaluators(policyResourceTrie, resource); + } + + List getDataMaskPolicyEvaluators() { return dataMaskPolicyEvaluators; } - public List getRowFilterPolicyEvaluators() { + List getDataMaskPolicyEvaluators(RangerAccessResource resource) { + String resourceStr = resource == null ? null : resource.getAsString(); + + return dataMaskResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getDataMaskPolicyEvaluators() : getPolicyEvaluators(dataMaskResourceTrie, resource); + } + + List getRowFilterPolicyEvaluators() { return rowFilterPolicyEvaluators; } + List getRowFilterPolicyEvaluators(RangerAccessResource resource) { + String resourceStr = resource == null ? null : resource.getAsString(); + + return rowFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getRowFilterPolicyEvaluators() : getPolicyEvaluators(rowFilterResourceTrie, resource); + } + AuditModeEnum getAuditModeEnum() { return auditModeEnum; } + + private List getPolicyEvaluators(Map resourceTrie, RangerAccessResource resource) { + List ret = null; + Set resourceKeys = resource == null ? null : resource.getKeys(); + + if(CollectionUtils.isNotEmpty(resourceKeys)) { + boolean isRetModifiable = false; + + for(String resourceName : resourceKeys) { + RangerResourceTrie trie = resourceTrie.get(resourceName); + + if(trie == null) { // if no trie exists for this resource level, ignore and continue to next level + continue; + } + + List resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); + + if(CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out + ret = null; + } else if(ret == null) { // initialize ret with policies found for this resource + ret = resourceEvaluators; + } else { // remove policies from ret that are not in resourceEvaluators + if(isRetModifiable) { + ret.retainAll(resourceEvaluators); + } else { + final List shorterList; + final List longerList; + + if (ret.size() < resourceEvaluators.size()) { + shorterList = ret; + longerList = resourceEvaluators; + } else { + shorterList = resourceEvaluators; + longerList = ret; + } + + ret = new ArrayList<>(shorterList); + ret.retainAll(longerList); + isRetModifiable = true; + } + } + + if(CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + + if(ret == null) { + ret = Collections.emptyList(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.getPolicyEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size()); + } + + return ret; + } + private RangerServiceDef normalizeAccessTypeDefs(RangerServiceDef serviceDef, final String componentType) { if (serviceDef != null && StringUtils.isNotBlank(componentType)) { @@ -283,10 +445,8 @@ private List normalizeAndPrunePolicyIte return policyItems; } - public static boolean isDelegateAdminPolicy(RangerPolicy policy) { - boolean ret = false; - - ret = hasDelegateAdminItems(policy.getPolicyItems()) + private static boolean isDelegateAdminPolicy(RangerPolicy policy) { + boolean ret = hasDelegateAdminItems(policy.getPolicyItems()) || hasDelegateAdminItems(policy.getDenyPolicyItems()) || hasDelegateAdminItems(policy.getAllowExceptions()) || hasDelegateAdminItems(policy.getDenyExceptions()); @@ -470,36 +630,27 @@ boolean setAuditEnabledFromCache(RangerAccessRequest request, RangerAccessResult LOG.debug("==> RangerPolicyRepository.setAuditEnabledFromCache()"); } - Boolean value = null; - - if (accessAuditCache != null) { - value = accessAuditCache.get(request.getResource().getAsString()); - } + final AuditInfo auditInfo = accessAuditCache != null ? accessAuditCache.get(request.getResource().getAsString()) : null; - if ((value != null)) { - result.setIsAudited(value); + if (auditInfo != null) { + result.setIsAudited(auditInfo.getIsAudited()); + result.setAuditPolicyId(auditInfo.getAuditPolicyId()); } if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyRepository.setAuditEnabledFromCache()"); + LOG.debug("<== RangerPolicyRepository.setAuditEnabledFromCache():" + (auditInfo != null)); } - return value != null; + return auditInfo != null; } - void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult ret) { + void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult result) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.storeAuditEnabledInCache()"); } - if ((ret.getIsAuditedDetermined())) { - String strResource = request.getResource().getAsString(); - - Boolean value = ret.getIsAudited() ? Boolean.TRUE : Boolean.FALSE; - - if (accessAuditCache != null) { - accessAuditCache.put(strResource, value); - } + if (accessAuditCache != null && result.getIsAuditedDetermined()) { + accessAuditCache.put(request.getResource().getAsString(), new AuditInfo(result.getIsAudited(), result.getAuditPolicyId())); } if (LOG.isDebugEnabled()) { @@ -507,11 +658,7 @@ void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult re } } - /** - * Remove nulls from policy resource values - * @param policy - */ - boolean scrubPolicy(RangerPolicy policy) { + private boolean scrubPolicy(RangerPolicy policy) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.scrubPolicy(" + policy + ")"); } @@ -538,6 +685,76 @@ boolean scrubPolicy(RangerPolicy policy) { return altered; } + void reorderPolicyEvaluators() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> reorderEvaluators()"); + } + + if(policyResourceTrie != null) { + reorderPolicyEvaluators(policyResourceTrie); + } else { + policyEvaluators = getReorderedPolicyEvaluators(policyEvaluators); + } + + if(dataMaskResourceTrie != null) { + reorderPolicyEvaluators(dataMaskResourceTrie); + } else { + dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(dataMaskPolicyEvaluators); + } + + if(rowFilterResourceTrie != null) { + reorderPolicyEvaluators(rowFilterResourceTrie); + } else { + rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(rowFilterPolicyEvaluators); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== reorderEvaluators()"); + } + } + + private void reorderPolicyEvaluators(Map trieMap) { + if(trieMap != null) { + for(Map.Entry entry : trieMap.entrySet()) { + RangerResourceTrie trie = entry.getValue(); + + if(trie != null) { + trie.reorderEvaluators(); + } + } + } + } + + private List getReorderedPolicyEvaluators(List evaluators) { + List ret = evaluators; + + if (CollectionUtils.isNotEmpty(evaluators)) { + + ret = new ArrayList(evaluators); + Collections.sort(ret); + + ret = Collections.unmodifiableList(ret); + } + + return ret; + } + + private Map createResourceTrieMap(List evaluators) { + final Map ret; + + if (CollectionUtils.isNotEmpty(evaluators) && serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) { + ret = new HashMap(); + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators)); + } + } else { + ret = null; + } + + return ret; + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java index adc7d8c10b..9b48dfe5ab 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java @@ -28,13 +28,20 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.ranger.plugin.util.ServiceDefUtil; + +import java.util.Map; public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerAbstractPolicyEvaluator.class); - private RangerPolicy policy = null; - private RangerServiceDef serviceDef = null; - private int evalOrder = 0; + private RangerPolicy policy = null; + private RangerServiceDef serviceDef = null; + private Integer leafResourceLevel = null; + private int evalOrder = 0; + protected long usageCount = 0; + protected boolean usageCountMutable = true; @Override @@ -43,14 +50,25 @@ public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyE LOG.debug("==> RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")"); } - this.policy = policy; - this.serviceDef = serviceDef; + this.policy = policy; + this.serviceDef = serviceDef; + this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(serviceDef, getPolicyResource()); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")"); } } + @Override + public long getId() { + return policy != null ? policy.getId() :-1; + } + + @Override + public Map getPolicyResource() { + return policy !=null ? policy.getResources() : null; + } + @Override public RangerPolicy getPolicy() { return policy; @@ -61,6 +79,11 @@ public RangerServiceDef getServiceDef() { return serviceDef; } + @Override + public Integer getLeafResourceLevel() { + return leafResourceLevel; + } + public boolean hasAllow() { return policy != null && CollectionUtils.isNotEmpty(policy.getPolicyItems()); } @@ -73,23 +96,44 @@ public boolean hasDeny() { return policy != null && CollectionUtils.isNotEmpty(policy.getDenyPolicyItems()); } + @Override + public long getUsageCount() { + return usageCount; + } + @Override public int getEvalOrder() { return evalOrder; } - @Override public boolean isAuditEnabled() { return policy != null && policy.getIsAuditEnabled(); } @Override - public int compareTo(RangerPolicyEvaluator other) { + public int compareTo(RangerPolicyResourceEvaluator obj) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()"); + LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()"); } - int result = Integer.compare(this.getEvalOrder(), other.getEvalOrder()); + int result; + + if(obj instanceof RangerPolicyEvaluator) { + RangerPolicyEvaluator other = (RangerPolicyEvaluator)obj; + + if (hasDeny() && !other.hasDeny()) { + result = -1; + } else if (!hasDeny() && other.hasDeny()) { + result = 1; + } else { + result = Long.compare(other.getUsageCount(), this.usageCount); + if (result == 0) { + result = Integer.compare(this.evalOrder, other.getEvalOrder()); + } + } + } else { + result = Long.compare(getId(), obj.getId()); + } if(LOG.isDebugEnabled()) { LOG.debug("<== RangerAbstractPolicyEvaluator.compareTo(), result:" + result); @@ -102,6 +146,22 @@ public void setEvalOrder(int evalOrder) { this.evalOrder = evalOrder; } + @Override + public void incrementUsageCount(int number) { + if (usageCountMutable) usageCount += number; + } + + @Override + public void setUsageCountImmutable() { + this.usageCountMutable = false; + } + + @Override + public void resetUsageCount() { + this.usageCount = 0; + this.usageCountMutable = true; + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java index 514884fba3..b36bc1f79e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java @@ -120,7 +120,8 @@ private int computeEvalOrder() { int evalOrder = RANGER_POLICY_ITEM_EVAL_ORDER_DEFAULT; if(policyItem != null) { - if(CollectionUtils.isNotEmpty(policyItem.getGroups()) && policyItem.getGroups().contains(RangerPolicyEngine.GROUP_PUBLIC)) { + if((CollectionUtils.isNotEmpty(policyItem.getGroups()) && policyItem.getGroups().contains(RangerPolicyEngine.GROUP_PUBLIC)) + || (CollectionUtils.isNotEmpty(policyItem.getUsers()) && policyItem.getUsers().contains(RangerPolicyEngine.USER_CURRENT))) { evalOrder -= RANGER_POLICY_ITEM_EVAL_ORDER_MAX_DISCOUNT_USERSGROUPS; } else { int userGroupCount = 0; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java index 91a53d8b61..7711765faa 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java @@ -26,6 +26,8 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import java.util.Map; + public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerCachedPolicyEvaluator.class); @@ -47,34 +49,38 @@ public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyE } @Override - public boolean isMatch(RangerAccessResource resource) { + public boolean isMatch(RangerAccessResource resource, Map evalContext) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerCachedPolicyEvaluator.isMatch(" + resource + ")"); + LOG.debug("==> RangerCachedPolicyEvaluator.isMatch(" + resource + ", " + evalContext + ")"); } boolean result = false; - // Check in the evaluator-owned cache for the match, if found return. else call super.isMatch(), add result to cache - RangerResourceAccessCache.LookupResult lookup = cache.lookup(resource); + if (needsDynamicEval()) { + result = super.isMatch(resource, evalContext); + } else { + // Check in the evaluator-owned cache for the match, if found return. else call super.isMatch(), add result to cache + RangerResourceAccessCache.LookupResult lookup = cache.lookup(resource); - if (lookup != RangerResourceAccessCache.LookupResult.IN_NOTMATCHED_CACHE) { - // We dont know definitely that this previously not matched - if (lookup != RangerResourceAccessCache.LookupResult.IN_MATCHED_CACHE) { - result = super.isMatch(resource); + if (lookup != RangerResourceAccessCache.LookupResult.IN_NOTMATCHED_CACHE) { + // We dont know definitely that this previously not matched + if (lookup != RangerResourceAccessCache.LookupResult.IN_MATCHED_CACHE) { + result = super.isMatch(resource, evalContext); - // update the cache with the result of the match - if(result) { - cache.add(resource, RangerResourceAccessCache.CacheType.MATCHED_CACHE); + // update the cache with the result of the match + if (result) { + cache.add(resource, RangerResourceAccessCache.CacheType.MATCHED_CACHE); + } else { + cache.add(resource, RangerResourceAccessCache.CacheType.NOTMATCHED_CACHE); + } } else { - cache.add(resource, RangerResourceAccessCache.CacheType.NOTMATCHED_CACHE); + result = true; } - } else { - result = true; } } if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerCachedPolicyEvaluator.isMatch(" + resource + "): " + result); + LOG.debug("<== RangerCachedPolicyEvaluator.isMatch(" + resource + ", " + evalContext + "): " + result); } return result; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 368a695440..867cd208a7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -51,6 +51,8 @@ import org.apache.ranger.plugin.policyengine.RangerRowFilterResult; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServiceDefUtil; @@ -69,13 +71,24 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator private int customConditionsCount = 0; private List dataMaskEvaluators = null; private List rowFilterEvaluators = null; + private String perfTag; + protected boolean needsDynamicEval() { return resourceMatcher != null ? resourceMatcher.getNeedsDynamicEval() : false; } + @Override public int getCustomConditionsCount() { return customConditionsCount; } + @Override + public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return resourceMatcher; } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return resourceMatcher != null ? resourceMatcher.getResourceMatcher(resourceName) : null; + } + @Override public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { if(LOG.isDebugEnabled()) { @@ -161,14 +174,14 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (!result.getIsAuditedDetermined()) { // Need to match request.resource first. If it matches (or head matches), then only more progress can be made if (!isResourceMatchAttempted) { - isResourceMatch = isMatch(request.getResource()); + isResourceMatch = isMatch(request.getResource(), request.getContext()); isResourceMatchAttempted = true; } // Try head match only if match was not found and ANY access was requested if (!isResourceMatch) { if (attemptResourceHeadMatch && !isResourceHeadMatchAttempted) { - isResourceHeadMatch = matchResourceHead(request.getResource()); + isResourceHeadMatch = matchResourceHead(request.getResource(), request.getContext()); isResourceHeadMatchAttempted = true; } } @@ -177,6 +190,7 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { // We are done for determining if audit is needed for this policy if (isAuditEnabled()) { result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getId()); } } } @@ -186,7 +200,7 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (hasMatchablePolicyItem(request)) { // Try Match only if it was not attempted as part of evaluating Audit requirement if (!isResourceMatchAttempted) { - isResourceMatch = isMatch(request.getResource()); + isResourceMatch = isMatch(request.getResource(), request.getContext()); isResourceMatchAttempted = true; } @@ -194,7 +208,7 @@ public void evaluate(RangerAccessRequest request, RangerAccessResult result) { // Audit requirement if (!isResourceMatch) { if (attemptResourceHeadMatch && !isResourceHeadMatchAttempted) { - isResourceHeadMatch = matchResourceHead(request.getResource()); + isResourceHeadMatch = matchResourceHead(request.getResource(), request.getContext()); isResourceHeadMatchAttempted = true; } } @@ -235,13 +249,13 @@ public void evaluate(RangerAccessRequest request, RangerDataMaskResult result) { if (!result.getIsAuditedDetermined()) { if (!isResourceMatchAttempted) { - isResourceMatch = isMatch(request.getResource()); + isResourceMatch = isMatch(request.getResource(), request.getContext()); isResourceMatchAttempted = true; } if (!isResourceMatch) { if (attemptResourceHeadMatch && !isResourceHeadMatchAttempted) { - isResourceHeadMatch = matchResourceHead(request.getResource()); + isResourceHeadMatch = matchResourceHead(request.getResource(), request.getContext()); isResourceHeadMatchAttempted = true; } } @@ -249,19 +263,20 @@ public void evaluate(RangerAccessRequest request, RangerDataMaskResult result) { if (isResourceMatch || isResourceHeadMatch) { if (isAuditEnabled()) { result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getId()); } } } if (!result.getIsAccessDetermined()) { if (!isResourceMatchAttempted) { - isResourceMatch = isMatch(request.getResource()); + isResourceMatch = isMatch(request.getResource(), request.getContext()); isResourceMatchAttempted = true; } if (!isResourceMatch) { if (attemptResourceHeadMatch && !isResourceHeadMatchAttempted) { - isResourceHeadMatch = matchResourceHead(request.getResource()); + isResourceHeadMatch = matchResourceHead(request.getResource(), request.getContext()); isResourceHeadMatchAttempted = true; } } @@ -270,6 +285,7 @@ public void evaluate(RangerAccessRequest request, RangerDataMaskResult result) { evaluatePolicyItems(request, result); } } + } RangerPerfTracer.log(perf); @@ -300,13 +316,13 @@ public void evaluate(RangerAccessRequest request, RangerRowFilterResult result) if (!result.getIsAuditedDetermined()) { if (!isResourceMatchAttempted) { - isResourceMatch = isMatch(request.getResource()); + isResourceMatch = isMatch(request.getResource(), request.getContext()); isResourceMatchAttempted = true; } if (!isResourceMatch) { if (attemptResourceHeadMatch && !isResourceHeadMatchAttempted) { - isResourceHeadMatch = matchResourceHead(request.getResource()); + isResourceHeadMatch = matchResourceHead(request.getResource(), request.getContext()); isResourceHeadMatchAttempted = true; } } @@ -314,19 +330,20 @@ public void evaluate(RangerAccessRequest request, RangerRowFilterResult result) if (isResourceMatch || isResourceHeadMatch) { if (isAuditEnabled()) { result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getId()); } } } if (!result.getIsAccessDetermined()) { if (!isResourceMatchAttempted) { - isResourceMatch = isMatch(request.getResource()); + isResourceMatch = isMatch(request.getResource(), request.getContext()); isResourceMatchAttempted = true; } if (!isResourceMatch) { if (attemptResourceHeadMatch && !isResourceHeadMatchAttempted) { - isResourceHeadMatch = matchResourceHead(request.getResource()); + isResourceHeadMatch = matchResourceHead(request.getResource(), request.getContext()); isResourceHeadMatchAttempted = true; } } @@ -345,9 +362,9 @@ public void evaluate(RangerAccessRequest request, RangerRowFilterResult result) } @Override - public boolean isMatch(RangerAccessResource resource) { + public boolean isMatch(RangerAccessResource resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; @@ -355,32 +372,32 @@ public boolean isMatch(RangerAccessResource resource) { RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_REQUEST_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_REQUEST_LOG, "RangerPolicyEvaluator.isMatch(resource=" + resource.getAsString() + "," + perfTag + ")"); + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_REQUEST_LOG, "RangerPolicyEvaluator.isMatch(resource=" + resource.getAsString() + "," + evalContext + "," + perfTag + ")"); } if(resourceMatcher != null) { - ret = resourceMatcher.isMatch(resource); + ret = resourceMatcher.isMatch(resource, evalContext); } RangerPerfTracer.log(perf); if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resource + "): " + ret); + LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resource + ", " + evalContext + "): " + ret); } return ret; } @Override - public boolean isCompleteMatch(RangerAccessResource resource) { + public boolean isCompleteMatch(RangerAccessResource resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.isCompleteMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.isCompleteMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; if(resourceMatcher != null) { - ret = resourceMatcher.isCompleteMatch(resource); + ret = resourceMatcher.isCompleteMatch(resource, evalContext); } if(LOG.isDebugEnabled()) { @@ -391,19 +408,19 @@ public boolean isCompleteMatch(RangerAccessResource resource) { } @Override - public boolean isCompleteMatch(Map resources) { + public boolean isCompleteMatch(Map resources, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.isCompleteMatch(" + resources + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.isCompleteMatch(" + resources + ", " + evalContext + ")"); } boolean ret = false; if(resourceMatcher != null) { - ret = resourceMatcher.isCompleteMatch(resources); + ret = resourceMatcher.isCompleteMatch(resources, evalContext); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyEvaluator.isCompleteMatch(" + resources + "): " + ret); + LOG.debug("<== RangerDefaultPolicyEvaluator.isCompleteMatch(" + resources + ", " + evalContext + "): " + ret); } return ret; @@ -415,7 +432,10 @@ public boolean isAccessAllowed(RangerAccessResource resource, String user, Set RangerDefaultPolicyEvaluator.isAccessAllowed(" + resource + ", " + user + ", " + userGroups + ", " + accessType + ")"); } - boolean ret = isAccessAllowed(user, userGroups, accessType) && isMatch(resource); + Map evalContext = new HashMap(); + RangerAccessRequestUtil.setCurrentUserInContext(evalContext, user); + + boolean ret = isAccessAllowed(user, userGroups, accessType) && isMatch(resource, evalContext); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.isAccessAllowed(" + resource + ", " + user + ", " + userGroups + ", " + accessType + "): " + ret); @@ -430,7 +450,10 @@ public boolean isAccessAllowed(Map resources, Stri LOG.debug("==> RangerDefaultPolicyEvaluator.isAccessAllowed(" + resources + ", " + user + ", " + userGroups + ", " + accessType + ")"); } - boolean ret = isAccessAllowed(user, userGroups, accessType) && isMatch(resources); + Map evalContext = new HashMap(); + RangerAccessRequestUtil.setCurrentUserInContext(evalContext, user); + + boolean ret = isAccessAllowed(user, userGroups, accessType) && isMatch(resources, evalContext); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.isAccessAllowed(" + resources + ", " + user + ", " + userGroups + ", " + accessType + "): " + ret); @@ -445,9 +468,9 @@ public void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAcc LOG.debug("==> RangerDefaultPolicyEvaluator.getResourceAccessInfo(" + request + ", " + result + ")"); } - final boolean isResourceMatch = isMatch(request.getResource()); + final boolean isResourceMatch = isMatch(request.getResource(), request.getContext()); final boolean attemptResourceHeadMatch = request.isAccessTypeAny() || request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS; - final boolean isResourceHeadMatch = (!isResourceMatch && attemptResourceHeadMatch) ? matchResourceHead(request.getResource()) : false; + final boolean isResourceHeadMatch = (!isResourceMatch && attemptResourceHeadMatch) ? matchResourceHead(request.getResource(), request.getContext()) : false; if(isResourceMatch || isResourceHeadMatch) { if (CollectionUtils.isNotEmpty(allowEvaluators)) { @@ -630,37 +653,37 @@ private void getResourceAccessInfo(RangerAccessRequest request, List evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.matchResourceHead(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.matchResourceHead(" + resource + ", " + evalContext + ")"); } boolean ret = false; if(resourceMatcher != null) { - ret = resourceMatcher.isHeadMatch(resource); + ret = resourceMatcher.isHeadMatch(resource, evalContext); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyEvaluator.matchResourceHead(" + resource + "): " + ret); + LOG.debug("<== RangerDefaultPolicyEvaluator.matchResourceHead(" + resource + ", " + evalContext + "): " + ret); } return ret; } - protected boolean isMatch(Map resources) { + protected boolean isMatch(Map resources, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + resources + ")"); + LOG.debug("==> RangerDefaultPolicyEvaluator.isMatch(" + resources + ", " + evalContext + ")"); } boolean ret = false; if(resourceMatcher != null) { - ret = resourceMatcher.isMatch(resources); + ret = resourceMatcher.isMatch(resources, evalContext); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resources + "): " + ret); + LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resources + ", " + evalContext + "): " + ret); } return ret; @@ -990,5 +1013,6 @@ private T getMatchingPolicyItem(String use } return ret; - } + } + } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java index e2c715f439..b899dd1c2b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; import org.apache.commons.collections.CollectionUtils; @@ -48,6 +49,8 @@ public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEv private static final Log PERF_POLICYCONDITION_INIT_LOG = RangerPerfTracer.getPerfLogger("policycondition.init"); private static final Log PERF_POLICYCONDITION_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policycondition.request"); + private boolean hasCurrentUser = false; + public RangerDefaultPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, int policyItemIndex, RangerPolicyEngineOptions options) { super(serviceDef, policy, policyItem, policyItemType, policyItemIndex, options); } @@ -100,6 +103,9 @@ public void init() { RangerPerfTracer.log(perf); } + List users = policyItem != null ? policyItem.getUsers() : null; + this.hasCurrentUser = CollectionUtils.isNotEmpty(users) && users.contains(RangerPolicyEngine.USER_CURRENT); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyItemEvaluator(policyId=" + policyId + ", conditionsCount=" + getConditionEvaluators().size() + ")"); } @@ -172,7 +178,7 @@ public boolean matchUserGroup(String user, Set userGroups) { if(policyItem != null) { if(!ret && user != null && policyItem.getUsers() != null) { - ret = policyItem.getUsers().contains(user); + ret = hasCurrentUser || policyItem.getUsers().contains(user); } if(!ret && userGroups != null && policyItem.getGroups() != null) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java index 2e777aedbf..fb854d0a70 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java @@ -40,14 +40,13 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator private boolean delegateAdmin = false; private boolean hasAllPerms = false; private boolean hasPublicGroup = false; - + private boolean hasCurrentUser = false; // For computation of priority private static final String RANGER_POLICY_EVAL_MATCH_ANY_PATTERN_STRING = "*"; private static final String RANGER_POLICY_EVAL_MATCH_ONE_CHARACTER_STRING = "?"; private static final int RANGER_POLICY_EVAL_SCORE_DEFAULT = 10000; - private static final int RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY = 4000; private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE = 100; private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS = 25; @@ -60,7 +59,7 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator private static final int RANGER_POLICY_EVAL_SCORE_RESOURCE_DISCOUNT_IS_EXCLUDES = 5; private static final int RANGER_POLICY_EVAL_SCORE_RESORUCE_DISCOUNT_IS_RECURSIVE = 5; private static final int RANGER_POLICY_EVAL_SCORE_CUSTOM_CONDITION_PENALTY = 5; - + private static final int RANGER_POLICY_EVAL_SCORE_DYNAMIC_RESOURCE_EVAL_PENALTY = 20; @Override public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { @@ -78,9 +77,17 @@ public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyE hasAllPerms = checkIfHasAllPerms(); + for (String user : users) { + if (user.equalsIgnoreCase(RangerPolicyEngine.USER_CURRENT)) { + hasCurrentUser = true; + break; + } + } + for (String group : groups) { if (group.equalsIgnoreCase(RangerPolicyEngine.GROUP_PUBLIC)) { hasPublicGroup = true; + break; } } @@ -186,10 +193,13 @@ public int computeEvalOrder() { } } } + if (needsDynamicEval()) { + evalOrder += RANGER_POLICY_EVAL_SCORE_DYNAMIC_RESOURCE_EVAL_PENALTY; + } evalOrder -= Math.min(RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE, resourceDiscount); - if (hasPublicGroup) { + if (hasPublicGroup || hasCurrentUser) { evalOrder -= RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS; } else { evalOrder -= Math.min(groups.size() + users.size(), RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS); @@ -202,10 +212,6 @@ public int computeEvalOrder() { evalOrder -= customConditionsDiscount; } - if (hasDeny()) { - evalOrder -= RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY; - } - if(LOG.isDebugEnabled()) { LOG.debug("<== RangerOptimizedPolicyEvaluator.computeEvalOrder(), policyName:" + policy.getName() + ", priority:" + evalOrder); } @@ -232,7 +238,7 @@ protected boolean isAccessAllowed(String user, Set userGroups, String ac protected boolean hasMatchablePolicyItem(RangerAccessRequest request) { boolean ret = false; - if (hasPublicGroup || users.contains(request.getUser()) || CollectionUtils.containsAny(groups, request.getUserGroups())) { + if (hasPublicGroup || hasCurrentUser || users.contains(request.getUser()) || CollectionUtils.containsAny(groups, request.getUserGroups())) { if(request.isAccessTypeDelegatedAdmin()) { ret = delegateAdmin; } else if(hasAllPerms) { @@ -248,7 +254,7 @@ protected boolean hasMatchablePolicyItem(RangerAccessRequest request) { private boolean hasMatchablePolicyItem(String user, Set userGroups, String accessType) { boolean ret = false; - if (hasPublicGroup || users.contains(user) || CollectionUtils.containsAny(groups, userGroups)) { + if (hasPublicGroup || hasCurrentUser || users.contains(user) || CollectionUtils.containsAny(groups, userGroups)) { boolean isAdminAccess = StringUtils.equals(accessType, RangerPolicyEngine.ADMIN_ACCESS); if(isAdminAccess) { @@ -281,6 +287,7 @@ private void preprocessPolicyItems(List groups.addAll(item.getGroups()); users.addAll(item.getUsers()); + } } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index be97830eff..92f15929b0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -33,9 +33,10 @@ import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo; import org.apache.ranger.plugin.policyengine.RangerRowFilterResult; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; -public interface RangerPolicyEvaluator extends Comparable { +public interface RangerPolicyEvaluator extends RangerPolicyResourceEvaluator { String EVALUATOR_TYPE_AUTO = "auto"; String EVALUATOR_TYPE_OPTIMIZED = "optimized"; String EVALUATOR_TYPE_CACHED = "cached"; @@ -52,6 +53,14 @@ public interface RangerPolicyEvaluator extends Comparable int getEvalOrder(); + long getUsageCount(); + + void incrementUsageCount(int number); + + void setUsageCountImmutable(); + + void resetUsageCount(); + int getCustomConditionsCount(); boolean isAuditEnabled(); @@ -62,15 +71,16 @@ public interface RangerPolicyEvaluator extends Comparable void evaluate(RangerAccessRequest request, RangerRowFilterResult result); - boolean isMatch(RangerAccessResource resource); + boolean isMatch(RangerAccessResource resource, Map evalContext); - boolean isCompleteMatch(RangerAccessResource resource); + boolean isCompleteMatch(RangerAccessResource resource, Map evalContext); - boolean isCompleteMatch(Map resources); + boolean isCompleteMatch(Map resources, Map evalContext); boolean isAccessAllowed(RangerAccessResource resource, String user, Set userGroups, String accessType); boolean isAccessAllowed(Map resources, String user, Set userGroups, String accessType); void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAccessInfo result); + } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java index 80e46f55be..20b08c8382 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java @@ -47,7 +47,6 @@ public interface RangerPolicyItemEvaluator extends Comparable userGroups); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index 6ea194d69c..5444e2b1e7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -48,6 +48,7 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM protected Map policyResources = null; private Map matchers = null; + private boolean needsDynamicEval = false; private List firstValidResourceDefHierarchy; @Override @@ -67,6 +68,11 @@ public void setPolicyResources(Map policyResources this.policyResources = policyResources; } + @Override + public boolean getNeedsDynamicEval() { + return needsDynamicEval; + } + @Override public void init() { if(LOG.isDebugEnabled()) { @@ -133,6 +139,9 @@ public void init() { RangerResourceMatcher matcher = createResourceMatcher(resourceDef, policyResource); if (matcher != null) { + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } matchers.put(resourceName, matcher); } else { LOG.error("failed to find matcher for resource " + resourceName); @@ -162,7 +171,7 @@ public void init() { String keysString = sb.toString(); String serviceDefName = serviceDef == null ? "" : serviceDef.getName(); String validHierarchy = ""; - if (CollectionUtils.isNotEmpty(firstValidResourceDefHierarchy)) { + if (serviceDef != null && CollectionUtils.isNotEmpty(firstValidResourceDefHierarchy)) { RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); List resourceDefNameOrderedList = serviceDefHelper.getAllResourceNamesOrdered(firstValidResourceDefHierarchy); @@ -180,9 +189,18 @@ public void init() { } @Override - public boolean isMatch(RangerAccessResource resource) { + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return matchers != null ? matchers.get(resourceName) : null; + } + + public boolean isMatch(RangerAccessResource resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; @@ -201,9 +219,9 @@ public boolean isMatch(RangerAccessResource resource) { // when no value exists for a resourceName, consider it a match only if: policy doesn't have a matcher OR matcher allows no-value resource if(StringUtils.isEmpty(resourceValue)) { - ret = matcher == null || matcher.isMatch(resourceValue); + ret = matcher == null || matcher.isMatch(resourceValue, evalContext); } else { - ret = matcher != null && matcher.isMatch(resourceValue); + ret = matcher != null && matcher.isMatch(resourceValue, evalContext); } if(! ret) { @@ -218,7 +236,7 @@ public boolean isMatch(RangerAccessResource resource) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.isMatch(" + resource + "): " + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isMatch(" + resource + ", " + evalContext + "): " + ret); } return ret; @@ -226,9 +244,9 @@ public boolean isMatch(RangerAccessResource resource) { @Override - public boolean isMatch(Map resources) { + public boolean isMatch(Map resources, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isMatch(" + resources + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isMatch(" + resources + ", " + evalContext + ")"); } boolean ret = false; @@ -247,10 +265,10 @@ public boolean isMatch(Map resources) { // when no value exists for a resourceName, consider it a match only if: policy doesn't have a matcher OR matcher allows no-value resource if(resourceValues == null || CollectionUtils.isEmpty(resourceValues.getValues())) { - ret = matcher == null || matcher.isMatch(null); + ret = matcher == null || matcher.isMatch(null, null); } else if(matcher != null) { for(String resourceValue : resourceValues.getValues()) { - ret = matcher.isMatch(resourceValue); + ret = matcher.isMatch(resourceValue, evalContext); if(! ret) { break; @@ -270,16 +288,16 @@ public boolean isMatch(Map resources) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.isMatch(" + resources + "): " + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isMatch(" + resources + ", " + evalContext + "): " + ret); } return ret; } @Override - public boolean isCompleteMatch(RangerAccessResource resource) { + public boolean isCompleteMatch(RangerAccessResource resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; @@ -301,9 +319,9 @@ public boolean isCompleteMatch(RangerAccessResource resource) { RangerResourceMatcher matcher = matchers == null ? null : matchers.get(resourceName); if(StringUtils.isEmpty(resourceValue)) { - ret = matcher == null || matcher.isCompleteMatch(resourceValue); + ret = matcher == null || matcher.isCompleteMatch(resourceValue, evalContext); } else { - ret = matcher != null && matcher.isCompleteMatch(resourceValue); + ret = matcher != null && matcher.isCompleteMatch(resourceValue, evalContext); } if(! ret) { @@ -318,21 +336,21 @@ public boolean isCompleteMatch(RangerAccessResource resource) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resource + "): " + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + "): " + ret); } return ret; } @Override - public boolean isHeadMatch(RangerAccessResource resource) { + public boolean isHeadMatch(RangerAccessResource resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isHeadMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isHeadMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; - + if (matchers == null) { LOG.debug("RangerDefaultPolicyResourceMatcher.isHeadMatch(): PolicyResourceMatcher not initialized correctly!!!"); @@ -345,22 +363,22 @@ public boolean isHeadMatch(RangerAccessResource resource) { } else { - ret = newIsHeadMatch(resource); + ret = newIsHeadMatch(resource, evalContext); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.matchResourceHead(" + resource + "): " + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.matchResourceHead(" + resource + ", " + evalContext + "): " + ret); } return ret; } @Override - public boolean isExactHeadMatch(RangerAccessResource resource) { + public boolean isExactHeadMatch(RangerAccessResource resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isExactHeadMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isExactHeadMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; @@ -382,21 +400,21 @@ public boolean isExactHeadMatch(RangerAccessResource resource) { } else { - ret = newIsHeadMatch(resource); + ret = newIsHeadMatch(resource, evalContext); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.isExactHeadMatch(" + resource + ")" + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isExactHeadMatch(" + resource + ", " + evalContext + ")" + ret); } return ret; } - private boolean newIsHeadMatch(RangerAccessResource resource) { + private boolean newIsHeadMatch(RangerAccessResource resource, Map evalContext) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.newIsHeadMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.newIsHeadMatch(" + resource + ", " + evalContext + ")"); } boolean skipped = false; @@ -414,7 +432,7 @@ private boolean newIsHeadMatch(RangerAccessResource resource) { if (!skipped) { - matched = matcher.isMatch(resourceValue); + matched = matcher.isMatch(resourceValue, evalContext); } else { @@ -434,7 +452,7 @@ private boolean newIsHeadMatch(RangerAccessResource resource) { } if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.newIsHeadMatch(" + resource + "): " + matched); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.newIsHeadMatch(" + resource + ", " + evalContext + "): " + matched); } return matched; @@ -510,9 +528,9 @@ protected static RangerResourceMatcher createResourceMatcher(RangerResourceDef r } @Override - public boolean isCompleteMatch(Map resources) { + public boolean isCompleteMatch(Map resources, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resources + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resources + ", " + evalContext + ")"); } boolean ret = false; @@ -551,7 +569,7 @@ public boolean isCompleteMatch(Map resources) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resources + "): " + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isCompleteMatch(" + resources + ", " + evalContext + "): " + ret); } return ret; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java new file mode 100644 index 0000000000..eed58e1ca5 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.policyresourcematcher; + + +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; + +import java.util.Map; + +public interface RangerPolicyResourceEvaluator extends Comparable { + long getId(); + + RangerPolicyResourceMatcher getPolicyResourceMatcher(); + + Map getPolicyResource(); + + RangerResourceMatcher getResourceMatcher(String resourceName); + + Integer getLeafResourceLevel(); +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index 54b9586cec..ea0f36c002 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -25,6 +25,7 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.RangerAccessResource; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; public interface RangerPolicyResourceMatcher { void setServiceDef(RangerServiceDef serviceDef); @@ -35,17 +36,23 @@ public interface RangerPolicyResourceMatcher { void init(); - boolean isMatch(RangerAccessResource resource); + RangerServiceDef getServiceDef(); - boolean isMatch(Map resources); + RangerResourceMatcher getResourceMatcher(String resourceName); - boolean isCompleteMatch(RangerAccessResource resource); + boolean isMatch(RangerAccessResource resource, Map evalContext); - boolean isHeadMatch(RangerAccessResource resource); + boolean isMatch(Map resources, Map evalContext); - boolean isExactHeadMatch(RangerAccessResource resource); + boolean isCompleteMatch(RangerAccessResource resource, Map evalContext); - boolean isCompleteMatch(Map resources); + boolean isHeadMatch(RangerAccessResource resource, Map evalContext); + + boolean isExactHeadMatch(RangerAccessResource resource, Map evalContext); + + boolean isCompleteMatch(Map resources, Map evalContext); + + boolean getNeedsDynamicEval(); StringBuilder toString(StringBuilder sb); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java index cd725c9f9c..26b9866b5b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java @@ -20,27 +20,33 @@ package org.apache.ranger.plugin.resourcematcher; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOCase; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; +import org.apache.ranger.plugin.util.ServiceDefUtil; public abstract class RangerAbstractResourceMatcher implements RangerResourceMatcher { private static final Log LOG = LogFactory.getLog(RangerAbstractResourceMatcher.class); public final static String WILDCARD_ASTERISK = "*"; - public final static String WILDCARDS = "*?"; - public final static String OPTIONS_SEP = ";"; - public final static String OPTION_NV_SEP = "="; public final static String OPTION_IGNORE_CASE = "ignoreCase"; public final static String OPTION_WILD_CARD = "wildCard"; + public final static String OPTION_REPLACE_TOKENS = "replaceTokens"; + public final static String OPTION_TOKEN_DELIMITER_START = "tokenDelimiterStart"; + public final static String OPTION_TOKEN_DELIMITER_END = "tokenDelimiterEnd"; + public final static String OPTION_TOKEN_DELIMITER_ESCAPE = "tokenDelimiterEscape"; + public final static String OPTION_TOKEN_DELIMITER_PREFIX = "tokenDelimiterPrefix"; protected RangerResourceDef resourceDef = null; protected RangerPolicyResource policyResource = null; @@ -48,9 +54,16 @@ public abstract class RangerAbstractResourceMatcher implements RangerResourceMat protected boolean optIgnoreCase = false; protected boolean optWildCard = false; - protected List policyValues = null; - protected boolean policyIsExcludes = false; - protected boolean isMatchAny = false; + protected List policyValues = null; + protected boolean policyIsExcludes = false; + protected boolean isMatchAny = false; + protected ResourceMatcherWrapper resourceMatchers = null; + + protected boolean optReplaceTokens = false; + protected char startDelimiterChar = '{'; + protected char endDelimiterChar = '}'; + protected char escapeChar = '\\'; + protected String tokenPrefix = ""; @Override public void setResourceDef(RangerResourceDef resourceDef) { @@ -68,44 +81,120 @@ public void init() { LOG.debug("==> RangerAbstractResourceMatcher.init()"); } - optIgnoreCase = getBooleanOption(OPTION_IGNORE_CASE, true); - optWildCard = getBooleanOption(OPTION_WILD_CARD, true); + Map options = resourceDef != null ? resourceDef.getMatcherOptions() : null; + + optIgnoreCase = getOptionIgnoreCase(options); + optWildCard = getOptionWildCard(options); - policyValues = new ArrayList(); + policyValues = new ArrayList(); policyIsExcludes = policyResource == null ? false : policyResource.getIsExcludes(); - if(policyResource != null && policyResource.getValues() != null) { - boolean isWildCardPresent = false; - for(String policyValue : policyResource.getValues()) { - if(StringUtils.isEmpty(policyValue)) { + if (policyResource != null && policyResource.getValues() != null) { + for (String policyValue : policyResource.getValues()) { + if (StringUtils.isEmpty(policyValue)) { continue; } - - if(optWildCard) { - if (StringUtils.containsOnly(policyValue, WILDCARD_ASTERISK)) { - isMatchAny = true; - } else if (!isWildCardPresent && StringUtils.containsAny(policyValue, WILDCARDS)) { - isWildCardPresent = true; - } - } policyValues.add(policyValue); } - optWildCard = optWildCard && isWildCardPresent; } - if(policyValues.isEmpty()) { - isMatchAny = true; + optReplaceTokens = getOptionReplaceTokens(options); + + if(optReplaceTokens) { + startDelimiterChar = getOptionDelimiterStart(options); + endDelimiterChar = getOptionDelimiterEnd(options); + escapeChar = getOptionDelimiterEscape(options); + tokenPrefix = getOptionDelimiterPrefix(options); + + if(escapeChar == startDelimiterChar || escapeChar == endDelimiterChar || + tokenPrefix.indexOf(escapeChar) != -1 || tokenPrefix.indexOf(startDelimiterChar) != -1 || + tokenPrefix.indexOf(endDelimiterChar) != -1) { + String resouceName = resourceDef == null ? "" : resourceDef.getName(); + + String msg = "Invalid token-replacement parameters for resource '" + resouceName + "': { "; + msg += (OPTION_TOKEN_DELIMITER_START + "='" + startDelimiterChar + "'; "); + msg += (OPTION_TOKEN_DELIMITER_END + "='" + endDelimiterChar + "'; "); + msg += (OPTION_TOKEN_DELIMITER_ESCAPE + "='" + escapeChar + "'; "); + msg += (OPTION_TOKEN_DELIMITER_PREFIX + "='" + tokenPrefix + "' }. "); + msg += "Token replacement disabled"; + + LOG.error(msg); + + optReplaceTokens = false; + } } + resourceMatchers = buildResourceMatchers(); + isMatchAny = resourceMatchers == null || CollectionUtils.isEmpty(resourceMatchers.getResourceMatchers()); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerAbstractResourceMatcher.init()"); } } @Override - public boolean isCompleteMatch(String resource) { + public boolean isMatchAny() { return isMatchAny; } + + public boolean getNeedsDynamicEval() { + return resourceMatchers != null && resourceMatchers.getNeedsDynamicEval(); + } + + public static boolean getOptionIgnoreCase(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_IGNORE_CASE, true); + } + + public static boolean getOptionWildCard(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_WILD_CARD, true); + } + + public static boolean getOptionReplaceTokens(Map options) { + return ServiceDefUtil.getBooleanOption(options, OPTION_REPLACE_TOKENS, true); + } + + public static char getOptionDelimiterStart(Map options) { + return ServiceDefUtil.getCharOption(options, OPTION_TOKEN_DELIMITER_START, '{'); + } + + public static char getOptionDelimiterEnd(Map options) { + return ServiceDefUtil.getCharOption(options, OPTION_TOKEN_DELIMITER_END, '}'); + } + + public static char getOptionDelimiterEscape(Map options) { + return ServiceDefUtil.getCharOption(options, OPTION_TOKEN_DELIMITER_ESCAPE, '\\'); + } + + public static String getOptionDelimiterPrefix(Map options) { + return ServiceDefUtil.getOption(options, OPTION_TOKEN_DELIMITER_PREFIX, ""); + } + protected ResourceMatcherWrapper buildResourceMatchers() { + List resourceMatchers = new ArrayList(); + boolean needsDynamicEval = false; + + for (String policyValue : policyValues) { + ResourceMatcher matcher = getMatcher(policyValue); + + if (matcher != null) { + if (matcher.isMatchAny()) { + resourceMatchers.clear(); + break; + } + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } + resourceMatchers.add(matcher); + } + } + + Collections.sort(resourceMatchers); + + return CollectionUtils.isNotEmpty(resourceMatchers) ? + new ResourceMatcherWrapper(needsDynamicEval, resourceMatchers) : null; + } + + @Override + public boolean isCompleteMatch(String resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAbstractResourceMatcher.isCompleteMatch(" + resource + ")"); + LOG.debug("==> RangerAbstractResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; @@ -114,7 +203,7 @@ public boolean isCompleteMatch(String resource) { ret = StringUtils.isEmpty(resource); } else if(policyValues.size() == 1) { String policyValue = policyValues.get(0); - + if(isMatchAny) { ret = StringUtils.containsOnly(resource, WILDCARD_ASTERISK); } else { @@ -127,53 +216,7 @@ public boolean isCompleteMatch(String resource) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAbstractResourceMatcher.isCompleteMatch(" + resource + "): " + ret); - } - - return ret; - } - - - public String getOption(String name) { - String ret = null; - - Map options = resourceDef != null ? resourceDef.getMatcherOptions() : null; - - if(options != null && name != null) { - ret = options.get(name); - } - - return ret; - } - - public String getOption(String name, String defaultValue) { - String ret = defaultValue; - String val = getOption(name); - - if(val != null) { - ret = val; - } - - return ret; - } - - public boolean getBooleanOption(String name, boolean defaultValue) { - boolean ret = defaultValue; - String val = getOption(name); - - if(val != null) { - ret = Boolean.parseBoolean(val); - } - - return ret; - } - - public char getCharOption(String name, char defaultValue) { - char ret = defaultValue; - String val = getOption(name); - - if(! StringUtils.isEmpty(val)) { - ret = val.charAt(0); + LOG.debug("<== RangerAbstractResourceMatcher.isCompleteMatch(" + resource + ", " + evalContext + "): " + ret); } return ret; @@ -218,7 +261,7 @@ public StringBuilder toString(StringBuilder sb) { sb.append("options={"); if(resourceDef != null && resourceDef.getMatcherOptions() != null) { for(Map.Entry e : resourceDef.getMatcherOptions().entrySet()) { - sb.append(e.getKey()).append("=").append(e.getValue()).append(OPTIONS_SEP); + sb.append(e.getKey()).append("=").append(e.getValue()).append(';'); } } sb.append("} "); @@ -228,11 +271,6 @@ public StringBuilder toString(StringBuilder sb) { return sb; } - /** - * Is resource asking to authorize all possible values at this level? - * @param resource - * @return - */ boolean isAllValuesRequested(String resource) { boolean result = StringUtils.isEmpty(resource) || WILDCARD_ASTERISK.equals(resource); if (LOG.isDebugEnabled()) { @@ -246,13 +284,191 @@ boolean isAllValuesRequested(String resource) { * - Resource denotes all possible values (i.e. resource in (null, "", "*") * - where as policy does not allow all possible values (i.e. policy.values().contains("*") * - * @param allValuesRequested - * @param resultWithoutExcludes - * @return */ public boolean applyExcludes(boolean allValuesRequested, boolean resultWithoutExcludes) { if (!policyIsExcludes) return resultWithoutExcludes; // not an excludes policy! if (allValuesRequested && !isMatchAny) return resultWithoutExcludes; // one case where excludes has no effect return !resultWithoutExcludes; // all other cases flip it } + + ResourceMatcher getMatcher(String policyValue) { + final int len = policyValue != null ? policyValue.length() : 0; + + if (len == 0) { + return null; + } + + final ResourceMatcher ret; + + int wildcardStartIdx = -1; + int wildcardEndIdx = -1; + boolean needWildcardMatch = false; + + // If optWildcard is true + // If ('?' found or non-contiguous '*'s found in policyValue) + // needWildcardMatch = true + // End + // + // wildcardStartIdx is set to index of first '*' in policyValue or -1 if '*' is not found in policyValue, and + // wildcardEndIdx is set to index of last '*' in policyValue or -1 if '*' is not found in policyValue + // Else + // needWildcardMatch is set to false + // End + if (optWildCard) { + for (int i = 0; i < len; i++) { + final char c = policyValue.charAt(i); + + if (c == '?') { + needWildcardMatch = true; + break; + } else if (c == '*') { + if (wildcardEndIdx == -1 || wildcardEndIdx == (i - 1)) { + wildcardEndIdx = i; + if (wildcardStartIdx == -1) { + wildcardStartIdx = i; + } + } else { + needWildcardMatch = true; + break; + } + } + } + } + + if (needWildcardMatch) { // test?, test*a*, test*a*b, *test*a + ret = optIgnoreCase ? new CaseInsensitiveWildcardMatcher(policyValue) : new CaseSensitiveWildcardMatcher(policyValue); + } else if (wildcardStartIdx == -1) { // test, testa, testab + ret = optIgnoreCase ? new CaseInsensitiveStringMatcher(policyValue) : new CaseSensitiveStringMatcher(policyValue); + } else if (wildcardStartIdx == 0) { // *test, **test, *testa, *testab + String matchStr = policyValue.substring(wildcardEndIdx + 1); + ret = optIgnoreCase ? new CaseInsensitiveEndsWithMatcher(matchStr) : new CaseSensitiveEndsWithMatcher(matchStr); + } else if (wildcardEndIdx != (len - 1)) { // test*a, test*ab + ret = optIgnoreCase ? new CaseInsensitiveWildcardMatcher(policyValue) : new CaseSensitiveWildcardMatcher(policyValue); + } else { // test*, test**, testa*, testab* + String matchStr = policyValue.substring(0, wildcardStartIdx); + ret = optIgnoreCase ? new CaseInsensitiveStartsWithMatcher(matchStr) : new CaseSensitiveStartsWithMatcher(matchStr); + } + + if(optReplaceTokens) { + ret.setDelimiters(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + return ret; + } +} + +final class CaseSensitiveStringMatcher extends ResourceMatcher { + CaseSensitiveStringMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.equals(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 1 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveStringMatcher extends ResourceMatcher { + CaseInsensitiveStringMatcher(String value) { super(value); } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.equalsIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() {return 2 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseSensitiveStartsWithMatcher extends ResourceMatcher { + CaseSensitiveStartsWithMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.startsWith(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 3 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} } + +final class CaseInsensitiveStartsWithMatcher extends ResourceMatcher { + CaseInsensitiveStartsWithMatcher(String value) { super(value); } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.startsWithIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 4 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseSensitiveEndsWithMatcher extends ResourceMatcher { + CaseSensitiveEndsWithMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.endsWith(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 3 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseInsensitiveEndsWithMatcher extends ResourceMatcher { + CaseInsensitiveEndsWithMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return StringUtils.endsWithIgnoreCase(resourceValue, getExpandedValue(evalContext)); + } + int getPriority() { return 4 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class CaseSensitiveWildcardMatcher extends ResourceMatcher { + CaseSensitiveWildcardMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return FilenameUtils.wildcardMatch(resourceValue, getExpandedValue(evalContext), IOCase.SENSITIVE); + } + int getPriority() { return 5 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + + +final class CaseInsensitiveWildcardMatcher extends ResourceMatcher { + CaseInsensitiveWildcardMatcher(String value) { + super(value); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return FilenameUtils.wildcardMatch(resourceValue, getExpandedValue(evalContext), IOCase.INSENSITIVE); + } + int getPriority() {return 6 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0); } +} + +final class ResourceMatcherWrapper { + private final boolean needsDynamicEval; + private final List resourceMatchers; + + ResourceMatcherWrapper() { + this(false, null); + } + + ResourceMatcherWrapper(boolean needsDynamicEval, List resourceMatchers) { + this.needsDynamicEval = needsDynamicEval; + this.resourceMatchers = resourceMatchers; + } + + boolean getNeedsDynamicEval() { + return needsDynamicEval; + } + + List getResourceMatchers() { + return resourceMatchers; + } +} + diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcher.java index 669cf0a112..c1508bf7af 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcher.java @@ -20,20 +20,19 @@ package org.apache.ranger.plugin.resourcematcher; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOCase; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.Map; + public class RangerDefaultResourceMatcher extends RangerAbstractResourceMatcher { private static final Log LOG = LogFactory.getLog(RangerDefaultResourceMatcher.class); @Override - public boolean isMatch(String resource) { + public boolean isMatch(String resource, Map evalContext) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultResourceMatcher.isMatch(" + resource + ")"); + LOG.debug("==> RangerDefaultResourceMatcher.isMatch(" + resource + ", " + evalContext + ")"); } boolean ret = false; @@ -42,19 +41,13 @@ public boolean isMatch(String resource) { if(allValuesRequested || isMatchAny) { ret = isMatchAny; } else { - for(String policyValue : policyValues) { - if(optWildCard) { - ret = optIgnoreCase ? FilenameUtils.wildcardMatch(resource, policyValue, IOCase.INSENSITIVE) - : FilenameUtils.wildcardMatch(resource, policyValue, IOCase.SENSITIVE); - } else { - ret = optIgnoreCase ? StringUtils.equalsIgnoreCase(resource, policyValue) - : StringUtils.equals(resource, policyValue); - } - - if(ret) { + for (ResourceMatcher resourceMatcher : resourceMatchers.getResourceMatchers()) { + ret = resourceMatcher.isMatch(resource, evalContext); + if (ret) { break; } } + } ret = applyExcludes(allValuesRequested, ret); @@ -74,7 +67,7 @@ public boolean isMatch(String resource) { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultResourceMatcher.isMatch(" + resource + "): " + ret); + LOG.debug("<== RangerDefaultResourceMatcher.isMatch(" + resource + ", " + evalContext + "): " + ret); } return ret; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java index 5c555eb9e4..aeb9fb0e17 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java @@ -19,27 +19,29 @@ package org.apache.ranger.plugin.resourcematcher; - -import java.util.ArrayList; -import java.util.List; - +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOCase; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.ServiceDefUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; -public class RangerPathResourceMatcher extends RangerAbstractResourceMatcher { +public class RangerPathResourceMatcher extends RangerDefaultResourceMatcher { private static final Log LOG = LogFactory.getLog(RangerPathResourceMatcher.class); - public static final String OPTION_PATH_SEPERATOR = "pathSeparatorChar"; - public static final char DEFAULT_PATH_SEPERATOR_CHAR = org.apache.hadoop.fs.Path.SEPARATOR_CHAR; + private static final String OPTION_PATH_SEPARATOR = "pathSeparatorChar"; + private static final char DEFAULT_PATH_SEPARATOR_CHAR = org.apache.hadoop.fs.Path.SEPARATOR_CHAR; - private boolean policyIsRecursive = false; - private char pathSeparatorChar = DEFAULT_PATH_SEPERATOR_CHAR; - private List policyValuesForMatch = null; + private boolean policyIsRecursive = false; + private char pathSeparatorChar = '/'; @Override public void init() { @@ -47,76 +49,98 @@ public void init() { LOG.debug("==> RangerPathResourceMatcher.init()"); } - super.init(); + Map options = resourceDef == null ? null : resourceDef.getMatcherOptions(); policyIsRecursive = policyResource == null ? false : policyResource.getIsRecursive(); - pathSeparatorChar = getCharOption(OPTION_PATH_SEPERATOR, DEFAULT_PATH_SEPERATOR_CHAR); + pathSeparatorChar = ServiceDefUtil.getCharOption(options, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR); - if(policyIsRecursive && optWildCard && !isMatchAny) { - policyValuesForMatch = new ArrayList(); + super.init(); - for(String policyValue : policyValues) { - if(policyValue.charAt(policyValue.length() - 1) == pathSeparatorChar) { - policyValuesForMatch.add(policyValue + WILDCARD_ASTERISK); - } else { - policyValuesForMatch.add(policyValue); + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPathResourceMatcher.init()"); + } + } + + @Override + + protected ResourceMatcherWrapper buildResourceMatchers() { + List resourceMatchers = new ArrayList(); + boolean needsDynamicEval = false; + + for (String policyValue : policyValues) { + if (optWildCard && policyIsRecursive) { + if (policyValue.charAt(policyValue.length() - 1) == pathSeparatorChar) { + policyValue += WILDCARD_ASTERISK; } } - } else { - policyValuesForMatch = policyValues; - } - if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerPathResourceMatcher.init()"); + ResourceMatcher matcher = getMatcher(policyValue); + + if (matcher != null) { + if (matcher.isMatchAny()) { + resourceMatchers.clear(); + break; + } + if (!needsDynamicEval && matcher.getNeedsDynamicEval()) { + needsDynamicEval = true; + } + resourceMatchers.add(matcher); + } } + + Collections.sort(resourceMatchers); + + return CollectionUtils.isNotEmpty(resourceMatchers) ? + new ResourceMatcherWrapper(needsDynamicEval, resourceMatchers) : null; } @Override - public boolean isMatch(String resource) { - if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerPathResourceMatcher.isMatch(" + resource + ")"); + ResourceMatcher getMatcher(String policyValue) { + if(! policyIsRecursive) { + return super.getMatcher(policyValue); } - boolean ret = false; - boolean allValuesRequested = isAllValuesRequested(resource); + final int len = policyValue != null ? policyValue.length() : 0; - if(allValuesRequested || isMatchAny) { - ret = isMatchAny; - } else { - IOCase caseSensitivity = optIgnoreCase ? IOCase.INSENSITIVE : IOCase.SENSITIVE; - - for(String policyValue : policyValuesForMatch) { - if(policyIsRecursive && optWildCard) { - ret = isRecursiveWildCardMatch(resource, policyValue, pathSeparatorChar, caseSensitivity); - } else if(policyIsRecursive) { - ret = optIgnoreCase ? StringUtils.startsWithIgnoreCase(resource, policyValue) - : StringUtils.startsWith(resource, policyValue); - } else if(optWildCard) { - ret = FilenameUtils.wildcardMatch(resource, policyValue, caseSensitivity); - } else { - ret = optIgnoreCase ? StringUtils.equalsIgnoreCase(resource, policyValue) - : StringUtils.equals(resource, policyValue); - } + if (len == 0) { + return null; + } + + // To ensure that when policyValue is single '*', ResourceMatcher created here returns true for isMatchAny() + if (optWildCard && policyValue.equals(WILDCARD_ASTERISK)) { + return new CaseInsensitiveStringMatcher(""); + } + + boolean isWildcardPresent = false; + + if (optWildCard) { + for (int i = 0; i < len; i++) { + final char c = policyValue.charAt(i); - if(ret) { + if (c == '?' || c == '*') { + isWildcardPresent = true; break; } } } - ret = applyExcludes(allValuesRequested, ret); + final ResourceMatcher ret; - if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerPathResourceMatcher.isMatch(" + resource + "): " + ret); + if (isWildcardPresent) { + ret = optIgnoreCase ? new CaseInsensitiveRecursiveWildcardMatcher(policyValue, pathSeparatorChar) + : new CaseSensitiveRecursiveWildcardMatcher(policyValue, pathSeparatorChar); + } else { + ret = optIgnoreCase ? new CaseInsensitiveRecursiveMatcher(policyValue, pathSeparatorChar) : new CaseSensitiveRecursiveMatcher(policyValue, pathSeparatorChar); + } + + if (optReplaceTokens) { + ret.setDelimiters(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); } return ret; } - private boolean isRecursiveWildCardMatch(String pathToCheck, String wildcardPath, char pathSeparatorChar, IOCase caseSensitivity) { - if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerPathResourceMatcher.isRecursiveWildCardMatch(" + pathToCheck + ", " + wildcardPath + ", " + pathSeparatorChar + ")"); - } + static boolean isRecursiveWildCardMatch(String pathToCheck, String wildcardPath, char pathSeparatorChar, IOCase caseSensitivity) { boolean ret = false; @@ -147,11 +171,6 @@ private boolean isRecursiveWildCardMatch(String pathToCheck, String wildcardPath ret = FilenameUtils.wildcardMatch(pathToCheck, wildcardPath, caseSensitivity) ; } } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerPathResourceMatcher.isRecursiveWildCardMatch(" + pathToCheck + ", " + wildcardPath + ", " + pathSeparatorChar + "): " + ret); - } - return ret; } @@ -167,3 +186,112 @@ public StringBuilder toString(StringBuilder sb) { return sb; } } + +final class CaseSensitiveRecursiveWildcardMatcher extends ResourceMatcher { + private final char levelSeparatorChar; + CaseSensitiveRecursiveWildcardMatcher(String value, char levelSeparatorChar) { + super(value); + this.levelSeparatorChar = levelSeparatorChar; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return RangerPathResourceMatcher.isRecursiveWildCardMatch(resourceValue, getExpandedValue(evalContext), levelSeparatorChar, IOCase.SENSITIVE); + } + int getPriority() { return 7 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveRecursiveWildcardMatcher extends ResourceMatcher { + private final char levelSeparatorChar; + CaseInsensitiveRecursiveWildcardMatcher(String value, char levelSeparatorChar) { + super(value); + this.levelSeparatorChar = levelSeparatorChar; + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + return RangerPathResourceMatcher.isRecursiveWildCardMatch(resourceValue, getExpandedValue(evalContext), levelSeparatorChar, IOCase.INSENSITIVE); + } + int getPriority() { return 8 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} + +} + +abstract class RecursiveMatcher extends ResourceMatcher { + final char levelSeparatorChar; + String valueWithoutSeparator = null; + String valueWithSeparator = null; + + RecursiveMatcher(String value, char levelSeparatorChar) { + super(value); + this.levelSeparatorChar = levelSeparatorChar; + } + + String getStringToCompare(String policyValue) { + return (policyValue.lastIndexOf(levelSeparatorChar) == policyValue.length()-1) ? + policyValue.substring(0, policyValue.length()-1) : policyValue; + } +} + +final class CaseSensitiveRecursiveMatcher extends RecursiveMatcher { + CaseSensitiveRecursiveMatcher(String value, char levelSeparatorChar) { + super(value, levelSeparatorChar); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + + final String noSeparator; + if (getNeedsDynamicEval()) { + noSeparator = getStringToCompare(getExpandedValue(evalContext)); + } else { + if (valueWithoutSeparator == null) { + valueWithoutSeparator = getStringToCompare(value); + valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); + } + noSeparator = valueWithoutSeparator; + } + + boolean ret = StringUtils.equals(resourceValue, noSeparator); + + if (!ret) { + final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; + ret = StringUtils.startsWith(resourceValue, withSeparator); + } + + return ret; + } + int getPriority() { return 7 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} +} + +final class CaseInsensitiveRecursiveMatcher extends RecursiveMatcher { + CaseInsensitiveRecursiveMatcher(String value, char levelSeparatorChar) { + super(value, levelSeparatorChar); + } + + @Override + boolean isMatch(String resourceValue, Map evalContext) { + + final String noSeparator; + if (getNeedsDynamicEval()) { + noSeparator = getStringToCompare(getExpandedValue(evalContext)); + } else { + if (valueWithoutSeparator == null) { + valueWithoutSeparator = getStringToCompare(value); + valueWithSeparator = valueWithoutSeparator + Character.toString(levelSeparatorChar); + } + noSeparator = valueWithoutSeparator; + } + + boolean ret = StringUtils.equalsIgnoreCase(resourceValue, noSeparator); + + if (!ret) { + final String withSeparator = getNeedsDynamicEval() ? noSeparator + Character.toString(levelSeparatorChar) : valueWithSeparator; + ret = StringUtils.startsWithIgnoreCase(resourceValue, withSeparator); + } + + return ret; + } + + int getPriority() { return 8 + (getNeedsDynamicEval() ? DYNAMIC_EVALUATION_PENALTY : 0);} + +} \ No newline at end of file diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java index e4d3ce5c83..c1d8366230 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java @@ -22,6 +22,8 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; +import java.util.Map; + public interface RangerResourceMatcher { void setResourceDef(RangerResourceDef resourceDef); @@ -29,7 +31,12 @@ public interface RangerResourceMatcher { void init(); - boolean isMatch(String resource); + boolean isMatchAny(); + + boolean isMatch(String resource, Map evalContext); + + boolean isCompleteMatch(String resource, Map evalContext); + + boolean getNeedsDynamicEval(); - boolean isCompleteMatch(String resource); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/ResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/ResourceMatcher.java new file mode 100644 index 0000000000..853c525978 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/ResourceMatcher.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.resourcematcher; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.StringTokenReplacer; + +import java.util.Map; + +abstract class ResourceMatcher implements Comparable { + private static final Log LOG = LogFactory.getLog(ResourceMatcher.class); + + protected final String value; + protected StringTokenReplacer tokenReplacer; + + static final int DYNAMIC_EVALUATION_PENALTY = 8; + + ResourceMatcher(String value) { this.value = value; } + + abstract boolean isMatch(String resourceValue, Map evalContext); + abstract int getPriority(); + + boolean isMatchAny() { return value != null && value.length() == 0; } + + boolean getNeedsDynamicEval() { + return tokenReplacer != null; + } + + @Override + public int compareTo(ResourceMatcher other) { return Integer.compare(getPriority(), other.getPriority()); } + + @Override + public String toString() { + return this.getClass().getName() + "(" + this.value + ")"; + } + + void setDelimiters(char startDelimiterChar, char endDelimiterChar, char escapeChar, String tokenPrefix) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> setDelimiters(value= " + value + ", startDelimiter=" + startDelimiterChar + + ", endDelimiter=" + endDelimiterChar + ", escapeChar=" + escapeChar + ", prefix=" + tokenPrefix); + } + + if(value != null && (value.indexOf(escapeChar) != -1 || (value.indexOf(startDelimiterChar) != -1 && value.indexOf(endDelimiterChar) != -1))) { + tokenReplacer = new StringTokenReplacer(startDelimiterChar, endDelimiterChar, escapeChar, tokenPrefix); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== setDelimiters(value= " + value + ", startDelimiter=" + startDelimiterChar + + ", endDelimiter=" + endDelimiterChar + ", escapeChar=" + escapeChar + ", prefix=" + tokenPrefix); + } + } + + String getExpandedValue(Map evalContext) { + final String ret; + + if(tokenReplacer != null) { + ret = tokenReplacer.replaceTokens(value, evalContext); + } else { + ret = value; + } + + return ret; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index a000125e08..172cb2f163 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -22,6 +22,8 @@ import java.util.Collection; import java.util.Hashtable; import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -30,7 +32,17 @@ import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerServiceDef; -import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor; +import org.apache.ranger.plugin.policyengine.RangerDataMaskResult; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo; +import org.apache.ranger.plugin.policyengine.RangerRowFilterResult; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.PolicyRefresher; @@ -51,6 +63,7 @@ public class RangerBasePlugin { private RangerAccessResultProcessor resultProcessor = null; private boolean useForwardedIPAddress = false; private String[] trustedProxyAddresses = null; + private Timer policyEngineRefreshTimer; Map logHistoryList = new Hashtable(); int logInterval = 30000; // 30 seconds @@ -120,12 +133,39 @@ public void init() { policyEngineOptions.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); policyEngineOptions.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); policyEngineOptions.disableTagPolicyEvaluation = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); + policyEngineOptions.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); RangerAdminClient admin = createAdminClient(serviceName, appId, propertyPrefix); refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir); refresher.setDaemon(true); refresher.startRefresher(); + + long policyReorderIntervalMs = RangerConfiguration.getInstance().getLong(propertyPrefix + ".policy.policyReorderInterval", 60 * 1000); + if (policyReorderIntervalMs >= 0 && policyReorderIntervalMs < 15 * 1000) { + policyReorderIntervalMs = 15 * 1000; + } + + if (LOG.isDebugEnabled()) { + LOG.debug(propertyPrefix + ".policy.policyReorderInterval:" + policyReorderIntervalMs); + } + + if (policyReorderIntervalMs > 0) { + policyEngineRefreshTimer = new Timer("PolicyEngineRefreshTimer", true); + try { + policyEngineRefreshTimer.schedule(new PolicyEngineRefresher(this), policyReorderIntervalMs, policyReorderIntervalMs); + if (LOG.isDebugEnabled()) { + LOG.debug("Scheduled PolicyEngineRefresher to reorder policies nbased on number of evaluations in and every " + policyReorderIntervalMs + " milliseconds"); + } + } catch (IllegalStateException exception) { + LOG.error("Error scheduling policyEngineRefresher:", exception); + LOG.error("*** PolicyEngine will NOT be reorderd based on number of evaluations every " + policyReorderIntervalMs + " milliseconds ***"); + policyEngineRefreshTimer = null; + } + } else { + LOG.info("Policies will NOT be reordered based on number of evaluations because " + + propertyPrefix + ".policy.policyReorderInterval is set to a negative number[" + policyReorderIntervalMs +"]"); + } } public void setPolicies(ServicePolicies policies) { @@ -154,17 +194,25 @@ public void cleanup() { RangerPolicyEngine policyEngine = this.policyEngine; + Timer policyEngineRefreshTimer = this.policyEngineRefreshTimer; + this.serviceName = null; this.policyEngine = null; this.refresher = null; + this.policyEngineRefreshTimer = null; if (refresher != null) { refresher.stopRefresher(); } + if (policyEngineRefreshTimer != null) { + policyEngineRefreshTimer.cancel(); + } + if (policyEngine != null) { policyEngine.cleanup(); } + } public void setResultProcessor(RangerAccessResultProcessor resultProcessor) { @@ -243,16 +291,6 @@ public RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest reques return null; } - public RangerAccessResult createAccessResult(RangerAccessRequest request) { - RangerPolicyEngine policyEngine = this.policyEngine; - - if(policyEngine != null) { - return policyEngine.createAccessResult(request); - } - - return null; - } - public void grantAccess(GrantRevokeRequest request, RangerAccessResultProcessor resultProcessor) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerAdminRESTClient.grantAccess(" + request + ")"); @@ -400,4 +438,19 @@ static class LogHistory { int counter=0; } + static private final class PolicyEngineRefresher extends TimerTask { + private final RangerBasePlugin plugin; + + PolicyEngineRefresher(RangerBasePlugin plugin) { + this.plugin = plugin; + } + + @Override + public void run() { + RangerPolicyEngine policyEngine = plugin.policyEngine; + if (policyEngine != null) { + policyEngine.reorderPolicyEvaluators(); + } + } + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index a20a9f8da9..780d3796e1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -195,7 +195,7 @@ public void deleteTagDefByName(String name) throws Exception { RangerTagDef existing = getTagDefByName(name); - if (existing == null) { + if (existing != null) { try { deleteTagDef(existing); } catch (Exception excp) { @@ -422,7 +422,9 @@ public void deleteTag(Long id) throws Exception { try { RangerTag tag = getTag(id); - deleteTag(tag); + if (tag != null) { + deleteTag(tag); + } } catch (Exception excp) { throw new Exception("failed to delete tag with ID=" + id, excp); } @@ -667,7 +669,9 @@ public void deleteServiceResource(Long id) throws Exception { try { RangerServiceResource resource = getServiceResource(id); - deleteServiceResource(resource); + if (resource != null) { + deleteServiceResource(resource); + } } catch (Exception excp) { throw new Exception("failed to delete service-resource with ID=" + id, excp); } @@ -686,7 +690,9 @@ public void deleteServiceResourceByGuid(String guid) throws Exception { try { RangerServiceResource resource = getServiceResourceByGuid(guid); - deleteServiceResource(resource); + if (resource != null) { + deleteServiceResource(resource); + } } catch (Exception excp) { throw new Exception("failed to delete service-resource with GUID=" + guid, excp); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java index 1cb2175022..4a00d63582 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java @@ -599,9 +599,7 @@ public ClientResponse run() { if(response != null && response.getStatus() == 200) { ret = response.getEntity(ServicePolicies.class); - } else if(response != null && response.getStatus() == 304) { - // no change - } else { + } else if(!(response != null && response.getStatus() == 304)) { RESTResponse resp = RESTResponse.fromClientResponse(response); throw new Exception(resp.getMessage()); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java index ddc42c0a69..3fe96ac70e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -36,7 +36,6 @@ public class PerfDataRecorder { private static volatile PerfDataRecorder instance = null; private Map perfStatistics = new HashMap(); - private boolean initPerfStatisticsOnce = true; public static void initialize(List names) { if (instance == null) { @@ -57,6 +56,13 @@ public static void printStatistics() { instance.dumpStatistics(); } } + + public static void clearStatistics() { + if (instance != null) { + instance.clear(); + } + } + public static void recordStatistic(String tag, long elapsedTime) { if (instance != null) { instance.record(tag, elapsedTime); @@ -73,19 +79,16 @@ private void dumpStatistics() { PerfStatistic perfStatistic = perfStatistics.get(tag); long averageTimeSpent = 0L; - long minTimeSpent = 0L; - long maxTimeSpent = 0L; + if (perfStatistic.numberOfInvocations.get() != 0L) { averageTimeSpent = perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get(); - minTimeSpent = perfStatistic.minTimeSpent.get(); - maxTimeSpent = perfStatistic.maxTimeSpent.get(); } String logMsg = "[" + tag + "]" + - " execCount:" + perfStatistic.numberOfInvocations + - ", totalTimeTaken:" + perfStatistic.millisecondsSpent + - ", maxTimeTaken:" + maxTimeSpent + - ", minTimeTaken:" + minTimeSpent + + " execCount:" + perfStatistic.numberOfInvocations.get() + + ", totalTimeTaken:" + perfStatistic.millisecondsSpent.get() + + ", maxTimeTaken:" + perfStatistic.maxTimeSpent.get() + + ", minTimeTaken:" + perfStatistic.minTimeSpent.get() + ", avgTimeTaken:" + averageTimeSpent; LOG.info(logMsg); @@ -93,23 +96,25 @@ private void dumpStatistics() { } } + private void clear() { + perfStatistics.clear(); + } + private void record(String tag, long elapsedTime) { PerfStatistic perfStatistic = perfStatistics.get(tag); - if (perfStatistic == null && !initPerfStatisticsOnce) { + if (perfStatistic == null) { synchronized (PerfDataRecorder.class) { perfStatistic = perfStatistics.get(tag); - if (perfStatistic == null) { + + if(perfStatistic == null) { perfStatistic = new PerfStatistic(); perfStatistics.put(tag, perfStatistic); } } } - if (perfStatistic != null) { - perfStatistic.addPerfDataItem(elapsedTime); - } - + perfStatistic.addPerfDataItem(elapsedTime); } private PerfDataRecorder(List names) { @@ -118,8 +123,6 @@ private PerfDataRecorder(List names) { // Create structure perfStatistics.put(name, new PerfStatistic()); } - } else { - initPerfStatisticsOnce = false; } } @@ -134,12 +137,12 @@ void addPerfDataItem(final long timeTaken) { millisecondsSpent.getAndAdd(timeTaken); long min = minTimeSpent.get(); - if (timeTaken < min) { + if(timeTaken < min) { minTimeSpent.compareAndSet(min, timeTaken); } long max = maxTimeSpent.get(); - if (timeTaken > max) { + if(timeTaken > max) { maxTimeSpent.compareAndSet(max, timeTaken); } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java index f865a2a42c..38e05d91c0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java @@ -179,6 +179,9 @@ private void loadPolicy() { if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.loadPolicy(serviceName=" + serviceName + ")"); + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory-freeMemory) + ", Free memory:" + freeMemory); } //load policy from PolicyAdmin @@ -195,6 +198,12 @@ private void loadPolicy() { RangerPerfTracer.log(perf); + if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { + long freeMemory = Runtime.getRuntime().freeMemory(); + long totalMemory = Runtime.getRuntime().totalMemory(); + PERF_POLICYENGINE_INIT_LOG.debug("In-Use memory: " + (totalMemory-freeMemory) + ", Free memory:" + freeMemory); + } + if (svcPolicies != null) { plugIn.setPolicies(svcPolicies); policiesSetInPlugin = true; @@ -213,6 +222,12 @@ private ServicePolicies loadPolicyfromPolicyAdmin() { ServicePolicies svcPolicies = null; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.loadPolicyFromPolicyAdmin(serviceName=" + serviceName + ")"); + } + try { svcPolicies = rangerAdmin.getServicePoliciesIfUpdated(lastKnownVersion); @@ -240,7 +255,9 @@ private ServicePolicies loadPolicyfromPolicyAdmin() { LOG.error("PolicyRefresher(serviceName=" + serviceName + "): failed to refresh policies. Will continue to use last known version of policies (" + lastKnownVersion + ")", excp); } - if(LOG.isDebugEnabled()) { + RangerPerfTracer.log(perf); + + if(LOG.isDebugEnabled()) { LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadPolicyfromPolicyAdmin()"); } @@ -261,6 +278,12 @@ private ServicePolicies loadFromCache() { if(cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) { Reader reader = null; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.loadFromCache(serviceName=" + serviceName + ")"); + } + try { reader = new FileReader(cacheFile); @@ -278,6 +301,8 @@ private ServicePolicies loadFromCache() { } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); } finally { + RangerPerfTracer.log(perf); + if(reader != null) { try { reader.close(); @@ -320,6 +345,13 @@ private void saveToCache(ServicePolicies policies) { } if(cacheFile != null) { + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "PolicyRefresher.saveToCache(serviceName=" + serviceName + ")"); + } + Writer writer = null; try { @@ -337,6 +369,9 @@ private void saveToCache(ServicePolicies policies) { } } } + + RangerPerfTracer.log(perf); + } } else { LOG.info("policies is null. Nothing to save in cache"); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java index 0ce3721f5c..2f3a39e63e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java @@ -37,6 +37,8 @@ public class RangerAccessRequestUtil { public static final String KEY_CONTEXT_TAG_OBJECT = "TAG_OBJECT"; public static final String KEY_CONTEXT_RESOURCE = "RESOURCE"; public static final String KEY_CONTEXT_REQUESTED_RESOURCES = "REQUESTED_RESOURCES"; + public static final String KEY_TOKEN_NAMESPACE = "token:"; + public static final String KEY_USER = "USER"; public static void setRequestTagsInContext(Map context, List tags) { if(CollectionUtils.isEmpty(tags)) { @@ -125,4 +127,22 @@ public static Map copyContext(Map context) { return ret; } + + public static void setCurrentUserInContext(Map context, String user) { + setTokenInContext(context, KEY_USER, user); + } + + public static String getCurrentUserFromContext(Map context) { + Object ret = getTokenFromContext(context, KEY_USER); + return ret != null ? ret.toString() : ""; + } + + public static void setTokenInContext(Map context, String tokenName, Object tokenValue) { + String tokenNameWithNamespace = KEY_TOKEN_NAMESPACE + tokenName; + context.put(tokenNameWithNamespace, tokenValue); + } + public static Object getTokenFromContext(Map context, String tokenName) { + String tokenNameWithNamespace = KEY_TOKEN_NAMESPACE + tokenName; + return MapUtils.isNotEmpty(context) ? context.get(tokenNameWithNamespace) : null; + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java index e130cc7ecf..4b17110dfe 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java @@ -71,6 +71,11 @@ public static void log(RangerPerfTracer tracer) { } } + public static void logAlways(RangerPerfTracer tracer) { + if(tracer != null) { + tracer.logAlways(); + } + } public RangerPerfTracer(Log logger, String tag, String data) { this.logger = logger; this.tag = tag; @@ -96,4 +101,8 @@ public void log() { logger.debug("[PERF] " + tag + data + ": " + elapsedTime); } } + public void logAlways() { + long elapsedTime = getElapsedTime(); + logger.debug("[PERF] " + tag + data + ": " + elapsedTime); + } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java index 7cfd040f24..8eb9b27e97 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java @@ -349,7 +349,7 @@ private TrustManager[] getTrustManagers() { private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { try { - if(kmList != null && tmList != null) { + if(tmList != null) { SSLContext sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE); sslContext.init(kmList, tmList, new SecureRandom()); @@ -360,8 +360,9 @@ private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) { LOG.error("SSL algorithm is available in the environment", e); } catch (KeyManagementException e) { LOG.error("Unable to initials the SSLContext", e); + }catch (Exception e) { + LOG.error("Unable to initialize the SSLContext", e); } - return null; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRequestedResources.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRequestedResources.java index 0f10deb698..cb3b84a0b3 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRequestedResources.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRequestedResources.java @@ -31,6 +31,7 @@ import javax.xml.bind.annotation.XmlRootElement; import java.util.ArrayList; import java.util.List; +import java.util.Map; @JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) @@ -67,7 +68,7 @@ public void addRequestedResource(RangerAccessResource requestedResource) { } } - public boolean isMutuallyExcluded(final List matchers) { + public boolean isMutuallyExcluded(final List matchers, final Map evalContext) { boolean ret = true; int matchedCount = 0; @@ -78,7 +79,7 @@ public boolean isMutuallyExcluded(final List matche for (RangerPolicyResourceMatcher matcher : matchers) { - if (matcher.isMatch(resource) && matchedCount++ > 0) { + if (matcher.isMatch(resource, evalContext) && matchedCount++ > 0) { ret = false; break; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java new file mode 100644 index 0000000000..b5c8fb07ef --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class RangerResourceTrie { + private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); + + private static final String DEFAULT_WILDCARD_CHARS = "*?"; + + private final String resourceName; + private final boolean optIgnoreCase; + private final boolean optWildcard; + private final String wildcardChars; + private final TrieNode root; + + public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List evaluators) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")"); + } + + Map matcherOptions = resourceDef.getMatcherOptions(); + + boolean optReplaceTokens = RangerAbstractResourceMatcher.getOptionReplaceTokens(matcherOptions); + + String tokenReplaceSpecialChars = ""; + + if(optReplaceTokens) { + char delimiterStart = RangerAbstractResourceMatcher.getOptionDelimiterStart(matcherOptions); + char delimiterEnd = RangerAbstractResourceMatcher.getOptionDelimiterEnd(matcherOptions); + char delimiterEscape = RangerAbstractResourceMatcher.getOptionDelimiterEscape(matcherOptions); + + tokenReplaceSpecialChars += delimiterStart; + tokenReplaceSpecialChars += delimiterEnd; + tokenReplaceSpecialChars += delimiterEscape; + } + + this.resourceName = resourceDef.getName(); + this.optIgnoreCase = RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions); + this.optWildcard = RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions); + this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS + tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars; + this.root = new TrieNode(Character.valueOf((char)0)); + + for(T evaluator : evaluators) { + Map policyResources = evaluator.getPolicyResource(); + RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null; + + if(policyResource == null) { + if(evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) { + root.addWildcardEvaluator(evaluator); + } + + continue; + } + + if(policyResource.getIsExcludes()) { + root.addWildcardEvaluator(evaluator); + } else { + RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName); + + if(resourceMatcher != null && (resourceMatcher.isMatchAny())) { + root.addWildcardEvaluator(evaluator); + } else { + if(CollectionUtils.isNotEmpty(policyResource.getValues())) { + for (String resource : policyResource.getValues()) { + insert(resource, policyResource.getIsRecursive(), evaluator); + } + } + } + } + } + + root.postSetup(null); + + LOG.info(toString()); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString()); + } + } + + public String getResourceName() { + return resourceName; + } + + public List getEvaluatorsForResource(String resource) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ")"); + } + + List ret = null; + + TrieNode curr = root; + + final int len = resource.length(); + for(int i = 0; i < len; i++) { + Character ch = getLookupChar(resource.charAt(i)); + TrieNode child = curr.getChild(ch); + + if(child == null) { + ret = curr.getWildcardEvaluators(); + curr = null; // so that curr.getEvaluators() will not be called below + break; + } + + curr = child; + } + + if(ret == null) { + if(curr != null) { + ret = curr.getEvaluators(); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + public TrieData getTrieData() { + TrieData ret = new TrieData(); + + root.populateTrieData(ret); + ret.maxDepth = getMaxDepth(); + + return ret; + } + + public int getMaxDepth() { + return root.getMaxDepth(); + } + + public void reorderEvaluators() { + root.reorderEvaluators(null); + } + + private final Character getLookupChar(char ch) { + if(optIgnoreCase) { + ch = Character.toLowerCase(ch); + } + + return Character.valueOf(ch); + } + + private void insert(String resource, boolean isRecursive, T evaluator) { + TrieNode curr = root; + boolean isWildcard = false; + + final int len = resource.length(); + for(int i = 0; i < len; i++) { + Character ch = getLookupChar(resource.charAt(i)); + + if(optWildcard) { + if (wildcardChars.indexOf(ch) != -1) { + isWildcard = true; + break; + } + } + + curr = curr.getOrCreateChild(ch); + } + + if(isWildcard || isRecursive) { + curr.addWildcardEvaluator(evaluator); + } else { + curr.addEvaluator(evaluator); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + TrieData trieData = getTrieData(); + + sb.append("resourceName=").append(resourceName); + sb.append("; optIgnoreCase=").append(optIgnoreCase); + sb.append("; optWildcard=").append(optWildcard); + sb.append("; wildcardChars=").append(wildcardChars); + sb.append("; nodeCount=").append(trieData.nodeCount); + sb.append("; leafNodeCount=").append(trieData.leafNodeCount); + sb.append("; singleChildNodeCount=").append(trieData.singleChildNodeCount); + sb.append("; maxDepth=").append(trieData.maxDepth); + sb.append("; evaluatorListCount=").append(trieData.evaluatorListCount); + sb.append("; wildcardEvaluatorListCount=").append(trieData.wildcardEvaluatorListCount); + sb.append("; evaluatorListRefCount=").append(trieData.evaluatorListRefCount); + sb.append("; wildcardEvaluatorListRefCount=").append(trieData.wildcardEvaluatorListRefCount); + + return sb.toString(); + } + + public class TrieData { + int nodeCount = 0; + int leafNodeCount = 0; + int singleChildNodeCount = 0; + int maxDepth = 0; + int evaluatorListCount = 0; + int wildcardEvaluatorListCount = 0; + int evaluatorListRefCount = 0; + int wildcardEvaluatorListRefCount = 0; + } +} + +class TrieNode { + private final Character c; + private Map children = null; + private List evaluators = null; + private List wildcardEvaluators = null; + private boolean isSharingParentWildcardEvaluators = false; + + TrieNode(Character c) { + this.c = c; + } + + Character getChar() { + return c; + } + + Map getChildren() { + return children; + } + + List getEvaluators() { + return evaluators; + } + + List getWildcardEvaluators() { + return wildcardEvaluators; + } + + TrieNode getChild(Character c) { + TrieNode ret = children == null ? null : children.get(c); + + return ret; + } + + void populateTrieData(RangerResourceTrie.TrieData trieData) { + trieData.nodeCount++; + + if(wildcardEvaluators != null) { + if(isSharingParentWildcardEvaluators) { + trieData.wildcardEvaluatorListRefCount++; + } else { + trieData.wildcardEvaluatorListCount++; + } + } + + if(evaluators != null) { + if(evaluators == wildcardEvaluators) { + trieData.evaluatorListRefCount++; + } else { + trieData.evaluatorListCount++; + } + } + + if(children != null && children.size() > 0) { + if(children.size() == 1) { + trieData.singleChildNodeCount++; + } + + for(Map.Entry entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.populateTrieData(trieData); + } + } else { + trieData.leafNodeCount++; + } + } + + int getMaxDepth() { + int ret = 0; + + if(children != null) { + for(Map.Entry entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + int maxChildDepth = child.getMaxDepth(); + + if(maxChildDepth > ret) { + ret = maxChildDepth; + } + } + } + + return ret + 1; + } + + TrieNode getOrCreateChild(Character c) { + if(children == null) { + children = new HashMap(); + } + + TrieNode child = children.get(c); + + if(child == null) { + child = new TrieNode(c); + children.put(c, child); + } + + return child; + } + + void addEvaluator(T evaluator) { + if(evaluators == null) { + evaluators = new ArrayList(); + } + + if(!evaluators.contains(evaluator)) { + evaluators.add(evaluator); + } + } + + void addWildcardEvaluator(T evaluator) { + if(wildcardEvaluators == null) { + wildcardEvaluators = new ArrayList(); + } + + if(!wildcardEvaluators.contains(evaluator)) { + wildcardEvaluators.add(evaluator); + } + } + + void postSetup(List parentWildcardEvaluators) { + // finalize wildcard-evaluators list by including parent's wildcard evaluators + if(parentWildcardEvaluators != null) { + if(CollectionUtils.isEmpty(this.wildcardEvaluators)) { + this.wildcardEvaluators = parentWildcardEvaluators; + } else { + for (T evaluator : parentWildcardEvaluators) { + addWildcardEvaluator(evaluator); + } + } + } + this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; + + // finalize evaluators list by including wildcard evaluators + if(wildcardEvaluators != null) { + if(CollectionUtils.isEmpty(this.evaluators)) { + this.evaluators = wildcardEvaluators; + } else { + for (T evaluator : wildcardEvaluators) { + addEvaluator(evaluator); + } + } + } + + if(!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { + Collections.sort(wildcardEvaluators); + } + + if(evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { + Collections.sort(evaluators); + } + + if(children != null) { + for(Map.Entry entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.postSetup(wildcardEvaluators); + } + } + } + + void reorderEvaluators(List parentWildcardEvaluators) { + boolean isEvaluatorsSameAsWildcardEvaluators = evaluators == wildcardEvaluators; + + if(isSharingParentWildcardEvaluators) { + wildcardEvaluators = parentWildcardEvaluators; + } else { + wildcardEvaluators = getSortedCopy(wildcardEvaluators); + } + + if(isEvaluatorsSameAsWildcardEvaluators) { + evaluators = wildcardEvaluators; + } else { + evaluators = getSortedCopy(evaluators); + } + + if(children != null) { + for(Map.Entry entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.reorderEvaluators(wildcardEvaluators); + } + } + } + + public void toString(String prefix, StringBuilder sb) { + String nodeValue = prefix; + + if(c != 0) { + nodeValue += c; + } + + sb.append("nodeValue=").append(nodeValue); + sb.append("; childCount=").append(children == null ? 0 : children.size()); + sb.append("; evaluators=[ "); + if(evaluators != null) { + for(T evaluator : evaluators) { + sb.append(evaluator.getId()).append(" "); + } + } + sb.append("]"); + + sb.append("; wildcardEvaluators=[ "); + if(wildcardEvaluators != null) { + for(T evaluator : wildcardEvaluators) { + sb.append(evaluator.getId()).append(" "); + } + } + sb.append("]"); + sb.append(Character.LINE_SEPARATOR); + + if(children != null) { + for(Map.Entry entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.toString(nodeValue, sb); + } + } + } + + public void clear() { + children = null; + evaluators = null; + wildcardEvaluators = null; + } + + private List getSortedCopy(List evaluators) { + final List ret; + + if(CollectionUtils.isNotEmpty(evaluators)) { + ret = new ArrayList(evaluators); + + Collections.sort(ret); + } else { + ret = evaluators; + } + + return ret; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index cac018127d..038c1c1be6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -96,18 +96,6 @@ public String getParam(String name) { return params == null ? null : params.get(name); } - public Long getParamAsLong(String name) { - - String stringValue = params == null ? null : params.get(name); - Long ret = null; - try { - ret = Long.valueOf(stringValue); - } catch (NumberFormatException exception) { - // Ignore - } - return ret; - } - public void setParam(String name, String value) { if(StringUtils.isEmpty(name) || StringUtils.isEmpty(value)) { return; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java index eaf60b799f..dbdc935015 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java @@ -22,6 +22,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; @@ -73,6 +74,64 @@ public static RangerServiceDef normalize(RangerServiceDef serviceDef) { return serviceDef; } + public static RangerResourceDef getResourceDef(RangerServiceDef serviceDef, String resource) { + RangerResourceDef ret = null; + + if(serviceDef != null && resource != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) { + for(RangerResourceDef resourceDef : serviceDef.getResources()) { + if(StringUtils.equalsIgnoreCase(resourceDef.getName(), resource)) { + ret = resourceDef; + break; + } + } + } + + return ret; + } + + public static Integer getLeafResourceLevel(RangerServiceDef serviceDef, Map policyResource) { + Integer ret = null; + + if(serviceDef != null && policyResource != null) { + for(Map.Entry entry : policyResource.entrySet()) { + String resource = entry.getKey(); + RangerResourceDef resourceDef = ServiceDefUtil.getResourceDef(serviceDef, resource); + + if(resourceDef != null && resourceDef.getLevel() != null) { + if(ret == null) { + ret = resourceDef.getLevel(); + } else if(ret < resourceDef.getLevel()) { + ret = resourceDef.getLevel(); + } + } + } + } + + return ret; + } + + public static String getOption(Map options, String name, String defaultValue) { + String ret = options != null && name != null ? options.get(name) : null; + + if(ret == null) { + ret = defaultValue; + } + + return ret; + } + + public static boolean getBooleanOption(Map options, String name, boolean defaultValue) { + String val = getOption(options, name, null); + + return val == null ? defaultValue : Boolean.parseBoolean(val); + } + + public static char getCharOption(Map options, String name, char defaultValue) { + String val = getOption(options, name, null); + + return StringUtils.isEmpty(val) ? defaultValue : val.charAt(0); + } + private static void normalizeDataMaskDef(RangerServiceDef serviceDef) { if(serviceDef != null && serviceDef.getDataMaskDef() != null) { List dataMaskResources = serviceDef.getDataMaskDef().getResources(); @@ -245,4 +304,5 @@ private static boolean getBooleanValue(Map map, String elementNa return ret; } + } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java index d450af1332..3764d1c8f6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java @@ -29,6 +29,7 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; @@ -50,6 +51,7 @@ public class ServicePolicies implements java.io.Serializable { private Date policyUpdateTime; private List policies; private RangerServiceDef serviceDef; + private String auditMode = RangerPolicyEngine.AUDIT_DEFAULT; private TagPolicies tagPolicies; /** @@ -124,6 +126,14 @@ public RangerServiceDef getServiceDef() { public void setServiceDef(RangerServiceDef serviceDef) { this.serviceDef = serviceDef; } + + public String getAuditMode() { + return auditMode; + } + + public void setAuditMode(String auditMode) { + this.auditMode = auditMode; + } /** * @return the tagPolicies */ @@ -146,6 +156,7 @@ public String toString() { .add("policyUpdateTime", policyUpdateTime) .add("policies", policies) .add("serviceDef", serviceDef) + .add("auditMode", auditMode) .add("tagPolicies", tagPolicies) .toString(); } @@ -164,6 +175,7 @@ public static class TagPolicies implements java.io.Serializable { private Date policyUpdateTime; private List policies; private RangerServiceDef serviceDef; + private String auditMode = RangerPolicyEngine.AUDIT_DEFAULT; /** * @return the serviceName */ @@ -237,16 +249,25 @@ public void setServiceDef(RangerServiceDef serviceDef) { this.serviceDef = serviceDef; } + public String getAuditMode() { + return auditMode; + } + + public void setAuditMode(String auditMode) { + this.auditMode = auditMode; + } + @Override public String toString() { return Objects.toStringHelper(this.getClass()) - .add("serviceName", serviceName) - .add("serviceId", serviceId) - .add("policyVersion", policyVersion) - .add("policyUpdateTime", policyUpdateTime) - .add("policies", policies) - .add("serviceDef", serviceDef) - .toString(); + .add("serviceName", serviceName) + .add("serviceId", serviceId) + .add("policyVersion", policyVersion) + .add("policyUpdateTime", policyUpdateTime) + .add("policies", policies) + .add("serviceDef", serviceDef) + .add("auditMode", auditMode) + .toString(); } } } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java new file mode 100644 index 0000000000..f7047f33c6 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import java.util.HashMap; +import java.util.Map; + +public class StringTokenReplacer { + private final char startChar; + private final char endChar; + private final char escapeChar; + private final String tokenPrefix; + + public StringTokenReplacer(char startChar, char endChar, char escapeChar, String tokenPrefix) { + this.startChar = startChar; + this.endChar = endChar; + this.escapeChar = escapeChar; + this.tokenPrefix = tokenPrefix; + } + + public String replaceTokens(String value, Map tokens) { + if(tokens == null || tokens.size() < 1 || value == null || value.length() < 1 || + (value.indexOf(startChar) == -1 && value.indexOf(endChar) == -1 && value.indexOf(escapeChar) == -1)) { + return value; + } + + StringBuilder ret = new StringBuilder(); + StringBuilder token = null; + + for(int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + + if(c == escapeChar) { + i++; + if(i < value.length()) { + c = value.charAt(i); + if(token != null) { + token.append(c); + } else { + ret.append(c); + } + } + continue; + } + + if(token == null) { // not in token + if(c == startChar) { + token = new StringBuilder(); + } else { + ret.append(c); + } + } else { // in token + if(c == endChar) { + String rawToken = token.toString(); + if (tokenPrefix.length() == 0 || rawToken.startsWith(tokenPrefix)) { + Object replaced = RangerAccessRequestUtil.getTokenFromContext(tokens, rawToken.substring(tokenPrefix.length())); + if (replaced != null) { + ret.append(replaced.toString()); + } + } else { + ret.append(startChar).append(token).append(endChar); + } + token = null; + } else { + token.append(c); + } + } + } + + if(token != null) { // if no endChar is found + ret.append(startChar).append(token); + } + + return ret.toString(); + } + + public static void main(String[] args) { + char startChar = '%'; + char endChar = '%'; + char escapeChar = '\\'; + String tokenPrefix = "ranger:"; + Map tokens = new HashMap(); + + tokens.put("USER", "testUser"); + tokens.put("COUNTRY", "USA"); + tokens.put("STATE", "CA"); + tokens.put("CITY", "Santa Clara"); + + StringTokenReplacer tokenReplacer = new StringTokenReplacer(startChar, endChar, escapeChar, tokenPrefix); + + if(args.length == 0) { + args = new String[] { + "/home/%USER%/*", + "/home/%ranger:USER%/*", + "tmp_%USER%", + "tmp_%ranger:USER%", + "%USER%_db", + "%ranger:USER%_db", + "\\%USER_db", + "\\%ranger:USER_db", + "\\%USER%_db", + "\\%ranger:USER%_db", + }; + } + + for(String str : args) { + System.out.println(str + " ==> " + tokenReplacer.replaceTokens(str, tokens)); + } + }} diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java new file mode 100644 index 0000000000..30190aba4f --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import org.apache.ranger.plugin.contextenricher.TestTagEnricher.TagEnricherTestCase.TestData; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.util.RangerAccessRequestUtil; +import org.apache.ranger.plugin.util.ServiceTags; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTagEnricher { + static Gson gsonBuilder = null; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer()) + .create(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void testTagEnricher_hive() { + String[] hiveTestResourceFiles = { "/contextenricher/test_tagenricher_hive.json" }; + + runTestsFromResourceFiles(hiveTestResourceFiles); + } + + private void runTestsFromResourceFiles(String[] resourceNames) { + for(String resourceName : resourceNames) { + InputStream inStream = this.getClass().getResourceAsStream(resourceName); + InputStreamReader reader = new InputStreamReader(inStream); + + runTests(reader, resourceName); + } + } + + private void runTests(InputStreamReader reader, String testName) { + TagEnricherTestCase testCase = gsonBuilder.fromJson(reader, TagEnricherTestCase.class); + + assertTrue("invalid input: " + testName, testCase != null && testCase.serviceDef != null && testCase.serviceResources != null && testCase.tests != null); + + ServiceTags serviceTags = new ServiceTags(); + serviceTags.setServiceName(testCase.serviceName); + serviceTags.setTagDefinitions(testCase.tagDefinitions); + serviceTags.setTags(testCase.tags); + serviceTags.setServiceResources(testCase.serviceResources); + serviceTags.setResourceToTagIds(testCase.resourceToTagIds); + + RangerTagEnricher tagEnricher = new RangerTagEnricher(); + + tagEnricher.setServiceName(testCase.serviceName); + tagEnricher.setServiceDef(testCase.serviceDef); + tagEnricher.setServiceTags(serviceTags); + + List expectedTags = new ArrayList(); + List resultTags = new ArrayList(); + + + for (TestData test : testCase.tests) { + RangerAccessRequestImpl request = new RangerAccessRequestImpl(test.resource, "", "testUser", null); + + tagEnricher.enrich(request); + + List expected = test.result; + List result = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + expectedTags.clear(); + if(expected != null) { + for (RangerTag tag : expected) { + expectedTags.add(tag.getType()); + } + Collections.sort(expectedTags); + } + + resultTags.clear(); + if(result != null) { + for(RangerTag tag : result) { + resultTags.add(tag.getType()); + } + Collections.sort(resultTags); + } + + assertEquals(test.name, expectedTags, resultTags); + } + } + + static class TagEnricherTestCase { + public String serviceName; + public RangerServiceDef serviceDef; + public Map tagDefinitions; + public Map tags; + public List serviceResources; + public Map> resourceToTagIds; + public List tests; + + + class TestData { + public String name; + public RangerAccessResource resource; + public List result; + } + } + + static class RangerResourceDeserializer implements JsonDeserializer { + @Override + public RangerAccessResource deserialize(JsonElement jsonObj, Type type, + JsonDeserializationContext context) throws JsonParseException { + return gsonBuilder.fromJson(jsonObj, RangerAccessResourceImpl.class); + } + } +} diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index dd15ff869d..aebc869775 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -58,7 +58,6 @@ public class TestPolicyEngine { - static RangerPolicyEngine policyEngine = null; static Gson gsonBuilder = null; @BeforeClass @@ -159,6 +158,13 @@ public static void setUpBeforeClass() throws Exception { public static void tearDownAfterClass() throws Exception { } + @Test + public void testPolicyEngine_hdfs_resourcespec() { + String[] hdfsTestResourceFiles = { "/policyengine/test_policyengine_hdfs_resourcespec.json" }; + + runTestsFromResourceFiles(hdfsTestResourceFiles); + } + @Test public void testPolicyEngine_hdfs() { String[] hdfsTestResourceFiles = { "/policyengine/test_policyengine_hdfs.json" }; @@ -166,6 +172,20 @@ public void testPolicyEngine_hdfs() { runTestsFromResourceFiles(hdfsTestResourceFiles); } + @Test + public void testPolicyEngine_hdfs_allaudit() { + String[] hdfsTestResourceFiles = { "/policyengine/test_policyengine_hdfs_allaudit.json" }; + + runTestsFromResourceFiles(hdfsTestResourceFiles); + } + + @Test + public void testPolicyEngine_hdfs_noaudit() { + String[] hdfsTestResourceFiles = { "/policyengine/test_policyengine_hdfs_noaudit.json" }; + + runTestsFromResourceFiles(hdfsTestResourceFiles); + } + @Test public void testPolicyEngine_hdfsForTag() { String[] hdfsTestResourceFiles = { "/policyengine/test_policyengine_tag_hdfs.json" }; @@ -194,6 +214,13 @@ public void testPolicyEngine_hbase() { runTestsFromResourceFiles(hbaseTestResourceFiles); } + @Test + public void testPolicyEngine_hbase_with_multiple_matching_policies() { + String[] hbaseTestResourceFiles = { "/policyengine/test_policyengine_hbase_multiple_matching_policies.json" }; + + runTestsFromResourceFiles(hbaseTestResourceFiles); + } + @Test public void testPolicyEngine_conditions() { String[] conditionsTestResourceFiles = { "/policyengine/test_policyengine_conditions.json" }; @@ -255,12 +282,19 @@ private void runTests(InputStreamReader reader, String testName) { servicePolicies.setServiceDef(testCase.serviceDef); servicePolicies.setPolicies(testCase.policies); + if (StringUtils.isNotBlank(testCase.auditMode)) { + servicePolicies.setAuditMode(testCase.auditMode); + } + if (null != testCase.tagPolicyInfo) { ServicePolicies.TagPolicies tagPolicies = new ServicePolicies.TagPolicies(); tagPolicies.setServiceName(testCase.tagPolicyInfo.serviceName); tagPolicies.setServiceDef(testCase.tagPolicyInfo.serviceDef); tagPolicies.setPolicies(testCase.tagPolicyInfo.tagPolicies); + if (StringUtils.isNotBlank(testCase.auditMode)) { + tagPolicies.setAuditMode(testCase.auditMode); + } servicePolicies.setTagPolicies(tagPolicies); } @@ -276,14 +310,18 @@ private void runTests(InputStreamReader reader, String testName) { trustedProxyAddresses[i] = trustedProxyAddresses[i].trim(); } } - policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); + RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); + long requestCount = 0L; RangerAccessRequest request = null; for(TestData test : testCase.tests) { request = test.request; + if ((requestCount++ % 10) == 1) { + policyEngine.reorderPolicyEvaluators(); + } if (request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_TAGS) || request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { // Create a new AccessRequest @@ -403,6 +441,7 @@ static class PolicyEngineTestCase { public RangerServiceDef serviceDef; public List policies; public TagPolicyInfo tagPolicyInfo; + public String auditMode; public List tests; class TestData { diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcherTest.java b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcherTest.java index 48bc6ee2af..e2c7c27087 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcherTest.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcherTest.java @@ -21,6 +21,8 @@ import org.junit.Test; +import java.util.Map; + import static org.junit.Assert.*; public class RangerAbstractResourceMatcherTest { @@ -40,9 +42,9 @@ public void test_isAllPossibleValues() { static class AbstractMatcherWrapper extends RangerAbstractResourceMatcher { @Override - public boolean isMatch(String resource) { + public boolean isMatch(String resource, Map evalContext) { fail("This method is not expected to be used by test!"); return false; } } -} \ No newline at end of file +} diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcherTest.java b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcherTest.java index 5576a098f2..75320611b3 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcherTest.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerDefaultResourceMatcherTest.java @@ -20,24 +20,29 @@ package org.apache.ranger.plugin.resourcematcher; import com.google.common.collect.Lists; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + import static org.junit.Assert.*; public class RangerDefaultResourceMatcherTest { Object[][] data = { // { resource, policy, excludes, result - { "*", "*", false, true }, // resource is all values - { "*", "*", true, false }, - { "*", "a*", false, false }, // but, policy is not match any - { "*", "a*", true, false }, // ==> compare with above: exclude flag has no effect here - { "a*", "a", false, false }, // resource has regex marker! - { "a*", "a", true, true }, - { "a", "a", false, true }, // exact match - { "a", "a", true, false }, - { "a1", "a*", false, true }, // trivial regex match - { "a1", "a*", true, false }, + { "*", "*", false, true, "user" }, // resource is all values + { "*", "*", true, false, "user" }, + { "*", "a*", false, false, "user" }, // but, policy is not match any + { "*", "a*", true, false, "user" }, // ==> compare with above: exclude flag has no effect here + { "a*", "a", false, false, "user" }, // resource has regex marker! + { "a*", "a", true, true, "user" }, + { "a", "a", false, true, "user" }, // exact match + { "a", "a", true, false, "user" }, + { "a1", "a*", false, true, "user" }, // trivial regex match + { "a1", "a*", true, false, "user" }, }; @Test @@ -47,9 +52,13 @@ public void testIsMatch() throws Exception { String policyValue = (String)row[1]; boolean excludes = (boolean)row[2]; boolean result = (boolean)row[3]; + String user = (String) row[4]; + + Map evalContext = new HashMap(); + RangerAccessRequestUtil.setCurrentUserInContext(evalContext, user); MatcherWrapper matcher = new MatcherWrapper(policyValue, excludes); - assertEquals(getMessage(row), result, matcher.isMatch(resource)); + assertEquals(getMessage(row), result, matcher.isMatch(resource, evalContext)); } } @@ -60,14 +69,16 @@ String getMessage(Object[] row) { static class MatcherWrapper extends RangerDefaultResourceMatcher { MatcherWrapper(String policyValue, boolean exclude) { - this.policyValues = Lists.newArrayList(policyValue); - if (WILDCARD_ASTERISK.equals(policyValue)) { - this.isMatchAny = true; - } + RangerPolicy.RangerPolicyResource policyResource = new RangerPolicy.RangerPolicyResource(); + policyResource.setIsExcludes(exclude); + policyResource.setValues(Lists.newArrayList(policyValue)); + setPolicyResource(policyResource); + if (policyValue.contains(WILDCARD_ASTERISK)) { this.optWildCard = true; } - this.policyIsExcludes = exclude; + this.optIgnoreCase = false; + init(); } } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java new file mode 100644 index 0000000000..da81d81a28 --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.resourcematcher; + +import com.google.common.collect.Lists; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.util.RangerAccessRequestUtil; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class RangerPathResourceMatcherTest { + + Object[][] data = { + // { resource, policy, optWildcard, recursive, result + { "/app/hive/test.db", "/", true, false, false, "user" }, + { "/app/hive/test.db", "/", true, true, true, "user" }, + { "/app/hive/test.db", "/*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/*", true, false, true, "user" }, + { "/app/hive/test.db", "/app", true, false, false, "user" }, + { "/app/hive/test.db", "/app/", true, false, false, "user" }, + { "/app/hive/test.db", "/app/", true, true, true, "user" }, + { "/app/hive/test.db", "/app/*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/*", true, false, true, "user" }, + { "/app/hive/test.db", "/app/hive/*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/hive/*", true, false, false, "user" }, + { "/app/hive/test.db", "/app/hive/test*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/hive/test*", true, false, false, "user" }, + { "/app/hive/test.db", "/app/hive/test.db", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/hive/test.db", true, false, false, "user" }, + }; + + @Test + public void testIsMatch() throws Exception { + for (Object[] row : data) { + String resource = (String)row[0]; + String policyValue = (String)row[1]; + boolean optWildcard = (boolean)row[2]; + boolean isRecursive = (boolean)row[3]; + boolean result = (boolean)row[4]; + String user = (String) row[5]; + + Map evalContext = new HashMap(); + RangerAccessRequestUtil.setCurrentUserInContext(evalContext, user); + + MatcherWrapper matcher = new MatcherWrapper(policyValue, optWildcard, isRecursive); + assertEquals(getMessage(row), result, matcher.isMatch(resource, evalContext)); + } + } + + String getMessage(Object[] row) { + return String.format("Resource=%s, Policy=%s, optWildcard=%s, recursive=%s, result=%s", + (String)row[0], (String)row[1], (boolean)row[2], (boolean)row[3], (boolean)row[4]); + } + + static class MatcherWrapper extends RangerPathResourceMatcher { + MatcherWrapper(String policyValue, boolean optWildcard, boolean isRecursive) { + super.optWildCard = optWildcard; + + RangerPolicy.RangerPolicyResource policyResource = new RangerPolicy.RangerPolicyResource(); + policyResource.setIsRecursive(isRecursive); + policyResource.setValues(Lists.newArrayList(policyValue)); + setPolicyResource(policyResource); + + init(); + } + } + +} \ No newline at end of file diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/TestResourceMatcher.java b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/TestResourceMatcher.java index 9b870d461c..2cb8fde1e9 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/TestResourceMatcher.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/TestResourceMatcher.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.List; +import java.util.Map; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; @@ -39,7 +40,7 @@ import com.google.gson.GsonBuilder; public class TestResourceMatcher { - static Gson gsonBuilder = null; + static Gson gsonBuilder = null; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -74,6 +75,20 @@ public void testResourceMatcher_path() throws Exception { runTestsFromResourceFiles(tests); } + @Test + public void testResourceMatcher_dynamic() throws Exception { + String[] tests = { "/resourcematcher/test_resourcematcher_dynamic.json"}; + + runTestsFromResourceFiles(tests); + } + + @Test + public void testResourceMatcher_wildcards_as_delimiters() throws Exception { + String[] tests = { "/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json"}; + + runTestsFromResourceFiles(tests); + } + private void runTestsFromResourceFiles(String[] resourceNames) throws Exception { for(String resourceName : resourceNames) { InputStream inStream = this.getClass().getResourceAsStream(resourceName); @@ -97,7 +112,7 @@ private void runTests(InputStreamReader reader, String testName) throws Exceptio } boolean expected = oneTest.result; - boolean result = matcher.isMatch(oneTest.input); + boolean result = matcher.isMatch(oneTest.input, oneTest.evalContext); assertEquals("isMatch() failed! " + testCase.name + ":" + oneTest.name + ": input=" + oneTest.input, expected, result); } @@ -130,6 +145,7 @@ class TestCase { class OneTest { String name; String input; + Map evalContext; boolean result; } } diff --git a/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json new file mode 100644 index 0000000000..317c651643 --- /dev/null +++ b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json @@ -0,0 +1,77 @@ +{ + "serviceName":"hivedev", + + "serviceDef":{ + "name":"hive", + "id":3, + "resources":[ + {"name":"database","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Database","description":"Hive Database"}, + {"name":"table","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Table","description":"Hive Table"}, + {"name":"udf","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive UDF","description":"Hive UDF"}, + {"name":"column","level":3,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Column","description":"Hive Column"} + ], + "accessTypes":[ + {"name":"select","label":"Select"}, + {"name":"update","label":"Update"}, + {"name":"create","label":"Create"}, + {"name":"drop","label":"Drop"}, + {"name":"alter","label":"Alter"}, + {"name":"index","label":"Index"}, + {"name":"lock","label":"Lock"}, + {"name":"all","label":"All"} + ] + }, + + "tagDefinitions": { + "1":{"name":"PII"}, + "2":{"name":"EXPIRES_ON","attributeDefs":[{"name":"expiry_date","type":"date"}]}, + "3":{"name":"FINANCE"} + }, + + "tags": { + "1":{"type":"PII"}, + "2":{"type":"EXPIRES_ON","attributes":{"expiry_date":"2015/12/31"}}, + "3":{"type":"FINANCE"} + }, + + "serviceResources": [ + {"id":1,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["hr"]},"table":{"values":["employee"]},"column":{"values":["ssn"]}}}, + {"id":2,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]}}}, + {"id":3,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]},"table":{"values":["tax_2010"]}}}, + {"id":4,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]},"table":{"values":["tax_2010"]},"column":{"values":["ssn"]}}} + ], + + "resourceToTagIds": { + "1":[1], + "2":[3], + "3":[2], + "4":[1] + }, + + "tests":[ + {"name":"hr.employee.ssn", + "resource":{"elements":{"database":"hr","table":"employee","column":"ssn"}}, + "result":[{"type":"PII"}] + } + , + {"name":"hr.employee.id", + "resource":{"elements":{"database":"hr","table":"employee","column":"id"}}, + "result":[] + } + , + {"name":"finance.tax_2010", + "resource":{"elements":{"database":"finance","table":"tax_2010"}}, + "result":[{"type":"EXPIRES_ON"},{"type":"FINANCE"}] + } + , + {"name":"finance.tax_2010.ssn", + "resource":{"elements":{"database":"finance","table":"tax_2010","column":"ssn"}}, + "result":[{"type":"EXPIRES_ON"},{"type":"PII"},{"type":"FINANCE"}] + } + , + {"name":"finance.tax_2010.id", + "resource":{"elements":{"database":"finance","table":"tax_2010","column":"id"}}, + "result":[{"type":"EXPIRES_ON"},{"type":"FINANCE"}] + } + ] +} diff --git a/agents-common/src/test/resources/log4j.xml b/agents-common/src/test/resources/log4j.xml index f9a613b243..d863cf1d69 100644 --- a/agents-common/src/test/resources/log4j.xml +++ b/agents-common/src/test/resources/log4j.xml @@ -1,3 +1,20 @@ + + diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hbase_multiple_matching_policies.json b/agents-common/src/test/resources/policyengine/test_policyengine_hbase_multiple_matching_policies.json new file mode 100644 index 0000000000..f9957bba7b --- /dev/null +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hbase_multiple_matching_policies.json @@ -0,0 +1,75 @@ +{ + "serviceName":"hbasedev", + + "serviceDef":{ + "name":"hbase", + "id":2, + "resources":[ + {"name":"table","level":1,"parent":"","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"HBase Table","description":"HBase Table"}, + {"name":"column-family","level":2,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"HBase Column-Family","description":"HBase Column-Family"}, + {"name":"column","level":3,"parent":"column-family","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"HBase Column","description":"HBase Column"} + ], + "accessTypes":[ + {"name":"read","label":"Read"}, + {"name":"write","label":"Write"}, + {"name":"create","label":"Create"}, + {"name":"admin","label":"Admin","impliedGrants":["read","write","create"]} + ] + }, + + "policies":[ + {"id":1,"name":"table=default,*; column-family=default,*; column=default, *: audit-all-access","isEnabled":true,"isAuditEnabled":true, + "resources":{"table":{"values":["default", "*"]},"column-family":{"values":["default", "*"]}, "column":{"values":["default", "*"]}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true},{"type":"write","isAllowed":true}, {"type":"create", "isAllowed":true}, + {"type":"admin", "isAllowed":true}],"users":["user1"],"groups":[],"delegateAdmin":false} + , + {"accesses":[{"type":"read","isAllowed":true}],"users":["hrt_qa"],"groups":[],"delegateAdmin":false} + ] + } + , + {"id":2,"name":"table=*; column-family=*; column=*: audit-all-access","isEnabled":true,"isAuditEnabled":true, + "resources":{"table":{"values":["*"]},"column-family":{"values":["*"]}, "column":{"values":["*"]}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true},{"type":"write","isAllowed":true}, {"type":"create", "isAllowed":true}, + {"type":"admin", "isAllowed":true}],"users":["user1"],"groups":[],"delegateAdmin":false} + , + {"accesses":[{"type":"read","isAllowed":true}, {"type":"write", "isAllowed":true}],"users":["hrt_qa"],"groups":[],"delegateAdmin":false} + ] + } + ], + + "tests":[ + {"name":"TEST!!! ALLOW 'scan finance restricted-cf;' for hrt_qa", + "request":{ + "resource":{"elements":{"table":"finance","column-family":"restricted-cf"}}, + "accessType":"read","user":"hrt_qa","userGroups":[],"requestData":"scan finance restricted-cf; for hrt_qa" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":1} + } + , + {"name":"TEST!!! ALLOW 'put finance restricted-cf;' for hrt_qa", + "request":{ + "resource":{"elements":{"table":"finance","column-family":"restricted-cf"}}, + "accessType":"write","user":"hrt_qa","userGroups":[],"requestData":"put finance restricted-cf; for hrt_qa" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + }, + {"name":"TEST!!! DENY 'create finance restricted-cf;' for hrt_qa", + "request":{ + "resource":{"elements":{"table":"finance","column-family":"restricted-cf"}}, + "accessType":"create","user":"hrt_qa","userGroups":[],"requestData":"create finance restricted-cf; for hrt_qa" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"TEST!!! ALLOW 'create finance restricted-cf;' for user1", + "request":{ + "resource":{"elements":{"table":"finance","column-family":"restricted-cf"}}, + "accessType":"create","user":"user1","userGroups":[],"requestData":"create finance restricted-cf; for user1" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":1} + } + ] +} + diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_allaudit.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_allaudit.json new file mode 100644 index 0000000000..8686251f6a --- /dev/null +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_allaudit.json @@ -0,0 +1,205 @@ +{ + "serviceName":"hdfsdev", + + "auditMode":"audit-all", + + "serviceDef":{ + "name":"hdfs", + "id":1, + "resources":[ + {"name":"path","type":"path","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Resource Path","description":"HDFS file or directory path"} + ], + "accessTypes":[ + {"name":"read","label":"Read"}, + {"name":"write","label":"Write"}, + {"name":"execute","label":"Execute"} + ], + "contextEnrichers": + [ + { + "itemId":1, + "name" : "GeolocationEnricher", + "enricher" : "org.apache.ranger.plugin.contextenricher.RangerFileBasedGeolocationProvider", + "enricherOptions" : { + "FilePath":"/etc/ranger/geo/geo.txt", "ForceRead":"false", "IPInDotFormat":"true" + ,"geolocation.meta.prefix": "TEST_" + } + } + ], + "policyConditions": [ + { + "itemId":1, + "name":"ScriptConditionEvaluator", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript"}, + "label":"Script", + "description": "Script to execute" + } + ] + }, + + "policies":[ + {"id":1,"name":"audit-all-access under /finance/restricted/","isEnabled":true,"isAuditEnabled":true, + "resources":{"path":{"values":["/finance/restricted/"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[],"users":[],"groups":["public"],"delegateAdmin":false} + ] + } + , + {"id":2,"name":"allow-read-to-all under /public/","isEnabled":true,"isAuditEnabled":false, + "resources":{"path":{"values":["/public/*"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true},{"type":"execute","isAllowed":true}],"users":[],"groups":["public"],"delegateAdmin":false} + ] + } + , + {"id":3,"name":"allow-read-to-finance under /finance/restricted","isEnabled":true,"isAuditEnabled":true, + "resources":{"path":{"values":["/finance/restricted"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true}],"users":[],"groups":["finance"],"delegateAdmin":false, + "conditions":[{ + "type":"ScriptConditionEvaluator", + "values":["var country_code = ctx.getRequestContextAttribute('LOCATION_TEST_COUNTRY_CODE'); ctx.result = !!country_code;"] + }]} + ] + } + ], + + "tests":[ + {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; valid clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", + "remoteIPAddress":"255.255.255.255" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":3} + } + , + {"name":"DENY 'read /finance/restricted/sales.db' for g=finance; invalid clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", + "remoteIPAddress":"128.101.101.99" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; no clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /finance/restricted/hr/payroll.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/hr/payroll.db", + "remoteIPAddress":"128.101.101.101" + + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":3} + } + , + {"name":"DENY 'read /operations/visitors.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/operations/visitors.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /operations/visitors.db", + "clientIPAddress":"128.101.101.99" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /public/technology/blogs.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + } + , + + {"name":"DENY 'read /finance/restricted/sales.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"FALSE 'read /finance/restricted/hr/payroll.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /finance/restricted/hr/payroll.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"DENY 'read /operations/visitors.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/operations/visitors.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /operations/visitors.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /public/technology/blogs.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + } + , + + {"name":"DENY 'read /finance/restricted/sales.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"DENY 'read /finance/restricted/hr/payroll.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /finance/restricted/hr/payroll.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"DENY 'read /operations/visitors.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/operations/visitors.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /operations/visitors.db" + }, + "result":{"isAudited":true,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /public/technology/blogs.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + } + , + {"name":"ALLOW 'read /public/technology' for u=user1", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + } + , + {"name":"ALLOW 'read /public/technology' for u=user1", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"execute","user":"user1","userGroups":[],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":2} + } + ] +} + diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_noaudit.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_noaudit.json new file mode 100644 index 0000000000..cb0ed432e0 --- /dev/null +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_noaudit.json @@ -0,0 +1,205 @@ +{ + "serviceName":"hdfsdev", + + "auditMode":"audit-none", + + "serviceDef":{ + "name":"hdfs", + "id":1, + "resources":[ + {"name":"path","type":"path","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Resource Path","description":"HDFS file or directory path"} + ], + "accessTypes":[ + {"name":"read","label":"Read"}, + {"name":"write","label":"Write"}, + {"name":"execute","label":"Execute"} + ], + "contextEnrichers": + [ + { + "itemId":1, + "name" : "GeolocationEnricher", + "enricher" : "org.apache.ranger.plugin.contextenricher.RangerFileBasedGeolocationProvider", + "enricherOptions" : { + "FilePath":"/etc/ranger/geo/geo.txt", "ForceRead":"false", "IPInDotFormat":"true" + ,"geolocation.meta.prefix": "TEST_" + } + } + ], + "policyConditions": [ + { + "itemId":1, + "name":"ScriptConditionEvaluator", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript"}, + "label":"Script", + "description": "Script to execute" + } + ] + }, + + "policies":[ + {"id":1,"name":"audit-all-access under /finance/restricted/","isEnabled":true,"isAuditEnabled":true, + "resources":{"path":{"values":["/finance/restricted/"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[],"users":[],"groups":["public"],"delegateAdmin":false} + ] + } + , + {"id":2,"name":"allow-read-to-all under /public/","isEnabled":true,"isAuditEnabled":false, + "resources":{"path":{"values":["/public/*"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true},{"type":"execute","isAllowed":true}],"users":[],"groups":["public"],"delegateAdmin":false} + ] + } + , + {"id":3,"name":"allow-read-to-finance under /finance/restricted","isEnabled":true,"isAuditEnabled":true, + "resources":{"path":{"values":["/finance/restricted"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true}],"users":[],"groups":["finance"],"delegateAdmin":false, + "conditions":[{ + "type":"ScriptConditionEvaluator", + "values":["var country_code = ctx.getRequestContextAttribute('LOCATION_TEST_COUNTRY_CODE'); ctx.result = !!country_code;"] + }]} + ] + } + ], + + "tests":[ + {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; valid clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", + "remoteIPAddress":"255.255.255.255" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":3} + } + , + {"name":"DENY 'read /finance/restricted/sales.db' for g=finance; invalid clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", + "remoteIPAddress":"128.101.101.99" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; no clientIPAddress", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /finance/restricted/hr/payroll.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/hr/payroll.db", + "remoteIPAddress":"128.101.101.101" + + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":3} + } + , + {"name":"DENY 'read /operations/visitors.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/operations/visitors.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /operations/visitors.db", + "clientIPAddress":"128.101.101.99" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /public/technology/blogs.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":2} + } + , + + {"name":"DENY 'read /finance/restricted/sales.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"FALSE 'read /finance/restricted/hr/payroll.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /finance/restricted/hr/payroll.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"DENY 'read /operations/visitors.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/operations/visitors.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /operations/visitors.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /public/technology/blogs.db' for g=hr", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":["hr"],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":2} + } + , + + {"name":"DENY 'read /finance/restricted/sales.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /finance/restricted/sales.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"DENY 'read /finance/restricted/hr/payroll.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /finance/restricted/hr/payroll.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"DENY 'read /operations/visitors.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/operations/visitors.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /operations/visitors.db" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /public/technology/blogs.db' for u=user1", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":2} + } + , + {"name":"ALLOW 'read /public/technology' for u=user1", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":2} + } + , + {"name":"ALLOW 'read /public/technology' for u=user1", + "request":{ + "resource":{"elements":{"path":"/public/technology/blogs.db"}}, + "accessType":"execute","user":"user1","userGroups":[],"requestData":"read /public/technology/blogs.db" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":2} + } + ] +} + diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_resourcespec.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_resourcespec.json new file mode 100644 index 0000000000..384beb895e --- /dev/null +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs_resourcespec.json @@ -0,0 +1,67 @@ +{ + "serviceName":"hdfsdev", + + "serviceDef":{ + "name":"hdfs", + "id":1, + "resources":[ + {"name":"path","type":"path","level":1,"mandatory":true,"lookupSupported":true, + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true}, + "startDelimiterChar": "{", "endDelimiterChar": "}", "escapeChar":"%", + "label":"Resource Path","description":"HDFS file or directory path"} + ], + "accessTypes":[ + {"name":"read","label":"Read"}, + {"name":"write","label":"Write"}, + {"name":"execute","label":"Execute"} + ], + "contextEnrichers": [ ], + "policyConditions": [ ] + }, + + "policies":[ + {"id":1,"name":"allow-read-to-finance under /finance/rest*ricted/","isEnabled":true,"isAuditEnabled":true, + "resources":{"path":{"values":["/finance/rest*ricted/"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true}],"users":[],"groups":["finance"],"delegateAdmin":false, "conditions":[ ] } + ] + } + , + {"id":2,"name":"allow-read-to-{USER} under /home/{USER}/","isEnabled":true,"isAuditEnabled":false, + "resources":{"path":{"values":["/home/{USER}/"],"isRecursive":true}}, + "policyItems":[ + {"accesses":[{"type":"read","isAllowed":true}],"users":["{USER}"],"groups":[],"delegateAdmin":false, "conditions":[ ] } + ] + } + ], + + "tests":[ + {"name":"DENY 'read /home/user1/tmp/sales.db' for user=user2", + "request":{ + "resource":{"elements":{"path":"/home/user1/tmp/sales.db"}}, + "accessType":"read","user":"user2","userGroups":[],"requestData":"DENY read /home/user1/tmp/sales.db to user2" + }, + "result":{"isAudited":false,"isAllowed":false,"policyId":-1} + } + , + {"name":"ALLOW 'read /home/user1/tmp/sales.db' for user=user1", + "request":{ + "resource":{"elements":{"path":"/home/user1/tmp/sales.db"}}, + "accessType":"read","user":"user1","userGroups":[],"requestData":"ALLOW read /home/user1/tmp/sales.db to user1" + }, + "result":{"isAudited":false,"isAllowed":true,"policyId":2} + } + , + + {"name":"ALLOW 'read /finance/restricted/tmp/sales.db' for g=finance", + "request":{ + "resource":{"elements":{"path":"/finance/restricted/tmp/sales.db"}}, + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/tmp/sales.db" + }, + "result":{"isAudited":true,"isAllowed":true,"policyId":1} + } + + ] +} + diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_default.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_default.json index 918c30fbc0..50b4cc32d6 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_default.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_default.json @@ -1,5 +1,28 @@ { "testCases":[ + { + "name":"values={USER}_simple, %{USER}_simple; wildCard=true; ignoreCase=true; startDelimiter={, endDelimiter=}, escapeChar=%", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true, "replaceTokens":true, "tokenDelimiterEscape":"%", "tokenDelimiterPrefix":"rangerToken:" } + }, + "policyResource":{ + "values": ["{rangerToken:USER}_simple", "simple_{USER}", "{USER}_simple"] + }, + "tests":[ + { "name":"all-lower","input":"admin_simple", "evalContext": { "token:USER": "admin"}, "result":true}, + { "name":"all-upper","input":"{USER}_SIMPLE", "result":true}, + { "name":"mixed-case","input":"_SiMpLe", "evalContext": { "token:USER": ""}, "result":true}, + { "name":"invalid-all-lower-wild","input":"other-simple", "result":false}, + { "name":"invalid-all-upper-wild","input":"OTHER-SIMPLE", "result":false}, + { "name":"invalid-mixed-case-wild","input":"OtHeR-SiMpLe", "result":false}, + { "name":"escaped-delimiter","input":"%{USER}_SiMpLe", "evalContext": { "token:USER": ""}, "result":false}, + { "name":"not-escaped-delimiter-with-evalContext","input":"{USER}_SiMpLe", "evalContext": { "token:USER": ""}, "result":true}, + { "name":"not-escaped-delimiter-no-evalContext","input":"{USER}_SiMpLe", "result":true}, + { "name":"with-no-prefix","input":"simple_admin", "evalContext": { "token:USER": "admin"}, "result":false}, + { "name":"with-no-prefix-straight-match","input":"simple_{USER}", "evalContext": { "token:USER": "admin"}, "result":true} + ] + }, { "name":"value=simple; wildCard=true; ignoreCase=true", "resourceDef":{ @@ -350,4 +373,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json new file mode 100644 index 0000000000..168a50f4aa --- /dev/null +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json @@ -0,0 +1,33 @@ +{ + "testCases":[ + { + "name":"value=/; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true, "replaceTokens":true, "tokenDelimiterStart":"%", "tokenDelimiterEnd":"%", "tokenDelimiterEscape":"@" } + }, + "policyResource":{ + "values": ["/abc%xyz%w", "/xyz%somestuff%z", "/abc@%xyz@w", "/mad@@%xyy%"], + "isRecursive":false + }, + "tests":[ + { "name":"exact-path","input":"/mad@new", "evalContext": {"token:xyy": "new"}, "result":true} + , + { "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + , + { "name":"exact-path","input":"/abc%xyz%w", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":false} + , + { "name":"exact-path","input":"/abcabcdw", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":true} + , + { "name":"exact-path","input":"/xyzsomethingelsez", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + , + { "name":"exact-path","input":"/abc@%xyz@w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} + , + { "name":"exact-path","input":"/abc%xyzw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + , + { "name":"exact-path","input":"/abcabcdw", "evalContext": {"token:somestuff": "somethingelse", "xyz":"abcd"}, "result":false} + + ] + } + ] +} diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json index 25b0eb77e5..97765f94d1 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json @@ -1,6 +1,352 @@ { "testCases":[ - { + { + "name": "1a:value=/test?; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/test?"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/testa", "result": true}, + {"name": "incomplete-path", "input": "/test", "result": false}, + {"name": "extra-path", "input": "/testab", "result": false} + ] + }, + { + "name": "1b:value=/test*a*; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/test*a*"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/testa", "result": true}, + {"name": "expanded-path-1", "input": "/test1a", "result": true}, + {"name": "expanded-path-2", "input": "/test1a2", "result": true}, + {"name": "incorrect-path", "input": "/tes1a2", "result": false} + ] + }, + { + "name": "1c:value=/test*a*b; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/test*a*b"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/testab", "result": true}, + {"name": "expanded-path-1", "input": "/test1ab", "result": true}, + {"name": "expanded-path-2", "input": "/test1a2b", "result": true}, + {"name": "incorrect-path", "input": "/tesa12b", "result": false} + ] + }, + { + "name": "1d:value=/*test*a; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/*test*a"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/testa", "result": true}, + {"name": "expanded-path-1", "input": "/1test2a", "result": true}, + {"name": "expanded-path-2", "input": "/01test23a", "result": true}, + {"name": "incorrect-path", "input": "/tesa", "result": false} + ] + }, + { + "name": "2:value=/test; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/test"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/test", "result": true}, + {"name": "incorrect-path-1", "input": "/testa", "result": false}, + {"name": "incorrect-path-2", "input": "/1test", "result": false} + ] + }, + { + "name": "3:value=/*test; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/*test", "**test"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/test", "result": true}, + {"name": "expanded-path-1", "input": "/1test", "result": true}, + {"name": "expanded-path-2", "input": "/12test", "result": true}, + {"name": "incorrect-path", "input": "/12testa", "result": false} + ] + }, + { + "name": "4a:value=/test*a; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/test*a"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/testa", "result": true}, + {"name": "expanded-path-1", "input": "/test1a", "result": true}, + {"name": "expanded-path-2", "input": "/test12a", "result": true}, + {"name": "incorrect-path", "input": "/testb", "result": false} + ] + }, + { + "name": "4b:value=/*test*a; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/*test*a"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/testa", "result": true}, + {"name": "expanded-path-1", "input": "/test1a", "result": true}, + {"name": "expanded-path-2", "input": "/test12a", "result": true}, + {"name": "expanded-path-3", "input": "/0test12a", "result": true}, + {"name": "incorrect-path", "input": "/0testb", "result": false} + ] + }, + { + "name": "5:value=/test*; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/test*", "/test**"], + "isRecursive": false + }, + "tests": [ + {"name": "correct-path", "input": "/test", "result": true}, + {"name": "expanded-path-1", "input": "/test1", "result": true}, + {"name": "expanded-path-2", "input": "/test12", "result": true}, + {"name": "incorrect-path-1", "input": "/0test", "result": false}, + {"name": "incorrect-path-2", "input": "/tes", "result": false} + ] + }, + { + "name": "Case 0:value=/home/; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/home/", "/home"], + "isRecursive": false + }, + "tests": [ + {"name": "seemingly-correct-path", "input": "/home/", "result": true}, + {"name": "without-slash-path", "input": "/home", "result": true}, + {"name": "incorrect-path", "input": "/home/a.txt", "result": false} + ] + }, + { + "name": "Case 0:value=/home/; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/home/"], + "isRecursive": true + }, + "tests": [ + {"name": "seemingly-correct-path", "input": "/home/", "result": false}, + {"name": "correct-path", "input": "/home/a.txt", "result": true} + ] + }, + { + "name": "Case 4:value=/home/*/a.txt; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/home/*/a.txt"], + "isRecursive": false + }, + "tests": [ + {"name": "incorrect-path", "input": "/home/1/b.txt", "result": false}, + {"name": "missing-level-path", "input": "/home/a.txt", "result": false}, + {"name": "one-level-path", "input": "/home/1/a.txt", "result": true}, + {"name": "multi-level-path", "input": "/home/1/2/a.txt", "result": true}, + {"name": "multi-level-path", "input": "/home/1/2/ba.txt", "result": false} + ] + } + , + { + "name": "Case 7:value=/home/*a.txt; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/home/*a.txt"], + "isRecursive": false + }, + "tests": [ + {"name": "multi-level-path", "input": "/home/1/2/3a.txt", "result": true}, + {"name": "incorrect-path", "input": "/homea.txt", "result": false}, + {"name": "exact-path", "input": "/home/a.txt", "result": true} + ] + } + , + { + "name": "Case 4:value=/home/*/a.txt; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/home/*/a.txt"], + "isRecursive": true + }, + "tests": [ + {"name": "missing-level-path", "input": "/home/a.txt", "result": false}, + {"name": "correct-path", "input": "/home/1/a.txt", "result": true}, + {"name": "incorrect-path", "input": "/home/1/b.txt", "result": false}, + {"name": "multi-level-path", "input": "/home/1/2/a.txt", "result": true}, + {"name": "multi-level-path", "input": "/home/1/2/ba.txt", "result": false} + ] + } + , + { + "name": "Case 4:value=/home/*/*/a.txt; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": { + "values": ["/home/*/*/a.txt"], + "isRecursive": true + }, + "tests": [ + {"name": "missing-levels-path", "input": "/home/a.txt", "result": false}, + {"name": "missing-level-path", "input": "/home/1/a.txt", "result": false}, + {"name": "incorrect-path", "input": "/home/1/b.txt", "result": false}, + {"name": "correct-path", "input": "/home/1/2/a.txt", "result": true} + ] + } + , + { + "name": "Case 3: value=/home/; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true} + }, + "policyResource": {"values": ["/home/"], "isRecursive": true}, + "tests": [ + {"name": "slash-at-end-path", "input": "/home/", "result": false}, + {"name": "correct-path", "input": "/home/a.txt", "result": true}, + {"name": "incomplete-path", "input": "/home", "result": false} + ] + } + , + { + "name":"value=/a/b*y.txt; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true} + }, + "policyResource":{ + "values": ["/a/b*y.txt"], + "isRecursive":true + }, + "tests":[ + { "name":"exact-path","input":"/a/b*y.txt", "result":true}, + { "name":"child-path","input":"/a/b/y.txt", "result":true}, + { "name":"grand-child-path","input":"/a/b1/b2/y.txt", "result":true}, + { "name":"descendant-child-path","input":"/a/b1/c1/d1/any.txt", "result":true}, + { "name":"mismatche-path","input":"/a/any.txt", "result":false} + + ] + } + , + { + "name":"value=/a/b*y.txt; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true} + }, + "policyResource":{ + "values": ["/a/b*y.txt"], + "isRecursive":false + }, + "tests":[ + { "name":"exact-path","input":"/a/b*y.txt", "result":true}, + { "name":"child-path","input":"/a/b/y.txt", "result":true}, + { "name":"grand-child-path","input":"/a/b1/b2/y.txt", "result":true}, + { "name":"descendant-child-path","input":"/a/b1/c1/d1/any.txt", "result":true}, + { "name":"mismatche-path","input":"/a/any.txt", "result":false} + ] + } + , + { + "name":"value=*; isRecursive=false; wildCard=true; ignoreCase=true", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true} + }, + "policyResource":{ + "values": ["*"], + "isRecursive":false + }, + "tests":[ + { "name":"exact-path","input":"*", "result":true}, + { "name":"child-path","input":"/path1", "result":true}, + { "name":"grand-child-path","input":"/path1/path2", "result":true}, + { "name":"no-slash-path","input":"path1", "result":true} + ] + } + , + { + "name":"value=*; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":true, "ignoreCase":true} + }, + "policyResource":{ + "values": ["*"], + "isRecursive":true + }, + "tests":[ + { "name":"exact-path","input":"*", "result":true}, + { "name":"child-path","input":"/path1", "result":true}, + { "name":"grand-child-path","input":"/path1/path2", "result":true}, + { "name":"no-slash-path","input":"path1", "result":true} + ] + } + , + { "name":"value=/; isRecursive=false; wildCard=true; ignoreCase=true", "resourceDef":{ "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", @@ -14,7 +360,7 @@ { "name":"exact-path","input":"/", "result":true}, { "name":"child-path","input":"/path1", "result":false}, { "name":"grand-child-path","input":"/path1/path2", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -33,7 +379,7 @@ { "name":"exact-path","input":"/path1", "result":true}, { "name":"child-path","input":"/path1/path2", "result":false}, { "name":"grand-child-path","input":"/path1/path2/path3", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -53,7 +399,7 @@ { "name":"child-path","input":"/path1/path2", "result":true}, { "name":"grand-child-path","input":"/path1/path2/path3", "result":true}, { "name":"sibling-path","input":"/path2/path3/path4", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -71,7 +417,7 @@ { "name":"exact-path","input":"/", "result":true}, { "name":"child-path","input":"/path1", "result":true}, { "name":"grand-child-path","input":"/path1/path2", "result":true}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -95,9 +441,31 @@ { "name":"grand-child-path-camel-case","input":"/Path1/Path2/Path3", "result":true}, { "name":"sibling-path","input":"/path2", "result":false}, { "name":"invalid-path","input":"path1", "result":false}, + { "name":"prefix-path","input":"/path12", "result":false}, + { "name":"no-path","input":"", "result":false} ] } , + { + "name": "value=/path1/path2; isRecursive=true; wildCard=true; ignoreCase=true", + "resourceDef": { + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": {"wildCard": true, "ignoreCase": true + } + }, + "policyResource": {"values": ["/path1/path2"], "isRecursive": true}, + "tests": [ + {"name": "exact-path", "input": "/path1/path2", "result": true}, + {"name": "parent-path", "input": "/path1", "result": false}, + {"name": "grand-parent-path", "input": "/", "result": false}, + {"name": "child-path", "input": "/path1/path2/path3", "result": true}, + {"name": "grandchild-path", "input": "/path1/path2/path3/path4", "result": true}, + {"name": "sibling-path", "input": "/path1/path3", "result": false}, + {"name": "prefix-path", "input": "/path1/path21", "result": false}, + {"name": "invalid-path", "input": "path1/path2*", "result": false} + ] + } + , { "name":"value=/path*; isRecursive=true; wildCard=true; ignoreCase=true", "resourceDef":{ @@ -120,7 +488,7 @@ { "name":"grand-child-path-camel-case","input":"/Path-To-Success/Is-Slow/And-Fun", "result":true}, { "name":"unmatched-path","input":"/pat1ha", "result":false}, { "name":"unmatched-child-path","input":"/pat1ha/path2b", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -145,7 +513,7 @@ { "name":"grand-child-path-camel-case","input":"/Path1/Path2/Path3", "result":true}, { "name":"unmatched-path","input":"/path1a", "result":false}, { "name":"unmatched-child-path","input":"/path1a/path2b", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -169,7 +537,7 @@ { "name":"grand-child-path","input":"/public/archive/2008/first-test", "result":true}, { "name":"unmatched-path","input":"/pat1ha", "result":false}, { "name":"unmatched-child-path","input":"/pat1ha/path2b", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -200,7 +568,7 @@ { "name":"grand-child-path","input":"/public/last-test/best-result/details", "result":true}, { "name":"unmatched-path","input":"/pat1ha", "result":false}, { "name":"unmatched-child-path","input":"/pat1ha/path2b", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -231,7 +599,7 @@ { "name":"grand-child-path","input":"/public/last-test/best-result/details", "result":true}, # TODO: should this be false since isRecursive=false? { "name":"unmatched-path","input":"/pat1ha", "result":false}, { "name":"unmatched-child-path","input":"/pat1ha/path2b", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -249,7 +617,7 @@ { "name":"exact-path","input":"root", "result":true}, { "name":"child-path","input":"root.default", "result":false}, { "name":"grand-child-path","input":"root.default.mycompany", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -271,7 +639,7 @@ { "name":"child-path","input":"root.default.mycompany1.test", "result":true}, # TODO: should this be false since isRecursive=false { "name":"child-path","input":"root.default.mycompany1.dev", "result":true}, # TODO: should this be false since isRecursive=false { "name":"sibling-path","input":"root.default.othercompany1.dev", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -289,7 +657,7 @@ { "name":"exact-path","input":"root", "result":true}, { "name":"child-path","input":"root.default", "result":true}, { "name":"grand-child-path","input":"root.default.mycompany", "result":true}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } , @@ -311,8 +679,9 @@ { "name":"child-path","input":"root.default.mycompany1.test", "result":true}, { "name":"child-path","input":"root.default.mycompany1.dev", "result":true}, { "name":"sibling-path","input":"root.default.othercompany1.dev", "result":false}, - { "name":"invalid-path","input":"path1", "result":false}, + { "name":"invalid-path","input":"path1", "result":false} ] } + ] -} \ No newline at end of file +} diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json new file mode 100644 index 0000000000..f896745c7f --- /dev/null +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json @@ -0,0 +1,28 @@ +{ + "testCases":[ + { + "name":"value=/; isRecursive=false; wildCard=false; ignoreCase=true", + "resourceDef":{ + "matcher":"org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions":{"wildCard":false, "ignoreCase":true, "replaceTokens":true, "tokenDelimiterStart":"*", "tokenDelimiterEnd":"?", "tokenDelimiterEscape":"@" } + }, + "policyResource":{ + "values": ["/abc*xyz?w", "/xyz*somestuff?z", "/abc@*xyz@w", "/mad@@*xyy?"], + "isRecursive":false + }, + "tests":[ + { "name":"exact-path","input":"/abc@*xyz@w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false} + , + { "name":"exact-path","input":"/mad@new", "evalContext": {"token:xyy": "new"}, "result":true} + , + { "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + , + { "name":"exact-path","input":"/abc*xyz?w", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":false} + , + { "name":"exact-path","input":"/xyzsomethingelsez", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + , + { "name":"exact-path","input":"/abc*xyzw", "evalContext": {"token:somestuff": "somethingelse"}, "result":true} + ] + } + ] +} diff --git a/agents-cred/pom.xml b/agents-cred/pom.xml index b21e3b0cb3..1b15c02927 100644 --- a/agents-cred/pom.xml +++ b/agents-cred/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/agents-cred/src/test/resources/log4j.properties b/agents-cred/src/test/resources/log4j.properties index c2463a76db..1825e2470f 100644 --- a/agents-cred/src/test/resources/log4j.properties +++ b/agents-cred/src/test/resources/log4j.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Define some default values that can be overridden by system properties ranger.root.logger=FATAL,console # Define the root logger to the system property "hbase.root.logger". diff --git a/agents-installer/pom.xml b/agents-installer/pom.xml index ab2b86eb3c..63eb565cb2 100644 --- a/agents-installer/pom.xml +++ b/agents-installer/pom.xml @@ -24,7 +24,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/credentialbuilder/pom.xml b/credentialbuilder/pom.xml index e07a30024e..ec9ccbd9e7 100644 --- a/credentialbuilder/pom.xml +++ b/credentialbuilder/pom.xml @@ -24,7 +24,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/dev-support/ranger-pmd-ruleset.xml b/dev-support/ranger-pmd-ruleset.xml index b3cf4fdd3e..dbd8ff0d83 100644 --- a/dev-support/ranger-pmd-ruleset.xml +++ b/dev-support/ranger-pmd-ruleset.xml @@ -46,9 +46,6 @@ - - - diff --git a/docs/README.txt b/docs/README.txt index d15551cdf3..05edfb928f 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -17,7 +17,7 @@ # Build Instruction for Apache Ranger Documentation # --------------------------------------------------------------------- -$ export DOC_SRC_DIR=incubator-ranger/docs +$ export DOC_SRC_DIR=ranger/docs $ cd ${DOC_SRC_DIR} $ mvn site @@ -29,7 +29,7 @@ $ mvn site DOC_DEPLOY_DIR=/tmp/doc_deploy_dir.$$ mkdir -p ${DOC_DEPLOY_DIR} -svn co https://svn.apache.org/repos/asf/incubator/ranger/site/trunk ranger +svn co https://svn.apache.org/repos/asf/ranger/site/trunk ranger cd ${DOC_SRC_DIR}/target rsync -avcn * ${DOC_DEPLOY_DIR}/ranger diff --git a/docs/pom.xml b/docs/pom.xml index 0a99073296..eb60f230f7 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -21,10 +21,10 @@ Apache Ranger is a framework to enable, monitor and manage comprehensive data security across the Hadoop platform. Apache Ranger currently provides a centralized security adminstration, fine grain access control and detailed auditing for user access within Apache Hadoop, Apache Hive, Apache HBase and other Apache components - http://ranger.incubator.apache.org/ + http://ranger.apache.org/ 4.0.0 - org.incubator.apache.ranger - 0.6.0 + org.apache.ranger + 0.6.4-SNAPSHOT ranger pom @@ -35,10 +35,10 @@ - scm:svn:https://svn.apache.org/repos/asf/incubator/ranger/site/trunk - scm:svn:https://svn.apache.org/repos/asf/incubator/ranger/site/trunk + scm:svn:https://svn.apache.org/repos/asf/ranger/site/trunk + scm:svn:https://svn.apache.org/repos/asf/ranger/site/trunk HEAD - http://svn.apache.org/repos/asf/incubator/ranger/site/trunk + http://svn.apache.org/repos/asf/ranger/site/trunk Jira @@ -47,29 +47,29 @@ User list - mailto:user-subscribe@ranger.incubator.apache.org + mailto:user-subscribe@ranger.apache.org - mailto:user-unsubscribe@ranger.incubator.apache.org + mailto:user-unsubscribe@ranger.apache.org - mailto:user@ranger.incubator.apache.org + mailto:user@ranger.apache.org http://mail-archives.apache.org/mod_mbox/ranger-user/ Development list - mailto:dev-subscribe@ranger.incubator.apache.org + mailto:dev-subscribe@ranger.apache.org - mailto:dev-unsubscribe@ranger.incubator.apache.org + mailto:dev-unsubscribe@ranger.apache.org - mailto:dev@ranger.incubator.apache.org + mailto:dev@ranger.apache.org http://mail-archives.apache.org/mod_mbox/ranger-dev/ Commit list - mailto:commits-subscribe@ranger.incubator.apache.org + mailto:commits-subscribe@ranger.apache.org - mailto:commits-unsubscribe@ranger.incubator.apache.org + mailto:commits-unsubscribe@ranger.apache.org - mailto:commits@ranger.incubator.apache.org + mailto:commits@ranger.apache.org http://mail-archives.apache.org/mod_mbox/ranger-commits/ @@ -108,7 +108,6 @@ Committer - Hortonworks @@ -170,7 +169,6 @@ Committer - Hortonworks @@ -207,7 +205,6 @@ Committer - Hortonworks @@ -293,6 +290,7 @@ vel@apache.org -5 + PMC Committer @@ -462,7 +460,7 @@ apache-website Apache website - scpexe://people.apache.org/www/incubator.apache.org/ranger + scpexe://people.apache.org/www/apache.org/ranger diff --git a/docs/src/site/apt/index.apt.vm b/docs/src/site/apt/index.apt.vm index 5e42bd646a..5ce2cef2d5 100644 --- a/docs/src/site/apt/index.apt.vm +++ b/docs/src/site/apt/index.apt.vm @@ -64,7 +64,7 @@ ${project.name} concern. Especially for fixing bugs it is crucial that the developers can reproduce your problem. Contributors can check out the source codefrom our - {{{http://git.apache.org/incubator-ranger.git/}Git repository}} + {{{http://git.apache.org/ranger.git/}Git repository}} * Disclaimer diff --git a/docs/src/site/site.xml b/docs/src/site/site.xml index 52ef98d43c..59ea5c3957 100644 --- a/docs/src/site/site.xml +++ b/docs/src/site/site.xml @@ -36,14 +36,6 @@ under the License. - - Apache Incubator - http://incubator.apache.org/images/egg-logo.png - 200px - 400px - http://incubator.apache.org/ - - @@ -69,6 +61,7 @@ under the License. + diff --git a/docs/src/site/xdoc/download.xml b/docs/src/site/xdoc/download.xml index e6b806b34b..4e178f13ca 100644 --- a/docs/src/site/xdoc/download.xml +++ b/docs/src/site/xdoc/download.xml @@ -31,22 +31,32 @@ LICENSE.txt and NOTICE.txt files contained in each release artifact. @@ -55,7 +65,7 @@ The older branch release is Apache Ranger 0.4.0:

When downloading from a mirror please check the SHA1/MD5 checksums as well as verifying the OpenPGP compatible signature available from the main Apache -site. The KEYS +site. The KEYS file contains the public keys used for signing the release. It is recommended that a web of trust is used to confirm the identity of these keys.

diff --git a/docs/src/site/xdoc/quick_start_guide.xml b/docs/src/site/xdoc/quick_start_guide.xml index 933ca122c6..bdfee7ded6 100644 --- a/docs/src/site/xdoc/quick_start_guide.xml +++ b/docs/src/site/xdoc/quick_start_guide.xml @@ -31,15 +31,15 @@ 1. Check out the code from GIT repository

    - git clone https://git-wip-us.apache.org/repos/asf/incubator-ranger.git - cd incubator-ranger + git clone https://git-wip-us.apache.org/repos/asf/ranger.git + cd ranger

Alternatively, you can checkout the code from github:

    - git clone https://github.com/apache/incubator-ranger - cd incubator-ranger + git clone https://github.com/apache/ranger + cd ranger

@@ -92,30 +92,30 @@

5. Now, the following files are ready to be published for release: - ./target/apache-ranger-incubating-%version-number%.tar.gz - ./target/apache-ranger-incubating-%version-number%.tar.gz.asc - ./target/apache-ranger-incubating-%version-number%.tar.gz.mds + ./target/apache-ranger-%version-number%.tar.gz + ./target/apache-ranger-%version-number%.tar.gz.asc + ./target/apache-ranger-%version-number%.tar.gz.mds

- 1. Download the release source file, apache-ranger-incubating-%version-number%.tar.gz (from URL provided in the release email) + 1. Download the release source file, apache-ranger-%version-number%.tar.gz (from URL provided in the release email)

- 2. Download the PGP signature for, apache-ranger-incubating-%version-number%.tar.gz which usaully named as apache-ranger-incubating-%version-number%.tar.gz.asc (from URL provided in the release email) + 2. Download the PGP signature for, apache-ranger-%version-number%.tar.gz which usaully named as apache-ranger-%version-number%.tar.gz.asc (from URL provided in the release email)

3. Execute the following command to verify the release - $ gpg --verify apache-ranger-incubating-%version-number%.tar.gz.asc apache-ranger-incubating-%version-number%.tar.gz + $ gpg --verify apache-ranger-%version-number%.tar.gz.asc apache-ranger-%version-number%.tar.gz

- 4. Download MD5/SHA hash for apache-ranger-incubating-%version-number%.tar.gz which usaully named as apache-ranger-incubating-%version-number%.tar.gz.mds (from URL provided in the release email) - $ gpg --print-mds apache-ranger-incubating-%version-number%.tar.gz - Compare MD5, SHA hash generated from the above command and the signature available in the apache-ranger-incubating-%version-number%.tar.gz.mds. + 4. Download MD5/SHA hash for apache-ranger-%version-number%.tar.gz which usaully named as apache-ranger-%version-number%.tar.gz.mds (from URL provided in the release email) + $ gpg --print-mds apache-ranger-%version-number%.tar.gz + Compare MD5, SHA hash generated from the above command and the signature available in the apache-ranger-%version-number%.tar.gz.mds.

diff --git a/embeddedwebserver/pom.xml b/embeddedwebserver/pom.xml index d45cc9286d..cd2702b280 100644 --- a/embeddedwebserver/pom.xml +++ b/embeddedwebserver/pom.xml @@ -24,7 +24,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/hbase-agent/pom.xml b/hbase-agent/pom.xml index d930951b7a..008e44f19a 100644 --- a/hbase-agent/pom.xml +++ b/hbase-agent/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/AuthorizationSession.java b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/AuthorizationSession.java index 3c31c09a8e..48b1b11add 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/AuthorizationSession.java +++ b/hbase-agent/src/main/java/org/apache/ranger/authorization/hbase/AuthorizationSession.java @@ -347,22 +347,7 @@ String getLogMessage(boolean allowed, String reason) { } /** - * Hand creates a result object and set it on the request for cases where we need not go to policy manager. - * @return - */ - AuthorizationSession knownPatternAllowedNotAudited(String reason) { - _result = buildResult(true, false, reason); - return this; - } - - AuthorizationSession knownPatternDisallowedNotAudited(String reason) { - _result = buildResult(false, false, reason); - - return this; - } - - /** - * This method could potentially null out an earlier audit handler -- which effectively would suppress audits. + * This method could potentially null out an earlier audit handler -- which effectively would suppress audits. * @param anAuditHandler * @return */ @@ -371,16 +356,6 @@ AuthorizationSession auditHandler(HbaseAuditHandler anAuditHandler) { return this; } - RangerAccessResult buildResult(boolean allowed, boolean audited, String reason) { - RangerAccessResult result = _authorizer.createAccessResult(_request); - if (result != null) { - result.setIsAllowed(allowed); - result.setReason(reason); - result.setIsAudited(audited); - } - return result; - } - AuthorizationSession resourceMatchingScope(RangerAccessRequest.ResourceMatchingScope scope) { _resourceMatchingScope = scope; return this; diff --git a/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java b/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java index 65e7be6db8..0e980b9bd6 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java +++ b/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java @@ -60,9 +60,7 @@ public HBaseClient(String serivceName,Map connectionProp) { for (Map.Entry entry: connectionProperties.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - if (rangerInternalPropertyKeys.contains(key)) { - // skip - } else { + if (!rangerInternalPropertyKeys.contains(key)) { conf.set(key, value); } } @@ -251,10 +249,9 @@ public List run() { HBaseAdmin.checkHBaseAvailable(conf); LOG.info("getTableList: no exception: HbaseAvailability true"); admin = new HBaseAdmin(conf) ; - for (HTableDescriptor htd : admin.listTables(tableNameMatching)) { - if (htd == null) { - LOG.error("getTableList: null HTableDescription received from HBaseAdmin.listTables"); - } else { + HTableDescriptor [] htds = admin.listTables(tableNameMatching); + if (htds != null) { + for (HTableDescriptor htd : admin.listTables(tableNameMatching)) { String tableName = htd.getNameAsString(); if (existingTableList != null && existingTableList.contains(tableName)) { continue; @@ -262,7 +259,9 @@ public List run() { tableList.add(htd.getNameAsString()); } } - } + } else { + LOG.error("getTableList: null HTableDescription received from HBaseAdmin.listTables"); + } } catch (ZooKeeperConnectionException zce) { String msgDesc = "getTableList: Unable to connect to `ZooKeeper` " + "using given config parameters."; diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml index 6758b776a3..c2fd6e0cd4 100644 --- a/hdfs-agent/pom.xml +++ b/hdfs-agent/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java index bc98d24c00..ecc406cb08 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java @@ -55,9 +55,7 @@ public HdfsClient(String serviceName, Map connectionProperties) { for (Map.Entry entry: connectionProperties.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - if (rangerInternalPropertyKeys.contains(key)) { - // skip - } else { + if (!rangerInternalPropertyKeys.contains(key)) { conf.set(key, value); } } @@ -87,7 +85,7 @@ private List listFilesInternal(String baseDir, String fileMatching, fina FileStatus[] fileStats = fs.listStatus(basePath) ; if(LOG.isDebugEnabled()) { - LOG.debug("<== HdfsClient fileStatus : " + fileStats + " PathList :" + pathList) ; + LOG.debug("<== HdfsClient fileStatus : " + fileStats.length + " PathList :" + pathList) ; } if (fileStats != null) { @@ -185,7 +183,7 @@ public static final void main(String[] args) { String baseDir = args[1] ; String fileNameToMatch = (args.length == 2 ? null : args[2]) ; - HdfsClient fs = new HdfsClient(repositoryName, null) ; + HdfsClient fs = new HdfsClient(repositoryName, new HashMap()) ; List fsList = null; try { fsList = fs.listFiles(baseDir, fileNameToMatch,null); diff --git a/hdfs-agent/src/test/resources/hdfs-policies.json b/hdfs-agent/src/test/resources/hdfs-policies.json index c99c811ccb..eb61796c53 100644 --- a/hdfs-agent/src/test/resources/hdfs-policies.json +++ b/hdfs-agent/src/test/resources/hdfs-policies.json @@ -81,6 +81,11 @@ "policyItems": [ { "accesses": [ + { + "type": "read", + "isAllowed": true + } + , { "type": "execute", "isAllowed": true @@ -95,6 +100,11 @@ }, { "accesses": [ + { + "type": "read", + "isAllowed": true + } + , { "type": "execute", "isAllowed": true diff --git a/hive-agent/pom.xml b/hive-agent/pom.xml index ee1977f669..c4938ba89b 100644 --- a/hive-agent/pom.xml +++ b/hive-agent/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java index a6bb357a73..d98fe813e0 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuditHandler.java @@ -24,6 +24,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerAccessResult; @@ -66,7 +67,7 @@ AuthzAuditEvent createAuditEvent(RangerAccessResult result) { if(result instanceof RangerDataMaskResult) { accessType = ((RangerDataMaskResult)result).getMaskType(); - if(StringUtils.equals(accessType, RangerHiveAuthorizer.MASK_TYPE_NONE)) { + if(StringUtils.equals(accessType, RangerPolicy.MASK_TYPE_NONE)) { return null; } diff --git a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java index 69fa2935ec..7dd4c905e4 100644 --- a/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java +++ b/hive-agent/src/main/java/org/apache/ranger/authorization/hive/authorizer/RangerHiveAuthorizer.java @@ -54,6 +54,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants; import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResult; @@ -71,9 +72,6 @@ public class RangerHiveAuthorizer extends RangerHiveAuthorizerBase { private static final Log LOG = LogFactory.getLog(RangerHiveAuthorizer.class) ; private static final char COLUMN_SEP = ','; - public static final String MASK_TYPE_NULL = "MASK_NULL"; - public static final String MASK_TYPE_NONE = "MASK_NONE"; - public static final String MASK_TYPE_CUSTOM = "CUSTOM"; private static volatile RangerHivePlugin hivePlugin = null ; @@ -598,7 +596,7 @@ private RangerRowFilterResult getRowFilterResult(RangerHiveAccessRequest request } private boolean isDataMaskEnabled(RangerDataMaskResult result) { - return result != null && result.isMaskEnabled() && !StringUtils.equalsIgnoreCase(result.getMaskType(), MASK_TYPE_NONE); + return result != null && result.isMaskEnabled() && !StringUtils.equalsIgnoreCase(result.getMaskType(), RangerPolicy.MASK_TYPE_NONE); } private boolean isRowFilterEnabled(RangerRowFilterResult result) { @@ -672,11 +670,14 @@ private String getCellValueTransformer(HiveAuthzContext context, String database if(isDataMaskEnabled(result)) { String maskType = result.getMaskType(); RangerDataMaskTypeDef maskTypeDef = result.getMaskTypeDef(); - String transformer = maskTypeDef.getTransformer(); + String transformer = null; + if (maskTypeDef != null) { + transformer = maskTypeDef.getTransformer(); + } - if(StringUtils.equalsIgnoreCase(maskType, MASK_TYPE_NULL)) { + if(StringUtils.equalsIgnoreCase(maskType, RangerPolicy.MASK_TYPE_NULL)) { ret = "NULL"; - } else if(StringUtils.equalsIgnoreCase(maskType, MASK_TYPE_CUSTOM)) { + } else if(StringUtils.equalsIgnoreCase(maskType, RangerPolicy.MASK_TYPE_CUSTOM)) { String maskedValue = result.getMaskedValue(); if(maskedValue == null) { @@ -1005,16 +1006,29 @@ private boolean isURIAccessAllowed(String userName, FsAction action, String uri, try { Path filePath = new Path(uri); FileSystem fs = FileSystem.get(filePath.toUri(), conf); - // Path path = FileUtils.getPathOrParentThatExists(fs, filePath); - // FileStatus fileStatus = fs.getFileStatus(path); - FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath); + FileStatus[] filestat = fs.globStatus(filePath); + + if(filestat != null && filestat.length > 0) { + boolean isDenied = false; + + for(FileStatus file : filestat) { + if (FileUtils.isOwnerOfFileHierarchy(fs, file, userName) || + FileUtils.isActionPermittedForFileHierarchy(fs, file, userName, action)) { + continue; + } else { + isDenied = true; + break; + } + } + ret = !isDenied; + } else { // if given path does not exist then check for parent + FileStatus file = FileUtils.getPathOrParentThatExists(fs, filePath); - if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) { + FileUtils.checkFileAccessWithImpersonation(fs, file, action, userName); ret = true; - } else { - ret = FileUtils.isActionPermittedForFileHierarchy(fs, fileStatus, userName, action); } } catch(Exception excp) { + ret = false; LOG.error("Error getting permissions for " + uri, excp); } } diff --git a/jisql/pom.xml b/jisql/pom.xml index 51d8a36008..8eb5699ed2 100644 --- a/jisql/pom.xml +++ b/jisql/pom.xml @@ -24,7 +24,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/jisql/src/main/java/org/apache/util/sql/MySQLPLRunner.java b/jisql/src/main/java/org/apache/util/sql/MySQLPLRunner.java index b2eda30a67..58ab85b6a9 100644 --- a/jisql/src/main/java/org/apache/util/sql/MySQLPLRunner.java +++ b/jisql/src/main/java/org/apache/util/sql/MySQLPLRunner.java @@ -135,13 +135,9 @@ private void runScript(Connection conn, Reader reader) throws IOException, } String trimmedLine = line.trim(); - if (trimmedLine.startsWith("--")) { + if (trimmedLine.length() < 1 || trimmedLine.startsWith("--") + || trimmedLine.startsWith("//")) { //NOPMD //println(trimmedLine); - } else if (trimmedLine.length() < 1 - || trimmedLine.startsWith("//")) { - // Do nothing - } else if (trimmedLine.length() < 1 - || trimmedLine.startsWith("--")) { // Do nothing } else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter()) diff --git a/kms/pom.xml b/kms/pom.xml index f3c41394b4..f31bcc87c5 100644 --- a/kms/pom.xml +++ b/kms/pom.xml @@ -24,7 +24,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index abfab25f05..f91fc501d9 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -69,12 +69,12 @@ public class RangerKeyStore extends KeyStoreSpi { // keys private static class KeyEntry { - Date date; // the creation date of this entry + Date date=new Date(); // the creation date of this entry }; // Secret key private static final class SecretKeyEntry { - Date date; // the creation date of this entry + Date date=new Date(); // the creation date of this entry SealedObject sealedKey; String cipher_field; int bit_length; @@ -127,12 +127,15 @@ public Key engineGetKey(String alias, char[] password)throws NoSuchAlgorithmExce @Override public Date engineGetCreationDate(String alias) { Object entry = keyEntries.get(convertAlias(alias)); + Date date=null; if (entry != null) { - return new Date(((KeyEntry)entry).date.getTime()); - } else { - return null; - } - } + KeyEntry keyEntry=(KeyEntry)entry; + if(keyEntry.date!=null){ + date=new Date(keyEntry.date.getTime()); + } + } + return date; + } public void addKeyEntry(String alias, Key key, char[] password, String cipher, int bitLength, String description, int version, String attributes) @@ -331,10 +334,14 @@ public void engineLoad(InputStream stream, char[] password) } keyEntries.clear(); - md = getKeyedMessageDigest(password); + if(password!=null){ + md = getKeyedMessageDigest(password); + } - byte computed[]; - computed = md.digest(); + byte computed[]={}; + if(md!=null){ + computed = md.digest(); + } for(XXRangerKeyStore rangerKey : rangerKeyDetails){ String encoded = rangerKey.getEncoded(); byte[] data = DatatypeConverter.parseBase64Binary(encoded); @@ -555,18 +562,19 @@ public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, char KeyStore ks; try { ks = KeyStore.getInstance(fileFormat); - ks.load(null, storePass); - String alias = null; - engineLoad(null, masterKey); - Enumeration e = engineAliases(); - Key key; - while (e.hasMoreElements()) { - alias = e.nextElement(); - key = engineGetKey(alias, masterKey); - ks.setKeyEntry(alias, key, keyPass, null); + if(ks!=null){ + ks.load(null, storePass); + String alias = null; + engineLoad(null, masterKey); + Enumeration e = engineAliases(); + Key key; + while (e.hasMoreElements()) { + alias = e.nextElement(); + key = engineGetKey(alias, masterKey); + ks.setKeyEntry(alias, key, keyPass, null); + } + ks.store(stream, storePass); } - - ks.store(stream, storePass); } catch (Throwable t) { logger.error("Unable to load keystore file ", t); throw new IOException(t) ; diff --git a/knox-agent/pom.xml b/knox-agent/pom.xml index b5e600a10b..e41b710005 100644 --- a/knox-agent/pom.xml +++ b/knox-agent/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/knox-agent/src/main/java/org/apache/ranger/services/knox/client/KnoxClient.java b/knox-agent/src/main/java/org/apache/ranger/services/knox/client/KnoxClient.java index 33ac863351..6e3b227e81 100644 --- a/knox-agent/src/main/java/org/apache/ranger/services/knox/client/KnoxClient.java +++ b/knox-agent/src/main/java/org/apache/ranger/services/knox/client/KnoxClient.java @@ -79,7 +79,7 @@ public List getTopologyList(String topologyNameMatching,List kn ClientResponse response = null; try { - client = Client.create();; + client = Client.create(); client.addFilter(new HTTPBasicAuthFilter(userName, password)); WebResource webResource = client.resource(knoxUrl); @@ -170,7 +170,7 @@ public List getServiceList(String topologyName, String serviceNameMatchi ClientResponse response = null; try { - client = Client.create();; + client = Client.create(); client.addFilter(new HTTPBasicAuthFilter(userName, password)); diff --git a/plugin-atlas/pom.xml b/plugin-atlas/pom.xml index bd6fa1e8ec..7e927c558e 100644 --- a/plugin-atlas/pom.xml +++ b/plugin-atlas/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml index badc8f27f8..3d7e791382 100644 --- a/plugin-kafka/pom.xml +++ b/plugin-kafka/pom.xml @@ -28,7 +28,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java index 1afedd5af7..4af9e297ac 100644 --- a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java +++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java @@ -62,8 +62,6 @@ public class RangerKafkaAuthorizer implements Authorizer { public static final String ACCESS_TYPE_KAFKA_ADMIN = "kafka_admin"; private static volatile RangerBasePlugin rangerPlugin = null; - long lastLogTime = 0; - int errorLogFreq = 30000; // Log after every 30 seconds public RangerKafkaAuthorizer() { } @@ -75,27 +73,31 @@ public RangerKafkaAuthorizer() { */ @Override public void configure(Map configs) { - if (rangerPlugin == null) { - try { - LoginManager loginManager = LoginManager.acquireLoginManager(LoginType.SERVER, true, configs); - Subject subject = loginManager.subject(); - UserGroupInformation ugi = MiscUtil - .createUGIFromSubject(subject); - if (ugi != null) { - MiscUtil.setUGILoginUser(ugi, subject); + RangerBasePlugin me = rangerPlugin; + if (me == null) { + synchronized(RangerKafkaAuthorizer.class) { + me = rangerPlugin; + if (me == null) { + try { + LoginManager loginManager = LoginManager.acquireLoginManager(LoginType.SERVER, true, configs); + Subject subject = loginManager.subject(); + UserGroupInformation ugi = MiscUtil + .createUGIFromSubject(subject); + if (ugi != null) { + MiscUtil.setUGILoginUser(ugi, subject); + } + logger.info("LoginUser=" + MiscUtil.getUGILoginUser()); + } catch (Throwable t) { + logger.error("Error getting principal.", t); + } + me = rangerPlugin = new RangerBasePlugin("kafka", "kafka"); } - logger.info("LoginUser=" + MiscUtil.getUGILoginUser()); - } catch (Throwable t) { - logger.error("Error getting principal.", t); } - - rangerPlugin = new RangerBasePlugin("kafka", "kafka"); - logger.info("Calling plugin.init()"); - rangerPlugin.init(); - - RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); - rangerPlugin.setResultProcessor(auditHandler); } + logger.info("Calling plugin.init()"); + rangerPlugin.init(); + RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); + rangerPlugin.setResultProcessor(auditHandler); } @Override @@ -173,7 +175,7 @@ public boolean authorize(Session session, Operation operation, if (resource.resourceType().equals(Topic$.MODULE$)) { rangerResource.setValue(KEY_TOPIC, resource.name()); - } else if (resource.resourceType().equals(Cluster$.MODULE$)) { + } else if (resource.resourceType().equals(Cluster$.MODULE$)) { //NOPMD // CLUSTER should go as null // rangerResource.setValue(KEY_CLUSTER, resource.name()); } else if (resource.resourceType().equals(Group$.MODULE$)) { @@ -183,11 +185,10 @@ public boolean authorize(Session session, Operation operation, validationFailed = true; } - boolean returnValue = true; + boolean returnValue = false; if (validationFailed) { MiscUtil.logErrorMessageByInterval(logger, validationStr + ", request=" + rangerRequest); - returnValue = false; } else { try { @@ -195,7 +196,6 @@ public boolean authorize(Session session, Operation operation, .isAccessAllowed(rangerRequest); if (result == null) { logger.error("Ranger Plugin returned null. Returning false"); - returnValue = false; } else { returnValue = result.getIsAllowed(); } diff --git a/plugin-kms/pom.xml b/plugin-kms/pom.xml index 5e93ef760e..388a62fb58 100644 --- a/plugin-kms/pom.xml +++ b/plugin-kms/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java index e61d0bca33..bf1f493d21 100755 --- a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java +++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSResourceMgr.java @@ -84,9 +84,11 @@ public static List getKMSResources(String serviceName, Map getKMSResource(String url, String username, String password, String rangerPrincipal, String rangerKeytab, String nameRules, String authType, String kmsKeyName, List kmsKeyList) { List topologyList = null; final KMSClient KMSClient = KMSConnectionMgr.getKMSClient(url, username, password, rangerPrincipal, rangerKeytab, nameRules, authType); - synchronized(KMSClient){ - topologyList = KMSClient.getKeyList(kmsKeyName, kmsKeyList); - } - return topologyList; - } + if(KMSClient!=null){ + synchronized(KMSClient){ + topologyList = KMSClient.getKeyList(kmsKeyName, kmsKeyList); + } + } + return topologyList; + } } diff --git a/plugin-nifi/pom.xml b/plugin-nifi/pom.xml index c7519a0223..e343514d18 100644 --- a/plugin-nifi/pom.xml +++ b/plugin-nifi/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/plugin-solr/pom.xml b/plugin-solr/pom.xml index 372664b289..7a1f6ddf5b 100644 --- a/plugin-solr/pom.xml +++ b/plugin-solr/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java index 57fd3edbfb..c994b02c26 100644 --- a/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java +++ b/plugin-solr/src/main/java/org/apache/ranger/authorization/solr/authorizer/RangerSolrAuthorizer.java @@ -105,11 +105,17 @@ public void init(Map initInfo) { } try { - if (solrPlugin == null) { - logger.info("RangerSolrAuthorizer(): init called"); - solrPlugin = new RangerBasePlugin("solr", "solr"); - solrPlugin.init(); + RangerBasePlugin me = solrPlugin; + if (me == null) { + synchronized(RangerSolrAuthorizer.class) { + me = solrPlugin; + logger.info("RangerSolrAuthorizer(): init called"); + if (me == null) { + me = solrPlugin = new RangerBasePlugin("solr", "solr"); + } + } } + solrPlugin.init(); } catch (Throwable t) { logger.fatal("Error creating and initializing RangerBasePlugin()"); } @@ -152,7 +158,7 @@ public void close() throws IOException { */ @Override public AuthorizationResponse authorize(AuthorizationContext context) { - boolean isDenied = false; + boolean isDenied = true; try { if (logger.isDebugEnabled()) { @@ -201,6 +207,8 @@ public AuthorizationResponse authorize(AuthorizationContext context) { isDenied = true; // rejecting on first failure break; + } else { + isDenied = false; } } } finally { @@ -353,13 +361,13 @@ String mapToRangerAccessType(AuthorizationContext context) { String accessType = ACCESS_TYPE_OTHERS; RequestType requestType = context.getRequestType(); - if (requestType.equals(RequestType.ADMIN)) { + if (RequestType.ADMIN.equals(requestType)) { accessType = ACCESS_TYPE_ADMIN; - } else if (requestType.equals(RequestType.READ)) { + } else if (RequestType.READ.equals(requestType)) { accessType = ACCESS_TYPE_QUERY; - } else if (requestType.equals(RequestType.WRITE)) { + } else if (RequestType.WRITE.equals(requestType)) { accessType = ACCESS_TYPE_UPDATE; - } else if (requestType.equals(RequestType.UNKNOWN)) { + } else if (RequestType.UNKNOWN.equals(requestType)) { logger.info("UNKNOWN request type. Mapping it to " + accessType + ". Resource=" + context.getResource()); accessType = ACCESS_TYPE_OTHERS; diff --git a/plugin-yarn/pom.xml b/plugin-yarn/pom.xml index ab0ed86a1e..8896db8657 100644 --- a/plugin-yarn/pom.xml +++ b/plugin-yarn/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index a0d5eb922a..0ef33fcfab 100644 --- a/pom.xml +++ b/pom.xml @@ -23,11 +23,11 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT Security for Enforcing Enterprise Policies pom ranger - http://ranger.incubator.apache.org/ + http://ranger.apache.org/ Apache 2.0 License @@ -36,9 +36,9 @@ - scm:git:git@github.com:apache/incubator-ranger.git - scm:git:https://git-wip-us.apache.org/repos/asf/incubator-ranger.git - scm:git:git@github.com:apache/incubator-ranger.git + scm:git:git@github.com:apache/ranger.git + scm:git:https://git-wip-us.apache.org/repos/asf/ranger.git + scm:git:git@github.com:apache/ranger.git HEAD @@ -51,21 +51,21 @@ Dev Mailing List - dev@ranger.incubator.apache.org - dev-subscribe@ranger.incubator.apache.org - dev-unsubscribe@ranger.incubator.apache.org + dev@ranger.apache.org + dev-subscribe@ranger.apache.org + dev-unsubscribe@ranger.apache.org User Mailing List - user@ranger.incubator.apache.org - user-subscribe@ranger.incubator.apache.org - user-unsubscribe@ranger.incubator.apache.org + user@ranger.apache.org + user-subscribe@ranger.apache.org + user-unsubscribe@ranger.apache.org Commits Mailing List - commits@ranger.incubator.apache.org - commits-subscribe@ranger.incubator.apache.org - commits-unsubscribe@ranger.incubator.apache.org + commits@ranger.apache.org + commits-subscribe@ranger.apache.org + commits-unsubscribe@ranger.apache.org @@ -112,6 +112,8 @@ plugin-atlas + 3.3.3 + 1.7 1.7 1.7 apache.staging.https @@ -253,6 +255,33 @@ unixauthpam + + sign-artifacts + + + sign-artifacts + true + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + @@ -332,29 +361,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - 1.4.1 - - - enforce-versions - - enforce - - - - - 3.0.4 - - - ${java.version} - - - - - - org.codehaus.mojo build-helper-maven-plugin @@ -475,6 +481,29 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4.1 + + + enforce-versions + + enforce + + + + + ${maven.version.required} + + + ${java.version.required} + + + + + + org.apache.maven.plugins maven-pmd-plugin @@ -534,11 +563,12 @@ **/.classpath/** **/*.iml **/target/** - **/bin/** *CHANGES* **/ISSUES **/patchprocess/** - **/test/resources/** + **/test/resources/**/*.json + **/test/resources/**/*.txt + **/test/resources/**/*.csv **/main/resources/**/*.json **/samples/**/*.json **/.externalToolBuilders/* diff --git a/ranger-atlas-plugin-shim/pom.xml b/ranger-atlas-plugin-shim/pom.xml index 63c72312a1..b2e90ed6ec 100644 --- a/ranger-atlas-plugin-shim/pom.xml +++ b/ranger-atlas-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-examples/conditions-enrichers/pom.xml b/ranger-examples/conditions-enrichers/pom.xml index 6dc8f536af..59839a370e 100644 --- a/ranger-examples/conditions-enrichers/pom.xml +++ b/ranger-examples/conditions-enrichers/pom.xml @@ -23,7 +23,7 @@ ranger-examples org.apache.ranger - 0.6.0 + 0.6.4-SNAPSHOT diff --git a/ranger-examples/plugin-sampleapp/pom.xml b/ranger-examples/plugin-sampleapp/pom.xml index 354d099a65..6fee233e1c 100644 --- a/ranger-examples/plugin-sampleapp/pom.xml +++ b/ranger-examples/plugin-sampleapp/pom.xml @@ -24,7 +24,7 @@ ranger-examples org.apache.ranger - 0.6.0 + 0.6.4-SNAPSHOT diff --git a/ranger-examples/pom.xml b/ranger-examples/pom.xml index 8648b83341..76deef534f 100644 --- a/ranger-examples/pom.xml +++ b/ranger-examples/pom.xml @@ -19,7 +19,7 @@ ranger org.apache.ranger - 0.6.0 + 0.6.4-SNAPSHOT 4.0.0 ranger-examples diff --git a/ranger-examples/sampleapp/pom.xml b/ranger-examples/sampleapp/pom.xml index 28fa94ed4c..155a168969 100644 --- a/ranger-examples/sampleapp/pom.xml +++ b/ranger-examples/sampleapp/pom.xml @@ -24,7 +24,7 @@ ranger-examples org.apache.ranger - 0.6.0 + 0.6.4-SNAPSHOT diff --git a/ranger-hbase-plugin-shim/pom.xml b/ranger-hbase-plugin-shim/pom.xml index f0382b26b3..a65df59b21 100644 --- a/ranger-hbase-plugin-shim/pom.xml +++ b/ranger-hbase-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-hdfs-plugin-shim/pom.xml b/ranger-hdfs-plugin-shim/pom.xml index 9195745a5b..2386b6b92a 100644 --- a/ranger-hdfs-plugin-shim/pom.xml +++ b/ranger-hdfs-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-hive-plugin-shim/pom.xml b/ranger-hive-plugin-shim/pom.xml index bf22ec4624..117f8c2977 100644 --- a/ranger-hive-plugin-shim/pom.xml +++ b/ranger-hive-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-hive-utils/pom.xml b/ranger-hive-utils/pom.xml index 18d111fd27..df130b0968 100644 --- a/ranger-hive-utils/pom.xml +++ b/ranger-hive-utils/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-kafka-plugin-shim/pom.xml b/ranger-kafka-plugin-shim/pom.xml index 4613c516a5..6b179e223c 100644 --- a/ranger-kafka-plugin-shim/pom.xml +++ b/ranger-kafka-plugin-shim/pom.xml @@ -28,7 +28,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-kms-plugin-shim/pom.xml b/ranger-kms-plugin-shim/pom.xml index 29ee80da22..847510dcc6 100644 --- a/ranger-kms-plugin-shim/pom.xml +++ b/ranger-kms-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-knox-plugin-shim/pom.xml b/ranger-knox-plugin-shim/pom.xml index a13ba94db2..7444d360d5 100644 --- a/ranger-knox-plugin-shim/pom.xml +++ b/ranger-knox-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-plugin-classloader/pom.xml b/ranger-plugin-classloader/pom.xml index b3a07ea058..3c9fe1f30d 100644 --- a/ranger-plugin-classloader/pom.xml +++ b/ranger-plugin-classloader/pom.xml @@ -28,7 +28,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT diff --git a/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoaderUtil.java b/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoaderUtil.java index ea18883146..fb33dcb8f4 100644 --- a/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoaderUtil.java +++ b/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoaderUtil.java @@ -38,7 +38,7 @@ public class RangerPluginClassLoaderUtil { private static final Logger LOG = LoggerFactory.getLogger(RangerPluginClassLoaderUtil.class) ; - private static RangerPluginClassLoaderUtil config = null; + private static volatile RangerPluginClassLoaderUtil config = null; private static String rangerPluginLibDir = "ranger-%-plugin-impl"; public static RangerPluginClassLoaderUtil getInstance() { diff --git a/ranger-solr-plugin-shim/pom.xml b/ranger-solr-plugin-shim/pom.xml index 32cf3e24b1..395456f055 100644 --- a/ranger-solr-plugin-shim/pom.xml +++ b/ranger-solr-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-storm-plugin-shim/pom.xml b/ranger-storm-plugin-shim/pom.xml index ff3e55d2a2..f9c008f56c 100644 --- a/ranger-storm-plugin-shim/pom.xml +++ b/ranger-storm-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties index 21f7fadc23..4ead8023d5 100644 --- a/ranger-tools/conf/log4j.properties +++ b/ranger-tools/conf/log4j.properties @@ -23,7 +23,7 @@ log4j.threshold=ALL log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.target=System.err log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%m%n +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n # # ranger.perf log level diff --git a/ranger-tools/pom.xml b/ranger-tools/pom.xml index e6c54b4ccb..ca3d0f880b 100644 --- a/ranger-tools/pom.xml +++ b/ranger-tools/pom.xml @@ -19,7 +19,7 @@ ranger org.apache.ranger - 0.6.0 + 0.6.4-SNAPSHOT 4.0.0 ranger-tools diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt index 7dc84e9303..55170d55b4 100644 --- a/ranger-tools/scripts/README.txt +++ b/ranger-tools/scripts/README.txt @@ -67,3 +67,16 @@ This file describes how to build, setup, configure and run the performance testi [RangerPolicyEngine.isAccessAllowed] execCount:64, totalTimeTaken:1873, maxTimeTaken:276, minTimeTaken:4, avgTimeTaken:29 + +RangerPluginPerfTester tool + +Steps 1 - 4 as above.. + +Run the tool with + + % ./ranger-plugin-perftester.sh -s -n -a -r -t -p -c -e + + Example: + % ./ranger-plugin-perftester.sh -s hive -n cl1_hive -a test_hive_plugin -r http://ranger_admin_host -t 30000 -p 30000 -c /tmp/hive/policycache -e nocache + + diff --git a/ranger-tools/scripts/ranger-plugin-perftester.sh b/ranger-tools/scripts/ranger-plugin-perftester.sh new file mode 100755 index 0000000000..eae2fdf656 --- /dev/null +++ b/ranger-tools/scripts/ranger-plugin-perftester.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cdir=$(cd "$(dirname "$0")"; pwd) +cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf:." + +if [ "${JAVA_HOME}" != "" ] +then + export JAVA_HOME + PATH="${JAVA_HOME}/bin:${PATH}" + export PATH +fi + +JAVA_CMD="java -cp ${cp} org.apache.ranger.policyengine.RangerPluginPerfTester" + +cd ${cdir} + +echo "JAVA command = $JAVA_CMD " "$@" +$JAVA_CMD "$@" diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index ab7d5323fb..e8edd9eeb8 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -52,6 +52,9 @@ public class CommandLineParser private int concurrentClientCount = 1; private int iterationsCount = 1; + private boolean isDynamicReorderingDisabled = true; + private boolean isTrieLookupPrefixDisabled = true; + private Options options = new Options(); CommandLineParser() {} @@ -60,7 +63,7 @@ final PerfTestOptions parse(final String[] args) { PerfTestOptions ret = null; if (parseArguments(args) && validateInputFiles()) { // Instantiate a data-object and return - ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount); + ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled); } else { showUsage(); } @@ -75,6 +78,7 @@ final PerfTestOptions parse(final String[] args) { -r request-file-name-list -n number-of-iterations -p modules-to-collect-stats + -o If the concurrent-client-count is more than the number of files in the request-file-name-list, then reuse the request-file-names in a round-robin way @@ -94,6 +98,8 @@ final boolean parseArguments(final String[] args) { options.addOption("p", "statistics", true, "Modules for stat collection File Name"); options.addOption("c", "clients", true, "Number of concurrent clients"); options.addOption("n", "cycles", true, "Number of iterations"); + options.addOption("o", "optimize", false, "Enable usage-based policy reordering"); + options.addOption("t", "trie-prefilter", false, "Enable trie-prefilter"); org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser(); @@ -120,9 +126,18 @@ final boolean parseArguments(final String[] args) { if (iterationsOptionValue != null) { iterationsCount = Integer.parseInt(iterationsOptionValue); } + if (commandLine.hasOption("o")) { + isDynamicReorderingDisabled = false; + } + if (commandLine.hasOption("t")) { + isTrieLookupPrefixDisabled = false; + } + if (LOG.isDebugEnabled()) { LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames)); LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount); + LOG.debug("isDynamicReorderingDisabled=" + isDynamicReorderingDisabled); + LOG.debug("isTrieLookupPrefixDisabled=" + isTrieLookupPrefixDisabled); } ret = true; diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java index 38c6e2f44c..8d89794be2 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java @@ -31,15 +31,22 @@ import java.io.Reader; import java.net.URL; import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicLong; public class PerfTestEngine { static final Log LOG = LogFactory.getLog(PerfTestEngine.class); + static private final long POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT = 100; private final URL servicePoliciesFileURL; + private final RangerPolicyEngineOptions policyEngineOptions; private RangerPolicyEngine policyEvaluationEngine; + private final boolean disableDynamicPolicyEvalReordering; + private AtomicLong requestCount = new AtomicLong(); - public PerfTestEngine(final URL servicePoliciesFileURL) { + public PerfTestEngine(final URL servicePoliciesFileURL, RangerPolicyEngineOptions policyEngineOptions, boolean disableDynamicPolicyEvalReordering) { this.servicePoliciesFileURL = servicePoliciesFileURL; + this.policyEngineOptions = policyEngineOptions; + this.disableDynamicPolicyEvalReordering = disableDynamicPolicyEvalReordering; } public boolean init() { @@ -64,10 +71,9 @@ public boolean init() { servicePolicies = gsonBuilder.fromJson(reader, ServicePolicies.class); - RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions(); - engineOptions.disableTagPolicyEvaluation = false; + policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, policyEngineOptions); - policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, engineOptions); + requestCount.set(0L); ret = true; @@ -101,6 +107,12 @@ public RangerAccessResult execute(final RangerAccessRequest request) { if (policyEvaluationEngine != null) { + long processedRequestCount = requestCount.getAndIncrement(); + + if (!disableDynamicPolicyEvalReordering && (processedRequestCount % POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT) == 0) { + policyEvaluationEngine.reorderPolicyEvaluators(); + } + policyEvaluationEngine.preProcess(request); ret = policyEvaluationEngine.isAccessAllowed(request, null); diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java index f30cbd7521..d6e04eaadf 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java @@ -26,17 +26,21 @@ public class PerfTestOptions { private final URL servicePoliciesFileURL; private final URL[] requestFileURLs; private final URL statCollectionFileURL; + private final boolean isDynamicReorderingDisabled; + private final boolean isTrieLookupPrefixDisabled; private final int concurrentClientCount; private final int iterationsCount; - PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) { + PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled) { this.servicePoliciesFileURL = servicePoliciesFileURL; this.requestFileURLs = requestFileURLs; this.statCollectionFileURL = statCollectionFileURL; this.iterationsCount = iterationsCount; this.concurrentClientCount = concurrentClientCount; + this.isDynamicReorderingDisabled = isDynamicReorderingDisabled; + this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled; } public URL getServicePoliciesFileURL() { @@ -57,4 +61,9 @@ public int getConcurrentClientCount() { public int getIterationsCount() { return iterationsCount; - }} + } + + public boolean getIsDynamicReorderingDisabled() { return isDynamicReorderingDisabled; } + + public boolean getIsTrieLookupPrefixDisabled() { return isTrieLookupPrefixDisabled; } +} diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPluginPerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPluginPerfTester.java new file mode 100644 index 0000000000..803d737e9e --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPluginPerfTester.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.policyengine; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.ranger.plugin.service.RangerBasePlugin; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.File; + +public class RangerPluginPerfTester { + + static RangerBasePlugin plugin = null; + + private static String serviceType; + private static String serviceName; + private static String appId; + private static String rangerHostName; + private static int socketReadTimeout = 30*1000; + private static int pollingInterval = 30*1000; + private static String policyCacheDir; + private static boolean useCachedPolicyEvaluator = false; + + private static Options options = new Options(); + + public static void main(String[] args) { + + if (!parseArguments(args)) { + System.err.println("Exiting.. "); + System.exit(-1); + } + + + + System.out.println("Arguments:"); + System.out.println("\t\tservice-type:\t\t\t" + serviceType); + System.out.println("\t\tservice-name:\t\t\t" + serviceName); + System.out.println("\t\tapp-id:\t\t\t\t" + appId); + System.out.println("\t\tranger-host:\t\t\t" + rangerHostName); + System.out.println("\t\tsocket-read-timeout:\t\t" + socketReadTimeout); + System.out.println("\t\tpolling-interval:\t\t" + pollingInterval); + System.out.println("\t\tpolicy-cache-dir:\t\t" + policyCacheDir); + System.out.println("\t\tuse-cached-policy-evaluator:\t" + useCachedPolicyEvaluator); + System.out.println("\n\n"); + + + Path filePath = buildConfigurationFile(); + + if (filePath != null) { + RangerConfiguration rangerConfig = RangerConfiguration.getInstance(); + rangerConfig.addResource(filePath); + + plugin = new RangerBasePlugin(serviceType, appId); + + Runtime runtime = Runtime.getRuntime(); + runtime.gc(); + + long totalMemory = runtime.totalMemory(); + long freeMemory = runtime.freeMemory(); + + System.out.println("Initial Memory Statistics:"); + System.out.println("\t\tMaximum Memory available for the process:\t" + runtime.maxMemory()); + System.out.println("\t\tInitial In-Use memory:\t\t\t\t" + (totalMemory-freeMemory)); + System.out.println("\t\tInitial Free memory:\t\t\t\t" + freeMemory); + + System.out.println("\n\n"); + + plugin.init(); + + while (true) { + + runtime.gc(); + + freeMemory = runtime.freeMemory(); + totalMemory = runtime.totalMemory(); + + System.out.println("Memory Statistics:"); + System.out.println("\t\tCurrently In-Use memory:\t" + (totalMemory-freeMemory)); + System.out.println("\t\tCurrently Free memory:\t\t" + freeMemory); + + System.out.println("\n\n"); + + try { + Thread.sleep(60 * 1000); + } catch (InterruptedException e) { + + System.err.println("Main thread interrupted..., exiting..."); + break; + } + } + } else { + System.err.println("Failed to build configuration file"); + } + } + + static Path buildConfigurationFile() { + + Path ret = null; + + String propertyPrefix = "ranger.plugin." + serviceType; + String policyEvaluatorType = useCachedPolicyEvaluator ? RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED : RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + + try { + + File file = File.createTempFile("ranger-plugin-test-site", ".xml") ; + file.deleteOnExit(); + + String filePathStr = file.getAbsolutePath() ; + + Path filePath = new Path(filePathStr); + + FileSystem fs = filePath.getFileSystem(new Configuration()); + + FSDataOutputStream outStream = fs.create(filePath, true); + + OutputStreamWriter writer = new OutputStreamWriter(outStream); + + writer.write("\n" + + " \n" + + " " + propertyPrefix + ".policy.pollIntervalMs\n" + + " " + pollingInterval + "\n" + + " \n" + + " \n" + + " " + propertyPrefix + ".policy.cache.dir\n" + + " " + policyCacheDir + "\n" + + " \n" + + " \n" + + " " + propertyPrefix + ".policy.rest.url\n" + + " " + rangerHostName + ":6080" + "\n" + + " \n" + + " \n" + + " " + propertyPrefix + ".policy.source.impl\n" + + " org.apache.ranger.admin.client.RangerAdminRESTClient\n" + + " \n" + + " \n" + + " " + propertyPrefix + ".policy.rest.client.read.timeoutMs\n" + + " " + socketReadTimeout + "\n" + + " \n" + + " \n" + + " " + propertyPrefix + ".policyengine.option.evaluator.type\n" + + " " + policyEvaluatorType + "\n" + + " \n" + + " \n" + + " " + propertyPrefix + ".service.name\n" + + " " + serviceName + "\n" + + " \n" + + + " \n" + + " xasecure.audit.is.enabled\n" + + " false\n" + + " \n" + + "\n"); + + writer.close(); + ret = filePath; + + } catch (IOException exception) { + //Ignore + } + + return ret; + + } + + + static boolean parseArguments(final String[] args) { + + boolean ret = false; + + options.addOption("h", "help", false, "show help."); + options.addOption("s", "service-type", true, "Service-Type"); + options.addOption("n", "service-name", true, "Ranger service-name "); + options.addOption("a", "app-id", true, "Application-Id"); + options.addOption("r", "ranger-host", true, "Ranger host-name"); + options.addOption("t", "socket-read-timeout", true, "Read timeout on socket in milliseconds"); + options.addOption("p", "polling-interval", true, "Polling Interval in milliseconds"); + options.addOption("c", "policy-cache-dir", true, "Policy-Cache directory "); + options.addOption("e", "policy-evaluator-type", true, "Policy-Evaluator-Type (Cached/Other"); + + DefaultParser commandLineParser = new DefaultParser(); + + try { + CommandLine commandLine = commandLineParser.parse(options, args); + + if (commandLine.hasOption("h")) { + showUsage(); + return false; + } + + serviceType = commandLine.getOptionValue("s"); + serviceName = commandLine.getOptionValue("n"); + appId = commandLine.getOptionValue("a"); + rangerHostName = commandLine.getOptionValue("r"); + policyCacheDir = commandLine.getOptionValue("c"); + + try { + String pollingIntervalStr = commandLine.getOptionValue("p"); + pollingInterval = Integer.parseInt(pollingIntervalStr); + } catch (NumberFormatException exception) { + // Ignore + } + + String useCachedPolicyEvaluatorStr = commandLine.getOptionValue("e"); + if (StringUtils.equalsIgnoreCase(useCachedPolicyEvaluatorStr, "cache")) { + useCachedPolicyEvaluator = true; + } + + try { + String socketReadTimeoutStr = commandLine.getOptionValue("t"); + socketReadTimeout = Integer.parseInt(socketReadTimeoutStr); + } catch (NumberFormatException exception) { + // Ignore + } + + ret = true; + + } catch (ParseException exception) { + System.err.println("Failed to parse arguments:" + exception); + } + + return ret; + } + + static void showUsage() { + HelpFormatter formater = new HelpFormatter(); + formater.printHelp("plugin-tester", options); + } + +} + diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index 42d7cde7ac..056c548703 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -21,6 +21,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.PerfDataRecorder; import java.io.BufferedReader; @@ -54,7 +56,12 @@ public static void main(String[] args) { URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL(); - PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL); + RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); + policyEngineOptions.disableTagPolicyEvaluation = false; + policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled(); + + PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled()); if (!perfTestEngine.init()) { LOG.error("Error initializing test data. Existing..."); System.exit(1); @@ -63,6 +70,26 @@ public static void main(String[] args) { URL[] requestFileURLs = perfTestOptions.getRequestFileURLs(); int requestFilesCount = requestFileURLs.length; + // warm-up policy engine + LOG.error("Warming up.."); + try { + for(URL requestFileURL : requestFileURLs) { + PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, 0, requestFileURL, 1); + + if (perfTestClient.init()) { + perfTestClient.start(); + perfTestClient.join(); + } else { + LOG.error("Error initializing warm-up PerfTestClient"); + } + } + } catch(Throwable t) { + LOG.error("Error during warmup", t); + } + LOG.error("Warmed up!"); + + PerfDataRecorder.clearStatistics(); + int clientsCount = perfTestOptions.getConcurrentClientCount(); List perfTestClients = new ArrayList(clientsCount); @@ -83,6 +110,15 @@ public static void main(String[] args) { LOG.debug("Number of perfTestClients=" + perfTestClients.size()); } + Runtime runtime = Runtime.getRuntime(); + runtime.gc(); + + long totalMemory = runtime.totalMemory(); + long freeMemory = runtime.freeMemory(); + + LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory); + + LOG.info("Starting " + perfTestClients.size() + " clients.."); for (PerfTestClient client : perfTestClients) { try { client.start(); @@ -90,19 +126,28 @@ public static void main(String[] args) { LOG.error("Error in starting client: " + client.getName(), t); } } + LOG.info("Started " + perfTestClients.size() + " clients"); LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up"); for (PerfTestClient client : perfTestClients) { - try { - if (client.isAlive()) { - LOG.info("Waiting for " + client.getName() + " to finish up."); - client.join(); + while (client.isAlive()) { + try { + client.join(1000); + + runtime.gc(); + + totalMemory = runtime.totalMemory(); + freeMemory = runtime.freeMemory(); + + LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory); + + } catch (InterruptedException interruptedException) { + LOG.error("PerfTestClient.join() was interrupted"); } - } catch (InterruptedException interruptedException) { - LOG.error("PerfTestClient.join() was interrupted"); } } + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyenginePerfTester.main()"); } diff --git a/ranger-tools/src/test/resources/testdata/test_modules.txt b/ranger-tools/src/test/resources/testdata/test_modules.txt index 8eb57469cf..a355ec87d9 100644 --- a/ranger-tools/src/test/resources/testdata/test_modules.txt +++ b/ranger-tools/src/test/resources/testdata/test_modules.txt @@ -21,4 +21,5 @@ RangerTagEnricher.setServiceTags RangerPolicyEngine.init RangerPolicyEngine.preProcess RangerPolicyEngine.isAccessAllowedNoAudit - +RangerPolicyEngine.reorderPolicyEvaluators +RangerPolicyEngine.usage diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt index f317aaf734..f3a2f82fde 100644 --- a/ranger-tools/testdata/test_modules.txt +++ b/ranger-tools/testdata/test_modules.txt @@ -20,6 +20,8 @@ PolicyRefresher.loadPolicy RangerPolicyEngine.init RangerPolicyEngine.cleanUp +RangerPolicyEngine.reorderPolicyEvaluators +RangerPolicyEngine.usage RangerContextEnricher.init RangerPolicyEvaluator.init RangerPolicyItemEvaluator.init @@ -58,4 +60,4 @@ ServiceREST.getPolicy ServiceREST.getPolicies ServiceREST.countPolicies ServiceREST.getServicePolicies -ServiceREST.getServicePoliciesIfUpdated \ No newline at end of file +ServiceREST.getServicePoliciesIfUpdated diff --git a/ranger-util/pom.xml b/ranger-util/pom.xml index 17612eb9d1..195f89bf29 100644 --- a/ranger-util/pom.xml +++ b/ranger-util/pom.xml @@ -20,7 +20,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT ranger-util Ranger Util diff --git a/ranger-yarn-plugin-shim/pom.xml b/ranger-yarn-plugin-shim/pom.xml index 51a7f0f71e..12dc73ebb5 100644 --- a/ranger-yarn-plugin-shim/pom.xml +++ b/ranger-yarn-plugin-shim/pom.xml @@ -27,7 +27,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT .. diff --git a/ranger_solrj/pom.xml b/ranger_solrj/pom.xml index e7267e5e52..787d062267 100644 --- a/ranger_solrj/pom.xml +++ b/ranger_solrj/pom.xml @@ -20,7 +20,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT ranger_solrj ranger_solrj diff --git a/release-build.xml b/release-build.xml index b968d859fa..8f0629d0e9 100644 --- a/release-build.xml +++ b/release-build.xml @@ -21,7 +21,7 @@ - + diff --git a/security-admin/db/oracle/create_dbversion_catalog.sql b/security-admin/db/oracle/create_dbversion_catalog.sql index 0017748c57..5820cc8c80 100644 --- a/security-admin/db/oracle/create_dbversion_catalog.sql +++ b/security-admin/db/oracle/create_dbversion_catalog.sql @@ -22,4 +22,5 @@ create table X_DB_VERSION_H ( updated_by VARCHAR(256) NOT NULL, active VARCHAR(1) DEFAULT 'Y' ); +CREATE SEQUENCE X_DB_VERSION_H_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; commit; diff --git a/security-admin/db/oracle/xa_core_db_oracle.sql b/security-admin/db/oracle/xa_core_db_oracle.sql index 2ede81037c..d377482b5e 100644 --- a/security-admin/db/oracle/xa_core_db_oracle.sql +++ b/security-admin/db/oracle/xa_core_db_oracle.sql @@ -31,7 +31,6 @@ CREATE SEQUENCE X_PORTAL_USER_ROLE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYC CREATE SEQUENCE X_RESOURCE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; CREATE SEQUENCE X_TRX_LOG_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; CREATE SEQUENCE X_USER_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; -CREATE SEQUENCE X_DB_VERSION_H_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; CREATE SEQUENCE V_TRX_LOG_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; CREATE SEQUENCE XA_ACCESS_AUDIT_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE; commit; diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 218276b7f2..47f23f1bcd 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -24,7 +24,7 @@ org.apache.ranger ranger - 0.6.0 + 0.6.4-SNAPSHOT diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index a2b26f568c..c104ecb78c 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -27,6 +27,7 @@ import time import datetime from time import gmtime, strftime +import socket globalDict = {} os_name = platform.system() @@ -41,6 +42,11 @@ elif os_name == "WINDOWS": RANGER_ADMIN_HOME = os.getenv("RANGER_ADMIN_HOME") +if socket.getfqdn().find('.')>=0: + client_host=socket.getfqdn() +else: + client_host=socket.gethostbyaddr(socket.gethostname())[0] + def check_output(query): if os_name == "LINUX": p = subprocess.Popen(shlex.split(query), stdout=subprocess.PIPE) @@ -105,11 +111,10 @@ def check_table(self, db_name, db_user, db_password, TABLE_NAME): log("[I] ---------- Verifying table ----------", "info") def import_db_file(self, db_name, db_user, db_password, file_name): - log("[I] ---------- Importing db schema ----------", "info") + log("[I] Importing DB file :"+file_name, "info") - def upgrade_db(self, db_name, db_user, db_password, DBVERSION_CATALOG_CREATION): - self.import_db_file(db_name, db_user, db_password, DBVERSION_CATALOG_CREATION) - log("[I] Baseline DB upgraded successfully", "info") + def create_version_history_table(self, db_name, db_user, db_password, DBVERSION_CATALOG_CREATION,TABLE_NAME): + log("[I] Creating version and patch history info table", "info") def apply_patches(self, db_name, db_user, db_password, PATCHES_PATH): #first get all patches and then apply each patch @@ -153,6 +158,9 @@ def create_synonym(db_name, db_user, db_password,audit_db_user): def change_admin_default_password(xa_db_host, db_user, db_password, db_name,userName,oldPassword,newPassword): log("[I] ----------------- Changing Ranger admin default password ------------", "info") + def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_table,last_table): + log("[I] ---------- Importing Core DB Schema ----------", "info") + class MysqlConf(BaseDB): # Constructor def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN): @@ -207,6 +215,7 @@ def grant_audit_db_user(self, db_user, audit_db_name, audit_db_user, audit_db_pa sys.exit(1) def import_db_file(self, db_name, db_user, db_password, file_name): + isImported=False name = basename(file_name) if os.path.isfile(file_name): log("[I] Importing db schema to database " + db_name + " from file: " + name,"info") @@ -220,13 +229,14 @@ def import_db_file(self, db_name, db_user, db_password, file_name): jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: - log("[I] "+name + " DB schema imported successfully","info") + log("[I] "+name + " file imported successfully","info") + isImported=True else: - log("[E] "+name + " DB schema import failed!","error") - sys.exit(1) + log("[E] "+name + " file import failed!","error") else: log("[E] DB schema file " + name+ " not found","error") sys.exit(1) + return isImported def import_db_patches(self, db_name, db_user, db_password, file_name): name = basename(file_name) @@ -257,11 +267,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), user(), now(), user(),'N') ;\"" %(version) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), user(), now(), user(),'N') ;\" -c ;" %(version) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -280,25 +290,33 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -336,11 +354,11 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a output = check_output(query) else: if os_name == "LINUX": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), user(), now(), user(),'N') ;\"" %(version) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), user(), now(), user(),'N') ;\" -c ;" %(version) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -360,25 +378,33 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -466,11 +492,11 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', now(), user(), now(), user(),'N') ;\"" %(version) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', now(), '%s', now(), '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', now(), user(), now(), user(),'N') ;\" -c ;" %(version) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', now(), '%s', now(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -489,33 +515,33 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] java patch "+ className +" is applied..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] java patch "+ className +" failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] applying java patch "+ className +" failed", "error") @@ -555,11 +581,11 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), user(), now(), user(),'N') ;\"" %(version) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), user(), now(), user(),'N') ;\" -c ;" %(version) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -578,38 +604,154 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] Ranger admin default password change request processed successfully..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) + def create_version_history_table(self, db_name, db_user, db_password, file_name,table_name): + name = basename(file_name) + if os.path.isfile(file_name): + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + if isTableExist==False: + log("[I] Importing "+table_name+" table schema to database " + db_name + " from file: " + name,"info") + while(isTableExist==False): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -input %s" %file_name + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -input %s -c ;" %file_name + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " file imported successfully","info") + else: + log("[E] "+name + " file import failed!","error") + time.sleep(30) + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + else: + log("[E] Table schema file " + name+ " not found","error") + sys.exit(1) + + def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_table,last_table): + version = 'CORE_DB_SCHEMA' + if os.path.isfile(file_name): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + log("[I] "+version+" is already imported" ,"info") + else: + if os_name == "LINUX": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + while(output.strip(version + " |")): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + jisql_log(query, db_password) + output = check_output(query) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\"" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\" -c ;" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret != 0: + log("[E] "+ version +" import failed", "error") + sys.exit(1) + isFirstTableExist = self.check_table(db_name, db_user, db_password, first_table) + isLastTableExist = self.check_table(db_name, db_user, db_password, last_table) + isSchemaCreated=False + if isFirstTableExist == True and isLastTableExist == True : + isSchemaCreated=True + elif isFirstTableExist == True and isLastTableExist == False : + while(isLastTableExist==False): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + elif isFirstTableExist == False and isLastTableExist == False : + isImported=self.import_db_file(db_name, db_user, db_password, file_name) + if(isImported==False): + log("[I] "+ version +" might being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + if isSchemaCreated == True: + if os_name == "LINUX": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+version +" import status has been updated", "info") + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] Updating "+version +" import status failed", "error") + sys.exit(1) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] "+version + " import failed!","error") + sys.exit(1) class OracleConf(BaseDB): # Constructor @@ -679,6 +821,7 @@ def grant_audit_db_user(self, audit_db_name ,db_user,audit_db_user,db_password,a sys.exit(1) def import_db_file(self, db_name, db_user, db_password, file_name): + isImported=False name = basename(file_name) if os.path.isfile(file_name): log("[I] Importing script " + db_name + " from file: " + name,"info") @@ -693,9 +836,13 @@ def import_db_file(self, db_name, db_user, db_password, file_name): ret = subprocess.call(query) if ret == 0: log("[I] "+name + " imported successfully","info") + isImported=True else: log("[E] "+name + " import failed!","error") - sys.exit(1) + else: + log("[E] DB schema file " + name+ " not found","error") + sys.exit(1) + return isImported def create_synonym(self,db_name, db_user, db_password,audit_db_user): log("[I] ----------------- Creating Synonym ------------", "info") @@ -750,11 +897,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, db_user, db_user) + query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, db_user, db_user) + query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -772,25 +919,33 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " Import failed!","error") @@ -828,11 +983,11 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a output = check_output(query) else: if os_name == "LINUX": - query = get_cmd1 + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, db_user, db_user) + query = get_cmd1 + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, db_user, db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -851,25 +1006,33 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd1 + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd1 + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd1 + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version, client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version, client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd1 + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd1 + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " Import failed!","error") @@ -963,24 +1126,45 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): log("[I] java patch "+ className +" is already applied" ,"info") else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = 'J%s' and active = 'Y';\"" %(version) + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = 'J%s' and active = 'N';\"" %(version) elif os_name == "WINDOWS": - query = get_cmd + " -query \"select version from x_db_version_h where version = 'J%s' and active = 'Y';\" -c ;" %(version) + query = get_cmd + " -query \"select version from x_db_version_h where version = 'J%s' and active = 'N';\" -c ;" %(version) jisql_log(query, db_password) output = check_output(query) - if output.strip(version + " |"): - while(output.strip(version + " |")): - log("[I] Java patch "+ className +" is being applied by some other process" ,"info") - time.sleep(300) - jisql_log(query, db_password) - output = check_output(query) + if output.strip(version + " |"):# + if os_name == "LINUX": + queryInstallAt = get_cmd + " -c \; -query \"select version from x_db_version_h where version = 'J%s' and active = 'N' and inst_at<=CURRENT_DATE-1;\"" %(version) + elif os_name == "WINDOWS": + queryInstallAt = get_cmd + " -query \"select version from x_db_version_h where version = 'J%s' and active = 'N' inst_at<=CURRENT_DATE-1;\" -c ;" %(version) + jisql_log(queryInstallAt, db_password) + outputInstallAt = check_output(queryInstallAt) + if outputInstallAt.strip(version + " |"): + if os_name == "LINUX": + queryUpdate = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s' and inst_at<=CURRENT_DATE-1;\"" %(version, client_host) + jisql_log(queryUpdate, db_password) + retUpdate = subprocess.call(shlex.split(queryUpdate)) + elif os_name == "WINDOWS": + queryUpdate = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s' and inst_at<=CURRENT_DATE-1;\" -c ;" %(version, client_host) + jisql_log(queryUpdate, db_password) + retUpdate = subprocess.call(queryUpdate) + if retUpdate == 0: + log ("[I] java patch "+ className +" status entry has been updated to continue upgrade process.","info") + else: + log("[E] java patch "+ className +" status entry update request failed, Please update/delete java patches entries having active status 'N' from x_db_version_h table and try again!", "error") + sys.exit(1) + else: + while(output.strip(version + " |")): + log("[I] Java patch "+ className +" is being applied by some other process" ,"info") + time.sleep(300) + jisql_log(query, db_password) + output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by) values ( X_DB_VERSION_H_SEQ.nextval,'J%s', sysdate, '%s', sysdate, '%s');\"" %(version, db_user, db_user) + query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'J%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by) values ( X_DB_VERSION_H_SEQ.nextval,'J%s', sysdate, '%s', sysdate, '%s');\" -c ;" %(version, db_user, db_user) + query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'J%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -992,40 +1176,40 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_cmd = "%s -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) + get_java_cmd = "%s -Djava.security.egd=file:///dev/urandom -Dlogdir=%s -Dlog4j.configuration=db_patch.log4j.xml -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,ranger_log,path,className) if os_name == "LINUX": - ret = subprocess.call(shlex.split(get_cmd)) + ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": - ret = subprocess.call(get_cmd) + ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\"" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] java patch "+ className +" is applied..","info") else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\"" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version, client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] java patch "+ className +" failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] applying java patch "+ className +" failed", "error") @@ -1065,11 +1249,11 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s');\"" %(version, db_user, db_user) + query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s');\" -c ;" %(version, db_user, db_user) + query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, client_host, client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1088,38 +1272,155 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] Ranger admin default password change request processed successfully..","info") else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) + def create_version_history_table(self, db_name, db_user, db_password, file_name,table_name): + name = basename(file_name) + if os.path.isfile(file_name): + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + if isTableExist==False: + log("[I] Importing "+table_name+" table schema from file: " + name,"info") + while(isTableExist==False): + get_cmd = self.get_jisql_cmd(db_user, db_password) + if os_name == "LINUX": + query = get_cmd + " -input %s -c \;" %file_name + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -input %s -c ;" %file_name + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " file imported successfully","info") + else: + log("[E] "+name + " file import failed!","error") + time.sleep(30) + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + else: + log("[E] Table schema file " + name+ " not found","error") + sys.exit(1) + + def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_table,last_table): + version = 'CORE_DB_SCHEMA' + if os.path.isfile(file_name): + get_cmd = self.get_jisql_cmd(db_user, db_password) + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + log("[I] "+version+" is already imported" ,"info") + else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'N';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + while(output.strip(version + " |")): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + jisql_log(query, db_password) + output = check_output(query) + else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\"" %(version, client_host, client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by,active) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s','N');\" -c ;" %(version, client_host, client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret != 0: + log("[E] "+ version +" import failed", "error") + sys.exit(1) + isFirstTableExist = self.check_table(db_name, db_user, db_password, first_table) + isLastTableExist = self.check_table(db_name, db_user, db_password, last_table) + isSchemaCreated=False + if isFirstTableExist == True and isLastTableExist == True : + isSchemaCreated=True + elif isFirstTableExist == True and isLastTableExist == False : + while(isLastTableExist==False): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + elif isFirstTableExist == False and isLastTableExist == False : + isImported=self.import_db_file(db_name, db_user, db_password, file_name) + if(isImported==False): + log("[I] "+ version +" might being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + if isSchemaCreated == True: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+version +" import status has been updated", "info") + else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] Updating "+version +" import status failed", "error") + sys.exit(1) + else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] "+version + " import failed!","error") + sys.exit(1) + class PostgresConf(BaseDB): # Constructor def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): @@ -1154,6 +1455,7 @@ def check_connection(self, db_name, db_user, db_password): sys.exit(1) def import_db_file(self, db_name, db_user, db_password, file_name): + isImported=False name = basename(file_name) if os.path.isfile(file_name): log("[I] Importing db schema to database " + db_name + " from file: " + name,"info") @@ -1168,9 +1470,13 @@ def import_db_file(self, db_name, db_user, db_password, file_name): ret = subprocess.call(query) if ret == 0: log("[I] "+name + " DB schema imported successfully","info") + isImported=True else: log("[E] "+name + " DB schema import failed!","error") - sys.exit(1) + else: + log("[E] DB schema file " + name+ " not found","error") + sys.exit(1) + return isImported def grant_audit_db_user(self, audit_db_name , db_user, audit_db_user, db_password, audit_db_password): log("[I] Granting permission to " + audit_db_user, "info") @@ -1254,11 +1560,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1276,25 +1582,33 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -1333,11 +1647,11 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a output = check_output(query) else: if os_name == "LINUX": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,db_user,db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1356,25 +1670,33 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -1465,11 +1787,11 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1488,33 +1810,33 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] java patch "+ className +" is applied..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] java patch "+ className +" failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] applying java patch "+ className +" failed", "error") @@ -1554,11 +1876,11 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1577,38 +1899,155 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] Ranger admin default password change request processed successfully..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\"" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) + def create_version_history_table(self, db_name, db_user, db_password, file_name,table_name): + name = basename(file_name) + if os.path.isfile(file_name): + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + if isTableExist==False: + log("[I] Importing "+table_name+" table schema to database " + db_name + " from file: " + name,"info") + while(isTableExist==False): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -input %s" %file_name + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -input %s -c ;" %file_name + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " file imported successfully","info") + else: + log("[E] "+name + " file import failed!","error") + time.sleep(30) + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + else: + log("[E] Table schema file " + name+ " not found","error") + sys.exit(1) + + def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_table,last_table): + version = 'CORE_DB_SCHEMA' + if os.path.isfile(file_name): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + log("[I] "+version+" is already imported" ,"info") + else: + if os_name == "LINUX": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + while(output.strip(version + " |")): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + jisql_log(query, db_password) + output = check_output(query) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', current_timestamp, '%s', current_timestamp, '%s','N') ;\"" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', now(), '%s', now(), '%s','N') ;\" -c ;" %(version) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret != 0: + log("[E] "+ version +" import failed", "error") + sys.exit(1) + isFirstTableExist = self.check_table(db_name, db_user, db_password, first_table) + isLastTableExist = self.check_table(db_name, db_user, db_password, last_table) + isSchemaCreated=False + if isFirstTableExist == True and isLastTableExist == True : + isSchemaCreated=True + elif isFirstTableExist == True and isLastTableExist == False : + while(isLastTableExist==False): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + elif isFirstTableExist == False and isLastTableExist == False : + isImported=self.import_db_file(db_name, db_user, db_password, file_name) + if(isImported==False): + log("[I] "+ version +" might being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + if isSchemaCreated == True: + if os_name == "LINUX": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+version +" import status has been updated", "info") + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] Updating "+version +" import status failed", "error") + sys.exit(1) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\"" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] "+version + " import failed!","error") + sys.exit(1) + class SqlServerConf(BaseDB): # Constructor def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): @@ -1643,6 +2082,7 @@ def check_connection(self, db_name, db_user, db_password): sys.exit(1) def import_db_file(self, db_name, db_user, db_password, file_name): + isImported=False name = basename(file_name) if os.path.isfile(file_name): log("[I] Importing db schema to database " + db_name + " from file: " + name,"info") @@ -1657,9 +2097,13 @@ def import_db_file(self, db_name, db_user, db_password, file_name): ret = subprocess.call(query) if ret == 0: log("[I] "+name + " DB schema imported successfully","info") + isImported=True else: log("[E] "+name + " DB Schema import failed!","error") - sys.exit(1) + else: + log("[E] DB schema file " + name+ " not found","error") + sys.exit(1) + return isImported def check_table(self, db_name, db_user, db_password, TABLE_NAME): get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) @@ -1721,11 +2165,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1743,25 +2187,33 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -1800,11 +2252,11 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a else: get_cmd2 = self.get_jisql_cmd(db_user, db_password, audit_db_name) if os_name == "LINUX": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1822,25 +2274,33 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -1915,11 +2375,11 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -1938,33 +2398,33 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] java patch "+ className +" is applied..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] java patch "+ className +" failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] applying java patch "+ className +" failed", "error") @@ -2004,11 +2464,11 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -2027,38 +2487,155 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] Ranger admin default password change request processed successfully..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) + def create_version_history_table(self, db_name, db_user, db_password, file_name,table_name): + name = basename(file_name) + if os.path.isfile(file_name): + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + if isTableExist==False: + log("[I] Importing "+table_name+" table schema to database " + db_name + " from file: " + name,"info") + while(isTableExist==False): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -input %s" %file_name + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -input %s" %file_name + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " file imported successfully","info") + else: + log("[E] "+name + " file import failed!","error") + time.sleep(30) + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + else: + log("[E] Table schema file " + name+ " not found","error") + sys.exit(1) + + def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_table,last_table): + version = 'CORE_DB_SCHEMA' + if os.path.isfile(file_name): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + log("[I] "+version+" is already imported" ,"info") + else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'N';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + while(output.strip(version + " |")): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + jisql_log(query, db_password) + output = check_output(query) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret != 0: + log("[E] "+ version +" import failed", "error") + sys.exit(1) + isFirstTableExist = self.check_table(db_name, db_user, db_password, first_table) + isLastTableExist = self.check_table(db_name, db_user, db_password, last_table) + isSchemaCreated=False + if isFirstTableExist == True and isLastTableExist == True : + isSchemaCreated=True + elif isFirstTableExist == True and isLastTableExist == False : + while(isLastTableExist==False): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + elif isFirstTableExist == False and isLastTableExist == False : + isImported=self.import_db_file(db_name, db_user, db_password, file_name) + if(isImported==False): + log("[I] "+ version +" might being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + if isSchemaCreated == True: + if os_name == "LINUX": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+version +" import status has been updated", "info") + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] Updating "+version +" import status failed", "error") + sys.exit(1) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] "+version + " import failed!","error") + sys.exit(1) + class SqlAnywhereConf(BaseDB): # Constructor def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): @@ -2092,6 +2669,7 @@ def check_connection(self, db_name, db_user, db_password): sys.exit(1) def import_db_file(self, db_name, db_user, db_password, file_name): + isImported=False name = basename(file_name) if os.path.isfile(file_name): log("[I] Importing db schema to database " + db_name + " from file: " + name,"info") @@ -2106,9 +2684,13 @@ def import_db_file(self, db_name, db_user, db_password, file_name): ret = subprocess.call(query) if ret == 0: log("[I] "+name + " DB schema imported successfully","info") + isImported=True else: log("[E] "+name + " DB Schema import failed!","error") - sys.exit(1) + else: + log("[E] DB schema file " + name+ " not found","error") + sys.exit(1) + return isImported def check_table(self, db_name, db_user, db_password, TABLE_NAME): self.set_options(db_name, db_user, db_password, TABLE_NAME) @@ -2171,11 +2753,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -2193,11 +2775,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): if ret == 0: log("[I] "+name + " patch applied","info") if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -2207,11 +2789,11 @@ def import_db_patches(self, db_name, db_user, db_password, file_name): sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -2249,11 +2831,11 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a output = check_output(query) else: if os_name == "LINUX": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -2272,25 +2854,33 @@ def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, a if ret == 0: if os_name == "LINUX": log("[I] "+name + " patch applied","info") - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd1 + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log("[I] Patch version updated", "info") else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) log("[E] Updating patch version failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] "+name + " import failed!","error") @@ -2365,11 +2955,11 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('J%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -2388,33 +2978,33 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name): ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] java patch "+ className +" is applied..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] java patch "+ className +" failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='J%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] applying java patch "+ className +" failed", "error") @@ -2475,11 +3065,11 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam output = check_output(query) else: if os_name == "LINUX": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,db_user,db_user) + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: @@ -2498,38 +3088,155 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam ret = subprocess.call(get_java_cmd) if ret == 0: if os_name == "LINUX": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) if ret == 0: log ("[I] Ranger admin default password change request processed successfully..","info") else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) else: if os_name == "LINUX": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c \;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(shlex.split(query)) elif os_name == "WINDOWS": - query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N';\" -c ;" %(version) + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) jisql_log(query, db_password) ret = subprocess.call(query) log("[E] Ranger admin default password change request failed", "error") sys.exit(1) + def create_version_history_table(self, db_name, db_user, db_password, file_name,table_name): + name = basename(file_name) + if os.path.isfile(file_name): + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + if isTableExist==False: + log("[I] Importing "+table_name+" table schema to database " + db_name + " from file: " + name,"info") + while(isTableExist==False): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -input %s" %file_name + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -input %s" %file_name + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " file imported successfully","info") + else: + log("[E] "+name + " file import failed!","error") + time.sleep(30) + isTableExist=self.check_table(db_name, db_user, db_password, table_name) + else: + log("[E] Table schema file " + name+ " not found","error") + sys.exit(1) + + def import_core_db_schema(self, db_name, db_user, db_password, file_name,first_table,last_table): + version = 'CORE_DB_SCHEMA' + if os.path.isfile(file_name): + get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + log("[I] "+version+" is already imported" ,"info") + else: + if os_name == "LINUX": + query = get_cmd + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'N';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"select version from x_db_version_h where version = '%s' and active = 'N';\" -c ;" %(version) + jisql_log(query, db_password) + output = check_output(query) + if output.strip(version + " |"): + while(output.strip(version + " |")): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + jisql_log(query, db_password) + output = check_output(query) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c \;" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by,active) values ('%s', GETDATE(), '%s', GETDATE(), '%s','N') ;\" -c ;" %(version,client_host,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret != 0: + log("[E] "+ version +" import failed", "error") + sys.exit(1) + isFirstTableExist = self.check_table(db_name, db_user, db_password, first_table) + isLastTableExist = self.check_table(db_name, db_user, db_password, last_table) + isSchemaCreated=False + if isFirstTableExist == True and isLastTableExist == True : + isSchemaCreated=True + elif isFirstTableExist == True and isLastTableExist == False : + while(isLastTableExist==False): + log("[I] "+ version +" is being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + elif isFirstTableExist == False and isLastTableExist == False : + isImported=self.import_db_file(db_name, db_user, db_password, file_name) + if(isImported==False): + log("[I] "+ version +" might being imported by some other process" ,"info") + time.sleep(300) + isLastTableExist=self.check_table(db_name, db_user, db_password, last_table) + if(isLastTableExist==True): + isSchemaCreated=True + if isSchemaCreated == True: + if os_name == "LINUX": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"update x_db_version_h set active='Y' where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + if ret == 0: + log("[I] "+version +" import status has been updated", "info") + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] Updating "+version +" import status failed", "error") + sys.exit(1) + else: + if os_name == "LINUX": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c \;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd + " -query \"delete from x_db_version_h where version='%s' and active='N' and updated_by='%s';\" -c ;" %(version,client_host) + jisql_log(query, db_password) + ret = subprocess.call(query) + log("[E] "+version + " import failed!","error") + sys.exit(1) + def main(argv): populate_global_dict() @@ -2597,7 +3304,6 @@ def main(argv): x_db_version = 'x_db_version_h' xa_access_audit = 'xa_access_audit' - x_user = 'x_portal_user' audit_db_name='' audit_db_user='' @@ -2622,6 +3328,8 @@ def main(argv): xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_patches) audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_auditdb_patches) + first_table='x_asset' + last_table='xa_access_audit' elif XA_DB_FLAVOR == "ORACLE": ORACLE_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] @@ -2630,6 +3338,8 @@ def main(argv): xa_db_core_file = os.path.join(RANGER_ADMIN_HOME ,oracle_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,oracle_patches) audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,oracle_auditdb_patches) + first_table='X_PORTAL_USER' + last_table='X_AUDIT_MAP' elif XA_DB_FLAVOR == "POSTGRES": db_user=db_user.lower() @@ -2640,6 +3350,8 @@ def main(argv): xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , postgres_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME , postgres_patches) audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,postgres_auditdb_patches) + first_table='x_portal_user' + last_table='x_group_module_perm' elif XA_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR = globalDict['SQL_CONNECTOR_JAR'] @@ -2648,6 +3360,8 @@ def main(argv): xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , sqlserver_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME , sqlserver_patches) audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,sqlserver_auditdb_patches) + first_table='x_portal_user' + last_table='x_group_module_perm' elif XA_DB_FLAVOR == "SQLA": if not os_name == "WINDOWS" : @@ -2660,6 +3374,8 @@ def main(argv): xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , sqlanywhere_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME , sqlanywhere_patches) audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,sqlanywhere_auditdb_patches) + first_table='x_portal_user' + last_table='x_group_module_perm' else: log("[E] --------- NO SUCH SUPPORTED DB FLAVOUR!! ---------", "error") @@ -2699,22 +3415,18 @@ def main(argv): xa_sqlObj.check_connection(db_name, db_user, db_password) if len(argv)==1: - - log("[I] --------- Verifying Ranger DB tables ---------","info") - if xa_sqlObj.check_table(db_name, db_user, db_password, x_user): - pass - else: - log("[I] --------- Importing Ranger Core DB Schema ---------","info") - xa_sqlObj.import_db_file(db_name, db_user, db_password, xa_db_core_file) - if XA_DB_FLAVOR == "ORACLE": - if xa_sqlObj.check_table(db_name, db_user, db_password, xa_access_audit): - if audit_db_user != "" and db_user != audit_db_user: - xa_sqlObj.create_synonym(db_name, db_user, db_password,audit_db_user) - log("[I] --------- Verifying upgrade history table ---------","info") + log("[I] --------- Verifying version history table ---------","info") output = xa_sqlObj.check_table(db_name, db_user, db_password, x_db_version) if output == False: - log("[I] --------- Creating version history table ---------","info") - xa_sqlObj.upgrade_db(db_name, db_user, db_password, xa_db_version_file) + xa_sqlObj.create_version_history_table(db_name, db_user, db_password, xa_db_version_file,x_db_version) + + log("[I] --------- Importing Ranger Core DB Schema ---------","info") + xa_sqlObj.import_core_db_schema(db_name, db_user, db_password, xa_db_core_file,first_table,last_table) + if XA_DB_FLAVOR == "ORACLE": + if xa_sqlObj.check_table(db_name, db_user, db_password, xa_access_audit): + if audit_db_user != "" and db_user != audit_db_user: + xa_sqlObj.create_synonym(db_name, db_user, db_password,audit_db_user) + log("[I] --------- Applying Ranger DB patches ---------","info") xa_sqlObj.apply_patches(db_name, db_user, db_password, xa_patch_file) if audit_store == "db" and audit_db_password!='': diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties index 77b034374c..46a0091639 100644 --- a/security-admin/scripts/install.properties +++ b/security-admin/scripts/install.properties @@ -72,7 +72,7 @@ audit_solr_password= audit_solr_zookeepers= #audit_db_name= #audit_db_user= -#audit_db_password +#audit_db_password= #------------------------- DB CONFIG - END ---------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/KmsKeyMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/KmsKeyMgr.java index 693e959624..d565ebfa3e 100755 --- a/security-admin/src/main/java/org/apache/ranger/biz/KmsKeyMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/KmsKeyMgr.java @@ -125,52 +125,54 @@ public VXKmsKeyList searchKeys(HttpServletRequest request, String repoName) thro } catch (Exception e1) { logger.error("checkKerberos(" + repoName + ") failed", e1); } - for (int i = 0; i < providers.length; i++) { - Client c = getClient(); - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); - String keyLists = KMS_KEY_LIST_URI.replaceAll( - Pattern.quote("${userName}"), currentUserLoginId); - connProvider = providers[i]; - String uri = providers[i] - + (providers[i].endsWith("/") ? keyLists : ("/" + keyLists)); - if(!isKerberos){ - uri = uri.concat("?user.name="+currentUserLoginId); - }else{ - uri = uri.concat("?doAs="+currentUserLoginId); - } - final WebResource r = c.resource(uri); - try { - String response = null; + if(providers!=null){ + for (int i = 0; i < providers.length; i++) { + Client c = getClient(); + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + String keyLists = KMS_KEY_LIST_URI.replaceAll( + Pattern.quote("${userName}"), currentUserLoginId); + connProvider = providers[i]; + String uri = providers[i] + + (providers[i].endsWith("/") ? keyLists : ("/" + keyLists)); if(!isKerberos){ - response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); + uri = uri.concat("?user.name="+currentUserLoginId); }else{ - Subject sub = getSubjectForKerberos(repoName); - response = Subject.doAs(sub, new PrivilegedAction() { - @Override - public String run() { - return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); - } - }); + uri = uri.concat("?doAs="+currentUserLoginId); } - Gson gson = new GsonBuilder().create(); - logger.debug(" Search Key RESPONSE: [" + response + "]"); - keys = gson.fromJson(response, List.class); - Collections.sort(keys); - VXKmsKeyList vxKmsKeyList2 = new VXKmsKeyList(); - List vXKeys2 = new ArrayList(); - for (String name : keys) { - VXKmsKey key = new VXKmsKey(); - key.setName(name); - vXKeys2.add(key); + final WebResource r = c.resource(uri); + try { + String response = null; + if(!isKerberos){ + response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); + }else{ + Subject sub = getSubjectForKerberos(repoName); + response = Subject.doAs(sub, new PrivilegedAction() { + @Override + public String run() { + return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); + } + }); + } + Gson gson = new GsonBuilder().create(); + logger.debug(" Search Key RESPONSE: [" + response + "]"); + keys = gson.fromJson(response, List.class); + Collections.sort(keys); + VXKmsKeyList vxKmsKeyList2 = new VXKmsKeyList(); + List vXKeys2 = new ArrayList(); + for (String name : keys) { + VXKmsKey key = new VXKmsKey(); + key.setName(name); + vXKeys2.add(key); + } + vxKmsKeyList2.setVXKeys(vXKeys2); + vxKmsKeyList = getFilteredKeyList(request, vxKmsKeyList2); + break; + } catch (Exception e) { + if (e instanceof UniformInterfaceException || i == providers.length - 1) + throw e; + else + continue; } - vxKmsKeyList2.setVXKeys(vXKeys2); - vxKmsKeyList = getFilteredKeyList(request, vxKmsKeyList2); - break; - } catch (Exception e) { - if (e instanceof UniformInterfaceException || i == providers.length - 1) - throw e; - else - continue; } } //details @@ -186,7 +188,7 @@ public String run() { request.getParameter("pageSize"), 0, "Invalid value for parameter pageSize", MessageEnums.INVALID_INPUT_DATA, null, "pageSize"); - pageSize = pageSize < 0 ? 0 : pageSize; + pageSize = pageSize < 0 ? 0 : pageSize; vxKmsKeyList.setResultSize(lstKMSKey.size()); vxKmsKeyList.setTotalCount(lstKMSKey.size()); @@ -196,14 +198,20 @@ public String run() { startIndex = startIndex >= lstKMSKey.size() ? 0 : startIndex; lstKMSKey = lstKMSKey.subList(startIndex, lstKMSKey.size()); } - for (VXKmsKey kmsKey : lstKMSKey) { - VXKmsKey key = getKeyFromUri(connProvider, kmsKey.getName(), isKerberos, repoName); - vXKeys.add(key); - } + if(CollectionUtils.isNotEmpty(lstKMSKey)){ + for (VXKmsKey kmsKey : lstKMSKey) { + if(kmsKey!=null){ + VXKmsKey key = getKeyFromUri(connProvider, kmsKey.getName(), isKerberos, repoName); + vXKeys.add(key); + } + } + } vxKmsKeyList.setStartIndex(startIndex); vxKmsKeyList.setPageSize(pageSize); } - vxKmsKeyList.setVXKeys(vXKeys); + if(vxKmsKeyList!=null){ + vxKmsKeyList.setVXKeys(vXKeys); + } return vxKmsKeyList; } @@ -221,40 +229,42 @@ public VXKmsKey rolloverKey(String provider, VXKmsKey vXKey) throws Exception{ } catch (Exception e1) { logger.error("checkKerberos(" + provider + ") failed", e1); } - for (int i = 0; i < providers.length; i++) { - Client c = getClient(); - String rollRest = KMS_ROLL_KEY_URI.replaceAll(Pattern.quote("${alias}"), vXKey.getName()); - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); - String uri = providers[i] + (providers[i].endsWith("/") ? rollRest : ("/" + rollRest)); - if(!isKerberos){ - uri = uri.concat("?user.name="+currentUserLoginId); - }else{ - uri = uri.concat("?doAs="+currentUserLoginId); - } - final WebResource r = c.resource(uri); - Gson gson = new GsonBuilder().create(); - final String jsonString = gson.toJson(vXKey); - try { - String response = null; + if(providers!=null){ + for (int i = 0; i < providers.length; i++) { + Client c = getClient(); + String rollRest = KMS_ROLL_KEY_URI.replaceAll(Pattern.quote("${alias}"), vXKey.getName()); + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + String uri = providers[i] + (providers[i].endsWith("/") ? rollRest : ("/" + rollRest)); if(!isKerberos){ - response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString);} - else{ - Subject sub = getSubjectForKerberos(provider); - response = Subject.doAs(sub, new PrivilegedAction() { - @Override - public String run() { - return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString); - } - }); - } - logger.debug("Roll RESPONSE: [" + response + "]"); - ret = gson.fromJson(response, VXKmsKey.class); - break; - } catch (Exception e) { - if (e instanceof UniformInterfaceException || i == providers.length - 1) - throw e; - else - continue; + uri = uri.concat("?user.name="+currentUserLoginId); + }else{ + uri = uri.concat("?doAs="+currentUserLoginId); + } + final WebResource r = c.resource(uri); + Gson gson = new GsonBuilder().create(); + final String jsonString = gson.toJson(vXKey); + try { + String response = null; + if(!isKerberos){ + response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString);} + else{ + Subject sub = getSubjectForKerberos(provider); + response = Subject.doAs(sub, new PrivilegedAction() { + @Override + public String run() { + return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString); + } + }); + } + logger.debug("Roll RESPONSE: [" + response + "]"); + ret = gson.fromJson(response, VXKmsKey.class); + break; + } catch (Exception e) { + if (e instanceof UniformInterfaceException || i == providers.length - 1) + throw e; + else + continue; + } } } return ret; @@ -273,39 +283,41 @@ public void deleteKey(String provider, String name) throws Exception{ } catch (Exception e1) { logger.error("checkKerberos(" + provider + ") failed", e1); } - for (int i = 0; i < providers.length; i++) { - Client c = getClient(); - String deleteRest = KMS_DELETE_KEY_URI.replaceAll(Pattern.quote("${alias}"), name); - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); - String uri = providers[i] + (providers[i].endsWith("/") ? deleteRest : ("/" + deleteRest)); - if(!isKerberos){ - uri = uri.concat("?user.name="+currentUserLoginId); - }else{ - uri = uri.concat("?doAs="+currentUserLoginId); - } - final WebResource r = c.resource(uri); - try { - String response = null; + if(providers!=null){ + for (int i = 0; i < providers.length; i++) { + Client c = getClient(); + String deleteRest = KMS_DELETE_KEY_URI.replaceAll(Pattern.quote("${alias}"), name); + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + String uri = providers[i] + (providers[i].endsWith("/") ? deleteRest : ("/" + deleteRest)); if(!isKerberos){ - response = r.delete(String.class) ; + uri = uri.concat("?user.name="+currentUserLoginId); }else{ - Subject sub = getSubjectForKerberos(provider); - response = Subject.doAs(sub, new PrivilegedAction() { - @Override - public String run() { - return r.delete(String.class); - } - }); + uri = uri.concat("?doAs="+currentUserLoginId); + } + final WebResource r = c.resource(uri); + try { + String response = null; + if(!isKerberos){ + response = r.delete(String.class) ; + }else{ + Subject sub = getSubjectForKerberos(provider); + response = Subject.doAs(sub, new PrivilegedAction() { + @Override + public String run() { + return r.delete(String.class); + } + }); + } + logger.debug("delete RESPONSE: [" + response + "]") ; + break; + } catch (Exception e) { + if (e instanceof UniformInterfaceException || i == providers.length - 1) + throw e; + else + continue; } - logger.debug("delete RESPONSE: [" + response + "]") ; - break; - } catch (Exception e) { - if (e instanceof UniformInterfaceException || i == providers.length - 1) - throw e; - else - continue; } - } + } } public VXKmsKey createKey(String provider, VXKmsKey vXKey) throws Exception{ @@ -323,39 +335,41 @@ public VXKmsKey createKey(String provider, VXKmsKey vXKey) throws Exception{ } catch (Exception e1) { logger.error("checkKerberos(" + provider + ") failed", e1); } - for (int i = 0; i < providers.length; i++) { - Client c = getClient(); - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); - String uri = providers[i] + (providers[i].endsWith("/") ? KMS_ADD_KEY_URI : ("/" + KMS_ADD_KEY_URI)); - if(!isKerberos){ - uri = uri.concat("?user.name="+currentUserLoginId); - }else{ - uri = uri.concat("?doAs="+currentUserLoginId); - } - final WebResource r = c.resource(uri); - Gson gson = new GsonBuilder().create(); - final String jsonString = gson.toJson(vXKey); - try { - String response = null; + if(providers!=null){ + for (int i = 0; i < providers.length; i++) { + Client c = getClient(); + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + String uri = providers[i] + (providers[i].endsWith("/") ? KMS_ADD_KEY_URI : ("/" + KMS_ADD_KEY_URI)); if(!isKerberos){ - response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString); + uri = uri.concat("?user.name="+currentUserLoginId); }else{ - Subject sub = getSubjectForKerberos(provider); - response = Subject.doAs(sub, new PrivilegedAction() { - @Override - public String run() { - return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString); - } - }); + uri = uri.concat("?doAs="+currentUserLoginId); + } + final WebResource r = c.resource(uri); + Gson gson = new GsonBuilder().create(); + final String jsonString = gson.toJson(vXKey); + try { + String response = null; + if(!isKerberos){ + response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString); + }else{ + Subject sub = getSubjectForKerberos(provider); + response = Subject.doAs(sub, new PrivilegedAction() { + @Override + public String run() { + return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).post(String.class, jsonString); + } + }); + } + logger.debug("Create RESPONSE: [" + response + "]"); + ret = gson.fromJson(response, VXKmsKey.class); + return ret; + } catch (Exception e) { + if (e instanceof UniformInterfaceException || i == providers.length - 1) + throw e; + else + continue; } - logger.debug("Create RESPONSE: [" + response + "]"); - ret = gson.fromJson(response, VXKmsKey.class); - return ret; - } catch (Exception e) { - if (e instanceof UniformInterfaceException || i == providers.length - 1) - throw e; - else - continue; } } return ret; @@ -374,39 +388,41 @@ public VXKmsKey getKey(String provider, String name) throws Exception{ } catch (Exception e1) { logger.error("checkKerberos(" + provider + ") failed", e1); } - for (int i = 0; i < providers.length; i++) { - Client c = getClient(); - String keyRest = KMS_KEY_METADATA_URI.replaceAll(Pattern.quote("${alias}"), name); - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); - String uri = providers[i] + (providers[i].endsWith("/") ? keyRest : ("/" + keyRest)); - if(!isKerberos){ - uri = uri.concat("?user.name="+currentUserLoginId); - }else{ - uri = uri.concat("?doAs="+currentUserLoginId); - } - final WebResource r = c.resource(uri); - try { - String response = null; + if(providers!=null){ + for (int i = 0; i < providers.length; i++) { + Client c = getClient(); + String keyRest = KMS_KEY_METADATA_URI.replaceAll(Pattern.quote("${alias}"), name); + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + String uri = providers[i] + (providers[i].endsWith("/") ? keyRest : ("/" + keyRest)); if(!isKerberos){ - response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); + uri = uri.concat("?user.name="+currentUserLoginId); }else{ - Subject sub = getSubjectForKerberos(provider); - response = Subject.doAs(sub, new PrivilegedAction() { - @Override - public String run() { - return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); - } - }); + uri = uri.concat("?doAs="+currentUserLoginId); + } + final WebResource r = c.resource(uri); + try { + String response = null; + if(!isKerberos){ + response = r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); + }else{ + Subject sub = getSubjectForKerberos(provider); + response = Subject.doAs(sub, new PrivilegedAction() { + @Override + public String run() { + return r.accept(MediaType.APPLICATION_JSON_TYPE).type(MediaType.APPLICATION_JSON_TYPE).get(String.class); + } + }); + } + Gson gson = new GsonBuilder().create(); + logger.debug("RESPONSE: [" + response + "]"); + VXKmsKey key = gson.fromJson(response, VXKmsKey.class); + return key; + } catch (Exception e) { + if (e instanceof UniformInterfaceException || i == providers.length - 1) + throw e; + else + continue; } - Gson gson = new GsonBuilder().create(); - logger.debug("RESPONSE: [" + response + "]"); - VXKmsKey key = gson.fromJson(response, VXKmsKey.class); - return key; - } catch (Exception e) { - if (e instanceof UniformInterfaceException || i == providers.length - 1) - throw e; - else - continue; } } return null; @@ -415,7 +431,7 @@ public String run() { public VXKmsKey getKeyFromUri(String provider, String name, boolean isKerberos, String repoName) throws Exception { Client c = getClient(); String keyRest = KMS_KEY_METADATA_URI.replaceAll(Pattern.quote("${alias}"), name); - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); String uri = provider + (provider.endsWith("/") ? keyRest : ("/" + keyRest)); if(!isKerberos){ uri = uri.concat("?user.name="+currentUserLoginId); @@ -438,7 +454,7 @@ public String run() { Gson gson = new GsonBuilder().create(); logger.debug("RESPONSE: [" + response + "]"); VXKmsKey key = gson.fromJson(response, VXKmsKey.class); - return key; + return key; } private String[] getKMSURL(String name) throws Exception{ @@ -446,21 +462,24 @@ private String[] getKMSURL(String name) throws Exception{ RangerService rangerService = null; try { rangerService = svcStore.getServiceByName(name); - String kmsUrl = rangerService.getConfigs().get(KMS_URL_CONFIG); - String dbKmsUrl = kmsUrl; - if(providerList.containsKey(kmsUrl)){ - kmsUrl = providerList.get(kmsUrl); + if(rangerService!=null){ + String kmsUrl = rangerService.getConfigs().get(KMS_URL_CONFIG); + String dbKmsUrl = kmsUrl; + if(providerList.containsKey(kmsUrl)){ + kmsUrl = providerList.get(kmsUrl); + }else{ + providerList.put(kmsUrl, kmsUrl); + } + providers = createProvider(dbKmsUrl,kmsUrl); }else{ - providerList.put(kmsUrl, kmsUrl); + throw new Exception("Service " + name + " not found"); } - providers = createProvider(dbKmsUrl,kmsUrl); } catch (Exception excp) { logger.error("getServiceByName(" + name + ") failed", excp); throw new Exception("getServiceByName(" + name + ") failed", excp); } - - if (rangerService == null || providers == null) { - throw new Exception("Provider " + name + " not found"); + if (providers == null) { + throw new Exception("Providers for service " + name + " not found"); } return providers; } @@ -554,7 +573,7 @@ private Subject getSubjectForKerberos(String provider) throws Exception{ } private String getKMSPassword(String srvName) throws Exception { - XXService rangerService = rangerDaoManagerBase.getXXService().findByName(srvName); + XXService rangerService = rangerDaoManagerBase.getXXService().findByName(srvName); XXServiceConfigMap xxConfigMap = rangerDaoManagerBase.getXXServiceConfigMap().findByServiceAndConfigKey(rangerService.getId(), KMS_PASSWORD); String encryptedPwd = xxConfigMap.getConfigvalue(); String pwd = PasswordUtils.decryptPassword(encryptedPwd); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index e0a98401a8..728b85ddd4 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -1061,7 +1061,10 @@ private int getRandomInt(int min, int max) { } else { int interval = max - min; int randomNum = random.nextInt(); - return ((Math.abs(randomNum) % interval) + min); + if(randomNum<0){ + randomNum=Math.abs(randomNum); + } + return ((randomNum % interval) + min); } } @@ -1383,12 +1386,12 @@ public static int getDBFlavor() { return AppConstants.DB_FLAVOR_SQLANYWHERE; }else { if(logger.isDebugEnabled()) { - logger.debug("DB Falvor could not be determined from property - " + propertyName + "=" + propertyValue); + logger.debug("DB Flavor could not be determined from property - " + propertyName + "=" + propertyValue); } } } - logger.error("DB Falvor could not be determined"); + logger.error("DB Flavor could not be determined"); return AppConstants.DB_FLAVOR_UNKNOWN; } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 9af5b5f05b..7004a4159b 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -56,6 +56,7 @@ import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.util.PasswordUtils; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; @@ -279,15 +280,19 @@ public void initStore() { final ServiceDBStore dbStore = this; predicateUtil = new ServicePredicateUtil(dbStore); - - txTemplate.execute(new TransactionCallback() { - @Override - public Object doInTransaction(TransactionStatus status) { - EmbeddedServiceDefsUtil.instance().init(dbStore); - getServiceUpgraded(); - return null; - } - }); + try { + txTemplate.execute(new TransactionCallback() { + @Override + public Object doInTransaction(TransactionStatus status) { + EmbeddedServiceDefsUtil.instance().init(dbStore); + getServiceUpgraded(); + createGenericUser(); + return null; + } + }); + } catch (Throwable ex) { + LOG.fatal("ServiceDefDBStore.initStore(): Failed to update DB: " + ex); + } legacyServiceDefsInitDone = true; } @@ -2023,9 +2028,20 @@ public void getPoliciesInCSV(List policies, HttpServletResponse re } finally { - in.close(); - out.flush(); - out.close(); + try{ + if(in!=null){ + in.close(); + in=null; + } + }catch(Exception ex){ + } + try{ + if(out!=null){ + out.flush(); + out.close(); + } + }catch(Exception ex){ + } } } @@ -2258,6 +2274,8 @@ public ServicePolicies getServicePolicies(String serviceName) throws Exception { List policies = null; ServicePolicies.TagPolicies tagPolicies = null; + String auditMode = getAuditMode(serviceDef.getName(), serviceName); + if (serviceDbObj.getIsenabled()) { if (serviceDbObj.getTagService() != null) { XXService tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService()); @@ -2282,6 +2300,7 @@ public ServicePolicies getServicePolicies(String serviceName) throws Exception { tagPolicies.setPolicyUpdateTime(tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyUpdateTime()); tagPolicies.setPolicies(getServicePoliciesFromDb(tagServiceDbObj)); tagPolicies.setServiceDef(tagServiceDef); + tagPolicies.setAuditMode(auditMode); } } @@ -2299,6 +2318,7 @@ public ServicePolicies getServicePolicies(String serviceName) throws Exception { ret.setPolicyUpdateTime(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime()); ret.setPolicies(policies); ret.setServiceDef(serviceDef); + ret.setAuditMode(auditMode); ret.setTagPolicies(tagPolicies); if (LOG.isDebugEnabled()) { @@ -3285,7 +3305,7 @@ private StringBuffer writeCSV(List policies, String cSVFileName, H break; } - if (serviceTypeId == 100) { + if (serviceTypeId!=null && serviceTypeId.equals(Long.valueOf(100L))) { Map resources = policy.getResources(); if (resources != null) { @@ -3784,4 +3804,28 @@ private void updateServiceWithCustomProperty() { LOG.error("Error getting Services : "+e.getMessage()); } } + + private String getAuditMode(String serviceTypeName, String serviceName) { + RangerConfiguration config = RangerConfiguration.getInstance(); + String ret = config.get("ranger.audit.global.mode"); + if (StringUtils.isNotBlank(ret)) { + return ret; + } + ret = config.get("ranger.audit.servicedef." + serviceTypeName + ".mode"); + if (StringUtils.isNotBlank(ret)) { + return ret; + } + ret = config.get("ranger.audit.service." + serviceName + ".mode"); + if (StringUtils.isNotBlank(ret)) { + return ret; + } + return RangerPolicyEngine.AUDIT_DEFAULT; + } + + private void createGenericUser() { + VXUser genericUser = new VXUser(); + genericUser.setName(RangerPolicyEngine.USER_CURRENT); + genericUser.setDescription(RangerPolicyEngine.USER_CURRENT); + xUserService.createXUserWithOutLogin(genericUser); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java index 0059884441..65d41fbda7 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java @@ -159,12 +159,12 @@ public VXResponse validateConfig(RangerService service, ServiceStore svcStore) t service.getConfigs().put(HadoopConfigHolder.RANGER_AUTH_TYPE, authType); } } - - Map newConfigs = rangerSvcService.getConfigsWithDecryptedPassword(service); - service.setConfigs(newConfigs); - - RangerBaseService svc = getRangerServiceByService(service, svcStore); - + RangerBaseService svc=null; + if(service!=null){ + Map newConfigs = rangerSvcService.getConfigsWithDecryptedPassword(service); + service.setConfigs(newConfigs); + svc = getRangerServiceByService(service, svcStore); + } if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceMgr.validateConfig for Service: (" + svc + ")"); } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/SessionMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/SessionMgr.java index 2e9d6d511f..4c77d01463 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/SessionMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/SessionMgr.java @@ -96,10 +96,6 @@ public SessionMgr() { private static final Long SESSION_UPDATE_INTERVAL_IN_MILLIS = 30 * DateUtils.MILLIS_PER_MINUTE; - public UserSessionBase processSuccessLogin(int authType, String userAgent) { - return processSuccessLogin(authType, userAgent, null); - } - public UserSessionBase processSuccessLogin(int authType, String userAgent, HttpServletRequest httpRequest) { boolean newSessionCreation = true; @@ -165,7 +161,7 @@ else if (!StringUtils.isEmpty(httpRequest.getRequestURI()) && !(httpRequest.getR }else if (StringUtils.isEmpty(httpRequest.getRequestURI())){ gjAuthSession = storeAuthSession(gjAuthSession); session.setAttribute("auditLoginId", gjAuthSession.getId()); - }else{ + }else{ //NOPMD //do not log the details for download policy and tag } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java index a5089268ad..136e75636b 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/UserMgr.java @@ -25,8 +25,8 @@ import java.util.List; import javax.persistence.Query; +import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; @@ -374,142 +374,108 @@ public void setUserRoles(Long userId, List vStringRolesList) { * @return */ public VXResponse changePassword(VXPasswordChange pwdChange) { - VXResponse ret = new VXResponse(); - - // First let's get the XXPortalUser for the current logged in user - String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); - XXPortalUser gjUserCurrent = daoManager.getXXPortalUser() - .findByLoginId(currentUserLoginId); - checkAccess(gjUserCurrent); - - String encryptedOldPwd = encrypt(gjUserCurrent.getLoginId(), - pwdChange.getOldPassword()); - - if (!stringUtil.equals(encryptedOldPwd, gjUserCurrent.getPassword())) { - logger.info("changePassword(). Invalid old password. userId=" - + pwdChange.getId()); - - throw restErrorUtil.createRESTException( - "serverMsg.userMgrPassword", - MessageEnums.OPER_NO_PERMISSION, null, null, - "" + pwdChange.getId()); - } - - // Get the user for whom we want to change the password - XXPortalUser gjUser = daoManager.getXXPortalUser().getById( - pwdChange.getId()); - if (gjUser == null) { - logger.warn("SECURITY:changePassword(). User not found. userId=" - + pwdChange.getId()); - throw restErrorUtil.createRESTException( - "serverMsg.userMgrInvalidUser", - MessageEnums.DATA_NOT_FOUND, null, null, - "" + pwdChange.getId()); - } - - if (!stringUtil - .validatePassword( - pwdChange.getUpdPassword(), - new String[] { gjUser.getFirstName(), - gjUser.getLastName(), gjUser.getLoginId(), - gjUserCurrent.getFirstName(), - gjUserCurrent.getLastName(), - gjUserCurrent.getLoginId() })) { - logger.warn("SECURITY:changePassword(). Invalid new password. userId=" - + pwdChange.getId()); - - throw restErrorUtil.createRESTException( - "serverMsg.userMgrNewPassword", - MessageEnums.INVALID_PASSWORD, null, null, - "" + pwdChange.getId()); - } - - String encryptedNewPwd = encrypt(gjUser.getLoginId(), - pwdChange.getUpdPassword()); - - String currentPassword = gjUser.getPassword(); - - if (!encryptedNewPwd.equals(currentPassword)) { - - List trxLogList = new ArrayList(); - XXTrxLog xTrxLog = new XXTrxLog(); - - xTrxLog.setAttributeName("Password"); - xTrxLog.setPreviousValue(currentPassword); - xTrxLog.setNewValue(encryptedNewPwd); - xTrxLog.setAction("password change"); - xTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_PASSWORD_CHANGE); - xTrxLog.setObjectId(pwdChange.getId()); - xTrxLog.setObjectName(pwdChange.getLoginId()); - trxLogList.add(xTrxLog); + VXResponse ret = new VXResponse(); + + // First let's get the XXPortalUser for the current logged in user + String currentUserLoginId = ContextUtil.getCurrentUserLoginId(); + XXPortalUser gjUserCurrent = daoManager.getXXPortalUser().findByLoginId(currentUserLoginId); + checkAccessForUpdate(gjUserCurrent); + + // Get the user of whom we want to change the password + XXPortalUser gjUser = daoManager.getXXPortalUser().findByLoginId(pwdChange.getLoginId()); + if (gjUser == null) { + logger.warn("SECURITY:changePassword(). User not found. LoginId="+ pwdChange.getLoginId()); + throw restErrorUtil.createRESTException("serverMsg.userMgrInvalidUser",MessageEnums.DATA_NOT_FOUND, null, null,pwdChange.getLoginId()); + } - msBizUtil.createTrxLog(trxLogList); + //check current password and provided old password is same or not + String encryptedOldPwd = encrypt(pwdChange.getLoginId(),pwdChange.getOldPassword()); + if (!stringUtil.equals(encryptedOldPwd, gjUser.getPassword())) { + logger.info("changePassword(). Invalid old password. LoginId="+ pwdChange.getLoginId()); + throw restErrorUtil.createRESTException("serverMsg.userMgrOldPassword",MessageEnums.INVALID_INPUT_DATA, null, null,pwdChange.getLoginId()); + } - gjUser.setPassword(encryptedNewPwd); - gjUser = daoManager.getXXPortalUser().update(gjUser); + //validate new password + if (!stringUtil.validatePassword(pwdChange.getUpdPassword(),new String[] { gjUser.getFirstName(),gjUser.getLastName(), gjUser.getLoginId()})) { + logger.warn("SECURITY:changePassword(). Invalid new password. LoginId="+ pwdChange.getLoginId()); + throw restErrorUtil.createRESTException("serverMsg.userMgrNewPassword",MessageEnums.INVALID_PASSWORD, null, null,pwdChange.getLoginId()); + } - ret.setMsgDesc("Password successfully updated"); - ret.setStatusCode(VXResponse.STATUS_SUCCESS); - } else { - ret.setMsgDesc("Password update failed"); - ret.setStatusCode(VXResponse.STATUS_ERROR); - throw restErrorUtil.createRESTException( - "serverMsg.userMgrOldPassword", - MessageEnums.INVALID_INPUT_DATA, gjUser.getId(), - "password", gjUser.toString()); - } - return ret; - } + String encryptedNewPwd = encrypt(pwdChange.getLoginId(),pwdChange.getUpdPassword()); + String currentPassword = gjUser.getPassword(); + if (!encryptedNewPwd.equals(currentPassword)) { + List trxLogList = new ArrayList(); + XXTrxLog xTrxLog = new XXTrxLog(); + xTrxLog.setAttributeName("Password"); + xTrxLog.setPreviousValue(currentPassword); + xTrxLog.setNewValue(encryptedNewPwd); + xTrxLog.setAction("password change"); + xTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_PASSWORD_CHANGE); + xTrxLog.setObjectId(pwdChange.getId()); + xTrxLog.setObjectName(pwdChange.getLoginId()); + trxLogList.add(xTrxLog); + msBizUtil.createTrxLog(trxLogList); + gjUser.setPassword(encryptedNewPwd); + gjUser = daoManager.getXXPortalUser().update(gjUser); + ret.setMsgDesc("Password successfully updated"); + ret.setStatusCode(VXResponse.STATUS_SUCCESS); + } else { + ret.setMsgDesc("Password update failed"); + ret.setStatusCode(VXResponse.STATUS_ERROR); + throw restErrorUtil.createRESTException("serverMsg.userMgrNewPassword",MessageEnums.INVALID_INPUT_DATA, gjUser.getId(),"password", gjUser.toString()); + } + return ret; + } /** * @param gjUser * @param changeEmail * @return */ - public VXPortalUser changeEmailAddress(XXPortalUser gjUser, - VXPasswordChange changeEmail) { - checkAccess(gjUser); - if (StringUtils.isEmpty(changeEmail.getEmailAddress())) { - throw restErrorUtil.createRESTException( - "serverMsg.userMgrInvalidEmail", - MessageEnums.INVALID_INPUT_DATA, changeEmail.getId(), - "emailAddress", changeEmail.toString()); - } + public VXPortalUser changeEmailAddress(XXPortalUser gjUser, + VXPasswordChange changeEmail) { + checkAccessForUpdate(gjUser); + if (stringUtil.isEmpty(changeEmail.getEmailAddress())) { + throw restErrorUtil.createRESTException( + "serverMsg.userMgrInvalidEmail", + MessageEnums.INVALID_INPUT_DATA, changeEmail.getId(), + "emailAddress", changeEmail.toString()); + } - String encryptedOldPwd = encrypt(gjUser.getLoginId(), - changeEmail.getOldPassword()); + String encryptedOldPwd = encrypt(gjUser.getLoginId(), + changeEmail.getOldPassword()); - if (!stringUtil.validateEmail(changeEmail.getEmailAddress())) { - logger.info("Invalid email address." + changeEmail); - throw restErrorUtil.createRESTException( - "serverMsg.userMgrInvalidEmail", - MessageEnums.INVALID_INPUT_DATA, changeEmail.getId(), - "emailAddress", changeEmail.toString()); + if (!stringUtil.validateEmail(changeEmail.getEmailAddress())) { + logger.info("Invalid email address." + changeEmail); + throw restErrorUtil.createRESTException( + "serverMsg.userMgrInvalidEmail", + MessageEnums.INVALID_INPUT_DATA, changeEmail.getId(), + "emailAddress", changeEmail.toString()); - } + } - if (!stringUtil.equals(encryptedOldPwd, gjUser.getPassword())) { - logger.info("changeEmailAddress(). Invalid password. changeEmail=" - + changeEmail); + if (!stringUtil.equals(encryptedOldPwd, gjUser.getPassword())) { + logger.info("changeEmailAddress(). Invalid password. changeEmail=" + + changeEmail); - throw restErrorUtil.createRESTException( - "serverMsg.userMgrWrongPassword", - MessageEnums.OPER_NO_PERMISSION, null, null, "" - + changeEmail); - } + throw restErrorUtil.createRESTException( + "serverMsg.userMgrWrongPassword", + MessageEnums.OPER_NO_PERMISSION, null, null, "" + + changeEmail); + } - // Normalize email. Make it lower case - gjUser.setEmailAddress(stringUtil.normalizeEmail(changeEmail - .getEmailAddress())); + // Normalize email. Make it lower case + gjUser.setEmailAddress(stringUtil.normalizeEmail(changeEmail + .getEmailAddress())); - String saltEncodedpasswd = encrypt(gjUser.getLoginId(), - changeEmail.getOldPassword()); + String saltEncodedpasswd = encrypt(gjUser.getLoginId(), + changeEmail.getOldPassword()); - gjUser.setPassword(saltEncodedpasswd); + gjUser.setPassword(saltEncodedpasswd); - daoManager.getXXPortalUser().update(gjUser); - return mapXXPortalUserVXPortalUser(gjUser); - } + daoManager.getXXPortalUser().update(gjUser); + return mapXXPortalUserVXPortalUser(gjUser); + } /** * @param userId @@ -529,7 +495,7 @@ public VXPortalUser deactivateUser(XXPortalUser gjUser) { public VXPortalUser getUserProfile(Long id) { XXPortalUser user = daoManager.getXXPortalUser().getById(id); if (user != null) { - checkAccessForRead(user); + checkAccess(user); return mapXXPortalUserVXPortalUser(user); } else { if (logger.isDebugEnabled()) { @@ -742,8 +708,8 @@ public VXPortalUserList searchUsers(SearchCriteria searchCriteria) { // Get total count first Query query = createUserSearchQuery(countQueryStr, null, searchCriteria); Long count = (Long) query.getSingleResult(); - int resultSize = Integer.parseInt(count.toString()); - if (count == null || count.longValue() == 0) { + int resultSize = count!=null ? count.intValue() :0; + if (resultSize == 0) { return returnList; } @@ -1075,7 +1041,7 @@ public void checkAccess(XXPortalUser gjUser) { } - public void checkAccessForRead(XXPortalUser gjUser) { + public void checkAccessForUpdate(XXPortalUser gjUser) { if (gjUser == null) { throw restErrorUtil .create403RESTException("serverMsg.userMgrWrongUser"); @@ -1084,7 +1050,7 @@ public void checkAccessForRead(XXPortalUser gjUser) { if (sess != null) { // Admin - if (sess != null && sess.isUserAdmin() || sess.isKeyAdmin()) { + if (sess != null && sess.isUserAdmin()) { return; } @@ -1094,11 +1060,14 @@ public void checkAccessForRead(XXPortalUser gjUser) { } } - throw restErrorUtil.create403RESTException("User " + VXResponse vXResponse = new VXResponse(); + vXResponse.setStatusCode(HttpServletResponse.SC_FORBIDDEN); + vXResponse.setMsgDesc("User " + " access denied. loggedInUser=" + (sess != null ? sess.getXXPortalUser().getId() : "Not Logged In") + ", accessing user=" + gjUser.getId()); + throw restErrorUtil.createRESTException(vXResponse); } @@ -1154,7 +1123,7 @@ public VXPortalUser createDefaultAccountUser(VXPortalUser userProfile) { xXPortalUser = this.createUser(userProfile, RangerCommonEnums.STATUS_ENABLED); } - } else { + } else { //NOPMD /* * throw restErrorUtil .createRESTException( "The login id " + * loginId + diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index 242a27eefc..f3d661ce32 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -168,6 +168,15 @@ public VXUser getXUserByUserName(String userName) { return vXUser; } + public VXGroup getGroupByGroupName(String groupName) { + VXGroup vxGroup = xGroupService.getGroupByGroupName(groupName); + if (vxGroup == null) { + throw restErrorUtil.createRESTException( + groupName + " is Not Found", MessageEnums.DATA_NOT_FOUND); + } + return vxGroup; + } + public VXUser createXUser(VXUser vXUser) { checkAdminAccess(); String userName = vXUser.getName(); @@ -234,35 +243,39 @@ public VXUser createXUser(VXUser vXUser) { } // xaBizUtil.createTrxLog(trxLogList); - - assignPermissionToUser(vXPortalUser, true); + if(vXPortalUser!=null){ + assignPermissionToUser(vXPortalUser, true); + } return createdXUser; } public void assignPermissionToUser(VXPortalUser vXPortalUser, boolean isCreate) { HashMap moduleNameId = getAllModuleNameAndIdMap(); + if(moduleNameId!=null && vXPortalUser!=null){ + if(CollectionUtils.isNotEmpty(vXPortalUser.getUserRoleList())){ + for (String role : vXPortalUser.getUserRoleList()) { + + if (role.equals(RangerConstants.ROLE_USER)) { + + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_RESOURCE_BASED_POLICIES), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_REPORTS), isCreate); + } else if (role.equals(RangerConstants.ROLE_SYS_ADMIN)) { + + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_REPORTS), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_RESOURCE_BASED_POLICIES), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_AUDIT), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_USER_GROUPS), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_TAG_BASED_POLICIES), isCreate); + } else if (role.equals(RangerConstants.ROLE_KEY_ADMIN)) { + + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_KEY_MANAGER), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_REPORTS), isCreate); + createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_RESOURCE_BASED_POLICIES), isCreate); + } - for (String role : vXPortalUser.getUserRoleList()) { - - if (role.equals(RangerConstants.ROLE_USER)) { - - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_RESOURCE_BASED_POLICIES), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_REPORTS), isCreate); - } else if (role.equals(RangerConstants.ROLE_SYS_ADMIN)) { - - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_REPORTS), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_RESOURCE_BASED_POLICIES), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_AUDIT), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_USER_GROUPS), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_TAG_BASED_POLICIES), isCreate); - } else if (role.equals(RangerConstants.ROLE_KEY_ADMIN)) { - - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_KEY_MANAGER), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_REPORTS), isCreate); - createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_RESOURCE_BASED_POLICIES), isCreate); + } } - } } @@ -838,18 +851,26 @@ public VXModuleDef updateXModuleDefPermission(VXModuleDef vXModuleDef) { VXModuleDef vModuleDefPopulateOld = xModuleDefService.populateViewBean(xModuleDef); List xgroupPermissionList = daoManager.getXXGroupPermission().findByModuleId(vXModuleDef.getId(), true); - - for (XXGroupPermission xGrpPerm : xgroupPermissionList) { - VXGroupPermission vXGrpPerm = xGroupPermissionService.populateViewBean(xGrpPerm); - groupPermListOld.add(vXGrpPerm); + Map xXGroupMap=xGroupService.getXXGroupIdXXGroupMap(); + if(xXGroupMap==null || xXGroupMap.isEmpty()){ + for (XXGroupPermission xGrpPerm : xgroupPermissionList) { + VXGroupPermission vXGrpPerm = xGroupPermissionService.populateViewBean(xGrpPerm); + groupPermListOld.add(vXGrpPerm); + } + }else{ + groupPermListOld=xGroupPermissionService.getPopulatedVXGroupPermissionList(xgroupPermissionList,xXGroupMap,vModuleDefPopulateOld); } vModuleDefPopulateOld.setGroupPermList(groupPermListOld); List xuserPermissionList = daoManager.getXXUserPermission().findByModuleId(vXModuleDef.getId(), true); - - for (XXUserPermission xUserPerm : xuserPermissionList) { - VXUserPermission vUserPerm = xUserPermissionService.populateViewBean(xUserPerm); - userPermListOld.add(vUserPerm); + Map xXPortalUserIdXXUserMap=xUserService.getXXPortalUserIdXXUserMap(); + if(xXPortalUserIdXXUserMap==null || xXPortalUserIdXXUserMap.isEmpty()){ + for (XXUserPermission xUserPerm : xuserPermissionList) { + VXUserPermission vUserPerm = xUserPermissionService.populateViewBean(xUserPerm); + userPermListOld.add(vUserPerm); + } + }else{ + userPermListOld=xUserPermissionService.getPopulatedVXUserPermissionList(xuserPermissionList,xXPortalUserIdXXUserMap,vModuleDefPopulateOld); } vModuleDefPopulateOld.setUserPermList(userPermListOld); @@ -1606,7 +1627,10 @@ public void deleteXUser(Long id, boolean force) { } XXPortalUserDao xXPortalUserDao=daoManager.getXXPortalUser(); XXPortalUser xXPortalUser=xXPortalUserDao.findByLoginId(vXUser.getName().trim()); - VXPortalUser vXPortalUser=xPortalUserService.populateViewBean(xXPortalUser); + VXPortalUser vXPortalUser=null; + if(xXPortalUser!=null){ + vXPortalUser=xPortalUserService.populateViewBean(xXPortalUser); + } if(vXPortalUser==null ||StringUtil.isEmpty(vXPortalUser.getLoginId())){ throw restErrorUtil.createRESTException("No user found with id=" + id); } @@ -1772,11 +1796,8 @@ public void deleteXUser(Long id, boolean force) { xXPortalUserDao.remove(xXPortalUserId); List trxLogList =xUserService.getTransactionLog(xUserService.populateViewBean(xXUser), "delete"); xaBizUtil.createTrxLog(trxLogList); - if (xXPortalUser != null) { - trxLogList=xPortalUserService - .getTransactionLog(xPortalUserService.populateViewBean(xXPortalUser), "delete"); - xaBizUtil.createTrxLog(trxLogList); - } + trxLogList=xPortalUserService.getTransactionLog(xPortalUserService.populateViewBean(xXPortalUser), "delete"); + xaBizUtil.createTrxLog(trxLogList); } } } diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java index a68b2153fe..1cc97938e1 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -77,7 +77,9 @@ public void dump() { ServicePoliciesWrapper cachedServicePoliciesWrapper = null; for (String serviceName : serviceNames) { - cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName); + synchronized (this) { + cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName); + } if (LOG.isDebugEnabled()) { LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServicePoliciesWrapper); } diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java index 66c1562296..5521523a38 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java @@ -77,7 +77,9 @@ public void dump() { ServiceTagsWrapper cachedServiceTagsWrapper = null; for (String serviceName : serviceNames) { - cachedServiceTagsWrapper = serviceTagsMap.get(serviceName); + synchronized (this) { + cachedServiceTagsWrapper = serviceTagsMap.get(serviceName); + } if (LOG.isDebugEnabled()) { LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServiceTagsWrapper); } diff --git a/security-admin/src/main/java/org/apache/ranger/common/SearchField.java b/security-admin/src/main/java/org/apache/ranger/common/SearchField.java index 1891edb43b..2d6ab14853 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/SearchField.java +++ b/security-admin/src/main/java/org/apache/ranger/common/SearchField.java @@ -213,5 +213,7 @@ public void setJoinCriteria(String joinCriteria) { public String getCustomCondition() { return customCondition; } - + public void setCustomCondition(String conditions) { + customCondition=conditions; + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/common/SearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/SearchUtil.java index a91eb72fac..df48d54f1d 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/SearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/SearchUtil.java @@ -665,7 +665,7 @@ protected void resolveQueryParams(Query query, SearchCriteria searchCriteria, if (searchCriteria.getNullParamList().contains( searchField.getClientFieldName()) || searchCriteria.getNotNullParamList().contains( - searchField.getClientFieldName())) { + searchField.getClientFieldName())) { //NOPMD // Already addressed while building where clause } else if (searchField.getDataType() == SearchField.DATA_TYPE.INT_LIST || isListValue diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java index 7355e3dd46..eb4ba2dc04 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java @@ -1226,10 +1226,7 @@ public GrantRevokeRequest toGrantRevokeRequest(VXPolicy vXPolicy) { Integer assetType = toAssetType(serviceType); - if(assetType == null) { - // nothing to do - } - else if (assetType == RangerCommonEnums.ASSET_HIVE) { + if (assetType == RangerCommonEnums.ASSET_HIVE) { String database = StringUtils.isEmpty(vXPolicy.getDatabases()) ? "*" : vXPolicy.getDatabases(); String table = getTableOrUdf(vXPolicy); @@ -1286,7 +1283,7 @@ else if ( assetType == RangerCommonEnums.ASSET_HBASE) { if ( AppConstants.getEnumFor_XAPermType(perm) != 0 ) { if (perm.equalsIgnoreCase("Admin")) { delegatedAdmin=true; - if ( assetType != RangerCommonEnums.ASSET_HBASE) { + if (assetType!=null && assetType.intValue() != RangerCommonEnums.ASSET_HBASE) { continue; } } @@ -1340,11 +1337,7 @@ public boolean isValidateHttpsAuthentication( String serviceName, HttpServletReq RangerService service = null; try { - if(null != request.getAttribute("downloadPolicy") && StringUtils.equalsIgnoreCase(request.getAttribute("downloadPolicy").toString(), "secure")){ - service = svcStore.getServiceByNameForDP(serviceName); - }else{ - service = svcStore.getServiceByName(serviceName); - } + service = svcStore.getServiceByName(serviceName); } catch (Exception e) { LOG.error("Requested Service not found. serviceName=" + serviceName); throw restErrorUtil.createRESTException("Service:" + serviceName + " not found", @@ -1461,6 +1454,43 @@ public boolean isValidateHttpsAuthentication( String serviceName, HttpServletReq return isValidAuthentication; } + public boolean isValidService(String serviceName, HttpServletRequest request){ + boolean isValid = true; + if (serviceName == null || serviceName.isEmpty()) { + LOG.error("ServiceName not provided"); + isValid = false; + throw restErrorUtil.createRESTException("Unauthorized access.", + MessageEnums.OPER_NOT_ALLOWED_FOR_ENTITY); + } + + RangerService service = null; + try { + if(null != request.getAttribute("downloadPolicy") && StringUtils.equalsIgnoreCase(request.getAttribute("downloadPolicy").toString(), "secure")){ + service = svcStore.getServiceByNameForDP(serviceName); + }else{ + service = svcStore.getServiceByName(serviceName); + } + } catch (Exception e) { + isValid = false; + LOG.error("Requested Service not found. serviceName=" + serviceName); + throw restErrorUtil.createRESTException("Service:" + serviceName + " not found", + MessageEnums.DATA_NOT_FOUND); + } + if(service==null){ + isValid = false; + LOG.error("Requested Service not found. serviceName=" + serviceName); + throw restErrorUtil.createRESTException("Service:" + serviceName + " not found", + MessageEnums.DATA_NOT_FOUND); + } + if(!service.getIsEnabled()){ + isValid = false; + LOG.error("Requested Service is disabled. serviceName=" + serviceName); + throw restErrorUtil.createRESTException("Unauthorized access.", + MessageEnums.OPER_NOT_ALLOWED_FOR_STATE); + } + return isValid; + } + private boolean matchNames(String target, String source, boolean wildcardMatch) { boolean matched = false; if(target != null && source != null) { diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java index 13607d3bbe..f6b2a1469b 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java @@ -228,9 +228,8 @@ public void setIdentityInsert(boolean identityInsert) { String tableName = table.name(); - Connection conn = entityMgr.unwrap(Connection.class); try { - conn.createStatement().execute("SET IDENTITY_INSERT " + tableName + " " + identityInsertStr); + entityMgr.unwrap(Connection.class).createStatement().execute("SET IDENTITY_INSERT " + tableName + " " + identityInsertStr); } catch (SQLException e) { logger.error("Error while settion identity_insert " + identityInsertStr, e); } @@ -238,16 +237,17 @@ public void setIdentityInsert(boolean identityInsert) { public void updateUserIDReference(String paramName,long oldID) { Table table = tClass.getAnnotation(Table.class); - if(table == null) { + if(table != null) { + String tableName = table.name(); + String query = "update " + tableName + " set " + paramName+"=null" + + " where " +paramName+"=" + oldID; + int count=getEntityManager().createNativeQuery(query).executeUpdate(); + if(count>0){ + logger.warn(count + " records updated in table '" + tableName + "' with: set " + paramName + "=null where " + paramName + "=" + oldID); + } + }else{ logger.warn("Required annotation `Table` not found"); } - String tableName = table.name(); - String query = "update " + tableName + " set " + paramName+"=null" - + " where " +paramName+"=" + oldID; - int count=getEntityManager().createNativeQuery(query).executeUpdate(); - if(count>0){ - logger.warn(count + " records updated in table '" + tableName + "' with: set " + paramName + "=null where " + paramName + "=" + oldID); - } } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java index e25540b55d..5623517100 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java @@ -123,7 +123,7 @@ public void updateSequence() { updateSequence("X_POLICY_SEQ", maxId + 1); } public List findByUserId(Long userId) { - if(userId == null || userId.equals(0)) { + if(userId == null || userId.equals(Long.valueOf(0L))) { return new ArrayList(); } try { @@ -135,7 +135,7 @@ public List findByUserId(Long userId) { } } public List findByGroupId(Long groupId) { - if(groupId == null || groupId.equals(0)) { + if(groupId == null || groupId.equals(Long.valueOf(0L))) { return new ArrayList(); } try { diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java index 52910453bd..f1535fe223 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java @@ -120,10 +120,6 @@ private void updateTagVersionAndTagUpdateTime(List service currentTagVersion = Long.valueOf(0); } - if (updateTime == null) { - updateTime = new Date(); - } - serviceVersionInfo.setTagVersion(currentTagVersion + 1); serviceVersionInfo.setTagUpdateTime(updateTime); } diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXContextEnricherDef.java b/security-admin/src/main/java/org/apache/ranger/entity/XXContextEnricherDef.java index e035e58e4f..77eb061fa0 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXContextEnricherDef.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXContextEnricherDef.java @@ -313,10 +313,14 @@ public boolean equals(Object obj) { */ @Override public String toString() { - return "XXContextEnricherDef [id=" + id + ", defId=" + defId + ", itemId=" + itemId + String str = "XXContextEnricherDef={"; + str += super.toString(); + str+=" [id=" + id + ", defId=" + defId + ", itemId=" + itemId + ", name=" + name + ", enricher=" + enricherOptions + ", enricherOptions=" + enricherOptions + ", order=" + order + "]"; + str += "}"; + return str; } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java index 8564d43f56..aebe38ca84 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyBase.java @@ -352,9 +352,13 @@ public boolean equals(Object obj) { */ @Override public String toString() { - return "XXPolicyBase [guid=" + guid + ", version=" + version + ", service=" + service + ", name=" + name + String str = "XXPolicyBase={"; + str += super.toString(); + str += " [guid=" + guid + ", version=" + version + ", service=" + service + ", name=" + name + ", policyType=" + policyType + ", description=" + description + ", resourceSignature=" + resourceSignature + ", isEnabled=" + isEnabled + ", isAuditEnabled=" + isAuditEnabled + "]"; + str += "}"; + return str; } } diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyConditionDef.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyConditionDef.java index d7388410a9..6b12d94037 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyConditionDef.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyConditionDef.java @@ -558,7 +558,9 @@ public boolean equals(Object obj) { */ @Override public String toString() { - return "XXPolicyConditionDef [id=" + id + ", defId=" + defId + ", itemId=" + itemId + String str = "XXPolicyConditionDef={"; + str += super.toString(); + str += " [id=" + id + ", defId=" + defId + ", itemId=" + itemId + ", name=" + name + ", evaluator=" + evaluator + ", evaluatorOptions=" + evaluatorOptions + ", label=" + label + ", validationRegEx=" + validationRegEx @@ -568,6 +570,8 @@ public String toString() { + ", rbKeyValidationMessage=" + rbKeyValidationMessage + ", rbKeyDescription=" + rbKeyDescription + ", order=" + order + "]"; + str += "}"; + return str; } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyItemUserPerm.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyItemUserPerm.java index 874ca20355..69c47df968 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyItemUserPerm.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyItemUserPerm.java @@ -205,9 +205,13 @@ public boolean equals(Object obj) { */ @Override public String toString() { - return "XXPolicyItemUserPerm [id=" + id + ", policyItemId=" + String str = "XXPolicyItemUserPerm={"; + str += super.toString(); + str += " [id=" + id + ", policyItemId=" + policyItemId + ", userId=" + userId + ", order=" + order + "]"; + str += "}"; + return str; } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java index 283f44f502..f25c3f77f3 100644 --- a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java @@ -236,6 +236,15 @@ else if(auditTableVersion==6){ System.out.println("Total Number of Migrated Audit logs:"+totalMigratedLogs); logger.info("Total Number of Migrated Audit logs:"+totalMigratedLogs); } + if(solrClient!=null){ + try { + solrClient.close(); + } catch (IOException e) { + logger.error("Error while closing solr connection", e); + }finally{ + solrClient=null; + } + } System.out.println("Migration process finished!!"); } @@ -401,8 +410,7 @@ private SolrClient createSolrClient() throws Exception { zkHosts); solrCloudClient .setDefaultCollection(collectionName); - solrClient = solrCloudClient; - solrCloudClient.close(); + return solrCloudClient; } catch (Exception e) { logger.fatal( "Can't connect to Solr server. ZooKeepers=" diff --git a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java index 3d649df6cd..b1a21590be 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java @@ -39,6 +39,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.biz.AssetMgr; @@ -502,12 +503,17 @@ public Response getXResourceFile(@Context HttpServletRequest request, VXResource resource = getXResource(id); - File file = assetMgr.getXResourceFile(resource, fileType); - return Response - .ok(file, MediaType.APPLICATION_OCTET_STREAM) - .header("Content-Disposition", - "attachment;filename=" + file.getName()).build(); + Response response=null; + if(resource!=null && StringUtils.isNotEmpty(fileType)){ + File file = null; + file=assetMgr.getXResourceFile(resource, fileType); + if(file!=null){ + response=Response.ok(file, MediaType.APPLICATION_OCTET_STREAM).header("Content-Disposition","attachment;filename=" + file.getName()).build(); + file=null; + } + } + return response; } @GET diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index 6ecb356bb3..13c2fcf604 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -39,6 +39,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; +import java.util.ArrayList; import java.util.List; @Path("public/v2") @@ -282,6 +283,26 @@ public RangerPolicy getPolicy(@PathParam("id") Long id) { return serviceREST.getPolicy(id); } + @GET + @Path("/api/policy/") + @Produces({ "application/json", "application/xml" }) + public List getPolicies(@Context HttpServletRequest request) { + + List ret = new ArrayList(); + + if(logger.isDebugEnabled()) { + logger.debug("==> PublicAPIsv2.getPolicies()"); + } + + ret = serviceREST.getPolicies(request).getPolicies(); + + if(logger.isDebugEnabled()) { + logger.debug("<== PublicAPIsv2.getPolicies(Request: " + request.getQueryString() + " Result Size: " + ret.size() ); + } + + return ret; + } + @GET @Path("/api/service/{servicename}/policy/{policyname}") @Produces({ "application/json", "application/xml" }) @@ -326,8 +347,8 @@ public RangerPolicy createPolicy(RangerPolicy policy , @Context HttpServletReque @POST @Path("/api/policy/apply/") @Produces({ "application/json", "application/xml" }) - public RangerPolicy applyPolicy(RangerPolicy policy) { // new API - return serviceREST.applyPolicy(policy); + public RangerPolicy applyPolicy(RangerPolicy policy, @Context HttpServletRequest request) { // new API + return serviceREST.applyPolicy(policy, request); } @PUT diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 0d1e552dcc..26e29069d8 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -83,6 +83,7 @@ import org.apache.ranger.plugin.store.PList; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.util.GrantRevokeRequest; +import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; @@ -455,15 +456,24 @@ public RangerService createService(RangerService service) { validator.validate(service, Action.CREATE); UserSessionBase session = ContextUtil.getCurrentUserSession(); + XXServiceDef xxServiceDef = daoManager.getXXServiceDef().findByName(service.getType()); if(session != null && !session.isSpnegoEnabled()){ bizUtil.hasAdminPermissions("Services"); // TODO: As of now we are allowing SYS_ADMIN to create all the // services including KMS - - XXServiceDef xxServiceDef = daoManager.getXXServiceDef().findByName(service.getType()); bizUtil.hasKMSPermissions("Service", xxServiceDef.getImplclassname()); } + if(session != null && session.isSpnegoEnabled()){ + if (session.isKeyAdmin() && !xxServiceDef.getImplclassname().equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + throw restErrorUtil.createRESTException("KeyAdmin can create/update/delete only KMS ", + MessageEnums.OPER_NO_PERMISSION); + } + if ((!session.isKeyAdmin() && !session.isUserAdmin()) && xxServiceDef.getImplclassname().equals(EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + throw restErrorUtil.createRESTException("User cannot create/update/delete KMS Service", + MessageEnums.OPER_NO_PERMISSION); + } + } ret = svcStore.createService(service); } catch(WebApplicationException excp) { throw excp; @@ -861,132 +871,26 @@ public RESTResponse grantAccess(@PathParam("serviceName") String serviceName, Gr RESTResponse ret = new RESTResponse(); RangerPerfTracer perf = null; - if (serviceUtil.isValidateHttpsAuthentication(serviceName, request)) { - - try { - if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.grantAccess(serviceName=" + serviceName + ")"); - } - - String userName = grantRequest.getGrantor(); - Set userGroups = userMgr.getGroupsForUser(userName); - RangerAccessResource resource = new RangerAccessResourceImpl(grantRequest.getResource()); - - boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); - - if(!isAdmin) { - throw restErrorUtil.createGrantRevokeRESTException( "User doesn't have necessary permission to grant access"); - } - - RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource); - - if(policy != null) { - boolean policyUpdated = false; - policyUpdated = ServiceRESTUtil.processGrantRequest(policy, grantRequest); - - if(policyUpdated) { - svcStore.updatePolicy(policy); - } else { - LOG.error("processGrantRequest processing failed"); - throw new Exception("processGrantRequest processing failed"); - } - } else { - policy = new RangerPolicy(); - policy.setService(serviceName); - policy.setName("grant-" + System.currentTimeMillis()); // TODO: better policy name - policy.setDescription("created by grant"); - policy.setIsAuditEnabled(grantRequest.getEnableAudit()); - policy.setCreatedBy(userName); - - Map policyResources = new HashMap(); - Set resourceNames = resource.getKeys(); - - if(! CollectionUtils.isEmpty(resourceNames)) { - for(String resourceName : resourceNames) { - RangerPolicyResource policyResource = new RangerPolicyResource(resource.getValue(resourceName)); - policyResource.setIsRecursive(grantRequest.getIsRecursive()); - - policyResources.put(resourceName, policyResource); - } - } - policy.setResources(policyResources); - - RangerPolicyItem policyItem = new RangerPolicyItem(); - - policyItem.setDelegateAdmin(grantRequest.getDelegateAdmin()); - policyItem.getUsers().addAll(grantRequest.getUsers()); - policyItem.getGroups().addAll(grantRequest.getGroups()); + if(grantRequest!=null){ + if (serviceUtil.isValidateHttpsAuthentication(serviceName, request)) { - for(String accessType : grantRequest.getAccessTypes()) { - policyItem.getAccesses().add(new RangerPolicyItemAccess(accessType, Boolean.TRUE)); + try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.grantAccess(serviceName=" + serviceName + ")"); } - policy.getPolicyItems().add(policyItem); - - svcStore.createPolicy(policy); - } - } catch(WebApplicationException excp) { - throw excp; - } catch(Throwable excp) { - LOG.error("grantAccess(" + serviceName + ", " + grantRequest + ") failed", excp); + validateGrantRevokeRequest(grantRequest); + String userName = grantRequest.getGrantor(); + Set userGroups = userMgr.getGroupsForUser(userName); + RangerAccessResource resource = new RangerAccessResourceImpl(grantRequest.getResource()); - throw restErrorUtil.createRESTException(excp.getMessage()); - } finally { - RangerPerfTracer.log(perf); - } - - ret.setStatusCode(RESTResponse.STATUS_SUCCESS); - } + boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceREST.grantAccess(" + serviceName + ", " + grantRequest + "): " + ret); - } - - return ret; - } - - @POST - @Path("/secure/services/grant/{serviceName}") - @Produces({ "application/json", "application/xml" }) - public RESTResponse secureGrantAccess(@PathParam("serviceName") String serviceName, GrantRevokeRequest grantRequest, @Context HttpServletRequest request) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceREST.secureGrantAccess(" + serviceName + ", " + grantRequest + ")"); - } - RESTResponse ret = new RESTResponse(); - RangerPerfTracer perf = null; - boolean isAllowed = false; - boolean isKeyAdmin = bizUtil.isKeyAdmin(); - if (serviceUtil.isValidateHttpsAuthentication(serviceName, request)) { - try { - if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.scureGrantAccess(serviceName=" + serviceName + ")"); - } - String userName = grantRequest.getGrantor(); - Set userGroups = userMgr.getGroupsForUser(userName); - RangerAccessResource resource = new RangerAccessResourceImpl(grantRequest.getResource()); - boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); - - XXService xService = daoManager.getXXService().findByName(serviceName); - XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); - RangerService rangerService = svcStore.getServiceByName(serviceName); - - if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - if (isKeyAdmin) { - isAllowed = true; - }else { - isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); - } - }else{ - if (isAdmin) { - isAllowed = true; + if(!isAdmin) { + throw restErrorUtil.createGrantRevokeRESTException( "User doesn't have necessary permission to grant access"); } - else{ - isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); - } - } - - if (isAllowed) { - RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource); + + RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource, userName); if(policy != null) { boolean policyUpdated = false; @@ -995,8 +899,8 @@ public RESTResponse secureGrantAccess(@PathParam("serviceName") String serviceNa if(policyUpdated) { svcStore.updatePolicy(policy); } else { - LOG.error("processSecureGrantRequest processing failed"); - throw new Exception("processSecureGrantRequest processing failed"); + LOG.error("processGrantRequest processing failed"); + throw new Exception("processGrantRequest processing failed"); } } else { policy = new RangerPolicy(); @@ -1033,21 +937,134 @@ public RESTResponse secureGrantAccess(@PathParam("serviceName") String serviceNa svcStore.createPolicy(policy); } - }else{ - LOG.error("secureGrantAccess(" + serviceName + ", " + grantRequest + ") failed as User doesn't have permission to grant Policy"); - throw restErrorUtil.createGrantRevokeRESTException( "User doesn't have necessary permission to grant access"); + } catch(WebApplicationException excp) { + throw excp; + } catch(Throwable excp) { + LOG.error("grantAccess(" + serviceName + ", " + grantRequest + ") failed", excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } finally { + RangerPerfTracer.log(perf); } - } catch(WebApplicationException excp) { - throw excp; - } catch(Throwable excp) { - LOG.error("secureGrantAccess(" + serviceName + ", " + grantRequest + ") failed", excp); - - throw restErrorUtil.createRESTException(excp.getMessage()); - } finally { - RangerPerfTracer.log(perf); + + ret.setStatusCode(RESTResponse.STATUS_SUCCESS); } + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceREST.grantAccess(" + serviceName + ", " + grantRequest + "): " + ret); + } + + return ret; + } - ret.setStatusCode(RESTResponse.STATUS_SUCCESS); + @POST + @Path("/secure/services/grant/{serviceName}") + @Produces({ "application/json", "application/xml" }) + public RESTResponse secureGrantAccess(@PathParam("serviceName") String serviceName, GrantRevokeRequest grantRequest, @Context HttpServletRequest request) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceREST.secureGrantAccess(" + serviceName + ", " + grantRequest + ")"); + } + RESTResponse ret = new RESTResponse(); + RangerPerfTracer perf = null; + boolean isAllowed = false; + boolean isKeyAdmin = bizUtil.isKeyAdmin(); + if(grantRequest!=null){ + if (serviceUtil.isValidService(serviceName, request)) { + try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.scureGrantAccess(serviceName=" + serviceName + ")"); + } + + validateGrantRevokeRequest(grantRequest); + + String userName = grantRequest.getGrantor(); + Set userGroups = userMgr.getGroupsForUser(userName); + RangerAccessResource resource = new RangerAccessResourceImpl(grantRequest.getResource()); + boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); + + XXService xService = daoManager.getXXService().findByName(serviceName); + XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); + RangerService rangerService = svcStore.getServiceByName(serviceName); + + if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (isKeyAdmin) { + isAllowed = true; + }else { + isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); + } + }else{ + if (isAdmin) { + isAllowed = true; + } + else{ + isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); + } + } + + if (isAllowed) { + RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource, userName); + + if(policy != null) { + boolean policyUpdated = false; + policyUpdated = ServiceRESTUtil.processGrantRequest(policy, grantRequest); + + if(policyUpdated) { + svcStore.updatePolicy(policy); + } else { + LOG.error("processSecureGrantRequest processing failed"); + throw new Exception("processSecureGrantRequest processing failed"); + } + } else { + policy = new RangerPolicy(); + policy.setService(serviceName); + policy.setName("grant-" + System.currentTimeMillis()); // TODO: better policy name + policy.setDescription("created by grant"); + policy.setIsAuditEnabled(grantRequest.getEnableAudit()); + policy.setCreatedBy(userName); + + Map policyResources = new HashMap(); + Set resourceNames = resource.getKeys(); + + if(! CollectionUtils.isEmpty(resourceNames)) { + for(String resourceName : resourceNames) { + RangerPolicyResource policyResource = new RangerPolicyResource(resource.getValue(resourceName)); + policyResource.setIsRecursive(grantRequest.getIsRecursive()); + + policyResources.put(resourceName, policyResource); + } + } + policy.setResources(policyResources); + + RangerPolicyItem policyItem = new RangerPolicyItem(); + + policyItem.setDelegateAdmin(grantRequest.getDelegateAdmin()); + policyItem.getUsers().addAll(grantRequest.getUsers()); + policyItem.getGroups().addAll(grantRequest.getGroups()); + + for(String accessType : grantRequest.getAccessTypes()) { + policyItem.getAccesses().add(new RangerPolicyItemAccess(accessType, Boolean.TRUE)); + } + + policy.getPolicyItems().add(policyItem); + + svcStore.createPolicy(policy); + } + }else{ + LOG.error("secureGrantAccess(" + serviceName + ", " + grantRequest + ") failed as User doesn't have permission to grant Policy"); + throw restErrorUtil.createGrantRevokeRESTException( "User doesn't have necessary permission to grant access"); + } + } catch(WebApplicationException excp) { + throw excp; + } catch(Throwable excp) { + LOG.error("secureGrantAccess(" + serviceName + ", " + grantRequest + ") failed", excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } finally { + RangerPerfTracer.log(perf); + } + + ret.setStatusCode(RESTResponse.STATUS_SUCCESS); + } } if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceREST.secureGrantAccess(" + serviceName + ", " + grantRequest + "): " + ret); @@ -1065,52 +1082,52 @@ public RESTResponse revokeAccess(@PathParam("serviceName") String serviceName, G RESTResponse ret = new RESTResponse(); RangerPerfTracer perf = null; + if(revokeRequest!=null){ + if (serviceUtil.isValidateHttpsAuthentication(serviceName,request)) { - if (serviceUtil.isValidateHttpsAuthentication(serviceName,request)) { + try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.revokeAccess(serviceName=" + serviceName + ")"); + } - try { - if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.revokeAccess(serviceName=" + serviceName + ")"); - } + validateGrantRevokeRequest(revokeRequest); - String userName = revokeRequest.getGrantor(); - Set userGroups = userMgr.getGroupsForUser(userName); - RangerAccessResource resource = new RangerAccessResourceImpl(revokeRequest.getResource()); + String userName = revokeRequest.getGrantor(); + Set userGroups = userMgr.getGroupsForUser(userName); + RangerAccessResource resource = new RangerAccessResourceImpl(revokeRequest.getResource()); - boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); - - if(!isAdmin) { - throw restErrorUtil.createGrantRevokeRESTException("User doesn't have necessary permission to revoke access"); - } - - RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource); - - if(policy != null) { - boolean policyUpdated = false; - policyUpdated = ServiceRESTUtil.processRevokeRequest(policy, revokeRequest); + boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); - if(policyUpdated) { - svcStore.updatePolicy(policy); - } else { - LOG.error("processRevokeRequest processing failed"); - throw new Exception("processRevokeRequest processing failed"); + if(!isAdmin) { + throw restErrorUtil.createGrantRevokeRESTException("User doesn't have necessary permission to revoke access"); } - } else { - // nothing to revoke! + + RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource, userName); + + if(policy != null) { + boolean policyUpdated = false; + policyUpdated = ServiceRESTUtil.processRevokeRequest(policy, revokeRequest); + + if(policyUpdated) { + svcStore.updatePolicy(policy); + } else { + LOG.error("processRevokeRequest processing failed"); + throw new Exception("processRevokeRequest processing failed"); + } + } + } catch(WebApplicationException excp) { + throw excp; + } catch(Throwable excp) { + LOG.error("revokeAccess(" + serviceName + ", " + revokeRequest + ") failed", excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } finally { + RangerPerfTracer.log(perf); } - } catch(WebApplicationException excp) { - throw excp; - } catch(Throwable excp) { - LOG.error("revokeAccess(" + serviceName + ", " + revokeRequest + ") failed", excp); - - throw restErrorUtil.createRESTException(excp.getMessage()); - } finally { - RangerPerfTracer.log(perf); + + ret.setStatusCode(RESTResponse.STATUS_SUCCESS); } - - ret.setStatusCode(RESTResponse.STATUS_SUCCESS); } - if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceREST.revokeAccess(" + serviceName + ", " + revokeRequest + "): " + ret); } @@ -1127,68 +1144,71 @@ public RESTResponse secureRevokeAccess(@PathParam("serviceName") String serviceN } RESTResponse ret = new RESTResponse(); RangerPerfTracer perf = null; - if (serviceUtil.isValidateHttpsAuthentication(serviceName,request)) { - try { - if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.secureRevokeAccess(serviceName=" + serviceName + ")"); - } - String userName = revokeRequest.getGrantor(); - Set userGroups = userMgr.getGroupsForUser(userName); - RangerAccessResource resource = new RangerAccessResourceImpl(revokeRequest.getResource()); - boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); - boolean isAllowed = false; - boolean isKeyAdmin = bizUtil.isKeyAdmin(); - - XXService xService = daoManager.getXXService().findByName(serviceName); - XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); - RangerService rangerService = svcStore.getServiceByName(serviceName); - - if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { - if (isKeyAdmin) { - isAllowed = true; - }else { - isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); - } - }else{ - if (isAdmin) { - isAllowed = true; + if(revokeRequest!=null){ + if (serviceUtil.isValidService(serviceName,request)) { + try { + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.secureRevokeAccess(serviceName=" + serviceName + ")"); } - else{ - isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); + + validateGrantRevokeRequest(revokeRequest); + + String userName = revokeRequest.getGrantor(); + Set userGroups = userMgr.getGroupsForUser(userName); + RangerAccessResource resource = new RangerAccessResourceImpl(revokeRequest.getResource()); + boolean isAdmin = hasAdminAccess(serviceName, userName, userGroups, resource); + boolean isAllowed = false; + boolean isKeyAdmin = bizUtil.isKeyAdmin(); + + XXService xService = daoManager.getXXService().findByName(serviceName); + XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById(xService.getType()); + RangerService rangerService = svcStore.getServiceByName(serviceName); + + if (StringUtils.equals(xServiceDef.getImplclassname(), EmbeddedServiceDefsUtil.KMS_IMPL_CLASS_NAME)) { + if (isKeyAdmin) { + isAllowed = true; + }else { + isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); + } + }else{ + if (isAdmin) { + isAllowed = true; + } + else{ + isAllowed = bizUtil.isUserAllowedForGrantRevoke(rangerService, Allowed_User_List_For_Grant_Revoke, userName); + } } - } - - if (isAllowed) { - RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource); - - if(policy != null) { - boolean policyUpdated = false; - policyUpdated = ServiceRESTUtil.processRevokeRequest(policy, revokeRequest); - - if(policyUpdated) { - svcStore.updatePolicy(policy); - } else { - LOG.error("processSecureRevokeRequest processing failed"); - throw new Exception("processSecureRevokeRequest processing failed"); + + if (isAllowed) { + RangerPolicy policy = getExactMatchPolicyForResource(serviceName, resource, userName); + + if(policy != null) { + boolean policyUpdated = false; + policyUpdated = ServiceRESTUtil.processRevokeRequest(policy, revokeRequest); + + if(policyUpdated) { + svcStore.updatePolicy(policy); + } else { + LOG.error("processSecureRevokeRequest processing failed"); + throw new Exception("processSecureRevokeRequest processing failed"); + } } - } else { - // nothing to revoke! + }else{ + LOG.error("secureRevokeAccess(" + serviceName + ", " + revokeRequest + ") failed as User doesn't have permission to revoke Policy"); + throw restErrorUtil.createGrantRevokeRESTException("User doesn't have necessary permission to revoke access"); } - }else{ - LOG.error("secureRevokeAccess(" + serviceName + ", " + revokeRequest + ") failed as User doesn't have permission to revoke Policy"); - throw restErrorUtil.createGrantRevokeRESTException("User doesn't have necessary permission to revoke access"); + } catch(WebApplicationException excp) { + throw excp; + } catch(Throwable excp) { + LOG.error("secureRevokeAccess(" + serviceName + ", " + revokeRequest + ") failed", excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } finally { + RangerPerfTracer.log(perf); } - } catch(WebApplicationException excp) { - throw excp; - } catch(Throwable excp) { - LOG.error("secureRevokeAccess(" + serviceName + ", " + revokeRequest + ") failed", excp); - - throw restErrorUtil.createRESTException(excp.getMessage()); - } finally { - RangerPerfTracer.log(perf); + + ret.setStatusCode(RESTResponse.STATUS_SUCCESS); } - - ret.setStatusCode(RESTResponse.STATUS_SUCCESS); } if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceREST.secureRevokeAccess(" + serviceName + ", " + revokeRequest + "): " + ret); @@ -1294,7 +1314,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques @POST @Path("/policies/apply") @Produces({ "application/json", "application/xml" }) - public RangerPolicy applyPolicy(RangerPolicy policy) { + public RangerPolicy applyPolicy(RangerPolicy policy, @Context HttpServletRequest request) { if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.applyPolicy(" + policy + ")"); } @@ -1309,7 +1329,8 @@ public RangerPolicy applyPolicy(RangerPolicy policy) { throw new Exception("Applied policy contains condition(s); not supported:" + policy); } - RangerPolicy existingPolicy = getExactMatchPolicyForResource(policy.getService(), policy.getResources()); + String user = request.getRemoteUser(); + RangerPolicy existingPolicy = getExactMatchPolicyForResource(policy.getService(), policy.getResources(), StringUtils.isNotBlank(user) ? user :"admin"); if (existingPolicy == null) { ret = createPolicy(policy, null); @@ -1325,7 +1346,7 @@ public RangerPolicy applyPolicy(RangerPolicy policy) { throw restErrorUtil.createRESTException(exception.getMessage()); } } else { - throw restErrorUtil.createRESTException("Non-existing service specified:" + policy == null ? null : policy.getService()); + throw restErrorUtil.createRESTException("Non-existing service specified:"); } if (LOG.isDebugEnabled()) { @@ -1846,7 +1867,7 @@ public ServicePolicies getSecureServicePoliciesIfUpdated(@PathParam("serviceName boolean isAdmin = bizUtil.isAdmin(); boolean isKeyAdmin = bizUtil.isKeyAdmin(); request.setAttribute("downloadPolicy", "secure"); - if (serviceUtil.isValidateHttpsAuthentication(serviceName, request)) { + if (serviceUtil.isValidService(serviceName, request)) { if (lastKnownVersion == null) { lastKnownVersion = Long.valueOf(-1); } @@ -1863,10 +1884,12 @@ public ServicePolicies getSecureServicePoliciesIfUpdated(@PathParam("serviceName if (isKeyAdmin) { isAllowed = true; }else { - isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Download); - if(!isAllowed){ - isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Grant_Revoke); - } + if(rangerService!=null){ + isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Download); + if(!isAllowed){ + isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Grant_Revoke); + } + } } }else{ rangerService = svcStore.getServiceByName(serviceName); @@ -1874,10 +1897,12 @@ public ServicePolicies getSecureServicePoliciesIfUpdated(@PathParam("serviceName isAllowed = true; } else{ - isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Download); - if(!isAllowed){ - isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Grant_Revoke); - } + if(rangerService!=null){ + isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Download); + if(!isAllowed){ + isAllowed = bizUtil.isUserAllowed(rangerService, Allowed_User_List_For_Grant_Revoke); + } + } } } if (isAllowed) { @@ -1936,14 +1961,18 @@ private void createPolicyDownloadAudit(String serviceName, Long lastKnownVersion } } - private RangerPolicy getExactMatchPolicyForResource(String serviceName, RangerAccessResource resource) throws Exception { + private RangerPolicy getExactMatchPolicyForResource(String serviceName, RangerAccessResource resource, String user) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceREST.getExactMatchPolicyForResource(" + resource + ")"); + LOG.debug("==> ServiceREST.getExactMatchPolicyForResource(" + resource + ", " + user + ")"); } RangerPolicy ret = null; RangerPolicyEngine policyEngine = getPolicyEngine(serviceName); - List policies = policyEngine != null ? policyEngine.getExactMatchPolicies(resource) : null; + + Map evalContext = new HashMap(); + RangerAccessRequestUtil.setCurrentUserInContext(evalContext, user); + + List policies = policyEngine != null ? policyEngine.getExactMatchPolicies(resource, evalContext) : null; if(CollectionUtils.isNotEmpty(policies)) { // at this point, ret is a policy in policy-engine; the caller might update the policy (for grant/revoke); so get a copy from the store @@ -1951,20 +1980,24 @@ private RangerPolicy getExactMatchPolicyForResource(String serviceName, RangerAc } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceREST.getExactMatchPolicyForResource(" + resource + "): " + ret); + LOG.debug("<== ServiceREST.getExactMatchPolicyForResource(" + resource + ", " + user + "): " + ret); } return ret; } - private RangerPolicy getExactMatchPolicyForResource(String serviceName, Map resources) throws Exception { + private RangerPolicy getExactMatchPolicyForResource(String serviceName, Map resources, String user) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceREST.getExactMatchPolicyForResource(" + resources + ")"); + LOG.debug("==> ServiceREST.getExactMatchPolicyForResource(" + resources + ", " + user + ")"); } RangerPolicy ret = null; RangerPolicyEngine policyEngine = getPolicyEngine(serviceName); - List policies = policyEngine != null ? policyEngine.getExactMatchPolicies(resources) : null; + + Map evalContext = new HashMap(); + RangerAccessRequestUtil.setCurrentUserInContext(evalContext, user); + + List policies = policyEngine != null ? policyEngine.getExactMatchPolicies(resources, evalContext) : null; if(CollectionUtils.isNotEmpty(policies)) { // at this point, ret is a policy in policy-engine; the caller might update the policy (for grant/revoke); so get a copy from the store @@ -1972,7 +2005,7 @@ private RangerPolicy getExactMatchPolicyForResource(String serviceName, Map ServiceRESTUtil.processRevokeRequest()"); } @@ -93,29 +93,75 @@ static public boolean processRevokeRequest(RangerPolicy policy, GrantRevokeReque // remove all existing privileges for users and groups if (revokeRequest.getReplaceExistingPermissions()) { - policyUpdated = removeUsersAndGroupsFromPolicy(policy, revokeRequest.getUsers(), revokeRequest.getGroups()); + policyUpdated = removeUsersAndGroupsFromPolicy(existingRangerPolicy, revokeRequest.getUsers(), revokeRequest.getGroups()); } else { //Build a policy and set up policyItem in it to mimic revoke request - RangerPolicy appliedPolicy = new RangerPolicy(); + RangerPolicy appliedRangerPolicy = new RangerPolicy(); - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + RangerPolicy.RangerPolicyItem appliedRangerPolicyItem = new RangerPolicy.RangerPolicyItem(); - policyItem.setDelegateAdmin(revokeRequest.getDelegateAdmin()); - policyItem.getUsers().addAll(revokeRequest.getUsers()); - policyItem.getGroups().addAll(revokeRequest.getGroups()); + appliedRangerPolicyItem.setDelegateAdmin(revokeRequest.getDelegateAdmin()); + appliedRangerPolicyItem.getUsers().addAll(revokeRequest.getUsers()); + appliedRangerPolicyItem.getGroups().addAll(revokeRequest.getGroups()); - List accesses = new ArrayList(); + List appliedRangerPolicyItemAccess = new ArrayList(); - Set accessTypes = revokeRequest.getAccessTypes(); - for (String accessType : accessTypes) { - accesses.add(new RangerPolicy.RangerPolicyItemAccess(accessType, true)); + Set appliedPolicyItemAccessType = revokeRequest.getAccessTypes(); + for (String accessType : appliedPolicyItemAccessType) { + appliedRangerPolicyItemAccess.add(new RangerPolicy.RangerPolicyItemAccess(accessType, false)); } - policyItem.setAccesses(accesses); - - appliedPolicy.getDenyPolicyItems().add(policyItem); - - processApplyPolicy(policy, appliedPolicy); + appliedRangerPolicyItem.setAccesses(appliedRangerPolicyItemAccess); + + appliedRangerPolicy.getPolicyItems().add(appliedRangerPolicyItem); + + //List appliedRangerPolicyItems = appliedRangerPolicy.getPolicyItems(); + processApplyPolicyForItemType(existingRangerPolicy, appliedRangerPolicy, POLICYITEM_TYPE.ALLOW); + /*if (CollectionUtils.isNotEmpty(appliedRangerPolicyItems)) { + Set users = new HashSet(); + Set groups = new HashSet(); + + Map userPolicyItems = new HashMap(); + Map groupPolicyItems = new HashMap(); + + // Extract users and groups specified in appliedPolicy items + extractUsersAndGroups(appliedRangerPolicyItems, users, groups); + + // Split existing policyItems for users and groups extracted from appliedPolicyItem into userPolicyItems and groupPolicyItems + splitExistingPolicyItems(existingRangerPolicy, users, userPolicyItems, groups, groupPolicyItems); + + for (RangerPolicy.RangerPolicyItem tempPolicyItem : appliedRangerPolicyItems) { + List appliedPolicyItemsUser = tempPolicyItem.getUsers(); + for (String user : appliedPolicyItemsUser) { + RangerPolicy.RangerPolicyItem[] rangerPolicyItems = userPolicyItems.get(user); + if(rangerPolicyItems!=null && rangerPolicyItems.length>0){ + removeAccesses(rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()], tempPolicyItem.getAccesses()); + if(!CollectionUtils.isEmpty(rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()].getAccesses())){ + rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()].setDelegateAdmin(revokeRequest.getDelegateAdmin()); + }else{ + rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()].setDelegateAdmin(Boolean.FALSE); + } + } + } + } + for (RangerPolicy.RangerPolicyItem tempPolicyItem : appliedRangerPolicyItems) { + List appliedPolicyItemsGroup = tempPolicyItem.getGroups(); + for (String group : appliedPolicyItemsGroup) { + RangerPolicy.RangerPolicyItem[] rangerPolicyItems = groupPolicyItems.get(group); + if(rangerPolicyItems!=null && rangerPolicyItems.length>0){ + removeAccesses(rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()], tempPolicyItem.getAccesses()); + if(!CollectionUtils.isEmpty(rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()].getAccesses())){ + rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()].setDelegateAdmin(revokeRequest.getDelegateAdmin()); + }else{ + rangerPolicyItems[POLICYITEM_TYPE.ALLOW.ordinal()].setDelegateAdmin(Boolean.FALSE); + } + } + } + } + // Add modified/new policyItems back to existing policy + mergeProcessedPolicyItems(existingRangerPolicy, userPolicyItems, groupPolicyItems); + compactPolicy(existingRangerPolicy); + }*/ policyUpdated = true; } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java index 3dfb250ff5..8aef9a8d61 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java @@ -191,7 +191,9 @@ public void deleteTagDefByGuid(@PathParam("guid") String guid) { try { RangerTagDef exist = tagStore.getTagDefByGuid(guid); - tagStore.deleteTagDef(exist.getId()); + if(exist!=null){ + tagStore.deleteTagDef(exist.getId()); + } } catch(Exception excp) { LOG.error("deleteTagDef(" + guid + ") failed", excp); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/UserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/UserREST.java index 3fad7b4377..63729d6a47 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/UserREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/UserREST.java @@ -296,22 +296,24 @@ public String suggestUserFirstName(@QueryParam("letters") String letters, @Path("{userId}/passwordchange") @Produces({ "application/xml", "application/json" }) public VXResponse changePassword(@PathParam("userId") Long userId, - VXPasswordChange changePassword) { - logger.info("changePassword:" + userId); - - XXPortalUser gjUser = daoManager.getXXPortalUser().getById(userId); - if (gjUser == null) { - logger.warn("SECURITY:changePassword(): Invalid userId provided: userId=" - + userId); - throw restErrorUtil.createRESTException("serverMsg.userRestUser", - MessageEnums.DATA_NOT_FOUND, null, null, "" + userId); - } - - userManager.checkAccess(gjUser); - changePassword.setId(userId); - VXResponse ret = userManager.changePassword(changePassword); - return ret; - } + VXPasswordChange changePassword) { + if(changePassword==null || stringUtil.isEmpty(changePassword.getLoginId())){ + logger.warn("SECURITY:changePassword(): Invalid loginId provided. loginId was empty or null"); + throw restErrorUtil.createRESTException("serverMsg.userRestUser",MessageEnums.DATA_NOT_FOUND, null, null,""); + } + + logger.info("changePassword:" + changePassword.getLoginId()); + XXPortalUser gjUser = daoManager.getXXPortalUser().findByLoginId(changePassword.getLoginId()); + if (gjUser == null) { + logger.warn("SECURITY:changePassword(): Invalid loginId provided: loginId="+ changePassword.getLoginId()); + throw restErrorUtil.createRESTException("serverMsg.userRestUser",MessageEnums.DATA_NOT_FOUND, null, null, changePassword.getLoginId()); + } + + userManager.checkAccessForUpdate(gjUser); + changePassword.setId(gjUser.getId()); + VXResponse ret = userManager.changePassword(changePassword); + return ret; + } /** * @@ -322,22 +324,24 @@ public VXResponse changePassword(@PathParam("userId") Long userId, @POST @Path("{userId}/emailchange") @Produces({ "application/xml", "application/json" }) - public VXPortalUser changeEmailAddress(@PathParam("userId") Long userId, - VXPasswordChange changeEmail) { - logger.info("changeEmail:" + userId); - - XXPortalUser gjUser = daoManager.getXXPortalUser().getById(userId); - if (gjUser == null) { - logger.warn("SECURITY:changeEmail(): Invalid userId provided: userId=" - + userId); - throw restErrorUtil.createRESTException("serverMsg.userRestUser", - MessageEnums.DATA_NOT_FOUND, null, null, "" + userId); - } - - userManager.checkAccess(gjUser); - changeEmail.setId(userId); - VXPortalUser ret = userManager.changeEmailAddress(gjUser, changeEmail); - return ret; - } + public VXPortalUser changeEmailAddress(@PathParam("userId") Long userId, + VXPasswordChange changeEmail) { + if(changeEmail==null || stringUtil.isEmpty(changeEmail.getLoginId())){ + logger.warn("SECURITY:changeEmail(): Invalid loginId provided. loginId was empty or null"); + throw restErrorUtil.createRESTException("serverMsg.userRestUser",MessageEnums.DATA_NOT_FOUND, null, null,""); + } + + logger.info("changeEmail:" + changeEmail.getLoginId()); + XXPortalUser gjUser = daoManager.getXXPortalUser().findByLoginId(changeEmail.getLoginId()); + if (gjUser == null) { + logger.warn("SECURITY:changeEmail(): Invalid loginId provided: loginId="+ changeEmail.getLoginId()); + throw restErrorUtil.createRESTException("serverMsg.userRestUser",MessageEnums.DATA_NOT_FOUND, null, null, changeEmail.getLoginId()); + } + + userManager.checkAccessForUpdate(gjUser); + changeEmail.setId(gjUser.getId()); + VXPortalUser ret = userManager.changeEmailAddress(gjUser, changeEmail); + return ret; + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java index 899d8662c9..fe225c7508 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java @@ -97,7 +97,7 @@ public boolean isAPISpnegoAccessible(){ UserSessionBase userSession = ContextUtil.getCurrentUserSession(); if (userSession != null && userSession.isSpnegoEnabled()) { return true; - }else if(userSession != null && userSession.isUserAdmin()){ + }else if(userSession != null && (userSession.isUserAdmin() || userSession.isKeyAdmin())){ return true; } throw restErrorUtil.createRESTException(HttpServletResponse.SC_FORBIDDEN, "User is not allowed to access the API", true); diff --git a/security-admin/src/main/java/org/apache/ranger/security/handler/RangerAuthenticationProvider.java b/security-admin/src/main/java/org/apache/ranger/security/handler/RangerAuthenticationProvider.java index 3fa3436e64..00541cb5b9 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/handler/RangerAuthenticationProvider.java +++ b/security-admin/src/main/java/org/apache/ranger/security/handler/RangerAuthenticationProvider.java @@ -569,12 +569,15 @@ private Authentication getJDBCAuthentication(Authentication authentication,Strin } authenticator.setSaltSource(saltSource); - - String userName = authentication.getName(); + String userName =""; String userPassword = ""; - if (authentication.getCredentials() != null) { - userPassword = authentication.getCredentials().toString(); + if(authentication!=null){ + userName = authentication.getName(); + if (authentication.getCredentials() != null) { + userPassword = authentication.getCredentials().toString(); + } } + String rangerLdapDefaultRole = PropertiesUtil.getProperty("ranger.ldap.default.role", "ROLE_USER"); if (userName != null && userPassword != null && !userName.trim().isEmpty()&& !userPassword.trim().isEmpty()) { final List grantedAuths = new ArrayList<>(); diff --git a/security-admin/src/main/java/org/apache/ranger/security/listener/SpringEventListener.java b/security-admin/src/main/java/org/apache/ranger/security/listener/SpringEventListener.java index 5876445f66..e26365d198 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/listener/SpringEventListener.java +++ b/security-admin/src/main/java/org/apache/ranger/security/listener/SpringEventListener.java @@ -50,7 +50,7 @@ public void onApplicationEvent(AbstractAuthenticationEvent event) { process((AuthenticationFailureBadCredentialsEvent) event); } else if (event instanceof AuthenticationFailureDisabledEvent) { process((AuthenticationFailureDisabledEvent) event); - } else { + // } else { // igonre all other events } diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthFailureHandler.java b/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthFailureHandler.java index 1859ebc11b..5c0738c61f 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthFailureHandler.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthFailureHandler.java @@ -113,9 +113,9 @@ public void onAuthenticationFailure(HttpServletRequest request, if (logger.isDebugEnabled()) { logger.debug("Sending login failed response : " + jsonResp); } - } else { + }// else { // super.onAuthenticationFailure(request, response, exception); - } + //} } } diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthenticationEntryPoint.java b/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthenticationEntryPoint.java index 2c06f58c3c..8a7c6414fe 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthenticationEntryPoint.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/authentication/RangerAuthenticationEntryPoint.java @@ -129,9 +129,10 @@ public void commence(HttpServletRequest request, response.sendError(ajaxReturnCode, ""); } else if (!(requestURL.startsWith(reqServletPath))) { if(requestURL.contains(RangerSSOAuthenticationFilter.LOCAL_LOGIN_URL)){ - if (request.getSession() != null) + if (request.getSession() != null){ request.getSession().setAttribute("locallogin","true"); request.getServletContext().setAttribute(request.getSession().getId(), "locallogin"); + } } if(request.getHeader("x-forwarded-server") != null){ super.setUseForward(true); diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java index d431bc1a9e..47836083e5 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSSOAuthenticationFilter.java @@ -115,14 +115,15 @@ public void init(FilterConfig filterConfig) throws ServletException { public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)throws IOException, ServletException { HttpServletRequest httpRequest = (HttpServletRequest)servletRequest; - if (httpRequest.getRequestedSessionId() != null && !httpRequest.isRequestedSessionIdValid()) - { - if(httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()) != null && httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()).toString().equals("locallogin")){ - ssoEnabled = false; - httpRequest.getSession().setAttribute("locallogin","true"); - httpRequest.getServletContext().removeAttribute(httpRequest.getRequestedSessionId()); - } - } + if (httpRequest.getRequestedSessionId() != null && !httpRequest.isRequestedSessionIdValid()){ + synchronized(httpRequest.getServletContext()){ + if(httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()) != null && httpRequest.getServletContext().getAttribute(httpRequest.getRequestedSessionId()).toString().equals("locallogin")){ + ssoEnabled = false; + httpRequest.getSession().setAttribute("locallogin","true"); + httpRequest.getServletContext().removeAttribute(httpRequest.getRequestedSessionId()); + } + } + } RangerSecurityContext context = RangerContextHolder.getSecurityContext(); UserSessionBase session = context != null ? context.getUserSession() : null; diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java index 7314782fe6..3763687459 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java @@ -90,14 +90,15 @@ public void doFilter(ServletRequest request, ServletResponse response, Authentication auth = SecurityContextHolder.getContext() .getAuthentication(); - if (auth instanceof AnonymousAuthenticationToken) { - // ignore - } else { + if (!(auth instanceof AnonymousAuthenticationToken)) { HttpServletRequest httpRequest = (HttpServletRequest) request; HttpSession httpSession = httpRequest.getSession(false); // [1]get the context from session - RangerSecurityContext context = (RangerSecurityContext) httpSession.getAttribute(AKA_SC_SESSION_KEY); + RangerSecurityContext context = null; + if(httpSession!=null){ + context=(RangerSecurityContext) httpSession.getAttribute(AKA_SC_SESSION_KEY); + } int clientTimeOffset = 0; if (context == null) { context = new RangerSecurityContext(); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java index 65ad5adc59..e94bad44a5 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java @@ -68,7 +68,7 @@ public void createObjectDataHistory(RangerBaseModelObject baseModelObj, String a String objectGuid = baseModelObj.getGuid(); Date currentDate = DateUtil.getUTCDate(); - XXDataHist xDataHist = new XXDataHist();; + XXDataHist xDataHist = new XXDataHist(); xDataHist.setObjectId(baseModelObj.getId()); xDataHist.setObjectGuid(objectGuid); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java index 4b792de7b7..5616406113 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java @@ -281,7 +281,7 @@ private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName, oldValue = String.valueOf(processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled())); } } - if (oldValue == null || value.equalsIgnoreCase(oldValue)) { + if (oldValue == null || oldValue.equalsIgnoreCase(value)) { return null; } else if (fieldName.equalsIgnoreCase(POLICY_RESOURCE_CLASS_FIELD_NAME)) { // Compare old and new resources diff --git a/security-admin/src/main/java/org/apache/ranger/service/XGroupPermissionService.java b/security-admin/src/main/java/org/apache/ranger/service/XGroupPermissionService.java index 20c3b67d25..3df523309b 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XGroupPermissionService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XGroupPermissionService.java @@ -17,12 +17,17 @@ package org.apache.ranger.service; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.SearchField; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXGroupPermission; import org.apache.ranger.view.VXGroupPermission; +import org.apache.ranger.view.VXModuleDef; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; @@ -77,4 +82,25 @@ public VXGroupPermission populateViewBean(XXGroupPermission xObj) { vObj.setGroupName(xGroup.getName()); return vObj; } + + public List getPopulatedVXGroupPermissionList(List xgroupPermissionList,Map xXGroupMap,VXModuleDef vModuleDef){ + List vXGroupPermissionList = new ArrayList(); + XXGroup xXGroup=null; + for(XXGroupPermission xgroupPermission:xgroupPermissionList){ + if(xXGroupMap.containsKey(xgroupPermission.getGroupId())){ + xXGroup =xXGroupMap.get(xgroupPermission.getGroupId()); + VXGroupPermission vXGrpPerm=new VXGroupPermission(); + vXGrpPerm.setId(xgroupPermission.getId()); + vXGrpPerm.setGroupId(xgroupPermission.getGroupId()); + vXGrpPerm.setModuleId(xgroupPermission.getModuleId()); + vXGrpPerm.setIsAllowed(xgroupPermission.getIsAllowed()); + vXGrpPerm.setCreateDate(xgroupPermission.getCreateTime()); + vXGrpPerm.setUpdateDate(xgroupPermission.getUpdateTime()); + vXGrpPerm.setGroupName(xXGroup.getName()); + vXGrpPerm.setModuleName(vModuleDef.getModule()); + vXGroupPermissionList.add(vXGrpPerm); + } + } + return vXGroupPermissionList; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java b/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java index 19c3d1981b..5c5d59a261 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XGroupService.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.MessageEnums; @@ -41,6 +42,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service @Scope("singleton") @@ -253,5 +255,18 @@ protected XXGroup mapViewToEntityBean(VXGroup vObj, XXGroup mObj, int OPERATION_ protected VXGroup mapEntityToViewBean(VXGroup vObj, XXGroup mObj) { super.mapEntityToViewBean(vObj, mObj); return vObj; - } + } + + public Map getXXGroupIdXXGroupMap(){ + Map xXGroupMap=new HashMap(); + try{ + List xXGroupList=rangerDaoManager.getXXGroup().getAll(); + if(!CollectionUtils.isEmpty(xXGroupList)){ + for(XXGroup xXGroup:xXGroupList){ + xXGroupMap.put(xXGroup.getId(), xXGroup); + } + } + }catch(Exception ex){} + return xXGroupMap; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XModuleDefService.java b/security-admin/src/main/java/org/apache/ranger/service/XModuleDefService.java index 2e006433b3..b8403e87a7 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XModuleDefService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XModuleDefService.java @@ -19,12 +19,14 @@ import java.util.ArrayList; import java.util.List; - +import java.util.Map; import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.SearchField; import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXGroup; import org.apache.ranger.entity.XXGroupPermission; import org.apache.ranger.entity.XXModuleDef; +import org.apache.ranger.entity.XXUser; import org.apache.ranger.entity.XXUserPermission; import org.apache.ranger.view.VXGroupPermission; import org.apache.ranger.view.VXModuleDef; @@ -32,6 +34,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service @Scope("singleton") @@ -49,6 +52,11 @@ public class XModuleDefService extends @Autowired XGroupPermissionService xGrpPermService; + @Autowired + XUserService xUserService; + + @Autowired + XGroupService xGroupService; public XModuleDefService() { searchFields.add(new SearchField("module", "obj.module", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL)); @@ -84,34 +92,33 @@ protected void validateForUpdate(VXModuleDef vObj, XXModuleDef mObj) { @Override public VXModuleDef populateViewBean(XXModuleDef xObj) { - VXModuleDef vModuleDef = super.populateViewBean(xObj); + Map xXPortalUserIdXXUserMap=xUserService.getXXPortalUserIdXXUserMap(); + Map xXGroupMap=xGroupService.getXXGroupIdXXGroupMap(); List vXUserPermissionList = new ArrayList(); List vXGroupPermissionList = new ArrayList(); - List xuserPermissionList = rangerDaoManager .getXXUserPermission().findByModuleId(xObj.getId(), false); List xgroupPermissionList = rangerDaoManager .getXXGroupPermission().findByModuleId(xObj.getId(), false); - for (XXUserPermission xUserPerm : xuserPermissionList) { - - VXUserPermission vXUserPerm = xUserPermService - .populateViewBean(xUserPerm); - vXUserPermissionList.add(vXUserPerm); - + if(CollectionUtils.isEmpty(xXPortalUserIdXXUserMap)){ + for (XXUserPermission xUserPerm : xuserPermissionList) { + VXUserPermission vXUserPerm = xUserPermService.populateViewBean(xUserPerm); + vXUserPermissionList.add(vXUserPerm); + } + }else{ + vXUserPermissionList=xUserPermService.getPopulatedVXUserPermissionList(xuserPermissionList,xXPortalUserIdXXUserMap,vModuleDef); } - - for (XXGroupPermission xGrpPerm : xgroupPermissionList) { - - VXGroupPermission vXGrpPerm = xGrpPermService - .populateViewBean(xGrpPerm); - vXGroupPermissionList.add(vXGrpPerm); - + if(CollectionUtils.isEmpty(xXGroupMap)){ + for (XXGroupPermission xGrpPerm : xgroupPermissionList) { + VXGroupPermission vXGrpPerm = xGrpPermService.populateViewBean(xGrpPerm); + vXGroupPermissionList.add(vXGrpPerm); + } + }else{ + vXGroupPermissionList=xGrpPermService.getPopulatedVXGroupPermissionList(xgroupPermissionList,xXGroupMap,vModuleDef); } - vModuleDef.setUserPermList(vXUserPermissionList); vModuleDef.setGroupPermList(vXGroupPermissionList); return vModuleDef; } - } diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserPermissionService.java b/security-admin/src/main/java/org/apache/ranger/service/XUserPermissionService.java index bd3a50df9d..3a97ef84da 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XUserPermissionService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XUserPermissionService.java @@ -17,11 +17,17 @@ package org.apache.ranger.service; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.apache.ranger.common.SearchField; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXModuleDef; import org.apache.ranger.entity.XXPortalUser; +import org.apache.ranger.entity.XXUser; import org.apache.ranger.entity.XXUserPermission; +import org.apache.ranger.view.VXModuleDef; import org.apache.ranger.view.VXUserPermission; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; @@ -71,4 +77,25 @@ public VXUserPermission populateViewBean(XXUserPermission xObj) { return vObj; } + public List getPopulatedVXUserPermissionList(List xuserPermissionList,Map xXPortalUserIdXXUserMap,VXModuleDef vModuleDef){ + List vXUserPermissionList = new ArrayList(); + XXUser xXUser=null; + for(XXUserPermission xuserPermission:xuserPermissionList){ + if(xXPortalUserIdXXUserMap.containsKey(xuserPermission.getUserId())){ + xXUser =xXPortalUserIdXXUserMap.get(xuserPermission.getUserId()); + VXUserPermission vXUserPerm=new VXUserPermission(); + vXUserPerm.setId(xuserPermission.getId()); + vXUserPerm.setUserId(xXUser.getId()); + vXUserPerm.setModuleId(xuserPermission.getModuleId()); + vXUserPerm.setIsAllowed(xuserPermission.getIsAllowed()); + vXUserPerm.setCreateDate(xuserPermission.getCreateTime()); + vXUserPerm.setUpdateDate(xuserPermission.getUpdateTime()); + vXUserPerm.setModuleName(vModuleDef.getModule()); + vXUserPerm.setLoginId(xXUser.getName()); + vXUserPerm.setUserName(xXUser.getName()); + vXUserPermissionList.add(vXUserPerm); + } + } + return vXUserPermissionList; + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java index 8210650b18..813079f294 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.ranger.biz.RangerBizUtil; @@ -47,6 +48,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service @Scope("singleton") @@ -383,5 +385,27 @@ public List getTransactionLog(VXUser vObj, VXPortalUser mObj, return trxLogList; } + public Map getXXPortalUserIdXXUserMap(){ + Map xXPortalUserIdXXUserMap=new HashMap(); + try{ + Map xXUserMap=new HashMap(); + List xXUserList=daoManager.getXXUser().getAll(); + if(!CollectionUtils.isEmpty(xXUserList)){ + for(XXUser xxUser:xXUserList){ + xXUserMap.put(xxUser.getName(), xxUser); + } + } + xXUserList=null; + List xXPortalUserList=daoManager.getXXPortalUser().getAll(); + if(!CollectionUtils.isEmpty(xXPortalUserList)){ + for(XXPortalUser xXPortalUser:xXPortalUserList){ + if(xXUserMap.containsKey(xXPortalUser.getLoginId())){ + xXPortalUserIdXXUserMap.put(xXPortalUser.getId(),xXUserMap.get(xXPortalUser.getLoginId())); + } + } + } + }catch(Exception ex){} + return xXPortalUserIdXXUserMap; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java b/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java index 0e5bb08929..2e7881e979 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/service/filter/RangerRESTAPIFilter.java @@ -120,12 +120,7 @@ public ContainerResponse filter(ContainerRequest request, ContainerResponse response) { if (logStdOut) { // If it is image, then don't call super - if (response.getMediaType() != null) { - // logger.info("DELETE ME: Response= mediaType=" - // + response.getMediaType() + ", getType()" - // + response.getMediaType().getType() + ", getSubType()=" - // + response.getMediaType().getSubtype()); - } else { + if (response.getMediaType() == null) { logger.info("DELETE ME: Response= mediaType is null"); } if (response.getMediaType() == null diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java index 322f4426cf..d2255f2b0a 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java @@ -215,10 +215,10 @@ private VXAccessAudit populateViewBean(SolrDocument doc) { accessAudit.setClientIP(value.toString()); } value = doc.getFieldValue("logType"); - if (value != null) { + //if (value != null) { // TODO: Need to see what logType maps to in UI // accessAudit.setAuditType(solrUtil.toInt(value)); - } + //} value = doc.getFieldValue("result"); if (value != null) { accessAudit.setAccessResult(solrUtil.toInt(value)); diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java index 1b5793f7b2..b924646bd1 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java @@ -184,8 +184,12 @@ private void init() { } public SolrClient getSolrClient() { - if (solrClient == null) { - connect(); + if(solrClient!=null){ + return solrClient; + }else{ + synchronized(this){ + connect(); + } } return solrClient; } diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java index e912cfb35d..b09a73b6eb 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java @@ -122,7 +122,7 @@ public QueryResponse searchResources(SearchCriteria searchCriteria, } else if (searchField.getSearchType() == SEARCH_TYPE.GREATER_EQUAL_THAN || searchField.getSearchType() == SEARCH_TYPE.GREATER_THAN || searchField.getSearchType() == SEARCH_TYPE.LESS_EQUAL_THAN - || searchField.getSearchType() == SEARCH_TYPE.LESS_THAN) { + || searchField.getSearchType() == SEARCH_TYPE.LESS_THAN) { //NOPMD // TODO: Need to handle range here } else { String fq = setField(fieldName, paramValue); diff --git a/security-admin/src/main/webapp/WEB-INF/log4j.xml b/security-admin/src/main/webapp/WEB-INF/log4j.xml index 54abd065a2..359ed93600 100644 --- a/security-admin/src/main/webapp/WEB-INF/log4j.xml +++ b/security-admin/src/main/webapp/WEB-INF/log4j.xml @@ -85,7 +85,7 @@ diff --git a/security-admin/src/main/webapp/robots.txt b/security-admin/src/main/webapp/robots.txt index 941749507a..2d850ec9d1 100644 --- a/security-admin/src/main/webapp/robots.txt +++ b/security-admin/src/main/webapp/robots.txt @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # robotstxt.org User-agent: * diff --git a/security-admin/src/main/webapp/scripts/controllers/Controller.js b/security-admin/src/main/webapp/scripts/controllers/Controller.js index fc564962c3..041c6b7226 100755 --- a/security-admin/src/main/webapp/scripts/controllers/Controller.js +++ b/security-admin/src/main/webapp/scripts/controllers/Controller.js @@ -346,7 +346,7 @@ define(function(require) { var modulePermission = new ModulePermission({id : moduleId}); var that = this modulePermission.collection = new ModulePermissionList(); - modulePermission.fetch({cache : true}).done(function(){ + modulePermission.fetch({cache : false}).done(function(){ App.rContent.show(new view({ model : modulePermission, groupList : that.groupList, diff --git a/security-admin/src/main/webapp/scripts/models/VXPortalUser.js b/security-admin/src/main/webapp/scripts/models/VXPortalUser.js index f2b7a104a5..b0319747df 100644 --- a/security-admin/src/main/webapp/scripts/models/VXPortalUser.js +++ b/security-admin/src/main/webapp/scripts/models/VXPortalUser.js @@ -23,7 +23,6 @@ define(function(require){ var VXPortalUserBase = require('model_bases/VXPortalUserBase'); var XAEnums = require('utils/XAEnums'); - var XAUtils = require('utils/XAUtils'); var localization = require('utils/XALangSupport'); var VXPortalUser = VXPortalUserBase.extend( @@ -94,6 +93,7 @@ define(function(require){ userRoleList : { type : 'Select', options : function(callback, editor){ + var XAUtils = require('utils/XAUtils'); var userTypes = _.filter(XAEnums.UserRoles,function(m){return m.label != 'Unknown'}); var nvPairs = XAUtils.enumToSelectPairs(userTypes); callback(nvPairs); diff --git a/security-admin/src/main/webapp/scripts/models/XABaseModel.js b/security-admin/src/main/webapp/scripts/models/XABaseModel.js index 7d3a9151e3..03b4e13c43 100644 --- a/security-admin/src/main/webapp/scripts/models/XABaseModel.js +++ b/security-admin/src/main/webapp/scripts/models/XABaseModel.js @@ -27,7 +27,6 @@ define(function(require){ 'use strict'; var Backbone = require('backbone'); - var XAUtils = require('utils/XAUtils'); var XABaseModel = Backbone.Model.extend( /** @lends XABaseModel.prototype */ @@ -41,7 +40,11 @@ define(function(require){ }, bindErrorEvents :function(){ - this.bind("error", XAUtils.defaultErrorHandler); + //Moved require inside fuctn expression due to ie issue + this.bind("error", function(){ + var XAUtils = require('utils/XAUtils'); + XAUtils.defaultErrorHandler(); + }); }, /** * toString for a model. Every model should implement this function. diff --git a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js index 5604237823..7d050852b9 100644 --- a/security-admin/src/main/webapp/scripts/modules/XAOverrides.js +++ b/security-admin/src/main/webapp/scripts/modules/XAOverrides.js @@ -407,6 +407,7 @@ renderResource : function() { var that = this; if(!_.isNull(this.value) && !_.isEmpty(this.value)){ + this.value.values = _.map(this.value.values, function(val){ return _.escape(val); }); this.$resource.val(this.value.values.toString()) //to preserve resources values to text field if(!_.isUndefined(this.value.resourceType)){ diff --git a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js index de12935ab4..fb7a3265cc 100644 --- a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js +++ b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js @@ -233,7 +233,9 @@ define(function(require) { componentPermissions : 'Component Permissions', selectDataMaskTypes : 'Select Data Mask Types', accessTypes : 'Access Types', - rowLevelFilter : 'Row Level Filter' + rowLevelFilter : 'Row Level Filter', + selectAndAddUser : 'Select and Add User', + selectAndAddGroup : 'Select and Add Group', }, btn : { add : 'Add', @@ -338,7 +340,10 @@ define(function(require) { grpUpdatedSucc : 'Group updated successfully', grpCreatedSucc : 'Group created successfully', errorLoadingAuditLogs : 'Unable to connect to Audit store !!', - enterCustomMask : 'Please enter custom masked value or expression !!' + enterCustomMask : 'Please enter custom masked value or expression !!', + pleaseSelectUser : 'Please select user.', + pleaseSelectGroup : 'Please select group.', + addSelectedUserGroup : 'Please add selected user/group to permissions else user/group will not be added.' diff --git a/security-admin/src/main/webapp/scripts/utils/XAUtils.js b/security-admin/src/main/webapp/scripts/utils/XAUtils.js index 7c4e445f29..b776e8d39a 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAUtils.js +++ b/security-admin/src/main/webapp/scripts/utils/XAUtils.js @@ -430,7 +430,7 @@ define(function(require) { var groupArr = _.uniq(_.compact(_.map(rawValue.models, function(m, i) { if (m.has('groupName')) - return m.get('groupName'); + return _.escape(m.get('groupName')); }))); if (groupArr.length > 0) { if (rawValue.first().has('resourceId')) @@ -492,14 +492,14 @@ define(function(require) { if (i >= 4) { return ''; + + _.escape(name) + ''; } else if (i == 3 && groupArr.length > 4) { showMoreLess = true; return '' + name + ''; + + '-id="' + model.id + '">' + _.escape(name) + ''; } else { return '' + name + ''; + + '-id="' + model.id + '">' + _.escape(name) + ''; } }); if (showMoreLess) { @@ -521,24 +521,29 @@ define(function(require) { }; XAUtils.showGroupsOrUsers = function(rawValue, model, userOrGroups) { - var showMoreLess = false, objArr = []; + var showMoreLess = false, objArr, lastShowMoreCnt = 1, j = 1, listShownCnt = 5000; if (!_.isArray(rawValue) && rawValue.length == 0) return '--'; - if (userOrGroups == 'groups') { - _.each(rawValue, function(perm) { - objArr = _.union(objArr, perm.groupName) - }); - } else if (userOrGroups == 'users') { - _.each(rawValue, function(perm) { - objArr = _.union(objArr, perm.userName) - }); - } - + objArr = (userOrGroups == 'groups') ? _.pluck(rawValue, 'groupName') : _.pluck(rawValue, 'userName'); var newObjArr = _.map(objArr, function(name, i) { if (i >= 4) { - return ''; + var eleStr = '', span = '' + + _.escape(name) + ''; + if( (i + listShownCnt ) === (listShownCnt*j) + 4){ + eleStr = '
'+span; + if(i == objArr.length - 1){ + eleStr += '
'; + } + lastShowMoreCnt = ( listShownCnt*j ) + 4; + j++; + }else if(i === lastShowMoreCnt - 1 || i == objArr.length - 1){ + eleStr = span + ''; + + }else{ + eleStr = span; + } + return eleStr; } else if (i == 3 && objArr.length > 4) { showMoreLess = true; return ''); newObjArr.push(''); return newObjArr.length ? newObjArr.join(' ') : '--'; - }; XAUtils.defaultErrorHandler = function(model, error) { diff --git a/security-admin/src/main/webapp/scripts/views/common/AddGroup.js b/security-admin/src/main/webapp/scripts/views/common/AddGroup.js index 1246e577de..81fd9014d0 100644 --- a/security-admin/src/main/webapp/scripts/views/common/AddGroup.js +++ b/security-admin/src/main/webapp/scripts/views/common/AddGroup.js @@ -101,7 +101,7 @@ define(function(require){ values = $(that.el).find('.select2-container-multi').select2('data') } else { var groupNameList = that.model.get('groupNameList'); - values = _.map(that.model.get('groupIdList'),function(id,i){ return {'id': id, 'text': groupNameList[i]};}); + values = _.map(that.model.get('groupIdList'),function(id,i){ return {'id': id, 'text': _.escape(groupNameList[i]) };}); } valArr = _.map(values,function(val,i){ @@ -138,7 +138,7 @@ define(function(require){ }, getSelect2Options :function(){ var that = this,groupCnt = 0; - var tags = _.map(that.model.get('groupIdList'),function(id,i){ return {'id': id, 'text': that.model.get('groupNameList')[i]};}); + var tags = _.map(that.model.get('groupIdList'),function(id,i){ return {'id': id, 'text': _.escape(that.model.get('groupNameList')[i]) };}); return{ closeOnSelect : true, placeholder : 'Select Group', @@ -171,7 +171,7 @@ define(function(require){ selectedVals = that.$('.tags').data('editable').input.$input.val().split(','); } if(data.resultSize != "0"){ - results = data.vXGroups.map(function(m, i){ return {id : (m.id).toString(), text: m.name}; }); + results = data.vXGroups.map(function(m, i){ return {id : (m.id).toString(), text: _.escape(m.name) }; }); if(!_.isEmpty(selectedVals)) { results = XAUtil.filterResultByIds(results, selectedVals); } @@ -211,4 +211,4 @@ define(function(require){ }); return AddGroup; -}); \ No newline at end of file +}); diff --git a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionCreate.js b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionCreate.js index 02b879d307..0d1c92f1c9 100644 --- a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionCreate.js +++ b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionCreate.js @@ -92,29 +92,9 @@ define(function(require){ initializePlugins: function(){ }, renderForm : function(){ - var VXGroupList = require('collections/VXGroupList'); - var VXUserList = require('collections/VXUserList'); - var params = {sortBy : 'name'}; - this.userList = new VXUserList(); - this.userList.setPageSize(100,{fetch:true}); - this.userList.fetch({ - cache :false, - data: params, - async : false - }); - this.groupList = new VXGroupList(); - this.groupList.setPageSize(100,{fetch:true}); - this.groupList.fetch({ - cache :false, - data : params, - async : false - }); - var that = this; this.form = new ModulePermissionForm({ template : require('hbs!tmpl/permissions/ModulePermissionForm_tmpl'), - model : that.model, - groupList : that.groupList, - userList : that.userList + model : this.model, }); this.rForm.show(this.form); }, diff --git a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js index 9a97dc14dd..76154a92ed 100644 --- a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js +++ b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermissionForm.js @@ -48,20 +48,34 @@ define(function(require) { templateHelpers :function(){ }, templateData : function(){ - return { 'id' : this.model.id, 'permHeaders' : this.getPermHeaders() }; + return { + 'id' : this.model.id, + 'permHeaders' : this.getPermHeaders(), + 'userList' : this.model.get('userPermList'), + 'groupList' : this.model.get('groupPermList') + }; }, initialize : function(options) { - _.extend(this, _.pick(options, 'groupList','userList')); + _.extend(this, _.pick(options)); if (!this.model.isNew()){ this.setupFieldsforEditModule(); } Backbone.Form.prototype.initialize.call(this, options); }, ui : { - /*selectGroups : 'div[data-fields="selectGroups"]', - selectUsers : 'div[data-fields="selectUsers"]',*/ + selectGroups : 'div[data-editors="selectGroups"]', + selectUsers : 'div[data-editors="selectUsers"]', + addGroupBtn : '[data-id="addGroupBtn"]', + addUserBtn : '[data-id="addUserBtn"]', + }, - events : { + events : function(){ + var events = {}; + events['click ' + this.ui.addGroupBtn ] = 'onAddGroup'; + events['click ' + this.ui.addUserBtn ] = 'onAddUser'; + events['click ' + '[data-js="addUser"]'] = 'onAddUserClick'; + events['click ' + '[data-js="selectedGroupSpan"]'] = 'onRemovedGroupClick'; + return events; }, /** fields for the form */ @@ -81,13 +95,13 @@ define(function(require) { selectGroups : { type : 'Select2Remote', editorAttrs : {'placeholder' :'Select Group','tokenSeparators': [",", " "],multiple:true}, - pluginAttr: this.getPlugginAttr(true,{'lookupURL':"service/xusers/groups",'permList':that.model.get('groupPermList'),'idKey':'groupId','textKey':'groupName'}), + pluginAttr: this.getPlugginAttr(true,{'lookupURL':"service/xusers/groups",'idKey':'groupId','textKey':'groupName'}), title : localization.tt('lbl.selectGroup')+' *' }, selectUsers : { type : 'Select2Remote', editorAttrs : {'placeholder' :'Select User','tokenSeparators': [",", " "],multiple:true}, - pluginAttr: this.getPlugginAttr(true,{'lookupURL':"service/xusers/users",'permList':that.model.get('userPermList'),'idKey':'userId','textKey':'userName'}), + pluginAttr: this.getPlugginAttr(true,{'lookupURL':"service/xusers/users",'idKey':'userId','textKey':'userName'}), title : localization.tt('lbl.selectUser')+' *', }, isAllowed : { @@ -98,29 +112,28 @@ define(function(require) { } }, render: function(options) { - var that = this; Backbone.Form.prototype.render.call(this, options); - + this.$el.find('[data-js="selectedGroupList"] span i').on('click', this.removeGroup.bind(this)); + this.$el.find('[data-js="selectedUserList"] span i').on('click', this.removeUser.bind(this)); + if(this.model.get('groupPermList').length <= 0){ + this.$el.find('.emptySelectedGroups').show(); + }else{ + this.$el.find('.emptySelectedGroups').hide(); + } + if(this.model.get('userPermList').length <= 0){ + this.$el.find('.emptySelectedUsers').show(); + }else{ + this.$el.find('.emptySelectedUsers').hide(); + } }, setupFieldsforEditModule : function(){ - var groupsNVList=[],usersNVList =[]; - groupsNVList = _.map(this.model.get('groupPermList'),function(gPerm){ - return {'id': Number(gPerm.groupId), 'text':gPerm.groupName}; - }); - this.model.set('selectGroups', groupsNVList); - - usersNVList = _.map(this.model.get('userPermList'),function(uPerm){ - return {'id': Number(uPerm.userId), 'text':uPerm.userName}; - }); - this.model.set('selectUsers', usersNVList); - + this.addedGroups = _.map(this.model.get('groupPermList'), function(g){ return { 'id' : g.groupId, 'text' : g.groupName} }); + this.addedUsers = _.map(this.model.get('userPermList'), function(u){ return { 'id' : u.userId, 'text' : u.userName} }); }, getPermHeaders : function(){ var permList = []; - permList.unshift(localization.tt('lbl.allowAccess')); - permList.unshift(localization.tt('lbl.selectUser')); - permList.unshift(localization.tt('lbl.selectGroup')); -// permList.push(""); + permList.unshift(localization.tt('lbl.selectAndAddUser')); + permList.unshift(localization.tt('lbl.selectAndAddGroup')); return permList; }, getPlugginAttr :function(autocomplete, options){ @@ -131,25 +144,7 @@ define(function(require) { return { closeOnSelect : true, multiple: true, - minimumInputLength: 0, tokenSeparators: [",", " "], - initSelection : function (element, callback) { - var data = []; - _.each(options.permList,function (elem) { - data.push({id: elem[options.idKey], text: elem[options.textKey]}); - }); - callback(data); - }, - createSearchChoice: function(term, data) { - if ($(data).filter(function() { - return this.text.localeCompare(term) === 0; - }).length === 0) { - return { - id : term, - text: term - }; - } - }, ajax: { url: options.lookupURL, type : 'GET', @@ -169,13 +164,15 @@ define(function(require) { selectedVals = that.getSelectedValues(options); if(data.resultSize != "0"){ if(!_.isUndefined(data.vXGroups)){ - results = data.vXGroups.map(function(m, i){ return {id : m.id+"", text: m.name}; }); - } else if(!_.isUndefined(data.vXUsers)){ - results = data.vXUsers.map(function(m, i){ return {id : m.id+"", text: m.name}; }); - if(!_.isEmpty(selectedVals)){ - results = XAUtil.filterResultByText(results, selectedVals); - } + results = data.vXGroups.map(function(m, i){ return {id : m.id, text: _.escape(m.name)}; }); + } + else if(!_.isUndefined(data.vXUsers)){ + results = data.vXUsers.map(function(m, i){ return {id : m.id, text: _.escape(m.name)}; }); } + if(!_.isEmpty(selectedVals)){ + results = XAUtil.filterResultByText(results, selectedVals); + } + } return { results : results}; }, @@ -195,80 +192,136 @@ define(function(require) { return result.text; }, formatNoMatches : function(term){ - return options.textKey == 'groupName' ? 'No group found.' : 'No user found.'; + switch (term){ + default : return "No Matches found"; + } } }; } }, getSelectedValues : function(options){ var vals = [],selectedVals = []; - var type = options.textKey == 'groupName' ? 'selectGroups' : 'selectUsers'; - var $select = this.$('[name="'+type+'"]'); - if(!_.isEmpty($select.select2('data'))){ - selectedVals = _.map($select.select2('data'),function(obj){ return obj.text; }); + var added = options.textKey == 'groupName' ? this.addedGroups : this.addedUsers; + if(!_.isEmpty(added)){ + selectedVals = _.map(added, function(obj){ return obj.text; }); } vals.push.apply(vals , selectedVals); - vals = $.unique(vals); return vals; }, beforeSaveModulePermissions : function(){ - if(this.model.get('module') != ''){ - var groupValStr = this.fields.selectGroups.getValue(); - var userValStr = this.fields.selectUsers.getValue(); - this.compareAndUpdateObj(groupValStr,{'mode':'groups','permList':this.model.get('groupPermList'),'idKey':'groupId','textKey':'groupName'}); - this.compareAndUpdateObj(userValStr,{'mode':'users','permList':this.model.get('userPermList'),'idKey':'userId','textKey':'userName'}); + if(!_.isEmpty(this.fields.selectGroups.editor.$el.select2('data')) + || !_.isEmpty(this.fields.selectUsers.editor.$el.select2('data'))){ + XAUtil.alertPopup({ + msg :localization.tt('msg.addSelectedUserGroup'), + }); + return false; } + this.model.unset('selectUsers'); + this.model.unset('selectGroups'); + return true; }, - compareAndUpdateObj: function(objValsStr,options){ + onAddGroup : function(e){ + var that = this, newPerms = []; + var selectedGroups = this.fields.selectGroups.editor.$el.select2('data'); + _.each(selectedGroups, function(obj){ + var self = that; + this.$el.find('[data-js="selectedGroupList"]').append(' '+obj.text+'') + this.addedGroups.push(obj) + this.$el.find('[data-js="selectedGroupList"] :last').on('click',this.removeGroup.bind(this)); + this.fields.selectGroups.editor.$el.select2('data',[]); + var addedGroupPerm =_.findWhere(this.model.get('groupPermList'), {'groupId': parseInt(obj.id) }); + if(!_.isUndefined(addedGroupPerm)){ + addedGroupPerm.isAllowed = XAEnums.AccessResult.ACCESS_RESULT_ALLOWED.value; + }else{ + var perm = {}; + perm['moduleId'] = that.model.get('id'); + perm['groupId'] = obj['id']; + perm.isAllowed = XAEnums.AccessResult.ACCESS_RESULT_ALLOWED.value; + newPerms.push(perm); + } + }, this); + if(!_.isEmpty(newPerms)){ + var permissions = this.model.get('groupPermList'); + this.model.set('groupPermList', permissions.concat(newPerms)); + } + + this.emptyCheck(); + if(_.isEmpty(selectedGroups)) alert(localization.tt("msg.pleaseSelectGroup")); + return false; + + }, - var selectedVals = (!_.isNull(objValsStr)) ? objValsStr.toString().split(',') : []; - var selectedIdList=[]; - selectedVals = _.each(selectedVals, function(eachVal){ - //Ignoring any non existing Group Name - if(_.isNumber(parseInt(eachVal)) && !_.isNaN(parseInt(eachVal))){ - selectedIdList.push(Number(eachVal)); + removeGroup : function(e){ + var ele = $(e.currentTarget); + var id = ele.attr('data-id'); + ele.parent().remove(); + this.addedGroups = _.reject(this.addedGroups, function(d){ return d.id == id; }); + var removedGroupPerm =_.findWhere(this.model.get('groupPermList'), {'groupId': parseInt(id) }); + if(!_.isUndefined(removedGroupPerm)){ + if(!_.has(removedGroupPerm, 'id')){ + this.model.set('groupPermList', _.reject(this.model.get('groupPermList'), function(perm){ return perm.groupId == removedGroupPerm.groupId })); + }else{ + removedGroupPerm.isAllowed = XAEnums.AccessResult.ACCESS_RESULT_DENIED.value; } - }); - var modelPermList = options.permList; - var modelPerms = _.unique(_.pluck(options.permList, options.idKey)); - if(!_.isEmpty(selectedIdList)){ - //Look for equals - if(_.isEqual(selectedIdList,modelPerms)) { - //No changes in Selected Users - } else { - //look for new values - - //loop through each new element and check if it has any non matching ids - var diff = _.filter(selectedIdList, function(value){ return !_.contains(modelPerms, value); }); - var that = this; - if(!_.isEmpty(diff)){ - //push new elements to model groupPermList - _.each(diff, function(newEl){ - var newObj = {}; - newObj[options.idKey] = newEl; - newObj['moduleId'] = that.model.get('id'); - newObj['isAllowed'] = 1; - options.permList.push(newObj); - }); - } - //Look for removed users/groups - //loop through each model element and check new selected groups is missing from any original list of group ids - var updDiff = _.filter(modelPerms, function(value){ return !_.contains(selectedIdList, value); }); - if(!_.isEmpty(updDiff)){ - _.each(options.permList, function(origElem){ - if(_.contains(updDiff, origElem[options.idKey])) - origElem.isAllowed = 0; - }); - } + } + this.emptyCheck(); + }, + onAddUser : function(e){ + var that = this, newPerms = []; + var selectedUsers = this.fields.selectUsers.editor.$el.select2('data'); + _.each(selectedUsers, function(obj){ + var self = that; + this.$el.find('[data-js="selectedUserList"]').append(' '+obj.text+'') + this.addedUsers.push(obj) + this.$el.find('[data-js="selectedUserList"] :last').on('click',this.removeUser.bind(this)); + this.fields.selectUsers.editor.$el.select2('data', []); + var addedUserPerm =_.findWhere(this.model.get('userPermList'), {'userId': parseInt(obj.id) }); + if(!_.isUndefined(addedUserPerm)){ + addedUserPerm.isAllowed = XAEnums.AccessResult.ACCESS_RESULT_ALLOWED.value; + }else{ + var perm = {}; + perm['moduleId'] = that.model.get('id'); + perm['userId'] = obj['id']; + perm.isAllowed = XAEnums.AccessResult.ACCESS_RESULT_ALLOWED.value; + newPerms.push(perm); } + }, this); + if(!_.isEmpty(newPerms)){ + var permissions = this.model.get('userPermList'); + this.model.set('userPermList', permissions.concat(newPerms)); + } + this.emptyCheck(); + if(_.isEmpty(selectedUsers)) alert(localization.tt("msg.pleaseSelectUser")); + return false; + }, + removeUser : function(e){ + var ele = $(e.currentTarget); + var id = ele.attr('data-id'); + ele.parent().remove(); + this.addedUsers = _.reject(this.addedUsers, function(d){ return d.id == id; }); + var removedUserPerm =_.findWhere(this.model.get('userPermList'), {'userId': parseInt(id) }); + if(!_.isUndefined(removedUserPerm)){ + if(!_.has(removedUserPerm, 'id')){ + this.model.set('userPermList', _.reject(this.model.get('userPermList'), function(perm){ return perm.userId == removedUserPerm.userId })); + }else{ + removedUserPerm.isAllowed = XAEnums.AccessResult.ACCESS_RESULT_DENIED.value; + } + } + this.emptyCheck(); + }, + emptyCheck : function(type){ + if(this.$el.find('[data-js="selectedGroupList"] span').length > 0){ + this.$el.find('.emptySelectedGroups').hide(); + }else{ + this.$el.find('.emptySelectedGroups').show(); + } - } else { - //Remove permissions from all objects which earlier had permission - _.each(options.permList, function(perm){ - perm.isAllowed = 0; - }); + if(this.$el.find('[data-js="selectedUserList"] span').length > 0){ + this.$el.find('.emptySelectedUsers').hide(); + }else{ + this.$el.find('.emptySelectedUsers').show(); } - } }); diff --git a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermsTableLayout.js b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermsTableLayout.js index 76dc027cff..92c07c75cd 100644 --- a/security-admin/src/main/webapp/scripts/views/permissions/ModulePermsTableLayout.js +++ b/security-admin/src/main/webapp/scripts/views/permissions/ModulePermsTableLayout.js @@ -184,9 +184,21 @@ define(function(require){ attrName = 'policy-users-id'; } var $td = $(e.currentTarget).parents('td'); - $td.find('['+attrName+'="'+id+'"]').show(); - $td.find('[data-id="showLess"]['+attrName+'="'+id+'"]').show(); - $td.find('[data-id="showMore"]['+attrName+'="'+id+'"]').hide(); + var show = false, shownCnt = 1; + $.each($td.find('[data-id="moreSpans"]'), function(i, div){ + if($(div).is(':hidden') && !show){ + $(div).show(); + show = true; + return false; + } + if(!$(div).is(':hidden')){ + shownCnt++; + } + }) + if($td.find('[data-id="moreSpans"]').length == shownCnt){ + $td.find('[data-id="showLess"]['+attrName+'="'+id+'"]').show(); + $td.find('[data-id="showMore"]['+attrName+'="'+id+'"]').hide(); + } $td.find('[data-id="showMore"]['+attrName+'="'+id+'"]').parents('div[data-id="groupsDiv"]').addClass('set-height-groups'); }, onShowLess : function(e){ @@ -197,7 +209,7 @@ define(function(require){ attrName = 'policy-users-id'; } var $td = $(e.currentTarget).parents('td'); - $td.find('['+attrName+'="'+id+'"]').slice(4).hide(); + $td.find('[data-id="moreSpans"]').hide(); $td.find('[data-id="showLess"]['+attrName+'="'+id+'"]').hide(); $td.find('[data-id="showMore"]['+attrName+'="'+id+'"]').show(); $td.find('[data-id="showMore"]['+attrName+'="'+id+'"]').parents('div[data-id="groupsDiv"]').removeClass('set-height-groups'); diff --git a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js index b48e3065c7..6a61f553e2 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js +++ b/security-admin/src/main/webapp/scripts/views/policies/PermissionList.js @@ -115,10 +115,10 @@ define(function(require) { }); if(this.model.has('editMode') && this.model.get('editMode')){ if(!_.isUndefined(this.model.get('groupName')) && !_.isNull(this.model.get('groupName'))){ - this.ui.selectGroups.val(this.model.get('groupName')); + this.ui.selectGroups.val(_.map(this.model.get('groupName'), function(name){ return _.escape(name); })); } if(!_.isUndefined(this.model.get('userName')) && !_.isNull(this.model.get('userName'))){ - this.ui.selectUsers.val(this.model.get('userName')); + this.ui.selectUsers.val(_.map(this.model.get('userName'), function(name){ return _.escape(name); })); } if(!_.isUndefined(this.model.get('conditions'))){ @@ -200,6 +200,7 @@ define(function(require) { }); } var tags = list.map(function(m){ +// return { id : m.id+"" , text : _.escape(m.get('name'))}; return { id : m.id+"" , text : m.get('name')}; }); @@ -213,6 +214,7 @@ define(function(require) { initSelection : function (element, callback) { var data = [], names = (typeGroup) ? that.model.get('groupName') : that.model.get('userName'); _.each(names, function (name) { +// name = _.escape(name); var obj = _.findWhere(tags, {text: name }); data.push({ id : obj.id, text : name }) }); @@ -230,9 +232,9 @@ define(function(require) { selectedVals = that.getSelectedValues($select, typeGroup); if(data.resultSize != "0"){ if(typeGroup){ - results = data.vXGroups.map(function(m, i){ return {id : m.id+"", text: m.name}; }); + results = data.vXGroups.map(function(m, i){ return {id : m.id+"", text: _.escape(m.name) }; }); } else { - results = data.vXUsers.map(function(m, i){ return {id : m.id+"", text: m.name}; }); + results = data.vXUsers.map(function(m, i){ return {id : m.id+"", text: _.escape(m.name) }; }); } if(!_.isEmpty(selectedVals)){ results = XAUtil.filterResultByText(results, selectedVals); @@ -470,10 +472,11 @@ define(function(require) { val = pcond['evaluatorOptions']['engineName'] + ' Condition' } i++; - return ''+name+' : '+ val + ''; + return ''+name+' : '+ _.escape(val) + ''; }); var cond = _.map(value, function(val, name) { - return {'type' : name, 'values' : !_.isArray(val) ? val.split(', ') : val}; + val = _.escape(val); + return {'type' : name, 'values' : !_.isArray(val) ? val.split(',') : val}; }); that.model.set('conditions', cond); @@ -653,7 +656,7 @@ define(function(require) { return; } that.model.set('rowFilterInfo', {'filterExpr': value }); - $(this).html("" + value + ""); + $(this).html("" + _.escape(value) + ""); that.ui.addRowFilterSpan.find('i').attr('class', 'icon-pencil'); that.ui.addRowFilterSpan.attr('title','edit'); }, @@ -776,4 +779,4 @@ define(function(require) { } }); -}); \ No newline at end of file +}); diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js index b3f7ccd37f..317c6ee47a 100644 --- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js +++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyForm.js @@ -380,10 +380,10 @@ define(function(require){ });// 'isRecursive' attribute of model is updated //set sameLevel fieldAttr value with resource name _.each(this.model.attributes, function(val, key) { - if(key.indexOf("sameLevel") >= 0){ - this.model.set(val.resourceType,val); - that.model.unset(key); - } + if(key.indexOf("sameLevel") >= 0 && !_.isNull(val)){ + this.model.set(val.resourceType,val); + that.model.unset(key); + } },this); //To set resource values //Check for masking policies @@ -519,7 +519,7 @@ define(function(require){ setTimeout(function(){ p.abort(); console.log('connection timeout for resource path request...!!'); - }, 7000); + }, 10000); }, open : function(){ $(this).removeClass('working'); @@ -620,7 +620,7 @@ define(function(require){ url: options.lookupURL, type : 'POST', params : { - timeout: 3000, + timeout: 10000, contentType: "application/json; charset=utf-8", }, cache: false, diff --git a/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js b/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js index 7ff1b0e211..0788a4eca9 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js +++ b/security-admin/src/main/webapp/scripts/views/reports/PlugableServiceDiffDetail.js @@ -112,13 +112,15 @@ define(function(require){ cache : false, async : false }) - this.rangerServiceDefModel = new RangerServiceDef(); - this.rangerServiceDefModel.url = XAUtils.getRangerServiceDef(rangerService.get('type')); - this.rangerServiceDefModel.fetch({ - cache : false, - async : false - }) - this.repositoryType = this.rangerServiceDefModel.get('name'); + if(!_.isUndefined(rangerService.get('type'))){ + this.rangerServiceDefModel = new RangerServiceDef(); + this.rangerServiceDefModel.url = XAUtils.getRangerServiceDef(rangerService.get('type')); + this.rangerServiceDefModel.fetch({ + cache : false, + async : false + }) + this.repositoryType = this.rangerServiceDefModel.get('name'); + } } //get policy created/updated date/owner diff --git a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js index 159d18a956..0e2401ec76 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/UserAccessLayout.js @@ -214,9 +214,9 @@ define(function(require) {'use strict'; } else { _.each(model.get('groups'),function(group,index){ if(index < 4) { - group_str += '' + group + endSpanEle + " "; + group_str += '' + _.escape(group) + endSpanEle + " "; } else { - group_str += '