From 0ace4f1e4f89cf5a5d260f43b41b330165072c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 23 Mar 2018 11:30:22 +0100 Subject: [PATCH 01/65] SOLR-12120: New AuditLoggerPlugin type allowing custom Audit logger plugins --- solr/CHANGES.txt | 2 + .../org/apache/solr/core/CoreContainer.java | 40 ++ .../org/apache/solr/security/AuditEvent.java | 324 ++++++++++++ .../solr/security/AuditLoggerPlugin.java | 127 +++++ .../security/MultiDestinationAuditLogger.java | 100 ++++ .../security/SolrLogAuditLoggerPlugin.java | 72 +++ .../org/apache/solr/servlet/HttpSolrCall.java | 14 + .../solr/servlet/SolrDispatchFilter.java | 12 + .../apache/solr/security/AuditEventTest.java | 468 ++++++++++++++++++ .../MultiDestinationAuditLoggerTest.java | 48 ++ .../SolrLogAuditLoggerPluginTest.java | 43 ++ 11 files changed, 1250 insertions(+) create mode 100644 solr/core/src/java/org/apache/solr/security/AuditEvent.java create mode 100644 solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java create mode 100644 solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java create mode 100644 solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java create mode 100644 solr/core/src/test/org/apache/solr/security/AuditEventTest.java create mode 100644 solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java create mode 100644 solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9053b5ec4a11..c7989602da87 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -68,6 +68,8 @@ Bug Fixes * SOLR-12108: Fixed the fallback behavior of [raw] and [xml] transformers when an incompatble 'wt' was specified, the field value was lost if documentCache was not used. (hossman) +* SOLR-12120: New AuditLoggerPlugin type allowing custom Audit logger plugins (janhoy) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index c73507154de2..b1fa7eb7ee83 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -99,6 +99,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.search.SolrFieldCacheBean; +import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.AuthorizationPlugin; import org.apache.solr.security.HttpClientBuilderPlugin; @@ -182,6 +183,8 @@ public CoreLoadFailure(CoreDescriptor cd, Exception loadFailure) { private SecurityPluginHolder authenticationPlugin; + private SecurityPluginHolder auditloggerPlugin; + private BackupRepositoryFactory backupRepoFactory; protected SolrMetricManager metricManager; @@ -328,6 +331,38 @@ private synchronized void initializeAuthorizationPlugin(Map auth } } + private void initializeAuditloggerPlugin(Map auditConf) { + auditConf = Utils.getDeepCopy(auditConf, 4); + //Initialize the Auditlog module + SecurityPluginHolder old = auditloggerPlugin; + SecurityPluginHolder auditloggerPlugin = null; + if (auditConf != null) { + String klas = (String) auditConf.get("class"); + if (klas == null) { + throw new SolrException(ErrorCode.SERVER_ERROR, "class is required for auditlogger plugin"); + } + if (old != null && old.getZnodeVersion() == readVersion(auditConf)) { + return; + } + log.info("Initializing auditlogger plugin: " + klas); + auditloggerPlugin = new SecurityPluginHolder<>(readVersion(auditConf), + getResourceLoader().newInstance(klas, AuditLoggerPlugin.class)); + + // Read and pass the authorization context to the plugin + auditloggerPlugin.plugin.init(auditConf); + } else { + log.debug("Security conf doesn't exist. Skipping setup for audit logging module."); + } + this.auditloggerPlugin = auditloggerPlugin; + if (old != null) { + try { + old.plugin.close(); + } catch (Exception e) { + } + } + } + + private synchronized void initializeAuthenticationPlugin(Map authenticationConfig) { authenticationConfig = Utils.getDeepCopy(authenticationConfig, 4); String pluginClassName = null; @@ -710,6 +745,7 @@ private void reloadSecurityProperties() { SecurityConfHandler.SecurityConfig securityConfig = securityConfHandler.getSecurityConfig(false); initializeAuthorizationPlugin((Map) securityConfig.getData().get("authorization")); initializeAuthenticationPlugin((Map) securityConfig.getData().get("authentication")); + initializeAuditloggerPlugin((Map) securityConfig.getData().get("auditlogging")); } private static void checkForDuplicateCoreNames(List cds) { @@ -1654,6 +1690,10 @@ public AuthenticationPlugin getAuthenticationPlugin() { return authenticationPlugin == null ? null : authenticationPlugin.plugin; } + public AuditLoggerPlugin getAuditLoggerPlugin() { + return auditloggerPlugin == null ? null : auditloggerPlugin.plugin; + } + public NodeConfig getNodeConfig() { return cfg; } diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java new file mode 100644 index 000000000000..63a7db4ee0bc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import javax.servlet.http.HttpServletRequest; +import java.lang.invoke.MethodHandles; +import java.security.Principal; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.security.AuditEvent.EventType.ANONYMOUS; + +/** + * Audit event that takes request and auth context as input to be able to audit log custom things + */ +public class AuditEvent { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private String message; + private Level level; + private Date date; + private String username; + private String session; + private String clientIp; + private List collections; + private Map context; + private HashMap headers; + private Map solrParams; + private String solrHost; + private int solrPort; + private String solrIp; + private String resource; + private String httpMethod; + private String queryString; + private EventType eventType; + private AuthorizationResponse autResponse; + private String requestType; + + /* Predefined event types. Custom types can be made through constructor */ + public enum EventType { + AUTHENTICATED("Authenticated", "User successfully authenticated", Level.INFO), + REJECTED("Rejected", "Authentication request rejected", Level.WARN), + ANONYMOUS("Anonymous", "Request proceeds with unknown user", Level.INFO), + ANONYMOUS_REJECTED("AnonymousRejected", "Request from unknown user rejected", Level.WARN), + AUTHORIZED("Authorized", "Authorization succeeded", Level.INFO), + UNAUTHORIZED("Unauthorized", "Authorization failed", Level.WARN), + ERROR("Error", "Request failed due to an error", Level.ERROR); + + private final String message; + private String explanation; + private final Level level; + + EventType(String message, String explanation, Level level) { + this.message = message; + this.explanation = explanation; + this.level = level; + } + } + + /** + * Empty event, must be filled by user using setters. + * Message and Loglevel will be initialized from EventType but can + * be overridden with setters afterwards. + * @param eventType a predefined or custom EventType + */ + public AuditEvent(EventType eventType) { + this.date = new Date(); + this.eventType = eventType; + this.level = eventType.level; + this.message = eventType.message; + } + + /** + * Event based on an HttpServletRequest, typically used during authentication. + * Solr will fill in details such as ip, http method etc from the request, and + * username if Principal exists on the request. + * @param eventType a predefined or custom EventType + * @param httpRequest the request to initialize from + */ + public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { + this(eventType); + this.solrHost = httpRequest.getLocalName(); + this.solrPort = httpRequest.getLocalPort(); + this.solrIp = httpRequest.getLocalAddr(); + this.clientIp = httpRequest.getRemoteAddr(); + this.resource = httpRequest.getContextPath(); + this.httpMethod = httpRequest.getMethod(); + this.queryString = httpRequest.getQueryString(); + this.headers = getHeadersFromRequest(httpRequest); + + Principal principal = httpRequest.getUserPrincipal(); + if (principal != null) { + this.username = httpRequest.getUserPrincipal().getName(); + } else if (eventType.equals(EventType.AUTHENTICATED)) { + this.eventType = ANONYMOUS; + this.message = ANONYMOUS.message; + this.level = ANONYMOUS.level; + log.debug("Audit event type changed from AUTHENTICATED to ANONYMOUS since no Principal found on request"); + } + } + + /** + * Event based on an AuthorizationContext and reponse. Solr will fill in details + * such as collections, , ip, http method etc from the context. + * @param eventType a predefined or custom EventType + * @param authorizationContext the context to initialize from + */ + public AuditEvent(EventType eventType, HttpServletRequest httpRequest, AuthorizationContext authorizationContext, AuthorizationResponse authResponse) { + this(eventType, httpRequest); + this.collections = authorizationContext.getCollectionRequests() + .stream().map(r -> r.collectionName).collect(Collectors.toList()); + this.resource = authorizationContext.getResource(); + this.requestType = authorizationContext.getRequestType().toString(); + authorizationContext.getParams().getAll(this.solrParams); + this.autResponse = authResponse; + } + + + private HashMap getHeadersFromRequest(HttpServletRequest httpRequest) { + HashMap h = new HashMap<>(); + Enumeration headersEnum = httpRequest.getHeaderNames(); + while (headersEnum != null && headersEnum.hasMoreElements()) { + String name = headersEnum.nextElement(); + h.put(name, httpRequest.getHeader(name)); + } + return h; + } + + public enum Level { + INFO, // Used for normal successful events + WARN, // Used when a user is blocked etc + ERROR // Used when there is an exception or error during auth / authz + } + + public String getMessage() { + return message; + } + + public Level getLevel() { + return level; + } + + public Date getDate() { + return date; + } + + public String getUsername() { + return username; + } + + public String getSession() { + return session; + } + + public String getClientIp() { + return clientIp; + } + + public Map getContext() { + return context; + } + + public List getCollections() { + return collections; + } + + public String getResource() { + return resource; + } + + public String getHttpMethod() { + return httpMethod; + } + + public String getQueryString() { + return queryString; + } + + public EventType getEventType() { + return eventType; + } + + public String getSolrHost() { + return solrHost; + } + + public String getSolrIp() { + return solrIp; + } + + public int getSolrPort() { + return solrPort; + } + + public HashMap getHeaders() { + return headers; + } + + public Map getSolrParams() { + return solrParams; + } + + public AuthorizationResponse getAutResponse() { + return autResponse; + } + + // Setters, builder style + + public AuditEvent setSession(String session) { + this.session = session; + return this; + } + + public AuditEvent setClientIp(String clientIp) { + this.clientIp = clientIp; + return this; + } + + public AuditEvent setContext(Map context) { + this.context = context; + return this; + } + + public AuditEvent setContextEntry(String key, Object value) { + this.context.put(key, value); + return this; + } + + public AuditEvent setMessage(String message) { + this.message = message; + return this; + } + + public AuditEvent setLevel(Level level) { + this.level = level; + return this; + } + + public AuditEvent setDate(Date date) { + this.date = date; + return this; + } + + public AuditEvent setUsername(String username) { + this.username = username; + return this; + } + + public AuditEvent setCollections(List collections) { + this.collections = collections; + return this; + } + + public AuditEvent setResource(String resource) { + this.resource = resource; + return this; + } + + public AuditEvent setHttpMethod(String httpMethod) { + this.httpMethod = httpMethod; + return this; + } + + public AuditEvent setQueryString(String queryString) { + this.queryString = queryString; + return this; + } + + public AuditEvent setSolrHost(String solrHost) { + this.solrHost = solrHost; + return this; + } + + public AuditEvent setSolrPort(int solrPort) { + this.solrPort = solrPort; + return this; + } + + public AuditEvent setSolrIp(String solrIp) { + this.solrIp = solrIp; + return this; + } + + public AuditEvent setHeaders(HashMap headers) { + this.headers = headers; + return this; + } + + public AuditEvent setSolrParams(Map solrParams) { + this.solrParams = solrParams; + return this; + } + + public AuditEvent setAutResponse(AuthorizationResponse autResponse) { + this.autResponse = autResponse; + return this; + } + + public AuditEvent setRequestType(String requestType) { + this.requestType = requestType; + return this; + } +} diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java new file mode 100644 index 000000000000..eb6de4803801 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.Closeable; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.solr.common.SolrException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for Audit logger plugins + */ +public abstract class AuditLoggerPlugin implements Closeable, Runnable { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String PARAM_BLOCKASYNC = "blockAsync"; + private static final String PARAM_QUEUE_SIZE = "queueSize"; + protected AuditEventFormatter formatter; + private BlockingQueue queue; + private boolean blockAsync; + private int blockingQueueSize; + + /** + * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. + * @param event + */ + public final void auditAsync(AuditEvent event) { + if (blockAsync) { + try { + queue.put(event); + } catch (InterruptedException e) { + log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); + } + } else { + if (!queue.offer(event)) { + log.warn("Audit log async queue is full, not blocking since " + PARAM_BLOCKASYNC + "==false"); + } + } + } + + /** + * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. + * If an event was submitted asynchronously with {@link #auditAsync(AuditEvent)} then this + * method will be called by the framework by the background thread. + * @param event + */ + public abstract void audit(AuditEvent event); + + /** + * Initialize the plugin from security.json. + * @param pluginConfig the config for the plugin + */ + public void init(Map pluginConfig) { + blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); + blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, 1024))); + queue = new ArrayBlockingQueue<>(blockingQueueSize); + formatter = new JSONAuditEventFormatter(); + } + + public void setFormatter(AuditEventFormatter formatter) { + this.formatter = formatter; + } + + /** + * Interface for formatting the event + */ + public interface AuditEventFormatter { + String formatEvent(AuditEvent event); + } + + /** + * Event formatter that returns event as JSON string + */ + public static class JSONAuditEventFormatter implements AuditEventFormatter { + /** + * Formats an audit event as a JSON string + */ + @Override + public String formatEvent(AuditEvent event) { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); + try { + StringWriter sw = new StringWriter(); + mapper.writeValue(sw, event); + return sw.toString(); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error converting Event to JSON", e); + } + } + } + + /** + * Pick next event from async queue and call {@link #audit(AuditEvent)} + */ + @Override + public void run() { + try { + audit(queue.take()); + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for next audit log event"); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java new file mode 100644 index 000000000000..225b716cf9b7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.analysis.util.ResourceLoader; +import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.solr.common.SolrException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; + +/** + * Audit logger that chains other loggers. Lets you configure logging to multiple destinations. + * The config is simply a list of configs for the sub plugins: + * 
+ *   "class" : "solr.MultiDestinationAuditLogger",
+ *   "plugins" : [
+ *     { "class" : "solr.SolrLogAuditLoggerPlugin" },
+ *     { "class" : "solr.MyOtherAuditPlugin"}
+ *   ]
+ *
+ */ +public class MultiDestinationAuditLogger extends AuditLoggerPlugin implements ResourceLoaderAware { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String PARAM_PLUGINS = "plugins"; + private List plugins = new ArrayList<>();; + private ResourceLoader loader; + + /** + * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. + * @param event + */ + @Override + public void audit(AuditEvent event) { + plugins.forEach(plugin -> { + log.debug("Passing auditEvent to plugin {}", plugin.getClass().getName()); + plugin.audit(event); + }); + } + + /** + * Initialize the plugin from security.json + * @param pluginConfig the config for the plugin + */ + @Override + public void init(Map pluginConfig) { + if (!pluginConfig.containsKey(PARAM_PLUGINS)) { + log.warn("No plugins configured"); + } + List> pluginList = (List>) pluginConfig.get(PARAM_PLUGINS); + pluginList.forEach(pluginConf -> { + plugins.add(createPlugin(pluginConf)); + }); + log.info("Initialized {} audit plugins", pluginList.size()); + } + + private AuditLoggerPlugin createPlugin(Map auditConf) { + if (auditConf != null) { + String klas = (String) auditConf.get("class"); + if (klas == null) { + throw new SolrException(SERVER_ERROR, "class is required for auditlogger plugin"); + } + log.info("Initializing auditlogger plugin: " + klas); + AuditLoggerPlugin p = loader.newInstance(klas, AuditLoggerPlugin.class); + p.init(auditConf); + return p; + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Empty config when creating audit plugin"); + } + } + + @Override + public void close() throws IOException {} + + @Override + public void inform(ResourceLoader loader) throws IOException { + this.loader = loader; + } +} diff --git a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java new file mode 100644 index 000000000000..b51cdd97e220 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Audit logger that writes to the Solr log + * @lucene.experimental + */ +public class SolrLogAuditLoggerPlugin extends AuditLoggerPlugin { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Initialize the plugin from security.json + * @param pluginConfig the config for the plugin + */ + @Override + public void init(Map pluginConfig) { + super.init(pluginConfig); + setFormatter(event -> + "type='" + event.getEventType().name() + '\'' + + ", message='" + event.getMessage() + '\'' + + ", method='" + event.getHttpMethod() + '\'' + + ", username='" + event.getUsername() + '\'' + + ", resource='" + event.getResource() + '\'' + + ", collections=" + event.getCollections()); + } + + /** + * Audit logs an event. The event should be a {@link AuditEvent} to be able to pull context info + * @param event the event to log + */ + @Override + public void audit(AuditEvent event) { + switch (event.getLevel()) { + case INFO: + log.info(formatter.formatEvent(event)); + break; + + case WARN: + log.warn(formatter.formatEvent(event)); + break; + + case ERROR: + log.error(formatter.formatEvent(event)); + break; + } + } + + @Override + public void close() throws IOException {} +} diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 50fa71c0452d..86405626d707 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -100,6 +100,8 @@ import org.apache.solr.security.AuthorizationContext.RequestType; import org.apache.solr.security.AuthorizationResponse; import org.apache.solr.security.PKIAuthenticationPlugin; +import org.apache.solr.security.AuditEvent; +import org.apache.solr.security.AuditEvent.EventType; import org.apache.solr.servlet.SolrDispatchFilter.Action; import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; import org.apache.solr.servlet.cache.Method; @@ -464,6 +466,9 @@ public Action call() throws IOException { if (solrDispatchFilter.abortErrorMessage != null) { sendError(500, solrDispatchFilter.abortErrorMessage); + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ERROR, getReq())); + } return RETURN; } @@ -483,13 +488,22 @@ public Action call() throws IOException { for (Map.Entry e : headers.entrySet()) response.setHeader(e.getKey(), e.getValue()); } log.debug("USER_REQUIRED "+req.getHeader("Authorization")+" "+ req.getUserPrincipal()); + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.REJECTED, req, context, authResponse)); + } } if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED) && !(authResponse.statusCode == HttpStatus.SC_OK)) { log.info("USER_REQUIRED auth header {} context : {} ", req.getHeader("Authorization"), context); sendError(authResponse.statusCode, "Unauthorized request, Response code: " + authResponse.statusCode); + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.UNAUTHORIZED, req, context, authResponse)); + } return RETURN; } + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.AUTHORIZED, req, context, authResponse)); + } } HttpServletResponse resp = response; diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index edf616e54d77..660044f96508 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -72,6 +72,7 @@ import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; +import org.apache.solr.security.AuditEvent; import org.apache.solr.util.SolrFileCleaningTracker; import org.apache.solr.util.StartupLoggingUtils; import org.apache.solr.util.configuration.SSLConfigurationsFactory; @@ -83,6 +84,8 @@ import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; +import static org.apache.solr.security.AuditEvent.EventType; + /** * This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml * @@ -442,6 +445,9 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo final AtomicBoolean isAuthenticated = new AtomicBoolean(false); AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin(); if (authenticationPlugin == null) { + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.ANONYMOUS, request)); + } return true; } else { // /admin/info/key must be always open. see SOLR-9188 @@ -472,8 +478,14 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo // multiple code paths. if (!requestContinues || !isAuthenticated.get()) { response.flushBuffer(); + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.REJECTED, request)); + } return false; } + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.AUTHENTICATED, request)); + } return true; } diff --git a/solr/core/src/test/org/apache/solr/security/AuditEventTest.java b/solr/core/src/test/org/apache/solr/security/AuditEventTest.java new file mode 100644 index 000000000000..fe8d29b8054b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/AuditEventTest.java @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.security; + +import javax.servlet.AsyncContext; +import javax.servlet.DispatcherType; +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletInputStream; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import javax.servlet.http.HttpUpgradeHandler; +import javax.servlet.http.Part; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.Principal; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.apache.commons.collections.iterators.EnumerationIterator; +import org.apache.http.auth.BasicUserPrincipal; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class AuditEventTest { + @Test + public void fromAuthorizationContext() throws Exception { + AuditEvent event = new AuditEvent(AuditEvent.EventType.AUTHENTICATED, new HttpServletRequest() { + @Override + public String getAuthType() { + return null; + } + + @Override + public Cookie[] getCookies() { + return new Cookie[0]; + } + + @Override + public long getDateHeader(String name) { + return 0; + } + + @Override + public String getHeader(String name) { + return name.equals("foo") ? "fooval" : "barval"; + } + + @Override + public Enumeration getHeaders(String name) { + return null; + } + + @Override + public Enumeration getHeaderNames() { + return Collections.enumeration(Arrays.asList("foo", "bar")); + } + + @Override + public int getIntHeader(String name) { + return 0; + } + + @Override + public String getMethod() { + return "GET"; + } + + @Override + public String getPathInfo() { + return null; + } + + @Override + public String getPathTranslated() { + return null; + } + + @Override + public String getContextPath() { + return null; + } + + @Override + public String getQueryString() { + return "?foo&bar"; + } + + @Override + public String getRemoteUser() { + return null; + } + + @Override + public boolean isUserInRole(String role) { + return false; + } + + @Override + public Principal getUserPrincipal() { + return new BasicUserPrincipal("George"); + } + + @Override + public String getRequestedSessionId() { + return null; + } + + @Override + public String getRequestURI() { + return null; + } + + @Override + public StringBuffer getRequestURL() { + return null; + } + + @Override + public String getServletPath() { + return null; + } + + @Override + public HttpSession getSession(boolean create) { + return null; + } + + @Override + public HttpSession getSession() { + return null; + } + + @Override + public String changeSessionId() { + return null; + } + + @Override + public boolean isRequestedSessionIdValid() { + return false; + } + + @Override + public boolean isRequestedSessionIdFromCookie() { + return false; + } + + @Override + public boolean isRequestedSessionIdFromURL() { + return false; + } + + @Override + public boolean isRequestedSessionIdFromUrl() { + return false; + } + + @Override + public boolean authenticate(HttpServletResponse response) throws IOException, ServletException { + return false; + } + + @Override + public void login(String username, String password) throws ServletException { + + } + + @Override + public void logout() throws ServletException { + + } + + @Override + public Collection getParts() throws IOException, ServletException { + return null; + } + + @Override + public Part getPart(String name) throws IOException, ServletException { + return null; + } + + @Override + public T upgrade(Class handlerClass) throws IOException, ServletException { + return null; + } + + @Override + public Object getAttribute(String name) { + return null; + } + + @Override + public Enumeration getAttributeNames() { + return null; + } + + @Override + public String getCharacterEncoding() { + return null; + } + + @Override + public void setCharacterEncoding(String env) throws UnsupportedEncodingException { + + } + + @Override + public int getContentLength() { + return 0; + } + + @Override + public long getContentLengthLong() { + return 0; + } + + @Override + public String getContentType() { + return null; + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return null; + } + + @Override + public String getParameter(String name) { + return null; + } + + @Override + public Enumeration getParameterNames() { + return null; + } + + @Override + public String[] getParameterValues(String name) { + return new String[0]; + } + + @Override + public Map getParameterMap() { + return null; + } + + @Override + public String getProtocol() { + return null; + } + + @Override + public String getScheme() { + return null; + } + + @Override + public String getServerName() { + return null; + } + + @Override + public int getServerPort() { + return 0; + } + + @Override + public BufferedReader getReader() throws IOException { + return null; + } + + @Override + public String getRemoteAddr() { + return null; + } + + @Override + public String getRemoteHost() { + return null; + } + + @Override + public void setAttribute(String name, Object o) { + + } + + @Override + public void removeAttribute(String name) { + + } + + @Override + public Locale getLocale() { + return null; + } + + @Override + public Enumeration getLocales() { + return null; + } + + @Override + public boolean isSecure() { + return false; + } + + @Override + public RequestDispatcher getRequestDispatcher(String path) { + return null; + } + + @Override + public String getRealPath(String path) { + return null; + } + + @Override + public int getRemotePort() { + return 0; + } + + @Override + public String getLocalName() { + return null; + } + + @Override + public String getLocalAddr() { + return null; + } + + @Override + public int getLocalPort() { + return 0; + } + + @Override + public ServletContext getServletContext() { + return null; + } + + @Override + public AsyncContext startAsync() throws IllegalStateException { + return null; + } + + @Override + public AsyncContext startAsync(ServletRequest servletRequest, ServletResponse servletResponse) throws IllegalStateException { + return null; + } + + @Override + public boolean isAsyncStarted() { + return false; + } + + @Override + public boolean isAsyncSupported() { + return false; + } + + @Override + public AsyncContext getAsyncContext() { + return null; + } + + @Override + public DispatcherType getDispatcherType() { + return null; + } + }, new AuthorizationContext() { + @Override + public SolrParams getParams() { + return new MapSolrParams(Collections.singletonMap("q", "hello")); + } + + @Override + public Principal getUserPrincipal() { + return new BasicUserPrincipal("George"); + } + + @Override + public String getHttpHeader(String header) { + return "MyHeader"; + } + + @Override + public Enumeration getHeaderNames() { + return Collections.enumeration(Arrays.asList("Header1", "Header2")); + } + + @Override + public String getRemoteAddr() { + return "127.0.0.1"; + } + + @Override + public String getRemoteHost() { + return "localhost"; + } + + @Override + public List getCollectionRequests() { + return Arrays.asList(new CollectionRequest("coll1")); + } + + @Override + public RequestType getRequestType() { + return RequestType.READ; + } + + @Override + public String getResource() { + return "/solr/admin/info"; + } + + @Override + public String getHttpMethod() { + return "GET"; + } + + @Override + public Object getHandler() { + return "/select"; + } + }, new AuthorizationResponse(200)); + assertEquals("{\"message\":\"Authenticated\",\"level\":\"INFO\",\"date\":" + event.getDate().getTime() + ",\"username\":\"George\",\"session\":null,\"clientIp\":null,\"collections\":[\"coll1\"],\"context\":null,\"headers\":{\"bar\":\"barval\",\"foo\":\"fooval\"},\"solrParams\":null,\"solrHost\":null,\"solrPort\":0,\"solrIp\":null,\"resource\":\"/solr/admin/info\",\"httpMethod\":\"GET\",\"queryString\":\"?foo&bar\",\"eventType\":\"AUTHENTICATED\",\"autResponse\":{\"statusCode\":200,\"message\":null}}", + new AuditLoggerPlugin.JSONAuditEventFormatter().formatEvent(event)); + } + + @Test + public void manual() throws Exception { + AuditEvent event = new AuditEvent(AuditEvent.EventType.REJECTED).setHttpMethod("GET"); + assertTrue(event.getDate() != null); + assertEquals("GET", event.getHttpMethod()); + } +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java new file mode 100644 index 000000000000..8d0cc6abda0e --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.SolrResourceLoader; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class MultiDestinationAuditLoggerTest { + @Test + public void init() throws Exception { + MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); + Map config = new HashMap<>(); + config.put("class", "solr.MultiDestinationAuditLogger"); + ArrayList> plugins = new ArrayList>(); + + Map myPlugin = new HashMap<>(); + myPlugin.put("class", "solr.SolrLogAuditLoggerPlugin"); + plugins.add(myPlugin); + config.put("plugins", plugins); + + al.inform(new SolrResourceLoader()); + al.init(config); + + al.audit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); + } + +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java new file mode 100644 index 000000000000..d8dfe6407975 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.security; + +import java.util.Arrays; +import java.util.HashMap; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SolrLogAuditLoggerPluginTest { + private SolrLogAuditLoggerPlugin plugin; + + @Before + public void setUp() throws Exception { + plugin = new SolrLogAuditLoggerPlugin(); + } + + @Test + public void init() throws Exception { + HashMap config = new HashMap<>(); + plugin.init(config); + plugin.audit(new AuditEvent(AuditEvent.EventType.AUTHENTICATED)); + } + +} \ No newline at end of file From bdc1124697097ce198e2ff04019de641dcc8571e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 23 Mar 2018 12:17:10 +0100 Subject: [PATCH 02/65] Fixed async audit --- .../org/apache/solr/security/AuditLoggerPlugin.java | 5 +++++ .../solr/security/SolrLogAuditLoggerPluginTest.java | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index eb6de4803801..3e3e5c3c1764 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -23,10 +23,13 @@ import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,6 +81,8 @@ public void init(Map pluginConfig) { blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, 1024))); queue = new ArrayBlockingQueue<>(blockingQueueSize); formatter = new JSONAuditEventFormatter(); + ExecutorService executorService = Executors.newSingleThreadExecutor(new SolrjNamedThreadFactory("audit")); + executorService.submit(this); } public void setFormatter(AuditEventFormatter formatter) { diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index d8dfe6407975..2fd1cb4c3fec 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -37,7 +37,16 @@ public void setUp() throws Exception { public void init() throws Exception { HashMap config = new HashMap<>(); plugin.init(config); - plugin.audit(new AuditEvent(AuditEvent.EventType.AUTHENTICATED)); + plugin.audit(new AuditEvent(AuditEvent.EventType.REJECTED) + .setUsername("Jan") + .setHttpMethod("POST") + .setMessage("Wrong password") + .setResource("/collection1")); + plugin.auditAsync(new AuditEvent(AuditEvent.EventType.AUTHORIZED) + .setUsername("Per") + .setHttpMethod("GET") + .setMessage("Async") + .setResource("/collection1")); } } \ No newline at end of file From 07b4b3947ee67a1742686c514e0784a8ff4db51a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 23 Mar 2018 12:42:43 +0100 Subject: [PATCH 03/65] Fix init bug, log error if config params not consumed --- .../apache/solr/security/AuditLoggerPlugin.java | 5 ++++- .../security/MultiDestinationAuditLogger.java | 17 ++++++++++++----- .../MultiDestinationAuditLoggerTest.java | 13 ++++++++++++- .../security/SolrLogAuditLoggerPluginTest.java | 3 +++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 3e3e5c3c1764..aaa3982f5941 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -74,17 +74,20 @@ public final void auditAsync(AuditEvent event) { /** * Initialize the plugin from security.json. + * This method removes parameters from config object after consuming, so subclasses can check for config errors. * @param pluginConfig the config for the plugin */ public void init(Map pluginConfig) { blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, 1024))); + pluginConfig.remove(PARAM_BLOCKASYNC); + pluginConfig.remove(PARAM_QUEUE_SIZE); queue = new ArrayBlockingQueue<>(blockingQueueSize); formatter = new JSONAuditEventFormatter(); ExecutorService executorService = Executors.newSingleThreadExecutor(new SolrjNamedThreadFactory("audit")); executorService.submit(this); } - + public void setFormatter(AuditEventFormatter formatter) { this.formatter = formatter; } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 225b716cf9b7..698aeced40a9 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -65,14 +65,21 @@ public void audit(AuditEvent event) { */ @Override public void init(Map pluginConfig) { + super.init(pluginConfig); if (!pluginConfig.containsKey(PARAM_PLUGINS)) { log.warn("No plugins configured"); + } else { + List> pluginList = (List>) pluginConfig.get(PARAM_PLUGINS); + pluginList.forEach(pluginConf -> { + plugins.add(createPlugin(pluginConf)); + }); + pluginConfig.remove(PARAM_PLUGINS); } - List> pluginList = (List>) pluginConfig.get(PARAM_PLUGINS); - pluginList.forEach(pluginConf -> { - plugins.add(createPlugin(pluginConf)); - }); - log.info("Initialized {} audit plugins", pluginList.size()); + pluginConfig.remove("class"); + if (pluginConfig.size() > 0) { + log.error("Plugin config was not fully consumed. Remaining parameters are {}", pluginConfig); + } + log.info("Initialized {} audit plugins", plugins.size()); } private AuditLoggerPlugin createPlugin(Map auditConf) { diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 8d0cc6abda0e..1828b414bb1f 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -42,7 +42,18 @@ public void init() throws Exception { al.inform(new SolrResourceLoader()); al.init(config); - al.audit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); + al.auditAsync(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); + + assertEquals(0, config.size()); } + @Test + public void wrongConfigParam() throws Exception { + MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); + Map config = new HashMap<>(); + config.put("class", "solr.MultiDestinationAuditLogger"); + config.put("foo", "Should complain"); + al.init(config); + assertEquals(1, config.size()); + } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 2fd1cb4c3fec..9aed5b74daaa 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -36,6 +36,8 @@ public void setUp() throws Exception { @Test public void init() throws Exception { HashMap config = new HashMap<>(); + config.put("blockAsync", true); + config.put("queueSize", 1); plugin.init(config); plugin.audit(new AuditEvent(AuditEvent.EventType.REJECTED) .setUsername("Jan") @@ -47,6 +49,7 @@ public void init() throws Exception { .setHttpMethod("GET") .setMessage("Async") .setResource("/collection1")); + assertEquals(0, config.size()); } } \ No newline at end of file From 51bb99907b7870edf292edd231e7b947bb3fa941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 24 Mar 2018 01:07:44 +0100 Subject: [PATCH 04/65] Changes based on review comments * New abstract AsyncAuditLoggerPlugin for async logging * Changed formatter to use StingBuilder --- .../org/apache/solr/core/CoreContainer.java | 6 +- .../solr/security/AsyncAuditLoggerPlugin.java | 97 ++++ .../org/apache/solr/security/AuditEvent.java | 1 - .../solr/security/AuditLoggerPlugin.java | 53 +- .../security/MultiDestinationAuditLogger.java | 13 +- .../security/SolrLogAuditLoggerPlugin.java | 15 +- .../org/apache/solr/servlet/HttpSolrCall.java | 6 +- .../solr/servlet/SolrDispatchFilter.java | 6 +- .../solr/analysis/TokenizerChainTest.java | 1 + .../apache/solr/security/AuditEventTest.java | 468 ------------------ .../solr/security/MockAuditLoggerPlugin.java | 59 +++ .../MultiDestinationAuditLoggerTest.java | 20 +- .../SolrLogAuditLoggerPluginTest.java | 13 +- 13 files changed, 199 insertions(+), 559 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java delete mode 100644 solr/core/src/test/org/apache/solr/security/AuditEventTest.java create mode 100644 solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index b1fa7eb7ee83..aa03262fa09b 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -327,6 +327,7 @@ private synchronized void initializeAuthorizationPlugin(Map auth try { old.plugin.close(); } catch (Exception e) { + log.error("Exception while attempting to close old authorization plugin", e); } } } @@ -358,6 +359,7 @@ private void initializeAuditloggerPlugin(Map auditConf) { try { old.plugin.close(); } catch (Exception e) { + log.error("Exception while attempting to close old auditlogger plugin", e); } } } @@ -403,7 +405,9 @@ private synchronized void initializeAuthenticationPlugin(Map aut this.authenticationPlugin = authenticationPlugin; try { if (old != null) old.plugin.close(); - } catch (Exception e) {/*do nothing*/ } + } catch (Exception e) { + log.error("Exception while attempting to close old authentication plugin", e); + } } diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java new file mode 100644 index 000000000000..37c1181f02bc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.lang.invoke.MethodHandles; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; + +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrjNamedThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for asynchronous audit logging. Extend this class for queued logging events + */ +public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implements Runnable { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String PARAM_BLOCKASYNC = "blockAsync"; + private static final String PARAM_QUEUE_SIZE = "queueSize"; + private BlockingQueue queue; + private boolean blockAsync; + + /** + * Enqueues an {@link AuditEvent} to a queue and returns immediately. + * A background thread will pull events from this queue and call {@link #auditCallback(AuditEvent)} + * @param event the audit event + */ + public final void audit(AuditEvent event) { + if (blockAsync) { + try { + queue.put(event); + } catch (InterruptedException e) { + log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); + } + } else { + if (!queue.offer(event)) { + log.warn("Audit log async queue is full, not blocking since " + PARAM_BLOCKASYNC + "==false"); + } + } + } + + /** + * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. + * This method will be called by the audit background thread as it pulls events from the + * queue. This is where the actual logging work shall be done. + * @param event the audit event + */ + public abstract void auditCallback(AuditEvent event); + + /** + * Initialize the plugin from security.json. + * This method removes parameters from config object after consuming, so subclasses can check for config errors. + * @param pluginConfig the config for the plugin + */ + public void init(Map pluginConfig) { + blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); + int blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, 4000))); + pluginConfig.remove(PARAM_BLOCKASYNC); + pluginConfig.remove(PARAM_QUEUE_SIZE); + queue = new ArrayBlockingQueue<>(blockingQueueSize); + ExecutorService executorService = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrjNamedThreadFactory("audit")); + executorService.submit(this); + } + + /** + * Pick next event from async queue and call {@link #auditCallback(AuditEvent)} + */ + @Override + public void run() { + while (true) { + try { + auditCallback(queue.take()); + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for next audit log event"); + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 63a7db4ee0bc..d4a27760046a 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -136,7 +136,6 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza this.autResponse = authResponse; } - private HashMap getHeadersFromRequest(HttpServletRequest httpRequest) { HashMap h = new HashMap<>(); Enumeration headersEnum = httpRequest.getHeaderNames(); diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index aaa3982f5941..4737fcf5be88 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -21,54 +21,24 @@ import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Base class for Audit logger plugins */ -public abstract class AuditLoggerPlugin implements Closeable, Runnable { +public abstract class AuditLoggerPlugin implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String PARAM_BLOCKASYNC = "blockAsync"; - private static final String PARAM_QUEUE_SIZE = "queueSize"; protected AuditEventFormatter formatter; - private BlockingQueue queue; - private boolean blockAsync; - private int blockingQueueSize; /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. - * @param event - */ - public final void auditAsync(AuditEvent event) { - if (blockAsync) { - try { - queue.put(event); - } catch (InterruptedException e) { - log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); - } - } else { - if (!queue.offer(event)) { - log.warn("Audit log async queue is full, not blocking since " + PARAM_BLOCKASYNC + "==false"); - } - } - } - - /** - * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. - * If an event was submitted asynchronously with {@link #auditAsync(AuditEvent)} then this - * method will be called by the framework by the background thread. - * @param event + * @param event the audit event */ public abstract void audit(AuditEvent event); @@ -78,14 +48,7 @@ public final void auditAsync(AuditEvent event) { * @param pluginConfig the config for the plugin */ public void init(Map pluginConfig) { - blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); - blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, 1024))); - pluginConfig.remove(PARAM_BLOCKASYNC); - pluginConfig.remove(PARAM_QUEUE_SIZE); - queue = new ArrayBlockingQueue<>(blockingQueueSize); formatter = new JSONAuditEventFormatter(); - ExecutorService executorService = Executors.newSingleThreadExecutor(new SolrjNamedThreadFactory("audit")); - executorService.submit(this); } public void setFormatter(AuditEventFormatter formatter) { @@ -120,16 +83,4 @@ public String formatEvent(AuditEvent event) { } } } - - /** - * Pick next event from async queue and call {@link #audit(AuditEvent)} - */ - @Override - public void run() { - try { - audit(queue.take()); - } catch (InterruptedException e) { - log.warn("Interrupted while waiting for next audit log event"); - } - } } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 698aeced40a9..9e353e6fe653 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -44,12 +44,12 @@ public class MultiDestinationAuditLogger extends AuditLoggerPlugin implements ResourceLoaderAware { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_PLUGINS = "plugins"; - private List plugins = new ArrayList<>();; private ResourceLoader loader; + List plugins = new ArrayList<>(); /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. - * @param event + * @param event the audit event */ @Override public void audit(AuditEvent event) { @@ -69,10 +69,9 @@ public void init(Map pluginConfig) { if (!pluginConfig.containsKey(PARAM_PLUGINS)) { log.warn("No plugins configured"); } else { + @SuppressWarnings("unchecked") List> pluginList = (List>) pluginConfig.get(PARAM_PLUGINS); - pluginList.forEach(pluginConf -> { - plugins.add(createPlugin(pluginConf)); - }); + pluginList.forEach(pluginConf -> plugins.add(createPlugin(pluginConf))); pluginConfig.remove(PARAM_PLUGINS); } pluginConfig.remove("class"); @@ -98,10 +97,10 @@ private AuditLoggerPlugin createPlugin(Map auditConf) { } @Override - public void close() throws IOException {} + public void close() {} @Override - public void inform(ResourceLoader loader) throws IOException { + public void inform(ResourceLoader loader) { this.loader = loader; } } diff --git a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java index b51cdd97e220..a6cad9659ff2 100644 --- a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java @@ -38,16 +38,17 @@ public class SolrLogAuditLoggerPlugin extends AuditLoggerPlugin { public void init(Map pluginConfig) { super.init(pluginConfig); setFormatter(event -> - "type='" + event.getEventType().name() + '\'' + - ", message='" + event.getMessage() + '\'' + - ", method='" + event.getHttpMethod() + '\'' + - ", username='" + event.getUsername() + '\'' + - ", resource='" + event.getResource() + '\'' + - ", collections=" + event.getCollections()); + new StringBuilder() + .append("type=\"").append(event.getEventType().name()).append("\"") + .append(" message=\"").append(event.getMessage()).append("\"") + .append(" method=\"").append(event.getHttpMethod()).append("\"") + .append(" username=\"").append(event.getUsername()).append("\"") + .append(" resource=\"").append(event.getResource()).append("\"") + .append(" collections=").append(event.getCollections()).toString()); } /** - * Audit logs an event. The event should be a {@link AuditEvent} to be able to pull context info + * Audit logs an event to Solr log. The event should be a {@link AuditEvent} to be able to pull context info * @param event the event to log */ @Override diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 86405626d707..746b53794433 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -489,7 +489,7 @@ public Action call() throws IOException { } log.debug("USER_REQUIRED "+req.getHeader("Authorization")+" "+ req.getUserPrincipal()); if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.REJECTED, req, context, authResponse)); + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, req, context, authResponse)); } } if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED) && !(authResponse.statusCode == HttpStatus.SC_OK)) { @@ -497,12 +497,12 @@ public Action call() throws IOException { sendError(authResponse.statusCode, "Unauthorized request, Response code: " + authResponse.statusCode); if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.UNAUTHORIZED, req, context, authResponse)); + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.UNAUTHORIZED, req, context, authResponse)); } return RETURN; } if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.AUTHORIZED, req, context, authResponse)); + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHORIZED, req, context, authResponse)); } } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 660044f96508..1ff5d81b2cac 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -446,7 +446,7 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin(); if (authenticationPlugin == null) { if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.ANONYMOUS, request)); + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ANONYMOUS, request)); } return true; } else { @@ -479,12 +479,12 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo if (!requestContinues || !isAuthenticated.get()) { response.flushBuffer(); if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.REJECTED, request)); + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, request)); } return false; } if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().auditAsync(new AuditEvent(EventType.AUTHENTICATED, request)); + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHENTICATED, request)); } return true; } diff --git a/solr/core/src/test/org/apache/solr/analysis/TokenizerChainTest.java b/solr/core/src/test/org/apache/solr/analysis/TokenizerChainTest.java index 2e4c67ab6588..d40911975b2d 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TokenizerChainTest.java +++ b/solr/core/src/test/org/apache/solr/analysis/TokenizerChainTest.java @@ -39,5 +39,6 @@ public void testNormalization() throws Exception { tff); assertEquals(new BytesRef("fooba"), tokenizerChain.normalize(fieldName, "FOOB\u00c4")); + tokenizerChain.close(); } } diff --git a/solr/core/src/test/org/apache/solr/security/AuditEventTest.java b/solr/core/src/test/org/apache/solr/security/AuditEventTest.java deleted file mode 100644 index fe8d29b8054b..000000000000 --- a/solr/core/src/test/org/apache/solr/security/AuditEventTest.java +++ /dev/null @@ -1,468 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.security; - -import javax.servlet.AsyncContext; -import javax.servlet.DispatcherType; -import javax.servlet.RequestDispatcher; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.ServletInputStream; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpSession; -import javax.servlet.http.HttpUpgradeHandler; -import javax.servlet.http.Part; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.security.Principal; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Enumeration; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -import org.apache.commons.collections.iterators.EnumerationIterator; -import org.apache.http.auth.BasicUserPrincipal; -import org.apache.solr.common.params.MapSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.junit.Test; - -import static org.junit.Assert.*; - -public class AuditEventTest { - @Test - public void fromAuthorizationContext() throws Exception { - AuditEvent event = new AuditEvent(AuditEvent.EventType.AUTHENTICATED, new HttpServletRequest() { - @Override - public String getAuthType() { - return null; - } - - @Override - public Cookie[] getCookies() { - return new Cookie[0]; - } - - @Override - public long getDateHeader(String name) { - return 0; - } - - @Override - public String getHeader(String name) { - return name.equals("foo") ? "fooval" : "barval"; - } - - @Override - public Enumeration getHeaders(String name) { - return null; - } - - @Override - public Enumeration getHeaderNames() { - return Collections.enumeration(Arrays.asList("foo", "bar")); - } - - @Override - public int getIntHeader(String name) { - return 0; - } - - @Override - public String getMethod() { - return "GET"; - } - - @Override - public String getPathInfo() { - return null; - } - - @Override - public String getPathTranslated() { - return null; - } - - @Override - public String getContextPath() { - return null; - } - - @Override - public String getQueryString() { - return "?foo&bar"; - } - - @Override - public String getRemoteUser() { - return null; - } - - @Override - public boolean isUserInRole(String role) { - return false; - } - - @Override - public Principal getUserPrincipal() { - return new BasicUserPrincipal("George"); - } - - @Override - public String getRequestedSessionId() { - return null; - } - - @Override - public String getRequestURI() { - return null; - } - - @Override - public StringBuffer getRequestURL() { - return null; - } - - @Override - public String getServletPath() { - return null; - } - - @Override - public HttpSession getSession(boolean create) { - return null; - } - - @Override - public HttpSession getSession() { - return null; - } - - @Override - public String changeSessionId() { - return null; - } - - @Override - public boolean isRequestedSessionIdValid() { - return false; - } - - @Override - public boolean isRequestedSessionIdFromCookie() { - return false; - } - - @Override - public boolean isRequestedSessionIdFromURL() { - return false; - } - - @Override - public boolean isRequestedSessionIdFromUrl() { - return false; - } - - @Override - public boolean authenticate(HttpServletResponse response) throws IOException, ServletException { - return false; - } - - @Override - public void login(String username, String password) throws ServletException { - - } - - @Override - public void logout() throws ServletException { - - } - - @Override - public Collection getParts() throws IOException, ServletException { - return null; - } - - @Override - public Part getPart(String name) throws IOException, ServletException { - return null; - } - - @Override - public T upgrade(Class handlerClass) throws IOException, ServletException { - return null; - } - - @Override - public Object getAttribute(String name) { - return null; - } - - @Override - public Enumeration getAttributeNames() { - return null; - } - - @Override - public String getCharacterEncoding() { - return null; - } - - @Override - public void setCharacterEncoding(String env) throws UnsupportedEncodingException { - - } - - @Override - public int getContentLength() { - return 0; - } - - @Override - public long getContentLengthLong() { - return 0; - } - - @Override - public String getContentType() { - return null; - } - - @Override - public ServletInputStream getInputStream() throws IOException { - return null; - } - - @Override - public String getParameter(String name) { - return null; - } - - @Override - public Enumeration getParameterNames() { - return null; - } - - @Override - public String[] getParameterValues(String name) { - return new String[0]; - } - - @Override - public Map getParameterMap() { - return null; - } - - @Override - public String getProtocol() { - return null; - } - - @Override - public String getScheme() { - return null; - } - - @Override - public String getServerName() { - return null; - } - - @Override - public int getServerPort() { - return 0; - } - - @Override - public BufferedReader getReader() throws IOException { - return null; - } - - @Override - public String getRemoteAddr() { - return null; - } - - @Override - public String getRemoteHost() { - return null; - } - - @Override - public void setAttribute(String name, Object o) { - - } - - @Override - public void removeAttribute(String name) { - - } - - @Override - public Locale getLocale() { - return null; - } - - @Override - public Enumeration getLocales() { - return null; - } - - @Override - public boolean isSecure() { - return false; - } - - @Override - public RequestDispatcher getRequestDispatcher(String path) { - return null; - } - - @Override - public String getRealPath(String path) { - return null; - } - - @Override - public int getRemotePort() { - return 0; - } - - @Override - public String getLocalName() { - return null; - } - - @Override - public String getLocalAddr() { - return null; - } - - @Override - public int getLocalPort() { - return 0; - } - - @Override - public ServletContext getServletContext() { - return null; - } - - @Override - public AsyncContext startAsync() throws IllegalStateException { - return null; - } - - @Override - public AsyncContext startAsync(ServletRequest servletRequest, ServletResponse servletResponse) throws IllegalStateException { - return null; - } - - @Override - public boolean isAsyncStarted() { - return false; - } - - @Override - public boolean isAsyncSupported() { - return false; - } - - @Override - public AsyncContext getAsyncContext() { - return null; - } - - @Override - public DispatcherType getDispatcherType() { - return null; - } - }, new AuthorizationContext() { - @Override - public SolrParams getParams() { - return new MapSolrParams(Collections.singletonMap("q", "hello")); - } - - @Override - public Principal getUserPrincipal() { - return new BasicUserPrincipal("George"); - } - - @Override - public String getHttpHeader(String header) { - return "MyHeader"; - } - - @Override - public Enumeration getHeaderNames() { - return Collections.enumeration(Arrays.asList("Header1", "Header2")); - } - - @Override - public String getRemoteAddr() { - return "127.0.0.1"; - } - - @Override - public String getRemoteHost() { - return "localhost"; - } - - @Override - public List getCollectionRequests() { - return Arrays.asList(new CollectionRequest("coll1")); - } - - @Override - public RequestType getRequestType() { - return RequestType.READ; - } - - @Override - public String getResource() { - return "/solr/admin/info"; - } - - @Override - public String getHttpMethod() { - return "GET"; - } - - @Override - public Object getHandler() { - return "/select"; - } - }, new AuthorizationResponse(200)); - assertEquals("{\"message\":\"Authenticated\",\"level\":\"INFO\",\"date\":" + event.getDate().getTime() + ",\"username\":\"George\",\"session\":null,\"clientIp\":null,\"collections\":[\"coll1\"],\"context\":null,\"headers\":{\"bar\":\"barval\",\"foo\":\"fooval\"},\"solrParams\":null,\"solrHost\":null,\"solrPort\":0,\"solrIp\":null,\"resource\":\"/solr/admin/info\",\"httpMethod\":\"GET\",\"queryString\":\"?foo&bar\",\"eventType\":\"AUTHENTICATED\",\"autResponse\":{\"statusCode\":200,\"message\":null}}", - new AuditLoggerPlugin.JSONAuditEventFormatter().formatEvent(event)); - } - - @Test - public void manual() throws Exception { - AuditEvent event = new AuditEvent(AuditEvent.EventType.REJECTED).setHttpMethod("GET"); - assertTrue(event.getDate() != null); - assertEquals("GET", event.getHttpMethod()); - } -} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java new file mode 100644 index 000000000000..75429286eef6 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MockAuditLoggerPlugin extends AuditLoggerPlugin { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public List events = new ArrayList<>(); + public Map typeCounts = new HashMap<>(); + + /** + * Audits an event to an internal list that can be inspected later by the test code + * @param event the audit event + */ + @Override + public void audit(AuditEvent event) { + events.add(event); + incrementType(event.getEventType().name()); + log.info("#{} - {}", events.size(), typeCounts); + } + + private void incrementType(String type) { + if (!typeCounts.containsKey(type)) + typeCounts.put(type, new AtomicInteger(0)); + typeCounts.get(type).incrementAndGet(); + } + + @Override + public void close() throws IOException { /* ignored */ } + + public void reset() { + events.clear(); + typeCounts.clear(); + } +} diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 1828b414bb1f..a3dce92c59f2 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -17,38 +17,38 @@ package org.apache.solr.security; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; +import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.core.SolrResourceLoader; import org.junit.Test; -import static org.junit.Assert.*; - -public class MultiDestinationAuditLoggerTest { +public class MultiDestinationAuditLoggerTest extends LuceneTestCase { @Test - public void init() throws Exception { + public void init() { MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); ArrayList> plugins = new ArrayList>(); - Map myPlugin = new HashMap<>(); - myPlugin.put("class", "solr.SolrLogAuditLoggerPlugin"); - plugins.add(myPlugin); + plugins.add(Collections.singletonMap("class", "solr.SolrLogAuditLoggerPlugin")); + plugins.add(Collections.singletonMap("class", "solr.MockAuditLoggerPlugin")); config.put("plugins", plugins); al.inform(new SolrResourceLoader()); al.init(config); - al.auditAsync(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); + al.audit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); + assertEquals(1, ((MockAuditLoggerPlugin)al.plugins.get(1)).events.size()); assertEquals(0, config.size()); + al.close(); } @Test - public void wrongConfigParam() throws Exception { + public void wrongConfigParam() { MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 9aed5b74daaa..5d51e4ebc405 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -17,34 +17,31 @@ package org.apache.solr.security; -import java.util.Arrays; import java.util.HashMap; +import org.apache.lucene.util.LuceneTestCase; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; - -public class SolrLogAuditLoggerPluginTest { +public class SolrLogAuditLoggerPluginTest extends LuceneTestCase { private SolrLogAuditLoggerPlugin plugin; @Before public void setUp() throws Exception { + super.setUp(); plugin = new SolrLogAuditLoggerPlugin(); } @Test - public void init() throws Exception { + public void init() { HashMap config = new HashMap<>(); - config.put("blockAsync", true); - config.put("queueSize", 1); plugin.init(config); plugin.audit(new AuditEvent(AuditEvent.EventType.REJECTED) .setUsername("Jan") .setHttpMethod("POST") .setMessage("Wrong password") .setResource("/collection1")); - plugin.auditAsync(new AuditEvent(AuditEvent.EventType.AUTHORIZED) + plugin.audit(new AuditEvent(AuditEvent.EventType.AUTHORIZED) .setUsername("Per") .setHttpMethod("GET") .setMessage("Async") From 5c675d288d4be6595aa7a3af474dfc5e07169e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 24 Mar 2018 23:40:35 +0100 Subject: [PATCH 05/65] Simplified with new private method auditIfConfigured(event) Fix another InterruptedException catch --- .../solr/security/AsyncAuditLoggerPlugin.java | 1 + .../org/apache/solr/servlet/HttpSolrCall.java | 26 ++++++++++--------- .../solr/servlet/SolrDispatchFilter.java | 22 +++++++++------- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java index 37c1181f02bc..23684c80b75e 100644 --- a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java @@ -49,6 +49,7 @@ public final void audit(AuditEvent event) { queue.put(event); } catch (InterruptedException e) { log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); + Thread.currentThread().interrupt(); } } else { if (!queue.offer(event)) { diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 746b53794433..79e32e1e658e 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -466,9 +466,7 @@ public Action call() throws IOException { if (solrDispatchFilter.abortErrorMessage != null) { sendError(500, solrDispatchFilter.abortErrorMessage); - if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ERROR, getReq())); - } + auditIfConfigured(new AuditEvent(EventType.ERROR, getReq())); return RETURN; } @@ -488,22 +486,16 @@ public Action call() throws IOException { for (Map.Entry e : headers.entrySet()) response.setHeader(e.getKey(), e.getValue()); } log.debug("USER_REQUIRED "+req.getHeader("Authorization")+" "+ req.getUserPrincipal()); - if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, req, context, authResponse)); - } + auditIfConfigured(new AuditEvent(EventType.REJECTED, req, context, authResponse)); } if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED) && !(authResponse.statusCode == HttpStatus.SC_OK)) { log.info("USER_REQUIRED auth header {} context : {} ", req.getHeader("Authorization"), context); sendError(authResponse.statusCode, "Unauthorized request, Response code: " + authResponse.statusCode); - if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.UNAUTHORIZED, req, context, authResponse)); - } + auditIfConfigured(new AuditEvent(EventType.UNAUTHORIZED, req, context, authResponse)); return RETURN; } - if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHORIZED, req, context, authResponse)); - } + auditIfConfigured(new AuditEvent(EventType.AUTHORIZED, req, context, authResponse)); } HttpServletResponse resp = response; @@ -562,6 +554,16 @@ public Action call() throws IOException { } + /** + * Calls auditIfConfigured logging API if enabled + * @param auditEvent + */ + private void auditIfConfigured(AuditEvent auditEvent) { + if (cores.getAuditLoggerPlugin() != null) { + cores.getAuditLoggerPlugin().audit(auditEvent); + } + } + private boolean shouldAuthorize() { if(PKIAuthenticationPlugin.PATH.equals(path)) return false; //admin/info/key is the path where public key is exposed . it is always unsecured diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 1ff5d81b2cac..0c423f02ac79 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -445,9 +445,7 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo final AtomicBoolean isAuthenticated = new AtomicBoolean(false); AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin(); if (authenticationPlugin == null) { - if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ANONYMOUS, request)); - } + auditIfConfigured(new AuditEvent(EventType.ANONYMOUS, request)); return true; } else { // /admin/info/key must be always open. see SOLR-9188 @@ -478,17 +476,23 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo // multiple code paths. if (!requestContinues || !isAuthenticated.get()) { response.flushBuffer(); - if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, request)); - } + auditIfConfigured(new AuditEvent(EventType.REJECTED, request)); return false; } + auditIfConfigured(new AuditEvent(EventType.AUTHENTICATED, request)); + return true; + } + + /** + * Calls auditIfConfigured logging API if enabled + * @param auditEvent + */ + private void auditIfConfigured(AuditEvent auditEvent) { if (cores.getAuditLoggerPlugin() != null) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHENTICATED, request)); + cores.getAuditLoggerPlugin().audit(auditEvent); } - return true; } - + /** * Wrap the request's input stream with a close shield, as if by a {@link CloseShieldInputStream}. If this is a * retry, we will assume that the stream has already been wrapped and do nothing. From 8841373a64475ef5a1849a14807ea5017d2648a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 2 Apr 2018 22:02:18 +0200 Subject: [PATCH 06/65] Restructuring refguide docs WIP --- .../org/apache/solr/servlet/HttpSolrCall.java | 4 +- .../solr/servlet/SolrDispatchFilter.java | 4 +- solr/solr-ref-guide/src/audit-logging.adoc | 51 +++++++++++++++++++ ...hentication-and-authorization-plugins.adoc | 3 +- solr/solr-ref-guide/src/securing-solr.adoc | 44 +++++++++++++--- ...olrcloud-configuration-and-parameters.adoc | 2 +- 6 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 solr/solr-ref-guide/src/audit-logging.adoc diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 79e32e1e658e..539bfab23306 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -555,8 +555,8 @@ public Action call() throws IOException { } /** - * Calls auditIfConfigured logging API if enabled - * @param auditEvent + * Calls audit logging API if enabled + * @param auditEvent the audit event */ private void auditIfConfigured(AuditEvent auditEvent) { if (cores.getAuditLoggerPlugin() != null) { diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 0c423f02ac79..c370f86b8848 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -484,8 +484,8 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo } /** - * Calls auditIfConfigured logging API if enabled - * @param auditEvent + * Calls audit logging API if enabled + * @param auditEvent the audit event */ private void auditIfConfigured(AuditEvent auditEvent) { if (cores.getAuditLoggerPlugin() != null) { diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc new file mode 100644 index 000000000000..674a354061ef --- /dev/null +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -0,0 +1,51 @@ += Audit Logging +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr has the ability to log an audit trail of all events in the system. +Audit loggers are pluggable to suit any possible format or log destination. + +[quote] +An audit trail (also called audit log) is a security-relevant chronological record, set of records, and/or destination and source of records that provide documentary evidence of the sequence of activities that have affected at any time a specific operation, procedure, or event. (https://en.wikipedia.org/wiki/Audit_trail[Wikipedia]) + +== Events being logged +These are the event types that will be logged by this framework: + +[%header,format=csv,separator=;] +|=== +foo;bar +hello +|=== + + +An example `security.json` showing both sections is shown below to show how these plugins can work together: + +[source,json] +---- +{ +"authentication":{ <1> + "blockUnknown": true, <2> + "class":"solr.BasicAuthPlugin", + "credentials":{"solr":"IV0EHq1OnNrj6gvRCwvFwTrZ1+z1oBbnQdiVC3otuq0= Ndd7LKvVBAaZIF0QAVi1ekCfAJXr1GGfLtRUXhgrF8c="} <3> +}, +"authorization":{ + "class":"solr.RuleBasedAuthorizationPlugin", + "permissions":[{"name":"security-edit", + "role":"admin"}], <4> + "user-role":{"solr":"admin"} <5> +}} +---- diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index 971dbcdf3714..de96064e6975 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -1,5 +1,4 @@ -= Authentication and Authorization Plugins -:page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, rule-based-authorization-plugin += Configuring security plugins in security.json // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index fb81e54f553d..f8f898d19cb2 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -1,5 +1,5 @@ = Securing Solr -:page-children: authentication-and-authorization-plugins, enabling-ssl +:page-children: authentication-and-authorization-plugins, kerberos-authentication-plugin, basic-authentication-plugin, rule-based-authorization-plugin, enabling-ssl, audit-logging, zookeeper-access-control // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -19,13 +19,41 @@ When planning how to secure Solr, you should consider which of the available features or approaches are right for you. -* Authentication or authorization of users using: -** <> -** <> -** <> -** <> -* <> -* If using SolrCloud, <> +== Encryption with TLS certificates +Ecrypting traffic to/from Solr and between Solr nodes prevents sensitive data to be leaked out on +the network. TLS is also a requirement to secure the password when using Basic Authentication. + +See the page <> for details. + +== Authentication, Authorization and Audit logging +Plugins for authentication, authorization and audit logging are configured in the `security.json` configuration file, +and enabling any of these will immediately take effect across the whole cluster. + +Read the chapter <> +to learn how to work with the `security.json` file. + +=== Authentication plugins +Authentication makes sure you know the identity of your users. Supported authentication plugins are: + +* <> +* <> +* <> + +=== Authorization plugins +Authorization makes sure that only users with the necessary roles/permissions can access any given resource. +The authorization plugins shipping with Solr are: + +* <> + +=== Audit logging plugins +Audit logging will record an audit trail of important events in your cluster, such as users being authenticated, +or access being denied to admin APIs. Learn more about audit logging and how to implement an audit logger plugin here: + +* <> + +== Securing Zookeeper traffic +Zookeeper is a central and important part of a SolrCloud cluster and understanding how to secure +its content is covered in the <> page. [WARNING] ==== diff --git a/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc b/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc index 0d79a261ece3..6f739f022836 100644 --- a/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc +++ b/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc @@ -1,5 +1,5 @@ = SolrCloud Configuration and Parameters -:page-children: setting-up-an-external-zookeeper-ensemble, using-zookeeper-to-manage-configuration-files, zookeeper-access-control, collections-api, parameter-reference, command-line-utilities, solrcloud-with-legacy-configuration-files, configsets-api +:page-children: setting-up-an-external-zookeeper-ensemble, using-zookeeper-to-manage-configuration-files, collections-api, parameter-reference, command-line-utilities, solrcloud-with-legacy-configuration-files, configsets-api // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From fd9fc6c38b752db9c5a31e911b7bbbc6fa77bb34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 9 Apr 2018 15:10:13 +0200 Subject: [PATCH 07/65] Audit log completed requests including time and status --- .../org/apache/solr/security/AuditEvent.java | 77 +++++++++++++++++-- .../org/apache/solr/servlet/HttpSolrCall.java | 9 ++- .../SolrLogAuditLoggerPluginTest.java | 5 +- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index d4a27760046a..3f265a3d82fb 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.stream.Collectors; +import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +57,9 @@ public class AuditEvent { private EventType eventType; private AuthorizationResponse autResponse; private String requestType; + private double QTime = -1; + private int status = 0; + private Throwable exception; /* Predefined event types. Custom types can be made through constructor */ public enum EventType { @@ -65,7 +69,8 @@ public enum EventType { ANONYMOUS_REJECTED("AnonymousRejected", "Request from unknown user rejected", Level.WARN), AUTHORIZED("Authorized", "Authorization succeeded", Level.INFO), UNAUTHORIZED("Unauthorized", "Authorization failed", Level.WARN), - ERROR("Error", "Request failed due to an error", Level.ERROR); + COMPLETED("Completed", "Request completed", Level.INFO), + ERROR("Error", "Request was not executed due to an error", Level.ERROR); private final String message; private String explanation; @@ -91,14 +96,19 @@ public AuditEvent(EventType eventType) { this.message = eventType.message; } + public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { + this(eventType, null, httpRequest); + } + /** * Event based on an HttpServletRequest, typically used during authentication. * Solr will fill in details such as ip, http method etc from the request, and * username if Principal exists on the request. * @param eventType a predefined or custom EventType + * @param exception * @param httpRequest the request to initialize from */ - public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { + public AuditEvent(EventType eventType, Throwable exception, HttpServletRequest httpRequest) { this(eventType); this.solrHost = httpRequest.getLocalName(); this.solrPort = httpRequest.getLocalPort(); @@ -108,6 +118,7 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { this.httpMethod = httpRequest.getMethod(); this.queryString = httpRequest.getQueryString(); this.headers = getHeadersFromRequest(httpRequest); + setException(exception); Principal principal = httpRequest.getUserPrincipal(); if (principal != null) { @@ -121,21 +132,49 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { } /** - * Event based on an AuthorizationContext and reponse. Solr will fill in details - * such as collections, , ip, http method etc from the context. + * Event based on request and AuthorizationContext. Solr will fill in details + * such as collections, ip, http method etc from the context. * @param eventType a predefined or custom EventType + * @param httpRequest the request to initialize from * @param authorizationContext the context to initialize from */ - public AuditEvent(EventType eventType, HttpServletRequest httpRequest, AuthorizationContext authorizationContext, AuthorizationResponse authResponse) { + public AuditEvent(EventType eventType, HttpServletRequest httpRequest, AuthorizationContext authorizationContext) { this(eventType, httpRequest); this.collections = authorizationContext.getCollectionRequests() .stream().map(r -> r.collectionName).collect(Collectors.toList()); this.resource = authorizationContext.getResource(); this.requestType = authorizationContext.getRequestType().toString(); authorizationContext.getParams().getAll(this.solrParams); - this.autResponse = authResponse; } + /** + * Event to log completed requests. Takes time and status. Solr will fill in details + * such as collections, ip, http method etc from the HTTP request and context. + * + * @param eventType a predefined or custom EventType + * @param httpRequest the request to initialize from + * @param authorizationContext the context to initialize from + * @param qTime query time + * @param exception exception from query response, or null if OK + */ + public AuditEvent(EventType eventType, HttpServletRequest httpRequest, AuthorizationContext authorizationContext, double qTime, Throwable exception) { + this(eventType, httpRequest, authorizationContext); + setQTime(qTime); + setException(exception); + } + + private int statusFromException(Throwable exception) { + int status = 0; + if (exception != null ) { + if (exception instanceof SolrException) + status = ((SolrException)exception).code(); + else + status = 500; + } + return status; + } + + private HashMap getHeadersFromRequest(HttpServletRequest httpRequest) { HashMap h = new HashMap<>(); Enumeration headersEnum = httpRequest.getHeaderNames(); @@ -224,6 +263,18 @@ public AuthorizationResponse getAutResponse() { return autResponse; } + public long getStatus() { + return status; + } + + public double getQTime() { + return QTime; + } + + public Throwable getException() { + return exception; + } + // Setters, builder style public AuditEvent setSession(String session) { @@ -320,4 +371,18 @@ public AuditEvent setRequestType(String requestType) { this.requestType = requestType; return this; } + + public void setQTime(double QTime) { + this.QTime = QTime; + } + + public void setStatus(int status) { + this.status = status; + } + + public void setException(Throwable exception) { + this.exception = exception; + setStatus(statusFromException(exception)); + } + } diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 539bfab23306..05c96ff7ea26 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -486,16 +486,16 @@ public Action call() throws IOException { for (Map.Entry e : headers.entrySet()) response.setHeader(e.getKey(), e.getValue()); } log.debug("USER_REQUIRED "+req.getHeader("Authorization")+" "+ req.getUserPrincipal()); - auditIfConfigured(new AuditEvent(EventType.REJECTED, req, context, authResponse)); + auditIfConfigured(new AuditEvent(EventType.REJECTED, req, context)); } if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED) && !(authResponse.statusCode == HttpStatus.SC_OK)) { log.info("USER_REQUIRED auth header {} context : {} ", req.getHeader("Authorization"), context); sendError(authResponse.statusCode, "Unauthorized request, Response code: " + authResponse.statusCode); - auditIfConfigured(new AuditEvent(EventType.UNAUTHORIZED, req, context, authResponse)); + auditIfConfigured(new AuditEvent(EventType.UNAUTHORIZED, req, context)); return RETURN; } - auditIfConfigured(new AuditEvent(EventType.AUTHORIZED, req, context, authResponse)); + auditIfConfigured(new AuditEvent(EventType.AUTHORIZED, req, context)); } HttpServletResponse resp = response; @@ -521,6 +521,7 @@ public Action call() throws IOException { */ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp)); execute(solrRsp); + auditIfConfigured(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod); Iterator> headers = solrRsp.httpHeaders(); while (headers.hasNext()) { @@ -535,6 +536,7 @@ public Action call() throws IOException { default: return action; } } catch (Throwable ex) { + auditIfConfigured(new AuditEvent(EventType.ERROR, ex, req)); sendError(ex); // walk the the entire cause chain to search for an Error Throwable t = ex; @@ -738,6 +740,7 @@ private void handleAdminRequest() throws IOException { QueryResponseWriter respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS.get(solrReq.getParams().get(CommonParams.WT)); if (respWriter == null) respWriter = getResponseWriter(); writeResponse(solrResp, respWriter, Method.getMethod(req.getMethod())); + auditIfConfigured(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); } protected QueryResponseWriter getResponseWriter() { diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 5d51e4ebc405..b44ad649ede0 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -25,17 +25,18 @@ public class SolrLogAuditLoggerPluginTest extends LuceneTestCase { private SolrLogAuditLoggerPlugin plugin; + private HashMap config; @Before public void setUp() throws Exception { super.setUp(); plugin = new SolrLogAuditLoggerPlugin(); + config = new HashMap<>(); + plugin.init(config); } @Test public void init() { - HashMap config = new HashMap<>(); - plugin.init(config); plugin.audit(new AuditEvent(AuditEvent.EventType.REJECTED) .setUsername("Jan") .setHttpMethod("POST") From e76827c062cd7f47e9539420a52e2e47f85a44fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 9 Aug 2018 21:58:23 +0200 Subject: [PATCH 08/65] Remove deprecation use --- .../src/java/org/apache/solr/security/AuditLoggerPlugin.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 4737fcf5be88..e301efac36d1 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.solr.common.SolrException; @@ -73,7 +74,7 @@ public static class JSONAuditEventFormatter implements AuditEventFormatter { public String formatEvent(AuditEvent event) { ObjectMapper mapper = new ObjectMapper(); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false); + mapper.setSerializationInclusion(Include.NON_NULL); try { StringWriter sw = new StringWriter(); mapper.writeValue(sw, event); From fcdf15aa7aa1a40ad730640f18916b73d7c107bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 17 Aug 2018 16:53:27 +0200 Subject: [PATCH 09/65] * RefGuide docs * Support for configuring what "eventTypes" that shall be logged by default * Add more tests, own test for AuditLogger * Validate that all params are consumed --- .../solr/security/AsyncAuditLoggerPlugin.java | 3 +- .../org/apache/solr/security/AuditEvent.java | 2 +- .../solr/security/AuditLoggerPlugin.java | 32 +++++ .../security/MultiDestinationAuditLogger.java | 1 - .../security/SolrLogAuditLoggerPlugin.java | 7 +- .../org/apache/solr/servlet/HttpSolrCall.java | 4 +- .../solr/servlet/SolrDispatchFilter.java | 4 +- .../solr/security/AuditLoggerPluginTest.java | 121 ++++++++++++++++++ .../MultiDestinationAuditLoggerTest.java | 8 +- .../SolrLogAuditLoggerPluginTest.java | 32 +++-- solr/solr-ref-guide/src/audit-logging.adoc | 73 ++++++++--- 11 files changed, 249 insertions(+), 38 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java index 23684c80b75e..310ab3c8d5e9 100644 --- a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java @@ -35,6 +35,7 @@ public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implement private static final String PARAM_BLOCKASYNC = "blockAsync"; private static final String PARAM_QUEUE_SIZE = "queueSize"; + private static final int DEFAULT_QUEUE_SIZE = 4096; private BlockingQueue queue; private boolean blockAsync; @@ -73,7 +74,7 @@ public final void audit(AuditEvent event) { */ public void init(Map pluginConfig) { blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); - int blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, 4000))); + int blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))); pluginConfig.remove(PARAM_BLOCKASYNC); pluginConfig.remove(PARAM_QUEUE_SIZE); queue = new ArrayBlockingQueue<>(blockingQueueSize); diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 3f265a3d82fb..470ec16ed2b3 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -58,7 +58,7 @@ public class AuditEvent { private AuthorizationResponse autResponse; private String requestType; private double QTime = -1; - private int status = 0; + private int status = -1; private Throwable exception; /* Predefined event types. Custom types can be made through constructor */ diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index e301efac36d1..b15764c73069 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -20,12 +20,15 @@ import java.io.IOException; import java.io.StringWriter; import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.solr.common.SolrException; +import org.apache.solr.security.AuditEvent.EventType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,9 +37,18 @@ */ public abstract class AuditLoggerPlugin implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String PARAM_EVENT_TYPES = "eventTypes"; protected AuditEventFormatter formatter; + // Event types to be logged by default + protected List eventTypes = Arrays.asList( + EventType.COMPLETED.name(), + EventType.ERROR.name(), + EventType.REJECTED.name(), + EventType.UNAUTHORIZED.name(), + EventType.ANONYMOUS_REJECTED.name()); + /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. * @param event the audit event @@ -50,6 +62,12 @@ public abstract class AuditLoggerPlugin implements Closeable { */ public void init(Map pluginConfig) { formatter = new JSONAuditEventFormatter(); + if (pluginConfig.containsKey(PARAM_EVENT_TYPES)) { + eventTypes = (List) pluginConfig.get(PARAM_EVENT_TYPES); + } + pluginConfig.remove(PARAM_EVENT_TYPES); + pluginConfig.remove("class"); + log.debug("AuditLogger initialized with event types {}", eventTypes); } public void setFormatter(AuditEventFormatter formatter) { @@ -63,6 +81,20 @@ public interface AuditEventFormatter { String formatEvent(AuditEvent event); } + /** + * Checks whether this event should be logged based on "eventTypes" config parameter. + * The framework will call this method and avoid logging if false. + * @param event the event to consider + * @return true if this event should be logged + */ + public boolean shouldLog(AuditEvent event) { + boolean shouldLog = eventTypes.contains(event.getEventType().name()); + if (!shouldLog) { + log.debug("Event type {} is not configured for audit logging", event.getEventType().name()); + } + return shouldLog; + } + /** * Event formatter that returns event as JSON string */ diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 9e353e6fe653..6d49ccb71921 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -74,7 +74,6 @@ public void init(Map pluginConfig) { pluginList.forEach(pluginConf -> plugins.add(createPlugin(pluginConf))); pluginConfig.remove(PARAM_PLUGINS); } - pluginConfig.remove("class"); if (pluginConfig.size() > 0) { log.error("Plugin config was not fully consumed. Remaining parameters are {}", pluginConfig); } diff --git a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java index a6cad9659ff2..f13dbe3e0579 100644 --- a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java @@ -20,6 +20,7 @@ import java.lang.invoke.MethodHandles; import java.util.Map; +import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +46,12 @@ public void init(Map pluginConfig) { .append(" username=\"").append(event.getUsername()).append("\"") .append(" resource=\"").append(event.getResource()).append("\"") .append(" collections=").append(event.getCollections()).toString()); + if (pluginConfig.size() > 0) { + throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Plugin config was not fully consumed. Remaining parameters are " + pluginConfig); + } + log.debug("Initialized SolrLogAuditLoggerPlugin"); } - + /** * Audit logs an event to Solr log. The event should be a {@link AuditEvent} to be able to pull context info * @param event the event to log diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 0da1d27c4496..6fd09dc921bc 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -556,11 +556,11 @@ public Action call() throws IOException { } /** - * Calls audit logging API if enabled + * Calls audit logging API if enabled and if the event type is configured for logging * @param auditEvent the audit event */ private void auditIfConfigured(AuditEvent auditEvent) { - if (cores.getAuditLoggerPlugin() != null) { + if (cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(auditEvent)) { cores.getAuditLoggerPlugin().audit(auditEvent); } } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 6f67aea3e586..8df18484e987 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -532,11 +532,11 @@ public void setWriteListener(WriteListener arg0) { /** - * Calls audit logging API if enabled + * Calls audit logging API if enabled and if the event type is configured for logging * @param auditEvent the audit event */ private void auditIfConfigured(AuditEvent auditEvent) { - if (cores.getAuditLoggerPlugin() != null) { + if (cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(auditEvent)) { cores.getAuditLoggerPlugin().audit(auditEvent); } } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java new file mode 100644 index 000000000000..e2c0bdfaa017 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.security; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class AuditLoggerPluginTest { + protected static final Date SAMPLE_DATE = new Date(1234567890); + protected static final AuditEvent EVENT_ANONYMOUS = new AuditEvent(AuditEvent.EventType.ANONYMOUS) + .setHttpMethod("GET") + .setMessage("Anonymous") + .setResource("/collection1") + .setDate(SAMPLE_DATE); + protected static final AuditEvent EVENT_ANONYMOUS_REJECTED = new AuditEvent(AuditEvent.EventType.ANONYMOUS_REJECTED) + .setHttpMethod("GET") + .setMessage("Anonymous rejected") + .setResource("/collection1"); + protected static final AuditEvent EVENT_AUTHENTICATED = new AuditEvent(AuditEvent.EventType.AUTHENTICATED) + .setUsername("Jan") + .setHttpMethod("GET") + .setMessage("Authenticated") + .setDate(SAMPLE_DATE) + .setResource("/collection1"); + protected static final AuditEvent EVENT_REJECTED = new AuditEvent(AuditEvent.EventType.REJECTED) + .setUsername("Jan") + .setHttpMethod("POST") + .setMessage("Wrong password") + .setDate(SAMPLE_DATE) + .setResource("/collection1"); + protected static final AuditEvent EVENT_AUTHORIZED = new AuditEvent(AuditEvent.EventType.AUTHORIZED) + .setUsername("Per") + .setHttpMethod("GET") + .setMessage("Async") + .setDate(SAMPLE_DATE) + .setResource("/collection1"); + protected static final AuditEvent EVENT_UNAUTHORIZED = new AuditEvent(AuditEvent.EventType.UNAUTHORIZED) + .setUsername("Jan") + .setHttpMethod("POST") + .setMessage("No access to collection1") + .setDate(SAMPLE_DATE) + .setResource("/collection1"); + protected static final AuditEvent EVENT_ERROR = new AuditEvent(AuditEvent.EventType.ERROR) + .setUsername("Jan") + .setHttpMethod("POST") + .setMessage("Error occurred") + .setDate(SAMPLE_DATE) + .setResource("/collection1"); + + private MockAuditLoggerPlugin plugin; + private HashMap config; + + @Before + public void setUp() throws Exception { + plugin = new MockAuditLoggerPlugin(); + config = new HashMap<>(); + plugin.init(config); + } + + @Test + public void init() { + config = new HashMap<>(); + config.put("eventTypes", Arrays.asList("REJECTED")); + plugin.init(config); + assertTrue(plugin.shouldLog(EVENT_REJECTED)); + assertFalse(plugin.shouldLog(EVENT_UNAUTHORIZED)); + } + + @Test + public void shouldLog() { + // Default types + assertTrue(plugin.shouldLog(EVENT_ANONYMOUS_REJECTED)); + assertTrue(plugin.shouldLog(EVENT_REJECTED)); + assertTrue(plugin.shouldLog(EVENT_UNAUTHORIZED)); + assertTrue(plugin.shouldLog(EVENT_ERROR)); + assertFalse(plugin.shouldLog(EVENT_ANONYMOUS)); + assertFalse(plugin.shouldLog(EVENT_AUTHENTICATED)); + assertFalse(plugin.shouldLog(EVENT_AUTHORIZED)); + } + + @Test + public void audit() { + plugin.audit(EVENT_ANONYMOUS_REJECTED); + plugin.audit(EVENT_REJECTED); + assertEquals(1, plugin.typeCounts.get("ANONYMOUS_REJECTED").get()); + assertEquals(1, plugin.typeCounts.get("REJECTED").get()); + assertEquals(2, plugin.events.size()); + } + + @Test + public void jsonEventFormatter() { + assertEquals("{\"message\":\"Anonymous\",\"level\":\"INFO\",\"date\":" + SAMPLE_DATE.getTime() + ",\"solrPort\":0,\"resource\":\"/collection1\",\"httpMethod\":\"GET\",\"eventType\":\"ANONYMOUS\",\"status\":-1,\"qtime\":-1.0}", + plugin.formatter.formatEvent(EVENT_ANONYMOUS)); + assertEquals("{\"message\":\"Authenticated\",\"level\":\"INFO\",\"date\":1234567890,\"username\":\"Jan\",\"solrPort\":0,\"resource\":\"/collection1\",\"httpMethod\":\"GET\",\"eventType\":\"AUTHENTICATED\",\"status\":-1,\"qtime\":-1.0}", + plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); + } + +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index a3dce92c59f2..d246ca577df6 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -33,8 +33,12 @@ public void init() { config.put("class", "solr.MultiDestinationAuditLogger"); ArrayList> plugins = new ArrayList>(); - plugins.add(Collections.singletonMap("class", "solr.SolrLogAuditLoggerPlugin")); - plugins.add(Collections.singletonMap("class", "solr.MockAuditLoggerPlugin")); + Map conf1 = new HashMap<>(); + conf1.put("class", "solr.SolrLogAuditLoggerPlugin"); + plugins.add(conf1); + Map conf2 = new HashMap<>(); + conf2.put("class", "solr.MockAuditLoggerPlugin"); + plugins.add(conf2); config.put("plugins", plugins); al.inform(new SolrResourceLoader()); diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index b44ad649ede0..2661ecfe00a3 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -20,9 +20,13 @@ import java.util.HashMap; import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.common.SolrException; import org.junit.Before; import org.junit.Test; +import static org.apache.solr.security.AuditLoggerPluginTest.EVENT_ANONYMOUS; +import static org.apache.solr.security.AuditLoggerPluginTest.EVENT_AUTHENTICATED; + public class SolrLogAuditLoggerPluginTest extends LuceneTestCase { private SolrLogAuditLoggerPlugin plugin; private HashMap config; @@ -35,19 +39,23 @@ public void setUp() throws Exception { plugin.init(config); } + @Test(expected = SolrException.class) + public void badConfig() { + config = new HashMap<>(); + config.put("invalid", "parameter"); + plugin.init(config); + } + @Test - public void init() { - plugin.audit(new AuditEvent(AuditEvent.EventType.REJECTED) - .setUsername("Jan") - .setHttpMethod("POST") - .setMessage("Wrong password") - .setResource("/collection1")); - plugin.audit(new AuditEvent(AuditEvent.EventType.AUTHORIZED) - .setUsername("Per") - .setHttpMethod("GET") - .setMessage("Async") - .setResource("/collection1")); - assertEquals(0, config.size()); + public void audit() { + plugin.audit(EVENT_ANONYMOUS); } + @Test + public void eventFormatter() { + assertEquals("type=\"ANONYMOUS\" message=\"Anonymous\" method=\"GET\" username=\"null\" resource=\"/collection1\" collections=null", + plugin.formatter.formatEvent(EVENT_ANONYMOUS)); + assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" username=\"Jan\" resource=\"/collection1\" collections=null", + plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); + } } \ No newline at end of file diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 674a354061ef..65b7252cb304 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -22,30 +22,71 @@ Audit loggers are pluggable to suit any possible format or log destination. [quote] An audit trail (also called audit log) is a security-relevant chronological record, set of records, and/or destination and source of records that provide documentary evidence of the sequence of activities that have affected at any time a specific operation, procedure, or event. (https://en.wikipedia.org/wiki/Audit_trail[Wikipedia]) -== Events being logged -These are the event types that will be logged by this framework: +== Event types +These are the event types triggered by the framework: [%header,format=csv,separator=;] |=== -foo;bar -hello +EventType;Usage +AUTHENTICATED;User successfully authenticated +REJECTED;Authentication request rejected +ANONYMOUS;Request proceeds with unknown user +ANONYMOUS_REJECTED;Request from unknown user rejected +AUTHORIZED;Authorization succeeded +UNAUTHORIZED;Authorization failed +COMPLETED;Request completed successfully +ERROR;Request was not executed due to an error |=== +By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. -An example `security.json` showing both sections is shown below to show how these plugins can work together: +== Configuration in security.json +Audit logging is configured in `security.json` under the `auditlogging` key. + +The example `security.json` below configures audit logging to Solr default log file, enabling logging of all available event types. + +[source,json] +---- +{ + "auditlogging":{ + "class":"solr.SolrLogAuditLoggerPlugin", + "eventTypes": ["AUTHENTICATED", "REJECTED", "ANONYMOUS", + "ANONYMOUS_REJECTED", "AUTHORIZED", "UNAUTHORIZED", + "COMPLETED", "ERROR"] + } +} +---- + +=== Chaining multiple loggers +Using the `MultiDestinationAuditLogger` you can configure multiple audit logger plugins in a chain, to log to multiple destinations, as follows: [source,json] ---- { -"authentication":{ <1> - "blockUnknown": true, <2> - "class":"solr.BasicAuthPlugin", - "credentials":{"solr":"IV0EHq1OnNrj6gvRCwvFwTrZ1+z1oBbnQdiVC3otuq0= Ndd7LKvVBAaZIF0QAVi1ekCfAJXr1GGfLtRUXhgrF8c="} <3> -}, -"authorization":{ - "class":"solr.RuleBasedAuthorizationPlugin", - "permissions":[{"name":"security-edit", - "role":"admin"}], <4> - "user-role":{"solr":"admin"} <5> -}} + "auditlogging":{ + "class" : "solr.MultiDestinationAuditLogger", + "plugins" : [ + { "class" : "solr.SolrLogAuditLoggerPlugin" }, + { "class" : "solr.MyOtherAuditPlugin", + "customParam" : "value" + } + ] + } +} ---- + +=== Synchronous vs asynchronous audit logging +AuditLoggerPlugin developers can choose to make audit logging asynchronous by subclassing the `AsyncAuditLoggerPlugin` base class intead of the normal `AuditLoggerPlugin`. This will cause the event to but put on a queue and consumed and logged by a background thread. For Audit loggers with async support, you can also configure queue size and whether it should block when the queue is full: + +[source,json] +---- +{ + "auditlogging":{ + "class" : "solr.MyAsyncAuditLogger", + "blockAsync" : false, + "queueSize" : 4096 + } +} +---- + +It is not possible to enable async audit logging unless the plugin extends the `AsyncAuditLoggerPlugin`. \ No newline at end of file From 7c68f68731b901e5ce0c6a33fae53c25f24c7a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 17 Aug 2018 17:14:26 +0200 Subject: [PATCH 10/65] Fix precommit --- solr/core/src/java/org/apache/solr/security/AuditEvent.java | 1 - .../org/apache/solr/security/MultiDestinationAuditLogger.java | 1 - solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java | 1 - 3 files changed, 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 470ec16ed2b3..7c9ffe333974 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -105,7 +105,6 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { * Solr will fill in details such as ip, http method etc from the request, and * username if Principal exists on the request. * @param eventType a predefined or custom EventType - * @param exception * @param httpRequest the request to initialize from */ public AuditEvent(EventType eventType, Throwable exception, HttpServletRequest httpRequest) { diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 6d49ccb71921..5e6cce722ed5 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -16,7 +16,6 @@ */ package org.apache.solr.security; -import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 6fd09dc921bc..0da561ec722a 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -97,7 +97,6 @@ import org.apache.solr.security.AuthorizationContext.CollectionRequest; import org.apache.solr.security.AuthorizationContext.RequestType; import org.apache.solr.security.AuthorizationResponse; -import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.AuditEvent; import org.apache.solr.security.AuditEvent.EventType; import org.apache.solr.security.PublicKeyHandler; From bf53d87ad121d95d19d13983673338e0ff197f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 17 Aug 2018 17:26:15 +0200 Subject: [PATCH 11/65] Fix another precommit --- .../apache/solr/security/MultiDestinationAuditLoggerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index d246ca577df6..1c45ba953e27 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -17,7 +17,6 @@ package org.apache.solr.security; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -59,5 +58,6 @@ public void wrongConfigParam() { config.put("foo", "Should complain"); al.init(config); assertEquals(1, config.size()); + al.close(); } } \ No newline at end of file From 20cc385a2bdf2e1a79e104be28f68303cc2f6af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 17 Aug 2018 23:56:32 +0200 Subject: [PATCH 12/65] Move the meat of the auditIfConfigured methods to a static method on AuditLoggerPlugin and add debug logging if an event type is skipped --- .../solr/security/AuditLoggerPlugin.java | 18 ++++++++++++++++++ .../org/apache/solr/servlet/HttpSolrCall.java | 5 ++--- .../solr/servlet/SolrDispatchFilter.java | 5 ++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index b15764c73069..624701e72fca 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -49,6 +49,24 @@ public abstract class AuditLoggerPlugin implements Closeable { EventType.UNAUTHORIZED.name(), EventType.ANONYMOUS_REJECTED.name()); + /** + * Static method that can be called to audit log if logging is configured and event type is supported + * @param auditLoggerPlugin reference to the configured logger or null if not configured (will simply return) + * @param auditEvent the event to log if logging is configured + */ + public static void auditIfConfigured(AuditLoggerPlugin auditLoggerPlugin, AuditEvent auditEvent) { + if (auditLoggerPlugin != null) { + if (auditLoggerPlugin.shouldLog(auditEvent)) { + auditLoggerPlugin.audit(auditEvent); + } else { + log.debug("Not logging audit event of type {}, method {}, path {}", + auditEvent.getEventType().name(), + auditEvent.getHttpMethod(), + auditEvent.getResource()); + } + } + } + /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. * @param event the audit event diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 0da561ec722a..d96cedb299ed 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -92,6 +92,7 @@ import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.QueryResponseWriterUtil; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.AuthorizationContext.CollectionRequest; @@ -559,9 +560,7 @@ public Action call() throws IOException { * @param auditEvent the audit event */ private void auditIfConfigured(AuditEvent auditEvent) { - if (cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(auditEvent)) { - cores.getAuditLoggerPlugin().audit(auditEvent); - } + AuditLoggerPlugin.auditIfConfigured(cores.getAuditLoggerPlugin(), auditEvent); } private boolean shouldAuthorize() { diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 8df18484e987..7c9d098ac824 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -73,6 +73,7 @@ import org.apache.solr.metrics.OperatingSystemMetricSet; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.AuditEvent; @@ -536,9 +537,7 @@ public void setWriteListener(WriteListener arg0) { * @param auditEvent the audit event */ private void auditIfConfigured(AuditEvent auditEvent) { - if (cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(auditEvent)) { - cores.getAuditLoggerPlugin().audit(auditEvent); - } + AuditLoggerPlugin.auditIfConfigured(cores.getAuditLoggerPlugin(), auditEvent); } /** From 878119f2e570624656ac7c6c367f922e4bd26cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 10:05:06 +0100 Subject: [PATCH 13/65] Refactor auditlogger to avoid creating AuditEvent when not necessary --- .../solr/security/AuditLoggerPlugin.java | 48 ++++++------------- .../org/apache/solr/servlet/HttpSolrCall.java | 42 +++++++++------- .../solr/servlet/SolrDispatchFilter.java | 22 +++++---- .../solr/security/AuditLoggerPluginTest.java | 18 +++---- 4 files changed, 64 insertions(+), 66 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 624701e72fca..ace33e41a2ea 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -42,31 +42,13 @@ public abstract class AuditLoggerPlugin implements Closeable { protected AuditEventFormatter formatter; // Event types to be logged by default - protected List eventTypes = Arrays.asList( + protected static List eventTypes = Arrays.asList( EventType.COMPLETED.name(), EventType.ERROR.name(), EventType.REJECTED.name(), EventType.UNAUTHORIZED.name(), EventType.ANONYMOUS_REJECTED.name()); - /** - * Static method that can be called to audit log if logging is configured and event type is supported - * @param auditLoggerPlugin reference to the configured logger or null if not configured (will simply return) - * @param auditEvent the event to log if logging is configured - */ - public static void auditIfConfigured(AuditLoggerPlugin auditLoggerPlugin, AuditEvent auditEvent) { - if (auditLoggerPlugin != null) { - if (auditLoggerPlugin.shouldLog(auditEvent)) { - auditLoggerPlugin.audit(auditEvent); - } else { - log.debug("Not logging audit event of type {}, method {}, path {}", - auditEvent.getEventType().name(), - auditEvent.getHttpMethod(), - auditEvent.getResource()); - } - } - } - /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. * @param event the audit event @@ -88,6 +70,20 @@ public void init(Map pluginConfig) { log.debug("AuditLogger initialized with event types {}", eventTypes); } + /** + * Checks whether this event type should be logged based on "eventTypes" config parameter. + * + * @param eventType the event type to consider + * @return true if this event type should be logged + */ + public static boolean shouldLog(EventType eventType) { + boolean shouldLog = eventTypes.contains(eventType.name()); + if (!shouldLog) { + log.debug("Event type {} is not configured for audit logging", eventType.name()); + } + return shouldLog; + } + public void setFormatter(AuditEventFormatter formatter) { this.formatter = formatter; } @@ -99,20 +95,6 @@ public interface AuditEventFormatter { String formatEvent(AuditEvent event); } - /** - * Checks whether this event should be logged based on "eventTypes" config parameter. - * The framework will call this method and avoid logging if false. - * @param event the event to consider - * @return true if this event should be logged - */ - public boolean shouldLog(AuditEvent event) { - boolean shouldLog = eventTypes.contains(event.getEventType().name()); - if (!shouldLog) { - log.debug("Event type {} is not configured for audit logging", event.getEventType().name()); - } - return shouldLog; - } - /** * Event formatter that returns event as JSON string */ diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 8a994ec51475..f8690aefb7af 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -92,14 +92,14 @@ import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.QueryResponseWriterUtil; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.AuditEvent; +import org.apache.solr.security.AuditEvent.EventType; import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.AuthorizationContext.CollectionRequest; import org.apache.solr.security.AuthorizationContext.RequestType; import org.apache.solr.security.AuthorizationResponse; -import org.apache.solr.security.AuditEvent; -import org.apache.solr.security.AuditEvent.EventType; import org.apache.solr.security.PublicKeyHandler; import org.apache.solr.servlet.SolrDispatchFilter.Action; import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; @@ -465,7 +465,9 @@ public Action call() throws IOException { if (solrDispatchFilter.abortErrorMessage != null) { sendError(500, solrDispatchFilter.abortErrorMessage); - auditIfConfigured(new AuditEvent(EventType.ERROR, getReq())); + if (shouldAudit(EventType.ERROR)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ERROR, getReq())); + } return RETURN; } @@ -485,16 +487,22 @@ public Action call() throws IOException { for (Map.Entry e : headers.entrySet()) response.setHeader(e.getKey(), e.getValue()); } log.debug("USER_REQUIRED "+req.getHeader("Authorization")+" "+ req.getUserPrincipal()); - auditIfConfigured(new AuditEvent(EventType.REJECTED, req, context)); + if (shouldAudit(EventType.REJECTED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, req, context)); + } } if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED) && !(authResponse.statusCode == HttpStatus.SC_OK)) { log.info("USER_REQUIRED auth header {} context : {} ", req.getHeader("Authorization"), context); sendError(authResponse.statusCode, "Unauthorized request, Response code: " + authResponse.statusCode); - auditIfConfigured(new AuditEvent(EventType.UNAUTHORIZED, req, context)); + if (shouldAudit(EventType.UNAUTHORIZED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.UNAUTHORIZED, req, context)); + } return RETURN; } - auditIfConfigured(new AuditEvent(EventType.AUTHORIZED, req, context)); + if (shouldAudit(EventType.AUTHORIZED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHORIZED, req, context)); + } } HttpServletResponse resp = response; @@ -520,7 +528,9 @@ public Action call() throws IOException { */ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp)); execute(solrRsp); - auditIfConfigured(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); + if (shouldAudit(EventType.COMPLETED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); + } HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod); Iterator> headers = solrRsp.httpHeaders(); while (headers.hasNext()) { @@ -535,7 +545,9 @@ public Action call() throws IOException { default: return action; } } catch (Throwable ex) { - auditIfConfigured(new AuditEvent(EventType.ERROR, ex, req)); + if (shouldAudit(EventType.ERROR)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ERROR, ex, req)); + } sendError(ex); // walk the the entire cause chain to search for an Error Throwable t = ex; @@ -555,12 +567,8 @@ public Action call() throws IOException { } - /** - * Calls audit logging API if enabled and if the event type is configured for logging - * @param auditEvent the audit event - */ - private void auditIfConfigured(AuditEvent auditEvent) { - AuditLoggerPlugin.auditIfConfigured(cores.getAuditLoggerPlugin(), auditEvent); + private boolean shouldAudit(AuditEvent.EventType eventType) { + return cores.getAuditLoggerPlugin() != null && AuditLoggerPlugin.shouldLog(eventType); } private boolean shouldAuthorize() { @@ -593,7 +601,7 @@ void destroy() { //TODO using Http2Client private void remoteQuery(String coreUrl, HttpServletResponse resp) throws IOException { - HttpRequestBase method = null; + HttpRequestBase method; HttpEntity httpEntity = null; try { String urlstr = coreUrl + queryParams.toQueryString(); @@ -738,7 +746,9 @@ private void handleAdminRequest() throws IOException { QueryResponseWriter respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS.get(solrReq.getParams().get(CommonParams.WT)); if (respWriter == null) respWriter = getResponseWriter(); writeResponse(solrResp, respWriter, Method.getMethod(req.getMethod())); - auditIfConfigured(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); + if (shouldAudit(EventType.COMPLETED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); + } } protected QueryResponseWriter getResponseWriter() { diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index b1b9f158bc30..39e32027ffc2 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -457,7 +457,9 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo final AtomicBoolean isAuthenticated = new AtomicBoolean(false); AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin(); if (authenticationPlugin == null) { - auditIfConfigured(new AuditEvent(EventType.ANONYMOUS, request)); + if (shouldAudit(EventType.ANONYMOUS)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ANONYMOUS, request)); + } return true; } else { // /admin/info/key must be always open. see SOLR-9188 @@ -500,10 +502,14 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo // multiple code paths. if (!requestContinues || !isAuthenticated.get()) { response.flushBuffer(); - auditIfConfigured(new AuditEvent(EventType.REJECTED, request)); + if (shouldAudit(EventType.REJECTED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, request)); + } return false; } - auditIfConfigured(new AuditEvent(EventType.AUTHENTICATED, request)); + if (shouldAudit(EventType.AUTHENTICATED)) { + cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHENTICATED, request)); + } return true; } @@ -563,13 +569,13 @@ public void setWriteListener(WriteListener arg0) { /** - * Calls audit logging API if enabled and if the event type is configured for logging - * @param auditEvent the audit event + * Check if audit logging is enabled and should happen for given event type + * @param eventType the audit event */ - private void auditIfConfigured(AuditEvent auditEvent) { - AuditLoggerPlugin.auditIfConfigured(cores.getAuditLoggerPlugin(), auditEvent); + private boolean shouldAudit(AuditEvent.EventType eventType) { + return cores.getAuditLoggerPlugin() != null && AuditLoggerPlugin.shouldLog(eventType); } - + /** * Wrap the request's input stream with a close shield. If this is a * retry, we will assume that the stream has already been wrapped and do nothing. diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index e2c0bdfaa017..0176367f7c2b 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -85,20 +85,20 @@ public void init() { config = new HashMap<>(); config.put("eventTypes", Arrays.asList("REJECTED")); plugin.init(config); - assertTrue(plugin.shouldLog(EVENT_REJECTED)); - assertFalse(plugin.shouldLog(EVENT_UNAUTHORIZED)); + assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); + assertFalse(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); } @Test public void shouldLog() { // Default types - assertTrue(plugin.shouldLog(EVENT_ANONYMOUS_REJECTED)); - assertTrue(plugin.shouldLog(EVENT_REJECTED)); - assertTrue(plugin.shouldLog(EVENT_UNAUTHORIZED)); - assertTrue(plugin.shouldLog(EVENT_ERROR)); - assertFalse(plugin.shouldLog(EVENT_ANONYMOUS)); - assertFalse(plugin.shouldLog(EVENT_AUTHENTICATED)); - assertFalse(plugin.shouldLog(EVENT_AUTHORIZED)); + assertTrue(plugin.shouldLog(EVENT_ANONYMOUS_REJECTED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_ERROR.getEventType())); + assertFalse(plugin.shouldLog(EVENT_ANONYMOUS.getEventType())); + assertFalse(plugin.shouldLog(EVENT_AUTHENTICATED.getEventType())); + assertFalse(plugin.shouldLog(EVENT_AUTHORIZED.getEventType())); } @Test From daa3394c3c9e30f18ea2853d6dcf4ced88ba0856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 10:17:15 +0100 Subject: [PATCH 14/65] Precommit --- .../src/java/org/apache/solr/servlet/SolrDispatchFilter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 39e32027ffc2..94380cd134b8 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -72,11 +72,10 @@ import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.OperatingSystemMetricSet; import org.apache.solr.metrics.SolrMetricManager; -import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.security.AuditEvent; import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; -import org.apache.solr.security.AuditEvent; import org.apache.solr.security.PublicKeyHandler; import org.apache.solr.util.SolrFileCleaningTracker; import org.apache.solr.util.StartupLoggingUtils; From 94caac83b8a0562485228b88963de3480e93c841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 10:20:51 +0100 Subject: [PATCH 15/65] Static access --- .../solr/security/AuditLoggerPluginTest.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 0176367f7c2b..30675ab20e7f 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -85,20 +85,20 @@ public void init() { config = new HashMap<>(); config.put("eventTypes", Arrays.asList("REJECTED")); plugin.init(config); - assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); - assertFalse(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); + assertTrue(AuditLoggerPlugin.shouldLog(EVENT_REJECTED.getEventType())); + assertFalse(AuditLoggerPlugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); } @Test public void shouldLog() { // Default types - assertTrue(plugin.shouldLog(EVENT_ANONYMOUS_REJECTED.getEventType())); - assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); - assertTrue(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); - assertTrue(plugin.shouldLog(EVENT_ERROR.getEventType())); - assertFalse(plugin.shouldLog(EVENT_ANONYMOUS.getEventType())); - assertFalse(plugin.shouldLog(EVENT_AUTHENTICATED.getEventType())); - assertFalse(plugin.shouldLog(EVENT_AUTHORIZED.getEventType())); + assertTrue(AuditLoggerPlugin.shouldLog(EVENT_ANONYMOUS_REJECTED.getEventType())); + assertTrue(AuditLoggerPlugin.shouldLog(EVENT_REJECTED.getEventType())); + assertTrue(AuditLoggerPlugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); + assertTrue(AuditLoggerPlugin.shouldLog(EVENT_ERROR.getEventType())); + assertFalse(AuditLoggerPlugin.shouldLog(EVENT_ANONYMOUS.getEventType())); + assertFalse(AuditLoggerPlugin.shouldLog(EVENT_AUTHENTICATED.getEventType())); + assertFalse(AuditLoggerPlugin.shouldLog(EVENT_AUTHORIZED.getEventType())); } @Test From 47ca7e51a3740e9d2bd28d1374add210effafda5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 10:32:11 +0100 Subject: [PATCH 16/65] Extend LuceneTestCase --- .../org/apache/solr/security/AuditLoggerPluginTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 30675ab20e7f..74b2e7528050 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -21,14 +21,11 @@ import java.util.Date; import java.util.HashMap; +import org.apache.lucene.util.LuceneTestCase; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class AuditLoggerPluginTest { +public class AuditLoggerPluginTest extends LuceneTestCase { protected static final Date SAMPLE_DATE = new Date(1234567890); protected static final AuditEvent EVENT_ANONYMOUS = new AuditEvent(AuditEvent.EventType.ANONYMOUS) .setHttpMethod("GET") From 754615bab37ece62512689941184e90bda1c2fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 10:42:49 +0100 Subject: [PATCH 17/65] Make eventTypes non-static --- .../solr/security/AuditLoggerPlugin.java | 4 ++-- .../org/apache/solr/servlet/HttpSolrCall.java | 2 +- .../solr/servlet/SolrDispatchFilter.java | 2 +- .../solr/security/AuditLoggerPluginTest.java | 19 ++++++++++--------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index ace33e41a2ea..6acbb76e990d 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -42,7 +42,7 @@ public abstract class AuditLoggerPlugin implements Closeable { protected AuditEventFormatter formatter; // Event types to be logged by default - protected static List eventTypes = Arrays.asList( + protected List eventTypes = Arrays.asList( EventType.COMPLETED.name(), EventType.ERROR.name(), EventType.REJECTED.name(), @@ -76,7 +76,7 @@ public void init(Map pluginConfig) { * @param eventType the event type to consider * @return true if this event type should be logged */ - public static boolean shouldLog(EventType eventType) { + public boolean shouldLog(EventType eventType) { boolean shouldLog = eventTypes.contains(eventType.name()); if (!shouldLog) { log.debug("Event type {} is not configured for audit logging", eventType.name()); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index f8690aefb7af..7ba0fd9302bc 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -568,7 +568,7 @@ public Action call() throws IOException { } private boolean shouldAudit(AuditEvent.EventType eventType) { - return cores.getAuditLoggerPlugin() != null && AuditLoggerPlugin.shouldLog(eventType); + return cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(eventType); } private boolean shouldAuthorize() { diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 94380cd134b8..8b4358fa2e08 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -572,7 +572,7 @@ public void setWriteListener(WriteListener arg0) { * @param eventType the audit event */ private boolean shouldAudit(AuditEvent.EventType eventType) { - return cores.getAuditLoggerPlugin() != null && AuditLoggerPlugin.shouldLog(eventType); + return cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(eventType); } /** diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 74b2e7528050..2e14805a9f12 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -72,6 +72,7 @@ public class AuditLoggerPluginTest extends LuceneTestCase { @Before public void setUp() throws Exception { + super.setUp(); plugin = new MockAuditLoggerPlugin(); config = new HashMap<>(); plugin.init(config); @@ -82,20 +83,20 @@ public void init() { config = new HashMap<>(); config.put("eventTypes", Arrays.asList("REJECTED")); plugin.init(config); - assertTrue(AuditLoggerPlugin.shouldLog(EVENT_REJECTED.getEventType())); - assertFalse(AuditLoggerPlugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); + assertFalse(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); } @Test public void shouldLog() { // Default types - assertTrue(AuditLoggerPlugin.shouldLog(EVENT_ANONYMOUS_REJECTED.getEventType())); - assertTrue(AuditLoggerPlugin.shouldLog(EVENT_REJECTED.getEventType())); - assertTrue(AuditLoggerPlugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); - assertTrue(AuditLoggerPlugin.shouldLog(EVENT_ERROR.getEventType())); - assertFalse(AuditLoggerPlugin.shouldLog(EVENT_ANONYMOUS.getEventType())); - assertFalse(AuditLoggerPlugin.shouldLog(EVENT_AUTHENTICATED.getEventType())); - assertFalse(AuditLoggerPlugin.shouldLog(EVENT_AUTHORIZED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_ANONYMOUS_REJECTED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); + assertTrue(plugin.shouldLog(EVENT_ERROR.getEventType())); + assertFalse(plugin.shouldLog(EVENT_ANONYMOUS.getEventType())); + assertFalse(plugin.shouldLog(EVENT_AUTHENTICATED.getEventType())); + assertFalse(plugin.shouldLog(EVENT_AUTHORIZED.getEventType())); } @Test From fe7f6fc31ee4888bc6cc536af7f4f12f4118cf2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 11:20:56 +0100 Subject: [PATCH 18/65] Support for numThreads, catch exception in Runnable --- .../solr/security/AsyncAuditLoggerPlugin.java | 13 ++++++++++--- .../solr/security/MultiDestinationAuditLogger.java | 5 +++++ .../java/org/apache/solr/servlet/HttpSolrCall.java | 3 +-- .../org/apache/solr/servlet/SolrDispatchFilter.java | 1 - solr/solr-ref-guide/src/audit-logging.adoc | 3 ++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java index 310ab3c8d5e9..fb860eb341dd 100644 --- a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java @@ -35,9 +35,12 @@ public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implement private static final String PARAM_BLOCKASYNC = "blockAsync"; private static final String PARAM_QUEUE_SIZE = "queueSize"; + private static final String PARAM_NUM_THREADS = "numThreads"; private static final int DEFAULT_QUEUE_SIZE = 4096; + private static final int DEFAULT_NUM_THREADS = 1; private BlockingQueue queue; private boolean blockAsync; + private int blockingQueueSize; /** * Enqueues an {@link AuditEvent} to a queue and returns immediately. @@ -54,7 +57,7 @@ public final void audit(AuditEvent event) { } } else { if (!queue.offer(event)) { - log.warn("Audit log async queue is full, not blocking since " + PARAM_BLOCKASYNC + "==false"); + log.warn("Audit log async queue is full (size={}), not blocking since {}", blockingQueueSize, PARAM_BLOCKASYNC + "==false"); } } } @@ -74,11 +77,13 @@ public final void audit(AuditEvent event) { */ public void init(Map pluginConfig) { blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); - int blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))); + blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))); + int numThreads = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS)));; pluginConfig.remove(PARAM_BLOCKASYNC); pluginConfig.remove(PARAM_QUEUE_SIZE); + pluginConfig.remove(PARAM_NUM_THREADS); queue = new ArrayBlockingQueue<>(blockingQueueSize); - ExecutorService executorService = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrjNamedThreadFactory("audit")); + ExecutorService executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); executorService.submit(this); } @@ -93,6 +98,8 @@ public void run() { } catch (InterruptedException e) { log.warn("Interrupted while waiting for next audit log event"); Thread.currentThread().interrupt(); + } catch (Exception ex) { + log.warn("Exception when attempting to audit log asynchronously", ex); } } } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 5e6cce722ed5..6a2337aa75ce 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -79,6 +79,11 @@ public void init(Map pluginConfig) { log.info("Initialized {} audit plugins", plugins.size()); } + @Override + public boolean shouldLog(AuditEvent.EventType eventType) { + return plugins.stream().anyMatch(p -> p.shouldLog(eventType)); + } + private AuditLoggerPlugin createPlugin(Map auditConf) { if (auditConf != null) { String klas = (String) auditConf.get("class"); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 7ba0fd9302bc..dc4cf8104352 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -94,7 +94,6 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuditEvent; import org.apache.solr.security.AuditEvent.EventType; -import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.AuthorizationContext.CollectionRequest; @@ -695,7 +694,7 @@ protected void sendError(Throwable ex) throws IOException { solrParams = SolrRequestParsers.parseQueryString(req.getQueryString()); } else { // we have no params at all, use empty ones: - solrParams = new MapSolrParams(Collections.emptyMap()); + solrParams = new MapSolrParams(Collections.emptyMap()); } solrReq = new SolrQueryRequestBase(core, solrParams) { }; diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 8b4358fa2e08..604a9c9f50e6 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -73,7 +73,6 @@ import org.apache.solr.metrics.OperatingSystemMetricSet; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.security.AuditEvent; -import org.apache.solr.security.AuditLoggerPlugin; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.PublicKeyHandler; diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 65b7252cb304..43e1633fd532 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -76,7 +76,7 @@ Using the `MultiDestinationAuditLogger` you can configure multiple audit logger ---- === Synchronous vs asynchronous audit logging -AuditLoggerPlugin developers can choose to make audit logging asynchronous by subclassing the `AsyncAuditLoggerPlugin` base class intead of the normal `AuditLoggerPlugin`. This will cause the event to but put on a queue and consumed and logged by a background thread. For Audit loggers with async support, you can also configure queue size and whether it should block when the queue is full: +AuditLoggerPlugin developers can choose to make audit logging asynchronous by subclassing the `AsyncAuditLoggerPlugin` base class intead of the normal `AuditLoggerPlugin`. This will cause the event to but put on a queue and consumed and logged by a background thread. For Audit loggers with async support, you can also configure queue size, number of threads and whether it should block when the queue is full: [source,json] ---- @@ -84,6 +84,7 @@ AuditLoggerPlugin developers can choose to make audit logging asynchronous by su "auditlogging":{ "class" : "solr.MyAsyncAuditLogger", "blockAsync" : false, + "numThreads" : 2, "queueSize" : 4096 } } From 6df580dfe1e58abc80ef23cef4a4f94a468698b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 11:27:57 +0100 Subject: [PATCH 19/65] Fix refguide precommit --- .../src/authentication-and-authorization-plugins.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index 8979ca6802b1..400ebdb546c8 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -1,4 +1,5 @@ = Configuring security plugins in security.json +:page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, rule-based-authorization-plugin // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From 2f7bad1f169329dbd6c8d3a302c3dde0f8feba2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 11:56:56 +0100 Subject: [PATCH 20/65] Fix refguide precommit --- solr/solr-ref-guide/src/securing-solr.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index f8f898d19cb2..c761551d9aa7 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -1,5 +1,5 @@ = Securing Solr -:page-children: authentication-and-authorization-plugins, kerberos-authentication-plugin, basic-authentication-plugin, rule-based-authorization-plugin, enabling-ssl, audit-logging, zookeeper-access-control +:page-children: authentication-and-authorization-plugins, enabling-ssl, audit-logging, zookeeper-access-control // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From 24eb9b07db8e22009d408cd2f99a0e4ab59c5689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 7 Feb 2019 16:55:15 +0100 Subject: [PATCH 21/65] SOLR-12120: Set requestType and nodeName --- .../org/apache/solr/security/AuditEvent.java | 23 ++++++++++++++++++- solr/solr-ref-guide/src/audit-logging.adoc | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 7c9ffe333974..6d19f536d963 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -27,8 +27,10 @@ import java.util.stream.Collectors; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkStateReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import static org.apache.solr.security.AuditEvent.EventType.ANONYMOUS; @@ -37,6 +39,7 @@ */ public class AuditEvent { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private String nodeName; private String message; private Level level; @@ -117,6 +120,16 @@ public AuditEvent(EventType eventType, Throwable exception, HttpServletRequest h this.httpMethod = httpRequest.getMethod(); this.queryString = httpRequest.getQueryString(); this.headers = getHeadersFromRequest(httpRequest); + this.nodeName = MDC.get(ZkStateReader.NODE_NAME_PROP); + + switch (this.httpMethod) { + case "GET": + this.requestType = AuthorizationContext.RequestType.READ.name(); + case "POST": + case "PUT": + this.requestType = AuthorizationContext.RequestType.WRITE.name(); + } + setException(exception); Principal principal = httpRequest.getUserPrincipal(); @@ -143,7 +156,7 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza .stream().map(r -> r.collectionName).collect(Collectors.toList()); this.resource = authorizationContext.getResource(); this.requestType = authorizationContext.getRequestType().toString(); - authorizationContext.getParams().getAll(this.solrParams); + authorizationContext.getParams().toMap(this.solrParams); } /** @@ -262,6 +275,14 @@ public AuthorizationResponse getAutResponse() { return autResponse; } + public String getNodeName() { + return nodeName; + } + + public String getRequestType() { + return requestType; + } + public long getStatus() { return status; } diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 43e1633fd532..c3c3e7384ad8 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -76,7 +76,7 @@ Using the `MultiDestinationAuditLogger` you can configure multiple audit logger ---- === Synchronous vs asynchronous audit logging -AuditLoggerPlugin developers can choose to make audit logging asynchronous by subclassing the `AsyncAuditLoggerPlugin` base class intead of the normal `AuditLoggerPlugin`. This will cause the event to but put on a queue and consumed and logged by a background thread. For Audit loggers with async support, you can also configure queue size, number of threads and whether it should block when the queue is full: +AuditLoggerPlugin developers can choose to make audit logging asynchronous by subclassing the `AsyncAuditLoggerPlugin` base class intead of the normal `AuditLoggerPlugin`. This will cause the event to be put on a queue and for asynchronous loggging by a background thread. For Audit loggers with async support, you can also configure queue size, number of threads and whether it should block when the queue is full: [source,json] ---- From 244d797d528ae30c32ad800c24db9b3beddd2125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 15 Mar 2019 14:31:23 +0100 Subject: [PATCH 22/65] Add metrics support to AuditLogger --- .../solr/security/AsyncAuditLoggerPlugin.java | 15 +- .../org/apache/solr/security/AuditEvent.java | 5 +- .../solr/security/AuditLoggerPlugin.java | 80 +++++- .../security/MultiDestinationAuditLogger.java | 4 + .../security/SolrLogAuditLoggerPlugin.java | 4 +- .../org/apache/solr/servlet/HttpSolrCall.java | 14 +- .../solr/servlet/SolrDispatchFilter.java | 6 +- .../security/AuditLoggerIntegrationTest.java | 252 ++++++++++++++++++ .../solr/security/AuditLoggerPluginTest.java | 4 +- .../MultiDestinationAuditLoggerTest.java | 2 +- .../SolrLogAuditLoggerPluginTest.java | 2 +- 11 files changed, 369 insertions(+), 19 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java index fb860eb341dd..9e98513f1a58 100644 --- a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java @@ -24,11 +24,15 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrjNamedThreadFactory; +import org.apache.solr.metrics.SolrMetricManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Base class for asynchronous audit logging. Extend this class for queued logging events + * Base class for asynchronous audit logging. Extend this class for queued logging events. + * This interface may change in next release and is marked experimental + * @since 8.1.0 + * @lucene.experimental */ public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implements Runnable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -41,6 +45,7 @@ public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implement private BlockingQueue queue; private boolean blockAsync; private int blockingQueueSize; + /** * Enqueues an {@link AuditEvent} to a queue and returns immediately. @@ -103,4 +108,12 @@ public void run() { } } } + + @Override + public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { + super.initializeMetrics(manager, registryName, tag, scope); + manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueSizeMax", true, "queueSizeMax", getCategory().toString()); + manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString()); + metricNames.add("queueSize"); + } } diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 6d19f536d963..82f19b836ec9 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -35,7 +35,10 @@ import static org.apache.solr.security.AuditEvent.EventType.ANONYMOUS; /** - * Audit event that takes request and auth context as input to be able to audit log custom things + * Audit event that takes request and auth context as input to be able to audit log custom things. + * This interface may change in next release and is marked experimental + * @since 8.1.0 + * @lucene.experimental */ public class AuditEvent { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 6acbb76e990d..62dbea12aa30 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -23,23 +23,44 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.solr.common.SolrException; +import org.apache.solr.core.SolrInfoBean; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.security.AuditEvent.EventType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Base class for Audit logger plugins + * Base class for Audit logger plugins. + * This interface may change in next release and is marked experimental + * @since 8.1.0 + * @lucene.experimental */ -public abstract class AuditLoggerPlugin implements Closeable { +public abstract class AuditLoggerPlugin implements Closeable, SolrInfoBean, SolrMetricProducer { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_EVENT_TYPES = "eventTypes"; protected AuditEventFormatter formatter; + MetricRegistry registry; + Set metricNames = ConcurrentHashMap.newKeySet(); + + protected String registryName; + protected SolrMetricManager metricManager; + protected Meter numErrors = new Meter(); + protected Meter numLogged = new Meter(); + protected Timer requestTimes = new Timer(); + protected Counter totalTime = new Counter(); // Event types to be logged by default protected List eventTypes = Arrays.asList( @@ -55,6 +76,23 @@ public abstract class AuditLoggerPlugin implements Closeable { */ public abstract void audit(AuditEvent event); + /** + * Called by the framework, and takes care of metrics + */ + public final void doAudit(AuditEvent event) { + Timer.Context timer = requestTimes.time(); + numLogged.mark(); + try { + audit(event); + } catch(Exception e) { + numErrors.mark(); + throw e; + } finally { + long elapsed = timer.stop(); + totalTime.inc(elapsed); + } + } + /** * Initialize the plugin from security.json. * This method removes parameters from config object after consuming, so subclasses can check for config errors. @@ -88,6 +126,44 @@ public void setFormatter(AuditEventFormatter formatter) { this.formatter = formatter; } + @Override + public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, final String scope) { + this.metricManager = manager; + this.registryName = registryName; + // Metrics + registry = manager.registry(registryName); + numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope); + numLogged = manager.meter(this, registryName, "logged", getCategory().toString(), scope); + requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope); + totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope); + metricNames.addAll(Arrays.asList("errors", "logged", "requestTimes", "totalTime")); + } + + @Override + public String getName() { + return this.getClass().getName(); + } + + @Override + public String getDescription() { + return "Auditlogger Plugin " + this.getClass().getName(); + } + + @Override + public Category getCategory() { + return Category.SECURITY; + } + + @Override + public Set getMetricNames() { + return metricNames; + } + + @Override + public MetricRegistry getMetricRegistry() { + return registry; + } + /** * Interface for formatting the event */ diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 6a2337aa75ce..cd0aa464ffee 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -39,6 +39,10 @@ * { "class" : "solr.MyOtherAuditPlugin"} * ] * + * + * This interface may change in next release and is marked experimental + * @since 8.1.0 + * @lucene.experimental */ public class MultiDestinationAuditLogger extends AuditLoggerPlugin implements ResourceLoaderAware { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java index f13dbe3e0579..7f9b972e46b6 100644 --- a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java @@ -25,7 +25,9 @@ import org.slf4j.LoggerFactory; /** - * Audit logger that writes to the Solr log + * Audit logger that writes to the Solr log. + * This interface may change in next release and is marked experimental + * @since 8.1.0 * @lucene.experimental */ public class SolrLogAuditLoggerPlugin extends AuditLoggerPlugin { diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index bee48eb945df..108ee36ec26f 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -465,7 +465,7 @@ public Action call() throws IOException { if (solrDispatchFilter.abortErrorMessage != null) { sendError(500, solrDispatchFilter.abortErrorMessage); if (shouldAudit(EventType.ERROR)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ERROR, getReq())); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.ERROR, getReq())); } return RETURN; } @@ -487,7 +487,7 @@ public Action call() throws IOException { } log.debug("USER_REQUIRED "+req.getHeader("Authorization")+" "+ req.getUserPrincipal()); if (shouldAudit(EventType.REJECTED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, req, context)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.REJECTED, req, context)); } } if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED) && !(authResponse.statusCode == HttpStatus.SC_OK)) { @@ -495,12 +495,12 @@ public Action call() throws IOException { sendError(authResponse.statusCode, "Unauthorized request, Response code: " + authResponse.statusCode); if (shouldAudit(EventType.UNAUTHORIZED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.UNAUTHORIZED, req, context)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.UNAUTHORIZED, req, context)); } return RETURN; } if (shouldAudit(EventType.AUTHORIZED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHORIZED, req, context)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.AUTHORIZED, req, context)); } } @@ -529,7 +529,7 @@ public Action call() throws IOException { SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp)); execute(solrRsp); if (shouldAudit(EventType.COMPLETED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); } HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod); Iterator> headers = solrRsp.httpHeaders(); @@ -546,7 +546,7 @@ public Action call() throws IOException { } } catch (Throwable ex) { if (shouldAudit(EventType.ERROR)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ERROR, ex, req)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.ERROR, ex, req)); } sendError(ex); // walk the the entire cause chain to search for an Error @@ -747,7 +747,7 @@ private void handleAdminRequest() throws IOException { if (respWriter == null) respWriter = getResponseWriter(); writeResponse(solrResp, respWriter, Method.getMethod(req.getMethod())); if (shouldAudit(EventType.COMPLETED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); } } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 604a9c9f50e6..f22b429f02b0 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -456,7 +456,7 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin(); if (authenticationPlugin == null) { if (shouldAudit(EventType.ANONYMOUS)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.ANONYMOUS, request)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.ANONYMOUS, request)); } return true; } else { @@ -501,12 +501,12 @@ private boolean authenticateRequest(HttpServletRequest request, HttpServletRespo if (!requestContinues || !isAuthenticated.get()) { response.flushBuffer(); if (shouldAudit(EventType.REJECTED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.REJECTED, request)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.REJECTED, request)); } return false; } if (shouldAudit(EventType.AUTHENTICATED)) { - cores.getAuditLoggerPlugin().audit(new AuditEvent(EventType.AUTHENTICATED, request)); + cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.AUTHENTICATED, request)); } return true; } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java new file mode 100644 index 000000000000..70bfb2604816 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.cloud.SolrCloudAuthTestCase; +import org.apache.solr.common.util.Pair; +import org.apache.solr.common.util.Utils; +import org.jose4j.jwk.PublicJsonWebKey; +import org.jose4j.jwk.RsaJsonWebKey; +import org.jose4j.jwk.RsaJwkGenerator; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jws.JsonWebSignature; +import org.jose4j.jwt.JwtClaims; +import org.jose4j.lang.JoseException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Validate that audit logging works in a live cluster + */ +@LuceneTestCase.Slow +public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { + protected static final int NUM_SERVERS = 2; + protected static final int NUM_SHARDS = 2; + protected static final int REPLICATION_FACTOR = 1; + private final String COLLECTION = "jwtColl"; + private String jwtTestToken; + private String baseUrl; + private JsonWebSignature jws; + private String jwtTokenWrongSignature; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + configureCluster(NUM_SERVERS)// nodes + .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json")) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .withDefaultClusterProperty("useLegacyReplicaAssignment", "false") + .configure(); + baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + + String jwkJSON = "{\n" + + " \"kty\": \"RSA\",\n" + + " \"d\": \"i6pyv2z3o-MlYytWsOr3IE1olu2RXZBzjPRBNgWAP1TlLNaphHEvH5aHhe_CtBAastgFFMuP29CFhaL3_tGczkvWJkSveZQN2AHWHgRShKgoSVMspkhOt3Ghha4CvpnZ9BnQzVHnaBnHDTTTfVgXz7P1ZNBhQY4URG61DKIF-JSSClyh1xKuMoJX0lILXDYGGcjVTZL_hci4IXPPTpOJHV51-pxuO7WU5M9252UYoiYyCJ56ai8N49aKIMsqhdGuO4aWUwsGIW4oQpjtce5eEojCprYl-9rDhTwLAFoBtjy6LvkqlR2Ae5dKZYpStljBjK8PJrBvWZjXAEMDdQ8PuQ\",\n" + + " \"e\": \"AQAB\",\n" + + " \"use\": \"sig\",\n" + + " \"kid\": \"test\",\n" + + " \"alg\": \"RS256\",\n" + + " \"n\": \"jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ\"\n" + + "}"; + + PublicJsonWebKey jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); + JwtClaims claims = JWTAuthPluginTest.generateClaims(); + jws = new JsonWebSignature(); + jws.setPayload(claims.toJson()); + jws.setKey(jwk.getPrivateKey()); + jws.setKeyIdHeaderValue(jwk.getKeyId()); + jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); + + jwtTestToken = jws.getCompactSerialization(); + + PublicJsonWebKey jwk2 = RsaJwkGenerator.generateJwk(2048); + jwk2.setKeyId("k2"); + JsonWebSignature jws2 = new JsonWebSignature(); + jws2.setPayload(claims.toJson()); + jws2.setKey(jwk2.getPrivateKey()); + jws2.setKeyIdHeaderValue(jwk2.getKeyId()); + jws2.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); + jwtTokenWrongSignature = jws2.getCompactSerialization(); + + cluster.waitForAllNodes(10); + } + + @Override + @After + public void tearDown() throws Exception { + shutdownCluster(); + super.tearDown(); + } + + @Test(expected = IOException.class) + public void infoRequestWithoutToken() throws Exception { + get(baseUrl + "/admin/info/system", null); + } + + @Test + public void testMetrics() throws Exception { + boolean isUseV2Api = random().nextBoolean(); + String authcPrefix = "/admin/authentication"; + if(isUseV2Api){ + authcPrefix = "/____v2/cluster/security/authentication"; + } + String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + CloseableHttpClient cl = HttpClientUtil.createClient(null); + + createCollection(COLLECTION); + + // Missing token + getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", null); + assertAuthMetricsMinimums(2, 1, 0, 0, 1, 0); + executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: false}}", jws); + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "false", 20, jws); + // Pass through + verifySecurityStatus(cl, baseUrl + "/admin/info/key", "key", NOT_NULL_PREDICATE, 20); + // Now succeeds since blockUnknown=false + get(baseUrl + "/" + COLLECTION + "/query?q=*:*", null); + executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", null); + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "true", 20, jws); + + assertAuthMetricsMinimums(9, 4, 4, 0, 1, 0); + + // Wrong Credentials + getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTokenWrongSignature); + assertAuthMetricsMinimums(10, 4, 4, 1, 1, 0); + + // JWT parse error + getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", "foozzz"); + assertAuthMetricsMinimums(11, 4, 4, 1, 1, 1); + + HttpClientUtil.close(cl); + } + + @Test + public void createCollectionUpdateAndQueryDistributed() throws Exception { + // Admin request will use PKI inter-node auth from Overseer, and succeed + createCollection(COLLECTION); + + // Now update three documents + assertAuthMetricsMinimums(1, 1, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); + Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); + + // First a non distributed query + result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); + + // Now do a distributed query, using JWTAuth for inter-node + result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + assertAuthMetricsMinimums(9, 9, 0, 0, 0, 0); + + // Delete + assertEquals(200, get(baseUrl + "/admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); + assertAuthMetricsMinimums(10, 10, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); + } + + private void getAndFail(String url, String token) { + try { + get(url, token); + fail("Request to " + url + " with token " + token + " should have failed"); + } catch(Exception e) { /* Fall through */ } + } + + private Pair get(String url, String token) throws IOException { + URL createUrl = new URL(url); + HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); + if (token != null) + createConn.setRequestProperty("Authorization", "Bearer " + token); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent(), StandardCharsets.UTF_8)); + String result = br2.lines().collect(Collectors.joining("\n")); + int code = createConn.getResponseCode(); + createConn.disconnect(); + return new Pair<>(result, code); + } + + private Pair post(String url, String json, String token) throws IOException { + URL createUrl = new URL(url); + HttpURLConnection con = (HttpURLConnection) createUrl.openConnection(); + con.setRequestMethod("POST"); + con.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); + if (token != null) + con.setRequestProperty("Authorization", "Bearer " + token); + + con.setDoOutput(true); + OutputStream os = con.getOutputStream(); + os.write(json.getBytes(StandardCharsets.UTF_8)); + os.flush(); + os.close(); + + con.connect(); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent(), StandardCharsets.UTF_8)); + String result = br2.lines().collect(Collectors.joining("\n")); + int code = con.getResponseCode(); + con.disconnect(); + return new Pair<>(result, code); + } + + private void createCollection(String collectionName) throws IOException { + assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + collectionName + "&numShards=2", jwtTestToken).second().intValue()); + cluster.waitForActiveCollection(collectionName, 2, 2); + } + + private void executeCommand(String url, HttpClient cl, String payload, JsonWebSignature jws) throws IOException, JoseException { + HttpPost httpPost; + HttpResponse r; + httpPost = new HttpPost(url); + if (jws != null) + setAuthorizationHeader(httpPost, "Bearer " + jws.getCompactSerialization()); + httpPost.setEntity(new ByteArrayEntity(payload.getBytes(UTF_8))); + httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); + r = cl.execute(httpPost); + String response = IOUtils.toString(r.getEntity().getContent(), StandardCharsets.UTF_8); + assertEquals("Non-200 response code. Response was " + response, 200, r.getStatusLine().getStatusCode()); + assertFalse("Response contained errors: " + response, response.contains("errorMessages")); + Utils.consumeFully(r.getEntity()); + } +} diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 2e14805a9f12..fe907b52855e 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -101,8 +101,8 @@ public void shouldLog() { @Test public void audit() { - plugin.audit(EVENT_ANONYMOUS_REJECTED); - plugin.audit(EVENT_REJECTED); + plugin.doAudit(EVENT_ANONYMOUS_REJECTED); + plugin.doAudit(EVENT_REJECTED); assertEquals(1, plugin.typeCounts.get("ANONYMOUS_REJECTED").get()); assertEquals(1, plugin.typeCounts.get("REJECTED").get()); assertEquals(2, plugin.events.size()); diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 1c45ba953e27..49bdaaacad46 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -43,7 +43,7 @@ public void init() { al.inform(new SolrResourceLoader()); al.init(config); - al.audit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); + al.doAudit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); assertEquals(1, ((MockAuditLoggerPlugin)al.plugins.get(1)).events.size()); assertEquals(0, config.size()); diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 2661ecfe00a3..ac18fb65d385 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -48,7 +48,7 @@ public void badConfig() { @Test public void audit() { - plugin.audit(EVENT_ANONYMOUS); + plugin.doAudit(EVENT_ANONYMOUS); } @Test From 98c941de7b36622676c8801b6feffa4d55db726a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 15 Mar 2019 20:50:54 +0100 Subject: [PATCH 23/65] Add metrics support to AuditLogger. Start of Integration test --- .../org/apache/solr/security/AuditEvent.java | 15 +- .../security/auditlog_plugin_security.json | 6 + .../security/AuditLoggerIntegrationTest.java | 268 ++++++------------ .../security/CallbackAuditLoggerPlugin.java | 61 ++++ solr/solr-ref-guide/src/audit-logging.adoc | 5 +- 5 files changed, 168 insertions(+), 187 deletions(-) create mode 100644 solr/core/src/test-files/solr/security/auditlog_plugin_security.json create mode 100644 solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 82f19b836ec9..52c46c66042d 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -59,7 +59,7 @@ public class AuditEvent { private String solrIp; private String resource; private String httpMethod; - private String queryString; + private String httpQueryString; private EventType eventType; private AuthorizationResponse autResponse; private String requestType; @@ -121,7 +121,7 @@ public AuditEvent(EventType eventType, Throwable exception, HttpServletRequest h this.clientIp = httpRequest.getRemoteAddr(); this.resource = httpRequest.getContextPath(); this.httpMethod = httpRequest.getMethod(); - this.queryString = httpRequest.getQueryString(); + this.httpQueryString = httpRequest.getQueryString(); this.headers = getHeadersFromRequest(httpRequest); this.nodeName = MDC.get(ZkStateReader.NODE_NAME_PROP); @@ -159,7 +159,8 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza .stream().map(r -> r.collectionName).collect(Collectors.toList()); this.resource = authorizationContext.getResource(); this.requestType = authorizationContext.getRequestType().toString(); - authorizationContext.getParams().toMap(this.solrParams); + // TODO: Insert params??? + //authorizationContext.getParams().toMap(this.solrParams); } /** @@ -246,8 +247,8 @@ public String getHttpMethod() { return httpMethod; } - public String getQueryString() { - return queryString; + public String getHttpQueryString() { + return httpQueryString; } public EventType getEventType() { @@ -355,8 +356,8 @@ public AuditEvent setHttpMethod(String httpMethod) { return this; } - public AuditEvent setQueryString(String queryString) { - this.queryString = queryString; + public AuditEvent setHttpQueryString(String httpQueryString) { + this.httpQueryString = httpQueryString; return this; } diff --git a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json new file mode 100644 index 000000000000..80d190c78cb5 --- /dev/null +++ b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json @@ -0,0 +1,6 @@ +{ + "auditlogging": { + "class": "solr.CallbackAuditLoggerPlugin", + "callbackPort": "_PORT_" + } +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 70bfb2604816..1f5ce3c8f163 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -17,190 +17,93 @@ package org.apache.solr.security; import java.io.BufferedReader; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStream; +import java.lang.invoke.MethodHandles; import java.net.HttpURLConnection; +import java.net.ServerSocket; +import java.net.Socket; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.commons.io.FileUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.SolrCloudAuthTestCase; import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; -import org.jose4j.jwk.PublicJsonWebKey; -import org.jose4j.jwk.RsaJsonWebKey; -import org.jose4j.jwk.RsaJwkGenerator; -import org.jose4j.jws.AlgorithmIdentifiers; -import org.jose4j.jws.JsonWebSignature; -import org.jose4j.jwt.JwtClaims; -import org.jose4j.lang.JoseException; import org.junit.After; import org.junit.Before; import org.junit.Test; - -import static java.nio.charset.StandardCharsets.UTF_8; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Validate that audit logging works in a live cluster */ @LuceneTestCase.Slow +@SolrTestCaseJ4.SuppressSSL public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { - protected static final int NUM_SERVERS = 2; - protected static final int NUM_SHARDS = 2; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected static final int NUM_SERVERS = 1; + protected static final int NUM_SHARDS = 1; protected static final int REPLICATION_FACTOR = 1; - private final String COLLECTION = "jwtColl"; - private String jwtTestToken; + private final String COLLECTION = "auditCollection"; private String baseUrl; - private JsonWebSignature jws; - private String jwtTokenWrongSignature; @Override @Before public void setUp() throws Exception { super.setUp(); - - configureCluster(NUM_SERVERS)// nodes - .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json")) - .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .withDefaultClusterProperty("useLegacyReplicaAssignment", "false") - .configure(); - baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - - String jwkJSON = "{\n" + - " \"kty\": \"RSA\",\n" + - " \"d\": \"i6pyv2z3o-MlYytWsOr3IE1olu2RXZBzjPRBNgWAP1TlLNaphHEvH5aHhe_CtBAastgFFMuP29CFhaL3_tGczkvWJkSveZQN2AHWHgRShKgoSVMspkhOt3Ghha4CvpnZ9BnQzVHnaBnHDTTTfVgXz7P1ZNBhQY4URG61DKIF-JSSClyh1xKuMoJX0lILXDYGGcjVTZL_hci4IXPPTpOJHV51-pxuO7WU5M9252UYoiYyCJ56ai8N49aKIMsqhdGuO4aWUwsGIW4oQpjtce5eEojCprYl-9rDhTwLAFoBtjy6LvkqlR2Ae5dKZYpStljBjK8PJrBvWZjXAEMDdQ8PuQ\",\n" + - " \"e\": \"AQAB\",\n" + - " \"use\": \"sig\",\n" + - " \"kid\": \"test\",\n" + - " \"alg\": \"RS256\",\n" + - " \"n\": \"jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ\"\n" + - "}"; - - PublicJsonWebKey jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); - JwtClaims claims = JWTAuthPluginTest.generateClaims(); - jws = new JsonWebSignature(); - jws.setPayload(claims.toJson()); - jws.setKey(jwk.getPrivateKey()); - jws.setKeyIdHeaderValue(jwk.getKeyId()); - jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); - - jwtTestToken = jws.getCompactSerialization(); - - PublicJsonWebKey jwk2 = RsaJwkGenerator.generateJwk(2048); - jwk2.setKeyId("k2"); - JsonWebSignature jws2 = new JsonWebSignature(); - jws2.setPayload(claims.toJson()); - jws2.setKey(jwk2.getPrivateKey()); - jws2.setKeyIdHeaderValue(jwk2.getKeyId()); - jws2.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); - jwtTokenWrongSignature = jws2.getCompactSerialization(); - - cluster.waitForAllNodes(10); } @Override @After public void tearDown() throws Exception { - shutdownCluster(); super.tearDown(); } - @Test(expected = IOException.class) - public void infoRequestWithoutToken() throws Exception { - get(baseUrl + "/admin/info/system", null); - } - @Test - public void testMetrics() throws Exception { - boolean isUseV2Api = random().nextBoolean(); - String authcPrefix = "/admin/authentication"; - if(isUseV2Api){ - authcPrefix = "/____v2/cluster/security/authentication"; + public void test() throws Exception { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + try (CallbackReceiver receiver = new CallbackReceiver()) { + int callbackPort = receiver.getPort(); + + executorService.submit(receiver); + + log.info("Starting cluster with callbackPort {}", callbackPort); + String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); + securityJson = securityJson.replace("_PORT_", Integer.toString(callbackPort)); + configureCluster(NUM_SERVERS)// nodes + .withSecurityJson(securityJson) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .withDefaultClusterProperty("useLegacyReplicaAssignment", "false") + .configure(); + baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + + cluster.waitForAllNodes(10); + + String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + + createCollection(COLLECTION); + + get(baseUrl + "/" + COLLECTION + "/query?q=*:*"); + assertEquals(1, receiver.getCount()); + } finally { + shutdownCluster(); + executorService.shutdown(); } - String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - CloseableHttpClient cl = HttpClientUtil.createClient(null); - - createCollection(COLLECTION); - - // Missing token - getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", null); - assertAuthMetricsMinimums(2, 1, 0, 0, 1, 0); - executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: false}}", jws); - verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "false", 20, jws); - // Pass through - verifySecurityStatus(cl, baseUrl + "/admin/info/key", "key", NOT_NULL_PREDICATE, 20); - // Now succeeds since blockUnknown=false - get(baseUrl + "/" + COLLECTION + "/query?q=*:*", null); - executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", null); - verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "true", 20, jws); - - assertAuthMetricsMinimums(9, 4, 4, 0, 1, 0); - - // Wrong Credentials - getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTokenWrongSignature); - assertAuthMetricsMinimums(10, 4, 4, 1, 1, 0); - - // JWT parse error - getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", "foozzz"); - assertAuthMetricsMinimums(11, 4, 4, 1, 1, 1); - - HttpClientUtil.close(cl); - } - - @Test - public void createCollectionUpdateAndQueryDistributed() throws Exception { - // Admin request will use PKI inter-node auth from Overseer, and succeed - createCollection(COLLECTION); - - // Now update three documents - assertAuthMetricsMinimums(1, 1, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); - Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); - - // First a non distributed query - result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); - - // Now do a distributed query, using JWTAuth for inter-node - result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(9, 9, 0, 0, 0, 0); - - // Delete - assertEquals(200, get(baseUrl + "/admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); - assertAuthMetricsMinimums(10, 10, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); - } - - private void getAndFail(String url, String token) { - try { - get(url, token); - fail("Request to " + url + " with token " + token + " should have failed"); - } catch(Exception e) { /* Fall through */ } } - private Pair get(String url, String token) throws IOException { + private Pair get(String url) throws IOException { URL createUrl = new URL(url); HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); - if (token != null) - createConn.setRequestProperty("Authorization", "Bearer " + token); BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent(), StandardCharsets.UTF_8)); String result = br2.lines().collect(Collectors.joining("\n")); int code = createConn.getResponseCode(); @@ -208,45 +111,52 @@ private Pair get(String url, String token) throws IOException { return new Pair<>(result, code); } - private Pair post(String url, String json, String token) throws IOException { - URL createUrl = new URL(url); - HttpURLConnection con = (HttpURLConnection) createUrl.openConnection(); - con.setRequestMethod("POST"); - con.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); - if (token != null) - con.setRequestProperty("Authorization", "Bearer " + token); - - con.setDoOutput(true); - OutputStream os = con.getOutputStream(); - os.write(json.getBytes(StandardCharsets.UTF_8)); - os.flush(); - os.close(); - - con.connect(); - BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent(), StandardCharsets.UTF_8)); - String result = br2.lines().collect(Collectors.joining("\n")); - int code = con.getResponseCode(); - con.disconnect(); - return new Pair<>(result, code); - } - private void createCollection(String collectionName) throws IOException { - assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + collectionName + "&numShards=2", jwtTestToken).second().intValue()); - cluster.waitForActiveCollection(collectionName, 2, 2); + assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + collectionName + "&numShards=" + NUM_SHARDS).second().intValue()); + cluster.waitForActiveCollection(collectionName, NUM_SHARDS, REPLICATION_FACTOR); } - private void executeCommand(String url, HttpClient cl, String payload, JsonWebSignature jws) throws IOException, JoseException { - HttpPost httpPost; - HttpResponse r; - httpPost = new HttpPost(url); - if (jws != null) - setAuthorizationHeader(httpPost, "Bearer " + jws.getCompactSerialization()); - httpPost.setEntity(new ByteArrayEntity(payload.getBytes(UTF_8))); - httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); - r = cl.execute(httpPost); - String response = IOUtils.toString(r.getEntity().getContent(), StandardCharsets.UTF_8); - assertEquals("Non-200 response code. Response was " + response, 200, r.getStatusLine().getStatusCode()); - assertFalse("Response contained errors: " + response, response.contains("errorMessages")); - Utils.consumeFully(r.getEntity()); + private class CallbackReceiver implements Runnable, AutoCloseable { + private final ServerSocket serverSocket; + private AtomicInteger count = new AtomicInteger(); + private AtomicInteger errors = new AtomicInteger(); + + public CallbackReceiver() throws IOException { + serverSocket = new ServerSocket(0); + } + + public int getCount() { + return count.get(); + } + + public int getErrors() { + return errors.get(); + } + + public int getPort() { + return serverSocket.getLocalPort(); + } + + @Override + public void run() { + try { + Socket socket = serverSocket.accept(); + InputStream is = socket.getInputStream(); + DataInputStream dis = new DataInputStream(is); + int b; + while (true) { + b = dis.readByte(); + log.info("Callback=" + b); + count.incrementAndGet(); + } + } catch (IOException e) { + log.info("Socket closed"); + } + } + + @Override + public void close() throws Exception { + serverSocket.close(); + } } } diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java new file mode 100644 index 000000000000..f5dce9f317cf --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.invoke.MethodHandles; +import java.net.Socket; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CallbackAuditLoggerPlugin extends AuditLoggerPlugin { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private int callbackPort; + private Socket socket; + private PrintWriter out; + + /** + * Opens a socket to send a callback, e.g. to a running test client + * @param event the audit event + */ + @Override + public void audit(AuditEvent event) { + log.info("Received audit event, type={}", event.getEventType()); + out.append('l'); + } + + @Override + public void init(Map pluginConfig) { + super.init(pluginConfig); + callbackPort = Integer.parseInt((String) pluginConfig.get("callbackPort")); + try { + socket = new Socket("127.0.0.1", callbackPort); +// socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), callbackPort)); + out = new PrintWriter(socket.getOutputStream(), true); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws IOException { + socket.close(); + } +} diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index c3c3e7384ad8..1856fa46460c 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -90,4 +90,7 @@ AuditLoggerPlugin developers can choose to make audit logging asynchronous by su } ---- -It is not possible to enable async audit logging unless the plugin extends the `AsyncAuditLoggerPlugin`. \ No newline at end of file +It is not possible to enable async audit logging unless the plugin extends the `AsyncAuditLoggerPlugin`. + +== Metrics +AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. \ No newline at end of file From 6ca85a98380871f00f3ee83420caa4d783fc60a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 09:54:34 +0100 Subject: [PATCH 24/65] Add metrics support to AuditLogger. Start of Integration test --- .../solr/security/AsyncAuditLoggerPlugin.java | 239 +++++++++--------- .../solr/security/AuditLoggerPlugin.java | 117 ++++++++- .../security/MultiDestinationAuditLogger.java | 3 - .../security/SolrLogAuditLoggerPlugin.java | 4 - .../MultiDestinationAuditLoggerTest.java | 6 +- .../SolrLogAuditLoggerPluginTest.java | 10 +- 6 files changed, 236 insertions(+), 143 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java index 9e98513f1a58..fa0c8a2f2f50 100644 --- a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java @@ -1,119 +1,120 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security; - -import java.lang.invoke.MethodHandles; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; - -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.SolrjNamedThreadFactory; -import org.apache.solr.metrics.SolrMetricManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for asynchronous audit logging. Extend this class for queued logging events. - * This interface may change in next release and is marked experimental - * @since 8.1.0 - * @lucene.experimental - */ -public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implements Runnable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String PARAM_BLOCKASYNC = "blockAsync"; - private static final String PARAM_QUEUE_SIZE = "queueSize"; - private static final String PARAM_NUM_THREADS = "numThreads"; - private static final int DEFAULT_QUEUE_SIZE = 4096; - private static final int DEFAULT_NUM_THREADS = 1; - private BlockingQueue queue; - private boolean blockAsync; - private int blockingQueueSize; - - - /** - * Enqueues an {@link AuditEvent} to a queue and returns immediately. - * A background thread will pull events from this queue and call {@link #auditCallback(AuditEvent)} - * @param event the audit event - */ - public final void audit(AuditEvent event) { - if (blockAsync) { - try { - queue.put(event); - } catch (InterruptedException e) { - log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); - Thread.currentThread().interrupt(); - } - } else { - if (!queue.offer(event)) { - log.warn("Audit log async queue is full (size={}), not blocking since {}", blockingQueueSize, PARAM_BLOCKASYNC + "==false"); - } - } - } - - /** - * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. - * This method will be called by the audit background thread as it pulls events from the - * queue. This is where the actual logging work shall be done. - * @param event the audit event - */ - public abstract void auditCallback(AuditEvent event); - - /** - * Initialize the plugin from security.json. - * This method removes parameters from config object after consuming, so subclasses can check for config errors. - * @param pluginConfig the config for the plugin - */ - public void init(Map pluginConfig) { - blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); - blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))); - int numThreads = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS)));; - pluginConfig.remove(PARAM_BLOCKASYNC); - pluginConfig.remove(PARAM_QUEUE_SIZE); - pluginConfig.remove(PARAM_NUM_THREADS); - queue = new ArrayBlockingQueue<>(blockingQueueSize); - ExecutorService executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); - executorService.submit(this); - } - - /** - * Pick next event from async queue and call {@link #auditCallback(AuditEvent)} - */ - @Override - public void run() { - while (true) { - try { - auditCallback(queue.take()); - } catch (InterruptedException e) { - log.warn("Interrupted while waiting for next audit log event"); - Thread.currentThread().interrupt(); - } catch (Exception ex) { - log.warn("Exception when attempting to audit log asynchronously", ex); - } - } - } - - @Override - public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { - super.initializeMetrics(manager, registryName, tag, scope); - manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueSizeMax", true, "queueSizeMax", getCategory().toString()); - manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString()); - metricNames.add("queueSize"); - } -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one or more +// * contributor license agreements. See the NOTICE file distributed with +// * this work for additional information regarding copyright ownership. +// * The ASF licenses this file to You under the Apache License, Version 2.0 +// * (the "License"); you may not use this file except in compliance with +// * the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +//package org.apache.solr.security; +// +//import java.lang.invoke.MethodHandles; +//import java.util.Map; +//import java.util.concurrent.ArrayBlockingQueue; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.ExecutorService; +// +//import org.apache.solr.common.util.ExecutorUtil; +//import org.apache.solr.common.util.SolrjNamedThreadFactory; +//import org.apache.solr.metrics.SolrMetricManager; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +///** +// * Base class for asynchronous audit logging. Extend this class for queued logging events. +// * This interface may change in next release and is marked experimental +// * @since 8.1.0 +// * @lucene.experimental +// */ +//public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implements Runnable { +// private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); +// +// private static final String PARAM_ASYNC = "async"; +// private static final String PARAM_BLOCKASYNC = "blockAsync"; +// private static final String PARAM_QUEUE_SIZE = "queueSize"; +// private static final String PARAM_NUM_THREADS = "numThreads"; +// private static final int DEFAULT_QUEUE_SIZE = 4096; +// private static final int DEFAULT_NUM_THREADS = 1; +// private BlockingQueue queue; +// private boolean blockAsync; +// private int blockingQueueSize; +// +// +// /** +// * Enqueues an {@link AuditEvent} to a queue and returns immediately. +// * A background thread will pull events from this queue and call {@link #auditCallback(AuditEvent)} +// * @param event the audit event +// */ +// public final void audit(AuditEvent event) { +// if (blockAsync) { +// try { +// queue.put(event); +// } catch (InterruptedException e) { +// log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); +// Thread.currentThread().interrupt(); +// } +// } else { +// if (!queue.offer(event)) { +// log.warn("Audit log async queue is full (size={}), not blocking since {}", blockingQueueSize, PARAM_BLOCKASYNC + "==false"); +// } +// } +// } +// +// /** +// * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. +// * This method will be called by the audit background thread as it pulls events from the +// * queue. This is where the actual logging work shall be done. +// * @param event the audit event +// */ +// public abstract void auditCallback(AuditEvent event); +// +// /** +// * Initialize the plugin from security.json. +// * This method removes parameters from config object after consuming, so subclasses can check for config errors. +// * @param pluginConfig the config for the plugin +// */ +// public void init(Map pluginConfig) { +// blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); +// blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))); +// int numThreads = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS)));; +// pluginConfig.remove(PARAM_BLOCKASYNC); +// pluginConfig.remove(PARAM_QUEUE_SIZE); +// pluginConfig.remove(PARAM_NUM_THREADS); +// queue = new ArrayBlockingQueue<>(blockingQueueSize); +// ExecutorService executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); +// executorService.submit(this); +// } +// +// /** +// * Pick next event from async queue and call {@link #auditCallback(AuditEvent)} +// */ +// @Override +// public void run() { +// while (true) { +// try { +// auditCallback(queue.take()); +// } catch (InterruptedException e) { +// log.warn("Interrupted while waiting for next audit log event"); +// Thread.currentThread().interrupt(); +// } catch (Exception ex) { +// log.warn("Exception when attempting to audit log asynchronously", ex); +// } +// } +// } +// +// @Override +// public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { +// super.initializeMetrics(manager, registryName, tag, scope); +// manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueSizeMax", true, "queueSizeMax", getCategory().toString()); +// manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString()); +// metricNames.add("queueSize"); +// } +//} diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 62dbea12aa30..ddb8b282ec95 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -24,7 +24,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; @@ -34,6 +37,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.core.SolrInfoBean; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricProducer; @@ -47,9 +52,21 @@ * @since 8.1.0 * @lucene.experimental */ -public abstract class AuditLoggerPlugin implements Closeable, SolrInfoBean, SolrMetricProducer { +public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfoBean, SolrMetricProducer { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_EVENT_TYPES = "eventTypes"; + + private static final String PARAM_ASYNC = "async"; + private static final String PARAM_BLOCKASYNC = "blockAsync"; + private static final String PARAM_QUEUE_SIZE = "queueSize"; + private static final String PARAM_NUM_THREADS = "numThreads"; + private static final int DEFAULT_QUEUE_SIZE = 4096; + private static final int DEFAULT_NUM_THREADS = 1; + + private BlockingQueue queue; + private boolean async; + private boolean blockAsync; + private int blockingQueueSize; protected AuditEventFormatter formatter; MetricRegistry registry; @@ -69,6 +86,7 @@ public abstract class AuditLoggerPlugin implements Closeable, SolrInfoBean, Solr EventType.REJECTED.name(), EventType.UNAUTHORIZED.name(), EventType.ANONYMOUS_REJECTED.name()); + private ExecutorService executorService; /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. @@ -80,16 +98,65 @@ public abstract class AuditLoggerPlugin implements Closeable, SolrInfoBean, Solr * Called by the framework, and takes care of metrics */ public final void doAudit(AuditEvent event) { - Timer.Context timer = requestTimes.time(); - numLogged.mark(); - try { - audit(event); - } catch(Exception e) { - numErrors.mark(); - throw e; - } finally { - long elapsed = timer.stop(); - totalTime.inc(elapsed); + if (async) { + auditAsync(event); + } else { + Timer.Context timer = requestTimes.time(); + numLogged.mark(); + try { + audit(event); + } catch(Exception e) { + numErrors.mark(); + throw e; + } finally { + totalTime.inc(timer.stop()); + } + } + } + + /** + * Enqueues an {@link AuditEvent} to a queue and returns immediately. + * A background thread will pull events from this queue and call {@link #audit(AuditEvent)} + * @param event the audit event + */ + public final void auditAsync(AuditEvent event) { + if (blockAsync) { + try { + queue.put(event); + } catch (InterruptedException e) { + log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); + Thread.currentThread().interrupt(); + } + } else { + if (!queue.offer(event)) { + log.warn("Audit log async queue is full (size={}), not blocking since {}", blockingQueueSize, PARAM_BLOCKASYNC + "==false"); + } + } + } + + /** + * Pick next event from async queue and call {@link #audit(AuditEvent)} + */ + @Override + public void run() { + while (true) { + Timer.Context timer = null; + try { + AuditEvent event = queue.take(); + numLogged.mark(); + timer = requestTimes.time(); + audit(event); + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for next audit log event"); + Thread.currentThread().interrupt(); + } catch (Exception ex) { + log.warn("Exception when attempting to audit log asynchronously", ex); + numErrors.mark(); + } finally { + if (timer != null) { + totalTime.inc(timer.stop()); + } + } } } @@ -104,8 +171,22 @@ public void init(Map pluginConfig) { eventTypes = (List) pluginConfig.get(PARAM_EVENT_TYPES); } pluginConfig.remove(PARAM_EVENT_TYPES); + + async = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_ASYNC, false))); + blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); + blockingQueueSize = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))) : 1; + int numThreads = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS))) : 1; + pluginConfig.remove(PARAM_ASYNC); + pluginConfig.remove(PARAM_BLOCKASYNC); + pluginConfig.remove(PARAM_QUEUE_SIZE); + pluginConfig.remove(PARAM_NUM_THREADS); + queue = new ArrayBlockingQueue<>(blockingQueueSize); + if (async) { + executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); + executorService.submit(this); + } pluginConfig.remove("class"); - log.debug("AuditLogger initialized with event types {}", eventTypes); + log.debug("AuditLogger initialized in {} mode with event types {}", async?"async":"syncronous", eventTypes); } /** @@ -136,7 +217,10 @@ public void initializeMetrics(SolrMetricManager manager, String registryName, St numLogged = manager.meter(this, registryName, "logged", getCategory().toString(), scope); requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope); totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope); - metricNames.addAll(Arrays.asList("errors", "logged", "requestTimes", "totalTime")); + manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueCapacity", true, "queueCapacity", getCategory().toString()); + manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString()); + manager.registerGauge(this, registryName, () -> async,"async", true, "async", getCategory().toString()); + metricNames.addAll(Arrays.asList("errors", "logged", "requestTimes", "totalTime", "queueCapacity", "queueSize", "async")); } @Override @@ -192,4 +276,11 @@ public String formatEvent(AuditEvent event) { } } } + + @Override + public void close() throws IOException { + if (executorService != null) { + executorService.shutdownNow(); + } + } } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index cd0aa464ffee..734b2d3d6adb 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -103,9 +103,6 @@ private AuditLoggerPlugin createPlugin(Map auditConf) { } } - @Override - public void close() {} - @Override public void inform(ResourceLoader loader) { this.loader = loader; diff --git a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java index 7f9b972e46b6..55c594556961 100644 --- a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java @@ -16,7 +16,6 @@ */ package org.apache.solr.security; -import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Map; @@ -74,7 +73,4 @@ public void audit(AuditEvent event) { break; } } - - @Override - public void close() throws IOException {} } diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 49bdaaacad46..bcc42201c9df 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -16,17 +16,19 @@ */ package org.apache.solr.security; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.core.SolrResourceLoader; +import org.junit.After; import org.junit.Test; public class MultiDestinationAuditLoggerTest extends LuceneTestCase { @Test - public void init() { + public void init() throws IOException { MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); @@ -51,7 +53,7 @@ public void init() { } @Test - public void wrongConfigParam() { + public void wrongConfigParam() throws IOException { MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index ac18fb65d385..ec79d44d3f15 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -17,10 +17,13 @@ package org.apache.solr.security; +import java.io.Closeable; +import java.io.IOException; import java.util.HashMap; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.SolrException; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -37,10 +40,13 @@ public void setUp() throws Exception { plugin = new SolrLogAuditLoggerPlugin(); config = new HashMap<>(); plugin.init(config); + closeAfterTest(plugin); } @Test(expected = SolrException.class) - public void badConfig() { + public void badConfig() throws IOException { + plugin.close(); + plugin = new SolrLogAuditLoggerPlugin(); config = new HashMap<>(); config.put("invalid", "parameter"); plugin.init(config); @@ -57,5 +63,5 @@ public void eventFormatter() { plugin.formatter.formatEvent(EVENT_ANONYMOUS)); assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" username=\"Jan\" resource=\"/collection1\" collections=null", plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); - } + } } \ No newline at end of file From 9cabd73f4730a39bf28a7e73887bbde486433799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 12:58:52 +0100 Subject: [PATCH 25/65] Properly close AuditLogger on CoreContainer shutdown() Merge Async capabilities into AuditLoggerPlugin, configurable Added integration test with CallbackAuditLoggerPlugin, and get tests passing Updated refGuide --- .../org/apache/solr/core/CoreContainer.java | 13 ++ .../solr/security/AsyncAuditLoggerPlugin.java | 120 --------------- .../solr/security/AuditLoggerPlugin.java | 74 +++++----- .../security/auditlog_plugin_security.json | 3 +- .../security/AuditLoggerIntegrationTest.java | 139 ++++++++---------- .../solr/security/AuditLoggerPluginTest.java | 4 +- .../security/CallbackAuditLoggerPlugin.java | 12 +- .../MultiDestinationAuditLoggerTest.java | 5 +- .../SolrLogAuditLoggerPluginTest.java | 6 +- solr/solr-ref-guide/src/audit-logging.adoc | 41 +++--- 10 files changed, 147 insertions(+), 270 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 32405b2aac93..0535508ecacb 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -855,6 +855,9 @@ private void reloadSecurityProperties() { pkiAuthenticationPlugin.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, "/authentication/pki"); } initializeAuditloggerPlugin((Map) securityConfig.getData().get("auditlogging")); + if (auditloggerPlugin != null) { + auditloggerPlugin.plugin.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, "/auditlogging"); + } } private static void checkForDuplicateCoreNames(List cds) { @@ -1023,6 +1026,16 @@ public void shutdown() { log.warn("Exception while closing authentication plugin.", e); } + // It should be safe to close the auditlogger plugin at this point. + try { + if (auditloggerPlugin != null) { + auditloggerPlugin.plugin.close(); + auditloggerPlugin = null; + } + } catch (Exception e) { + log.warn("Exception while closing auditlogger plugin.", e); + } + org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort } diff --git a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java deleted file mode 100644 index fa0c8a2f2f50..000000000000 --- a/solr/core/src/java/org/apache/solr/security/AsyncAuditLoggerPlugin.java +++ /dev/null @@ -1,120 +0,0 @@ -///* -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -//package org.apache.solr.security; -// -//import java.lang.invoke.MethodHandles; -//import java.util.Map; -//import java.util.concurrent.ArrayBlockingQueue; -//import java.util.concurrent.BlockingQueue; -//import java.util.concurrent.ExecutorService; -// -//import org.apache.solr.common.util.ExecutorUtil; -//import org.apache.solr.common.util.SolrjNamedThreadFactory; -//import org.apache.solr.metrics.SolrMetricManager; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -// -///** -// * Base class for asynchronous audit logging. Extend this class for queued logging events. -// * This interface may change in next release and is marked experimental -// * @since 8.1.0 -// * @lucene.experimental -// */ -//public abstract class AsyncAuditLoggerPlugin extends AuditLoggerPlugin implements Runnable { -// private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); -// -// private static final String PARAM_ASYNC = "async"; -// private static final String PARAM_BLOCKASYNC = "blockAsync"; -// private static final String PARAM_QUEUE_SIZE = "queueSize"; -// private static final String PARAM_NUM_THREADS = "numThreads"; -// private static final int DEFAULT_QUEUE_SIZE = 4096; -// private static final int DEFAULT_NUM_THREADS = 1; -// private BlockingQueue queue; -// private boolean blockAsync; -// private int blockingQueueSize; -// -// -// /** -// * Enqueues an {@link AuditEvent} to a queue and returns immediately. -// * A background thread will pull events from this queue and call {@link #auditCallback(AuditEvent)} -// * @param event the audit event -// */ -// public final void audit(AuditEvent event) { -// if (blockAsync) { -// try { -// queue.put(event); -// } catch (InterruptedException e) { -// log.warn("Interrupted while waiting to insert AuditEvent into blocking queue"); -// Thread.currentThread().interrupt(); -// } -// } else { -// if (!queue.offer(event)) { -// log.warn("Audit log async queue is full (size={}), not blocking since {}", blockingQueueSize, PARAM_BLOCKASYNC + "==false"); -// } -// } -// } -// -// /** -// * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. -// * This method will be called by the audit background thread as it pulls events from the -// * queue. This is where the actual logging work shall be done. -// * @param event the audit event -// */ -// public abstract void auditCallback(AuditEvent event); -// -// /** -// * Initialize the plugin from security.json. -// * This method removes parameters from config object after consuming, so subclasses can check for config errors. -// * @param pluginConfig the config for the plugin -// */ -// public void init(Map pluginConfig) { -// blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); -// blockingQueueSize = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))); -// int numThreads = Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS)));; -// pluginConfig.remove(PARAM_BLOCKASYNC); -// pluginConfig.remove(PARAM_QUEUE_SIZE); -// pluginConfig.remove(PARAM_NUM_THREADS); -// queue = new ArrayBlockingQueue<>(blockingQueueSize); -// ExecutorService executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); -// executorService.submit(this); -// } -// -// /** -// * Pick next event from async queue and call {@link #auditCallback(AuditEvent)} -// */ -// @Override -// public void run() { -// while (true) { -// try { -// auditCallback(queue.take()); -// } catch (InterruptedException e) { -// log.warn("Interrupted while waiting for next audit log event"); -// Thread.currentThread().interrupt(); -// } catch (Exception ex) { -// log.warn("Exception when attempting to audit log asynchronously", ex); -// } -// } -// } -// -// @Override -// public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { -// super.initializeMetrics(manager, registryName, tag, scope); -// manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueSizeMax", true, "queueSizeMax", getCategory().toString()); -// manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString()); -// metricNames.add("queueSize"); -// } -//} diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index ddb8b282ec95..4c6b45162605 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -61,7 +61,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo private static final String PARAM_QUEUE_SIZE = "queueSize"; private static final String PARAM_NUM_THREADS = "numThreads"; private static final int DEFAULT_QUEUE_SIZE = 4096; - private static final int DEFAULT_NUM_THREADS = 1; + private static final int DEFAULT_NUM_THREADS = 2; private BlockingQueue queue; private boolean async; @@ -87,6 +87,36 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo EventType.UNAUTHORIZED.name(), EventType.ANONYMOUS_REJECTED.name()); private ExecutorService executorService; + private boolean closed; + + /** + * Initialize the plugin from security.json. + * This method removes parameters from config object after consuming, so subclasses can check for config errors. + * @param pluginConfig the config for the plugin + */ + public void init(Map pluginConfig) { + formatter = new JSONAuditEventFormatter(); + if (pluginConfig.containsKey(PARAM_EVENT_TYPES)) { + eventTypes = (List) pluginConfig.get(PARAM_EVENT_TYPES); + } + pluginConfig.remove(PARAM_EVENT_TYPES); + + async = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_ASYNC, true))); + blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); + blockingQueueSize = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))) : 1; + int numThreads = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS))) : 1; + pluginConfig.remove(PARAM_ASYNC); + pluginConfig.remove(PARAM_BLOCKASYNC); + pluginConfig.remove(PARAM_QUEUE_SIZE); + pluginConfig.remove(PARAM_NUM_THREADS); + queue = new ArrayBlockingQueue<>(blockingQueueSize); + if (async) { + executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); + executorService.submit(this); + } + pluginConfig.remove("class"); + log.debug("AuditLogger initialized in {} mode with event types {}", async?"async":"syncronous", eventTypes); + } /** * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. @@ -139,56 +169,23 @@ public final void auditAsync(AuditEvent event) { */ @Override public void run() { - while (true) { - Timer.Context timer = null; + while (!closed && !Thread.currentThread().isInterrupted()) { try { AuditEvent event = queue.take(); - numLogged.mark(); - timer = requestTimes.time(); + Timer.Context timer = requestTimes.time(); audit(event); + numLogged.mark(); + totalTime.inc(timer.stop()); } catch (InterruptedException e) { log.warn("Interrupted while waiting for next audit log event"); Thread.currentThread().interrupt(); } catch (Exception ex) { log.warn("Exception when attempting to audit log asynchronously", ex); numErrors.mark(); - } finally { - if (timer != null) { - totalTime.inc(timer.stop()); - } } } } - /** - * Initialize the plugin from security.json. - * This method removes parameters from config object after consuming, so subclasses can check for config errors. - * @param pluginConfig the config for the plugin - */ - public void init(Map pluginConfig) { - formatter = new JSONAuditEventFormatter(); - if (pluginConfig.containsKey(PARAM_EVENT_TYPES)) { - eventTypes = (List) pluginConfig.get(PARAM_EVENT_TYPES); - } - pluginConfig.remove(PARAM_EVENT_TYPES); - - async = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_ASYNC, false))); - blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); - blockingQueueSize = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))) : 1; - int numThreads = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS))) : 1; - pluginConfig.remove(PARAM_ASYNC); - pluginConfig.remove(PARAM_BLOCKASYNC); - pluginConfig.remove(PARAM_QUEUE_SIZE); - pluginConfig.remove(PARAM_NUM_THREADS); - queue = new ArrayBlockingQueue<>(blockingQueueSize); - if (async) { - executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); - executorService.submit(this); - } - pluginConfig.remove("class"); - log.debug("AuditLogger initialized in {} mode with event types {}", async?"async":"syncronous", eventTypes); - } - /** * Checks whether this event type should be logged based on "eventTypes" config parameter. * @@ -279,6 +276,7 @@ public String formatEvent(AuditEvent event) { @Override public void close() throws IOException { + closed = true; if (executorService != null) { executorService.shutdownNow(); } diff --git a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json index 80d190c78cb5..6bbddfe05346 100644 --- a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json +++ b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json @@ -1,6 +1,7 @@ { "auditlogging": { "class": "solr.CallbackAuditLoggerPlugin", - "callbackPort": "_PORT_" + "callbackPort": "_PORT_", + "async": _ASYNC_ } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 1f5ce3c8f163..c9fbb1018b15 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -17,28 +17,22 @@ package org.apache.solr.security; import java.io.BufferedReader; -import java.io.DataInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.lang.invoke.MethodHandles; -import java.net.HttpURLConnection; import java.net.ServerSocket; import java.net.Socket; -import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.SolrCloudAuthTestCase; -import org.apache.solr.common.util.Pair; -import org.junit.After; -import org.junit.Before; +import org.apache.solr.util.DefaultSolrThreadFactory; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +40,6 @@ /** * Validate that audit logging works in a live cluster */ -@LuceneTestCase.Slow @SolrTestCaseJ4.SuppressSSL public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -55,82 +48,68 @@ public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SHARDS = 1; protected static final int REPLICATION_FACTOR = 1; private final String COLLECTION = "auditCollection"; - private String baseUrl; - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - } - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); + @Test + public void testSynchronous() throws Exception { + doTest(false); } @Test - public void test() throws Exception { - ExecutorService executorService = Executors.newSingleThreadExecutor(); - try (CallbackReceiver receiver = new CallbackReceiver()) { - int callbackPort = receiver.getPort(); - - executorService.submit(receiver); - - log.info("Starting cluster with callbackPort {}", callbackPort); - String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); - securityJson = securityJson.replace("_PORT_", Integer.toString(callbackPort)); - configureCluster(NUM_SERVERS)// nodes - .withSecurityJson(securityJson) - .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .withDefaultClusterProperty("useLegacyReplicaAssignment", "false") - .configure(); - baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - - cluster.waitForAllNodes(10); - - String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - - createCollection(COLLECTION); - - get(baseUrl + "/" + COLLECTION + "/query?q=*:*"); - assertEquals(1, receiver.getCount()); - } finally { - shutdownCluster(); - executorService.shutdown(); - } + public void testAsync() throws Exception { + doTest(true); } - private Pair get(String url) throws IOException { - URL createUrl = new URL(url); - HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); - BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent(), StandardCharsets.UTF_8)); - String result = br2.lines().collect(Collectors.joining("\n")); - int code = createConn.getResponseCode(); - createConn.disconnect(); - return new Pair<>(result, code); - } + void doTest(boolean async) throws Exception { + CallbackReceiver receiver = new CallbackReceiver(); + int callbackPort = receiver.getPort(); + + // Kicking off background thread for listening to the audit logger callbacks + Thread receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver); + receiverThread.start(); + + String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); + securityJson = securityJson.replace("_PORT_", Integer.toString(callbackPort)); + securityJson = securityJson.replace("_ASYNC_", Boolean.toString(async)); + configureCluster(NUM_SERVERS)// nodes + .withSecurityJson(securityJson) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + cluster.waitForAllNodes(10); - private void createCollection(String collectionName) throws IOException { - assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + collectionName + "&numShards=" + NUM_SHARDS).second().intValue()); - cluster.waitForActiveCollection(collectionName, NUM_SHARDS, REPLICATION_FACTOR); + CloudSolrClient client = cluster.getSolrClient(); + client.request(CollectionAdminRequest.Create.createCollection(COLLECTION, 1, 1)); + client.query(COLLECTION, params("q", "*:*")); + + if (async) Thread.sleep(1000); // Allow for async callbacks to arrive + + assertEquals(3, receiver.getTotalCount()); + assertEquals(1, receiver.getCountForPath("/select")); + assertEquals(1, receiver.getCountForPath("/admin/collections")); + assertEquals(1, receiver.getCountForPath("/admin/cores")); + receiverThread.interrupt(); + receiver.close(); + shutdownCluster(); } + /** + * Listening for socket callbacks in background thread from the custom CallbackAuditLoggerPlugin + */ private class CallbackReceiver implements Runnable, AutoCloseable { private final ServerSocket serverSocket; private AtomicInteger count = new AtomicInteger(); - private AtomicInteger errors = new AtomicInteger(); - + private Map resourceCounts = new HashMap<>(); + public CallbackReceiver() throws IOException { serverSocket = new ServerSocket(0); } - public int getCount() { + public int getTotalCount() { return count.get(); } - public int getErrors() { - return errors.get(); + public int getCountForPath(String path) { + return resourceCounts.getOrDefault(path, new AtomicInteger()).get(); } public int getPort() { @@ -139,18 +118,26 @@ public int getPort() { @Override public void run() { + log.info("Listening for audit callbacks on on port {}", serverSocket.getLocalPort()); try { Socket socket = serverSocket.accept(); - InputStream is = socket.getInputStream(); - DataInputStream dis = new DataInputStream(is); - int b; - while (true) { - b = dis.readByte(); - log.info("Callback=" + b); + BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); + while (!Thread.currentThread().isInterrupted()) { + if (!reader.ready()) continue; + + String r = reader.readLine(); + log.info("Received audit event for path " + r); count.incrementAndGet(); + AtomicInteger resourceCounter = resourceCounts.get(r); + if (resourceCounter == null) { + resourceCounter = new AtomicInteger(1); + resourceCounts.put(r, resourceCounter); + } else { + resourceCounter.incrementAndGet(); + } } } catch (IOException e) { - log.info("Socket closed"); + log.info("Socket closed", e); } } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index fe907b52855e..72e758976003 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -21,11 +21,11 @@ import java.util.Date; import java.util.HashMap; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; import org.junit.Before; import org.junit.Test; -public class AuditLoggerPluginTest extends LuceneTestCase { +public class AuditLoggerPluginTest extends SolrTestCaseJ4 { protected static final Date SAMPLE_DATE = new Date(1234567890); protected static final AuditEvent EVENT_ANONYMOUS = new AuditEvent(AuditEvent.EventType.ANONYMOUS) .setHttpMethod("GET") diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java index f5dce9f317cf..b5d1242f9b18 100644 --- a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -25,6 +25,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Special test-only audit logger which will send the path (e.g. /select) as a callback to the running test + */ public class CallbackAuditLoggerPlugin extends AuditLoggerPlugin { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private int callbackPort; @@ -37,8 +40,9 @@ public class CallbackAuditLoggerPlugin extends AuditLoggerPlugin { */ @Override public void audit(AuditEvent event) { - log.info("Received audit event, type={}", event.getEventType()); - out.append('l'); + out.write(event.getResource() + "\n"); + out.flush(); + log.info("Sent audit callback {} to localhost:{}", event.getResource(), callbackPort); } @Override @@ -46,8 +50,7 @@ public void init(Map pluginConfig) { super.init(pluginConfig); callbackPort = Integer.parseInt((String) pluginConfig.get("callbackPort")); try { - socket = new Socket("127.0.0.1", callbackPort); -// socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), callbackPort)); + socket = new Socket("localhost", callbackPort); out = new PrintWriter(socket.getOutputStream(), true); } catch (IOException e) { throw new RuntimeException(e); @@ -56,6 +59,7 @@ public void init(Map pluginConfig) { @Override public void close() throws IOException { + super.close(); socket.close(); } } diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index bcc42201c9df..792894a49eae 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -21,12 +21,11 @@ import java.util.HashMap; import java.util.Map; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrResourceLoader; -import org.junit.After; import org.junit.Test; -public class MultiDestinationAuditLoggerTest extends LuceneTestCase { +public class MultiDestinationAuditLoggerTest extends SolrTestCaseJ4 { @Test public void init() throws IOException { MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index ec79d44d3f15..1c1357018df5 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -17,20 +17,18 @@ package org.apache.solr.security; -import java.io.Closeable; import java.io.IOException; import java.util.HashMap; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; -import org.junit.After; import org.junit.Before; import org.junit.Test; import static org.apache.solr.security.AuditLoggerPluginTest.EVENT_ANONYMOUS; import static org.apache.solr.security.AuditLoggerPluginTest.EVENT_AUTHENTICATED; -public class SolrLogAuditLoggerPluginTest extends LuceneTestCase { +public class SolrLogAuditLoggerPluginTest extends SolrTestCaseJ4 { private SolrLogAuditLoggerPlugin plugin; private HashMap config; diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 1856fa46460c..1c5f045a21d0 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -43,54 +43,51 @@ By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHO == Configuration in security.json Audit logging is configured in `security.json` under the `auditlogging` key. -The example `security.json` below configures audit logging to Solr default log file, enabling logging of all available event types. +The example `security.json` below configures synchronous audit logging to Solr default log file. [source,json] ---- { "auditlogging":{ - "class":"solr.SolrLogAuditLoggerPlugin", - "eventTypes": ["AUTHENTICATED", "REJECTED", "ANONYMOUS", - "ANONYMOUS_REJECTED", "AUTHORIZED", "UNAUTHORIZED", - "COMPLETED", "ERROR"] + "class": "solr.SolrLogAuditLoggerPlugin" } } ---- -=== Chaining multiple loggers -Using the `MultiDestinationAuditLogger` you can configure multiple audit logger plugins in a chain, to log to multiple destinations, as follows: +To make audit logging happen asynchronously in the backgroun, add the parameter `async: true`. This will cause the events to be put on a queue for asynchronous logging by one or more background threads. You may optionally also configure queue size, number of threads and whether it should block when the queue is full or discard events. [source,json] ---- { "auditlogging":{ - "class" : "solr.MultiDestinationAuditLogger", - "plugins" : [ - { "class" : "solr.SolrLogAuditLoggerPlugin" }, - { "class" : "solr.MyOtherAuditPlugin", - "customParam" : "value" - } - ] + "class": "solr.SolrLogAuditLoggerPlugin", + "async": true, + "blockAsync" : false, + "numThreads" : 2, + "queueSize" : 4096 } } ---- -=== Synchronous vs asynchronous audit logging -AuditLoggerPlugin developers can choose to make audit logging asynchronous by subclassing the `AsyncAuditLoggerPlugin` base class intead of the normal `AuditLoggerPlugin`. This will cause the event to be put on a queue and for asynchronous loggging by a background thread. For Audit loggers with async support, you can also configure queue size, number of threads and whether it should block when the queue is full: +The defaults are `async: false`, `blockAsync: false`, `queueSize: 4096` and `numThreads: 2`. + +=== Chaining multiple loggers +Using the `MultiDestinationAuditLogger` you can configure multiple audit logger plugins in a chain, to log to multiple destinations, as follows: [source,json] ---- { "auditlogging":{ - "class" : "solr.MyAsyncAuditLogger", - "blockAsync" : false, - "numThreads" : 2, - "queueSize" : 4096 + "class" : "solr.MultiDestinationAuditLogger", + "plugins" : [ + { "class" : "solr.SolrLogAuditLoggerPlugin" }, + { "class" : "solr.MyOtherAuditPlugin", + "customParam" : "value" + } + ] } } ---- -It is not possible to enable async audit logging unless the plugin extends the `AsyncAuditLoggerPlugin`. - == Metrics AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. \ No newline at end of file From d17aaef8b995cbc28d8cc83ca8394b712646ba86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 13:10:25 +0100 Subject: [PATCH 26/65] Graceful shutdown of auditlogger background thread, allows to emty queue --- .../src/java/org/apache/solr/core/CoreContainer.java | 1 - .../org/apache/solr/security/AuditLoggerPlugin.java | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 0535508ecacb..7a4d43108c30 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -382,7 +382,6 @@ private void initializeAuditloggerPlugin(Map auditConf) { auditloggerPlugin = new SecurityPluginHolder<>(readVersion(auditConf), getResourceLoader().newInstance(klas, AuditLoggerPlugin.class)); - // Read and pass the authorization context to the plugin auditloggerPlugin.plugin.init(auditConf); } else { log.debug("Security conf doesn't exist. Skipping setup for audit logging module."); diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 4c6b45162605..ec9c198c6705 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -28,6 +28,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; @@ -276,9 +277,16 @@ public String formatEvent(AuditEvent event) { @Override public void close() throws IOException { - closed = true; if (executorService != null) { - executorService.shutdownNow(); + log.info("Shutting down async Auditlogger background thread(s)"); + executorService.shutdown(); + try { + executorService.awaitTermination(20, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.info("Auditlogger background threads did not complete work in 20 seconds, queue size is {}. Forcing termination.", queue.size()); + executorService.shutdownNow(); + } } + closed = true; } } From 149a6f08baea3a2222c1871d2effd38e2686a74b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 14:46:24 +0100 Subject: [PATCH 27/65] MultiDestination logger always synchronous Fix draining of queue at shutdown, new test using delay Add class name to metrics path Simplify Integration test to not create collection, speeds up test --- .../solr/security/AuditLoggerPlugin.java | 44 ++++++++++--------- .../security/MultiDestinationAuditLogger.java | 17 ++++--- .../security/auditlog_plugin_security.json | 3 +- .../security/AuditLoggerIntegrationTest.java | 28 +++++++----- .../solr/security/AuditLoggerPluginTest.java | 7 ++- .../security/CallbackAuditLoggerPlugin.java | 9 ++++ .../MultiDestinationAuditLoggerTest.java | 3 ++ .../SolrLogAuditLoggerPluginTest.java | 17 ++++--- 8 files changed, 83 insertions(+), 45 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index ec9c198c6705..722283e74af3 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -64,10 +64,10 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo private static final int DEFAULT_QUEUE_SIZE = 4096; private static final int DEFAULT_NUM_THREADS = 2; - private BlockingQueue queue; - private boolean async; - private boolean blockAsync; - private int blockingQueueSize; + BlockingQueue queue; + boolean async; + boolean blockAsync; + int blockingQueueSize; protected AuditEventFormatter formatter; MetricRegistry registry; @@ -172,7 +172,8 @@ public final void auditAsync(AuditEvent event) { public void run() { while (!closed && !Thread.currentThread().isInterrupted()) { try { - AuditEvent event = queue.take(); + AuditEvent event = queue.poll(1000, TimeUnit.MILLISECONDS); + if (event == null) continue; Timer.Context timer = requestTimes.time(); audit(event); numLogged.mark(); @@ -209,15 +210,16 @@ public void setFormatter(AuditEventFormatter formatter) { public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, final String scope) { this.metricManager = manager; this.registryName = registryName; + String className = this.getClass().getSimpleName(); // Metrics registry = manager.registry(registryName); - numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope); - numLogged = manager.meter(this, registryName, "logged", getCategory().toString(), scope); - requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope); - totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope); - manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueCapacity", true, "queueCapacity", getCategory().toString()); - manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString()); - manager.registerGauge(this, registryName, () -> async,"async", true, "async", getCategory().toString()); + numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope, className); + numLogged = manager.meter(this, registryName, "count", getCategory().toString(), scope, className); + requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope, className); + totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope, className); + manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueCapacity", true, "queueCapacity", getCategory().toString(), scope, className); + manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString(), scope, className); + manager.registerGauge(this, registryName, () -> async,"async", true, "async", getCategory().toString(), scope, className); metricNames.addAll(Arrays.asList("errors", "logged", "requestTimes", "totalTime", "queueCapacity", "queueSize", "async")); } @@ -278,15 +280,17 @@ public String formatEvent(AuditEvent event) { @Override public void close() throws IOException { if (executorService != null) { - log.info("Shutting down async Auditlogger background thread(s)"); - executorService.shutdown(); - try { - executorService.awaitTermination(20, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.info("Auditlogger background threads did not complete work in 20 seconds, queue size is {}. Forcing termination.", queue.size()); - executorService.shutdownNow(); + int timeSlept = 0; + while (!queue.isEmpty() && timeSlept < 30) { + try { + log.info("Async auditlogger queue still has {} elements, sleeping to let it drain...", queue.size()); + Thread.sleep(1000); + timeSlept ++; + } catch (InterruptedException e) {} } + closed = true; + log.info("Shutting down async Auditlogger background thread(s)"); + executorService.shutdownNow(); } - closed = true; } } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 734b2d3d6adb..181ac64ad65c 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; @@ -49,16 +50,17 @@ public class MultiDestinationAuditLogger extends AuditLoggerPlugin implements Re private static final String PARAM_PLUGINS = "plugins"; private ResourceLoader loader; List plugins = new ArrayList<>(); + List pluginNames = new ArrayList<>(); /** - * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. + * Passes the AuditEvent to all sub plugins in parallel. The event should be a {@link AuditEvent} to be able to pull context info. * @param event the audit event */ @Override public void audit(AuditEvent event) { - plugins.forEach(plugin -> { - log.debug("Passing auditEvent to plugin {}", plugin.getClass().getName()); - plugin.audit(event); + log.debug("Passing auditEvent to plugins {}", pluginNames); + plugins.parallelStream().forEach(plugin -> { + plugin.doAudit(event); }); } @@ -69,6 +71,10 @@ public void audit(AuditEvent event) { @Override public void init(Map pluginConfig) { super.init(pluginConfig); + if (async) { + log.warn(MultiDestinationAuditLogger.class.getName() + " cannot run in async mode"); + async = false; + } if (!pluginConfig.containsKey(PARAM_PLUGINS)) { log.warn("No plugins configured"); } else { @@ -76,11 +82,12 @@ public void init(Map pluginConfig) { List> pluginList = (List>) pluginConfig.get(PARAM_PLUGINS); pluginList.forEach(pluginConf -> plugins.add(createPlugin(pluginConf))); pluginConfig.remove(PARAM_PLUGINS); + pluginNames = plugins.stream().map(AuditLoggerPlugin::getName).collect(Collectors.toList()); } if (pluginConfig.size() > 0) { log.error("Plugin config was not fully consumed. Remaining parameters are {}", pluginConfig); } - log.info("Initialized {} audit plugins", plugins.size()); + log.info("Initialized {} audit plugins: {}", plugins.size(), pluginNames); } @Override diff --git a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json index 6bbddfe05346..7d9305699b58 100644 --- a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json +++ b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json @@ -2,6 +2,7 @@ "auditlogging": { "class": "solr.CallbackAuditLoggerPlugin", "callbackPort": "_PORT_", - "async": _ASYNC_ + "async": _ASYNC_, + "delay": "_DELAY_" } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index c9fbb1018b15..b7e104ce73e0 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -51,15 +51,20 @@ public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { @Test public void testSynchronous() throws Exception { - doTest(false); + doTest(false, 0); } @Test public void testAsync() throws Exception { - doTest(true); + doTest(true, 0); + } + + @Test + public void testAsyncWithQueue() throws Exception { + doTest(true, 100); } - void doTest(boolean async) throws Exception { + void doTest(boolean async, int delay) throws Exception { CallbackReceiver receiver = new CallbackReceiver(); int callbackPort = receiver.getPort(); @@ -70,6 +75,7 @@ void doTest(boolean async) throws Exception { String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); securityJson = securityJson.replace("_PORT_", Integer.toString(callbackPort)); securityJson = securityJson.replace("_ASYNC_", Boolean.toString(async)); + securityJson = securityJson.replace("_DELAY_", Integer.toString(delay)); configureCluster(NUM_SERVERS)// nodes .withSecurityJson(securityJson) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) @@ -78,18 +84,16 @@ void doTest(boolean async) throws Exception { cluster.waitForAllNodes(10); CloudSolrClient client = cluster.getSolrClient(); - client.request(CollectionAdminRequest.Create.createCollection(COLLECTION, 1, 1)); - client.query(COLLECTION, params("q", "*:*")); - - if (async) Thread.sleep(1000); // Allow for async callbacks to arrive + CollectionAdminRequest.listCollections(client); + client.request(CollectionAdminRequest.getClusterStatus()); + client.request(CollectionAdminRequest.getOverseerStatus()); + shutdownCluster(); assertEquals(3, receiver.getTotalCount()); - assertEquals(1, receiver.getCountForPath("/select")); - assertEquals(1, receiver.getCountForPath("/admin/collections")); - assertEquals(1, receiver.getCountForPath("/admin/cores")); + assertEquals(3, receiver.getCountForPath("/admin/collections")); + receiverThread.interrupt(); receiver.close(); - shutdownCluster(); } /** @@ -118,8 +122,8 @@ public int getPort() { @Override public void run() { - log.info("Listening for audit callbacks on on port {}", serverSocket.getLocalPort()); try { + log.info("Listening for audit callbacks on on port {}", serverSocket.getLocalPort()); Socket socket = serverSocket.accept(); BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); while (!Thread.currentThread().isInterrupted()) { diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 72e758976003..3e7301d19f38 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.SolrTestCaseJ4; import org.junit.Before; @@ -75,6 +76,7 @@ public void setUp() throws Exception { super.setUp(); plugin = new MockAuditLoggerPlugin(); config = new HashMap<>(); + config.put("async", false); plugin.init(config); } @@ -82,6 +84,7 @@ public void setUp() throws Exception { public void init() { config = new HashMap<>(); config.put("eventTypes", Arrays.asList("REJECTED")); + config.put("async", false); plugin.init(config); assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); assertFalse(plugin.shouldLog(EVENT_UNAUTHORIZED.getEventType())); @@ -103,8 +106,8 @@ public void shouldLog() { public void audit() { plugin.doAudit(EVENT_ANONYMOUS_REJECTED); plugin.doAudit(EVENT_REJECTED); - assertEquals(1, plugin.typeCounts.get("ANONYMOUS_REJECTED").get()); - assertEquals(1, plugin.typeCounts.get("REJECTED").get()); + assertEquals(1, plugin.typeCounts.getOrDefault("ANONYMOUS_REJECTED", new AtomicInteger()).get()); + assertEquals(1, plugin.typeCounts.getOrDefault("REJECTED", new AtomicInteger()).get()); assertEquals(2, plugin.events.size()); } diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java index b5d1242f9b18..778f73fc6944 100644 --- a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -33,6 +33,7 @@ public class CallbackAuditLoggerPlugin extends AuditLoggerPlugin { private int callbackPort; private Socket socket; private PrintWriter out; + private int delay; /** * Opens a socket to send a callback, e.g. to a running test client @@ -40,6 +41,13 @@ public class CallbackAuditLoggerPlugin extends AuditLoggerPlugin { */ @Override public void audit(AuditEvent event) { + if (delay > 0) { + log.info("Sleeping for {}ms before sending callback", delay); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + } + } out.write(event.getResource() + "\n"); out.flush(); log.info("Sent audit callback {} to localhost:{}", event.getResource(), callbackPort); @@ -49,6 +57,7 @@ public void audit(AuditEvent event) { public void init(Map pluginConfig) { super.init(pluginConfig); callbackPort = Integer.parseInt((String) pluginConfig.get("callbackPort")); + delay = Integer.parseInt((String) pluginConfig.get("delay")); try { socket = new Socket("localhost", callbackPort); out = new PrintWriter(socket.getOutputStream(), true); diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 792894a49eae..85d45636df37 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -31,13 +31,16 @@ public void init() throws IOException { MultiDestinationAuditLogger al = new MultiDestinationAuditLogger(); Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); + config.put("async", false); ArrayList> plugins = new ArrayList>(); Map conf1 = new HashMap<>(); conf1.put("class", "solr.SolrLogAuditLoggerPlugin"); + conf1.put("async", false); plugins.add(conf1); Map conf2 = new HashMap<>(); conf2.put("class", "solr.MockAuditLoggerPlugin"); + conf2.put("async", false); plugins.add(conf2); config.put("plugins", plugins); diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 1c1357018df5..840e252fa510 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -22,6 +22,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -32,34 +33,40 @@ public class SolrLogAuditLoggerPluginTest extends SolrTestCaseJ4 { private SolrLogAuditLoggerPlugin plugin; private HashMap config; + @Override @Before public void setUp() throws Exception { super.setUp(); plugin = new SolrLogAuditLoggerPlugin(); config = new HashMap<>(); - plugin.init(config); - closeAfterTest(plugin); + config.put("async", false); } @Test(expected = SolrException.class) public void badConfig() throws IOException { - plugin.close(); - plugin = new SolrLogAuditLoggerPlugin(); - config = new HashMap<>(); config.put("invalid", "parameter"); plugin.init(config); } @Test public void audit() { + plugin.init(config); plugin.doAudit(EVENT_ANONYMOUS); } @Test public void eventFormatter() { + plugin.init(config); assertEquals("type=\"ANONYMOUS\" message=\"Anonymous\" method=\"GET\" username=\"null\" resource=\"/collection1\" collections=null", plugin.formatter.formatEvent(EVENT_ANONYMOUS)); assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" username=\"Jan\" resource=\"/collection1\" collections=null", plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + plugin.close(); + } } \ No newline at end of file From 78cbaf36e00e76a5bc2a8bae440f84dcb067a5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 14:54:43 +0100 Subject: [PATCH 28/65] MultiDestination plugin properly close its children --- .../solr/security/AuditLoggerPlugin.java | 3 ++- .../security/MultiDestinationAuditLogger.java | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 722283e74af3..ab5d85ecd178 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -208,9 +208,10 @@ public void setFormatter(AuditEventFormatter formatter) { @Override public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, final String scope) { + String className = this.getClass().getSimpleName(); + log.debug("Initializing metrics for {}", className); this.metricManager = manager; this.registryName = registryName; - String className = this.getClass().getSimpleName(); // Metrics registry = manager.registry(registryName); numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope, className); diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 181ac64ad65c..224054b73385 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -16,6 +16,7 @@ */ package org.apache.solr.security; +import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; @@ -25,6 +26,7 @@ import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; import org.apache.solr.common.SolrException; +import org.apache.solr.metrics.SolrMetricManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,4 +116,22 @@ private AuditLoggerPlugin createPlugin(Map auditConf) { public void inform(ResourceLoader loader) { this.loader = loader; } + + @Override + public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { + super.initializeMetrics(manager, registryName, tag, scope); + plugins.forEach(p -> p.initializeMetrics(manager, registryName, tag, scope)); + } + + @Override + public void close() throws IOException { + super.close(); + plugins.forEach(p -> { + try { + p.close(); + } catch (IOException e) { + log.error("Exception trying to close {}", p.getName()); + } + }); + } } From 28ca24b97e582f670b7de882855133facf3c7b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 15:02:30 +0100 Subject: [PATCH 29/65] Fix precommit --- .../apache/solr/security/AuditLoggerIntegrationTest.java | 2 +- .../org/apache/solr/security/CallbackAuditLoggerPlugin.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index b7e104ce73e0..4c3f8df51f09 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -125,7 +125,7 @@ public void run() { try { log.info("Listening for audit callbacks on on port {}", serverSocket.getLocalPort()); Socket socket = serverSocket.accept(); - BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); + BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)); while (!Thread.currentThread().isInterrupted()) { if (!reader.ready()) continue; diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java index 778f73fc6944..fbe6fdae8d1e 100644 --- a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -16,10 +16,14 @@ */ package org.apache.solr.security; +import java.io.BufferedWriter; import java.io.IOException; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.net.Socket; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Map; import org.slf4j.Logger; @@ -60,7 +64,7 @@ public void init(Map pluginConfig) { delay = Integer.parseInt((String) pluginConfig.get("delay")); try { socket = new Socket("localhost", callbackPort); - out = new PrintWriter(socket.getOutputStream(), true); + out = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8), true); } catch (IOException e) { throw new RuntimeException(e); } From 5b152aac1143888717d118b841237a91810f35fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 15:45:38 +0100 Subject: [PATCH 30/65] Add test for audit logging metrics --- .../security/AuditLoggerIntegrationTest.java | 2 + .../security/CallbackAuditLoggerPlugin.java | 2 - .../solr/cloud/SolrCloudAuthTestCase.java | 46 ++++++++++++++----- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 4c3f8df51f09..14d52eb2b0a6 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -88,6 +88,8 @@ void doTest(boolean async, int delay) throws Exception { client.request(CollectionAdminRequest.getClusterStatus()); client.request(CollectionAdminRequest.getOverseerStatus()); + assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + shutdownCluster(); assertEquals(3, receiver.getTotalCount()); assertEquals(3, receiver.getCountForPath("/admin/collections")); diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java index fbe6fdae8d1e..9c6b31dd6936 100644 --- a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -16,13 +16,11 @@ */ package org.apache.solr.security; -import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.net.Socket; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Map; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java index 4a0e6ed06766..9254b58bad87 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java @@ -59,12 +59,15 @@ public class SolrCloudAuthTestCase extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final List AUTH_METRICS_KEYS = Arrays.asList("errors", "requests", "authenticated", "passThrough", "failWrongCredentials", "failMissingCredentials", "requestTimes", "totalTime"); - private static final List AUTH_METRICS_METER_KEYS = Arrays.asList("errors"); + private static final List AUTH_METRICS_METER_KEYS = Arrays.asList("errors", "count"); private static final List AUTH_METRICS_TIMER_KEYS = Collections.singletonList("requestTimes"); private static final String METRICS_PREFIX_PKI = "SECURITY./authentication/pki."; private static final String METRICS_PREFIX = "SECURITY./authentication."; public static final Predicate NOT_NULL_PREDICATE = o -> o != null; - + private static final List AUDIT_METRICS_KEYS = Arrays.asList("count"); + private static final List AUTH_METRICS_TO_COMPARE = Arrays.asList("requests", "authenticated", "passThrough", "failWrongCredentials", "failMissingCredentials", "errors"); + private static final List AUDIT_METRICS_TO_COMPARE = Arrays.asList("count"); + /** * Used to check metric counts for PKI auth */ @@ -85,7 +88,7 @@ protected void assertAuthMetricsMinimums(int requests, int authenticated, int pa * Common test method to be able to check security from any authentication plugin * @param prefix the metrics key prefix, currently "SECURITY./authentication." for basic auth and "SECURITY./authentication/pki." for PKI */ - Map countAuthMetrics(String prefix) { + Map countSecurityMetrics(String prefix, List keys) { List> metrics = new ArrayList<>(); cluster.getJettySolrRunners().forEach(r -> { MetricRegistry registry = r.getCoreContainer().getMetricManager().registry("solr.node"); @@ -94,14 +97,14 @@ Map countAuthMetrics(String prefix) { }); Map counts = new HashMap<>(); - AUTH_METRICS_KEYS.forEach(k -> { + keys.forEach(k -> { counts.put(k, sumCount(prefix, k, metrics)); }); return counts; } /** - * Common test method to be able to check security from any authentication plugin + * Common test method to be able to check auth metrics from any authentication plugin * @param prefix the metrics key prefix, currently "SECURITY./authentication." for basic auth and "SECURITY./authentication/pki." for PKI */ private void assertAuthMetricsMinimums(String prefix, int requests, int authenticated, int passThrough, int failWrongCredentials, int failMissingCredentials, int errors) throws InterruptedException { @@ -113,13 +116,13 @@ private void assertAuthMetricsMinimums(String prefix, int requests, int authenti expectedCounts.put("failMissingCredentials", (long) failMissingCredentials); expectedCounts.put("errors", (long) errors); - Map counts = countAuthMetrics(prefix); - boolean success = isMetricsEqualOrLarger(expectedCounts, counts); + Map counts = countSecurityMetrics(prefix, AUTH_METRICS_KEYS); + boolean success = isMetricsEqualOrLarger(AUTH_METRICS_TO_COMPARE, expectedCounts, counts); if (!success) { log.info("First metrics count assert failed, pausing 2s before re-attempt"); Thread.sleep(2000); - counts = countAuthMetrics(prefix); - success = isMetricsEqualOrLarger(expectedCounts, counts); + counts = countSecurityMetrics(prefix, AUTH_METRICS_KEYS); + success = isMetricsEqualOrLarger(AUTH_METRICS_TO_COMPARE, expectedCounts, counts); } assertTrue("Expected metric minimums for prefix " + prefix + ": " + expectedCounts + ", but got: " + counts, success); @@ -130,8 +133,29 @@ private void assertAuthMetricsMinimums(String prefix, int requests, int authenti } } - private boolean isMetricsEqualOrLarger(Map expectedCounts, Map actualCounts) { - return Stream.of("requests", "authenticated", "passThrough", "failWrongCredentials", "failMissingCredentials", "errors") + /** + * Common test method to be able to check audit metrics + * @param className the class name to be used for composing prefix, e.g. "SECURITY./auditlogging/SolrLogAuditLoggerPlugin" + */ + protected void assertAuditMetricsMinimums(String className, int count, int errors) throws InterruptedException { + String prefix = "SECURITY./auditlogging." + className + "."; + Map expectedCounts = new HashMap<>(); + expectedCounts.put("count", (long) count); + + Map counts = countSecurityMetrics(prefix, AUDIT_METRICS_KEYS); + boolean success = isMetricsEqualOrLarger(AUDIT_METRICS_TO_COMPARE, expectedCounts, counts); + if (!success) { + log.info("First metrics count assert failed, pausing 2s before re-attempt"); + Thread.sleep(2000); + counts = countSecurityMetrics(prefix, AUDIT_METRICS_KEYS); + success = isMetricsEqualOrLarger(AUDIT_METRICS_TO_COMPARE, expectedCounts, counts); + } + + assertTrue("Expected metric minimums for prefix " + prefix + ": " + expectedCounts + ", but got: " + counts, success); + } + + private boolean isMetricsEqualOrLarger(List metricsToCompare, Map expectedCounts, Map actualCounts) { + return metricsToCompare.stream() .allMatch(k -> actualCounts.get(k).intValue() >= expectedCounts.get(k).intValue()); } From faea5e836a0dbc2e90d1aad37bf707aafdbd5663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 15:51:59 +0100 Subject: [PATCH 31/65] Fix precommit --- .../src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java index 9254b58bad87..2550f32406c7 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.Predicate; -import java.util.stream.Stream; import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; From e12becb833ce108c09b1913081faa766b240f2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 15:55:09 +0100 Subject: [PATCH 32/65] List JWT plugin in docs --- solr/solr-ref-guide/src/securing-solr.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index c761551d9aa7..fc4d25abc10b 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -38,6 +38,7 @@ Authentication makes sure you know the identity of your users. Supported authent * <> * <> * <> +* <> === Authorization plugins Authorization makes sure that only users with the necessary roles/permissions can access any given resource. From fadc8b521bb428e40a48c91f58b0a06787da751e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 16:15:43 +0100 Subject: [PATCH 33/65] Fix refguide warnings --- solr/solr-ref-guide/src/audit-logging.adoc | 1 + solr/solr-ref-guide/src/securing-solr.adoc | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 1c5f045a21d0..c52615fe2b78 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -22,6 +22,7 @@ Audit loggers are pluggable to suit any possible format or log destination. [quote] An audit trail (also called audit log) is a security-relevant chronological record, set of records, and/or destination and source of records that provide documentary evidence of the sequence of activities that have affected at any time a specific operation, procedure, or event. (https://en.wikipedia.org/wiki/Audit_trail[Wikipedia]) +[#audit-event-types] == Event types These are the event types triggered by the framework: diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index fc4d25abc10b..909319a33809 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -32,6 +32,7 @@ and enabling any of these will immediately take effect across the whole cluster. Read the chapter <> to learn how to work with the `security.json` file. +[#securing-solr-auth-plugins] === Authentication plugins Authentication makes sure you know the identity of your users. Supported authentication plugins are: @@ -50,7 +51,7 @@ The authorization plugins shipping with Solr are: Audit logging will record an audit trail of important events in your cluster, such as users being authenticated, or access being denied to admin APIs. Learn more about audit logging and how to implement an audit logger plugin here: -* <> +* <> == Securing Zookeeper traffic Zookeeper is a central and important part of a SolrCloud cluster and understanding how to secure From 4d2ec72cff57c8a6209a15b9d135c6ffd6da788a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 17:37:29 +0100 Subject: [PATCH 34/65] Make the audit() method protected, since it will only ever be called from the parent --- .../src/java/org/apache/solr/security/AuditLoggerPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index ab5d85ecd178..91d9e6d119ed 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -120,10 +120,10 @@ public void init(Map pluginConfig) { } /** - * Audits an event. The event should be a {@link AuditEvent} to be able to pull context info. + * This is the method that each Audit plugin has to implement to do the actual logging. * @param event the audit event */ - public abstract void audit(AuditEvent event); + protected abstract void audit(AuditEvent event); /** * Called by the framework, and takes care of metrics From b2d3b635743cd20fc204fa207f2079090f888892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 18 Mar 2019 18:19:19 +0100 Subject: [PATCH 35/65] Make the auditAsync() method protected, since it will only ever be called internally --- .../src/java/org/apache/solr/security/AuditLoggerPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 91d9e6d119ed..8f2fd28d8490 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -150,7 +150,7 @@ public final void doAudit(AuditEvent event) { * A background thread will pull events from this queue and call {@link #audit(AuditEvent)} * @param event the audit event */ - public final void auditAsync(AuditEvent event) { + protected final void auditAsync(AuditEvent event) { if (blockAsync) { try { queue.put(event); From aa2424c9d128a18915888a82d3b3b7134ec7e0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 09:55:22 +0100 Subject: [PATCH 36/65] More integration tests and validation of events --- .../org/apache/solr/security/AuditEvent.java | 38 ++- .../solr/security/AuditLoggerPlugin.java | 3 +- .../security/SolrLogAuditLoggerPlugin.java | 3 + .../org/apache/solr/servlet/HttpSolrCall.java | 14 +- .../security/auditlog_plugin_security.json | 2 +- .../security/AuditLoggerIntegrationTest.java | 216 ++++++++++++++++-- .../solr/security/AuditLoggerPluginTest.java | 4 +- .../security/CallbackAuditLoggerPlugin.java | 4 +- .../SolrLogAuditLoggerPluginTest.java | 4 +- 9 files changed, 247 insertions(+), 41 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 52c46c66042d..a29ebe946ef5 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -33,6 +33,7 @@ import org.slf4j.MDC; import static org.apache.solr.security.AuditEvent.EventType.ANONYMOUS; +import static org.apache.solr.security.AuditEvent.EventType.ERROR; /** * Audit event that takes request and auth context as input to be able to audit log custom things. @@ -53,7 +54,7 @@ public class AuditEvent { private List collections; private Map context; private HashMap headers; - private Map solrParams; + private Map solrParams = new HashMap<>(); private String solrHost; private int solrPort; private String solrIp; @@ -98,6 +99,15 @@ public enum EventType { public AuditEvent(EventType eventType) { this.date = new Date(); this.eventType = eventType; + if (EventType.REJECTED == this.eventType) { + this.status = 401; + } + if (EventType.UNAUTHORIZED == this.eventType) { + this.status = 403; + } + if (EventType.ERROR == this.eventType) { + this.status = 500; + } this.level = eventType.level; this.message = eventType.message; } @@ -106,6 +116,10 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { this(eventType, null, httpRequest); } + // Constructor for testing only + protected AuditEvent() { + } + /** * Event based on an HttpServletRequest, typically used during authentication. * Solr will fill in details such as ip, http method etc from the request, and @@ -133,7 +147,7 @@ public AuditEvent(EventType eventType, Throwable exception, HttpServletRequest h this.requestType = AuthorizationContext.RequestType.WRITE.name(); } - setException(exception); + if (exception != null) setException(exception); Principal principal = httpRequest.getUserPrincipal(); if (principal != null) { @@ -159,8 +173,9 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza .stream().map(r -> r.collectionName).collect(Collectors.toList()); this.resource = authorizationContext.getResource(); this.requestType = authorizationContext.getRequestType().toString(); - // TODO: Insert params??? - //authorizationContext.getParams().toMap(this.solrParams); + authorizationContext.getParams().forEach(p -> { + this.solrParams.put(p.getKey(), p.getValue()); + }); } /** @@ -275,6 +290,14 @@ public Map getSolrParams() { return solrParams; } + public String getSolrParamAsString(String key) { + Object v = getSolrParams().get(key); + if (v instanceof List && ((List) v).size() > 0) { + return String.valueOf(((List) v).get(0)); + } + return null; + } + public AuthorizationResponse getAutResponse() { return autResponse; } @@ -287,7 +310,7 @@ public String getRequestType() { return requestType; } - public long getStatus() { + public int getStatus() { return status; } @@ -406,6 +429,11 @@ public void setStatus(int status) { public void setException(Throwable exception) { this.exception = exception; + if (exception != null) { + this.eventType = ERROR; + this.level = ERROR.level; + this.message = ERROR.message; + } setStatus(statusFromException(exception)); } diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 8f2fd28d8490..05b290fe1c71 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -126,7 +126,8 @@ public void init(Map pluginConfig) { protected abstract void audit(AuditEvent event); /** - * Called by the framework, and takes care of metrics + * Called by the framework, and takes care of metrics tracking and to dispatch + * to either synchronous or async logging. */ public final void doAudit(AuditEvent event) { if (async) { diff --git a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java index 55c594556961..2268d43e096d 100644 --- a/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/SolrLogAuditLoggerPlugin.java @@ -44,8 +44,11 @@ public void init(Map pluginConfig) { .append("type=\"").append(event.getEventType().name()).append("\"") .append(" message=\"").append(event.getMessage()).append("\"") .append(" method=\"").append(event.getHttpMethod()).append("\"") + .append(" status=\"").append(event.getStatus()).append("\"") + .append(" requestType=\"").append(event.getRequestType()).append("\"") .append(" username=\"").append(event.getUsername()).append("\"") .append(" resource=\"").append(event.getResource()).append("\"") + .append(" queryString=\"").append(event.getHttpQueryString()).append("\"") .append(" collections=").append(event.getCollections()).toString()); if (pluginConfig.size() > 0) { throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Plugin config was not fully consumed. Remaining parameters are " + pluginConfig); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 108ee36ec26f..c39bb773e927 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -528,8 +528,12 @@ public Action call() throws IOException { */ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp)); execute(solrRsp); - if (shouldAudit(EventType.COMPLETED)) { - cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); + if (shouldAudit()) { + EventType eventType = solrRsp.getException() == null ? EventType.COMPLETED : EventType.ERROR; + if (shouldAudit(eventType)) { + cores.getAuditLoggerPlugin().doAudit( + new AuditEvent(eventType, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrRsp.getException())); + } } HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod); Iterator> headers = solrRsp.httpHeaders(); @@ -567,8 +571,12 @@ public Action call() throws IOException { } + private boolean shouldAudit() { + return cores.getAuditLoggerPlugin() != null; + } + private boolean shouldAudit(AuditEvent.EventType eventType) { - return cores.getAuditLoggerPlugin() != null && cores.getAuditLoggerPlugin().shouldLog(eventType); + return shouldAudit() && cores.getAuditLoggerPlugin().shouldLog(eventType); } private boolean shouldAuthorize() { diff --git a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json index 7d9305699b58..18fc02a1a1e7 100644 --- a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json +++ b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json @@ -4,5 +4,5 @@ "callbackPort": "_PORT_", "async": _ASYNC_, "delay": "_DELAY_" - } + }_AUTH_ } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 14d52eb2b0a6..f35ccb38171c 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -23,20 +23,40 @@ import java.net.ServerSocket; import java.net.Socket; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.SolrCloudAuthTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.util.DefaultSolrThreadFactory; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.getClusterStatus; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.getOverseerStatus; +import static org.apache.solr.security.AuditEvent.EventType.COMPLETED; +import static org.apache.solr.security.AuditEvent.EventType.ERROR; +import static org.apache.solr.security.AuditEvent.EventType.REJECTED; +import static org.apache.solr.security.AuditEvent.EventType.UNAUTHORIZED; + + /** * Validate that audit logging works in a live cluster */ @@ -47,53 +67,187 @@ public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SERVERS = 1; protected static final int NUM_SHARDS = 1; protected static final int REPLICATION_FACTOR = 1; - private final String COLLECTION = "auditCollection"; + private CallbackReceiver receiver; + private int callbackPort; + private Thread receiverThread; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + receiver = new CallbackReceiver(); + callbackPort = receiver.getPort(); + receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver); + receiverThread.start(); + } @Test public void testSynchronous() throws Exception { - doTest(false, 0); + setupCluster(false, 0); + runAdminCommands(); + assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + shutdownCluster(); + assertThreeAdminEvents(receiver); } - + @Test public void testAsync() throws Exception { - doTest(true, 0); + setupCluster(true, 0); + runAdminCommands(); + assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + shutdownCluster(); + assertThreeAdminEvents(receiver); } @Test public void testAsyncWithQueue() throws Exception { - doTest(true, 100); + setupCluster(true, 100); + runAdminCommands(); + assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + shutdownCluster(); + assertThreeAdminEvents(receiver); } - - void doTest(boolean async, int delay) throws Exception { - CallbackReceiver receiver = new CallbackReceiver(); - int callbackPort = receiver.getPort(); - // Kicking off background thread for listening to the audit logger callbacks - Thread receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver); - receiverThread.start(); + @Test + public void searchWithException() throws Exception { + setupCluster(false, 0); + try { + cluster.getSolrClient().request(CollectionAdminRequest.createCollection("test", 1, 1)); + cluster.getSolrClient().query("test", new MapSolrParams(Collections.singletonMap("q", "a(bc"))); + fail("Query should fail"); + } catch (SolrException ex) { + waitForAuditEventCallbacks(3); + assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/cores"); + assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections"); + assertAuditEvent(receiver.popEvent(), ERROR,"/select", "READ", null, 500); + } + } + + @Test + public void auth() throws Exception { + setupCluster(false, 0, true); + CloudSolrClient client = cluster.getSolrClient(); + try { + CollectionAdminRequest.List request = new CollectionAdminRequest.List(); + client.request(request); + request.setBasicAuthCredentials("solr", "SolrRocks"); + client.request(request); + CollectionAdminRequest.Create createRequest = CollectionAdminRequest.createCollection("test", 1, 1); + client.request(createRequest); + fail("Call should fail with 401"); + } catch (SolrException ex) { + waitForAuditEventCallbacks(3); + assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "action", "LIST"); + assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "ADMIN", "solr", 0, "action", "LIST"); + assertAuditEvent(receiver.popEvent(), REJECTED, "/admin/collections", "ADMIN", null,401); + } + try { + CollectionAdminRequest.Create createRequest = CollectionAdminRequest.createCollection("test", 1, 1); + createRequest.setBasicAuthCredentials("solr", "wrongPW"); + client.request(createRequest); + fail("Call should fail with 403"); + } catch (SolrException ex) { + waitForAuditEventCallbacks(1); + assertAuditEvent(receiver.popEvent(), UNAUTHORIZED, "/admin/collections", "ADMIN", null,403); + } + } + + private void assertAuditEvent(AuditEvent e, AuditEvent.EventType type, String path, String... params) { + assertAuditEvent(e, type, path, null, null,null, params); + } + + private void assertAuditEvent(AuditEvent e, AuditEvent.EventType type, String path, String requestType, String username, Integer status, String... params) { + assertEquals(type, e.getEventType()); + assertEquals(path, e.getResource()); + if (requestType != null) { + assertEquals(requestType, e.getRequestType()); + } + if (username != null) { + assertEquals(username, e.getUsername()); + } + if (status != null) { + assertEquals(status.intValue(), e.getStatus()); + } + if (params != null && params.length > 0) { + List p = new LinkedList<>(Arrays.asList(params)); + while (p.size() >= 2) { + String val = e.getSolrParamAsString(p.get(0)); + assertEquals(p.get(1), val); + p.remove(0); + p.remove(0); + } + } + } + private void waitForAuditEventCallbacks(int number) throws InterruptedException { + while(receiver.count.get() < number) { + Thread.sleep(100); + } + } + + private void runAdminCommands() throws IOException, SolrServerException { + SolrClient client = cluster.getSolrClient(); + CollectionAdminRequest.listCollections(client); + client.request(getClusterStatus()); + client.request(getOverseerStatus()); + } + + private void assertThreeAdminEvents(CallbackReceiver receiver) { + assertEquals(3, receiver.getTotalCount()); + assertEquals(3, receiver.getCountForPath("/admin/collections")); + + AuditEvent e = receiver.getHistory().pop(); + assertEquals(COMPLETED, e.getEventType()); + assertEquals("GET", e.getHttpMethod()); + assertEquals("action=LIST&wt=javabin&version=2", e.getHttpQueryString()); + assertEquals("LIST", e.getSolrParamAsString("action")); + assertEquals("javabin", e.getSolrParamAsString("wt")); + + e = receiver.getHistory().pop(); + assertEquals(COMPLETED, e.getEventType()); + assertEquals("GET", e.getHttpMethod()); + assertEquals("CLUSTERSTATUS", e.getSolrParamAsString("action")); + + e = receiver.getHistory().pop(); + assertEquals(COMPLETED, e.getEventType()); + assertEquals("GET", e.getHttpMethod()); + assertEquals("OVERSEERSTATUS", e.getSolrParamAsString("action")); + } + + private void setupCluster(boolean async, int delay) throws Exception { + setupCluster(async, delay, false); + } + + private static String AUTH_SECTION = ",\n" + + " \"authentication\":{\n" + + " \"blockUnknown\":\"false\",\n" + + " \"class\":\"solr.BasicAuthPlugin\",\n" + + " \"credentials\":{\"solr\":\"orwp2Ghgj39lmnrZOTm7Qtre1VqHFDfwAEzr0ApbN3Y= Ju5osoAqOX8iafhWpPP01E5P+sg8tK8tHON7rCYZRRw=\"}},\n" + + " \"authorization\":{\n" + + " \"class\":\"solr.RuleBasedAuthorizationPlugin\",\n" + + " \"user-role\":{\"solr\":\"admin\"},\n" + + " \"permissions\":[{\"name\":\"collection-admin-edit\",\"role\":\"admin\"}]\n" + + " }\n"; + + void setupCluster(boolean async, int delay, boolean enableAuth) throws Exception { String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); securityJson = securityJson.replace("_PORT_", Integer.toString(callbackPort)); securityJson = securityJson.replace("_ASYNC_", Boolean.toString(async)); securityJson = securityJson.replace("_DELAY_", Integer.toString(delay)); + securityJson = securityJson.replace("_AUTH_", enableAuth ? AUTH_SECTION : ""); configureCluster(NUM_SERVERS)// nodes .withSecurityJson(securityJson) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) .configure(); cluster.waitForAllNodes(10); + } - CloudSolrClient client = cluster.getSolrClient(); - CollectionAdminRequest.listCollections(client); - client.request(CollectionAdminRequest.getClusterStatus()); - client.request(CollectionAdminRequest.getOverseerStatus()); - - assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); - - shutdownCluster(); - assertEquals(3, receiver.getTotalCount()); - assertEquals(3, receiver.getCountForPath("/admin/collections")); - + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + shutdownCluster(); receiverThread.interrupt(); receiver.close(); } @@ -105,6 +259,7 @@ private class CallbackReceiver implements Runnable, AutoCloseable { private final ServerSocket serverSocket; private AtomicInteger count = new AtomicInteger(); private Map resourceCounts = new HashMap<>(); + private LinkedList history = new LinkedList<>(); public CallbackReceiver() throws IOException { serverSocket = new ServerSocket(0); @@ -130,8 +285,11 @@ public void run() { BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)); while (!Thread.currentThread().isInterrupted()) { if (!reader.ready()) continue; - - String r = reader.readLine(); + ObjectMapper om = new ObjectMapper(); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + AuditEvent event = om.readValue(reader.readLine(), AuditEvent.class); + history.add(event); + String r = event.getResource(); log.info("Received audit event for path " + r); count.incrementAndGet(); AtomicInteger resourceCounter = resourceCounts.get(r); @@ -151,5 +309,13 @@ public void run() { public void close() throws Exception { serverSocket.close(); } + + public LinkedList getHistory() { + return history; + } + + public AuditEvent popEvent() { + return history.pop(); + } } } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 3e7301d19f38..d15af07570f2 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -113,9 +113,9 @@ public void audit() { @Test public void jsonEventFormatter() { - assertEquals("{\"message\":\"Anonymous\",\"level\":\"INFO\",\"date\":" + SAMPLE_DATE.getTime() + ",\"solrPort\":0,\"resource\":\"/collection1\",\"httpMethod\":\"GET\",\"eventType\":\"ANONYMOUS\",\"status\":-1,\"qtime\":-1.0}", + assertEquals("{\"message\":\"Anonymous\",\"level\":\"INFO\",\"date\":" + SAMPLE_DATE.getTime() + ",\"solrParams\":{},\"solrPort\":0,\"resource\":\"/collection1\",\"httpMethod\":\"GET\",\"eventType\":\"ANONYMOUS\",\"status\":-1,\"qtime\":-1.0}", plugin.formatter.formatEvent(EVENT_ANONYMOUS)); - assertEquals("{\"message\":\"Authenticated\",\"level\":\"INFO\",\"date\":1234567890,\"username\":\"Jan\",\"solrPort\":0,\"resource\":\"/collection1\",\"httpMethod\":\"GET\",\"eventType\":\"AUTHENTICATED\",\"status\":-1,\"qtime\":-1.0}", + assertEquals("{\"message\":\"Authenticated\",\"level\":\"INFO\",\"date\":" + SAMPLE_DATE.getTime() + ",\"username\":\"Jan\",\"solrParams\":{},\"solrPort\":0,\"resource\":\"/collection1\",\"httpMethod\":\"GET\",\"eventType\":\"AUTHENTICATED\",\"status\":-1,\"qtime\":-1.0}", plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); } diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java index 9c6b31dd6936..872112982589 100644 --- a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -50,9 +50,9 @@ public void audit(AuditEvent event) { } catch (InterruptedException e) { } } - out.write(event.getResource() + "\n"); + out.write(formatter.formatEvent(event) + "\n"); out.flush(); - log.info("Sent audit callback {} to localhost:{}", event.getResource(), callbackPort); + log.info("Sent audit callback {} to localhost:{}", formatter.formatEvent(event), callbackPort); } @Override diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 840e252fa510..233b12e97c1a 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -57,9 +57,9 @@ public void audit() { @Test public void eventFormatter() { plugin.init(config); - assertEquals("type=\"ANONYMOUS\" message=\"Anonymous\" method=\"GET\" username=\"null\" resource=\"/collection1\" collections=null", + assertEquals("type=\"ANONYMOUS\" message=\"Anonymous\" method=\"GET\" requestType=\"null\" username=\"null\" resource=\"/collection1\" params=\"null\" collections=null", plugin.formatter.formatEvent(EVENT_ANONYMOUS)); - assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" username=\"Jan\" resource=\"/collection1\" collections=null", + assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" requestType=\"null\" username=\"Jan\" resource=\"/collection1\" params=\"null\" collections=null", plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); } From c6c8196b169a65a6135cc398f6b59cc0cd0f2506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 11:33:22 +0100 Subject: [PATCH 37/65] Only override status code if exception is set --- solr/core/src/java/org/apache/solr/security/AuditEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index a29ebe946ef5..e0f1a2f9bf18 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -433,8 +433,8 @@ public void setException(Throwable exception) { this.eventType = ERROR; this.level = ERROR.level; this.message = ERROR.message; + setStatus(statusFromException(exception)); } - setStatus(statusFromException(exception)); } } From d168d77ff3fe41e413b13913ac431e50e5d49aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 13:08:57 +0100 Subject: [PATCH 38/65] Fix some tests Always initialize status codes, use 200=OK. -1=undefined Make integration test beastable, with tests not sharing cluster etc --- .../org/apache/solr/security/AuditEvent.java | 49 ++----- .../security/AuditLoggerIntegrationTest.java | 138 +++++++++++------- .../SolrLogAuditLoggerPluginTest.java | 4 +- .../solr/cloud/SolrCloudAuthTestCase.java | 14 +- 4 files changed, 109 insertions(+), 96 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index e0f1a2f9bf18..0cfb7057fdf6 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -70,23 +70,25 @@ public class AuditEvent { /* Predefined event types. Custom types can be made through constructor */ public enum EventType { - AUTHENTICATED("Authenticated", "User successfully authenticated", Level.INFO), - REJECTED("Rejected", "Authentication request rejected", Level.WARN), - ANONYMOUS("Anonymous", "Request proceeds with unknown user", Level.INFO), - ANONYMOUS_REJECTED("AnonymousRejected", "Request from unknown user rejected", Level.WARN), - AUTHORIZED("Authorized", "Authorization succeeded", Level.INFO), - UNAUTHORIZED("Unauthorized", "Authorization failed", Level.WARN), - COMPLETED("Completed", "Request completed", Level.INFO), - ERROR("Error", "Request was not executed due to an error", Level.ERROR); + AUTHENTICATED("Authenticated", "User successfully authenticated", Level.INFO, -1), + REJECTED("Rejected", "Authentication request rejected", Level.WARN, 401), + ANONYMOUS("Anonymous", "Request proceeds with unknown user", Level.INFO, -1), + ANONYMOUS_REJECTED("AnonymousRejected", "Request from unknown user rejected", Level.WARN, 401), + AUTHORIZED("Authorized", "Authorization succeeded", Level.INFO, -1), + UNAUTHORIZED("Unauthorized", "Authorization failed", Level.WARN, 403), + COMPLETED("Completed", "Request completed", Level.INFO, 200), + ERROR("Error", "Request was not executed due to an error", Level.ERROR, 500); private final String message; private String explanation; private final Level level; + private int status; - EventType(String message, String explanation, Level level) { + EventType(String message, String explanation, Level level, int status) { this.message = message; this.explanation = explanation; this.level = level; + this.status = status; } } @@ -99,15 +101,7 @@ public enum EventType { public AuditEvent(EventType eventType) { this.date = new Date(); this.eventType = eventType; - if (EventType.REJECTED == this.eventType) { - this.status = 401; - } - if (EventType.UNAUTHORIZED == this.eventType) { - this.status = 403; - } - if (EventType.ERROR == this.eventType) { - this.status = 500; - } + this.status = eventType.status; this.level = eventType.level; this.message = eventType.message; } @@ -117,8 +111,7 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { } // Constructor for testing only - protected AuditEvent() { - } + protected AuditEvent() { } /** * Event based on an HttpServletRequest, typically used during authentication. @@ -194,18 +187,6 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza setException(exception); } - private int statusFromException(Throwable exception) { - int status = 0; - if (exception != null ) { - if (exception instanceof SolrException) - status = ((SolrException)exception).code(); - else - status = 500; - } - return status; - } - - private HashMap getHeadersFromRequest(HttpServletRequest httpRequest) { HashMap h = new HashMap<>(); Enumeration headersEnum = httpRequest.getHeaderNames(); @@ -433,8 +414,8 @@ public void setException(Throwable exception) { this.eventType = ERROR; this.level = ERROR.level; this.message = ERROR.message; - setStatus(statusFromException(exception)); + if (exception instanceof SolrException) + status = ((SolrException)exception).code(); } } - } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index f35ccb38171c..3dd14b388112 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -39,6 +39,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudAuthTestCase; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; @@ -56,7 +57,6 @@ import static org.apache.solr.security.AuditEvent.EventType.REJECTED; import static org.apache.solr.security.AuditEvent.EventType.UNAUTHORIZED; - /** * Validate that audit logging works in a live cluster */ @@ -67,66 +67,70 @@ public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SERVERS = 1; protected static final int NUM_SHARDS = 1; protected static final int REPLICATION_FACTOR = 1; - private CallbackReceiver receiver; - private int callbackPort; - private Thread receiverThread; + // Use a harness per thread to be able to beast this test + ThreadLocal testHarness = new ThreadLocal<>(); @Override @Before public void setUp() throws Exception { super.setUp(); - receiver = new CallbackReceiver(); - callbackPort = receiver.getPort(); - receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver); - receiverThread.start(); + testHarness.set(new AuditTestHarness()); } + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + testHarness.get().close(); + } + @Test public void testSynchronous() throws Exception { - setupCluster(false, 0); + setupCluster(false, 0, false); runAdminCommands(); - assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); - shutdownCluster(); - assertThreeAdminEvents(receiver); + assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + testHarness.get().shutdownCluster(); + assertThreeAdminEvents(); } @Test public void testAsync() throws Exception { - setupCluster(true, 0); + setupCluster(true, 0, false); runAdminCommands(); - assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); - shutdownCluster(); - assertThreeAdminEvents(receiver); + assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + testHarness.get().shutdownCluster(); + assertThreeAdminEvents(); } @Test public void testAsyncWithQueue() throws Exception { - setupCluster(true, 100); + setupCluster(true, 100, false); runAdminCommands(); - assertAuditMetricsMinimums(CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); - shutdownCluster(); - assertThreeAdminEvents(receiver); + assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + testHarness.get().shutdownCluster(); + assertThreeAdminEvents(); } @Test public void searchWithException() throws Exception { - setupCluster(false, 0); + setupCluster(false, 0, false); try { - cluster.getSolrClient().request(CollectionAdminRequest.createCollection("test", 1, 1)); - cluster.getSolrClient().query("test", new MapSolrParams(Collections.singletonMap("q", "a(bc"))); + testHarness.get().cluster.getSolrClient().request(CollectionAdminRequest.createCollection("test", 1, 1)); + testHarness.get().cluster.getSolrClient().query("test", new MapSolrParams(Collections.singletonMap("q", "a(bc"))); fail("Query should fail"); } catch (SolrException ex) { waitForAuditEventCallbacks(3); + CallbackReceiver receiver = testHarness.get().receiver; assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/cores"); assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections"); - assertAuditEvent(receiver.popEvent(), ERROR,"/select", "READ", null, 500); + assertAuditEvent(receiver.popEvent(), ERROR,"/select", "READ", null, 400); } } @Test public void auth() throws Exception { setupCluster(false, 0, true); - CloudSolrClient client = cluster.getSolrClient(); + CloudSolrClient client = testHarness.get().cluster.getSolrClient(); try { CollectionAdminRequest.List request = new CollectionAdminRequest.List(); client.request(request); @@ -137,8 +141,9 @@ public void auth() throws Exception { fail("Call should fail with 401"); } catch (SolrException ex) { waitForAuditEventCallbacks(3); + CallbackReceiver receiver = testHarness.get().receiver; assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "action", "LIST"); - assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "ADMIN", "solr", 0, "action", "LIST"); + assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "ADMIN", "solr", 200, "action", "LIST"); assertAuditEvent(receiver.popEvent(), REJECTED, "/admin/collections", "ADMIN", null,401); } try { @@ -148,6 +153,7 @@ public void auth() throws Exception { fail("Call should fail with 403"); } catch (SolrException ex) { waitForAuditEventCallbacks(1); + CallbackReceiver receiver = testHarness.get().receiver; assertAuditEvent(receiver.popEvent(), UNAUTHORIZED, "/admin/collections", "ADMIN", null,403); } } @@ -180,44 +186,45 @@ private void assertAuditEvent(AuditEvent e, AuditEvent.EventType type, String pa } private void waitForAuditEventCallbacks(int number) throws InterruptedException { - while(receiver.count.get() < number) { - Thread.sleep(100); + CallbackReceiver receiver = testHarness.get().receiver; + int count = 0; + while(receiver.buffer.size() < number) { + Thread.sleep(100); + if (++count >= 10) fail("Failed waiting for " + number + " callbacks"); } } private void runAdminCommands() throws IOException, SolrServerException { - SolrClient client = cluster.getSolrClient(); + SolrClient client = testHarness.get().cluster.getSolrClient(); CollectionAdminRequest.listCollections(client); client.request(getClusterStatus()); client.request(getOverseerStatus()); } - private void assertThreeAdminEvents(CallbackReceiver receiver) { + private void assertThreeAdminEvents() throws Exception { + CallbackReceiver receiver = testHarness.get().receiver; + waitForAuditEventCallbacks(3); assertEquals(3, receiver.getTotalCount()); assertEquals(3, receiver.getCountForPath("/admin/collections")); - AuditEvent e = receiver.getHistory().pop(); + AuditEvent e = receiver.getBuffer().pop(); assertEquals(COMPLETED, e.getEventType()); assertEquals("GET", e.getHttpMethod()); assertEquals("action=LIST&wt=javabin&version=2", e.getHttpQueryString()); assertEquals("LIST", e.getSolrParamAsString("action")); assertEquals("javabin", e.getSolrParamAsString("wt")); - e = receiver.getHistory().pop(); + e = receiver.getBuffer().pop(); assertEquals(COMPLETED, e.getEventType()); assertEquals("GET", e.getHttpMethod()); assertEquals("CLUSTERSTATUS", e.getSolrParamAsString("action")); - e = receiver.getHistory().pop(); + e = receiver.getBuffer().pop(); assertEquals(COMPLETED, e.getEventType()); assertEquals("GET", e.getHttpMethod()); assertEquals("OVERSEERSTATUS", e.getSolrParamAsString("action")); } - private void setupCluster(boolean async, int delay) throws Exception { - setupCluster(async, delay, false); - } - private static String AUTH_SECTION = ",\n" + " \"authentication\":{\n" + " \"blockUnknown\":\"false\",\n" + @@ -231,26 +238,19 @@ private void setupCluster(boolean async, int delay) throws Exception { void setupCluster(boolean async, int delay, boolean enableAuth) throws Exception { String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); - securityJson = securityJson.replace("_PORT_", Integer.toString(callbackPort)); + securityJson = securityJson.replace("_PORT_", Integer.toString(testHarness.get().callbackPort)); securityJson = securityJson.replace("_ASYNC_", Boolean.toString(async)); securityJson = securityJson.replace("_DELAY_", Integer.toString(delay)); securityJson = securityJson.replace("_AUTH_", enableAuth ? AUTH_SECTION : ""); - configureCluster(NUM_SERVERS)// nodes + MiniSolrCloudCluster myCluster = new Builder(NUM_SERVERS, createTempDir()) .withSecurityJson(securityJson) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .configure(); + .build(); - cluster.waitForAllNodes(10); + myCluster.waitForAllNodes(10); + testHarness.get().setCluster(myCluster); } - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - shutdownCluster(); - receiverThread.interrupt(); - receiver.close(); - } /** * Listening for socket callbacks in background thread from the custom CallbackAuditLoggerPlugin @@ -259,7 +259,7 @@ private class CallbackReceiver implements Runnable, AutoCloseable { private final ServerSocket serverSocket; private AtomicInteger count = new AtomicInteger(); private Map resourceCounts = new HashMap<>(); - private LinkedList history = new LinkedList<>(); + private LinkedList buffer = new LinkedList<>(); public CallbackReceiver() throws IOException { serverSocket = new ServerSocket(0); @@ -288,7 +288,7 @@ public void run() { ObjectMapper om = new ObjectMapper(); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); AuditEvent event = om.readValue(reader.readLine(), AuditEvent.class); - history.add(event); + buffer.add(event); String r = event.getResource(); log.info("Received audit event for path " + r); count.incrementAndGet(); @@ -310,12 +310,42 @@ public void close() throws Exception { serverSocket.close(); } - public LinkedList getHistory() { - return history; + public LinkedList getBuffer() { + return buffer; } public AuditEvent popEvent() { - return history.pop(); + return buffer.pop(); + } + } + + private class AuditTestHarness implements AutoCloseable { + CallbackReceiver receiver; + int callbackPort; + Thread receiverThread; + private MiniSolrCloudCluster cluster; + + public AuditTestHarness() throws IOException { + receiver = new CallbackReceiver(); + callbackPort = receiver.getPort(); + receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver);; + receiverThread.start(); + } + + @Override + public void close() throws Exception { + shutdownCluster(); + receiverThread.interrupt(); + receiver.close(); + receiverThread = null; + } + + public void shutdownCluster() throws Exception { + cluster.shutdown(); + } + + public void setCluster(MiniSolrCloudCluster cluster) { + this.cluster = cluster; } } } diff --git a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java index 233b12e97c1a..40df3da0d437 100644 --- a/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/SolrLogAuditLoggerPluginTest.java @@ -57,9 +57,9 @@ public void audit() { @Test public void eventFormatter() { plugin.init(config); - assertEquals("type=\"ANONYMOUS\" message=\"Anonymous\" method=\"GET\" requestType=\"null\" username=\"null\" resource=\"/collection1\" params=\"null\" collections=null", + assertEquals("type=\"ANONYMOUS\" message=\"Anonymous\" method=\"GET\" status=\"-1\" requestType=\"null\" username=\"null\" resource=\"/collection1\" queryString=\"null\" collections=null", plugin.formatter.formatEvent(EVENT_ANONYMOUS)); - assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" requestType=\"null\" username=\"Jan\" resource=\"/collection1\" params=\"null\" collections=null", + assertEquals("type=\"AUTHENTICATED\" message=\"Authenticated\" method=\"GET\" status=\"-1\" requestType=\"null\" username=\"Jan\" resource=\"/collection1\" queryString=\"null\" collections=null", plugin.formatter.formatEvent(EVENT_AUTHENTICATED)); } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java index 2550f32406c7..3baee2c6bb4f 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java @@ -85,9 +85,11 @@ protected void assertAuthMetricsMinimums(int requests, int authenticated, int pa /** * Common test method to be able to check security from any authentication plugin + * @param cluster the MiniSolrCloudCluster to fetch metrics from * @param prefix the metrics key prefix, currently "SECURITY./authentication." for basic auth and "SECURITY./authentication/pki." for PKI + * @param keys what keys to examine */ - Map countSecurityMetrics(String prefix, List keys) { + Map countSecurityMetrics(MiniSolrCloudCluster cluster, String prefix, List keys) { List> metrics = new ArrayList<>(); cluster.getJettySolrRunners().forEach(r -> { MetricRegistry registry = r.getCoreContainer().getMetricManager().registry("solr.node"); @@ -115,12 +117,12 @@ private void assertAuthMetricsMinimums(String prefix, int requests, int authenti expectedCounts.put("failMissingCredentials", (long) failMissingCredentials); expectedCounts.put("errors", (long) errors); - Map counts = countSecurityMetrics(prefix, AUTH_METRICS_KEYS); + Map counts = countSecurityMetrics(cluster, prefix, AUTH_METRICS_KEYS); boolean success = isMetricsEqualOrLarger(AUTH_METRICS_TO_COMPARE, expectedCounts, counts); if (!success) { log.info("First metrics count assert failed, pausing 2s before re-attempt"); Thread.sleep(2000); - counts = countSecurityMetrics(prefix, AUTH_METRICS_KEYS); + counts = countSecurityMetrics(cluster, prefix, AUTH_METRICS_KEYS); success = isMetricsEqualOrLarger(AUTH_METRICS_TO_COMPARE, expectedCounts, counts); } @@ -136,17 +138,17 @@ private void assertAuthMetricsMinimums(String prefix, int requests, int authenti * Common test method to be able to check audit metrics * @param className the class name to be used for composing prefix, e.g. "SECURITY./auditlogging/SolrLogAuditLoggerPlugin" */ - protected void assertAuditMetricsMinimums(String className, int count, int errors) throws InterruptedException { + protected void assertAuditMetricsMinimums(MiniSolrCloudCluster cluster, String className, int count, int errors) throws InterruptedException { String prefix = "SECURITY./auditlogging." + className + "."; Map expectedCounts = new HashMap<>(); expectedCounts.put("count", (long) count); - Map counts = countSecurityMetrics(prefix, AUDIT_METRICS_KEYS); + Map counts = countSecurityMetrics(cluster, prefix, AUDIT_METRICS_KEYS); boolean success = isMetricsEqualOrLarger(AUDIT_METRICS_TO_COMPARE, expectedCounts, counts); if (!success) { log.info("First metrics count assert failed, pausing 2s before re-attempt"); Thread.sleep(2000); - counts = countSecurityMetrics(prefix, AUDIT_METRICS_KEYS); + counts = countSecurityMetrics(cluster, prefix, AUDIT_METRICS_KEYS); success = isMetricsEqualOrLarger(AUDIT_METRICS_TO_COMPARE, expectedCounts, counts); } From 30fe6575fb76d416c4993d80901f9e0bb0c82398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 13:25:02 +0100 Subject: [PATCH 39/65] Do not initialize queue-size gauges for non-async instances --- .../apache/solr/security/AuditLoggerPlugin.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 05b290fe1c71..a407aff4acf0 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -110,8 +110,8 @@ public void init(Map pluginConfig) { pluginConfig.remove(PARAM_BLOCKASYNC); pluginConfig.remove(PARAM_QUEUE_SIZE); pluginConfig.remove(PARAM_NUM_THREADS); - queue = new ArrayBlockingQueue<>(blockingQueueSize); if (async) { + queue = new ArrayBlockingQueue<>(blockingQueueSize); executorService = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrjNamedThreadFactory("audit")); executorService.submit(this); } @@ -152,6 +152,7 @@ public final void doAudit(AuditEvent event) { * @param event the audit event */ protected final void auditAsync(AuditEvent event) { + assert(async); if (blockAsync) { try { queue.put(event); @@ -171,6 +172,7 @@ protected final void auditAsync(AuditEvent event) { */ @Override public void run() { + assert(async); while (!closed && !Thread.currentThread().isInterrupted()) { try { AuditEvent event = queue.poll(1000, TimeUnit.MILLISECONDS); @@ -219,9 +221,11 @@ public void initializeMetrics(SolrMetricManager manager, String registryName, St numLogged = manager.meter(this, registryName, "count", getCategory().toString(), scope, className); requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope, className); totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope, className); - manager.registerGauge(this, registryName, () -> blockingQueueSize,"queueCapacity", true, "queueCapacity", getCategory().toString(), scope, className); - manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(),"queueSize", true, "queueSize", getCategory().toString(), scope, className); - manager.registerGauge(this, registryName, () -> async,"async", true, "async", getCategory().toString(), scope, className); + if (async) { + manager.registerGauge(this, registryName, () -> blockingQueueSize, "queueCapacity", true, "queueCapacity", getCategory().toString(), scope, className); + manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(), "queueSize", true, "queueSize", getCategory().toString(), scope, className); + } + manager.registerGauge(this, registryName, () -> async, "async", true, "async", getCategory().toString(), scope, className); metricNames.addAll(Arrays.asList("errors", "logged", "requestTimes", "totalTime", "queueCapacity", "queueSize", "async")); } @@ -281,7 +285,7 @@ public String formatEvent(AuditEvent event) { @Override public void close() throws IOException { - if (executorService != null) { + if (async && executorService != null) { int timeSlept = 0; while (!queue.isEmpty() && timeSlept < 30) { try { From e36cb900c40e7f1376014b08b149a04700190fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 13:42:29 +0100 Subject: [PATCH 40/65] Fix bug in shouldLog() --- .../org/apache/solr/security/AuditLoggerPlugin.java | 2 +- .../solr/security/MultiDestinationAuditLogger.java | 2 +- .../solr/security/MultiDestinationAuditLoggerTest.java | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index a407aff4acf0..e236dfce3d3d 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -198,7 +198,7 @@ public void run() { * @return true if this event type should be logged */ public boolean shouldLog(EventType eventType) { - boolean shouldLog = eventTypes.contains(eventType.name()); + boolean shouldLog = eventTypes.contains(eventType); if (!shouldLog) { log.debug("Event type {} is not configured for audit logging", eventType.name()); } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 224054b73385..066a3fa854ae 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -94,7 +94,7 @@ public void init(Map pluginConfig) { @Override public boolean shouldLog(AuditEvent.EventType eventType) { - return plugins.stream().anyMatch(p -> p.shouldLog(eventType)); + return super.shouldLog(eventType) || plugins.stream().anyMatch(p -> p.shouldLog(eventType)); } private AuditLoggerPlugin createPlugin(Map auditConf) { diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 85d45636df37..5695166d0ee4 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -32,15 +33,18 @@ public void init() throws IOException { Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); config.put("async", false); + config.put("eventTypes", Arrays.asList(AuditEvent.EventType.COMPLETED)); ArrayList> plugins = new ArrayList>(); Map conf1 = new HashMap<>(); conf1.put("class", "solr.SolrLogAuditLoggerPlugin"); conf1.put("async", false); + conf1.put("eventTypes", Arrays.asList(AuditEvent.EventType.ANONYMOUS)); plugins.add(conf1); Map conf2 = new HashMap<>(); conf2.put("class", "solr.MockAuditLoggerPlugin"); conf2.put("async", false); + conf2.put("eventTypes", Arrays.asList(AuditEvent.EventType.AUTHENTICATED)); plugins.add(conf2); config.put("plugins", plugins); @@ -49,6 +53,12 @@ public void init() throws IOException { al.doAudit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); assertEquals(1, ((MockAuditLoggerPlugin)al.plugins.get(1)).events.size()); + + assertFalse(al.shouldLog(AuditEvent.EventType.ERROR)); + assertFalse(al.shouldLog(AuditEvent.EventType.UNAUTHORIZED)); + assertTrue(al.shouldLog(AuditEvent.EventType.COMPLETED)); + assertTrue(al.shouldLog(AuditEvent.EventType.ANONYMOUS)); + assertTrue(al.shouldLog(AuditEvent.EventType.AUTHENTICATED)); assertEquals(0, config.size()); al.close(); From b4533e2f4e8eb567d81121bf85af088a43f7e949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 14:06:17 +0100 Subject: [PATCH 41/65] Test eventTypes, force multiDestinationAuditLogger to synchronous, add tests --- .../org/apache/solr/security/AuditLoggerPlugin.java | 4 ++-- .../solr/security/MultiDestinationAuditLogger.java | 5 +---- .../solr/security/AuditLoggerIntegrationTest.java | 2 +- .../solr/security/MultiDestinationAuditLoggerTest.java | 10 ++++++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index e236dfce3d3d..bc2034ba528f 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -57,7 +57,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_EVENT_TYPES = "eventTypes"; - private static final String PARAM_ASYNC = "async"; + static final String PARAM_ASYNC = "async"; private static final String PARAM_BLOCKASYNC = "blockAsync"; private static final String PARAM_QUEUE_SIZE = "queueSize"; private static final String PARAM_NUM_THREADS = "numThreads"; @@ -198,7 +198,7 @@ public void run() { * @return true if this event type should be logged */ public boolean shouldLog(EventType eventType) { - boolean shouldLog = eventTypes.contains(eventType); + boolean shouldLog = eventTypes.contains(eventType.name()); if (!shouldLog) { log.debug("Event type {} is not configured for audit logging", eventType.name()); } diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 066a3fa854ae..692291eaac8f 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -72,11 +72,8 @@ public void audit(AuditEvent event) { */ @Override public void init(Map pluginConfig) { + pluginConfig.put(PARAM_ASYNC, false); // Force the multi plugin to synchronous operation super.init(pluginConfig); - if (async) { - log.warn(MultiDestinationAuditLogger.class.getName() + " cannot run in async mode"); - async = false; - } if (!pluginConfig.containsKey(PARAM_PLUGINS)) { log.warn("No plugins configured"); } else { diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 3dd14b388112..f95e003fe1b1 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -190,7 +190,7 @@ private void waitForAuditEventCallbacks(int number) throws InterruptedException int count = 0; while(receiver.buffer.size() < number) { Thread.sleep(100); - if (++count >= 10) fail("Failed waiting for " + number + " callbacks"); + if (++count >= 30) fail("Failed waiting for " + number + " callbacks"); } } diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 5695166d0ee4..5db373f357dc 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -33,22 +33,23 @@ public void init() throws IOException { Map config = new HashMap<>(); config.put("class", "solr.MultiDestinationAuditLogger"); config.put("async", false); - config.put("eventTypes", Arrays.asList(AuditEvent.EventType.COMPLETED)); + config.put("eventTypes", Arrays.asList(AuditEvent.EventType.COMPLETED.name())); ArrayList> plugins = new ArrayList>(); Map conf1 = new HashMap<>(); conf1.put("class", "solr.SolrLogAuditLoggerPlugin"); conf1.put("async", false); - conf1.put("eventTypes", Arrays.asList(AuditEvent.EventType.ANONYMOUS)); + conf1.put("eventTypes", Arrays.asList(AuditEvent.EventType.ANONYMOUS.name())); plugins.add(conf1); Map conf2 = new HashMap<>(); conf2.put("class", "solr.MockAuditLoggerPlugin"); conf2.put("async", false); - conf2.put("eventTypes", Arrays.asList(AuditEvent.EventType.AUTHENTICATED)); + conf2.put("eventTypes", Arrays.asList(AuditEvent.EventType.AUTHENTICATED.name())); plugins.add(conf2); config.put("plugins", plugins); - al.inform(new SolrResourceLoader()); + SolrResourceLoader loader = new SolrResourceLoader(); + al.inform(loader); al.init(config); al.doAudit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); @@ -62,6 +63,7 @@ public void init() throws IOException { assertEquals(0, config.size()); al.close(); + loader.close(); } @Test From de0b9b758623571dc68e075df3d34ff4785e7e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 14:14:19 +0100 Subject: [PATCH 42/65] Only distribute events of correct type to child loggers --- .../solr/security/MultiDestinationAuditLogger.java | 3 ++- .../solr/security/MultiDestinationAuditLoggerTest.java | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index 692291eaac8f..a5aa9401ffe5 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -62,7 +62,8 @@ public class MultiDestinationAuditLogger extends AuditLoggerPlugin implements Re public void audit(AuditEvent event) { log.debug("Passing auditEvent to plugins {}", pluginNames); plugins.parallelStream().forEach(plugin -> { - plugin.doAudit(event); + if (plugin.shouldLog(event.getEventType())) + plugin.doAudit(event); }); } diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 5db373f357dc..1cfc36ccfe8a 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -22,8 +22,10 @@ import java.util.HashMap; import java.util.Map; +import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrResourceLoader; +import org.junit.Before; import org.junit.Test; public class MultiDestinationAuditLoggerTest extends SolrTestCaseJ4 { @@ -53,7 +55,9 @@ public void init() throws IOException { al.init(config); al.doAudit(new AuditEvent(AuditEvent.EventType.ANONYMOUS).setUsername("me")); - assertEquals(1, ((MockAuditLoggerPlugin)al.plugins.get(1)).events.size()); + assertEquals(0, ((MockAuditLoggerPlugin)al.plugins.get(1)).events.size()); // not configured for ANONYMOUS + al.doAudit(new AuditEvent(AuditEvent.EventType.AUTHENTICATED).setUsername("me")); + assertEquals(1, ((MockAuditLoggerPlugin)al.plugins.get(1)).events.size()); // configured for authenticated assertFalse(al.shouldLog(AuditEvent.EventType.ERROR)); assertFalse(al.shouldLog(AuditEvent.EventType.UNAUTHORIZED)); @@ -62,6 +66,9 @@ public void init() throws IOException { assertTrue(al.shouldLog(AuditEvent.EventType.AUTHENTICATED)); assertEquals(0, config.size()); + + + al.close(); loader.close(); } From c785b6f46f05945781d06acee92cfcb156b32ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 14:22:11 +0100 Subject: [PATCH 43/65] Prevent nesting of MDALP. Call super.close after local close --- .../apache/solr/security/MultiDestinationAuditLogger.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java index a5aa9401ffe5..9aaae5c1403d 100644 --- a/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java +++ b/solr/core/src/java/org/apache/solr/security/MultiDestinationAuditLogger.java @@ -103,6 +103,9 @@ private AuditLoggerPlugin createPlugin(Map auditConf) { } log.info("Initializing auditlogger plugin: " + klas); AuditLoggerPlugin p = loader.newInstance(klas, AuditLoggerPlugin.class); + if (p.getClass().equals(this.getClass())) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Cannot nest MultiDestinationAuditLogger"); + } p.init(auditConf); return p; } else { @@ -123,7 +126,6 @@ public void initializeMetrics(SolrMetricManager manager, String registryName, St @Override public void close() throws IOException { - super.close(); plugins.forEach(p -> { try { p.close(); @@ -131,5 +133,6 @@ public void close() throws IOException { log.error("Exception trying to close {}", p.getName()); } }); + super.close(); } } From 434c9cc9deefd5b6e88df9e427fb22ae7f8f9c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 14:27:37 +0100 Subject: [PATCH 44/65] Handle COMPLETED/ERROR for admin response --- .../src/java/org/apache/solr/servlet/HttpSolrCall.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 4e4daf5da1aa..5f3c4c7c7205 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -754,8 +754,12 @@ private void handleAdminRequest() throws IOException { QueryResponseWriter respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS.get(solrReq.getParams().get(CommonParams.WT)); if (respWriter == null) respWriter = getResponseWriter(); writeResponse(solrResp, respWriter, Method.getMethod(req.getMethod())); - if (shouldAudit(EventType.COMPLETED)) { - cores.getAuditLoggerPlugin().doAudit(new AuditEvent(EventType.COMPLETED, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); + if (shouldAudit()) { + EventType eventType = solrResp.getException() == null ? EventType.COMPLETED : EventType.ERROR; + if (shouldAudit(eventType)) { + cores.getAuditLoggerPlugin().doAudit( + new AuditEvent(eventType, req, getAuthCtx(), solrReq.getRequestTimer().getTime(), solrResp.getException())); + } } } From b937b6d4e9b202c498523de86bd13d5060a6e002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 14:45:23 +0100 Subject: [PATCH 45/65] Fix close method --- .../org/apache/solr/security/AuditLoggerIntegrationTest.java | 2 +- .../apache/solr/security/MultiDestinationAuditLoggerTest.java | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index f95e003fe1b1..5e1ebe223731 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -80,8 +80,8 @@ public void setUp() throws Exception { @Override @After public void tearDown() throws Exception { - super.tearDown(); testHarness.get().close(); + super.tearDown(); } @Test diff --git a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java index 1cfc36ccfe8a..6b3a51f9a33b 100644 --- a/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java +++ b/solr/core/src/test/org/apache/solr/security/MultiDestinationAuditLoggerTest.java @@ -22,10 +22,8 @@ import java.util.HashMap; import java.util.Map; -import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrResourceLoader; -import org.junit.Before; import org.junit.Test; public class MultiDestinationAuditLoggerTest extends SolrTestCaseJ4 { @@ -67,8 +65,6 @@ public void init() throws IOException { assertEquals(0, config.size()); - - al.close(); loader.close(); } From c810b517f939f93cb342f230bdb5bd0aaaf215f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 22 Mar 2019 14:54:54 +0100 Subject: [PATCH 46/65] Update refguide docs with defaults for async etc --- solr/solr-ref-guide/src/audit-logging.adoc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index c52615fe2b78..3dd3b4cce9de 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -39,7 +39,7 @@ COMPLETED;Request completed successfully ERROR;Request was not executed due to an error |=== -By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. +By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. What eventTypes are logged can be configured with the `eventTypes` configuration parameter. == Configuration in security.json Audit logging is configured in `security.json` under the `auditlogging` key. @@ -55,7 +55,7 @@ The example `security.json` below configures synchronous audit logging to Solr d } ---- -To make audit logging happen asynchronously in the backgroun, add the parameter `async: true`. This will cause the events to be put on a queue for asynchronous logging by one or more background threads. You may optionally also configure queue size, number of threads and whether it should block when the queue is full or discard events. +By default any AuditLogger plugin configured will log asynchronously in the background to avoid slowing down the requests. To make audit logging happen synchronously, add the parameter `async: false`. For async logging, you may optionally also configure queue size, number of threads and whether it should block when the queue is full or discard events: [source,json] ---- @@ -65,12 +65,17 @@ To make audit logging happen asynchronously in the backgroun, add the parameter "async": true, "blockAsync" : false, "numThreads" : 2, - "queueSize" : 4096 + "queueSize" : 4096, + "eventTypes": ["REJECTED", "ANONYMOUS_REJECTED", "UNAUTHORIZED", "COMPLETED" "ERROR"] } } ---- -The defaults are `async: false`, `blockAsync: false`, `queueSize: 4096` and `numThreads: 2`. +The defaults are `async: true`, `blockAsync: false`, `queueSize: 4096` and `numThreads: 2`. + +Other parameters supported are: + + === Chaining multiple loggers Using the `MultiDestinationAuditLogger` you can configure multiple audit logger plugins in a chain, to log to multiple destinations, as follows: From 95bb111889355e78d0f78dbc58efec4b566e0158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 11:17:25 +0100 Subject: [PATCH 47/65] * Add muteRules for muting certain requests, collections, users, ip etc * New RequestType enum ADMIN, SEARCH, UPDATE, STREAMING, UNKNOWN always filled, using URL matching if necessary * New AuditEvent field requestUrl containing full URL from request * Add unit and integration tests --- .../org/apache/solr/security/AuditEvent.java | 105 +++++++++++++----- .../solr/security/AuditLoggerPlugin.java | 93 +++++++++++++++- .../security/auditlog_plugin_security.json | 5 +- .../security/AuditLoggerIntegrationTest.java | 44 +++++--- .../solr/security/AuditLoggerPluginTest.java | 52 ++++++++- .../solr/security/MockAuditLoggerPlugin.java | 4 - solr/solr-ref-guide/src/audit-logging.adoc | 81 ++++++++++---- 7 files changed, 314 insertions(+), 70 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 0cfb7057fdf6..67be61c21e43 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -19,11 +19,14 @@ import javax.servlet.http.HttpServletRequest; import java.lang.invoke.MethodHandles; import java.security.Principal; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.solr.common.SolrException; @@ -43,8 +46,8 @@ */ public class AuditEvent { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private StringBuffer requestUrl; private String nodeName; - private String message; private Level level; private Date date; @@ -53,7 +56,7 @@ public class AuditEvent { private String clientIp; private List collections; private Map context; - private HashMap headers; + private Map headers; private Map solrParams = new HashMap<>(); private String solrHost; private int solrPort; @@ -63,11 +66,11 @@ public class AuditEvent { private String httpQueryString; private EventType eventType; private AuthorizationResponse autResponse; - private String requestType; + private RequestType requestType; private double QTime = -1; private int status = -1; private Throwable exception; - + /* Predefined event types. Custom types can be made through constructor */ public enum EventType { AUTHENTICATED("Authenticated", "User successfully authenticated", Level.INFO, -1), @@ -79,10 +82,10 @@ public enum EventType { COMPLETED("Completed", "Request completed", Level.INFO, 200), ERROR("Error", "Request was not executed due to an error", Level.ERROR, 500); - private final String message; - private String explanation; - private final Level level; - private int status; + public final String message; + public String explanation; + public final Level level; + public int status; EventType(String message, String explanation, Level level, int status) { this.message = message; @@ -110,7 +113,7 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest) { this(eventType, null, httpRequest); } - // Constructor for testing only + // Constructor for testing and deserialization only protected AuditEvent() { } /** @@ -130,15 +133,10 @@ public AuditEvent(EventType eventType, Throwable exception, HttpServletRequest h this.httpMethod = httpRequest.getMethod(); this.httpQueryString = httpRequest.getQueryString(); this.headers = getHeadersFromRequest(httpRequest); + this.requestUrl = httpRequest.getRequestURL(); this.nodeName = MDC.get(ZkStateReader.NODE_NAME_PROP); - switch (this.httpMethod) { - case "GET": - this.requestType = AuthorizationContext.RequestType.READ.name(); - case "POST": - case "PUT": - this.requestType = AuthorizationContext.RequestType.WRITE.name(); - } + setRequestType(findRequestType()); if (exception != null) setException(exception); @@ -165,7 +163,7 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza this.collections = authorizationContext.getCollectionRequests() .stream().map(r -> r.collectionName).collect(Collectors.toList()); this.resource = authorizationContext.getResource(); - this.requestType = authorizationContext.getRequestType().toString(); + this.requestType = RequestType.convertType(authorizationContext.getRequestType()); authorizationContext.getParams().forEach(p -> { this.solrParams.put(p.getKey(), p.getValue()); }); @@ -203,6 +201,23 @@ public enum Level { ERROR // Used when there is an exception or error during auth / authz } + public enum RequestType { + ADMIN, SEARCH, UPDATE, STREAMING, UNKNOWN; + + static RequestType convertType(AuthorizationContext.RequestType ctxReqType) { + switch (ctxReqType) { + case ADMIN: + return RequestType.ADMIN; + case READ: + return RequestType.SEARCH; + case WRITE: + return RequestType.UPDATE; + default: + return RequestType.UNKNOWN; + } + } + } + public String getMessage() { return message; } @@ -263,7 +278,7 @@ public int getSolrPort() { return solrPort; } - public HashMap getHeaders() { + public Map getHeaders() { return headers; } @@ -287,7 +302,7 @@ public String getNodeName() { return nodeName; } - public String getRequestType() { + public RequestType getRequestType() { return requestType; } @@ -302,9 +317,18 @@ public double getQTime() { public Throwable getException() { return exception; } - + + public StringBuffer getRequestUrl() { + return requestUrl; + } + // Setters, builder style + public AuditEvent setRequestUrl(StringBuffer requestUrl) { + this.requestUrl = requestUrl; + return this; + } + public AuditEvent setSession(String session) { this.session = session; return this; @@ -380,7 +404,7 @@ public AuditEvent setSolrIp(String solrIp) { return this; } - public AuditEvent setHeaders(HashMap headers) { + public AuditEvent setHeaders(Map headers) { this.headers = headers; return this; } @@ -395,20 +419,22 @@ public AuditEvent setAutResponse(AuthorizationResponse autResponse) { return this; } - public AuditEvent setRequestType(String requestType) { + public AuditEvent setRequestType(RequestType requestType) { this.requestType = requestType; return this; } - public void setQTime(double QTime) { + public AuditEvent setQTime(double QTime) { this.QTime = QTime; + return this; } - public void setStatus(int status) { + public AuditEvent setStatus(int status) { this.status = status; + return this; } - public void setException(Throwable exception) { + public AuditEvent setException(Throwable exception) { this.exception = exception; if (exception != null) { this.eventType = ERROR; @@ -417,5 +443,34 @@ public void setException(Throwable exception) { if (exception instanceof SolrException) status = ((SolrException)exception).code(); } + return this; } + + private RequestType findRequestType() { + if (ADMIN_PATH_REGEXES.stream().map(Pattern::compile) + .anyMatch(p -> p.matcher(resource).matches())) return RequestType.ADMIN; + if (SEARCH_PATH_REGEXES.stream().map(Pattern::compile) + .anyMatch(p -> p.matcher(resource).matches())) return RequestType.SEARCH; + if (INDEXING_PATH_REGEXES.stream().map(Pattern::compile) + .anyMatch(p -> p.matcher(resource).matches())) return RequestType.UPDATE; + if (STREAMING_PATH_REGEXES.stream().map(Pattern::compile) + .anyMatch(p -> p.matcher(resource).matches())) return RequestType.STREAMING; + return RequestType.UNKNOWN; + } + + private static final List ADMIN_PATH_REGEXES = Arrays.asList( + "^/solr/admin/.*", + "^/api/(c|collections)/$", + "^/api/(c|collections)/[^/]+/config$", + "^/api/(c|collections)/[^/]+/schema$", + "^/api/(c|collections)/[^/]+/shards.*", + "^/api/cores.*$", + "^/api/node$", + "^/api/cluster$"); + + private static final List STREAMING_PATH_REGEXES = Collections.singletonList(".*/stream.*"); + + private static final List INDEXING_PATH_REGEXES = Collections.singletonList(".*/update.*"); + + private static final List SEARCH_PATH_REGEXES = Arrays.asList(".*/select.*", ".*/query.*"); } diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index bc2034ba528f..351f551393dc 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.io.StringWriter; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -61,6 +63,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo private static final String PARAM_BLOCKASYNC = "blockAsync"; private static final String PARAM_QUEUE_SIZE = "queueSize"; private static final String PARAM_NUM_THREADS = "numThreads"; + private static final String PARAM_MUTE_RULES = "muteRules"; private static final int DEFAULT_QUEUE_SIZE = 4096; private static final int DEFAULT_NUM_THREADS = 2; @@ -89,6 +92,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo EventType.ANONYMOUS_REJECTED.name()); private ExecutorService executorService; private boolean closed; + private MuteRules muteRules; /** * Initialize the plugin from security.json. @@ -106,6 +110,7 @@ public void init(Map pluginConfig) { blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); blockingQueueSize = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))) : 1; int numThreads = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS))) : 1; + muteRules = new MuteRules(pluginConfig.get(PARAM_MUTE_RULES)); pluginConfig.remove(PARAM_ASYNC); pluginConfig.remove(PARAM_BLOCKASYNC); pluginConfig.remove(PARAM_QUEUE_SIZE); @@ -130,6 +135,10 @@ public void init(Map pluginConfig) { * to either synchronous or async logging. */ public final void doAudit(AuditEvent event) { + if (shouldMute(event)) { + log.debug("Event muted due to mute rule(s)"); + return; + } if (async) { auditAsync(event); } else { @@ -146,6 +155,15 @@ public final void doAudit(AuditEvent event) { } } + /** + * Returns true if any of the configured mute rules matches. The inner lists are ORed, while rules inside + * inner lists are ANDed + * @param event the audit event + */ + protected boolean shouldMute(AuditEvent event) { + return muteRules.shouldMute(event); + } + /** * Enqueues an {@link AuditEvent} to a queue and returns immediately. * A background thread will pull events from this queue and call {@link #audit(AuditEvent)} @@ -292,11 +310,84 @@ public void close() throws IOException { log.info("Async auditlogger queue still has {} elements, sleeping to let it drain...", queue.size()); Thread.sleep(1000); timeSlept ++; - } catch (InterruptedException e) {} + } catch (InterruptedException ignored) {} } closed = true; log.info("Shutting down async Auditlogger background thread(s)"); executorService.shutdownNow(); } } + + /** + * Set of rules for when audit logging should be muted. + */ + private class MuteRules { + private List> rules; + + MuteRules(Object o) { + rules = new ArrayList<>(); + if (o != null) { + if (o instanceof List) { + ((List)o).forEach(l -> { + if (l instanceof String) { + rules.add(Collections.singletonList(parseRule(l))); + } else if (l instanceof List) { + List rl = new ArrayList<>(); + ((List) l).forEach(r -> { + rl.add(parseRule(r)); + }); + rules.add(rl); + } + }); + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The " + PARAM_MUTE_RULES + " configuration must be a list"); + } + } + } + + private MuteRule parseRule(Object ruleObj) { + try { + String rule = (String) ruleObj; + try { + AuditEvent.RequestType requestType = AuditEvent.RequestType.valueOf(rule); + return event -> event.getRequestType() != null && event.getRequestType().equals(requestType); + } catch (IllegalArgumentException e2) { + if (rule.startsWith("collection:")) { + return event -> event.getCollections().contains(rule.substring("collection:".length())); + } + if (rule.startsWith("user:")) { + return event -> event.getUsername() != null && event.getUsername().equals(rule.substring("user:".length())); + } + if (rule.startsWith("path:")) { + return event -> event.getResource().startsWith(rule.substring("path:".length())); + } + if (rule.startsWith("ip:")) { + return event -> event.getClientIp().equals(rule.substring("ip:".length())); + } + if (rule.startsWith("param:")) { + String[] kv = rule.substring("param:".length()).split("="); + if (kv.length == 2) { + return event -> event.getSolrParams().getOrDefault(kv[0],"").equals(kv[1]); + } + } + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unkonwn mute rule " + rule); + } + } catch (ClassCastException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The rules in " + PARAM_MUTE_RULES + " configuration must be strings or list fo strings"); + } + } + + /** + * Returns true if any of the configured mute rules matches. The inner lists are ORed, while rules inside + * inner lists are ANDed + */ + boolean shouldMute(AuditEvent event) { + if (rules == null) return false; + return rules.stream().anyMatch(rl -> rl.stream().allMatch(r -> r.shouldMute(event))); + } + } + + public interface MuteRule { + boolean shouldMute(AuditEvent event); + } } diff --git a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json index 18fc02a1a1e7..254e61659bb3 100644 --- a/solr/core/src/test-files/solr/security/auditlog_plugin_security.json +++ b/solr/core/src/test-files/solr/security/auditlog_plugin_security.json @@ -3,6 +3,7 @@ "class": "solr.CallbackAuditLoggerPlugin", "callbackPort": "_PORT_", "async": _ASYNC_, - "delay": "_DELAY_" - }_AUTH_ + "delay": "_DELAY_", + "muteRules": _MUTERULES_ + },_AUTH_ } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 5e1ebe223731..3dc8e327bc74 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -43,6 +43,8 @@ import org.apache.solr.cloud.SolrCloudAuthTestCase; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.security.AuditEvent.EventType; +import org.apache.solr.security.AuditEvent.RequestType; import org.apache.solr.util.DefaultSolrThreadFactory; import org.junit.After; import org.junit.Before; @@ -56,6 +58,8 @@ import static org.apache.solr.security.AuditEvent.EventType.ERROR; import static org.apache.solr.security.AuditEvent.EventType.REJECTED; import static org.apache.solr.security.AuditEvent.EventType.UNAUTHORIZED; +import static org.apache.solr.security.AuditEvent.RequestType.ADMIN; +import static org.apache.solr.security.AuditEvent.RequestType.SEARCH; /** * Validate that audit logging works in a live cluster @@ -86,7 +90,7 @@ public void tearDown() throws Exception { @Test public void testSynchronous() throws Exception { - setupCluster(false, 0, false); + setupCluster(false, 0, false, null); runAdminCommands(); assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); testHarness.get().shutdownCluster(); @@ -95,7 +99,7 @@ public void testSynchronous() throws Exception { @Test public void testAsync() throws Exception { - setupCluster(true, 0, false); + setupCluster(true, 0, false, null); runAdminCommands(); assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); testHarness.get().shutdownCluster(); @@ -104,16 +108,24 @@ public void testAsync() throws Exception { @Test public void testAsyncWithQueue() throws Exception { - setupCluster(true, 100, false); + setupCluster(true, 100, false, null); runAdminCommands(); assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); testHarness.get().shutdownCluster(); assertThreeAdminEvents(); } + @Test + public void testMuteAdminListCollections() throws Exception { + setupCluster(false, 0, false, "[ [ \"path:/admin\", \"param:action=LIST\" ], \"UNKNOWN\" ]"); + runAdminCommands(); + CallbackReceiver receiver = testHarness.get().receiver; + assertEquals(2, receiver.getBuffer().size()); // Only get two events, since the LIST event was muted + } + @Test public void searchWithException() throws Exception { - setupCluster(false, 0, false); + setupCluster(false, 0, false, null); try { testHarness.get().cluster.getSolrClient().request(CollectionAdminRequest.createCollection("test", 1, 1)); testHarness.get().cluster.getSolrClient().query("test", new MapSolrParams(Collections.singletonMap("q", "a(bc"))); @@ -123,13 +135,13 @@ public void searchWithException() throws Exception { CallbackReceiver receiver = testHarness.get().receiver; assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/cores"); assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections"); - assertAuditEvent(receiver.popEvent(), ERROR,"/select", "READ", null, 400); + assertAuditEvent(receiver.popEvent(), ERROR,"/select", SEARCH, null, 400); } } @Test public void auth() throws Exception { - setupCluster(false, 0, true); + setupCluster(false, 0, true, null); CloudSolrClient client = testHarness.get().cluster.getSolrClient(); try { CollectionAdminRequest.List request = new CollectionAdminRequest.List(); @@ -142,9 +154,11 @@ public void auth() throws Exception { } catch (SolrException ex) { waitForAuditEventCallbacks(3); CallbackReceiver receiver = testHarness.get().receiver; - assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "action", "LIST"); - assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", "ADMIN", "solr", 200, "action", "LIST"); - assertAuditEvent(receiver.popEvent(), REJECTED, "/admin/collections", "ADMIN", null,401); + assertAuditEvent(receiver.popEvent(), COMPLETED, "/admin/collections", ADMIN, null, 200, "action", "LIST"); + AuditEvent e = receiver.popEvent(); + System.out.println(new AuditLoggerPlugin.JSONAuditEventFormatter().formatEvent(e)); + assertAuditEvent(e, COMPLETED, "/admin/collections", ADMIN, "solr", 200, "action", "LIST"); + assertAuditEvent(receiver.popEvent(), REJECTED, "/admin/collections", ADMIN, null,401); } try { CollectionAdminRequest.Create createRequest = CollectionAdminRequest.createCollection("test", 1, 1); @@ -154,15 +168,15 @@ public void auth() throws Exception { } catch (SolrException ex) { waitForAuditEventCallbacks(1); CallbackReceiver receiver = testHarness.get().receiver; - assertAuditEvent(receiver.popEvent(), UNAUTHORIZED, "/admin/collections", "ADMIN", null,403); + assertAuditEvent(receiver.popEvent(), UNAUTHORIZED, "/admin/collections", ADMIN, null,403); } } - private void assertAuditEvent(AuditEvent e, AuditEvent.EventType type, String path, String... params) { + private void assertAuditEvent(AuditEvent e, EventType type, String path, String... params) { assertAuditEvent(e, type, path, null, null,null, params); } - private void assertAuditEvent(AuditEvent e, AuditEvent.EventType type, String path, String requestType, String username, Integer status, String... params) { + private void assertAuditEvent(AuditEvent e, EventType type, String path, RequestType requestType, String username, Integer status, String... params) { assertEquals(type, e.getEventType()); assertEquals(path, e.getResource()); if (requestType != null) { @@ -218,6 +232,7 @@ private void assertThreeAdminEvents() throws Exception { assertEquals(COMPLETED, e.getEventType()); assertEquals("GET", e.getHttpMethod()); assertEquals("CLUSTERSTATUS", e.getSolrParamAsString("action")); + System.out.println("*** " + new AuditLoggerPlugin.JSONAuditEventFormatter().formatEvent(e)); e = receiver.getBuffer().pop(); assertEquals(COMPLETED, e.getEventType()); @@ -236,12 +251,13 @@ private void assertThreeAdminEvents() throws Exception { " \"permissions\":[{\"name\":\"collection-admin-edit\",\"role\":\"admin\"}]\n" + " }\n"; - void setupCluster(boolean async, int delay, boolean enableAuth) throws Exception { + void setupCluster(boolean async, int delay, boolean enableAuth, String muteRulesJson) throws Exception { String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); securityJson = securityJson.replace("_PORT_", Integer.toString(testHarness.get().callbackPort)); securityJson = securityJson.replace("_ASYNC_", Boolean.toString(async)); securityJson = securityJson.replace("_DELAY_", Integer.toString(delay)); securityJson = securityJson.replace("_AUTH_", enableAuth ? AUTH_SECTION : ""); + securityJson = securityJson.replace("_MUTERULES_", muteRulesJson != null ? muteRulesJson : "[]"); MiniSolrCloudCluster myCluster = new Builder(NUM_SERVERS, createTempDir()) .withSecurityJson(securityJson) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) @@ -341,7 +357,7 @@ public void close() throws Exception { } public void shutdownCluster() throws Exception { - cluster.shutdown(); + if (cluster != null) cluster.shutdown(); } public void setCluster(MiniSolrCloudCluster cluster) { diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index d15af07570f2..916ab7292dde 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -17,12 +17,17 @@ package org.apache.solr.security; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -67,6 +72,22 @@ public class AuditLoggerPluginTest extends SolrTestCaseJ4 { .setMessage("Error occurred") .setDate(SAMPLE_DATE) .setResource("/collection1"); + protected static final AuditEvent EVENT_UPDATE = new AuditEvent(AuditEvent.EventType.ERROR) + .setUsername("updateuser") + .setHttpMethod("POST") + .setRequestType(AuditEvent.RequestType.UPDATE) + .setMessage("Success") + .setDate(SAMPLE_DATE) + .setCollections(Collections.singletonList("updatecoll")) + .setResource("/update"); + protected static final AuditEvent EVENT_STREAMING = new AuditEvent(AuditEvent.EventType.ERROR) + .setUsername("streaminguser") + .setHttpMethod("POST") + .setRequestType(AuditEvent.RequestType.STREAMING) + .setMessage("Success") + .setDate(SAMPLE_DATE) + .setCollections(Collections.singletonList("streamcoll")) + .setResource("/stream"); private MockAuditLoggerPlugin plugin; private HashMap config; @@ -79,11 +100,18 @@ public void setUp() throws Exception { config.put("async", false); plugin.init(config); } - + + @Override + @After + public void tearDown() throws Exception { + plugin.close(); + super.tearDown(); + } + @Test public void init() { config = new HashMap<>(); - config.put("eventTypes", Arrays.asList("REJECTED")); + config.put("eventTypes", Collections.singletonList("REJECTED")); config.put("async", false); plugin.init(config); assertTrue(plugin.shouldLog(EVENT_REJECTED.getEventType())); @@ -101,7 +129,27 @@ public void shouldLog() { assertFalse(plugin.shouldLog(EVENT_AUTHENTICATED.getEventType())); assertFalse(plugin.shouldLog(EVENT_AUTHORIZED.getEventType())); } + + @Test(expected = SolrException.class) + public void invalidMuteRule() { + config.put("muteRules", Collections.singletonList("foo:bar")); + plugin.init(config); + } + @Test + public void shouldMute() { + List rules = new ArrayList<>(); + rules.add("STREAMING"); + rules.add(Arrays.asList("user:updateuser", "collection:updatecoll")); + config.put("muteRules",rules); + plugin.init(config); + assertFalse(plugin.shouldMute(EVENT_ANONYMOUS)); + assertTrue(plugin.shouldMute(EVENT_STREAMING)); + assertTrue(plugin.shouldMute(EVENT_UPDATE)); + EVENT_UPDATE.setUsername("john"); + assertFalse(plugin.shouldMute(EVENT_UPDATE)); + } + @Test public void audit() { plugin.doAudit(EVENT_ANONYMOUS_REJECTED); diff --git a/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java index 75429286eef6..f1c7abb05cce 100644 --- a/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/MockAuditLoggerPlugin.java @@ -16,7 +16,6 @@ */ package org.apache.solr.security; -import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.HashMap; @@ -49,9 +48,6 @@ private void incrementType(String type) { typeCounts.get(type).incrementAndGet(); } - @Override - public void close() throws IOException { /* ignored */ } - public void reset() { events.clear(); typeCounts.clear(); diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 3dd3b4cce9de..e2ecfcbd9dbb 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -22,8 +22,40 @@ Audit loggers are pluggable to suit any possible format or log destination. [quote] An audit trail (also called audit log) is a security-relevant chronological record, set of records, and/or destination and source of records that provide documentary evidence of the sequence of activities that have affected at any time a specific operation, procedure, or event. (https://en.wikipedia.org/wiki/Audit_trail[Wikipedia]) +== Configuration in security.json +Audit logging is configured in `security.json` under the `auditlogging` key. + +The example `security.json` below configures synchronous audit logging to Solr default log file. + +[source,json] +---- +{ + "auditlogging":{ + "class": "solr.SolrLogAuditLoggerPlugin" + } +} +---- + +By default any AuditLogger plugin configured will log asynchronously in the background to avoid slowing down the requests. To make audit logging happen synchronously, add the parameter `async: false`. For async logging, you may optionally also configure queue size, number of threads and whether it should block when the queue is full or discard events: + +[source,json] +---- +{ + "auditlogging":{ + "class": "solr.SolrLogAuditLoggerPlugin", + "async": true, + "blockAsync" : false, + "numThreads" : 2, + "queueSize" : 4096, + "eventTypes": ["REJECTED", "ANONYMOUS_REJECTED", "UNAUTHORIZED", "COMPLETED" "ERROR"] + } +} +---- + +The defaults are `async: true`, `blockAsync: false`, `queueSize: 4096` and `numThreads: 2`. + [#audit-event-types] -== Event types +=== Event types These are the event types triggered by the framework: [%header,format=csv,separator=;] @@ -41,41 +73,46 @@ ERROR;Request was not executed due to an error By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. What eventTypes are logged can be configured with the `eventTypes` configuration parameter. -== Configuration in security.json -Audit logging is configured in `security.json` under the `auditlogging` key. +=== Muting certain events +The configuration parameter `muteRules` lets you mute logging for certain events. You may specify multiple rules and combination of rules that will cause muting. You can mute by event type, request type, username, collection name, path, request parameters or IP address. We'll explain through examples: -The example `security.json` below configures synchronous audit logging to Solr default log file. +The below example will mute logging for all `UPDATE` requests as well as all requests made my user johndoe or from IP address 192.168.0.10: [source,json] ---- { "auditlogging":{ - "class": "solr.SolrLogAuditLoggerPlugin" + "class": "solr.SolrLogAuditLoggerPlugin" + "muteRules": [ "UPDATE", "user:johndoe", "ip:192.168.0.10" ] } } ---- -By default any AuditLogger plugin configured will log asynchronously in the background to avoid slowing down the requests. To make audit logging happen synchronously, add the parameter `async: false`. For async logging, you may optionally also configure queue size, number of threads and whether it should block when the queue is full or discard events: +An update rule may also be a list, in which case all must be true. The following muteRules will mute update events from IP 192.168.0.10. It will also mute LIST collection admin requests as well as all collection admin requests for the collection named "test": [source,json] ---- { "auditlogging":{ - "class": "solr.SolrLogAuditLoggerPlugin", - "async": true, - "blockAsync" : false, - "numThreads" : 2, - "queueSize" : 4096, - "eventTypes": ["REJECTED", "ANONYMOUS_REJECTED", "UNAUTHORIZED", "COMPLETED" "ERROR"] + "class": "solr.SolrLogAuditLoggerPlugin" + "muteRules": [ + [ "UPDATE", "ip:192.168.0.10" ], + [ "path:/admin/collections", "param:action=LIST" ], + [ "path:/admin/collections", "param:collection=test" ] + ] } } ---- -The defaults are `async: true`, `blockAsync: false`, `queueSize: 4096` and `numThreads: 2`. +Valid rules are: -Other parameters supported are: +* `ADMIN`, `SEARCH`, `UPDATE`, `STREAMING`, `UNKNOWN` (request types) +* `collection:` (collection by name) +* `user:` (user by userid) +* `path:` (request path relative to `/solr` or for search/update requests relative to collection. Path is prefix matched, i.e. `/admin` will mute any sub path as well. +* `ip:` (IPv4-address) +* `param:=` (request parameter) - === Chaining multiple loggers Using the `MultiDestinationAuditLogger` you can configure multiple audit logger plugins in a chain, to log to multiple destinations, as follows: @@ -84,13 +121,13 @@ Using the `MultiDestinationAuditLogger` you can configure multiple audit logger ---- { "auditlogging":{ - "class" : "solr.MultiDestinationAuditLogger", - "plugins" : [ - { "class" : "solr.SolrLogAuditLoggerPlugin" }, - { "class" : "solr.MyOtherAuditPlugin", - "customParam" : "value" - } - ] + "class" : "solr.MultiDestinationAuditLogger", + "plugins" : [ + { "class" : "solr.SolrLogAuditLoggerPlugin" }, + { "class" : "solr.MyOtherAuditPlugin", + "customParam" : "value" + } + ] } } ---- From dc604e130ab2e5f5e07d5a68d1ff93fae6b7a007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 12:21:32 +0100 Subject: [PATCH 48/65] Clean up code and variable visibility --- .../solr/security/AuditLoggerPlugin.java | 25 ++++++++----------- .../security/AuditLoggerIntegrationTest.java | 18 ++++++------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 351f551393dc..b8ac94b0cac5 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -58,12 +58,11 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfoBean, SolrMetricProducer { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_EVENT_TYPES = "eventTypes"; - static final String PARAM_ASYNC = "async"; - private static final String PARAM_BLOCKASYNC = "blockAsync"; - private static final String PARAM_QUEUE_SIZE = "queueSize"; - private static final String PARAM_NUM_THREADS = "numThreads"; - private static final String PARAM_MUTE_RULES = "muteRules"; + static final String PARAM_BLOCKASYNC = "blockAsync"; + static final String PARAM_QUEUE_SIZE = "queueSize"; + static final String PARAM_NUM_THREADS = "numThreads"; + static final String PARAM_MUTE_RULES = "muteRules"; private static final int DEFAULT_QUEUE_SIZE = 4096; private static final int DEFAULT_NUM_THREADS = 2; @@ -73,8 +72,11 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo int blockingQueueSize; protected AuditEventFormatter formatter; - MetricRegistry registry; - Set metricNames = ConcurrentHashMap.newKeySet(); + private MetricRegistry registry; + private Set metricNames = ConcurrentHashMap.newKeySet(); + private ExecutorService executorService; + private boolean closed; + private MuteRules muteRules; protected String registryName; protected SolrMetricManager metricManager; @@ -90,9 +92,6 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo EventType.REJECTED.name(), EventType.UNAUTHORIZED.name(), EventType.ANONYMOUS_REJECTED.name()); - private ExecutorService executorService; - private boolean closed; - private MuteRules muteRules; /** * Initialize the plugin from security.json. @@ -333,9 +332,7 @@ private class MuteRules { rules.add(Collections.singletonList(parseRule(l))); } else if (l instanceof List) { List rl = new ArrayList<>(); - ((List) l).forEach(r -> { - rl.add(parseRule(r)); - }); + ((List) l).forEach(r -> rl.add(parseRule(r))); rules.add(rl); } }); @@ -373,7 +370,7 @@ private MuteRule parseRule(Object ruleObj) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unkonwn mute rule " + rule); } } catch (ClassCastException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The rules in " + PARAM_MUTE_RULES + " configuration must be strings or list fo strings"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The rules in " + PARAM_MUTE_RULES + " configuration must be list of strings or list of list of strings"); } } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 3dc8e327bc74..13ba324c5507 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -72,7 +72,7 @@ public class AuditLoggerIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SHARDS = 1; protected static final int REPLICATION_FACTOR = 1; // Use a harness per thread to be able to beast this test - ThreadLocal testHarness = new ThreadLocal<>(); + private ThreadLocal testHarness = new ThreadLocal<>(); @Override @Before @@ -251,7 +251,7 @@ private void assertThreeAdminEvents() throws Exception { " \"permissions\":[{\"name\":\"collection-admin-edit\",\"role\":\"admin\"}]\n" + " }\n"; - void setupCluster(boolean async, int delay, boolean enableAuth, String muteRulesJson) throws Exception { + private void setupCluster(boolean async, int delay, boolean enableAuth, String muteRulesJson) throws Exception { String securityJson = FileUtils.readFileToString(TEST_PATH().resolve("security").resolve("auditlog_plugin_security.json").toFile(), StandardCharsets.UTF_8); securityJson = securityJson.replace("_PORT_", Integer.toString(testHarness.get().callbackPort)); securityJson = securityJson.replace("_ASYNC_", Boolean.toString(async)); @@ -277,15 +277,15 @@ private class CallbackReceiver implements Runnable, AutoCloseable { private Map resourceCounts = new HashMap<>(); private LinkedList buffer = new LinkedList<>(); - public CallbackReceiver() throws IOException { + CallbackReceiver() throws IOException { serverSocket = new ServerSocket(0); } - public int getTotalCount() { + int getTotalCount() { return count.get(); } - public int getCountForPath(String path) { + int getCountForPath(String path) { return resourceCounts.getOrDefault(path, new AtomicInteger()).get(); } @@ -326,11 +326,11 @@ public void close() throws Exception { serverSocket.close(); } - public LinkedList getBuffer() { + protected LinkedList getBuffer() { return buffer; } - public AuditEvent popEvent() { + protected AuditEvent popEvent() { return buffer.pop(); } } @@ -341,10 +341,10 @@ private class AuditTestHarness implements AutoCloseable { Thread receiverThread; private MiniSolrCloudCluster cluster; - public AuditTestHarness() throws IOException { + AuditTestHarness() throws IOException { receiver = new CallbackReceiver(); callbackPort = receiver.getPort(); - receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver);; + receiverThread = new DefaultSolrThreadFactory("auditTestCallback").newThread(receiver); receiverThread.start(); } From 98c9a0371e72d9202cef442a0a12074eb4467060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 12:58:14 +0100 Subject: [PATCH 49/65] Fix test failure, needs to wait for callbacks --- .../org/apache/solr/security/AuditLoggerIntegrationTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 13ba324c5507..160380de9d5b 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -119,8 +119,9 @@ public void testAsyncWithQueue() throws Exception { public void testMuteAdminListCollections() throws Exception { setupCluster(false, 0, false, "[ [ \"path:/admin\", \"param:action=LIST\" ], \"UNKNOWN\" ]"); runAdminCommands(); - CallbackReceiver receiver = testHarness.get().receiver; - assertEquals(2, receiver.getBuffer().size()); // Only get two events, since the LIST event was muted + testHarness.get().shutdownCluster(); + waitForAuditEventCallbacks(2); + assertEquals(2, testHarness.get().receiver.getBuffer().size()); } @Test From caf6dfb3efc9c4c0b0cff62c4b4ec6cd6c0ffb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 14:23:15 +0100 Subject: [PATCH 50/65] Change requestType mute config into type: as the others Null-checks in shouldMatch() methods Change solrParams into Map>, fix bugs with mute param:k=v Remove System.out --- .../org/apache/solr/security/AuditEvent.java | 14 ++--- .../solr/security/AuditLoggerPlugin.java | 51 ++++++++++--------- .../security/AuditLoggerIntegrationTest.java | 3 +- .../solr/security/AuditLoggerPluginTest.java | 14 +++-- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditEvent.java b/solr/core/src/java/org/apache/solr/security/AuditEvent.java index 67be61c21e43..f9c45be1a256 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditEvent.java +++ b/solr/core/src/java/org/apache/solr/security/AuditEvent.java @@ -57,7 +57,7 @@ public class AuditEvent { private List collections; private Map context; private Map headers; - private Map solrParams = new HashMap<>(); + private Map> solrParams = new HashMap<>(); private String solrHost; private int solrPort; private String solrIp; @@ -165,7 +165,7 @@ public AuditEvent(EventType eventType, HttpServletRequest httpRequest, Authoriza this.resource = authorizationContext.getResource(); this.requestType = RequestType.convertType(authorizationContext.getRequestType()); authorizationContext.getParams().forEach(p -> { - this.solrParams.put(p.getKey(), p.getValue()); + this.solrParams.put(p.getKey(), Arrays.asList(p.getValue())); }); } @@ -282,14 +282,14 @@ public Map getHeaders() { return headers; } - public Map getSolrParams() { + public Map> getSolrParams() { return solrParams; } public String getSolrParamAsString(String key) { - Object v = getSolrParams().get(key); - if (v instanceof List && ((List) v).size() > 0) { - return String.valueOf(((List) v).get(0)); + List v = getSolrParams().get(key); + if (v != null && v.size() > 0) { + return String.valueOf((v).get(0)); } return null; } @@ -409,7 +409,7 @@ public AuditEvent setHeaders(Map headers) { return this; } - public AuditEvent setSolrParams(Map solrParams) { + public AuditEvent setSolrParams(Map> solrParams) { this.solrParams = solrParams; return this; } diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index b8ac94b0cac5..e98ff47b456e 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -135,7 +135,7 @@ public void init(Map pluginConfig) { */ public final void doAudit(AuditEvent event) { if (shouldMute(event)) { - log.debug("Event muted due to mute rule(s)"); + log.warn("Event muted due to mute rule(s)"); return; } if (async) { @@ -345,32 +345,33 @@ private class MuteRules { private MuteRule parseRule(Object ruleObj) { try { String rule = (String) ruleObj; - try { - AuditEvent.RequestType requestType = AuditEvent.RequestType.valueOf(rule); - return event -> event.getRequestType() != null && event.getRequestType().equals(requestType); - } catch (IllegalArgumentException e2) { - if (rule.startsWith("collection:")) { - return event -> event.getCollections().contains(rule.substring("collection:".length())); - } - if (rule.startsWith("user:")) { - return event -> event.getUsername() != null && event.getUsername().equals(rule.substring("user:".length())); - } - if (rule.startsWith("path:")) { - return event -> event.getResource().startsWith(rule.substring("path:".length())); - } - if (rule.startsWith("ip:")) { - return event -> event.getClientIp().equals(rule.substring("ip:".length())); - } - if (rule.startsWith("param:")) { - String[] kv = rule.substring("param:".length()).split("="); - if (kv.length == 2) { - return event -> event.getSolrParams().getOrDefault(kv[0],"").equals(kv[1]); - } + if (rule.startsWith("type:")) { + AuditEvent.RequestType muteType = AuditEvent.RequestType.valueOf(rule.substring("type:".length())); + return event -> event.getRequestType() != null && event.getRequestType().equals(muteType); + } + if (rule.startsWith("collection:")) { + return event -> event.getCollections() != null && event.getCollections().contains(rule.substring("collection:".length())); + } + if (rule.startsWith("user:")) { + return event -> event.getUsername() != null && event.getUsername().equals(rule.substring("user:".length())); + } + if (rule.startsWith("path:")) { + return event -> event.getResource() != null && event.getResource().startsWith(rule.substring("path:".length())); + } + if (rule.startsWith("ip:")) { + return event -> event.getClientIp() != null && event.getClientIp().equals(rule.substring("ip:".length())); + } + if (rule.startsWith("param:")) { + String[] kv = rule.substring("param:".length()).split("="); + if (kv.length == 2) { + return event -> event.getSolrParams() != null && kv[1].equals(event.getSolrParamAsString(kv[0])); + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The 'param' muteRule must be of format 'param:key=value', got " + rule); } - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unkonwn mute rule " + rule); } - } catch (ClassCastException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "The rules in " + PARAM_MUTE_RULES + " configuration must be list of strings or list of list of strings"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unkonwn mute rule " + rule); + } catch (ClassCastException | IllegalArgumentException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "There was a problem parsing muteRules. Must be a list of valid rule strings", e); } } diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 160380de9d5b..06fda49a9c3a 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -117,7 +117,7 @@ public void testAsyncWithQueue() throws Exception { @Test public void testMuteAdminListCollections() throws Exception { - setupCluster(false, 0, false, "[ [ \"path:/admin\", \"param:action=LIST\" ], \"UNKNOWN\" ]"); + setupCluster(false, 0, false, "[ \"type:UNKNOWN\", [ \"path:/admin\", \"param:action=LIST\" ] ]"); runAdminCommands(); testHarness.get().shutdownCluster(); waitForAuditEventCallbacks(2); @@ -233,7 +233,6 @@ private void assertThreeAdminEvents() throws Exception { assertEquals(COMPLETED, e.getEventType()); assertEquals("GET", e.getHttpMethod()); assertEquals("CLUSTERSTATUS", e.getSolrParamAsString("action")); - System.out.println("*** " + new AuditLoggerPlugin.JSONAuditEventFormatter().formatEvent(e)); e = receiver.getBuffer().pop(); assertEquals(COMPLETED, e.getEventType()); diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index 916ab7292dde..ba67310a4331 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -72,15 +72,16 @@ public class AuditLoggerPluginTest extends SolrTestCaseJ4 { .setMessage("Error occurred") .setDate(SAMPLE_DATE) .setResource("/collection1"); - protected static final AuditEvent EVENT_UPDATE = new AuditEvent(AuditEvent.EventType.ERROR) + protected static final AuditEvent EVENT_UPDATE = new AuditEvent(AuditEvent.EventType.COMPLETED) .setUsername("updateuser") .setHttpMethod("POST") .setRequestType(AuditEvent.RequestType.UPDATE) .setMessage("Success") .setDate(SAMPLE_DATE) .setCollections(Collections.singletonList("updatecoll")) + .setRequestType(AuditEvent.RequestType.UPDATE) .setResource("/update"); - protected static final AuditEvent EVENT_STREAMING = new AuditEvent(AuditEvent.EventType.ERROR) + protected static final AuditEvent EVENT_STREAMING = new AuditEvent(AuditEvent.EventType.COMPLETED) .setUsername("streaminguser") .setHttpMethod("POST") .setRequestType(AuditEvent.RequestType.STREAMING) @@ -139,15 +140,22 @@ public void invalidMuteRule() { @Test public void shouldMute() { List rules = new ArrayList<>(); - rules.add("STREAMING"); + rules.add("type:STREAMING"); rules.add(Arrays.asList("user:updateuser", "collection:updatecoll")); + rules.add(Arrays.asList("type:UPDATE", "param:commit=true")); + rules.add("ip:192.168.0.10"); config.put("muteRules",rules); plugin.init(config); assertFalse(plugin.shouldMute(EVENT_ANONYMOUS)); assertTrue(plugin.shouldMute(EVENT_STREAMING)); assertTrue(plugin.shouldMute(EVENT_UPDATE)); + assertTrue(plugin.shouldMute(EVENT_UPDATE)); EVENT_UPDATE.setUsername("john"); assertFalse(plugin.shouldMute(EVENT_UPDATE)); + EVENT_UPDATE.setSolrParams(Collections.singletonMap("commit", Collections.singletonList("true"))); + assertTrue(plugin.shouldMute(EVENT_UPDATE)); + EVENT_ANONYMOUS.setClientIp("192.168.0.10"); + assertTrue(plugin.shouldMute(EVENT_ANONYMOUS)); } @Test From 2bc9efeb0d41766808b8b0a5fe3465739a8e9fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 14:38:07 +0100 Subject: [PATCH 51/65] Remove muteRules parameter after configuring Update ref-guide with the type: syntax change --- .../java/org/apache/solr/security/AuditLoggerPlugin.java | 2 +- solr/solr-ref-guide/src/audit-logging.adoc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index e98ff47b456e..410774101673 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -109,7 +109,7 @@ public void init(Map pluginConfig) { blockAsync = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCKASYNC, false))); blockingQueueSize = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_QUEUE_SIZE, DEFAULT_QUEUE_SIZE))) : 1; int numThreads = async ? Integer.parseInt(String.valueOf(pluginConfig.getOrDefault(PARAM_NUM_THREADS, DEFAULT_NUM_THREADS))) : 1; - muteRules = new MuteRules(pluginConfig.get(PARAM_MUTE_RULES)); + muteRules = new MuteRules(pluginConfig.remove(PARAM_MUTE_RULES)); pluginConfig.remove(PARAM_ASYNC); pluginConfig.remove(PARAM_BLOCKASYNC); pluginConfig.remove(PARAM_QUEUE_SIZE); diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index e2ecfcbd9dbb..bbf8b249c2e0 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -83,7 +83,7 @@ The below example will mute logging for all `UPDATE` requests as well as all req { "auditlogging":{ "class": "solr.SolrLogAuditLoggerPlugin" - "muteRules": [ "UPDATE", "user:johndoe", "ip:192.168.0.10" ] + "muteRules": [ "type:SEARCH", "user:johndoe", "ip:192.168.0.10" ] } } ---- @@ -96,7 +96,7 @@ An update rule may also be a list, in which case all must be true. The following "auditlogging":{ "class": "solr.SolrLogAuditLoggerPlugin" "muteRules": [ - [ "UPDATE", "ip:192.168.0.10" ], + [ "type:UPDATE", "ip:192.168.0.10" ], [ "path:/admin/collections", "param:action=LIST" ], [ "path:/admin/collections", "param:collection=test" ] ] @@ -106,7 +106,7 @@ An update rule may also be a list, in which case all must be true. The following Valid rules are: -* `ADMIN`, `SEARCH`, `UPDATE`, `STREAMING`, `UNKNOWN` (request types) +* `type:` (request-type by name: `ADMIN`, `SEARCH`, `UPDATE`, `STREAMING`, `UNKNOWN`) * `collection:` (collection by name) * `user:` (user by userid) * `path:` (request path relative to `/solr` or for search/update requests relative to collection. Path is prefix matched, i.e. `/admin` will mute any sub path as well. From a052c2262526c7349d35fd622729386884447beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 15:18:59 +0100 Subject: [PATCH 52/65] Do not modify static test variables --- .../solr/security/AuditLoggerPluginTest.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java index ba67310a4331..7ee47ddb3809 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerPluginTest.java @@ -56,6 +56,7 @@ public class AuditLoggerPluginTest extends SolrTestCaseJ4 { .setResource("/collection1"); protected static final AuditEvent EVENT_AUTHORIZED = new AuditEvent(AuditEvent.EventType.AUTHORIZED) .setUsername("Per") + .setClientIp("192.168.0.10") .setHttpMethod("GET") .setMessage("Async") .setDate(SAMPLE_DATE) @@ -71,7 +72,8 @@ public class AuditLoggerPluginTest extends SolrTestCaseJ4 { .setHttpMethod("POST") .setMessage("Error occurred") .setDate(SAMPLE_DATE) - .setResource("/collection1"); + .setSolrParams(Collections.singletonMap("action", Collections.singletonList("DELETE"))) + .setResource("/admin/collections"); protected static final AuditEvent EVENT_UPDATE = new AuditEvent(AuditEvent.EventType.COMPLETED) .setUsername("updateuser") .setHttpMethod("POST") @@ -142,20 +144,16 @@ public void shouldMute() { List rules = new ArrayList<>(); rules.add("type:STREAMING"); rules.add(Arrays.asList("user:updateuser", "collection:updatecoll")); - rules.add(Arrays.asList("type:UPDATE", "param:commit=true")); + rules.add(Arrays.asList("path:/admin/collection", "param:action=DELETE")); rules.add("ip:192.168.0.10"); config.put("muteRules",rules); plugin.init(config); assertFalse(plugin.shouldMute(EVENT_ANONYMOUS)); - assertTrue(plugin.shouldMute(EVENT_STREAMING)); - assertTrue(plugin.shouldMute(EVENT_UPDATE)); - assertTrue(plugin.shouldMute(EVENT_UPDATE)); - EVENT_UPDATE.setUsername("john"); - assertFalse(plugin.shouldMute(EVENT_UPDATE)); - EVENT_UPDATE.setSolrParams(Collections.singletonMap("commit", Collections.singletonList("true"))); - assertTrue(plugin.shouldMute(EVENT_UPDATE)); - EVENT_ANONYMOUS.setClientIp("192.168.0.10"); - assertTrue(plugin.shouldMute(EVENT_ANONYMOUS)); + assertFalse(plugin.shouldMute(EVENT_AUTHENTICATED)); + assertTrue(plugin.shouldMute(EVENT_STREAMING)); // type:STREAMING + assertTrue(plugin.shouldMute(EVENT_UPDATE)); // updateuser, updatecoll + assertTrue(plugin.shouldMute(EVENT_ERROR)); // admin/collection action=DELETE + assertTrue(plugin.shouldMute(EVENT_AUTHORIZED)); // ip } @Test From 78e264a15b4055272f221aec19466852bd0ace10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 28 Mar 2019 20:41:52 +0100 Subject: [PATCH 53/65] log warn->debug --- .../src/java/org/apache/solr/security/AuditLoggerPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 410774101673..362d3f8ebe23 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -135,7 +135,7 @@ public void init(Map pluginConfig) { */ public final void doAudit(AuditEvent event) { if (shouldMute(event)) { - log.warn("Event muted due to mute rule(s)"); + log.debug("Event muted due to mute rule(s)"); return; } if (async) { From a9106116b98e53078ab07a469da9c4509f038a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 29 Mar 2019 10:33:43 +0100 Subject: [PATCH 54/65] RefGuide updates --- solr/solr-ref-guide/src/audit-logging.adoc | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index bbf8b249c2e0..aee9a2c683fc 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -74,9 +74,9 @@ ERROR;Request was not executed due to an error By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. What eventTypes are logged can be configured with the `eventTypes` configuration parameter. === Muting certain events -The configuration parameter `muteRules` lets you mute logging for certain events. You may specify multiple rules and combination of rules that will cause muting. You can mute by event type, request type, username, collection name, path, request parameters or IP address. We'll explain through examples: +The configuration parameter `muteRules` lets you mute logging for certain events. You may specify multiple rules and combination of rules that will cause muting. You can mute by request type, username, collection name, path, request parameters or IP address. We'll explain through examples: -The below example will mute logging for all `UPDATE` requests as well as all requests made my user johndoe or from IP address 192.168.0.10: +The below example will mute logging for all `SEARCH` requests as well as all requests made my user `johndoe` or from IP address `192.168.0.10`: [source,json] ---- @@ -88,7 +88,7 @@ The below example will mute logging for all `UPDATE` requests as well as all req } ---- -An update rule may also be a list, in which case all must be true. The following muteRules will mute update events from IP 192.168.0.10. It will also mute LIST collection admin requests as well as all collection admin requests for the collection named "test": +An mute rule may also be a list, in which case all must be true for muting to happen. The following muteRules will mute all events from IP `192.168.0.10`. It will also mute collection admin requests with action=LIST as well as all collection admin requests for the collection named `test`. Note how you can mix single string rules with lists of rules that must all match: [source,json] ---- @@ -96,7 +96,7 @@ An update rule may also be a list, in which case all must be true. The following "auditlogging":{ "class": "solr.SolrLogAuditLoggerPlugin" "muteRules": [ - [ "type:UPDATE", "ip:192.168.0.10" ], + "ip:192.168.0.10", [ "path:/admin/collections", "param:action=LIST" ], [ "path:/admin/collections", "param:collection=test" ] ] @@ -133,4 +133,12 @@ Using the `MultiDestinationAuditLogger` you can configure multiple audit logger ---- == Metrics -AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. \ No newline at end of file +AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. The metrics keys are all recorded on the `SECURITY` category, and each metric name are prefixed with a scope of `/auditlogging` and the class name of the logger, e.g. `SolrLogAuditLoggerPlugin`. The individual metrics are: + +* `count` (type: meter. Records number and rate of audit logs done) +* `errors` (type: meter. Records number and rate of errors) +* `requestTimes` (type: timer. Records latency and perceniles for logging performance) +* `totalTime` (type: counter. Records total time spent) +* `queueCapacity` (type: gauge. Records the max size of the async logging queue) +* `queueSize` (type: gauge. Records the number of events currently waiting in the queue) +* `async` (type: gauge. Tells whether this logger is in async mode) From 117ebad4b8f8831b91f1ffd0e3ee64e1d739bef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 29 Mar 2019 10:33:43 +0100 Subject: [PATCH 55/65] RefGuide updates Added new timer metric 'queuedTime' --- .../solr/security/AuditLoggerPlugin.java | 6 ++++++ solr/solr-ref-guide/src/audit-logging.adoc | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 362d3f8ebe23..8130b1635dfc 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Set; @@ -83,6 +84,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo protected Meter numErrors = new Meter(); protected Meter numLogged = new Meter(); protected Timer requestTimes = new Timer(); + protected Timer queuedTime = new Timer(); protected Counter totalTime = new Counter(); // Event types to be logged by default @@ -194,6 +196,9 @@ public void run() { try { AuditEvent event = queue.poll(1000, TimeUnit.MILLISECONDS); if (event == null) continue; + if (event.getDate() != null) { + queuedTime.update(new Date().getTime() - event.getDate().getTime(), TimeUnit.MILLISECONDS); + } Timer.Context timer = requestTimes.time(); audit(event); numLogged.mark(); @@ -241,6 +246,7 @@ public void initializeMetrics(SolrMetricManager manager, String registryName, St if (async) { manager.registerGauge(this, registryName, () -> blockingQueueSize, "queueCapacity", true, "queueCapacity", getCategory().toString(), scope, className); manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(), "queueSize", true, "queueSize", getCategory().toString(), scope, className); + queuedTime = manager.timer(this, registryName, "queuedTime", getCategory().toString(), scope, className); } manager.registerGauge(this, registryName, () -> async, "async", true, "async", getCategory().toString(), scope, className); metricNames.addAll(Arrays.asList("errors", "logged", "requestTimes", "totalTime", "queueCapacity", "queueSize", "async")); diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index bbf8b249c2e0..c2d62b64989e 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -74,9 +74,9 @@ ERROR;Request was not executed due to an error By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. What eventTypes are logged can be configured with the `eventTypes` configuration parameter. === Muting certain events -The configuration parameter `muteRules` lets you mute logging for certain events. You may specify multiple rules and combination of rules that will cause muting. You can mute by event type, request type, username, collection name, path, request parameters or IP address. We'll explain through examples: +The configuration parameter `muteRules` lets you mute logging for certain events. You may specify multiple rules and combination of rules that will cause muting. You can mute by request type, username, collection name, path, request parameters or IP address. We'll explain through examples: -The below example will mute logging for all `UPDATE` requests as well as all requests made my user johndoe or from IP address 192.168.0.10: +The below example will mute logging for all `SEARCH` requests as well as all requests made my user `johndoe` or from IP address `192.168.0.10`: [source,json] ---- @@ -88,7 +88,7 @@ The below example will mute logging for all `UPDATE` requests as well as all req } ---- -An update rule may also be a list, in which case all must be true. The following muteRules will mute update events from IP 192.168.0.10. It will also mute LIST collection admin requests as well as all collection admin requests for the collection named "test": +An mute rule may also be a list, in which case all must be true for muting to happen. The following muteRules will mute all events from IP `192.168.0.10`. It will also mute collection admin requests with action=LIST as well as all collection admin requests for the collection named `test`. Note how you can mix single string rules with lists of rules that must all match: [source,json] ---- @@ -96,7 +96,7 @@ An update rule may also be a list, in which case all must be true. The following "auditlogging":{ "class": "solr.SolrLogAuditLoggerPlugin" "muteRules": [ - [ "type:UPDATE", "ip:192.168.0.10" ], + "ip:192.168.0.10", [ "path:/admin/collections", "param:action=LIST" ], [ "path:/admin/collections", "param:collection=test" ] ] @@ -133,4 +133,13 @@ Using the `MultiDestinationAuditLogger` you can configure multiple audit logger ---- == Metrics -AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. \ No newline at end of file +AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. The metrics keys are all recorded on the `SECURITY` category, and each metric name are prefixed with a scope of `/auditlogging` and the class name of the logger, e.g. `SolrLogAuditLoggerPlugin`. The individual metrics are: + +* `count` (type: meter. Records number and rate of audit logs done) +* `errors` (type: meter. Records number and rate of errors) +* `requestTimes` (type: timer. Records latency and perceniles for logging performance) +* `totalTime` (type: counter. Records total time spent) +* `queueCapacity` (type: gauge. Records the max size of the async logging queue) +* `queueSize` (type: gauge. Records the number of events currently waiting in the queue) +* `queuedTime` (type: timer. Records the amount of time events waited in queue. Adding this with requestTimes you get total time from event to logging complete) +* `async` (type: gauge. Tells whether this logger is in async mode) From 161d24324c5957a3d5ce1c4da9b8614f82d6ddfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 29 Mar 2019 12:17:30 +0100 Subject: [PATCH 56/65] Test for the new timer queuedTime --- .../security/AuditLoggerIntegrationTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 06fda49a9c3a..8f9e295cd9a2 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -23,6 +23,7 @@ import java.net.ServerSocket; import java.net.Socket; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -31,6 +32,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.FileUtils; @@ -111,6 +114,10 @@ public void testAsyncWithQueue() throws Exception { setupCluster(true, 100, false, null); runAdminCommands(); assertAuditMetricsMinimums(testHarness.get().cluster, CallbackAuditLoggerPlugin.class.getSimpleName(), 3, 0); + ArrayList registries = getMetricsReigstries(testHarness.get().cluster); + Timer timer = ((Timer) registries.get(0).getMetrics().get("SECURITY./auditlogging.CallbackAuditLoggerPlugin.queuedTime")); + double meanTimeOnQueue = timer.getSnapshot().getMean() / 1000000; // Convert to ms + assertTrue(meanTimeOnQueue > 50); testHarness.get().shutdownCluster(); assertThreeAdminEvents(); } @@ -209,6 +216,16 @@ private void waitForAuditEventCallbacks(int number) throws InterruptedException } } + private ArrayList getMetricsReigstries(MiniSolrCloudCluster cluster) { + ArrayList registries = new ArrayList<>(); + cluster.getJettySolrRunners().forEach(r -> { + MetricRegistry registry = r.getCoreContainer().getMetricManager().registry("solr.node"); + assertNotNull(registry); + registries.add(registry); + }); + return registries; + } + private void runAdminCommands() throws IOException, SolrServerException { SolrClient client = testHarness.get().cluster.getSolrClient(); CollectionAdminRequest.listCollections(client); From cf56c037adbd6f99aabb42826ca12ff1d80ec7b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 13:40:25 +0200 Subject: [PATCH 57/65] Close socket only if non-null --- .../core/src/java/org/apache/solr/core/CoreContainer.java | 8 ++++---- .../apache/solr/security/CallbackAuditLoggerPlugin.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 7a4d43108c30..541b2514fb27 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -369,7 +369,7 @@ private void initializeAuditloggerPlugin(Map auditConf) { auditConf = Utils.getDeepCopy(auditConf, 4); //Initialize the Auditlog module SecurityPluginHolder old = auditloggerPlugin; - SecurityPluginHolder auditloggerPlugin = null; + SecurityPluginHolder newAuditloggerPlugin = null; if (auditConf != null) { String klas = (String) auditConf.get("class"); if (klas == null) { @@ -379,14 +379,14 @@ private void initializeAuditloggerPlugin(Map auditConf) { return; } log.info("Initializing auditlogger plugin: " + klas); - auditloggerPlugin = new SecurityPluginHolder<>(readVersion(auditConf), + newAuditloggerPlugin = new SecurityPluginHolder<>(readVersion(auditConf), getResourceLoader().newInstance(klas, AuditLoggerPlugin.class)); - auditloggerPlugin.plugin.init(auditConf); + newAuditloggerPlugin.plugin.init(auditConf); } else { log.debug("Security conf doesn't exist. Skipping setup for audit logging module."); } - this.auditloggerPlugin = auditloggerPlugin; + this.auditloggerPlugin = newAuditloggerPlugin; if (old != null) { try { old.plugin.close(); diff --git a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java index 872112982589..c0a829b2681d 100644 --- a/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/CallbackAuditLoggerPlugin.java @@ -70,7 +70,7 @@ public void init(Map pluginConfig) { @Override public void close() throws IOException { + if (socket != null) socket.close(); super.close(); - socket.close(); } } From 40c801a1305f707b059223f394c6e42137f0cbfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 14:03:24 +0200 Subject: [PATCH 58/65] Add 'lost' metric for lost events due to queue full. Change default for numThreads to max(CPU-cores/2, 2) --- .../src/java/org/apache/solr/security/AuditLoggerPlugin.java | 5 ++++- solr/solr-ref-guide/src/audit-logging.adoc | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java index 8130b1635dfc..36affa148b96 100644 --- a/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java @@ -65,7 +65,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo static final String PARAM_NUM_THREADS = "numThreads"; static final String PARAM_MUTE_RULES = "muteRules"; private static final int DEFAULT_QUEUE_SIZE = 4096; - private static final int DEFAULT_NUM_THREADS = 2; + private static final int DEFAULT_NUM_THREADS = Math.max(2, Runtime.getRuntime().availableProcessors() / 2); BlockingQueue queue; boolean async; @@ -82,6 +82,7 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo protected String registryName; protected SolrMetricManager metricManager; protected Meter numErrors = new Meter(); + protected Meter numLost = new Meter(); protected Meter numLogged = new Meter(); protected Timer requestTimes = new Timer(); protected Timer queuedTime = new Timer(); @@ -182,6 +183,7 @@ protected final void auditAsync(AuditEvent event) { } else { if (!queue.offer(event)) { log.warn("Audit log async queue is full (size={}), not blocking since {}", blockingQueueSize, PARAM_BLOCKASYNC + "==false"); + numLost.mark(); } } } @@ -240,6 +242,7 @@ public void initializeMetrics(SolrMetricManager manager, String registryName, St // Metrics registry = manager.registry(registryName); numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope, className); + numLost = manager.meter(this, registryName, "lost", getCategory().toString(), scope, className); numLogged = manager.meter(this, registryName, "count", getCategory().toString(), scope, className); requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope, className); totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope, className); diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index c2d62b64989e..9090c37b0473 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -52,7 +52,7 @@ By default any AuditLogger plugin configured will log asynchronously in the back } ---- -The defaults are `async: true`, `blockAsync: false`, `queueSize: 4096` and `numThreads: 2`. +The defaults are `async: true`, `blockAsync: false`, `queueSize: 4096`. The default for `numThreads` is 2, or if the server has more than 4 CPU-cores then we use CPU-cores/2. [#audit-event-types] === Event types @@ -137,9 +137,12 @@ AuditLoggerPlugins record metrics about count and timing of log requests, as wel * `count` (type: meter. Records number and rate of audit logs done) * `errors` (type: meter. Records number and rate of errors) +* `lost` (type: meter. Records number and rate of events lost due to queue full and `blockAsync=false`) * `requestTimes` (type: timer. Records latency and perceniles for logging performance) * `totalTime` (type: counter. Records total time spent) * `queueCapacity` (type: gauge. Records the max size of the async logging queue) * `queueSize` (type: gauge. Records the number of events currently waiting in the queue) * `queuedTime` (type: timer. Records the amount of time events waited in queue. Adding this with requestTimes you get total time from event to logging complete) * `async` (type: gauge. Tells whether this logger is in async mode) + +TIP: If you expect a very high request rate and have a slow audit logger plugin, you may see that the `queueSize` and `queuedTime` metrics increase, and in worst case start dropping events and see an increase in `lost` count. In this case you may want to increas the `numThreads` setting. \ No newline at end of file From 61f684c548f786837a6f5b2d73e8b043bf3bbe03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 14:09:41 +0200 Subject: [PATCH 59/65] Clarify documentation on muteRules --- solr/solr-ref-guide/src/audit-logging.adoc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adoc index 9090c37b0473..828faf3573bd 100644 --- a/solr/solr-ref-guide/src/audit-logging.adoc +++ b/solr/solr-ref-guide/src/audit-logging.adoc @@ -88,7 +88,7 @@ The below example will mute logging for all `SEARCH` requests as well as all req } ---- -An mute rule may also be a list, in which case all must be true for muting to happen. The following muteRules will mute all events from IP `192.168.0.10`. It will also mute collection admin requests with action=LIST as well as all collection admin requests for the collection named `test`. Note how you can mix single string rules with lists of rules that must all match: +An mute rule may also be a list, in which case all must be true for muting to happen. The configuration below has three mute rules: [source,json] ---- @@ -96,15 +96,21 @@ An mute rule may also be a list, in which case all must be true for muting to ha "auditlogging":{ "class": "solr.SolrLogAuditLoggerPlugin" "muteRules": [ - "ip:192.168.0.10", - [ "path:/admin/collections", "param:action=LIST" ], - [ "path:/admin/collections", "param:collection=test" ] + "ip:192.168.0.10", <1> + [ "path:/admin/collections", "param:action=LIST" ], <2> + [ "path:/admin/collections", "param:collection=test" ] <3> ] } } ---- -Valid rules are: +<1> The first will mute all events from client IP `192.168.0.10` +<2> The second rule will mute collection admin requests with action=LIST +<3> The third rule will mute collection admin requests for the collection named `test` + +Note how you can mix single string rules with lists of rules that must all match: + +*Valid mute rules are:* * `type:` (request-type by name: `ADMIN`, `SEARCH`, `UPDATE`, `STREAMING`, `UNKNOWN`) * `collection:` (collection by name) From 7bba2608dc23847fa0e49051a4f1d0748b590c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 14:39:19 +0200 Subject: [PATCH 60/65] Update the documentation --- ...hentication-and-authorization-plugins.adoc | 36 +++++++++---------- solr/solr-ref-guide/src/securing-solr.adoc | 9 ++--- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index 859968314383..db1ba7585d73 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -1,4 +1,4 @@ -= Configuring security plugins in security.json += Configuring security plugins (Authentication, Authorization, Auditlogging) :page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, rule-based-authorization-plugin, jwt-authentication-plugin // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file @@ -17,17 +17,17 @@ // specific language governing permissions and limitations // under the License. -Solr has security frameworks for supporting authentication and authorization of users. This allows for verifying a user's identity and for restricting access to resources in a Solr cluster. +Solr has security frameworks for supporting authentication, authorization and auditing of users. This allows for verifying a user's identity and for restricting access to resources in a Solr cluster. -Solr includes some plugins out of the box, and additional plugins can be developed using the authentication and authorization frameworks described below. +Solr includes some plugins out of the box, and additional plugins can be developed using the authentication, authorization and auditlogging frameworks described below. -All authentication and authorization plugins can work with Solr whether they are running in SolrCloud mode or standalone mode. All authentication and authorization configuration, including users and permission rules, are stored in a file named `security.json`. When using Solr in standalone mode, this file must be in the `$SOLR_HOME` directory (usually `server/solr`). When using SolrCloud, this file must be located in ZooKeeper. +All authentication, authorization and auditlogging plugins can work with Solr whether they are running in SolrCloud mode or standalone mode. All related configuration, including users and permission rules, are stored in a file named `security.json`. When using Solr in standalone mode, this file must be in the `$SOLR_HOME` directory (usually `server/solr`). When using SolrCloud, this file must be located in ZooKeeper. The following section describes how to enable plugins with `security.json` and place them in the proper locations for your mode of operation. == Enable Plugins with security.json -All of the information required to initialize either type of security plugin is stored in a `security.json` file. This file contains 2 sections, one each for authentication and authorization. +All of the information required to initialize either type of security plugin is stored in a `security.json` file. This file contains 3 sections, one each for authentication, authorization, and auditlogging. .Sample security.json [source,json] @@ -38,6 +38,9 @@ All of the information required to initialize either type of security plugin is }, "authorization": { "class": "class.that.implements.authorization" + }, + "auditlogging": { + "class": "class.that.implements.auditlogging" } } ---- @@ -81,7 +84,7 @@ This example also defines `security.json` on the command line, but you can also [WARNING] ==== -Depending on the authentication and authorization plugin that you use, you may have user information stored in `security.json`. If so, we highly recommend that you implement access control in your ZooKeeper nodes. Information about how to enable this is available in the section <>. +Whenever you use any security plugins and store `security.json` in ZooKeeper, we highly recommend that you implement access control in your ZooKeeper nodes. Information about how to enable this is available in the section <>. ==== Once `security.json` has been uploaded to ZooKeeper, you should use the appropriate APIs for the plugins you're using to update it. You can edit it manually, but you must take care to remove any version data so it will be properly updated across all ZooKeeper nodes. The version data is found at the end of the `security.json` file, and will appear as the letter "v" followed by a number, such as `{"v":138}`. @@ -103,7 +106,7 @@ An authentication plugin consists of two parts: . Server-side component, which intercepts and authenticates incoming requests to Solr using a mechanism defined in the plugin, such as Kerberos, Basic Auth or others. . Client-side component, i.e., an extension of `HttpClientConfigurer`, which enables a SolrJ client to make requests to a secure Solr instance using the authentication mechanism which the server understands. -=== Enabling a Plugin +=== Enabling an authentication Plugin * Specify the authentication plugin in `/security.json` as in this example: + @@ -118,20 +121,13 @@ An authentication plugin consists of two parts: * All of the content in the authentication block of `security.json` would be passed on as a map to the plugin during initialization. * An authentication plugin can also be used with a standalone Solr instance by passing in `-DauthenticationPlugin=` during startup. -=== Available Authentication Plugins - -Solr has the following implementations of authentication plugins: - -* <> -* <> -* <> -* <> +See <> for a list of available authentication plugins. == Authorization An authorization plugin can be written for Solr by extending the {solr-javadocs}/solr-core/org/apache/solr/security/AuthorizationPlugin.html[AuthorizationPlugin] interface. -=== Loading a Custom Plugin +=== Enabling an authorization Plugin * Make sure that the plugin implementation is in the classpath. * The plugin can then be initialized by specifying the same in `security.json` in the following manner: @@ -152,11 +148,13 @@ All of the content in the `authorization` block of `security.json` would be pass The authorization plugin is only supported in SolrCloud mode. Also, reloading the plugin isn't yet supported and requires a restart of the Solr installation (meaning, the JVM should be restarted, not simply a core reload). ==== -=== Available Authorization Plugins +See <> for a list of available authorization plugins. + +== Audit logging -Solr has one implementation of an authorization plugin: +<> plugins helps you keep an audit trail of events happening in your Solr cluster. Audit logging may e.g. ship data to an external audit service. A custom plugin can be implemented by extending the AuditLoggerPlugin class. -* <> +See <> for a list of available audit logger plugins. == Authenticating in the Admin UI diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index 909319a33809..55a11bcbda21 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -19,7 +19,7 @@ When planning how to secure Solr, you should consider which of the available features or approaches are right for you. -== Encryption with TLS certificates +== Encryption with TLS (SSL) certificates Ecrypting traffic to/from Solr and between Solr nodes prevents sensitive data to be leaked out on the network. TLS is also a requirement to secure the password when using Basic Authentication. @@ -29,12 +29,13 @@ See the page <> for details. Plugins for authentication, authorization and audit logging are configured in the `security.json` configuration file, and enabling any of these will immediately take effect across the whole cluster. -Read the chapter <> -to learn how to work with the `security.json` file. +Read the chapter <> to learn how to work with the `security.json` file. [#securing-solr-auth-plugins] === Authentication plugins -Authentication makes sure you know the identity of your users. Supported authentication plugins are: +Authentication makes sure you know the identity of your users. + +Supported authentication plugins are: * <> * <> From 42c1685c8f0140592af01e6805bbb69a89fae74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 15:04:59 +0200 Subject: [PATCH 61/65] Revert rest of refGuide docs --- ...hentication-and-authorization-plugins.adoc | 36 +++++++------- solr/solr-ref-guide/src/securing-solr.adoc | 47 ++++--------------- ...olrcloud-configuration-and-parameters.adoc | 2 +- 3 files changed, 28 insertions(+), 57 deletions(-) diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index db1ba7585d73..ff21ca40eeb7 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -1,4 +1,4 @@ -= Configuring security plugins (Authentication, Authorization, Auditlogging) += Authentication and Authorization Plugins :page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, rule-based-authorization-plugin, jwt-authentication-plugin // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file @@ -17,17 +17,17 @@ // specific language governing permissions and limitations // under the License. -Solr has security frameworks for supporting authentication, authorization and auditing of users. This allows for verifying a user's identity and for restricting access to resources in a Solr cluster. +Solr has security frameworks for supporting authentication and authorization of users. This allows for verifying a user's identity and for restricting access to resources in a Solr cluster. -Solr includes some plugins out of the box, and additional plugins can be developed using the authentication, authorization and auditlogging frameworks described below. +Solr includes some plugins out of the box, and additional plugins can be developed using the authentication and authorization frameworks described below. -All authentication, authorization and auditlogging plugins can work with Solr whether they are running in SolrCloud mode or standalone mode. All related configuration, including users and permission rules, are stored in a file named `security.json`. When using Solr in standalone mode, this file must be in the `$SOLR_HOME` directory (usually `server/solr`). When using SolrCloud, this file must be located in ZooKeeper. +All authentication and authorization plugins can work with Solr whether they are running in SolrCloud mode or standalone mode. All authentication and authorization configuration, including users and permission rules, are stored in a file named `security.json`. When using Solr in standalone mode, this file must be in the `$SOLR_HOME` directory (usually `server/solr`). When using SolrCloud, this file must be located in ZooKeeper. The following section describes how to enable plugins with `security.json` and place them in the proper locations for your mode of operation. == Enable Plugins with security.json -All of the information required to initialize either type of security plugin is stored in a `security.json` file. This file contains 3 sections, one each for authentication, authorization, and auditlogging. +All of the information required to initialize either type of security plugin is stored in a `security.json` file. This file contains 2 sections, one each for authentication and authorization. .Sample security.json [source,json] @@ -38,9 +38,6 @@ All of the information required to initialize either type of security plugin is }, "authorization": { "class": "class.that.implements.authorization" - }, - "auditlogging": { - "class": "class.that.implements.auditlogging" } } ---- @@ -84,7 +81,7 @@ This example also defines `security.json` on the command line, but you can also [WARNING] ==== -Whenever you use any security plugins and store `security.json` in ZooKeeper, we highly recommend that you implement access control in your ZooKeeper nodes. Information about how to enable this is available in the section <>. +Depending on the authentication and authorization plugin that you use, you may have user information stored in `security.json`. If so, we highly recommend that you implement access control in your ZooKeeper nodes. Information about how to enable this is available in the section <>. ==== Once `security.json` has been uploaded to ZooKeeper, you should use the appropriate APIs for the plugins you're using to update it. You can edit it manually, but you must take care to remove any version data so it will be properly updated across all ZooKeeper nodes. The version data is found at the end of the `security.json` file, and will appear as the letter "v" followed by a number, such as `{"v":138}`. @@ -106,7 +103,7 @@ An authentication plugin consists of two parts: . Server-side component, which intercepts and authenticates incoming requests to Solr using a mechanism defined in the plugin, such as Kerberos, Basic Auth or others. . Client-side component, i.e., an extension of `HttpClientConfigurer`, which enables a SolrJ client to make requests to a secure Solr instance using the authentication mechanism which the server understands. -=== Enabling an authentication Plugin +=== Enabling a Plugin * Specify the authentication plugin in `/security.json` as in this example: + @@ -121,13 +118,20 @@ An authentication plugin consists of two parts: * All of the content in the authentication block of `security.json` would be passed on as a map to the plugin during initialization. * An authentication plugin can also be used with a standalone Solr instance by passing in `-DauthenticationPlugin=` during startup. -See <> for a list of available authentication plugins. +=== Available Authentication Plugins + +Solr has the following implementations of authentication plugins: + +* <> +* <> +* <> +* <> == Authorization An authorization plugin can be written for Solr by extending the {solr-javadocs}/solr-core/org/apache/solr/security/AuthorizationPlugin.html[AuthorizationPlugin] interface. -=== Enabling an authorization Plugin +=== Loading a Custom Plugin * Make sure that the plugin implementation is in the classpath. * The plugin can then be initialized by specifying the same in `security.json` in the following manner: @@ -148,13 +152,11 @@ All of the content in the `authorization` block of `security.json` would be pass The authorization plugin is only supported in SolrCloud mode. Also, reloading the plugin isn't yet supported and requires a restart of the Solr installation (meaning, the JVM should be restarted, not simply a core reload). ==== -See <> for a list of available authorization plugins. - -== Audit logging +=== Available Authorization Plugins -<> plugins helps you keep an audit trail of events happening in your Solr cluster. Audit logging may e.g. ship data to an external audit service. A custom plugin can be implemented by extending the AuditLoggerPlugin class. +Solr has one implementation of an authorization plugin: -See <> for a list of available audit logger plugins. +* <> == Authenticating in the Admin UI diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index 55a11bcbda21..fb81e54f553d 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -1,5 +1,5 @@ = Securing Solr -:page-children: authentication-and-authorization-plugins, enabling-ssl, audit-logging, zookeeper-access-control +:page-children: authentication-and-authorization-plugins, enabling-ssl // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -19,44 +19,13 @@ When planning how to secure Solr, you should consider which of the available features or approaches are right for you. -== Encryption with TLS (SSL) certificates -Ecrypting traffic to/from Solr and between Solr nodes prevents sensitive data to be leaked out on -the network. TLS is also a requirement to secure the password when using Basic Authentication. - -See the page <> for details. - -== Authentication, Authorization and Audit logging -Plugins for authentication, authorization and audit logging are configured in the `security.json` configuration file, -and enabling any of these will immediately take effect across the whole cluster. - -Read the chapter <> to learn how to work with the `security.json` file. - -[#securing-solr-auth-plugins] -=== Authentication plugins -Authentication makes sure you know the identity of your users. - -Supported authentication plugins are: - -* <> -* <> -* <> -* <> - -=== Authorization plugins -Authorization makes sure that only users with the necessary roles/permissions can access any given resource. -The authorization plugins shipping with Solr are: - -* <> - -=== Audit logging plugins -Audit logging will record an audit trail of important events in your cluster, such as users being authenticated, -or access being denied to admin APIs. Learn more about audit logging and how to implement an audit logger plugin here: - -* <> - -== Securing Zookeeper traffic -Zookeeper is a central and important part of a SolrCloud cluster and understanding how to secure -its content is covered in the <> page. +* Authentication or authorization of users using: +** <> +** <> +** <> +** <> +* <> +* If using SolrCloud, <> [WARNING] ==== diff --git a/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc b/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc index 6f739f022836..0d79a261ece3 100644 --- a/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc +++ b/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc @@ -1,5 +1,5 @@ = SolrCloud Configuration and Parameters -:page-children: setting-up-an-external-zookeeper-ensemble, using-zookeeper-to-manage-configuration-files, collections-api, parameter-reference, command-line-utilities, solrcloud-with-legacy-configuration-files, configsets-api +:page-children: setting-up-an-external-zookeeper-ensemble, using-zookeeper-to-manage-configuration-files, zookeeper-access-control, collections-api, parameter-reference, command-line-utilities, solrcloud-with-legacy-configuration-files, configsets-api // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From 34c4ee3ba1c8855cc9dfd258ea3a89f754f91e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 15:06:18 +0200 Subject: [PATCH 62/65] Link to the new audit-logging page --- solr/solr-ref-guide/src/securing-solr.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index fb81e54f553d..dc7058284ba5 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -26,6 +26,7 @@ When planning how to secure Solr, you should consider which of the available fea ** <> * <> * If using SolrCloud, <> +* <> for recording an audit trail [WARNING] ==== From 43994ea3d677ad722c30146c0ee3e95b14fe3cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 15:13:30 +0200 Subject: [PATCH 63/65] Move zookeeper-access-control.adoc underneath securing-solr --- solr/solr-ref-guide/src/securing-solr.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index dc7058284ba5..35665933a7a7 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -1,5 +1,5 @@ = Securing Solr -:page-children: authentication-and-authorization-plugins, enabling-ssl +:page-children: authentication-and-authorization-plugins, enabling-ssl, zookeeper-access-control.adoc // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From 455337074481b4d5d818b78c1a33a240bc35e9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 15:15:29 +0200 Subject: [PATCH 64/65] Revert "Move zookeeper-access-control.adoc underneath securing-solr" This reverts commit 43994ea --- solr/solr-ref-guide/src/securing-solr.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index 35665933a7a7..dc7058284ba5 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -1,5 +1,5 @@ = Securing Solr -:page-children: authentication-and-authorization-plugins, enabling-ssl, zookeeper-access-control.adoc +:page-children: authentication-and-authorization-plugins, enabling-ssl // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From 99f753b9efc107b8e06a65fa5ab6ef9215254bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 4 Apr 2019 15:31:15 +0200 Subject: [PATCH 65/65] Hook page up with securing-solr --- solr/solr-ref-guide/src/securing-solr.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adoc index dc7058284ba5..daffc541213b 100644 --- a/solr/solr-ref-guide/src/securing-solr.adoc +++ b/solr/solr-ref-guide/src/securing-solr.adoc @@ -1,5 +1,5 @@ = Securing Solr -:page-children: authentication-and-authorization-plugins, enabling-ssl +:page-children: authentication-and-authorization-plugins, enabling-ssl, audit-logging // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information