From 2d3a7f4112509c9581d97c36a088715bd0aebca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 23 Mar 2018 17:34:50 +0100 Subject: [PATCH 01/88] SOLR-12121: JWT Token authentication plugin --- lucene/ivy-versions.properties | 4 +- solr/CHANGES.txt | 2 + solr/NOTICE.txt | 18 + solr/core/ivy.xml | 3 + .../security/BearerAuthSchemeProvider.java | 93 ++++ .../apache/solr/security/JWTAuthPlugin.java | 445 ++++++++++++++++++ .../solr/security/PrincipalWithUserRoles.java | 95 ++++ .../solr/security/TokenCredentials.java | 43 ++ .../solr/security/VerifiedUserRoles.java | 33 ++ .../security/jwt_plugin_jwk_security.json | 11 + .../security/jwt_plugin_jwk_url_security.json | 6 + .../solr/security/JWTAuthPluginTest.java | 331 +++++++++++++ solr/licenses/jose4j-0.6.3.jar.sha1 | 1 + solr/licenses/jose4j-LICENSE-ASL.txt | 272 +++++++++++ solr/licenses/jose4j-NOTICE.txt | 13 + ...hentication-and-authorization-plugins.adoc | 1 + .../src/jwt-authentication-plugin.adoc | 87 ++++ 17 files changed, 1457 insertions(+), 1 deletion(-) create mode 100644 solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java create mode 100644 solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java create mode 100644 solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java create mode 100644 solr/core/src/java/org/apache/solr/security/TokenCredentials.java create mode 100644 solr/core/src/java/org/apache/solr/security/VerifiedUserRoles.java create mode 100644 solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json create mode 100644 solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json create mode 100644 solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java create mode 100644 solr/licenses/jose4j-0.6.3.jar.sha1 create mode 100644 solr/licenses/jose4j-LICENSE-ASL.txt create mode 100644 solr/licenses/jose4j-NOTICE.txt create mode 100644 solr/solr-ref-guide/src/jwt-authentication-plugin.adoc diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index afbab2041ff5..b4d070a29c4d 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -221,6 +221,8 @@ org.apache.uima.version = 2.3.1 /org.aspectj/aspectjrt = 1.8.0 +/org.bitbucket.b_c/jose4j = 0.6.3 + org.bouncycastle.version = 1.54 /org.bouncycastle/bcmail-jdk15on = ${org.bouncycastle.version} /org.bouncycastle/bcpkix-jdk15on = ${org.bouncycastle.version} @@ -310,4 +312,4 @@ org.slf4j.version = 1.7.24 ua.net.nlp.morfologik-ukrainian-search.version = 3.9.0 /ua.net.nlp/morfologik-ukrainian-search = ${ua.net.nlp.morfologik-ukrainian-search.version} -/xerces/xercesImpl = 2.9.1 +/xerces/xercesImpl = 2.9.1 \ No newline at end of file diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 866c46fcceed..135b1c071d3a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -61,6 +61,8 @@ New Features * SOLR-12015: Add support "add-distinct" in AtomicURP so that we can use the 'add-distict' as a request parameter e.g: atomic.=add-distict (Amrit Sarkar via noble) +* SOLR-12121: JWT Token authentication plugin (janhoy) + Bug Fixes ---------------------- diff --git a/solr/NOTICE.txt b/solr/NOTICE.txt index fd954f4ef4f6..62328d2c9641 100644 --- a/solr/NOTICE.txt +++ b/solr/NOTICE.txt @@ -537,3 +537,21 @@ See http://www.restlet.org/ Protocol Buffers - Google's data interchange format Copyright 2008 Google Inc. http://code.google.com/apis/protocolbuffers/ + +========================================================================= +== Jose4j Notice == +========================================================================= + +jose4j +Copyright 2012-2015 Brian Campbell + +EcdsaUsingShaAlgorithm contains code for converting the concatenated +R & S values of the signature to and from DER, which was originally +derived from the Apache Santuario XML Security library's SignatureECDSA +implementation. http://santuario.apache.org/ + +The Base64 implementation in this software was derived from the +Apache Commons Codec project. http://commons.apache.org/proper/commons-codec/ + +JSON processing in this software was derived from the JSON.simple toolkit. +https://code.google.com/p/json-simple/ \ No newline at end of file diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index ff4fa48679dd..fb7ad019392e 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -151,6 +151,9 @@ + + + diff --git a/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java b/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java new file mode 100644 index 000000000000..d64566b89ebc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import org.apache.http.Header; +import org.apache.http.HttpRequest; +import org.apache.http.auth.AUTH; +import org.apache.http.auth.AuthScheme; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthenticationException; +import org.apache.http.auth.ContextAwareAuthScheme; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.MalformedChallengeException; +import org.apache.http.message.BufferedHeader; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.CharArrayBuffer; + +public class BearerAuthSchemeProvider implements AuthSchemeProvider { + + /** + * Creates an instance of {@link AuthScheme}. + * + * @param context the http context + * @return auth scheme. + */ + @Override + public AuthScheme create(HttpContext context) { + return new BearerAuthScheme(); + } + + private static class BearerAuthScheme implements ContextAwareAuthScheme { + private boolean complete = false; + + @Override + public void processChallenge(Header header) throws MalformedChallengeException { + this.complete = true; + } + + @Override + public Header authenticate(Credentials credentials, HttpRequest request) throws AuthenticationException { + return authenticate(credentials, request, null); + } + + @Override + public Header authenticate(Credentials credentials, HttpRequest request, HttpContext httpContext) + throws AuthenticationException { + CharArrayBuffer buffer = new CharArrayBuffer(128); + buffer.append(AUTH.WWW_AUTH_RESP); + buffer.append(": Bearer "); + buffer.append(credentials.getUserPrincipal().getName()); + return new BufferedHeader(buffer); + } + + @Override + public String getSchemeName() { + return "Bearer"; + } + + @Override + public String getParameter(String name) { + return null; + } + + @Override + public String getRealm() { + return null; + } + + @Override + public boolean isConnectionBased() { + return false; + } + + @Override + public boolean isComplete() { + return this.complete; + } + } +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java new file mode 100644 index 000000000000..6dd810a51ecc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import javax.servlet.FilterChain; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.Principal; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.regex.Pattern; + +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.BasicUserPrincipal; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.config.Lookup; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SpecProvider; +import org.apache.solr.common.util.Utils; +import org.apache.solr.common.util.ValidatingJsonMap; +import org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode; +import org.jose4j.jwa.AlgorithmConstraints; +import org.jose4j.jwk.HttpsJwks; +import org.jose4j.jwk.JsonWebKey; +import org.jose4j.jwk.JsonWebKeySet; +import org.jose4j.jwt.JwtClaims; +import org.jose4j.jwt.MalformedClaimException; +import org.jose4j.jwt.consumer.InvalidJwtException; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtConsumerBuilder; +import org.jose4j.keys.resolvers.HttpsJwksVerificationKeyResolver; +import org.jose4j.keys.resolvers.JwksVerificationKeyResolver; +import org.jose4j.keys.resolvers.VerificationKeyResolver; +import org.jose4j.lang.JoseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Authenticaion plugin that finds logged in user by validating the signature of a JWT token + */ +public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin, SpecProvider { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final static ThreadLocal
authHeader = new ThreadLocal<>(); + private static final String PARAM_BLOCK_UNKNOWN = "block_unknown"; + private static final String PARAM_JWK_URL = "jwk_url"; + private static final String PARAM_JWK = "jwk"; + private static final String PARAM_ISSUER = "iss"; + private static final String PARAM_AUDIENCE = "aud"; + private static final String PARAM_REQUIRE_SUBJECT = "require_sub"; + private static final String PARAM_PRINCIPAL_CLAIM = "principal_claim"; + private static final String PARAM_ROLES_CLAIM = "roles_claim"; + private static final String PARAM_REQUIRE_EXPIRATIONTIME = "require_exp"; + private static final String PARAM_ALG_WHITELIST = "alg_whitelist"; + private static final String PARAM_JWK_CACHE_DURATION = "jwk_cache_dur"; + private static final String PARAM_CLAIMS_MATCH = "claims_match"; + private static final String AUTH_REALM = "solr"; + + private String jwk_url; + private Map jwk; + private JwtConsumer jwtConsumer; + private String iss; + private String aud; + private boolean requireSubject; + private boolean requireExpirationTime; + private List algWhitelist; + private HttpsJwks httpsJkws; + private long jwkCacheDuration; + VerificationKeyResolver verificationKeyResolver; + private JsonWebKeySet jwks; + private String principalClaim; + private String rolesClaim; + private Map claimsMatch; + private HashMap claimsMatchCompiled; + private boolean blockUnknown; + + @Override + public void init(Map pluginConfig) { + blockUnknown = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCK_UNKNOWN, false))); + jwk_url = (String) pluginConfig.get(PARAM_JWK_URL); + jwk = (Map) pluginConfig.get(PARAM_JWK); + iss = (String) pluginConfig.get(PARAM_ISSUER); + aud = (String) pluginConfig.get(PARAM_AUDIENCE); + requireSubject = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_SUBJECT, "true"))); + requireExpirationTime = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_EXPIRATIONTIME, "true"))); + algWhitelist = (List) pluginConfig.get(PARAM_ALG_WHITELIST); + jwkCacheDuration = Long.parseLong((String) pluginConfig.getOrDefault(PARAM_JWK_CACHE_DURATION, "3600")); + principalClaim = (String) pluginConfig.getOrDefault(PARAM_PRINCIPAL_CLAIM, "sub"); + rolesClaim = (String) pluginConfig.get(PARAM_ROLES_CLAIM); + claimsMatch = (Map) pluginConfig.get(PARAM_CLAIMS_MATCH); + claimsMatchCompiled = new HashMap<>(); + if (claimsMatch != null) { + for (Map.Entry entry : claimsMatch.entrySet()) { + claimsMatchCompiled.put(entry.getKey(), Pattern.compile(entry.getValue())); + } + } + + jwtConsumer = null; + if (jwk_url != null) { + // The HttpsJwks retrieves and caches keys from a the given HTTPS JWKS endpoint. + try { + URL jwkUrl = new URL(jwk_url); + if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be an HTTPS url"); + } + } catch (MalformedURLException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be a valid https URL"); + } + httpsJkws = new HttpsJwks(jwk_url); + httpsJkws.setDefaultCacheDuration(jwkCacheDuration); + verificationKeyResolver = new HttpsJwksVerificationKeyResolver(httpsJkws); + initConsumer(); + } else if (jwk != null) { + try { + jwks = parseJwkSet(jwk); + verificationKeyResolver = new JwksVerificationKeyResolver(jwks.getJsonWebKeys()); + initConsumer(); + } catch (JoseException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid JWTAuthPlugin configuration, jwk field parse error", e); + } + } else { + log.warn("JWTAuthPlugin needs to specify either 'jwk' or 'jwk_url' parameters."); + } + } + + protected JsonWebKeySet parseJwkSet(Map jwkObj) throws JoseException { + JsonWebKeySet webKeySet = new JsonWebKeySet(); + if (jwkObj.containsKey("keys")) { + List jwkList = (List) jwkObj.get("keys"); + for (Object jwkO : jwkList) { + webKeySet.addJsonWebKey(JsonWebKey.Factory.newJwk((Map) jwkO)); + } + } else { + webKeySet = new JsonWebKeySet(JsonWebKey.Factory.newJwk(jwkObj)); + } + return webKeySet; + } + + @Override + public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws Exception { + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + + String header = request.getHeader(HttpHeaders.AUTHORIZATION); + + if (jwtConsumer == null) { + // + if (header == null && !blockUnknown) { + log.info("JWTAuth not configured, but allowing anonymous access since blockUnknown==false"); + response.setHeader(HttpHeaders.WWW_AUTHENTICATE, "Bearer realm=\"" + AUTH_REALM + "\""); + filterChain.doFilter(request, response); + return true; + } + log.warn("JWTAuth not configured"); + authenticationFailure(response, "JWTAuth not configured"); + return false; + } + + if (header != null) { + // Put the header on the thread for later use in inter-node request + authHeader.set(new BasicHeader(HttpHeaders.AUTHORIZATION, header)); + } + AuthenticationResponse authResponse = authenticate(header); + switch(authResponse.authCode) { + case AUTHENTICATED: + HttpServletRequestWrapper wrapper = new HttpServletRequestWrapper(request) { + @Override + public Principal getUserPrincipal() { + return authResponse.getPrincipal(); + } + }; + filterChain.doFilter(wrapper, response); + return true; + + case PASS_THROUGH: + log.debug("Unknown user, but allow due to block_unknown=false"); + response.setHeader(HttpHeaders.WWW_AUTHENTICATE, "Bearer realm=\"" + AUTH_REALM + "\""); + filterChain.doFilter(request, response); + return true; + + default: + log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); + if (authResponse.authCode.equals(AuthCode.JWT_VALIDATION_EXCEPTION)) { + log.debug("Exception: {}", authResponse.getJwtException().getMessage()); + } + authenticationFailure(response, authResponse.getAuthCode().msg); + return false; + } + } + + /** + * Testable authentication method + * + * @param authorizationHeader the http header "Authentication" + * @return AuthenticationResponse object + */ + protected AuthenticationResponse authenticate(String authorizationHeader) { + if (authorizationHeader != null) { + StringTokenizer st = new StringTokenizer(authorizationHeader); + if (st.hasMoreTokens()) { + String bearer = st.nextToken(); + if (bearer.equalsIgnoreCase("Bearer") && st.hasMoreTokens()) { + try { + String jwtCompact = st.nextToken(); + try { + JwtClaims jwtClaims = jwtConsumer.processToClaims(jwtCompact); + String principal = jwtClaims.getStringClaimValue(principalClaim); + if (principal == null || principal.isEmpty()) { + return new AuthenticationResponse(AuthCode.PRINCIPAL_MISSING, "Cannot identify principal from JWT. Required claim " + principalClaim + " missing. Cannot authenticate"); + } + if (claimsMatchCompiled != null) { + for (Map.Entry entry : claimsMatchCompiled.entrySet()) { + String claim = entry.getKey(); + if (jwtClaims.hasClaim(claim)) { + if (!entry.getValue().matcher(jwtClaims.getStringClaimValue(claim)).matches()) { + return new AuthenticationResponse(AuthCode.CLAIM_MISMATCH, + "Claim " + claim + "=" + jwtClaims.getStringClaimValue(claim) + + " does not match required regular expression " + entry.getValue().pattern()); + } + } else { + return new AuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Claim " + claim + " is required but does not exist in JWT"); + } + } + } + Set roles = Collections.emptySet(); + if (rolesClaim != null) { + if (!jwtClaims.hasClaim(rolesClaim)) { + return new AuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Roles claim " + rolesClaim + " is required but does not exist in JWT"); + } + Object rolesObj = jwtClaims.getClaimValue(rolesClaim); + if (rolesObj instanceof String) { + roles = Collections.singleton((String) rolesObj); + } else if (rolesObj instanceof List) { + roles = new HashSet<>(jwtClaims.getStringListClaimValue(rolesClaim)); + } + // Pass roles with principal to signal to any Authorization plugins that user has some verified role claims + return new AuthenticationResponse(AuthCode.AUTHENTICATED, new PrincipalWithUserRoles(principal, roles)); + } else { + return new AuthenticationResponse(AuthCode.AUTHENTICATED, new BasicUserPrincipal(principal)); + } + } catch (InvalidJwtException e) { + // Whether or not the JWT has expired being one common reason for invalidity + if (e.hasExpired()) { + return new AuthenticationResponse(AuthCode.JWT_EXPIRED, "Authentication failed due to expired JWT token. Expired at " + e.getJwtContext().getJwtClaims().getExpirationTime()); + } + return new AuthenticationResponse(AuthCode.JWT_VALIDATION_EXCEPTION, e); + } + } catch (MalformedClaimException e) { + return new AuthenticationResponse(AuthCode.JWT_PARSE_ERROR, "Malformed claim, error was: " + e.getMessage()); + } + } else { + return new AuthenticationResponse(AuthCode.AUTZ_HEADER_PROBLEM, "Authorization header is not in correct format"); + } + } else { + return new AuthenticationResponse(AuthCode.AUTZ_HEADER_PROBLEM, "Authorization header is not in correct format"); + } + } else { + // No Authorization header + if (blockUnknown) { + return new AuthenticationResponse(AuthCode.NO_AUTZ_HEADER, "Missing Authorization header"); + } else { + log.debug("No user authenticated, but block_unknown=false, so letting request through"); + return new AuthenticationResponse(AuthCode.PASS_THROUGH); + } + } + } + + private void initConsumer() { + JwtConsumerBuilder jwtConsumerBuilder = new JwtConsumerBuilder() + .setAllowedClockSkewInSeconds(30); // allow some leeway in validating time based claims to account for clock skew + if (iss != null) + jwtConsumerBuilder.setExpectedIssuer(iss); // whom the JWT needs to have been issued by + if (aud != null) { + jwtConsumerBuilder.setExpectedAudience(aud); // to whom the JWT is intended for + } else { + jwtConsumerBuilder.setSkipDefaultAudienceValidation(); + } + if (requireSubject) + jwtConsumerBuilder.setRequireSubject(); + if (requireExpirationTime) + jwtConsumerBuilder.setRequireExpirationTime(); + if (algWhitelist != null) + jwtConsumerBuilder.setJwsAlgorithmConstraints( // only allow the expected signature algorithm(s) in the given context + new AlgorithmConstraints(AlgorithmConstraints.ConstraintType.WHITELIST, algWhitelist.toArray(new String[0]))); + jwtConsumerBuilder.setVerificationKeyResolver(verificationKeyResolver); + jwtConsumer = jwtConsumerBuilder.build(); // create the JwtConsumer instance + } + + @Override + public void close() throws IOException { + + } + + @Override + public void closeRequest() { + authHeader.remove(); + } + + /** + * Gets a client builder for inter-node requests + * @param builder any existing builder or null to create a new one + * @return Returns an instance of a SolrHttpClientBuilder to be used for configuring the + * HttpClients for use with SolrJ clients. + * @lucene.experimental + */ + @Override + public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { + if (builder == null) { + builder = SolrHttpClientBuilder.create(); + } + builder.setAuthSchemeRegistryProvider(() -> { + Lookup authProviders = RegistryBuilder.create() + .register("Bearer", new BearerAuthSchemeProvider()) + .build(); + return authProviders; + }); + builder.setDefaultCredentialsProvider(() -> { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + // Pull the authorization bearer header from ThreadLocal + if (authHeader.get() == null) { + log.warn("Cannot find Authorization header on request thread"); + } else { + // TODO: Limit AuthScope? + credentialsProvider.setCredentials(AuthScope.ANY, new TokenCredentials(authHeader.get().getValue())); + } + return credentialsProvider; + }); + return builder; + } + + // NOCOMMIT: v2 api documentation + @Override + public ValidatingJsonMap getSpec() { + return Utils.getSpec("cluster.security.BasicAuth.Commands").getSpec(); + } + + private void authenticationFailure(HttpServletResponse response, String message) throws IOException { + response.setHeader(HttpHeaders.WWW_AUTHENTICATE, "Bearer realm=\"" + AUTH_REALM + "\""); + response.sendError(401, message); + } + + /** + * Response for authentication attempt + */ + static class AuthenticationResponse { + private final Principal principal; + private String errorMessage; + private AuthCode authCode; + private InvalidJwtException jwtException; + + enum AuthCode { + PASS_THROUGH("No user, pass through"), // Returned when no user authentication but block_unknown=false + AUTHENTICATED("Authenticated"), // Returned when authentication OK + PRINCIPAL_MISSING("No principal in JWT"), // JWT token does not contain necessary principal (typically sub) + JWT_PARSE_ERROR("Invalid JWT"), // Problems with parsing the JWT itself + AUTZ_HEADER_PROBLEM("Wrong header"), // The Authorization header exists but is not correct + NO_AUTZ_HEADER("Require authentication"), // The Authorization header is missing + JWT_EXPIRED("JWT token expired"), // JWT token has expired + CLAIM_MISMATCH("Required JWT claim missing"), // Some required claims are missing or wrong + JWT_VALIDATION_EXCEPTION("JWT validation failed"); // The JWT parser failed validation. More details in exception + + private final String msg; + + AuthCode(String msg) { + this.msg = msg; + } + } + + public AuthenticationResponse(AuthCode authCode, InvalidJwtException e) { + this.authCode = authCode; + this.jwtException = e; + principal = null; + this.errorMessage = e.getMessage(); + } + + public AuthenticationResponse(AuthCode authCode, String errorMessage) { + this.authCode = authCode; + this.errorMessage = errorMessage; + principal = null; + } + + public AuthenticationResponse(AuthCode authCode, Principal principal) { + this.authCode = authCode; + this.principal = principal; + } + + public AuthenticationResponse(AuthCode authCode) { + this.authCode = authCode; + principal = null; + } + + public boolean isAuthenticated() { + return authCode.equals(AuthCode.AUTHENTICATED); + } + + public Principal getPrincipal() { + return principal; + } + + public String getErrorMessage() { + return errorMessage; + } + + public InvalidJwtException getJwtException() { + return jwtException; + } + + public AuthCode getAuthCode() { + return authCode; + } + } + +} diff --git a/solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java b/solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java new file mode 100644 index 000000000000..6d685a8f1bf2 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.Serializable; +import java.security.Principal; +import java.util.Set; + +import org.apache.http.util.Args; + +/** + * Type of Principal object that can contain also a list of roles the user has. + * One use case can be to keep track of user-role mappings in an Identity Server + * external to Solr and pass the information to Solr in a signed JWT token or in + * another secure manner. The role information can then be used to authorize + * requests without the need to maintain or lookup what roles each user belongs to. + */ +public class PrincipalWithUserRoles implements Principal, VerifiedUserRoles, Serializable { + private static final long serialVersionUID = 4144666467522831388L; + private final String username; + + private final Set roles; + + /** + * User principal with user name as well as one or more roles that he/she belong to + * @param username string with user name for user + * @param roles a set of roles that we know this user belongs to, or empty list for no roles + */ + public PrincipalWithUserRoles(final String username, Set roles) { + super(); + Args.notNull(username, "User name"); + Args.notNull(roles, "User roles"); + this.username = username; + this.roles = roles; + } + + /** + * Returns the name of this principal. + * + * @return the name of this principal. + */ + @Override + public String getName() { + return this.username; + } + + /** + * Gets the list of roles + */ + @Override + public Set getVerifiedRoles() { + return roles; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + PrincipalWithUserRoles that = (PrincipalWithUserRoles) o; + + if (!username.equals(that.username)) return false; + return roles.equals(that.roles); + } + + @Override + public int hashCode() { + int result = username.hashCode(); + result = 31 * result + roles.hashCode(); + return result; + } + + @Override + public String toString() { + return "PrincipalWithUserRoles{" + + "username='" + username + '\'' + + ", roles=" + roles + + '}'; + } +} + diff --git a/solr/core/src/java/org/apache/solr/security/TokenCredentials.java b/solr/core/src/java/org/apache/solr/security/TokenCredentials.java new file mode 100644 index 000000000000..6433fd622d2f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/TokenCredentials.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.security.Principal; + +import org.apache.http.auth.BasicUserPrincipal; +import org.apache.http.auth.Credentials; + +/** + * Credentials implementation that holds on to a (JWT) token for later use in inter-node requests + */ +public class TokenCredentials implements Credentials { + private Principal userPrincipal; + + public TokenCredentials(String token) { + this.userPrincipal = new BasicUserPrincipal(token); + } + + @Override + public Principal getUserPrincipal() { + return userPrincipal; + } + + @Override + public String getPassword() { + return null; + } +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/security/VerifiedUserRoles.java b/solr/core/src/java/org/apache/solr/security/VerifiedUserRoles.java new file mode 100644 index 000000000000..ed574e076340 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/VerifiedUserRoles.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.util.Set; + +/** + * Interface used to pass verified user roles in a Principal object. + * An Authorization plugin may check for the presence of verified user + * roles on the Principal and choose to use those roles instead of + * explicitly configuring roles in config. Such roles may e.g. origin + * from a signed and validated JWT token. + */ +public interface VerifiedUserRoles { + /** + * Gets a set of roles that have been verified to belong to a user + */ + Set getVerifiedRoles(); +} diff --git a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json new file mode 100644 index 000000000000..b200361ec8d4 --- /dev/null +++ b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json @@ -0,0 +1,11 @@ +{ + "authentication": { + "class": "solr.JWTAuthPlugin", + "jwk": { + "e": "AQAB", + "kid": "k1", + "kty": "RSA", + "n": "3ZF6wBGPMsLzsS1KLghxaVpZtXD3nTLzDm0c974i9-KNU_1rhhBeiVfS64VfEQmP8SA470jEy7yWcvnz9GvG-YAlm9iOwVF7jLl2awdws0ocFjdSPT3SjPQKzOeMO7G9XqNTkrvoFCn1YAi26fbhhcqkwZDoeTcHQdRN32frzccuPhZrwImApIedroKLlKWv2IvPDnz2Bpe2WWVc2HdoWYqEVD3p_BEy8f-RTSHK3_8kDDF9yAwI9jx7CK1_C-eYxXltm-6rpS5NGyFm0UNTZMxVU28Tl7LX8Vb6CikyCQ9YRCtk_CvpKWmEuKEp9I28KHQNmGkDYT90nt3vjbCXxw" + } + } +} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json new file mode 100644 index 000000000000..0431ab86335b --- /dev/null +++ b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json @@ -0,0 +1,6 @@ +{ + "authentication" : { + "class": "solr.JWTAuthPlugin", + "jwk_url": "https://127.0.0.1:8999/this-will-fail.wks" + } +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java new file mode 100644 index 000000000000..e1065a8393ee --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.Principal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.Utils; +import org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse; +import org.jose4j.jwk.RsaJsonWebKey; +import org.jose4j.jwk.RsaJwkGenerator; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jws.JsonWebSignature; +import org.jose4j.jwt.JwtClaims; +import org.jose4j.keys.BigEndianBigInteger; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode.AUTZ_HEADER_PROBLEM; +import static org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode.NO_AUTZ_HEADER; + +public class JWTAuthPluginTest extends SolrTestCaseJ4 { + private static String testHeader; + private static String slimJwt; + private static String slimHeader; + private JWTAuthPlugin plugin; + private HashMap testJwk; + private static RsaJsonWebKey rsaJsonWebKey; + private static String testJwt; + private HashMap testConfig; + + + @BeforeClass + public static void beforeAll() throws Exception { + // Generate an RSA key pair, which will be used for signing and verification of the JWT, wrapped in a JWK + rsaJsonWebKey = RsaJwkGenerator.generateJwk(2048); + rsaJsonWebKey.setKeyId("k1"); + + JwtClaims claims = generateClaims(); + JsonWebSignature jws = new JsonWebSignature(); + jws.setPayload(claims.toJson()); + jws.setKey(rsaJsonWebKey.getPrivateKey()); + jws.setKeyIdHeaderValue(rsaJsonWebKey.getKeyId()); + jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); + + testJwt = jws.getCompactSerialization(); + testHeader = "Bearer" + " " + testJwt; + + System.out.println("Header:\n" + testHeader); + System.out.println("JWK:\n" + rsaJsonWebKey.toJson()); + + claims.unsetClaim("iss"); + claims.unsetClaim("aud"); + claims.unsetClaim("exp"); + jws.setPayload(claims.toJson()); + slimJwt = jws.getCompactSerialization(); + slimHeader = "Bearer" + " " + slimJwt; + } + + private static JwtClaims generateClaims() { + JwtClaims claims = new JwtClaims(); + claims.setIssuer("IDServer"); // who creates the token and signs it + claims.setAudience("Solr"); // to whom the token is intended to be sent + claims.setExpirationTimeMinutesInTheFuture(10); // time when the token will expire (10 minutes from now) + claims.setGeneratedJwtId(); // a unique identifier for the token + claims.setIssuedAtToNow(); // when the token was issued/created (now) + claims.setNotBeforeMinutesInThePast(2); // time before which the token is not yet valid (2 minutes ago) + claims.setSubject("solruser"); // the subject/principal is whom the token is about + claims.setClaim("name", "Solr User"); // additional claims/attributes about the subject can be added + claims.setClaim("customPrincipal", "custom"); // additional claims/attributes about the subject can be added + claims.setClaim("claim1", "foo"); // additional claims/attributes about the subject can be added + claims.setClaim("claim2", "bar"); // additional claims/attributes about the subject can be added + claims.setClaim("claim3", "foo"); // additional claims/attributes about the subject can be added + List groups = Arrays.asList("group-one", "other-group", "group-three"); + claims.setStringListClaim("groups", groups); // multi-valued claims work too and will end up as a JSON array + return claims; + } + + @Before + public void setUp() throws Exception { + super.setUp(); + // Create an auth plugin + plugin = new JWTAuthPlugin(); + + // Create a JWK config for security.json + testJwk = new HashMap<>(); + testJwk.put("kty", rsaJsonWebKey.getKeyType()); + testJwk.put("e", BigEndianBigInteger.toBase64Url(rsaJsonWebKey.getRsaPublicKey().getPublicExponent())); + testJwk.put("use", rsaJsonWebKey.getUse()); + testJwk.put("kid", rsaJsonWebKey.getKeyId()); + testJwk.put("alg", rsaJsonWebKey.getAlgorithm()); + testJwk.put("n", BigEndianBigInteger.toBase64Url(rsaJsonWebKey.getRsaPublicKey().getModulus())); + + testConfig = new HashMap<>(); + testConfig.put("class", "org.apache.solr.security.JWTAuthPlugin"); + testConfig.put("jwk", testJwk); + plugin.init(testConfig); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + plugin.close(); + } + + @Test + public void initWithoutRequired() throws Exception { + HashMap authConf = new HashMap(); + plugin = new JWTAuthPlugin(); + plugin.init(authConf); + assertEquals(AUTZ_HEADER_PROBLEM, plugin.authenticate("foo").getAuthCode()); + } + + @Test + public void initFromSecurityJSONLocalJWK() throws Exception { + Path securityJson = TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json"); + InputStream is = Files.newInputStream(securityJson); + Map securityConf = (Map) Utils.fromJSON(is); + Map authConf = (Map) securityConf.get("authentication"); + plugin.init(authConf); + } + + @Test + public void initFromSecurityJSONUrlJwk() throws Exception { + Path securityJson = TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_url_security.json"); + InputStream is = Files.newInputStream(securityJson); + Map securityConf = (Map) Utils.fromJSON(is); + Map authConf = (Map) securityConf.get("authentication"); + plugin.init(authConf); + + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + assertTrue(resp.getJwtException().getMessage().contains("Connection refused")); + } + + @Test + public void initWithJwk() throws Exception { + HashMap authConf = new HashMap(); + authConf.put("jwk", testJwk); + plugin = new JWTAuthPlugin(); + plugin.init(authConf); + } + + @Test + public void initWithJwkUrl() throws Exception { + HashMap authConf = new HashMap(); + authConf.put("jwk_url", "https://127.0.0.1:9999/foo.jwk"); + plugin = new JWTAuthPlugin(); + plugin.init(authConf); + } + + @Test + public void parseJwkSet() throws Exception { + plugin.parseJwkSet(testJwk); + + HashMap testJwks = new HashMap<>(); + List> keys = new ArrayList<>(); + keys.add(testJwk); + testJwks.put("keys", keys); + plugin.parseJwkSet(testJwks); + } + + @Test + public void authenticateOk() throws Exception { + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertTrue(resp.isAuthenticated()); + assertEquals("solruser", resp.getPrincipal().getName()); + } + + @Test + public void authFailedMissingSubject() throws Exception { + testConfig.put("iss", "NA"); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertFalse(resp.isAuthenticated()); + assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + + testConfig.put("iss", "IDServer"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertTrue(resp.isAuthenticated()); + } + + @Test + public void authFailedMissingAudience() throws Exception { + testConfig.put("aud", "NA"); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertFalse(resp.isAuthenticated()); + assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + + testConfig.put("aud", "Solr"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertTrue(resp.isAuthenticated()); + } + + @Test + public void authFailedMissingPrincipal() throws Exception { + testConfig.put("principal_claim", "customPrincipal"); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertTrue(resp.isAuthenticated()); + + testConfig.put("principal_claim", "NA"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertFalse(resp.isAuthenticated()); + assertEquals(AuthenticationResponse.AuthCode.PRINCIPAL_MISSING, resp.getAuthCode()); + } + + @Test + public void claimMatch() throws Exception { + // all custom claims match regex + Map shouldMatch = new HashMap<>(); + shouldMatch.put("claim1", "foo"); + shouldMatch.put("claim2", "foo|bar"); + shouldMatch.put("claim3", "f\\w{2}$"); + testConfig.put("claims_match", shouldMatch); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertTrue(resp.isAuthenticated()); + + // Required claim does not exist + shouldMatch.clear(); + shouldMatch.put("claim9", "NA"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertEquals(AuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); + + // Required claim does not match regex + shouldMatch.clear(); + shouldMatch.put("claim1", "NA"); + resp = plugin.authenticate(testHeader); + assertEquals(AuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); + } + + @Test + public void missingIssAudExp() throws Exception { + testConfig.put("require_exp", "false"); + testConfig.put("require_sub", "false"); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(slimHeader); + assertTrue(resp.isAuthenticated()); + + // Missing exp header + testConfig.put("require_exp", true); + plugin.init(testConfig); + resp = plugin.authenticate(slimHeader); + assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + + // Missing sub header + testConfig.put("require_sub", true); + plugin.init(testConfig); + resp = plugin.authenticate(slimHeader); + assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + } + + @Test + public void algWhitelist() throws Exception { + testConfig.put("alg_whitelist", Arrays.asList("PS384", "PS512")); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + assertTrue(resp.getErrorMessage().contains("not a whitelisted")); + } + + @Test + public void roles() throws Exception { + testConfig.put("roles_claim", "groups"); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(testHeader); + assertTrue(resp.isAuthenticated()); + + Principal principal = resp.getPrincipal(); + assertTrue(principal instanceof VerifiedUserRoles); + Set roles = ((VerifiedUserRoles)principal).getVerifiedRoles(); + assertEquals(3, roles.size()); + assertTrue(roles.contains("group-one")); + + // Wrong claim ID + testConfig.put("roles_claim", "NA"); + plugin.init(testConfig); + resp = plugin.authenticate(testHeader); + assertEquals(AuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); + } + + @Test + public void noHeaderBlockUnknown() throws Exception { + testConfig.put("block_unknown", true); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(null); + assertEquals(NO_AUTZ_HEADER, resp.getAuthCode()); + } + + @Test + public void noHeaderNotBlockUnknown() throws Exception { + testConfig.put("block_unknown", false); + plugin.init(testConfig); + AuthenticationResponse resp = plugin.authenticate(null); + assertEquals(AuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); + } + + // TODO: Add inter-node request tests +} \ No newline at end of file diff --git a/solr/licenses/jose4j-0.6.3.jar.sha1 b/solr/licenses/jose4j-0.6.3.jar.sha1 new file mode 100644 index 000000000000..968ae6dd27e2 --- /dev/null +++ b/solr/licenses/jose4j-0.6.3.jar.sha1 @@ -0,0 +1 @@ +05bf092aa7be6fe8894d11f8d2040a2b3b401a14 diff --git a/solr/licenses/jose4j-LICENSE-ASL.txt b/solr/licenses/jose4j-LICENSE-ASL.txt new file mode 100644 index 000000000000..ab3182e7776f --- /dev/null +++ b/solr/licenses/jose4j-LICENSE-ASL.txt @@ -0,0 +1,272 @@ +/* + * Apache License + * Version 2.0, January 2004 + * http://www.apache.org/licenses/ + * + * TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * + * 1. Definitions. + * + * "License" shall mean the terms and conditions for use, reproduction, + * and distribution as defined by Sections 1 through 9 of this document. + * + * "Licensor" shall mean the copyright owner or entity authorized by + * the copyright owner that is granting the License. + * + * "Legal Entity" shall mean the union of the acting entity and all + * other entities that control, are controlled by, or are under common + * control with that entity. For the purposes of this definition, + * "control" means (i) the power, direct or indirect, to cause the + * direction or management of such entity, whether by contract or + * otherwise, or (ii) ownership of fifty percent (50%) or more of the + * outstanding shares, or (iii) beneficial ownership of such entity. + * + * "You" (or "Your") shall mean an individual or Legal Entity + * exercising permissions granted by this License. + * + * "Source" form shall mean the preferred form for making modifications, + * including but not limited to software source code, documentation + * source, and configuration files. + * + * "Object" form shall mean any form resulting from mechanical + * transformation or translation of a Source form, including but + * not limited to compiled object code, generated documentation, + * and conversions to other media types. + * + * "Work" shall mean the work of authorship, whether in Source or + * Object form, made available under the License, as indicated by a + * copyright notice that is included in or attached to the work + * (an example is provided in the Appendix below). + * + * "Derivative Works" shall mean any work, whether in Source or Object + * form, that is based on (or derived from) the Work and for which the + * editorial revisions, annotations, elaborations, or other modifications + * represent, as a whole, an original work of authorship. For the purposes + * of this License, Derivative Works shall not include works that remain + * separable from, or merely link (or bind by name) to the interfaces of, + * the Work and Derivative Works thereof. + * + * "Contribution" shall mean any work of authorship, including + * the original version of the Work and any modifications or additions + * to that Work or Derivative Works thereof, that is intentionally + * submitted to Licensor for inclusion in the Work by the copyright owner + * or by an individual or Legal Entity authorized to submit on behalf of + * the copyright owner. For the purposes of this definition, "submitted" + * means any form of electronic, verbal, or written communication sent + * to the Licensor or its representatives, including but not limited to + * communication on electronic mailing lists, source code control systems, + * and issue tracking systems that are managed by, or on behalf of, the + * Licensor for the purpose of discussing and improving the Work, but + * excluding communication that is conspicuously marked or otherwise + * designated in writing by the copyright owner as "Not a Contribution." + * + * "Contributor" shall mean Licensor and any individual or Legal Entity + * on behalf of whom a Contribution has been received by Licensor and + * subsequently incorporated within the Work. + * + * 2. Grant of Copyright License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * copyright license to reproduce, prepare Derivative Works of, + * publicly display, publicly perform, sublicense, and distribute the + * Work and such Derivative Works in Source or Object form. + * + * 3. Grant of Patent License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * (except as stated in this section) patent license to make, have made, + * use, offer to sell, sell, import, and otherwise transfer the Work, + * where such license applies only to those patent claims licensable + * by such Contributor that are necessarily infringed by their + * Contribution(s) alone or by combination of their Contribution(s) + * with the Work to which such Contribution(s) was submitted. If You + * institute patent litigation against any entity (including a + * cross-claim or counterclaim in a lawsuit) alleging that the Work + * or a Contribution incorporated within the Work constitutes direct + * or contributory patent infringement, then any patent licenses + * granted to You under this License for that Work shall terminate + * as of the date such litigation is filed. + * + * 4. Redistribution. You may reproduce and distribute copies of the + * Work or Derivative Works thereof in any medium, with or without + * modifications, and in Source or Object form, provided that You + * meet the following conditions: + * + * (a) You must give any other recipients of the Work or + * Derivative Works a copy of this License; and + * + * (b) You must cause any modified files to carry prominent notices + * stating that You changed the files; and + * + * (c) You must retain, in the Source form of any Derivative Works + * that You distribute, all copyright, patent, trademark, and + * attribution notices from the Source form of the Work, + * excluding those notices that do not pertain to any part of + * the Derivative Works; and + * + * (d) If the Work includes a "NOTICE" text file as part of its + * distribution, then any Derivative Works that You distribute must + * include a readable copy of the attribution notices contained + * within such NOTICE file, excluding those notices that do not + * pertain to any part of the Derivative Works, in at least one + * of the following places: within a NOTICE text file distributed + * as part of the Derivative Works; within the Source form or + * documentation, if provided along with the Derivative Works; or, + * within a display generated by the Derivative Works, if and + * wherever such third-party notices normally appear. The contents + * of the NOTICE file are for informational purposes only and + * do not modify the License. You may add Your own attribution + * notices within Derivative Works that You distribute, alongside + * or as an addendum to the NOTICE text from the Work, provided + * that such additional attribution notices cannot be construed + * as modifying the License. + * + * You may add Your own copyright statement to Your modifications and + * may provide additional or different license terms and conditions + * for use, reproduction, or distribution of Your modifications, or + * for any such Derivative Works as a whole, provided Your use, + * reproduction, and distribution of the Work otherwise complies with + * the conditions stated in this License. + * + * 5. Submission of Contributions. Unless You explicitly state otherwise, + * any Contribution intentionally submitted for inclusion in the Work + * by You to the Licensor shall be under the terms and conditions of + * this License, without any additional terms or conditions. + * Notwithstanding the above, nothing herein shall supersede or modify + * the terms of any separate license agreement you may have executed + * with Licensor regarding such Contributions. + * + * 6. Trademarks. This License does not grant permission to use the trade + * names, trademarks, service marks, or product names of the Licensor, + * except as required for reasonable and customary use in describing the + * origin of the Work and reproducing the content of the NOTICE file. + * + * 7. Disclaimer of Warranty. Unless required by applicable law or + * agreed to in writing, Licensor provides the Work (and each + * Contributor provides its Contributions) on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied, including, without limitation, any warranties or conditions + * of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + * PARTICULAR PURPOSE. You are solely responsible for determining the + * appropriateness of using or redistributing the Work and assume any + * risks associated with Your exercise of permissions under this License. + * + * 8. Limitation of Liability. In no event and under no legal theory, + * whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law (such as deliberate and grossly + * negligent acts) or agreed to in writing, shall any Contributor be + * liable to You for damages, including any direct, indirect, special, + * incidental, or consequential damages of any character arising as a + * result of this License or out of the use or inability to use the + * Work (including but not limited to damages for loss of goodwill, + * work stoppage, computer failure or malfunction, or any and all + * other commercial damages or losses), even if such Contributor + * has been advised of the possibility of such damages. + * + * 9. Accepting Warranty or Additional Liability. While redistributing + * the Work or Derivative Works thereof, You may choose to offer, + * and charge a fee for, acceptance of support, warranty, indemnity, + * or other liability obligations and/or rights consistent with this + * License. However, in accepting such obligations, You may act only + * on Your own behalf and on Your sole responsibility, not on behalf + * of any other Contributor, and only if You agree to indemnify, + * defend, and hold each Contributor harmless for any liability + * incurred by, or claims asserted against, such Contributor by reason + * of your accepting any such warranty or additional liability. + * + * END OF TERMS AND CONDITIONS + * + * APPENDIX: How to apply the Apache License to your work. + * + * To apply the Apache License to your work, attach the following + * boilerplate notice, with the fields enclosed by brackets "[]" + * replaced with your own identifying information. (Don't include + * the brackets!) The text should be enclosed in the appropriate + * comment syntax for the file format. We also recommend that a + * file or class name and description of purpose be included on the + * same "printed page" as the copyright notice for easier + * identification within third-party archives. + * + * Copyright [yyyy] [name of copyright owner] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +W3C® SOFTWARE NOTICE AND LICENSE +http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 + +This work (and included software, documentation such as READMEs, or other +related items) is being provided by the copyright holders under the following +license. By obtaining, using and/or copying this work, you (the licensee) agree +that you have read, understood, and will comply with the following terms and +conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: + + 1. The full text of this NOTICE in a location viewable to users of the + redistributed or derivative work. + 2. Any pre-existing intellectual property disclaimers, notices, or terms + and conditions. If none exist, the W3C Software Short Notice should be + included (hypertext is preferred, text is permitted) within the body + of any redistributed or derivative code. + 3. Notice of any changes or modifications to the files, including the date + changes were made. (We recommend you provide URIs to the location from + which the code is derived.) + +THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE +NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT +THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY +PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. + +COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR DOCUMENTATION. + +The name and trademarks of copyright holders may NOT be used in advertising or +publicity pertaining to the software without specific, written prior permission. +Title to copyright in this software and any associated documentation will at +all times remain with copyright holders. + +____________________________________ + +This formulation of W3C's notice and license became active on December 31 2002. +This version removes the copyright ownership notice such that this license can +be used with materials other than those owned by the W3C, reflects that ERCIM +is now a host of the W3C, includes references to this specific dated version of +the license, and removes the ambiguous grant of "use". Otherwise, this version +is the same as the previous version and is written so as to preserve the Free +Software Foundation's assessment of GPL compatibility and OSI's certification +under the Open Source Definition. Please see our Copyright FAQ for common +questions about using materials from our site, including specific terms and +conditions for packages like libwww, Amaya, and Jigsaw. Other questions about +this notice can be directed to site-policy@w3.org. + +Joseph Reagle + +This license came from: http://www.megginson.com/SAX/copying.html + However please note future versions of SAX may be covered + under http://saxproject.org/?selected=pd + +SAX2 is Free! + +I hereby abandon any property rights to SAX 2.0 (the Simple API for +XML), and release all of the SAX 2.0 source code, compiled code, and +documentation contained in this distribution into the Public Domain. +SAX comes with NO WARRANTY or guarantee of fitness for any +purpose. + +David Megginson, david@megginson.com +2000-05-05 diff --git a/solr/licenses/jose4j-NOTICE.txt b/solr/licenses/jose4j-NOTICE.txt new file mode 100644 index 000000000000..f68819e4d384 --- /dev/null +++ b/solr/licenses/jose4j-NOTICE.txt @@ -0,0 +1,13 @@ +jose4j +Copyright 2012-2015 Brian Campbell + +EcdsaUsingShaAlgorithm contains code for converting the concatenated +R & S values of the signature to and from DER, which was originally +derived from the Apache Santuario XML Security library's SignatureECDSA +implementation. http://santuario.apache.org/ + +The Base64 implementation in this software was derived from the +Apache Commons Codec project. http://commons.apache.org/proper/commons-codec/ + +JSON processing in this software was derived from the JSON.simple toolkit. +https://code.google.com/p/json-simple/ \ No newline at end of file diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index 971dbcdf3714..7ba1c5b8d453 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -125,6 +125,7 @@ Solr has the following implementations of authentication plugins: * <> * <> * <> +* <> == Authorization diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc new file mode 100644 index 000000000000..11c157423bd7 --- /dev/null +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -0,0 +1,87 @@ += JWT Authentication Plugin +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token]] (JWT) based Bearer authentication for users with the use of the JWTAuthPlugin. This allows Solr to assert that a user is already authenticated with an external service by validating that the JWT is digitally signed by the Identity service. One possible use case is for https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect]. + +Authorization plugins are also available to configure Solr with permissions to perform various activities in the system. The authorization plugins are described in the section <>. + +== Enable JWT Authentication + +To use JWT authentication, you must first create a `security.json` file. This file and where to put it is described in detail in the section <>. + +The `security.json` file must have an `authentication` part which defines the class being used for authentication along with configuration parameters. + +The simplest possible `security.json` for registering the plugin without configuration is: + +[source,json] +---- +{ + "authentication": { + "class":"solr.JWTAuthPlugin" + } +} +---- + +The plugin will NOT block anonymous traffic in this mode, since the default for `block_unknown` is false. It is then possible to start configuring the plugin using REST calls. + +To start enforcing authentication for all users, requiring a valid JWT token in the `Authorization` header, you need to configure the plugin with one or more https://tools.ietf.org/html/rfc7517[JSON Web Key]s (JWK). This is a JSON document containing the key used to sign/encrypt the JWT. It could be a symmetric or asymmetric key. The JWK can either be fetched (and cached) from an external HTTPS endpoint or specified directly in `security.json`. Below is an example of the former: + +[source,json] +---- +{ + "authentication": { + "class": "org.apache.solr.security.JWTAuthPlugin", + "block_unknown": true, + "jwk_url": "https://my.key.server/jwk.json" + } +} +---- + +== Configuration parameters + +[%header,format=csv,separator=;] +|=== +Key ; Description ; Default +block_unknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` +jwk_url ; An https URL to a https://mkjwk.org[JWK] keys file ; +jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. See https://mkjwk.org for an online JWK generator ; +iss ; Validates that `iss` (issuer) equals this string ; (`iss` not required) +aud ; Validates that `aud` (audience) equals this string ; (`aud` not required) +require_sub ; Makes `sub` (subject) mandatory ; `true` +require_exp ; Makes `exp` (expiry time) mandatory ; `true` +alg_whitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; (any) +jwk_cache_dur ; Duration of JWK cache in seconds ; 3600 (1h) +principal_claim ; What claim id to pull principal from ; `sub` +roles_claim ; What claim id to pull roles from. If specified, all request MUST supply this claim ; +claims_match ; JSON object of claims (key) that must match a regular expression (value). ; (none) +|=== + + +== Editing Authentication Plugin Configuration + +**TODO: This plugin currently does not support edit API.** + +=== Using JWT Auth with SolrJ + +To use JWT Auth with SolrJ you must configure a `SolrHttpClientBuilder` that sets the `Authorization: Bearer` header. How to obtain a valid JWT token is up to the client application to choose. + +**TODO: Describe this ** + +=== Using the Solr Control Script with Basic Auth + +The control scripts (`bin/solr`) do not currently support JWT Auth. \ No newline at end of file From 91a32fdf4cf839ff9817e6854a6a9d3a80e91463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 23 Mar 2018 17:54:30 +0100 Subject: [PATCH 02/88] Document a more complex example --- .../src/jwt-authentication-plugin.adoc | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 11c157423bd7..b58545b97bf3 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -45,13 +45,49 @@ To start enforcing authentication for all users, requiring a valid JWT token in ---- { "authentication": { - "class": "org.apache.solr.security.JWTAuthPlugin", + "class": "solr.JWTAuthPlugin", "block_unknown": true, "jwk_url": "https://my.key.server/jwk.json" } } ---- +A more complex configuration: + +[source,json] +---- +{ + "authentication": { + "class": "solr.JWTAuthenticationPlugin", <1> + "block_unknown": true, <2> + "jwk": { <3> + "e": "AQAB", + "kid": "k1", + "kty": "RSA", + "n": "3ZF6wBGPMsLzsS1KLghxaVpZtXD3nTLzDm0c974i9-KNU_1rhhBeiVfS64VfEQmP8SA470jEy7yWcvnz9GvG-YAlm9iOwVF7jLl2awdws0ocFjdSPT3SjPQKzOeMO7G9XqNTkrvoFCn1YAi26fbhhcqkwZDoeTcHQdRN32frzccuPhZrwImApIedroKLlKWv2IvPDnz2Bpe2WWVc2HdoWYqEVD3p_BEy8f-RTSHK3_8kDDF9yAwI9jx7CK1_C-eYxXltm-6rpS5NGyFm0UNTZMxVU28Tl7LX8Vb6CikyCQ9YRCtk_CvpKWmEuKEp9I28KHQNmGkDYT90nt3vjbCXxw" + }, + "iss": "acme", <4> + "aud": "solr", <5> + "principal_claim": "solruid", <6> + "claims_match": { "roles" : "admin|search", "dept" : "IT" }, <7> + "roles_claim": "roles", <8> + "alg_whitelist" : [ "RS256", "RS384", "RS512" ] <9> + } +} +---- + +Let's comment on the config: + +<1> Plugin class +<2> Make sure to block anyone without a valid token +<3> Here we pass the JWK inline instead of referring to a URL with `jwk_url` +<4> The issuer claim must match "acme" +<5> The audience claim must match "solr" +<6> Fetch the user id from another claim than the default `sub` +<7> Require that the `roles` claim is one of "admin" or "search" and that the `dept` claim is "IT" +<8> Fetch user's roles from the "roles" claim and pass this on the request for use in Authorization +<9> Only accept RSA algorithms for signatures + == Configuration parameters [%header,format=csv,separator=;] From 8271356c10438fc4fd325f1b53818812107c59d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 4 Apr 2018 17:22:32 +0200 Subject: [PATCH 03/88] Return proper error and error_description in WWW-Authenticate header Throw exception if plugin not configured Return 400 error if Authorization header cannot be parsed --- .../apache/solr/security/JWTAuthPlugin.java | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 6dd810a51ecc..675c013849dd 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -27,6 +27,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.security.Principal; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -179,13 +180,11 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser // if (header == null && !blockUnknown) { log.info("JWTAuth not configured, but allowing anonymous access since blockUnknown==false"); - response.setHeader(HttpHeaders.WWW_AUTHENTICATE, "Bearer realm=\"" + AUTH_REALM + "\""); filterChain.doFilter(request, response); return true; } log.warn("JWTAuth not configured"); - authenticationFailure(response, "JWTAuth not configured"); - return false; + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin not correctly configured"); } if (header != null) { @@ -206,15 +205,29 @@ public Principal getUserPrincipal() { case PASS_THROUGH: log.debug("Unknown user, but allow due to block_unknown=false"); - response.setHeader(HttpHeaders.WWW_AUTHENTICATE, "Bearer realm=\"" + AUTH_REALM + "\""); filterChain.doFilter(request, response); return true; - default: + case AUTZ_HEADER_PROBLEM: + log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); + authenticationFailure(response, authResponse.getAuthCode().msg, HttpServletResponse.SC_BAD_REQUEST, BearerWwwAuthErrorCode.invalid_request); + return false; + + case CLAIM_MISMATCH: + case JWT_EXPIRED: + case JWT_PARSE_ERROR: + case JWT_VALIDATION_EXCEPTION: + case PRINCIPAL_MISSING: log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); - if (authResponse.authCode.equals(AuthCode.JWT_VALIDATION_EXCEPTION)) { - log.debug("Exception: {}", authResponse.getJwtException().getMessage()); + if (authResponse.getJwtException() != null) { + log.info("Exception: {}", authResponse.getJwtException().getMessage()); } + authenticationFailure(response, authResponse.getAuthCode().msg, HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.invalid_token); + return false; + + case NO_AUTZ_HEADER: + default: + log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); authenticationFailure(response, authResponse.getAuthCode().msg); return false; } @@ -367,10 +380,23 @@ public ValidatingJsonMap getSpec() { } private void authenticationFailure(HttpServletResponse response, String message) throws IOException { - response.setHeader(HttpHeaders.WWW_AUTHENTICATE, "Bearer realm=\"" + AUTH_REALM + "\""); - response.sendError(401, message); + authenticationFailure(response, message, HttpServletResponse.SC_UNAUTHORIZED, null); + } + + private enum BearerWwwAuthErrorCode { invalid_request, invalid_token, insufficient_scope}; + + private void authenticationFailure(HttpServletResponse response, String message, int httpCode, BearerWwwAuthErrorCode responseError) throws IOException { + List wwwAuthParams = new ArrayList<>(); + wwwAuthParams.add("Bearer realm=\"" + AUTH_REALM + "\""); + if (responseError != null) { + wwwAuthParams.add("error=\"" + responseError + "\""); + wwwAuthParams.add("error_description=\"" + message + "\""); + } + response.addHeader(HttpHeaders.WWW_AUTHENTICATE, org.apache.commons.lang.StringUtils.join(wwwAuthParams, ", ")); + response.sendError(httpCode, message); } + /** * Response for authentication attempt */ From 4c705a3a78e900a137d769d92a0e5ff9f408c246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 9 Apr 2018 13:28:33 +0200 Subject: [PATCH 04/88] Small doc fixes --- solr/solr-ref-guide/src/jwt-authentication-plugin.adoc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index b58545b97bf3..cb78aef3df1d 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -16,9 +16,7 @@ // specific language governing permissions and limitations // under the License. -Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token]] (JWT) based Bearer authentication for users with the use of the JWTAuthPlugin. This allows Solr to assert that a user is already authenticated with an external service by validating that the JWT is digitally signed by the Identity service. One possible use case is for https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect]. - -Authorization plugins are also available to configure Solr with permissions to perform various activities in the system. The authorization plugins are described in the section <>. +Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token] (JWT) based Bearer authentication for users with the use of the JWTAuthPlugin. This allows Solr to assert that a user is already authenticated with an external service by validating that the JWT is digitally signed by the Identity service. One possible use case is for https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect]. == Enable JWT Authentication @@ -58,7 +56,7 @@ A more complex configuration: ---- { "authentication": { - "class": "solr.JWTAuthenticationPlugin", <1> + "class": "solr.JWTAuthPlugin", <1> "block_unknown": true, <2> "jwk": { <3> "e": "AQAB", @@ -103,7 +101,7 @@ require_exp ; Makes `exp` (expiry time) mandatory ; `tr alg_whitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; (any) jwk_cache_dur ; Duration of JWK cache in seconds ; 3600 (1h) principal_claim ; What claim id to pull principal from ; `sub` -roles_claim ; What claim id to pull roles from. If specified, all request MUST supply this claim ; +roles_claim ; What claim id to pull roles from. If specified, all requests MUST supply this claim ; claims_match ; JSON object of claims (key) that must match a regular expression (value). ; (none) |=== From 1f06e9f1945e5068430ebfeeb8b90bd0eb517761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 18 Apr 2018 14:59:12 +0200 Subject: [PATCH 05/88] Put jwt-authentication-plugin page as child of authentication-and-authorization-plugins.adoc --- .../src/authentication-and-authorization-plugins.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index 7ba1c5b8d453..f83a50793660 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -1,5 +1,5 @@ = Authentication and Authorization Plugins -:page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, rule-based-authorization-plugin +:page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, rule-based-authorization-plugin, jwt-authentication-plugin // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information From 74ae5325b723e21c26b9c33f6ce1b6d4cde954ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 19 Apr 2018 10:36:36 +0200 Subject: [PATCH 06/88] Add debug logging for SUCCESS case --- .../src/java/org/apache/solr/security/JWTAuthPlugin.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 675c013849dd..69811cae39d0 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -200,11 +200,14 @@ public Principal getUserPrincipal() { return authResponse.getPrincipal(); } }; + if (log.isDebugEnabled()) + log.debug("Authentication SUCCESS"); filterChain.doFilter(wrapper, response); return true; case PASS_THROUGH: - log.debug("Unknown user, but allow due to block_unknown=false"); + if (log.isDebugEnabled()) + log.debug("Unknown user, but allow due to block_unknown=false"); filterChain.doFilter(request, response); return true; @@ -220,7 +223,7 @@ public Principal getUserPrincipal() { case PRINCIPAL_MISSING: log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); if (authResponse.getJwtException() != null) { - log.info("Exception: {}", authResponse.getJwtException().getMessage()); + log.warn("Exception: {}", authResponse.getJwtException().getMessage()); } authenticationFailure(response, authResponse.getAuthCode().msg, HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.invalid_token); return false; From 851276a5531ef9923b989ef343c1072660d89ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 30 Apr 2018 15:41:02 +0200 Subject: [PATCH 07/88] WIP integration test --- .../security/BearerAuthSchemeProvider.java | 90 +++++++------ .../apache/solr/security/JWTAuthPlugin.java | 10 +- .../security/jwt_plugin_jwk_security.json | 9 +- .../JWTAuthPluginIntegrationTest.java | 122 ++++++++++++++++++ .../solr/security/JWTAuthPluginTest.java | 2 +- 5 files changed, 186 insertions(+), 47 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java diff --git a/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java b/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java index d64566b89ebc..6cc25e449d47 100644 --- a/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java +++ b/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java @@ -16,7 +16,10 @@ */ package org.apache.solr.security; +import java.lang.invoke.MethodHandles; + import org.apache.http.Header; +import org.apache.http.HttpHeaders; import org.apache.http.HttpRequest; import org.apache.http.auth.AUTH; import org.apache.http.auth.AuthScheme; @@ -28,6 +31,9 @@ import org.apache.http.message.BufferedHeader; import org.apache.http.protocol.HttpContext; import org.apache.http.util.CharArrayBuffer; +import org.apache.solr.common.SolrException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BearerAuthSchemeProvider implements AuthSchemeProvider { @@ -43,51 +49,59 @@ public AuthScheme create(HttpContext context) { } private static class BearerAuthScheme implements ContextAwareAuthScheme { - private boolean complete = false; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private boolean complete = false; - @Override - public void processChallenge(Header header) throws MalformedChallengeException { - this.complete = true; - } + @Override + public void processChallenge(Header header) throws MalformedChallengeException { + this.complete = true; + } - @Override - public Header authenticate(Credentials credentials, HttpRequest request) throws AuthenticationException { - return authenticate(credentials, request, null); - } + @Override + public Header authenticate(Credentials credentials, HttpRequest request) throws AuthenticationException { + return authenticate(credentials, request, null); + } - @Override - public Header authenticate(Credentials credentials, HttpRequest request, HttpContext httpContext) - throws AuthenticationException { - CharArrayBuffer buffer = new CharArrayBuffer(128); - buffer.append(AUTH.WWW_AUTH_RESP); - buffer.append(": Bearer "); - buffer.append(credentials.getUserPrincipal().getName()); - return new BufferedHeader(buffer); - } + @Override + public Header authenticate(Credentials credentials, HttpRequest request, HttpContext httpContext) + throws AuthenticationException { + Header header = request.getFirstHeader(HttpHeaders.AUTHORIZATION); + CharArrayBuffer buffer = new CharArrayBuffer(128); + buffer.append(AUTH.WWW_AUTH_RESP); + buffer.append(": "); + if (header != null) { + buffer.append(header.getValue()); + log.debug("Propagating Authorization header from request: {}", header.getValue()); + } else { + buffer.append("Bearer " + credentials.getUserPrincipal().getName()); + log.debug("Using Authorization header from credentials provider: {}", credentials.getUserPrincipal().getName()); + } + return new BufferedHeader(buffer); + } - @Override - public String getSchemeName() { - return "Bearer"; - } + @Override + public String getSchemeName() { + return "Bearer"; + } - @Override - public String getParameter(String name) { - return null; - } + @Override + public String getParameter(String name) { + return null; + } - @Override - public String getRealm() { - return null; - } + @Override + public String getRealm() { + return null; + } - @Override - public boolean isConnectionBased() { - return false; - } + @Override + public boolean isConnectionBased() { + return false; + } - @Override - public boolean isComplete() { - return this.complete; - } + @Override + public boolean isComplete() { + return this.complete; } + } } \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 69811cae39d0..df8a1978d39d 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -88,6 +88,7 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String PARAM_JWK_CACHE_DURATION = "jwk_cache_dur"; private static final String PARAM_CLAIMS_MATCH = "claims_match"; private static final String AUTH_REALM = "solr"; + private static final String JWT_SYSPROP = "jwt"; private String jwk_url; private Map jwk; @@ -190,6 +191,7 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser if (header != null) { // Put the header on the thread for later use in inter-node request authHeader.set(new BasicHeader(HttpHeaders.AUTHORIZATION, header)); + log.info("**** Setting header on thread: {}, thread={}", header, Thread.currentThread().getName()); } AuthenticationResponse authResponse = authenticate(header); switch(authResponse.authCode) { @@ -364,12 +366,10 @@ public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) }); builder.setDefaultCredentialsProvider(() -> { CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - // Pull the authorization bearer header from ThreadLocal - if (authHeader.get() == null) { - log.warn("Cannot find Authorization header on request thread"); - } else { + String jwtProperty = System.getProperty(JWT_SYSPROP); + if (jwtProperty != null) { // TODO: Limit AuthScope? - credentialsProvider.setCredentials(AuthScope.ANY, new TokenCredentials(authHeader.get().getValue())); + credentialsProvider.setCredentials(AuthScope.ANY, new TokenCredentials(jwtProperty)); } return credentialsProvider; }); diff --git a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json index b200361ec8d4..3c249e64e9c2 100644 --- a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json +++ b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json @@ -1,11 +1,14 @@ { "authentication": { "class": "solr.JWTAuthPlugin", + "block_unknown": true, "jwk": { - "e": "AQAB", - "kid": "k1", "kty": "RSA", - "n": "3ZF6wBGPMsLzsS1KLghxaVpZtXD3nTLzDm0c974i9-KNU_1rhhBeiVfS64VfEQmP8SA470jEy7yWcvnz9GvG-YAlm9iOwVF7jLl2awdws0ocFjdSPT3SjPQKzOeMO7G9XqNTkrvoFCn1YAi26fbhhcqkwZDoeTcHQdRN32frzccuPhZrwImApIedroKLlKWv2IvPDnz2Bpe2WWVc2HdoWYqEVD3p_BEy8f-RTSHK3_8kDDF9yAwI9jx7CK1_C-eYxXltm-6rpS5NGyFm0UNTZMxVU28Tl7LX8Vb6CikyCQ9YRCtk_CvpKWmEuKEp9I28KHQNmGkDYT90nt3vjbCXxw" + "e": "AQAB", + "use": "sig", + "kid": "test", + "alg": "RS256", + "n": "jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ" } } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java new file mode 100644 index 000000000000..6a2b9f1286bb --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.message.BasicHeader; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.jose4j.jwk.PublicJsonWebKey; +import org.jose4j.jwk.RsaJsonWebKey; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jws.JsonWebSignature; +import org.jose4j.jwt.JwtClaims; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class JWTAuthPluginIntegrationTest extends SolrCloudTestCase { + protected static final int NUM_SERVERS = 2; + protected static final int NUM_SHARDS = 2; + protected static final int REPLICATION_FACTOR = 1; + private static Header tokenHeader; + private static String jwkJSON = "{\n" + + " \"kty\": \"RSA\",\n" + + " \"d\": \"i6pyv2z3o-MlYytWsOr3IE1olu2RXZBzjPRBNgWAP1TlLNaphHEvH5aHhe_CtBAastgFFMuP29CFhaL3_tGczkvWJkSveZQN2AHWHgRShKgoSVMspkhOt3Ghha4CvpnZ9BnQzVHnaBnHDTTTfVgXz7P1ZNBhQY4URG61DKIF-JSSClyh1xKuMoJX0lILXDYGGcjVTZL_hci4IXPPTpOJHV51-pxuO7WU5M9252UYoiYyCJ56ai8N49aKIMsqhdGuO4aWUwsGIW4oQpjtce5eEojCprYl-9rDhTwLAFoBtjy6LvkqlR2Ae5dKZYpStljBjK8PJrBvWZjXAEMDdQ8PuQ\",\n" + + " \"e\": \"AQAB\",\n" + + " \"use\": \"sig\",\n" + + " \"kid\": \"test\",\n" + + " \"alg\": \"RS256\",\n" + + " \"n\": \"jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ\"\n" + + "}"; + private static PublicJsonWebKey jwk; + + @BeforeClass + public static void setupClass() throws Exception { + configureCluster(NUM_SERVERS)// nodes + .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json")) + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); + JwtClaims claims = JWTAuthPluginTest.generateClaims(); + JsonWebSignature jws = new JsonWebSignature(); + jws.setPayload(claims.toJson()); + jws.setKey(jwk.getPrivateKey()); + jws.setKeyIdHeaderValue(jwk.getKeyId()); + jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); + + String testJwt = jws.getCompactSerialization(); + tokenHeader = new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + testJwt); + } + + @AfterClass + public static void tearDownClass() throws Exception { + System.clearProperty("java.security.auth.login.config"); + } + + @Test(expected = HttpSolrClient.RemoteSolrException.class) + public void testWithoutToken() throws Exception { + testCollectionCreateSearchDelete(); + // sometimes run a second test e.g. to test collection create-delete-create scenario + if (random().nextBoolean()) testCollectionCreateSearchDelete(); + } + + @Test + public void testWithToken() throws Exception { + enableToken(); + testCollectionCreateSearchDelete(); + // sometimes run a second test e.g. to test collection create-delete-create scenario + if (random().nextBoolean()) testCollectionCreateSearchDelete(); + } + + private void enableToken() { + + } + + protected void testCollectionCreateSearchDelete() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "jwtAuthTestColl"; + + // create collection + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", + NUM_SHARDS, REPLICATION_FACTOR); + create.process(solrClient); + + solrClient.add(collectionName, new SolrInputDocument("id", "1")); + solrClient.add(collectionName, new SolrInputDocument("id", "2")); + solrClient.commit(collectionName); + + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + QueryResponse rsp = solrClient.query(collectionName, query); + assertEquals(2, rsp.getResults().getNumFound()); + + CollectionAdminRequest.Delete deleteReq = CollectionAdminRequest.deleteCollection(collectionName); + deleteReq.process(solrClient); + AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, + solrClient.getZkStateReader(), true, true, 330); + } + +} diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index e1065a8393ee..f9529260ca78 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -82,7 +82,7 @@ public static void beforeAll() throws Exception { slimHeader = "Bearer" + " " + slimJwt; } - private static JwtClaims generateClaims() { + protected static JwtClaims generateClaims() { JwtClaims claims = new JwtClaims(); claims.setIssuer("IDServer"); // who creates the token and signs it claims.setAudience("Solr"); // to whom the token is intended to be sent From 9b6c85705b32db24e24d1cba20c216741e39e29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 6 Aug 2018 20:45:39 +0200 Subject: [PATCH 08/88] Bump jose version --- lucene/ivy-versions.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index cef5d7294221..6308a81f6121 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -213,7 +213,7 @@ org.apache.tika.version = 1.18 /org.aspectj/aspectjrt = 1.8.0 -/org.bitbucket.b_c/jose4j = 0.6.3 +/org.bitbucket.b_c/jose4j = 0.6.4 org.bouncycastle.version = 1.54 /org.bouncycastle/bcmail-jdk15on = ${org.bouncycastle.version} From a8a015a346834184ca1bbbf94d08b77d11d307c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 25 Aug 2018 23:25:27 +0200 Subject: [PATCH 09/88] Passing integration test. Still need work with initializing SolrJ for JWT on the client side --- .../handler/component/HttpShardHandler.java | 9 +- .../security/BearerAuthSchemeProvider.java | 107 -------- .../apache/solr/security/JWTAuthPlugin.java | 233 +++++++++++++----- .../security/PKIAuthenticationPlugin.java | 2 +- .../solr/security/PrincipalWithUserRoles.java | 95 ------- .../solr/security/TokenCredentials.java | 43 ---- .../JWTAuthPluginIntegrationTest.java | 185 +++++++++----- .../solr/security/JWTAuthPluginTest.java | 52 ++-- .../apache/solr/client/solrj/SolrRequest.java | 11 + .../client/solrj/impl/HttpSolrClient.java | 13 +- 10 files changed, 356 insertions(+), 394 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java delete mode 100644 solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java delete mode 100644 solr/core/src/java/org/apache/solr/security/TokenCredentials.java diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index a548031f7bd2..05988eb9c8f6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -18,6 +18,7 @@ import java.lang.invoke.MethodHandles; import java.net.ConnectException; +import java.security.Principal; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -56,6 +57,7 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -135,7 +137,11 @@ private List getURLs(String shard) { public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) { // do this outside of the callable for thread safety reasons final List urls = getURLs(shard); - + + // If request has a Principal (authenticated user), extract it for passing on to the new shard request + SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo(); + final Principal userPrincipal = requestInfo == null ? null : requestInfo.getReq().getUserPrincipal(); + Callable task = () -> { ShardResponse srsp = new ShardResponse(); @@ -154,6 +160,7 @@ public void submit(final ShardRequest sreq, final String shard, final Modifiable QueryRequest req = makeQueryRequest(sreq, params, shard); req.setMethod(SolrRequest.METHOD.POST); + req.setUserPrincipal(userPrincipal); // no need to set the response parser as binary is the default // req.setResponseParser(new BinaryResponseParser()); diff --git a/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java b/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java deleted file mode 100644 index 6cc25e449d47..000000000000 --- a/solr/core/src/java/org/apache/solr/security/BearerAuthSchemeProvider.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security; - -import java.lang.invoke.MethodHandles; - -import org.apache.http.Header; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpRequest; -import org.apache.http.auth.AUTH; -import org.apache.http.auth.AuthScheme; -import org.apache.http.auth.AuthSchemeProvider; -import org.apache.http.auth.AuthenticationException; -import org.apache.http.auth.ContextAwareAuthScheme; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.MalformedChallengeException; -import org.apache.http.message.BufferedHeader; -import org.apache.http.protocol.HttpContext; -import org.apache.http.util.CharArrayBuffer; -import org.apache.solr.common.SolrException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BearerAuthSchemeProvider implements AuthSchemeProvider { - - /** - * Creates an instance of {@link AuthScheme}. - * - * @param context the http context - * @return auth scheme. - */ - @Override - public AuthScheme create(HttpContext context) { - return new BearerAuthScheme(); - } - - private static class BearerAuthScheme implements ContextAwareAuthScheme { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private boolean complete = false; - - @Override - public void processChallenge(Header header) throws MalformedChallengeException { - this.complete = true; - } - - @Override - public Header authenticate(Credentials credentials, HttpRequest request) throws AuthenticationException { - return authenticate(credentials, request, null); - } - - @Override - public Header authenticate(Credentials credentials, HttpRequest request, HttpContext httpContext) - throws AuthenticationException { - Header header = request.getFirstHeader(HttpHeaders.AUTHORIZATION); - CharArrayBuffer buffer = new CharArrayBuffer(128); - buffer.append(AUTH.WWW_AUTH_RESP); - buffer.append(": "); - if (header != null) { - buffer.append(header.getValue()); - log.debug("Propagating Authorization header from request: {}", header.getValue()); - } else { - buffer.append("Bearer " + credentials.getUserPrincipal().getName()); - log.debug("Using Authorization header from credentials provider: {}", credentials.getUserPrincipal().getName()); - } - return new BufferedHeader(buffer); - } - - @Override - public String getSchemeName() { - return "Bearer"; - } - - @Override - public String getParameter(String name) { - return null; - } - - @Override - public String getRealm() { - return null; - } - - @Override - public boolean isConnectionBased() { - return false; - } - - @Override - public boolean isComplete() { - return this.complete; - } - } -} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index df8a1978d39d..a9a5f76b6a37 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -23,6 +23,7 @@ import javax.servlet.http.HttpServletRequestWrapper; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; import java.net.URL; @@ -33,25 +34,25 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.StringTokenizer; import java.util.regex.Pattern; -import org.apache.http.Header; +import org.apache.http.HttpException; import org.apache.http.HttpHeaders; -import org.apache.http.auth.AuthSchemeProvider; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.BasicUserPrincipal; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.config.Lookup; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.message.BasicHeader; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.Args; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; +import org.apache.solr.core.CoreContainer; import org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode; import org.jose4j.jwa.AlgorithmConstraints; import org.jose4j.jwk.HttpsJwks; @@ -74,7 +75,6 @@ */ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin, SpecProvider { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final static ThreadLocal
authHeader = new ThreadLocal<>(); private static final String PARAM_BLOCK_UNKNOWN = "block_unknown"; private static final String PARAM_JWK_URL = "jwk_url"; private static final String PARAM_JWK = "jwk"; @@ -88,40 +88,44 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String PARAM_JWK_CACHE_DURATION = "jwk_cache_dur"; private static final String PARAM_CLAIMS_MATCH = "claims_match"; private static final String AUTH_REALM = "solr"; - private static final String JWT_SYSPROP = "jwt"; - private String jwk_url; - private Map jwk; + private final PkiDelegationInterceptor interceptor = new PkiDelegationInterceptor(); + private JwtConsumer jwtConsumer; private String iss; private String aud; private boolean requireSubject; private boolean requireExpirationTime; private List algWhitelist; - private HttpsJwks httpsJkws; - private long jwkCacheDuration; - VerificationKeyResolver verificationKeyResolver; - private JsonWebKeySet jwks; + private VerificationKeyResolver verificationKeyResolver; private String principalClaim; private String rolesClaim; - private Map claimsMatch; private HashMap claimsMatchCompiled; private boolean blockUnknown; + private CoreContainer coreContainer; + + /** + * Initialize plugin with core container, this method is chosen by reflection at create time + * @param coreContainer instance of core container + */ + public JWTAuthPlugin(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } @Override public void init(Map pluginConfig) { blockUnknown = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCK_UNKNOWN, false))); - jwk_url = (String) pluginConfig.get(PARAM_JWK_URL); - jwk = (Map) pluginConfig.get(PARAM_JWK); + String jwk_url = (String) pluginConfig.get(PARAM_JWK_URL); + Map jwk = (Map) pluginConfig.get(PARAM_JWK); iss = (String) pluginConfig.get(PARAM_ISSUER); aud = (String) pluginConfig.get(PARAM_AUDIENCE); requireSubject = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_SUBJECT, "true"))); requireExpirationTime = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_EXPIRATIONTIME, "true"))); algWhitelist = (List) pluginConfig.get(PARAM_ALG_WHITELIST); - jwkCacheDuration = Long.parseLong((String) pluginConfig.getOrDefault(PARAM_JWK_CACHE_DURATION, "3600")); + long jwkCacheDuration = Long.parseLong((String) pluginConfig.getOrDefault(PARAM_JWK_CACHE_DURATION, "3600")); principalClaim = (String) pluginConfig.getOrDefault(PARAM_PRINCIPAL_CLAIM, "sub"); rolesClaim = (String) pluginConfig.get(PARAM_ROLES_CLAIM); - claimsMatch = (Map) pluginConfig.get(PARAM_CLAIMS_MATCH); + Map claimsMatch = (Map) pluginConfig.get(PARAM_CLAIMS_MATCH); claimsMatchCompiled = new HashMap<>(); if (claimsMatch != null) { for (Map.Entry entry : claimsMatch.entrySet()) { @@ -140,13 +144,13 @@ public void init(Map pluginConfig) { } catch (MalformedURLException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be a valid https URL"); } - httpsJkws = new HttpsJwks(jwk_url); + HttpsJwks httpsJkws = new HttpsJwks(jwk_url); httpsJkws.setDefaultCacheDuration(jwkCacheDuration); verificationKeyResolver = new HttpsJwksVerificationKeyResolver(httpsJkws); initConsumer(); } else if (jwk != null) { try { - jwks = parseJwkSet(jwk); + JsonWebKeySet jwks = parseJwkSet(jwk); verificationKeyResolver = new JwksVerificationKeyResolver(jwks.getJsonWebKeys()); initConsumer(); } catch (JoseException e) { @@ -157,7 +161,7 @@ public void init(Map pluginConfig) { } } - protected JsonWebKeySet parseJwkSet(Map jwkObj) throws JoseException { + JsonWebKeySet parseJwkSet(Map jwkObj) throws JoseException { JsonWebKeySet webKeySet = new JsonWebKeySet(); if (jwkObj.containsKey("keys")) { List jwkList = (List) jwkObj.get("keys"); @@ -170,6 +174,9 @@ protected JsonWebKeySet parseJwkSet(Map jwkObj) throws JoseExcep return webKeySet; } + /** + * Main authentication method that looks for correct JWT token in the Authorization header + */ @Override public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws Exception { HttpServletRequest request = (HttpServletRequest) servletRequest; @@ -188,11 +195,6 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin not correctly configured"); } - if (header != null) { - // Put the header on the thread for later use in inter-node request - authHeader.set(new BasicHeader(HttpHeaders.AUTHORIZATION, header)); - log.info("**** Setting header on thread: {}, thread={}", header, Thread.currentThread().getName()); - } AuthenticationResponse authResponse = authenticate(header); switch(authResponse.authCode) { case AUTHENTICATED: @@ -202,6 +204,9 @@ public Principal getUserPrincipal() { return authResponse.getPrincipal(); } }; + if (!(authResponse.getPrincipal() instanceof JWTPrincipal)) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin says AUTHENTICATED but no token extracted"); + } if (log.isDebugEnabled()) log.debug("Authentication SUCCESS"); filterChain.doFilter(wrapper, response); @@ -284,9 +289,9 @@ protected AuthenticationResponse authenticate(String authorizationHeader) { roles = new HashSet<>(jwtClaims.getStringListClaimValue(rolesClaim)); } // Pass roles with principal to signal to any Authorization plugins that user has some verified role claims - return new AuthenticationResponse(AuthCode.AUTHENTICATED, new PrincipalWithUserRoles(principal, roles)); + return new AuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipalWithUserRoles(principal, jwtCompact, jwtClaims.getClaimsMap(), roles)); } else { - return new AuthenticationResponse(AuthCode.AUTHENTICATED, new BasicUserPrincipal(principal)); + return new AuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipal(principal, jwtCompact, jwtClaims.getClaimsMap())); } } catch (InvalidJwtException e) { // Whether or not the JWT has expired being one common reason for invalidity @@ -338,16 +343,11 @@ private void initConsumer() { @Override public void close() throws IOException { - - } - - @Override - public void closeRequest() { - authHeader.remove(); + HttpClientUtil.removeRequestInterceptor(interceptor); } /** - * Gets a client builder for inter-node requests + * Register an interceptor to be able to add our header to inter-node requests * @param builder any existing builder or null to create a new one * @return Returns an instance of a SolrHttpClientBuilder to be used for configuring the * HttpClients for use with SolrJ clients. @@ -355,24 +355,8 @@ public void closeRequest() { */ @Override public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { - if (builder == null) { - builder = SolrHttpClientBuilder.create(); - } - builder.setAuthSchemeRegistryProvider(() -> { - Lookup authProviders = RegistryBuilder.create() - .register("Bearer", new BearerAuthSchemeProvider()) - .build(); - return authProviders; - }); - builder.setDefaultCredentialsProvider(() -> { - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - String jwtProperty = System.getProperty(JWT_SYSPROP); - if (jwtProperty != null) { - // TODO: Limit AuthScope? - credentialsProvider.setCredentials(AuthScope.ANY, new TokenCredentials(jwtProperty)); - } - return credentialsProvider; - }); + // Register interceptor for inter-node requests, that delegates to PKI if JWTPrincipal is not found on http context + HttpClientUtil.addRequestInterceptor(interceptor); return builder; } @@ -408,7 +392,7 @@ static class AuthenticationResponse { private String errorMessage; private AuthCode authCode; private InvalidJwtException jwtException; - + enum AuthCode { PASS_THROUGH("No user, pass through"), // Returned when no user authentication but block_unknown=false AUTHENTICATED("Authenticated"), // Returned when authentication OK @@ -471,4 +455,137 @@ public AuthCode getAuthCode() { } } + /** + * Principal object that carries JWT token and claims for authenticated user. + */ + public static class JWTPrincipal implements Principal, Serializable { + private static final long serialVersionUID = 4144666467522831388L; + final String username; + String token; + Map claims; + + /** + * User principal with user name as well as one or more roles that he/she belong to + * @param username string with user name for user + * @param token compact string representation of JWT token + * @param claims list of verified JWT claims as a map + */ + public JWTPrincipal(final String username, String token, Map claims) { + super(); + Args.notNull(username, "User name"); + Args.notNull(token, "JWT token"); + Args.notNull(claims, "JWT claims"); + this.token = token; + this.claims = claims; + this.username = username; + } + + @Override + public String getName() { + return this.username; + } + + public String getToken() { + return token; + } + + public Map getClaims() { + return claims; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JWTPrincipal that = (JWTPrincipal) o; + return Objects.equals(username, that.username) && + Objects.equals(token, that.token) && + Objects.equals(claims, that.claims); + } + + @Override + public int hashCode() { + return Objects.hash(username, token, claims); + } + + @Override + public String toString() { + return "JWTPrincipal{" + + "username='" + username + '\'' + + ", token='" + token + '\'' + + ", claims=" + claims + + '}'; + } + } + + /** + * JWT principal that contains username, token, claims and a list of roles the user has, + * so one can keep track of user-role mappings in an Identity Server external to Solr and + * pass the information to Solr in a signed JWT token. The role information can then be used to authorize + * requests without the need to maintain or lookup what roles each user belongs to.

+ */ + public static class JWTPrincipalWithUserRoles extends JWTPrincipal implements VerifiedUserRoles { + private final Set roles; + + public JWTPrincipalWithUserRoles(final String username, String token, Map claims, Set roles) { + super(username, token, claims); + Args.notNull(roles, "User roles"); + this.roles = roles; + } + + /** + * Gets the list of roles + */ + @Override + public Set getVerifiedRoles() { + return roles; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof JWTPrincipalWithUserRoles)) + return false; + JWTPrincipalWithUserRoles that = (JWTPrincipalWithUserRoles) o; + return super.equals(o) && roles.equals(that.roles); + } + + @Override + public int hashCode() { + return Objects.hash(username, token, claims, roles); + } + + @Override + public String toString() { + return "JWTPrincipalWithUserRoles{" + + "username='" + username + '\'' + + ", token='" + token + '\'' + + ", claims=" + claims + + ", roles=" + roles + + '}'; + } + } + + // The interceptor class that adds correct header or delegates to PKI + private class PkiDelegationInterceptor implements HttpRequestInterceptor { + @Override + public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { + if (context instanceof HttpClientContext) { + HttpClientContext httpClientContext = (HttpClientContext) context; + if (httpClientContext.getUserToken() instanceof JWTPrincipal) { + JWTPrincipal jwtPrincipal = (JWTPrincipal) httpClientContext.getUserToken(); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.token); + log.debug("Set JWT header on inter-node request"); + return; + } + } + + if (coreContainer.getPkiAuthenticationPlugin() != null) { + log.debug("Inter-node request delegated from JWTAuthPlugin to PKIAuthenticationPlugin"); + coreContainer.getPkiAuthenticationPlugin().setHeader(request); + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "JWTAuthPlugin wants to delegate inter-node request to PKI, but PKI plugin was not initialized"); + } + } + } } diff --git a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java index 43dac480168c..c0c0857839fa 100644 --- a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java @@ -244,7 +244,7 @@ public void process(HttpRequest httpRequest, HttpContext httpContext) throws Htt } @SuppressForbidden(reason = "Needs currentTimeMillis to set current time in header") - void setHeader(HttpRequest httpRequest) { + protected void setHeader(HttpRequest httpRequest) { SolrRequestInfo reqInfo = getRequestInfo(); String usr; if (reqInfo != null) { diff --git a/solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java b/solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java deleted file mode 100644 index 6d685a8f1bf2..000000000000 --- a/solr/core/src/java/org/apache/solr/security/PrincipalWithUserRoles.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security; - -import java.io.Serializable; -import java.security.Principal; -import java.util.Set; - -import org.apache.http.util.Args; - -/** - * Type of Principal object that can contain also a list of roles the user has. - * One use case can be to keep track of user-role mappings in an Identity Server - * external to Solr and pass the information to Solr in a signed JWT token or in - * another secure manner. The role information can then be used to authorize - * requests without the need to maintain or lookup what roles each user belongs to. - */ -public class PrincipalWithUserRoles implements Principal, VerifiedUserRoles, Serializable { - private static final long serialVersionUID = 4144666467522831388L; - private final String username; - - private final Set roles; - - /** - * User principal with user name as well as one or more roles that he/she belong to - * @param username string with user name for user - * @param roles a set of roles that we know this user belongs to, or empty list for no roles - */ - public PrincipalWithUserRoles(final String username, Set roles) { - super(); - Args.notNull(username, "User name"); - Args.notNull(roles, "User roles"); - this.username = username; - this.roles = roles; - } - - /** - * Returns the name of this principal. - * - * @return the name of this principal. - */ - @Override - public String getName() { - return this.username; - } - - /** - * Gets the list of roles - */ - @Override - public Set getVerifiedRoles() { - return roles; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - PrincipalWithUserRoles that = (PrincipalWithUserRoles) o; - - if (!username.equals(that.username)) return false; - return roles.equals(that.roles); - } - - @Override - public int hashCode() { - int result = username.hashCode(); - result = 31 * result + roles.hashCode(); - return result; - } - - @Override - public String toString() { - return "PrincipalWithUserRoles{" + - "username='" + username + '\'' + - ", roles=" + roles + - '}'; - } -} - diff --git a/solr/core/src/java/org/apache/solr/security/TokenCredentials.java b/solr/core/src/java/org/apache/solr/security/TokenCredentials.java deleted file mode 100644 index 6433fd622d2f..000000000000 --- a/solr/core/src/java/org/apache/solr/security/TokenCredentials.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security; - -import java.security.Principal; - -import org.apache.http.auth.BasicUserPrincipal; -import org.apache.http.auth.Credentials; - -/** - * Credentials implementation that holds on to a (JWT) token for later use in inter-node requests - */ -public class TokenCredentials implements Credentials { - private Principal userPrincipal; - - public TokenCredentials(String token) { - this.userPrincipal = new BasicUserPrincipal(token); - } - - @Override - public Principal getUserPrincipal() { - return userPrincipal; - } - - @Override - public String getPassword() { - return null; - } -} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 6a2b9f1286bb..3fd3641bdd17 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -16,17 +16,16 @@ */ package org.apache.solr.security; -import org.apache.http.Header; -import org.apache.http.HttpHeaders; -import org.apache.http.message.BasicHeader; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.stream.Collectors; + import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.Pair; import org.jose4j.jwk.PublicJsonWebKey; import org.jose4j.jwk.RsaJsonWebKey; import org.jose4j.jws.AlgorithmIdentifiers; @@ -40,26 +39,29 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudTestCase { protected static final int NUM_SERVERS = 2; protected static final int NUM_SHARDS = 2; protected static final int REPLICATION_FACTOR = 1; - private static Header tokenHeader; - private static String jwkJSON = "{\n" + - " \"kty\": \"RSA\",\n" + - " \"d\": \"i6pyv2z3o-MlYytWsOr3IE1olu2RXZBzjPRBNgWAP1TlLNaphHEvH5aHhe_CtBAastgFFMuP29CFhaL3_tGczkvWJkSveZQN2AHWHgRShKgoSVMspkhOt3Ghha4CvpnZ9BnQzVHnaBnHDTTTfVgXz7P1ZNBhQY4URG61DKIF-JSSClyh1xKuMoJX0lILXDYGGcjVTZL_hci4IXPPTpOJHV51-pxuO7WU5M9252UYoiYyCJ56ai8N49aKIMsqhdGuO4aWUwsGIW4oQpjtce5eEojCprYl-9rDhTwLAFoBtjy6LvkqlR2Ae5dKZYpStljBjK8PJrBvWZjXAEMDdQ8PuQ\",\n" + - " \"e\": \"AQAB\",\n" + - " \"use\": \"sig\",\n" + - " \"kid\": \"test\",\n" + - " \"alg\": \"RS256\",\n" + - " \"n\": \"jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ\"\n" + - "}"; - private static PublicJsonWebKey jwk; - + private static String jwtTestToken; + private static String baseUrl; + @BeforeClass public static void setupClass() throws Exception { configureCluster(NUM_SERVERS)// nodes .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json")) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) .configure(); + String hostport = cluster.getSolrClient().getClusterStateProvider().getLiveNodes().iterator().next().split("_")[0]; + baseUrl = "http://" + hostport + "/solr/"; + + String jwkJSON = "{\n" + + " \"kty\": \"RSA\",\n" + + " \"d\": \"i6pyv2z3o-MlYytWsOr3IE1olu2RXZBzjPRBNgWAP1TlLNaphHEvH5aHhe_CtBAastgFFMuP29CFhaL3_tGczkvWJkSveZQN2AHWHgRShKgoSVMspkhOt3Ghha4CvpnZ9BnQzVHnaBnHDTTTfVgXz7P1ZNBhQY4URG61DKIF-JSSClyh1xKuMoJX0lILXDYGGcjVTZL_hci4IXPPTpOJHV51-pxuO7WU5M9252UYoiYyCJ56ai8N49aKIMsqhdGuO4aWUwsGIW4oQpjtce5eEojCprYl-9rDhTwLAFoBtjy6LvkqlR2Ae5dKZYpStljBjK8PJrBvWZjXAEMDdQ8PuQ\",\n" + + " \"e\": \"AQAB\",\n" + + " \"use\": \"sig\",\n" + + " \"kid\": \"test\",\n" + + " \"alg\": \"RS256\",\n" + + " \"n\": \"jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ\"\n" + + "}"; - jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); + PublicJsonWebKey jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); JwtClaims claims = JWTAuthPluginTest.generateClaims(); JsonWebSignature jws = new JsonWebSignature(); jws.setPayload(claims.toJson()); @@ -67,56 +69,127 @@ public static void setupClass() throws Exception { jws.setKeyIdHeaderValue(jwk.getKeyId()); jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); - String testJwt = jws.getCompactSerialization(); - tokenHeader = new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + testJwt); + jwtTestToken = jws.getCompactSerialization(); } @AfterClass public static void tearDownClass() throws Exception { System.clearProperty("java.security.auth.login.config"); + shutdownCluster(); } - @Test(expected = HttpSolrClient.RemoteSolrException.class) - public void testWithoutToken() throws Exception { - testCollectionCreateSearchDelete(); - // sometimes run a second test e.g. to test collection create-delete-create scenario - if (random().nextBoolean()) testCollectionCreateSearchDelete(); + @Test(expected = IOException.class) + public void infoRequestWithoutToken() throws Exception { + get(baseUrl + "admin/info/system", null); } @Test - public void testWithToken() throws Exception { - enableToken(); - testCollectionCreateSearchDelete(); - // sometimes run a second test e.g. to test collection create-delete-create scenario - if (random().nextBoolean()) testCollectionCreateSearchDelete(); + public void infoRequestWithToken() throws IOException { + Pair result = get(baseUrl + "admin/info/system", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); } - private void enableToken() { + @Test + public void createCollectionAndQueryDistributed() throws Exception { + // Admin request will use PKI inter-node auth from Overseer, and succeed + assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); + // Now do a distributed query, using JWTAUth for inter-node + Pair result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); } - protected void testCollectionCreateSearchDelete() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String collectionName = "jwtAuthTestColl"; + + +// NOCOMMIT: Test using SolrJ as client +// private void testCollectionCreateSearchDelete(boolean enableJwt) throws Exception { +// if (enableJwt) { +// HttpClientUtil.setHttpClientBuilder(getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder(), testJwt)); +// } else { +// HttpClientUtil.resetHttpClientBuilder(); +// } + +// ClusterStateProvider csProv = cluster.getSolrClient().getClusterStateProvider(); +// CloudSolrClientBuilder builder = new CloudSolrClientBuilder(csProv); +// CloudSolrClient solrClient = builder.build(); + +// CloudSolrClient solrClient = cluster.getSolrClient(); + +// URL solrurl = new URL(baseUrl + "admin/info/system"); +// HttpURLConnection conn = (HttpURLConnection) solrurl.openConnection(); +// conn.setRequestProperty("Authorization", "Bearer " + testJwt); +// conn.connect(); +// BufferedReader br = new BufferedReader(new InputStreamReader((InputStream) conn.getContent())); +// br.lines().forEach(l -> { +// System.out.println(l); +// }); +// conn.disconnect(); + + +// Pair result; +// +// result = get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2"); +// System.out.println(result.first()); +// assertEquals(Integer.valueOf(200), result.second()); +// +// result = get(baseUrl + "mycoll/query?q=*:*"); +// System.out.println(result.first()); +// assertEquals(Integer.valueOf(200), result.second()); + + +// String collectionName = "jwtAuthTestColl"; // create collection - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", - NUM_SHARDS, REPLICATION_FACTOR); - create.process(solrClient); - - solrClient.add(collectionName, new SolrInputDocument("id", "1")); - solrClient.add(collectionName, new SolrInputDocument("id", "2")); - solrClient.commit(collectionName); - - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - QueryResponse rsp = solrClient.query(collectionName, query); - assertEquals(2, rsp.getResults().getNumFound()); - - CollectionAdminRequest.Delete deleteReq = CollectionAdminRequest.deleteCollection(collectionName); - deleteReq.process(solrClient); - AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, - solrClient.getZkStateReader(), true, true, 330); +// CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", +// NUM_SHARDS, REPLICATION_FACTOR); +// create.process(solrClient); +// +// solrClient.add(collectionName, new SolrInputDocument("id", "1")); +// solrClient.add(collectionName, new SolrInputDocument("id", "2")); +// solrClient.commit(collectionName); +// +// SolrQuery query = new SolrQuery(); +// query.setQuery("*:*"); +// QueryResponse rsp = solrClient.query(collectionName, query); +// assertEquals(2, rsp.getResults().getNumFound()); +// +// CollectionAdminRequest.Delete deleteReq = CollectionAdminRequest.deleteCollection(collectionName); +// deleteReq.process(solrClient); +// AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, +// solrClient.getZkStateReader(), true, true, 330); +// solrClient.close(); +// } + + private Pair get(String url, String token) throws IOException { + URL createUrl = new URL(url); + HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); + if (token != null) + createConn.setRequestProperty("Authorization", "Bearer " + token); + createConn.connect(); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent())); + String result = br2.lines().collect(Collectors.joining("\n")); + int code = createConn.getResponseCode(); + createConn.disconnect(); + return new Pair<>(result, code); } +// SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder, String token) { +// if (builder == null) { +// builder = SolrHttpClientBuilder.create(); +// } +// builder.setAuthSchemeRegistryProvider(() -> { +// Lookup authProviders = RegistryBuilder.create() +// .register("Bearer", new JWTAuthPlugin.JwtBearerAuthschemeProvider()) +// .build(); +// return authProviders; +// }); +// builder.setDefaultCredentialsProvider(() -> { +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(token, null)); +// return credentialsProvider; +// }); +// return builder; +// } + + } diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index f9529260ca78..725855176d62 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -46,12 +46,10 @@ public class JWTAuthPluginTest extends SolrTestCaseJ4 { private static String testHeader; - private static String slimJwt; private static String slimHeader; private JWTAuthPlugin plugin; private HashMap testJwk; private static RsaJsonWebKey rsaJsonWebKey; - private static String testJwt; private HashMap testConfig; @@ -68,21 +66,18 @@ public static void beforeAll() throws Exception { jws.setKeyIdHeaderValue(rsaJsonWebKey.getKeyId()); jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); - testJwt = jws.getCompactSerialization(); + String testJwt = jws.getCompactSerialization(); testHeader = "Bearer" + " " + testJwt; - - System.out.println("Header:\n" + testHeader); - System.out.println("JWK:\n" + rsaJsonWebKey.toJson()); claims.unsetClaim("iss"); claims.unsetClaim("aud"); claims.unsetClaim("exp"); jws.setPayload(claims.toJson()); - slimJwt = jws.getCompactSerialization(); + String slimJwt = jws.getCompactSerialization(); slimHeader = "Bearer" + " " + slimJwt; } - protected static JwtClaims generateClaims() { + static JwtClaims generateClaims() { JwtClaims claims = new JwtClaims(); claims.setIssuer("IDServer"); // who creates the token and signs it claims.setAudience("Solr"); // to whom the token is intended to be sent @@ -105,7 +100,7 @@ protected static JwtClaims generateClaims() { public void setUp() throws Exception { super.setUp(); // Create an auth plugin - plugin = new JWTAuthPlugin(); + plugin = new JWTAuthPlugin(null); // Create a JWK config for security.json testJwk = new HashMap<>(); @@ -130,9 +125,8 @@ public void tearDown() throws Exception { } @Test - public void initWithoutRequired() throws Exception { - HashMap authConf = new HashMap(); - plugin = new JWTAuthPlugin(); + public void initWithoutRequired() { + HashMap authConf = new HashMap<>(); plugin.init(authConf); assertEquals(AUTZ_HEADER_PROBLEM, plugin.authenticate("foo").getAuthCode()); } @@ -160,18 +154,18 @@ public void initFromSecurityJSONUrlJwk() throws Exception { } @Test - public void initWithJwk() throws Exception { - HashMap authConf = new HashMap(); + public void initWithJwk() { + HashMap authConf = new HashMap<>(); authConf.put("jwk", testJwk); - plugin = new JWTAuthPlugin(); + plugin = new JWTAuthPlugin(null); plugin.init(authConf); } @Test - public void initWithJwkUrl() throws Exception { - HashMap authConf = new HashMap(); + public void initWithJwkUrl() { + HashMap authConf = new HashMap<>(); authConf.put("jwk_url", "https://127.0.0.1:9999/foo.jwk"); - plugin = new JWTAuthPlugin(); + plugin = new JWTAuthPlugin(null); plugin.init(authConf); } @@ -187,14 +181,14 @@ public void parseJwkSet() throws Exception { } @Test - public void authenticateOk() throws Exception { + public void authenticateOk() { AuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); assertEquals("solruser", resp.getPrincipal().getName()); } @Test - public void authFailedMissingSubject() throws Exception { + public void authFailedMissingSubject() { testConfig.put("iss", "NA"); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(testHeader); @@ -208,7 +202,7 @@ public void authFailedMissingSubject() throws Exception { } @Test - public void authFailedMissingAudience() throws Exception { + public void authFailedMissingAudience() { testConfig.put("aud", "NA"); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(testHeader); @@ -222,7 +216,7 @@ public void authFailedMissingAudience() throws Exception { } @Test - public void authFailedMissingPrincipal() throws Exception { + public void authFailedMissingPrincipal() { testConfig.put("principal_claim", "customPrincipal"); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(testHeader); @@ -236,7 +230,7 @@ public void authFailedMissingPrincipal() throws Exception { } @Test - public void claimMatch() throws Exception { + public void claimMatch() { // all custom claims match regex Map shouldMatch = new HashMap<>(); shouldMatch.put("claim1", "foo"); @@ -262,7 +256,7 @@ public void claimMatch() throws Exception { } @Test - public void missingIssAudExp() throws Exception { + public void missingIssAudExp() { testConfig.put("require_exp", "false"); testConfig.put("require_sub", "false"); plugin.init(testConfig); @@ -283,7 +277,7 @@ public void missingIssAudExp() throws Exception { } @Test - public void algWhitelist() throws Exception { + public void algWhitelist() { testConfig.put("alg_whitelist", Arrays.asList("PS384", "PS512")); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(testHeader); @@ -292,7 +286,7 @@ public void algWhitelist() throws Exception { } @Test - public void roles() throws Exception { + public void roles() { testConfig.put("roles_claim", "groups"); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(testHeader); @@ -312,7 +306,7 @@ public void roles() throws Exception { } @Test - public void noHeaderBlockUnknown() throws Exception { + public void noHeaderBlockUnknown() { testConfig.put("block_unknown", true); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(null); @@ -320,12 +314,10 @@ public void noHeaderBlockUnknown() throws Exception { } @Test - public void noHeaderNotBlockUnknown() throws Exception { + public void noHeaderNotBlockUnknown() { testConfig.put("block_unknown", false); plugin.init(testConfig); AuthenticationResponse resp = plugin.authenticate(null); assertEquals(AuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); } - - // TODO: Add inter-node request tests } \ No newline at end of file diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java index 7dbaab90915c..04b94f7aae84 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.Serializable; +import java.security.Principal; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -36,6 +37,16 @@ * @since solr 1.3 */ public abstract class SolrRequest implements Serializable { + // This user principal is typically used by Auth plugins during distributed/sharded search + private Principal userPrincipal; + + public void setUserPrincipal(Principal userPrincipal) { + this.userPrincipal = userPrincipal; + } + + public Principal getUserPrincipal() { + return userPrincipal; + } public enum METHOD { GET, diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index 2b60e337e4a1..057122eb5fb3 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -24,6 +24,7 @@ import java.net.ConnectException; import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; +import java.security.Principal; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -252,7 +253,7 @@ public NamedList request(final SolrRequest request, final ResponseParser throws SolrServerException, IOException { HttpRequestBase method = createMethod(request, collection); setBasicAuthHeader(request, method); - return executeMethod(method, processor, isV2ApiRequest(request)); + return executeMethod(method, request.getUserPrincipal(), processor, isV2ApiRequest(request)); } private boolean isV2ApiRequest(final SolrRequest request) { @@ -296,7 +297,7 @@ public HttpUriRequestResponse httpUriRequest(final SolrRequest request, final Re ExecutorService pool = ExecutorUtil.newMDCAwareFixedThreadPool(1, new SolrjNamedThreadFactory("httpUriRequest")); try { MDC.put("HttpSolrClient.url", baseUrl); - mrr.future = pool.submit(() -> executeMethod(method, processor, isV2ApiRequest(request))); + mrr.future = pool.submit(() -> executeMethod(method, request.getUserPrincipal(), processor, isV2ApiRequest(request))); } finally { pool.shutdown(); @@ -517,7 +518,7 @@ private HttpEntityEnclosingRequestBase fillContentStream(SolrRequest request, Co private static final List errPath = Arrays.asList("metadata", "error-class");//Utils.getObjectByPath(err, false,"metadata/error-class") - protected NamedList executeMethod(HttpRequestBase method, final ResponseParser processor, final boolean isV2Api) throws SolrServerException { + protected NamedList executeMethod(HttpRequestBase method, Principal userPrincipal, final ResponseParser processor, final boolean isV2Api) throws SolrServerException { method.addHeader("User-Agent", AGENT); org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = HttpClientUtil.createDefaultRequestConfigBuilder(); @@ -539,6 +540,12 @@ protected NamedList executeMethod(HttpRequestBase method, final Response try { // Execute the method. HttpClientContext httpClientRequestContext = HttpClientUtil.createNewHttpClientRequestContext(); + if (userPrincipal != null) { + // Normally the context contains a static userToken to enable reuse resources. + // However, if a personal Principal object exists, we use that instead, also as a means + // to transfer authentication information to Auth plugins that wish to intercept the request later + httpClientRequestContext.setUserToken(userPrincipal); + } final HttpResponse response = httpClient.execute(method, httpClientRequestContext); int httpStatus = response.getStatusLine().getStatusCode(); From 1fa8aacab56aceaaa85f5bb1a0e749c0a9c3825f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 27 Aug 2018 13:18:37 +0200 Subject: [PATCH 10/88] POST test --- .../JWTAuthPluginIntegrationTest.java | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 3fd3641bdd17..5f80c87ef52f 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.stream.Collectors; @@ -97,10 +98,50 @@ public void createCollectionAndQueryDistributed() throws Exception { // Now do a distributed query, using JWTAUth for inter-node Pair result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); + + // Delete + assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); } - - + @Test + public void createCollectionAndUpdateDistributed() throws Exception { + // Admin request will use PKI inter-node auth from Overseer, and succeed + assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); + + // Now update two documents + Pair result = post(baseUrl + "mycoll/update", "{\"id\" : \"1\"}", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + result = post(baseUrl + "mycoll/update", "{\"id\" : \"2\"}", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + result = post(baseUrl + "mycoll/update", "{\"id\" : \"3\"}", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + + // Delete + assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); + } + + private Pair post(String url, String json, String token) throws IOException { + URL createUrl = new URL(url); + HttpURLConnection con = (HttpURLConnection) createUrl.openConnection(); + con.setRequestMethod("POST"); + if (token != null) + con.setRequestProperty("Authorization", "Bearer " + token); + + con.setDoOutput(true); + OutputStream os = con.getOutputStream(); + os.write(json.getBytes()); + os.flush(); + os.close(); + + con.connect(); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent())); + String result = br2.lines().collect(Collectors.joining("\n")); + int code = con.getResponseCode(); + con.disconnect(); + return new Pair<>(result, code); + } + + // NOCOMMIT: Test using SolrJ as client // private void testCollectionCreateSearchDelete(boolean enableJwt) throws Exception { // if (enableJwt) { From 51232a2eff3add9556a895c295f6cccfaa265263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 27 Aug 2018 14:26:37 +0200 Subject: [PATCH 11/88] Transfer principal from original request to distributed requests --- .../solr/update/SolrCmdDistributor.java | 3 + .../JWTAuthPluginIntegrationTest.java | 91 +++++++++++++------ 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index d5aafeca49ee..9b736d2032d0 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -47,6 +47,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.Diagnostics; +import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker; import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker; @@ -278,6 +279,8 @@ void addCommit(UpdateRequest ureq, CommitUpdateCommand cmd) { } private void submit(final Req req, boolean isCommit) { + // Copy user principal from the original request to the new update request, for later authentication interceptor use + req.uReq.setUserPrincipal(SolrRequestInfo.getRequestInfo().getReq().getUserPrincipal()); if (req.synchronous) { blockAndDoRetries(); diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 5f80c87ef52f..ce5ab73fbbf7 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -23,8 +23,17 @@ import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import org.apache.http.Header; +import org.apache.http.HttpException; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.entity.ContentType; +import org.apache.http.protocol.HttpContext; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.util.Pair; import org.jose4j.jwk.PublicJsonWebKey; @@ -33,6 +42,7 @@ import org.jose4j.jws.JsonWebSignature; import org.jose4j.jwt.JwtClaims; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -42,6 +52,9 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudTestCase { protected static final int REPLICATION_FACTOR = 1; private static String jwtTestToken; private static String baseUrl; + private static AtomicInteger jwtInterceptCount = new AtomicInteger(); + private static AtomicInteger pkiInterceptCount = new AtomicInteger(); + private static final CountInterceptor interceptor = new CountInterceptor(); @BeforeClass public static void setupClass() throws Exception { @@ -71,6 +84,9 @@ public static void setupClass() throws Exception { jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); jwtTestToken = jws.getCompactSerialization(); + + HttpClientUtil.removeRequestInterceptor(interceptor); + HttpClientUtil.addRequestInterceptor(interceptor); } @AfterClass @@ -79,6 +95,12 @@ public static void tearDownClass() throws Exception { shutdownCluster(); } + @Before + public void before() { + jwtInterceptCount.set(0); + pkiInterceptCount.set(0); + } + @Test(expected = IOException.class) public void infoRequestWithoutToken() throws Exception { get(baseUrl + "admin/info/system", null); @@ -88,6 +110,7 @@ public void infoRequestWithoutToken() throws Exception { public void infoRequestWithToken() throws IOException { Pair result = get(baseUrl + "admin/info/system", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); + verifyInterRequestHeaderCounts(0,0); } @Test @@ -101,6 +124,7 @@ public void createCollectionAndQueryDistributed() throws Exception { // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); + verifyInterRequestHeaderCounts(2,2); } @Test @@ -109,37 +133,14 @@ public void createCollectionAndUpdateDistributed() throws Exception { assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); // Now update two documents - Pair result = post(baseUrl + "mycoll/update", "{\"id\" : \"1\"}", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - result = post(baseUrl + "mycoll/update", "{\"id\" : \"2\"}", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - result = post(baseUrl + "mycoll/update", "{\"id\" : \"3\"}", jwtTestToken); + Pair result = post(baseUrl + "mycoll/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); + verifyInterRequestHeaderCounts(2,2); } - private Pair post(String url, String json, String token) throws IOException { - URL createUrl = new URL(url); - HttpURLConnection con = (HttpURLConnection) createUrl.openConnection(); - con.setRequestMethod("POST"); - if (token != null) - con.setRequestProperty("Authorization", "Bearer " + token); - - con.setDoOutput(true); - OutputStream os = con.getOutputStream(); - os.write(json.getBytes()); - os.flush(); - os.close(); - - con.connect(); - BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent())); - String result = br2.lines().collect(Collectors.joining("\n")); - int code = con.getResponseCode(); - con.disconnect(); - return new Pair<>(result, code); - } // NOCOMMIT: Test using SolrJ as client @@ -201,6 +202,11 @@ private Pair post(String url, String json, String token) throws // solrClient.close(); // } + private void verifyInterRequestHeaderCounts(int jwt, int pki) { + assertEquals(jwt, jwtInterceptCount.get()); + assertEquals(pki, jwtInterceptCount.get()); + } + private Pair get(String url, String token) throws IOException { URL createUrl = new URL(url); HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); @@ -214,6 +220,28 @@ private Pair get(String url, String token) throws IOException { return new Pair<>(result, code); } + private Pair post(String url, String json, String token) throws IOException { + URL createUrl = new URL(url); + HttpURLConnection con = (HttpURLConnection) createUrl.openConnection(); + con.setRequestMethod("POST"); + con.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); + if (token != null) + con.setRequestProperty("Authorization", "Bearer " + token); + + con.setDoOutput(true); + OutputStream os = con.getOutputStream(); + os.write(json.getBytes()); + os.flush(); + os.close(); + + con.connect(); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent())); + String result = br2.lines().collect(Collectors.joining("\n")); + int code = con.getResponseCode(); + con.disconnect(); + return new Pair<>(result, code); + } + // SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder, String token) { // if (builder == null) { // builder = SolrHttpClientBuilder.create(); @@ -232,5 +260,16 @@ private Pair get(String url, String token) throws IOException { // return builder; // } - + private static class CountInterceptor implements HttpRequestInterceptor { + @Override + public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { + Header ah = request.getFirstHeader(HttpHeaders.AUTHORIZATION); + if (ah != null && ah.getValue().startsWith("Bearer")) + jwtInterceptCount.addAndGet(1); + + Header ph = request.getFirstHeader(PKIAuthenticationPlugin.HEADER); + if (ph != null) + pkiInterceptCount.addAndGet(1); + } + } } From 711303281946e0fba768fe62ab9826a0090069e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 27 Aug 2018 15:03:33 +0200 Subject: [PATCH 12/88] Debug level for query test --- .../solr/security/JWTAuthPluginIntegrationTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index ce5ab73fbbf7..63c24f70c004 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -36,6 +36,7 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.util.Pair; +import org.apache.solr.util.LogLevel; import org.jose4j.jwk.PublicJsonWebKey; import org.jose4j.jwk.RsaJsonWebKey; import org.jose4j.jws.AlgorithmIdentifiers; @@ -114,13 +115,21 @@ public void infoRequestWithToken() throws IOException { } @Test + @LogLevel("org.apache.solr.security=DEBUG") public void createCollectionAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); + verifyInterRequestHeaderCounts(0,0); + // First a non distributed query + Pair result = get(baseUrl + "mycoll/query?q=*:*&distrib=false", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + verifyInterRequestHeaderCounts(0,0); + // Now do a distributed query, using JWTAUth for inter-node - Pair result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); + result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); + verifyInterRequestHeaderCounts(2,2); // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); From d7a1c38f503ac1df15af8e4007aa9722cc8028fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 5 Sep 2018 14:46:27 +0200 Subject: [PATCH 13/88] Support for well-known config and scopes --- .../apache/solr/security/JWTAuthPlugin.java | 406 ++++++++++-------- .../apache/solr/security/JWTPrincipal.java | 88 ++++ .../security/JWTPrincipalWithUserRoles.java | 71 +++ .../solr/security/jwt_well-known-config.json | 84 ++++ .../solr/security/JWTAuthPluginTest.java | 137 ++++-- .../src/jwt-authentication-plugin.adoc | 63 ++- 6 files changed, 604 insertions(+), 245 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/security/JWTPrincipal.java create mode 100644 solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java create mode 100644 solr/core/src/test-files/solr/security/jwt_well-known-config.json diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index a9a5f76b6a37..20a0de159319 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -22,19 +22,22 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequestWrapper; import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.Serializable; +import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.Charset; import java.security.Principal; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.StringTokenizer; import java.util.regex.Pattern; @@ -45,15 +48,15 @@ import org.apache.http.HttpRequestInterceptor; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.protocol.HttpContext; -import org.apache.http.util.Args; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; +import org.apache.solr.common.StringUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.core.CoreContainer; -import org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode; +import org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode; import org.jose4j.jwa.AlgorithmConstraints; import org.jose4j.jwk.HttpsJwks; import org.jose4j.jwk.JsonWebKey; @@ -82,14 +85,18 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String PARAM_AUDIENCE = "aud"; private static final String PARAM_REQUIRE_SUBJECT = "require_sub"; private static final String PARAM_PRINCIPAL_CLAIM = "principal_claim"; - private static final String PARAM_ROLES_CLAIM = "roles_claim"; private static final String PARAM_REQUIRE_EXPIRATIONTIME = "require_exp"; private static final String PARAM_ALG_WHITELIST = "alg_whitelist"; private static final String PARAM_JWK_CACHE_DURATION = "jwk_cache_dur"; private static final String PARAM_CLAIMS_MATCH = "claims_match"; - private static final String AUTH_REALM = "solr"; + private static final String PARAM_SCOPE = "scope"; + private static final String PARAM_ADMIN_SCOPE = "admin_scope"; + private static final String PARAM_CLIENT_ID = "client_id"; + private static final String PARAM_WELL_KNOWN_URL = "well_known_url"; + private static final String AUTH_REALM = "solr-jwt"; + private static final String CLAIM_SCOPE = "scope"; - private final PkiDelegationInterceptor interceptor = new PkiDelegationInterceptor(); + private final JwtPkiDelegationInterceptor interceptor = new JwtPkiDelegationInterceptor(); private JwtConsumer jwtConsumer; private String iss; @@ -99,9 +106,16 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private List algWhitelist; private VerificationKeyResolver verificationKeyResolver; private String principalClaim; - private String rolesClaim; private HashMap claimsMatchCompiled; private boolean blockUnknown; + private String adminScope; + private HashSet requiredScopes = new HashSet<>(); + private String clientId; + private long jwkCacheDuration; + private OidcDiscoveryConfig oidcDiscoveryConfig; + private String confIdpConfigUrl; + private Map pluginConfig; + private Instant lastInitTime = Instant.now(); private CoreContainer coreContainer; /** @@ -115,16 +129,41 @@ public JWTAuthPlugin(CoreContainer coreContainer) { @Override public void init(Map pluginConfig) { blockUnknown = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCK_UNKNOWN, false))); - String jwk_url = (String) pluginConfig.get(PARAM_JWK_URL); - Map jwk = (Map) pluginConfig.get(PARAM_JWK); - iss = (String) pluginConfig.get(PARAM_ISSUER); - aud = (String) pluginConfig.get(PARAM_AUDIENCE); + clientId = (String) pluginConfig.get(PARAM_CLIENT_ID); requireSubject = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_SUBJECT, "true"))); requireExpirationTime = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_EXPIRATIONTIME, "true"))); - algWhitelist = (List) pluginConfig.get(PARAM_ALG_WHITELIST); - long jwkCacheDuration = Long.parseLong((String) pluginConfig.getOrDefault(PARAM_JWK_CACHE_DURATION, "3600")); principalClaim = (String) pluginConfig.getOrDefault(PARAM_PRINCIPAL_CLAIM, "sub"); - rolesClaim = (String) pluginConfig.get(PARAM_ROLES_CLAIM); + confIdpConfigUrl = (String) pluginConfig.get(PARAM_WELL_KNOWN_URL); + + if (confIdpConfigUrl != null) { + log.debug("Initializing well-known oidc config from {}", confIdpConfigUrl); + oidcDiscoveryConfig = OidcDiscoveryConfig.parse(confIdpConfigUrl); + iss = oidcDiscoveryConfig.getIssuer(); + } + + if (pluginConfig.containsKey(PARAM_ISSUER)) { + if (iss != null) { + log.debug("Explicitly setting required issuer instead of using issuer from well-known config"); + } + iss = (String) pluginConfig.get(PARAM_ISSUER); + } + + if (pluginConfig.containsKey(PARAM_AUDIENCE)) { + if (clientId != null) { + log.debug("Explicitly setting required audience instead of using configured clientId"); + } + aud = (String) pluginConfig.get(PARAM_AUDIENCE); + } else { + aud = clientId; + } + + algWhitelist = (List) pluginConfig.get(PARAM_ALG_WHITELIST); + + String requiredScopesStr = (String) pluginConfig.get(PARAM_SCOPE); + if (!StringUtils.isEmpty(requiredScopesStr)) { + requiredScopes = new HashSet<>(Arrays.asList(requiredScopesStr.split("\\s+"))); + } + adminScope = (String) pluginConfig.get(PARAM_ADMIN_SCOPE); Map claimsMatch = (Map) pluginConfig.get(PARAM_CLAIMS_MATCH); claimsMatchCompiled = new HashMap<>(); if (claimsMatch != null) { @@ -133,32 +172,58 @@ public void init(Map pluginConfig) { } } + initJwk(pluginConfig); + + lastInitTime = Instant.now(); + } + + private void initJwk(Map pluginConfig) { + this.pluginConfig = pluginConfig; + String confJwkUrl = (String) pluginConfig.get(PARAM_JWK_URL); + Map confJwk = (Map) pluginConfig.get(PARAM_JWK); + jwkCacheDuration = Long.parseLong((String) pluginConfig.getOrDefault(PARAM_JWK_CACHE_DURATION, "3600")); + jwtConsumer = null; - if (jwk_url != null) { - // The HttpsJwks retrieves and caches keys from a the given HTTPS JWKS endpoint. - try { - URL jwkUrl = new URL(jwk_url); - if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be an HTTPS url"); - } - } catch (MalformedURLException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be a valid https URL"); - } - HttpsJwks httpsJkws = new HttpsJwks(jwk_url); - httpsJkws.setDefaultCacheDuration(jwkCacheDuration); - verificationKeyResolver = new HttpsJwksVerificationKeyResolver(httpsJkws); - initConsumer(); - } else if (jwk != null) { + int jwkConfigured = confIdpConfigUrl != null ? 1 : 0; + jwkConfigured += confJwkUrl != null ? 1 : 0; + jwkConfigured += confJwk != null ? 1 : 0; + if (jwkConfigured > 1) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuthPlugin needs to configure exactly one of " + + PARAM_WELL_KNOWN_URL + ", " + PARAM_JWK_URL + " and " + PARAM_JWK); + } + if (jwkConfigured == 0) { + log.warn("Initialized JWTAuthPlugin without any JWK config. No requests will succeed"); + } + if (oidcDiscoveryConfig != null) { + String jwkUrl = oidcDiscoveryConfig.getJwksUrl(); + setupJwkUrl(jwkUrl); + } else if (confJwkUrl != null) { + setupJwkUrl(confJwkUrl); + } else if (confJwk != null) { try { - JsonWebKeySet jwks = parseJwkSet(jwk); + JsonWebKeySet jwks = parseJwkSet(confJwk); verificationKeyResolver = new JwksVerificationKeyResolver(jwks.getJsonWebKeys()); - initConsumer(); } catch (JoseException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid JWTAuthPlugin configuration, jwk field parse error", e); } - } else { - log.warn("JWTAuthPlugin needs to specify either 'jwk' or 'jwk_url' parameters."); } + initConsumer(); + log.debug("JWK configured"); + } + + private void setupJwkUrl(String url) { + // The HttpsJwks retrieves and caches keys from a the given HTTPS JWKS endpoint. + try { + URL jwkUrl = new URL(url); + if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be an HTTPS url"); + } + } catch (MalformedURLException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be a valid URL"); + } + HttpsJwks httpsJkws = new HttpsJwks(url); + httpsJkws.setDefaultCacheDuration(jwkCacheDuration); + verificationKeyResolver = new HttpsJwksVerificationKeyResolver(httpsJkws); } JsonWebKeySet parseJwkSet(Map jwkObj) throws JoseException { @@ -185,18 +250,24 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser String header = request.getHeader(HttpHeaders.AUTHORIZATION); if (jwtConsumer == null) { - // if (header == null && !blockUnknown) { log.info("JWTAuth not configured, but allowing anonymous access since blockUnknown==false"); filterChain.doFilter(request, response); return true; } - log.warn("JWTAuth not configured"); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin not correctly configured"); + // Retry config + if (lastInitTime.plusSeconds(10).isAfter(Instant.now())) { + log.info("Retrying JWTAuthPlugin initialization"); + init(pluginConfig); + } + if (jwtConsumer == null) { + log.warn("JWTAuth not configured"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin not correctly configured"); + } } - AuthenticationResponse authResponse = authenticate(header); - switch(authResponse.authCode) { + JWTAuthenticationResponse authResponse = authenticate(header); + switch(authResponse.getAuthCode()) { case AUTHENTICATED: HttpServletRequestWrapper wrapper = new HttpServletRequestWrapper(request) { @Override @@ -219,8 +290,8 @@ public Principal getUserPrincipal() { return true; case AUTZ_HEADER_PROBLEM: - log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); - authenticationFailure(response, authResponse.getAuthCode().msg, HttpServletResponse.SC_BAD_REQUEST, BearerWwwAuthErrorCode.invalid_request); + log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); + authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_BAD_REQUEST, BearerWwwAuthErrorCode.invalid_request); return false; case CLAIM_MISMATCH: @@ -228,17 +299,22 @@ public Principal getUserPrincipal() { case JWT_PARSE_ERROR: case JWT_VALIDATION_EXCEPTION: case PRINCIPAL_MISSING: - log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); + log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); if (authResponse.getJwtException() != null) { log.warn("Exception: {}", authResponse.getJwtException().getMessage()); } - authenticationFailure(response, authResponse.getAuthCode().msg, HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.invalid_token); + authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.invalid_token); return false; + case SCOPE_MISSING: + log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); + authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.insufficient_scope); + return false; + case NO_AUTZ_HEADER: default: - log.debug("Authentication failed with reason {}, message {}", authResponse.authCode, authResponse.errorMessage); - authenticationFailure(response, authResponse.getAuthCode().msg); + log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); + authenticationFailure(response, authResponse.getAuthCode().getMsg()); return false; } } @@ -249,7 +325,7 @@ public Principal getUserPrincipal() { * @param authorizationHeader the http header "Authentication" * @return AuthenticationResponse object */ - protected AuthenticationResponse authenticate(String authorizationHeader) { + protected JWTAuthenticationResponse authenticate(String authorizationHeader) { if (authorizationHeader != null) { StringTokenizer st = new StringTokenizer(authorizationHeader); if (st.hasMoreTokens()) { @@ -261,61 +337,70 @@ protected AuthenticationResponse authenticate(String authorizationHeader) { JwtClaims jwtClaims = jwtConsumer.processToClaims(jwtCompact); String principal = jwtClaims.getStringClaimValue(principalClaim); if (principal == null || principal.isEmpty()) { - return new AuthenticationResponse(AuthCode.PRINCIPAL_MISSING, "Cannot identify principal from JWT. Required claim " + principalClaim + " missing. Cannot authenticate"); + return new JWTAuthenticationResponse(AuthCode.PRINCIPAL_MISSING, "Cannot identify principal from JWT. Required claim " + principalClaim + " missing. Cannot authenticate"); } if (claimsMatchCompiled != null) { for (Map.Entry entry : claimsMatchCompiled.entrySet()) { String claim = entry.getKey(); if (jwtClaims.hasClaim(claim)) { if (!entry.getValue().matcher(jwtClaims.getStringClaimValue(claim)).matches()) { - return new AuthenticationResponse(AuthCode.CLAIM_MISMATCH, + return new JWTAuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Claim " + claim + "=" + jwtClaims.getStringClaimValue(claim) + " does not match required regular expression " + entry.getValue().pattern()); } } else { - return new AuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Claim " + claim + " is required but does not exist in JWT"); + return new JWTAuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Claim " + claim + " is required but does not exist in JWT"); } } } - Set roles = Collections.emptySet(); - if (rolesClaim != null) { - if (!jwtClaims.hasClaim(rolesClaim)) { - return new AuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Roles claim " + rolesClaim + " is required but does not exist in JWT"); + if (!requiredScopes.isEmpty() && !jwtClaims.hasClaim(CLAIM_SCOPE)) { + // Fail if we require scopes but they don't exist + return new JWTAuthenticationResponse(AuthCode.CLAIM_MISMATCH, "Claim " + CLAIM_SCOPE + " is required but does not exist in JWT"); + } + Set scopes = Collections.emptySet(); + Object scopesObj = jwtClaims.getClaimValue(CLAIM_SCOPE); + if (scopesObj != null) { + if (scopesObj instanceof String) { + scopes = new HashSet<>(Arrays.asList(((String) scopesObj).split("\\s+"))); + } else if (scopesObj instanceof List) { + scopes = new HashSet<>(jwtClaims.getStringListClaimValue(CLAIM_SCOPE)); } - Object rolesObj = jwtClaims.getClaimValue(rolesClaim); - if (rolesObj instanceof String) { - roles = Collections.singleton((String) rolesObj); - } else if (rolesObj instanceof List) { - roles = new HashSet<>(jwtClaims.getStringListClaimValue(rolesClaim)); + // Validate that at least one of the required scopes are present in the scope claim + if (!requiredScopes.isEmpty()) { + if (scopes.stream().noneMatch(requiredScopes::contains)) { + return new JWTAuthenticationResponse(AuthCode.SCOPE_MISSING, "'scope' claim does not contain any of the required scopes: " + requiredScopes); + } } - // Pass roles with principal to signal to any Authorization plugins that user has some verified role claims - return new AuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipalWithUserRoles(principal, jwtCompact, jwtClaims.getClaimsMap(), roles)); + final Set roles = new HashSet<>(scopes); + roles.remove("openid"); // Remove standard claims + // Pass scopes with principal to signal to any Authorization plugins that user has some verified role claims + return new JWTAuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipalWithUserRoles(principal, jwtCompact, jwtClaims.getClaimsMap(), roles)); } else { - return new AuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipal(principal, jwtCompact, jwtClaims.getClaimsMap())); + return new JWTAuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipal(principal, jwtCompact, jwtClaims.getClaimsMap())); } } catch (InvalidJwtException e) { // Whether or not the JWT has expired being one common reason for invalidity if (e.hasExpired()) { - return new AuthenticationResponse(AuthCode.JWT_EXPIRED, "Authentication failed due to expired JWT token. Expired at " + e.getJwtContext().getJwtClaims().getExpirationTime()); + return new JWTAuthenticationResponse(AuthCode.JWT_EXPIRED, "Authentication failed due to expired JWT token. Expired at " + e.getJwtContext().getJwtClaims().getExpirationTime()); } - return new AuthenticationResponse(AuthCode.JWT_VALIDATION_EXCEPTION, e); + return new JWTAuthenticationResponse(AuthCode.JWT_VALIDATION_EXCEPTION, e); } } catch (MalformedClaimException e) { - return new AuthenticationResponse(AuthCode.JWT_PARSE_ERROR, "Malformed claim, error was: " + e.getMessage()); + return new JWTAuthenticationResponse(AuthCode.JWT_PARSE_ERROR, "Malformed claim, error was: " + e.getMessage()); } } else { - return new AuthenticationResponse(AuthCode.AUTZ_HEADER_PROBLEM, "Authorization header is not in correct format"); + return new JWTAuthenticationResponse(AuthCode.AUTZ_HEADER_PROBLEM, "Authorization header is not in correct format"); } } else { - return new AuthenticationResponse(AuthCode.AUTZ_HEADER_PROBLEM, "Authorization header is not in correct format"); + return new JWTAuthenticationResponse(AuthCode.AUTZ_HEADER_PROBLEM, "Authorization header is not in correct format"); } } else { // No Authorization header if (blockUnknown) { - return new AuthenticationResponse(AuthCode.NO_AUTZ_HEADER, "Missing Authorization header"); + return new JWTAuthenticationResponse(AuthCode.NO_AUTZ_HEADER, "Missing Authorization header"); } else { log.debug("No user authenticated, but block_unknown=false, so letting request through"); - return new AuthenticationResponse(AuthCode.PASS_THROUGH); + return new JWTAuthenticationResponse(AuthCode.PASS_THROUGH); } } } @@ -387,12 +472,12 @@ private void authenticationFailure(HttpServletResponse response, String message, /** * Response for authentication attempt */ - static class AuthenticationResponse { + static class JWTAuthenticationResponse { private final Principal principal; private String errorMessage; private AuthCode authCode; private InvalidJwtException jwtException; - + enum AuthCode { PASS_THROUGH("No user, pass through"), // Returned when no user authentication but block_unknown=false AUTHENTICATED("Authenticated"), // Returned when authentication OK @@ -402,183 +487,130 @@ enum AuthCode { NO_AUTZ_HEADER("Require authentication"), // The Authorization header is missing JWT_EXPIRED("JWT token expired"), // JWT token has expired CLAIM_MISMATCH("Required JWT claim missing"), // Some required claims are missing or wrong - JWT_VALIDATION_EXCEPTION("JWT validation failed"); // The JWT parser failed validation. More details in exception - + JWT_VALIDATION_EXCEPTION("JWT validation failed"), // The JWT parser failed validation. More details in exception + SCOPE_MISSING("Required scope missing in JWT"); // None of the required scopes were present in JWT + + public String getMsg() { + return msg; + } + private final String msg; - + AuthCode(String msg) { this.msg = msg; } } - - public AuthenticationResponse(AuthCode authCode, InvalidJwtException e) { + + JWTAuthenticationResponse(AuthCode authCode, InvalidJwtException e) { this.authCode = authCode; this.jwtException = e; principal = null; this.errorMessage = e.getMessage(); } - public AuthenticationResponse(AuthCode authCode, String errorMessage) { + JWTAuthenticationResponse(AuthCode authCode, String errorMessage) { this.authCode = authCode; this.errorMessage = errorMessage; principal = null; } - - public AuthenticationResponse(AuthCode authCode, Principal principal) { + + JWTAuthenticationResponse(AuthCode authCode, Principal principal) { this.authCode = authCode; this.principal = principal; } - public AuthenticationResponse(AuthCode authCode) { + JWTAuthenticationResponse(AuthCode authCode) { this.authCode = authCode; principal = null; } - - public boolean isAuthenticated() { + + boolean isAuthenticated() { return authCode.equals(AuthCode.AUTHENTICATED); } - + public Principal getPrincipal() { return principal; } - - public String getErrorMessage() { + + String getErrorMessage() { return errorMessage; } - - public InvalidJwtException getJwtException() { + + InvalidJwtException getJwtException() { return jwtException; } - - public AuthCode getAuthCode() { + + AuthCode getAuthCode() { return authCode; } } /** - * Principal object that carries JWT token and claims for authenticated user. + * Config object for a OpenId Connect well-known config + * Typically exposed through /.well-known/openid-configuration endpoint */ - public static class JWTPrincipal implements Principal, Serializable { - private static final long serialVersionUID = 4144666467522831388L; - final String username; - String token; - Map claims; + public static class OidcDiscoveryConfig { + private static Map securityConf; - /** - * User principal with user name as well as one or more roles that he/she belong to - * @param username string with user name for user - * @param token compact string representation of JWT token - * @param claims list of verified JWT claims as a map - */ - public JWTPrincipal(final String username, String token, Map claims) { - super(); - Args.notNull(username, "User name"); - Args.notNull(token, "JWT token"); - Args.notNull(claims, "JWT claims"); - this.token = token; - this.claims = claims; - this.username = username; + OidcDiscoveryConfig(Map securityConf) { + OidcDiscoveryConfig.securityConf = securityConf; } - @Override - public String getName() { - return this.username; - } - - public String getToken() { - return token; - } - - public Map getClaims() { - return claims; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - JWTPrincipal that = (JWTPrincipal) o; - return Objects.equals(username, that.username) && - Objects.equals(token, that.token) && - Objects.equals(claims, that.claims); - } - - @Override - public int hashCode() { - return Objects.hash(username, token, claims); + public static OidcDiscoveryConfig parse(String urlString) { + try { + URL url = new URL(urlString); + if (!Arrays.asList("https", "file").contains(url.getProtocol())) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Well-known config URL must be HTTPS or file"); + } + return parse(url.openStream()); + } catch (MalformedURLException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Well-known config URL " + urlString + " is malformed", e); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Well-known config could not be read from url " + urlString, e); + } } - - @Override - public String toString() { - return "JWTPrincipal{" + - "username='" + username + '\'' + - ", token='" + token + '\'' + - ", claims=" + claims + - '}'; + + public static OidcDiscoveryConfig parse(String json, Charset charset) { + return parse(new ByteArrayInputStream(json.getBytes(charset))); } - } - /** - * JWT principal that contains username, token, claims and a list of roles the user has, - * so one can keep track of user-role mappings in an Identity Server external to Solr and - * pass the information to Solr in a signed JWT token. The role information can then be used to authorize - * requests without the need to maintain or lookup what roles each user belongs to.

- */ - public static class JWTPrincipalWithUserRoles extends JWTPrincipal implements VerifiedUserRoles { - private final Set roles; - - public JWTPrincipalWithUserRoles(final String username, String token, Map claims, Set roles) { - super(username, token, claims); - Args.notNull(roles, "User roles"); - this.roles = roles; + public static OidcDiscoveryConfig parse(InputStream configStream) { + securityConf = (Map) Utils.fromJSON(configStream); + return new OidcDiscoveryConfig(securityConf); } + - /** - * Gets the list of roles - */ - @Override - public Set getVerifiedRoles() { - return roles; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof JWTPrincipalWithUserRoles)) - return false; - JWTPrincipalWithUserRoles that = (JWTPrincipalWithUserRoles) o; - return super.equals(o) && roles.equals(that.roles); + public String getJwksUrl() { + return (String) securityConf.get("jwks_uri"); } - - @Override - public int hashCode() { - return Objects.hash(username, token, claims, roles); + + public String getIssuer() { + return (String) securityConf.get("issuer"); } - - @Override - public String toString() { - return "JWTPrincipalWithUserRoles{" + - "username='" + username + '\'' + - ", token='" + token + '\'' + - ", claims=" + claims + - ", roles=" + roles + - '}'; + + public String getAuthorizationEndpoint() { + return (String) securityConf.get("authorization_endpoint"); } } - // The interceptor class that adds correct header or delegates to PKI - private class PkiDelegationInterceptor implements HttpRequestInterceptor { + /** + * The interceptor class that adds correct header or delegates to PKI. + */ + public class JwtPkiDelegationInterceptor implements HttpRequestInterceptor { + private final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + @Override public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { if (context instanceof HttpClientContext) { HttpClientContext httpClientContext = (HttpClientContext) context; if (httpClientContext.getUserToken() instanceof JWTPrincipal) { JWTPrincipal jwtPrincipal = (JWTPrincipal) httpClientContext.getUserToken(); - request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.token); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.getToken()); log.debug("Set JWT header on inter-node request"); return; } } - + if (coreContainer.getPkiAuthenticationPlugin() != null) { log.debug("Inter-node request delegated from JWTAuthPlugin to PKIAuthenticationPlugin"); coreContainer.getPkiAuthenticationPlugin().setHeader(request); diff --git a/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java b/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java new file mode 100644 index 000000000000..eaa14e4a72d5 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.security; + +import java.io.Serializable; +import java.security.Principal; +import java.util.Map; +import java.util.Objects; + +import org.apache.http.util.Args; + +/** + * Principal object that carries JWT token and claims for authenticated user. + */ +public class JWTPrincipal implements Principal, Serializable { + private static final long serialVersionUID = 4144666467522831388L; + final String username; + String token; + Map claims; + + /** + * User principal with user name as well as one or more roles that he/she belong to + * @param username string with user name for user + * @param token compact string representation of JWT token + * @param claims list of verified JWT claims as a map + */ + public JWTPrincipal(final String username, String token, Map claims) { + super(); + Args.notNull(username, "User name"); + Args.notNull(token, "JWT token"); + Args.notNull(claims, "JWT claims"); + this.token = token; + this.claims = claims; + this.username = username; + } + + @Override + public String getName() { + return this.username; + } + + public String getToken() { + return token; + } + + public Map getClaims() { + return claims; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JWTPrincipal that = (JWTPrincipal) o; + return Objects.equals(username, that.username) && + Objects.equals(token, that.token) && + Objects.equals(claims, that.claims); + } + + @Override + public int hashCode() { + return Objects.hash(username, token, claims); + } + + @Override + public String toString() { + return "JWTPrincipal{" + + "username='" + username + '\'' + + ", token='" + token + '\'' + + ", claims=" + claims + + '}'; + } +} diff --git a/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java b/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java new file mode 100644 index 000000000000..473f2f4f6e51 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.security; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import org.apache.http.util.Args; + +/** + * JWT principal that contains username, token, claims and a list of roles the user has, + * so one can keep track of user-role mappings in an Identity Server external to Solr and + * pass the information to Solr in a signed JWT token. The role information can then be used to authorize + * requests without the need to maintain or lookup what roles each user belongs to.

+ */ + public class JWTPrincipalWithUserRoles extends JWTPrincipal implements VerifiedUserRoles { + private final Set roles; + + public JWTPrincipalWithUserRoles(final String username, String token, Map claims, Set roles) { + super(username, token, claims); + Args.notNull(roles, "User roles"); + this.roles = roles; + } + + /** + * Gets the list of roles + */ + @Override + public Set getVerifiedRoles() { + return roles; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof JWTPrincipalWithUserRoles)) + return false; + JWTPrincipalWithUserRoles that = (JWTPrincipalWithUserRoles) o; + return super.equals(o) && roles.equals(that.roles); + } + + @Override + public int hashCode() { + return Objects.hash(username, token, claims, roles); + } + + @Override + public String toString() { + return "JWTPrincipalWithUserRoles{" + + "username='" + username + '\'' + + ", token='" + token + '\'' + + ", claims=" + claims + + ", roles=" + roles + + '}'; + } +} diff --git a/solr/core/src/test-files/solr/security/jwt_well-known-config.json b/solr/core/src/test-files/solr/security/jwt_well-known-config.json new file mode 100644 index 000000000000..ea6d9f10bb38 --- /dev/null +++ b/solr/core/src/test-files/solr/security/jwt_well-known-config.json @@ -0,0 +1,84 @@ +{ + "issuer":"http://acmepaymentscorp", + "authorization_endpoint":"http://acmepaymentscorp/oauth/auz/authorize", + "token_endpoint":"http://acmepaymentscorp/oauth/oauth20/token", + "userinfo_endpoint":"http://acmepaymentscorp/oauth/userinfo", + "jwks_uri":"https://acmepaymentscorp/oauth/jwks", + "scopes_supported":[ + "READ", + "WRITE", + "DELETE", + "openid", + "scope", + "profile", + "email", + "address", + "phone" + ], + "response_types_supported":[ + "code", + "code id_token", + "code token", + "code id_token token", + "token", + "id_token", + "id_token token" + ], + "grant_types_supported":[ + "authorization_code", + "implicit", + "password", + "client_credentials", + "urn:ietf:params:oauth:grant-type:jwt-bearer" + ], + "subject_types_supported":[ + "public" + ], + "id_token_signing_alg_values_supported":[ + "HS256", + "HS384", + "HS512", + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512" + ], + "id_token_encryption_alg_values_supported":[ + "RSA1_5", + "RSA-OAEP", + "RSA-OAEP-256", + "A128KW", + "A192KW", + "A256KW", + "A128GCMKW", + "A192GCMKW", + "A256GCMKW", + "dir" + ], + "id_token_encryption_enc_values_supported":[ + "A128CBC-HS256", + "A192CBC-HS384", + "A256CBC-HS512", + "A128GCM", + "A192GCM", + "A256GCM" + ], + "token_endpoint_auth_methods_supported":[ + "client_secret_post", + "client_secret_basic", + "client_secret_jwt", + "private_key_jwt" + ], + "token_endpoint_auth_signing_alg_values_supported":[ + "HS256", + "RS256" + ], + "claims_parameter_supported":false, + "request_parameter_supported":false, + "request_uri_parameter_supported":false +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index 725855176d62..8e15cd36e466 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -16,7 +16,9 @@ */ package org.apache.solr.security; +import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.Principal; @@ -27,9 +29,10 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; import org.apache.solr.common.util.Utils; -import org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse; import org.jose4j.jwk.RsaJsonWebKey; import org.jose4j.jwk.RsaJwkGenerator; import org.jose4j.jws.AlgorithmIdentifiers; @@ -41,8 +44,9 @@ import org.junit.BeforeClass; import org.junit.Test; -import static org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode.AUTZ_HEADER_PROBLEM; -import static org.apache.solr.security.JWTAuthPlugin.AuthenticationResponse.AuthCode.NO_AUTZ_HEADER; +import static org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.AUTZ_HEADER_PROBLEM; +import static org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.NO_AUTZ_HEADER; +import static org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.SCOPE_MISSING; public class JWTAuthPluginTest extends SolrTestCaseJ4 { private static String testHeader; @@ -51,8 +55,9 @@ public class JWTAuthPluginTest extends SolrTestCaseJ4 { private HashMap testJwk; private static RsaJsonWebKey rsaJsonWebKey; private HashMap testConfig; + private HashMap minimalConfig; + - @BeforeClass public static void beforeAll() throws Exception { // Generate an RSA key pair, which will be used for signing and verification of the JWT, wrapped in a JWK @@ -86,6 +91,7 @@ static JwtClaims generateClaims() { claims.setIssuedAtToNow(); // when the token was issued/created (now) claims.setNotBeforeMinutesInThePast(2); // time before which the token is not yet valid (2 minutes ago) claims.setSubject("solruser"); // the subject/principal is whom the token is about + claims.setStringClaim("scope", "solr:read"); claims.setClaim("name", "Solr User"); // additional claims/attributes about the subject can be added claims.setClaim("customPrincipal", "custom"); // additional claims/attributes about the subject can be added claims.setClaim("claim1", "foo"); // additional claims/attributes about the subject can be added @@ -115,6 +121,9 @@ public void setUp() throws Exception { testConfig.put("class", "org.apache.solr.security.JWTAuthPlugin"); testConfig.put("jwk", testJwk); plugin.init(testConfig); + + minimalConfig = new HashMap<>(); + minimalConfig.put("class", "org.apache.solr.security.JWTAuthPlugin"); } @Override @@ -126,8 +135,7 @@ public void tearDown() throws Exception { @Test public void initWithoutRequired() { - HashMap authConf = new HashMap<>(); - plugin.init(authConf); + plugin.init(testConfig); assertEquals(AUTZ_HEADER_PROBLEM, plugin.authenticate("foo").getAuthCode()); } @@ -148,8 +156,8 @@ public void initFromSecurityJSONUrlJwk() throws Exception { Map authConf = (Map) securityConf.get("authentication"); plugin.init(authConf); - AuthenticationResponse resp = plugin.authenticate(testHeader); - assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); assertTrue(resp.getJwtException().getMessage().contains("Connection refused")); } @@ -182,7 +190,7 @@ public void parseJwkSet() throws Exception { @Test public void authenticateOk() { - AuthenticationResponse resp = plugin.authenticate(testHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); assertEquals("solruser", resp.getPrincipal().getName()); } @@ -191,9 +199,9 @@ public void authenticateOk() { public void authFailedMissingSubject() { testConfig.put("iss", "NA"); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(testHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertFalse(resp.isAuthenticated()); - assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); testConfig.put("iss", "IDServer"); plugin.init(testConfig); @@ -205,9 +213,9 @@ public void authFailedMissingSubject() { public void authFailedMissingAudience() { testConfig.put("aud", "NA"); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(testHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertFalse(resp.isAuthenticated()); - assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); testConfig.put("aud", "Solr"); plugin.init(testConfig); @@ -219,14 +227,14 @@ public void authFailedMissingAudience() { public void authFailedMissingPrincipal() { testConfig.put("principal_claim", "customPrincipal"); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(testHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); testConfig.put("principal_claim", "NA"); plugin.init(testConfig); resp = plugin.authenticate(testHeader); assertFalse(resp.isAuthenticated()); - assertEquals(AuthenticationResponse.AuthCode.PRINCIPAL_MISSING, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.PRINCIPAL_MISSING, resp.getAuthCode()); } @Test @@ -238,7 +246,7 @@ public void claimMatch() { shouldMatch.put("claim3", "f\\w{2}$"); testConfig.put("claims_match", shouldMatch); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(testHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); // Required claim does not exist @@ -246,13 +254,13 @@ public void claimMatch() { shouldMatch.put("claim9", "NA"); plugin.init(testConfig); resp = plugin.authenticate(testHeader); - assertEquals(AuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); // Required claim does not match regex shouldMatch.clear(); shouldMatch.put("claim1", "NA"); resp = plugin.authenticate(testHeader); - assertEquals(AuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); } @Test @@ -260,56 +268,60 @@ public void missingIssAudExp() { testConfig.put("require_exp", "false"); testConfig.put("require_sub", "false"); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(slimHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(slimHeader); assertTrue(resp.isAuthenticated()); // Missing exp header testConfig.put("require_exp", true); plugin.init(testConfig); resp = plugin.authenticate(slimHeader); - assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); // Missing sub header testConfig.put("require_sub", true); plugin.init(testConfig); resp = plugin.authenticate(slimHeader); - assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); } @Test public void algWhitelist() { testConfig.put("alg_whitelist", Arrays.asList("PS384", "PS512")); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(testHeader); - assertEquals(AuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); assertTrue(resp.getErrorMessage().contains("not a whitelisted")); } @Test - public void roles() { - testConfig.put("roles_claim", "groups"); + public void scope() { + testConfig.put("scope", "solr:read solr:admin"); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(testHeader); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); Principal principal = resp.getPrincipal(); assertTrue(principal instanceof VerifiedUserRoles); Set roles = ((VerifiedUserRoles)principal).getVerifiedRoles(); - assertEquals(3, roles.size()); - assertTrue(roles.contains("group-one")); + assertEquals(1, roles.size()); + assertTrue(roles.contains("solr:read")); + } - // Wrong claim ID - testConfig.put("roles_claim", "NA"); + @Test + public void wrongScope() { + testConfig.put("scope", "wrong"); plugin.init(testConfig); - resp = plugin.authenticate(testHeader); - assertEquals(AuthenticationResponse.AuthCode.CLAIM_MISMATCH, resp.getAuthCode()); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); + assertFalse(resp.isAuthenticated()); + assertNull(resp.getPrincipal()); + assertEquals(SCOPE_MISSING, resp.getAuthCode()); } - + @Test public void noHeaderBlockUnknown() { testConfig.put("block_unknown", true); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(null); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); assertEquals(NO_AUTZ_HEADER, resp.getAuthCode()); } @@ -317,7 +329,60 @@ public void noHeaderBlockUnknown() { public void noHeaderNotBlockUnknown() { testConfig.put("block_unknown", false); plugin.init(testConfig); - AuthenticationResponse resp = plugin.authenticate(null); - assertEquals(AuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); + } + + @Test + public void minimalConfigPassThrough() { + testConfig.put("block_unknown", false); + plugin.init(testConfig); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); + } + + @Test + public void wellKnownConfig() throws IOException { + String wellKnownUrl = TEST_PATH().resolve("security").resolve("jwt_well-known-config.json").toAbsolutePath().toUri().toString(); + testConfig.put("well_known_url", wellKnownUrl); + testConfig.remove("jwk"); + plugin.init(testConfig); + JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); + assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); + } + + @Test(expected = SolrException.class) + public void onlyOneJwkConfig() throws IOException { + testConfig.put("jwk_url", "http://127.0.0.1:45678/.well-known/config"); + plugin.init(testConfig); + } + + @Test(expected = SolrException.class) + public void wellKnownConfigNotHttps() throws IOException { + testConfig.put("well_known_url", "http://127.0.0.1:45678/.well-known/config"); + plugin.init(testConfig); + } + + @Test(expected = SolrException.class) + public void wellKnownConfigNotReachable() { + testConfig.put("well_known_url", "https://127.0.0.1:45678/.well-known/config"); + plugin.init(testConfig); + } + + @Test + public void wellKnownConfigFromInputstream() throws IOException { + Path configJson = TEST_PATH().resolve("security").resolve("jwt_well-known-config.json"); + JWTAuthPlugin.OidcDiscoveryConfig config = JWTAuthPlugin.OidcDiscoveryConfig.parse(Files.newInputStream(configJson)); + assertEquals("https://acmepaymentscorp/oauth/jwks", config.getJwksUrl()); + } + + @Test + public void wellKnownConfigFromString() throws IOException { + Path configJson = TEST_PATH().resolve("security").resolve("jwt_well-known-config.json"); + String configString = StringUtils.join(Files.readAllLines(configJson), "\n"); + JWTAuthPlugin.OidcDiscoveryConfig config = JWTAuthPlugin.OidcDiscoveryConfig.parse(configString, StandardCharsets.UTF_8); + assertEquals("https://acmepaymentscorp/oauth/jwks", config.getJwksUrl()); + assertEquals("http://acmepaymentscorp", config.getIssuer()); + assertEquals("http://acmepaymentscorp/oauth/auz/authorize", config.getAuthorizationEndpoint()); } } \ No newline at end of file diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index cb78aef3df1d..6214b3b9dc6f 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -50,7 +50,22 @@ To start enforcing authentication for all users, requiring a valid JWT token in } ---- -A more complex configuration: +The next example shows configuring using https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] with a well-konwn URI. + +[source,json] +---- +{ + "authentication": { + "class": "solr.JWTAuthPlugin", + "block_unknown": true, + "well_known_url": "https://my.key.server/.well-known/openid-configuration" + } +} +---- + +In this case, `jwk_url` and `iss` will be automatically configured from the fetched configuration. + +A more complex configuration with static JWK: [source,json] ---- @@ -64,27 +79,29 @@ A more complex configuration: "kty": "RSA", "n": "3ZF6wBGPMsLzsS1KLghxaVpZtXD3nTLzDm0c974i9-KNU_1rhhBeiVfS64VfEQmP8SA470jEy7yWcvnz9GvG-YAlm9iOwVF7jLl2awdws0ocFjdSPT3SjPQKzOeMO7G9XqNTkrvoFCn1YAi26fbhhcqkwZDoeTcHQdRN32frzccuPhZrwImApIedroKLlKWv2IvPDnz2Bpe2WWVc2HdoWYqEVD3p_BEy8f-RTSHK3_8kDDF9yAwI9jx7CK1_C-eYxXltm-6rpS5NGyFm0UNTZMxVU28Tl7LX8Vb6CikyCQ9YRCtk_CvpKWmEuKEp9I28KHQNmGkDYT90nt3vjbCXxw" }, - "iss": "acme", <4> - "aud": "solr", <5> - "principal_claim": "solruid", <6> - "claims_match": { "roles" : "admin|search", "dept" : "IT" }, <7> - "roles_claim": "roles", <8> - "alg_whitelist" : [ "RS256", "RS384", "RS512" ] <9> + "client_id": "solr-client-12345", <4> + "iss": "https://example.com/idp", <5> + "aud": "https://example.com/solr", <6> + "principal_claim": "solruid", <7> + "claims_match": { "roles" : "admin|search", "dept" : "IT" }, <8> + "scope": "solr:read solr:write solr:admin", <9> + "alg_whitelist" : [ "RS256", "RS384", "RS512" ] <10> } } ---- -Let's comment on the config: +Let's comment on this config: <1> Plugin class <2> Make sure to block anyone without a valid token <3> Here we pass the JWK inline instead of referring to a URL with `jwk_url` -<4> The issuer claim must match "acme" -<5> The audience claim must match "solr" -<6> Fetch the user id from another claim than the default `sub` -<7> Require that the `roles` claim is one of "admin" or "search" and that the `dept` claim is "IT" -<8> Fetch user's roles from the "roles" claim and pass this on the request for use in Authorization -<9> Only accept RSA algorithms for signatures +<4> Set the client id registered with Identiy Provider +<5> The issuer claim must match "https://example.com/idp" +<6> The audience claim must match "https://example.com/solr" +<7> Fetch the user id from another claim than the default `sub` +<8> Require that the `roles` claim is one of "admin" or "search" and that the `dept` claim is "IT" +<9> Require one of the scopes `solr:read`, `solr:write` or `solr:amin` +<10> Only accept RSA algorithms for signatures == Configuration parameters @@ -92,17 +109,19 @@ Let's comment on the config: |=== Key ; Description ; Default block_unknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` -jwk_url ; An https URL to a https://mkjwk.org[JWK] keys file ; -jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. See https://mkjwk.org for an online JWK generator ; -iss ; Validates that `iss` (issuer) equals this string ; (`iss` not required) -aud ; Validates that `aud` (audience) equals this string ; (`aud` not required) +well_known_url ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; +client_id ; Client identifier for use with OpenID Connect ; +scope ; Whitespace separate list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; +jwk_url ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided +jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided +iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. +aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it require_sub ; Makes `sub` (subject) mandatory ; `true` require_exp ; Makes `exp` (expiry time) mandatory ; `true` -alg_whitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; (any) -jwk_cache_dur ; Duration of JWK cache in seconds ; 3600 (1h) +alg_whitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; +jwk_cache_dur ; Duration of JWK cache in seconds ; `3600` (1 hour) principal_claim ; What claim id to pull principal from ; `sub` -roles_claim ; What claim id to pull roles from. If specified, all requests MUST supply this claim ; -claims_match ; JSON object of claims (key) that must match a regular expression (value). ; (none) +claims_match ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "roles" : "A|B" }` will require the `roles` claim to be either "A" or "B". ; (none) |=== From 3e72685339728058d9d2a6b3e078e67014f2535c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 5 Sep 2018 15:07:35 +0200 Subject: [PATCH 14/88] Fix some precommit stuff --- .../apache/solr/security/JWTAuthPlugin.java | 6 ++-- .../JWTAuthPluginIntegrationTest.java | 32 +++++++++---------- .../src/jwt-authentication-plugin.adoc | 4 +-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 20a0de159319..38f6fde9fdcb 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -371,10 +371,10 @@ protected JWTAuthenticationResponse authenticate(String authorizationHeader) { return new JWTAuthenticationResponse(AuthCode.SCOPE_MISSING, "'scope' claim does not contain any of the required scopes: " + requiredScopes); } } - final Set roles = new HashSet<>(scopes); - roles.remove("openid"); // Remove standard claims + final Set finalScopes = new HashSet<>(scopes); + finalScopes.remove("openid"); // Remove standard scope // Pass scopes with principal to signal to any Authorization plugins that user has some verified role claims - return new JWTAuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipalWithUserRoles(principal, jwtCompact, jwtClaims.getClaimsMap(), roles)); + return new JWTAuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipalWithUserRoles(principal, jwtCompact, jwtClaims.getClaimsMap(), finalScopes)); } else { return new JWTAuthenticationResponse(AuthCode.AUTHENTICATED, new JWTPrincipal(principal, jwtCompact, jwtClaims.getClaimsMap())); } diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 63c24f70c004..5b106ead669a 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -75,7 +75,7 @@ public static void setupClass() throws Exception { " \"alg\": \"RS256\",\n" + " \"n\": \"jeyrvOaZrmKWjyNXt0myAc_pJ1hNt3aRupExJEx1ewPaL9J9HFgSCjMrYxCB1ETO1NDyZ3nSgjZis-jHHDqBxBjRdq_t1E2rkGFaYbxAyKt220Pwgme_SFTB9MXVrFQGkKyjmQeVmOmV6zM3KK8uMdKQJ4aoKmwBcF5Zg7EZdDcKOFgpgva1Jq-FlEsaJ2xrYDYo3KnGcOHIt9_0NQeLsqZbeWYLxYni7uROFncXYV5FhSJCeR4A_rrbwlaCydGxE0ToC_9HNYibUHlkJjqyUhAgORCbNS8JLCJH8NUi5sDdIawK9GTSyvsJXZ-QHqo4cMUuxWV5AJtaRGghuMUfqQ\"\n" + "}"; - + PublicJsonWebKey jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); JwtClaims claims = JWTAuthPluginTest.generateClaims(); JsonWebSignature jws = new JsonWebSignature(); @@ -84,7 +84,7 @@ public static void setupClass() throws Exception { jws.setKeyIdHeaderValue(jwk.getKeyId()); jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); - jwtTestToken = jws.getCompactSerialization(); + jwtTestToken = jws.getCompactSerialization(); HttpClientUtil.removeRequestInterceptor(interceptor); HttpClientUtil.addRequestInterceptor(interceptor); @@ -101,7 +101,7 @@ public void before() { jwtInterceptCount.set(0); pkiInterceptCount.set(0); } - + @Test(expected = IOException.class) public void infoRequestWithoutToken() throws Exception { get(baseUrl + "admin/info/system", null); @@ -120,7 +120,7 @@ public void createCollectionAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); verifyInterRequestHeaderCounts(0,0); - + // First a non distributed query Pair result = get(baseUrl + "mycoll/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); @@ -130,7 +130,7 @@ public void createCollectionAndQueryDistributed() throws Exception { result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(2,2); - + // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); verifyInterRequestHeaderCounts(2,2); @@ -140,7 +140,7 @@ public void createCollectionAndQueryDistributed() throws Exception { public void createCollectionAndUpdateDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); - + // Now update two documents Pair result = post(baseUrl + "mycoll/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); @@ -186,11 +186,11 @@ public void createCollectionAndUpdateDistributed() throws Exception { // result = get(baseUrl + "mycoll/query?q=*:*"); // System.out.println(result.first()); // assertEquals(Integer.valueOf(200), result.second()); - - + + // String collectionName = "jwtAuthTestColl"; - // create collection + // create collection // CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", // NUM_SHARDS, REPLICATION_FACTOR); // create.process(solrClient); @@ -224,7 +224,7 @@ private Pair get(String url, String token) throws IOException { createConn.connect(); BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent())); String result = br2.lines().collect(Collectors.joining("\n")); - int code = createConn.getResponseCode(); + int code = createConn.getResponseCode(); createConn.disconnect(); return new Pair<>(result, code); } @@ -242,15 +242,15 @@ private Pair post(String url, String json, String token) throws os.write(json.getBytes()); os.flush(); os.close(); - + con.connect(); BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent())); String result = br2.lines().collect(Collectors.joining("\n")); - int code = con.getResponseCode(); + int code = con.getResponseCode(); con.disconnect(); return new Pair<>(result, code); } - + // SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder, String token) { // if (builder == null) { // builder = SolrHttpClientBuilder.create(); @@ -268,17 +268,17 @@ private Pair post(String url, String json, String token) throws // }); // return builder; // } - + private static class CountInterceptor implements HttpRequestInterceptor { @Override public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { Header ah = request.getFirstHeader(HttpHeaders.AUTHORIZATION); if (ah != null && ah.getValue().startsWith("Bearer")) jwtInterceptCount.addAndGet(1); - + Header ph = request.getFirstHeader(PKIAuthenticationPlugin.HEADER); if (ph != null) pkiInterceptCount.addAndGet(1); - } + } } } diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 6214b3b9dc6f..6331676d118b 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -58,7 +58,7 @@ The next example shows configuring using https://openid.net/specs/openid-connect "authentication": { "class": "solr.JWTAuthPlugin", "block_unknown": true, - "well_known_url": "https://my.key.server/.well-known/openid-configuration" + "well_known_url": "https://idp.example.com/.well-known/openid-configuration" } } ---- @@ -121,7 +121,7 @@ require_exp ; Makes `exp` (expiry time) mandatory ; `tr alg_whitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; jwk_cache_dur ; Duration of JWK cache in seconds ; `3600` (1 hour) principal_claim ; What claim id to pull principal from ; `sub` -claims_match ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "roles" : "A|B" }` will require the `roles` claim to be either "A" or "B". ; (none) +claims_match ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) |=== From f24ada64b14e2277ea175ff938fc50b62b02c6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 5 Sep 2018 16:18:58 +0200 Subject: [PATCH 15/88] Fix javadoc formatting --- .../solr/security/JWTPrincipalWithUserRoles.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java b/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java index 473f2f4f6e51..ecad497929c9 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java +++ b/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java @@ -27,9 +27,9 @@ * JWT principal that contains username, token, claims and a list of roles the user has, * so one can keep track of user-role mappings in an Identity Server external to Solr and * pass the information to Solr in a signed JWT token. The role information can then be used to authorize - * requests without the need to maintain or lookup what roles each user belongs to.

- */ - public class JWTPrincipalWithUserRoles extends JWTPrincipal implements VerifiedUserRoles { + * requests without the need to maintain or lookup what roles each user belongs to. + */ +public class JWTPrincipalWithUserRoles extends JWTPrincipal implements VerifiedUserRoles { private final Set roles; public JWTPrincipalWithUserRoles(final String username, String token, Map claims, Set roles) { @@ -37,7 +37,7 @@ public JWTPrincipalWithUserRoles(final String username, String token, Map Date: Wed, 5 Sep 2018 19:44:32 +0200 Subject: [PATCH 16/88] More getters with tests --- .../apache/solr/security/JWTAuthPlugin.java | 34 ++++++++++++++----- .../solr/security/JWTAuthPluginTest.java | 6 ++-- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 38f6fde9fdcb..0371e9ab1be8 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -112,7 +112,7 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private HashSet requiredScopes = new HashSet<>(); private String clientId; private long jwkCacheDuration; - private OidcDiscoveryConfig oidcDiscoveryConfig; + private WellKnownDiscoveryConfig oidcDiscoveryConfig; private String confIdpConfigUrl; private Map pluginConfig; private Instant lastInitTime = Instant.now(); @@ -137,7 +137,7 @@ public void init(Map pluginConfig) { if (confIdpConfigUrl != null) { log.debug("Initializing well-known oidc config from {}", confIdpConfigUrl); - oidcDiscoveryConfig = OidcDiscoveryConfig.parse(confIdpConfigUrl); + oidcDiscoveryConfig = WellKnownDiscoveryConfig.parse(confIdpConfigUrl); iss = oidcDiscoveryConfig.getIssuer(); } @@ -549,14 +549,14 @@ AuthCode getAuthCode() { * Config object for a OpenId Connect well-known config * Typically exposed through /.well-known/openid-configuration endpoint */ - public static class OidcDiscoveryConfig { + public static class WellKnownDiscoveryConfig { private static Map securityConf; - OidcDiscoveryConfig(Map securityConf) { - OidcDiscoveryConfig.securityConf = securityConf; + WellKnownDiscoveryConfig(Map securityConf) { + WellKnownDiscoveryConfig.securityConf = securityConf; } - public static OidcDiscoveryConfig parse(String urlString) { + public static WellKnownDiscoveryConfig parse(String urlString) { try { URL url = new URL(urlString); if (!Arrays.asList("https", "file").contains(url.getProtocol())) { @@ -570,13 +570,13 @@ public static OidcDiscoveryConfig parse(String urlString) { } } - public static OidcDiscoveryConfig parse(String json, Charset charset) { + public static WellKnownDiscoveryConfig parse(String json, Charset charset) { return parse(new ByteArrayInputStream(json.getBytes(charset))); } - public static OidcDiscoveryConfig parse(InputStream configStream) { + public static WellKnownDiscoveryConfig parse(InputStream configStream) { securityConf = (Map) Utils.fromJSON(configStream); - return new OidcDiscoveryConfig(securityConf); + return new WellKnownDiscoveryConfig(securityConf); } @@ -591,6 +591,22 @@ public String getIssuer() { public String getAuthorizationEndpoint() { return (String) securityConf.get("authorization_endpoint"); } + + public String getUserInfoEndpoint() { + return (String) securityConf.get("userinfo_endpoint"); + } + + public String getTokenEndpoint() { + return (String) securityConf.get("token_endpoint"); + } + + public List getScopesSupported() { + return (List) securityConf.get("scopes_supported"); + } + + public List getResponseTypesSupported() { + return (List) securityConf.get("response_types_supported"); + } } /** diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index 8e15cd36e466..f8287531fc2f 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -372,7 +372,7 @@ public void wellKnownConfigNotReachable() { @Test public void wellKnownConfigFromInputstream() throws IOException { Path configJson = TEST_PATH().resolve("security").resolve("jwt_well-known-config.json"); - JWTAuthPlugin.OidcDiscoveryConfig config = JWTAuthPlugin.OidcDiscoveryConfig.parse(Files.newInputStream(configJson)); + JWTAuthPlugin.WellKnownDiscoveryConfig config = JWTAuthPlugin.WellKnownDiscoveryConfig.parse(Files.newInputStream(configJson)); assertEquals("https://acmepaymentscorp/oauth/jwks", config.getJwksUrl()); } @@ -380,9 +380,11 @@ public void wellKnownConfigFromInputstream() throws IOException { public void wellKnownConfigFromString() throws IOException { Path configJson = TEST_PATH().resolve("security").resolve("jwt_well-known-config.json"); String configString = StringUtils.join(Files.readAllLines(configJson), "\n"); - JWTAuthPlugin.OidcDiscoveryConfig config = JWTAuthPlugin.OidcDiscoveryConfig.parse(configString, StandardCharsets.UTF_8); + JWTAuthPlugin.WellKnownDiscoveryConfig config = JWTAuthPlugin.WellKnownDiscoveryConfig.parse(configString, StandardCharsets.UTF_8); assertEquals("https://acmepaymentscorp/oauth/jwks", config.getJwksUrl()); assertEquals("http://acmepaymentscorp", config.getIssuer()); assertEquals("http://acmepaymentscorp/oauth/auz/authorize", config.getAuthorizationEndpoint()); + assertEquals(Arrays.asList("READ", "WRITE", "DELETE", "openid", "scope", "profile", "email", "address", "phone"), config.getScopesSupported()); + assertEquals(Arrays.asList("code", "code id_token", "code token", "code id_token token", "token", "id_token", "id_token token"), config.getResponseTypesSupported()); } } \ No newline at end of file From 2ed173f0ffc5971db249be269f0989c12ebfbd98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 5 Sep 2018 20:43:33 +0200 Subject: [PATCH 17/88] Cleanup integration test --- .../JWTAuthPluginIntegrationTest.java | 122 +++--------------- 1 file changed, 18 insertions(+), 104 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 5b106ead669a..d2f35babc138 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -36,7 +36,6 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.util.Pair; -import org.apache.solr.util.LogLevel; import org.jose4j.jwk.PublicJsonWebKey; import org.jose4j.jwk.RsaJsonWebKey; import org.jose4j.jws.AlgorithmIdentifiers; @@ -47,6 +46,10 @@ import org.junit.BeforeClass; import org.junit.Test; +/** + * Validate that JWT token authentication works in a real cluster. + * TODO: Test also using SolrJ as client. But that requires a way to set Authorization header on request + */ public class JWTAuthPluginIntegrationTest extends SolrCloudTestCase { protected static final int NUM_SERVERS = 2; protected static final int NUM_SHARDS = 2; @@ -88,6 +91,8 @@ public static void setupClass() throws Exception { HttpClientUtil.removeRequestInterceptor(interceptor); HttpClientUtil.addRequestInterceptor(interceptor); + + cluster.waitForAllNodes(10); } @AfterClass @@ -97,7 +102,7 @@ public static void tearDownClass() throws Exception { } @Before - public void before() { + public void before() throws IOException, InterruptedException { jwtInterceptCount.set(0); pkiInterceptCount.set(0); } @@ -115,102 +120,30 @@ public void infoRequestWithToken() throws IOException { } @Test - @LogLevel("org.apache.solr.security=DEBUG") - public void createCollectionAndQueryDistributed() throws Exception { + public void createCollectionUpdateAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); - verifyInterRequestHeaderCounts(0,0); - - // First a non distributed query - Pair result = get(baseUrl + "mycoll/query?q=*:*&distrib=false", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(0,0); - // Now do a distributed query, using JWTAUth for inter-node - result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); + // Now update two documents + Pair result = post(baseUrl + "mycoll/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(2,2); - // Delete - assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); + // First a non distributed query + result = get(baseUrl + "mycoll/query?q=*:*&distrib=false", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(2,2); - } - - @Test - public void createCollectionAndUpdateDistributed() throws Exception { - // Admin request will use PKI inter-node auth from Overseer, and succeed - assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); - // Now update two documents - Pair result = post(baseUrl + "mycoll/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); + // Now do a distributed query, using JWTAUth for inter-node + result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - + verifyInterRequestHeaderCounts(6,6); + // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); - verifyInterRequestHeaderCounts(2,2); + verifyInterRequestHeaderCounts(6,6); } - - -// NOCOMMIT: Test using SolrJ as client -// private void testCollectionCreateSearchDelete(boolean enableJwt) throws Exception { -// if (enableJwt) { -// HttpClientUtil.setHttpClientBuilder(getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder(), testJwt)); -// } else { -// HttpClientUtil.resetHttpClientBuilder(); -// } - -// ClusterStateProvider csProv = cluster.getSolrClient().getClusterStateProvider(); -// CloudSolrClientBuilder builder = new CloudSolrClientBuilder(csProv); -// CloudSolrClient solrClient = builder.build(); - -// CloudSolrClient solrClient = cluster.getSolrClient(); - -// URL solrurl = new URL(baseUrl + "admin/info/system"); -// HttpURLConnection conn = (HttpURLConnection) solrurl.openConnection(); -// conn.setRequestProperty("Authorization", "Bearer " + testJwt); -// conn.connect(); -// BufferedReader br = new BufferedReader(new InputStreamReader((InputStream) conn.getContent())); -// br.lines().forEach(l -> { -// System.out.println(l); -// }); -// conn.disconnect(); - - -// Pair result; -// -// result = get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2"); -// System.out.println(result.first()); -// assertEquals(Integer.valueOf(200), result.second()); -// -// result = get(baseUrl + "mycoll/query?q=*:*"); -// System.out.println(result.first()); -// assertEquals(Integer.valueOf(200), result.second()); - - -// String collectionName = "jwtAuthTestColl"; - - // create collection -// CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", -// NUM_SHARDS, REPLICATION_FACTOR); -// create.process(solrClient); -// -// solrClient.add(collectionName, new SolrInputDocument("id", "1")); -// solrClient.add(collectionName, new SolrInputDocument("id", "2")); -// solrClient.commit(collectionName); -// -// SolrQuery query = new SolrQuery(); -// query.setQuery("*:*"); -// QueryResponse rsp = solrClient.query(collectionName, query); -// assertEquals(2, rsp.getResults().getNumFound()); -// -// CollectionAdminRequest.Delete deleteReq = CollectionAdminRequest.deleteCollection(collectionName); -// deleteReq.process(solrClient); -// AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, -// solrClient.getZkStateReader(), true, true, 330); -// solrClient.close(); -// } - private void verifyInterRequestHeaderCounts(int jwt, int pki) { assertEquals(jwt, jwtInterceptCount.get()); assertEquals(pki, jwtInterceptCount.get()); @@ -221,7 +154,6 @@ private Pair get(String url, String token) throws IOException { HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); if (token != null) createConn.setRequestProperty("Authorization", "Bearer " + token); - createConn.connect(); BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent())); String result = br2.lines().collect(Collectors.joining("\n")); int code = createConn.getResponseCode(); @@ -251,24 +183,6 @@ private Pair post(String url, String json, String token) throws return new Pair<>(result, code); } -// SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder, String token) { -// if (builder == null) { -// builder = SolrHttpClientBuilder.create(); -// } -// builder.setAuthSchemeRegistryProvider(() -> { -// Lookup authProviders = RegistryBuilder.create() -// .register("Bearer", new JWTAuthPlugin.JwtBearerAuthschemeProvider()) -// .build(); -// return authProviders; -// }); -// builder.setDefaultCredentialsProvider(() -> { -// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); -// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(token, null)); -// return credentialsProvider; -// }); -// return builder; -// } - private static class CountInterceptor implements HttpRequestInterceptor { @Override public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { From 47f591ff5176db97dbe2d2e2e1c59946bcc8127a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 5 Sep 2018 23:01:55 +0200 Subject: [PATCH 18/88] Convert to camel case config keys Fail if unknown config keys are present Enable editing through API Update refguide Document introspection API --- .../apache/solr/security/JWTAuthPlugin.java | 81 ++++++++++++++----- .../security/jwt_plugin_jwk_security.json | 2 +- .../security/jwt_plugin_jwk_url_security.json | 2 +- .../solr/security/JWTAuthPluginTest.java | 32 ++++---- .../src/jwt-authentication-plugin.adoc | 81 ++++++++++++------- .../cluster.security.JwtAuth.Commands.json | 18 +++++ 6 files changed, 152 insertions(+), 64 deletions(-) create mode 100644 solr/solrj/src/resources/apispec/cluster.security.JwtAuth.Commands.json diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 0371e9ab1be8..0c277aec9b79 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -41,7 +41,9 @@ import java.util.Set; import java.util.StringTokenizer; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import com.google.common.collect.ImmutableSet; import org.apache.http.HttpException; import org.apache.http.HttpHeaders; import org.apache.http.HttpRequest; @@ -53,6 +55,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.StringUtils; +import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.core.CoreContainer; @@ -76,27 +79,32 @@ /** * Authenticaion plugin that finds logged in user by validating the signature of a JWT token */ -public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin, SpecProvider { +public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin, SpecProvider, ConfigEditablePlugin { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String PARAM_BLOCK_UNKNOWN = "block_unknown"; - private static final String PARAM_JWK_URL = "jwk_url"; + private static final String PARAM_BLOCK_UNKNOWN = "blockUnknown"; + private static final String PARAM_JWK_URL = "jwkUrl"; private static final String PARAM_JWK = "jwk"; private static final String PARAM_ISSUER = "iss"; private static final String PARAM_AUDIENCE = "aud"; - private static final String PARAM_REQUIRE_SUBJECT = "require_sub"; - private static final String PARAM_PRINCIPAL_CLAIM = "principal_claim"; - private static final String PARAM_REQUIRE_EXPIRATIONTIME = "require_exp"; - private static final String PARAM_ALG_WHITELIST = "alg_whitelist"; - private static final String PARAM_JWK_CACHE_DURATION = "jwk_cache_dur"; - private static final String PARAM_CLAIMS_MATCH = "claims_match"; + private static final String PARAM_REQUIRE_SUBJECT = "requireSub"; + private static final String PARAM_PRINCIPAL_CLAIM = "principalClaim"; + private static final String PARAM_REQUIRE_EXPIRATIONTIME = "requireExp"; + private static final String PARAM_ALG_WHITELIST = "algWhitelist"; + private static final String PARAM_JWK_CACHE_DURATION = "jwkCacheDur"; + private static final String PARAM_CLAIMS_MATCH = "claimsMatch"; private static final String PARAM_SCOPE = "scope"; - private static final String PARAM_ADMIN_SCOPE = "admin_scope"; - private static final String PARAM_CLIENT_ID = "client_id"; - private static final String PARAM_WELL_KNOWN_URL = "well_known_url"; + private static final String PARAM_ADMIN_SCOPE = "adminScope"; + private static final String PARAM_CLIENT_ID = "clientId"; + private static final String PARAM_WELL_KNOWN_URL = "wellKnownUrl"; + private static final String AUTH_REALM = "solr-jwt"; private static final String CLAIM_SCOPE = "scope"; private final JwtPkiDelegationInterceptor interceptor = new JwtPkiDelegationInterceptor(); + static final Set supported_ops = ImmutableSet.of("set-user", "delete-user"); + private static final Set PROPS = ImmutableSet.of(PARAM_BLOCK_UNKNOWN, PARAM_JWK_URL, PARAM_JWK, PARAM_ISSUER, + PARAM_AUDIENCE, PARAM_REQUIRE_SUBJECT, PARAM_PRINCIPAL_CLAIM, PARAM_REQUIRE_EXPIRATIONTIME, PARAM_ALG_WHITELIST, + PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_ADMIN_SCOPE, PARAM_CLIENT_ID, PARAM_WELL_KNOWN_URL); private JwtConsumer jwtConsumer; private String iss; @@ -118,6 +126,7 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private Instant lastInitTime = Instant.now(); private CoreContainer coreContainer; + /** * Initialize plugin with core container, this method is chosen by reflection at create time * @param coreContainer instance of core container @@ -128,6 +137,12 @@ public JWTAuthPlugin(CoreContainer coreContainer) { @Override public void init(Map pluginConfig) { + List unknownKeys = pluginConfig.keySet().stream().filter(k -> !PROPS.contains(k)).collect(Collectors.toList()); + unknownKeys.remove("class"); + if (!unknownKeys.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid JwtAuth configuration parameter " + unknownKeys); + } + blockUnknown = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_BLOCK_UNKNOWN, false))); clientId = (String) pluginConfig.get(PARAM_CLIENT_ID); requireSubject = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_SUBJECT, "true"))); @@ -204,7 +219,7 @@ private void initJwk(Map pluginConfig) { JsonWebKeySet jwks = parseJwkSet(confJwk); verificationKeyResolver = new JwksVerificationKeyResolver(jwks.getJsonWebKeys()); } catch (JoseException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid JWTAuthPlugin configuration, jwk field parse error", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid JWTAuthPlugin configuration, " + PARAM_JWK + " parse error", e); } } initConsumer(); @@ -216,10 +231,10 @@ private void setupJwkUrl(String url) { try { URL jwkUrl = new URL(url); if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be an HTTPS url"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be an HTTPS url"); } } catch (MalformedURLException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jwk_url must be a valid URL"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be a valid URL"); } HttpsJwks httpsJkws = new HttpsJwks(url); httpsJkws.setDefaultCacheDuration(jwkCacheDuration); @@ -251,7 +266,7 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser if (jwtConsumer == null) { if (header == null && !blockUnknown) { - log.info("JWTAuth not configured, but allowing anonymous access since blockUnknown==false"); + log.info("JWTAuth not configured, but allowing anonymous access since {}==false", PARAM_BLOCK_UNKNOWN); filterChain.doFilter(request, response); return true; } @@ -285,7 +300,7 @@ public Principal getUserPrincipal() { case PASS_THROUGH: if (log.isDebugEnabled()) - log.debug("Unknown user, but allow due to block_unknown=false"); + log.debug("Unknown user, but allow due to {}=false", PARAM_BLOCK_UNKNOWN); filterChain.doFilter(request, response); return true; @@ -368,7 +383,7 @@ protected JWTAuthenticationResponse authenticate(String authorizationHeader) { // Validate that at least one of the required scopes are present in the scope claim if (!requiredScopes.isEmpty()) { if (scopes.stream().noneMatch(requiredScopes::contains)) { - return new JWTAuthenticationResponse(AuthCode.SCOPE_MISSING, "'scope' claim does not contain any of the required scopes: " + requiredScopes); + return new JWTAuthenticationResponse(AuthCode.SCOPE_MISSING, "Claim " + CLAIM_SCOPE + " does not contain any of the required scopes: " + requiredScopes); } } final Set finalScopes = new HashSet<>(scopes); @@ -399,7 +414,7 @@ protected JWTAuthenticationResponse authenticate(String authorizationHeader) { if (blockUnknown) { return new JWTAuthenticationResponse(AuthCode.NO_AUTZ_HEADER, "Missing Authorization header"); } else { - log.debug("No user authenticated, but block_unknown=false, so letting request through"); + log.debug("No user authenticated, but blockUnknown=false, so letting request through"); return new JWTAuthenticationResponse(AuthCode.PASS_THROUGH); } } @@ -455,6 +470,34 @@ private void authenticationFailure(HttpServletResponse response, String message) authenticationFailure(response, message, HttpServletResponse.SC_UNAUTHORIZED, null); } + /** + * Operate the commands on the latest conf and return a new conf object + * If there are errors in the commands , throw a SolrException. return a null + * if no changes are to be made as a result of this edit. It is the responsibility + * of the implementation to ensure that the returned config is valid . The framework + * does no validation of the data + * + * @param latestConf latest version of config + * @param commands the list of command operations to perform + */ + @Override + public Map edit(Map latestConf, List commands) { + for (CommandOperation command : commands) { + if (command.name.equals("set-property")) { + for (Map.Entry e : command.getDataMap().entrySet()) { + if (PROPS.contains(e.getKey())) { + latestConf.put(e.getKey(), e.getValue()); + return latestConf; + } else { + command.addError("Unknown property " + e.getKey()); + } + } + } + } + if (!CommandOperation.captureErrors(commands).isEmpty()) return null; + return latestConf; + } + private enum BearerWwwAuthErrorCode { invalid_request, invalid_token, insufficient_scope}; private void authenticationFailure(HttpServletResponse response, String message, int httpCode, BearerWwwAuthErrorCode responseError) throws IOException { diff --git a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json index 3c249e64e9c2..7daab7ac9cbd 100644 --- a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json +++ b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_security.json @@ -1,7 +1,7 @@ { "authentication": { "class": "solr.JWTAuthPlugin", - "block_unknown": true, + "blockUnknown": true, "jwk": { "kty": "RSA", "e": "AQAB", diff --git a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json index 0431ab86335b..74b86ef03c14 100644 --- a/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json +++ b/solr/core/src/test-files/solr/security/jwt_plugin_jwk_url_security.json @@ -1,6 +1,6 @@ { "authentication" : { "class": "solr.JWTAuthPlugin", - "jwk_url": "https://127.0.0.1:8999/this-will-fail.wks" + "jwkUrl": "https://127.0.0.1:8999/this-will-fail.wks" } } \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index f8287531fc2f..99f07687f144 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -172,7 +172,7 @@ public void initWithJwk() { @Test public void initWithJwkUrl() { HashMap authConf = new HashMap<>(); - authConf.put("jwk_url", "https://127.0.0.1:9999/foo.jwk"); + authConf.put("jwkUrl", "https://127.0.0.1:9999/foo.jwk"); plugin = new JWTAuthPlugin(null); plugin.init(authConf); } @@ -225,12 +225,12 @@ public void authFailedMissingAudience() { @Test public void authFailedMissingPrincipal() { - testConfig.put("principal_claim", "customPrincipal"); + testConfig.put("principalClaim", "customPrincipal"); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); - testConfig.put("principal_claim", "NA"); + testConfig.put("principalClaim", "NA"); plugin.init(testConfig); resp = plugin.authenticate(testHeader); assertFalse(resp.isAuthenticated()); @@ -244,7 +244,7 @@ public void claimMatch() { shouldMatch.put("claim1", "foo"); shouldMatch.put("claim2", "foo|bar"); shouldMatch.put("claim3", "f\\w{2}$"); - testConfig.put("claims_match", shouldMatch); + testConfig.put("claimsMatch", shouldMatch); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertTrue(resp.isAuthenticated()); @@ -265,20 +265,20 @@ public void claimMatch() { @Test public void missingIssAudExp() { - testConfig.put("require_exp", "false"); - testConfig.put("require_sub", "false"); + testConfig.put("requireExp", "false"); + testConfig.put("requireSub", "false"); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(slimHeader); assertTrue(resp.isAuthenticated()); // Missing exp header - testConfig.put("require_exp", true); + testConfig.put("requireExp", true); plugin.init(testConfig); resp = plugin.authenticate(slimHeader); assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); // Missing sub header - testConfig.put("require_sub", true); + testConfig.put("requireSub", true); plugin.init(testConfig); resp = plugin.authenticate(slimHeader); assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); @@ -286,7 +286,7 @@ public void missingIssAudExp() { @Test public void algWhitelist() { - testConfig.put("alg_whitelist", Arrays.asList("PS384", "PS512")); + testConfig.put("algWhitelist", Arrays.asList("PS384", "PS512")); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); @@ -319,7 +319,7 @@ public void wrongScope() { @Test public void noHeaderBlockUnknown() { - testConfig.put("block_unknown", true); + testConfig.put("blockUnknown", true); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); assertEquals(NO_AUTZ_HEADER, resp.getAuthCode()); @@ -327,7 +327,7 @@ public void noHeaderBlockUnknown() { @Test public void noHeaderNotBlockUnknown() { - testConfig.put("block_unknown", false); + testConfig.put("blockUnknown", false); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); @@ -335,7 +335,7 @@ public void noHeaderNotBlockUnknown() { @Test public void minimalConfigPassThrough() { - testConfig.put("block_unknown", false); + testConfig.put("blockUnknown", false); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); assertEquals(JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.PASS_THROUGH, resp.getAuthCode()); @@ -344,7 +344,7 @@ public void minimalConfigPassThrough() { @Test public void wellKnownConfig() throws IOException { String wellKnownUrl = TEST_PATH().resolve("security").resolve("jwt_well-known-config.json").toAbsolutePath().toUri().toString(); - testConfig.put("well_known_url", wellKnownUrl); + testConfig.put("wellKnownUrl", wellKnownUrl); testConfig.remove("jwk"); plugin.init(testConfig); JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(null); @@ -353,19 +353,19 @@ public void wellKnownConfig() throws IOException { @Test(expected = SolrException.class) public void onlyOneJwkConfig() throws IOException { - testConfig.put("jwk_url", "http://127.0.0.1:45678/.well-known/config"); + testConfig.put("jwkUrl", "http://127.0.0.1:45678/.well-known/config"); plugin.init(testConfig); } @Test(expected = SolrException.class) public void wellKnownConfigNotHttps() throws IOException { - testConfig.put("well_known_url", "http://127.0.0.1:45678/.well-known/config"); + testConfig.put("wellKnownUrl", "http://127.0.0.1:45678/.well-known/config"); plugin.init(testConfig); } @Test(expected = SolrException.class) public void wellKnownConfigNotReachable() { - testConfig.put("well_known_url", "https://127.0.0.1:45678/.well-known/config"); + testConfig.put("wellKnownUrl", "https://127.0.0.1:45678/.well-known/config"); plugin.init(testConfig); } diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 39c13f87884c..041f8c4b9d7c 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -44,8 +44,8 @@ To start enforcing authentication for all users, requiring a valid JWT token in { "authentication": { "class": "solr.JWTAuthPlugin", - "block_unknown": true, - "jwk_url": "https://my.key.server/jwk.json" + "blockUnknown": true, + "jwkUrl": "https://my.key.server/jwk.json" } } ---- @@ -57,8 +57,8 @@ The next example shows configuring using https://openid.net/specs/openid-connect { "authentication": { "class": "solr.JWTAuthPlugin", - "block_unknown": true, - "well_known_url": "https://idp.example.com/.well-known/openid-configuration" + "blockUnknown": true, + "wellKnownUrl": "https://idp.example.com/.well-known/openid-configuration" } } ---- @@ -72,20 +72,20 @@ A more complex configuration with static JWK: { "authentication": { "class": "solr.JWTAuthPlugin", <1> - "block_unknown": true, <2> + "blockUnknown": true, <2> "jwk": { <3> "e": "AQAB", "kid": "k1", "kty": "RSA", "n": "3ZF6wBGPMsLzsS1KLghxaVpZtXD3nTLzDm0c974i9-KNU_1rhhBeiVfS64VfEQmP8SA470jEy7yWcvnz9GvG-YAlm9iOwVF7jLl2awdws0ocFjdSPT3SjPQKzOeMO7G9XqNTkrvoFCn1YAi26fbhhcqkwZDoeTcHQdRN32frzccuPhZrwImApIedroKLlKWv2IvPDnz2Bpe2WWVc2HdoWYqEVD3p_BEy8f-RTSHK3_8kDDF9yAwI9jx7CK1_C-eYxXltm-6rpS5NGyFm0UNTZMxVU28Tl7LX8Vb6CikyCQ9YRCtk_CvpKWmEuKEp9I28KHQNmGkDYT90nt3vjbCXxw" }, - "client_id": "solr-client-12345", <4> + "clientId": "solr-client-12345", <4> "iss": "https://example.com/idp", <5> "aud": "https://example.com/solr", <6> - "principal_claim": "solruid", <7> - "claims_match": { "roles" : "admin|search", "dept" : "IT" }, <8> + "principalClaim": "solruid", <7> + "claimsMatch": { "foo" : "A|B", "dept" : "IT" }, <8> "scope": "solr:read solr:write solr:admin", <9> - "alg_whitelist" : [ "RS256", "RS384", "RS512" ] <10> + "algWhitelist" : [ "RS256", "RS384", "RS512" ] <10> } } ---- @@ -94,12 +94,12 @@ Let's comment on this config: <1> Plugin class <2> Make sure to block anyone without a valid token -<3> Here we pass the JWK inline instead of referring to a URL with `jwk_url` +<3> Here we pass the JWK inline instead of referring to a URL with `jwkUrl` <4> Set the client id registered with Identiy Provider <5> The issuer claim must match "https://example.com/idp" <6> The audience claim must match "https://example.com/solr" <7> Fetch the user id from another claim than the default `sub` -<8> Require that the `roles` claim is one of "admin" or "search" and that the `dept` claim is "IT" +<8> Require that the `roles` claim is one of "A" or "B" and that the `dept` claim is "IT" <9> Require one of the scopes `solr:read`, `solr:write` or `solr:amin` <10> Only accept RSA algorithms for signatures @@ -108,33 +108,60 @@ Let's comment on this config: [%header,format=csv,separator=;] |=== Key ; Description ; Default -block_unknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` -well_known_url ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; -client_id ; Client identifier for use with OpenID Connect ; +blockUnknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` +wellKnownUrl ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; +clientId ; Client identifier for use with OpenID Connect ; scope ; Whitespace separate list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; -jwk_url ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided +jwkUrl ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it -require_sub ; Makes `sub` (subject) mandatory ; `true` -require_exp ; Makes `exp` (expiry time) mandatory ; `true` -alg_whitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; -jwk_cache_dur ; Duration of JWK cache in seconds ; `3600` (1 hour) -principal_claim ; What claim id to pull principal from ; `sub` -claims_match ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) +requireSub ; Makes `sub` (subject) mandatory ; `true` +requireExp ; Makes `exp` (expiry time) mandatory ; `true` +algWhitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; +jwkCacheDur ; Duration of JWK cache in seconds ; `3600` (1 hour) +principalClaim ; What claim id to pull principal from ; `sub` +claimsMatch ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) |=== == Editing Authentication Plugin Configuration -**TODO: This plugin currently does not support edit API.** +All properties mentioned above can be set or changed using the <>. You can thus start with a simple configuration with only `class` configured and then configure the rest using the API. -=== Using JWT Auth with SolrJ +=== Set a Property -To use JWT Auth with SolrJ you must configure a `SolrHttpClientBuilder` that sets the `Authorization: Bearer` header. How to obtain a valid JWT token is up to the client application to choose. +Set properties for the authentication plugin. Each of the configuration keys in the table above can be used as parameter keys for the `set-property` command. -**TODO: Describe this ** +Example: -=== Using the Solr Control Script with Basic Auth +[.dynamic-tabs] +-- +[example.tab-pane#v1set-property] +==== +[.tab-label]*V1 API* -The control scripts (`bin/solr`) do not currently support JWT Auth. \ No newline at end of file +[source,bash] +---- +curl --user solr:SolrRocks http://localhost:8983/solr/admin/authentication -H 'Content-type:application/json' -d '{"set-property": {"blockUnknown":true, "wellKnownUrl": "https://example.com/.well-knwon/openid-configuration", "scope": "solr:read solr:write"}}' +---- +==== + +[example.tab-pane#v2set-property] +==== +[.tab-label]*V2 API* + +[source,bash] +---- +curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentication -H 'Content-type:application/json' -d '{"set-property": {"blockUnknown":true, "wellKnownUrl": "https://example.com/.well-knwon/openid-configuration", "scope": "solr:read solr:write"}}' +---- +==== +-- + +== Using JWT Auth with SolrJ + +SolrJ does not currently support setting JWT token per request. This is planned for a future release. + +== Using the Solr Control Script with Basic Auth + +The control script (`bin/solr`) does not currently support JWT Auth. \ No newline at end of file diff --git a/solr/solrj/src/resources/apispec/cluster.security.JwtAuth.Commands.json b/solr/solrj/src/resources/apispec/cluster.security.JwtAuth.Commands.json new file mode 100644 index 000000000000..e940d2f34f7a --- /dev/null +++ b/solr/solrj/src/resources/apispec/cluster.security.JwtAuth.Commands.json @@ -0,0 +1,18 @@ +{ + "documentation": "https://lucene.apache.org/solr/guide/jwt-authentication-plugin.html", + "description": "Modifies the configuration of JWT token authentication.", + "methods": [ + "POST" + ], + "url": { + "paths": [ + "/cluster/security/authentication" + ] + }, + "commands": { + "set-property": { + "type":"object", + "description": "The set-property command lets you set any of the configuration parameters supported by this plugin" + } + } +} From 54b80cf9e2a227fbd8aed112178620119fcd5e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 11 Sep 2018 23:07:40 +0200 Subject: [PATCH 19/88] Refguide wiip --- .../src/jwt-authentication-plugin.adoc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 041f8c4b9d7c..f501126830ae 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -158,9 +158,19 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica ==== -- -== Using JWT Auth with SolrJ +== Using clients with JWT Auth -SolrJ does not currently support setting JWT token per request. This is planned for a future release. +=== SolrJ + +SolrJ does not currently support setting JWT tokens per request. This is planned for a future release. + +=== cURL + +To authenticate with Solr with cURL + +=== Admin UI + +The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. Because of this the Admin UI will stop working after enabling this plugin. To circumvent this you can install a browser plugin that adds HTTP headers to requests, and add a valid JWT token to the `Authorization` header by hand. == Using the Solr Control Script with Basic Auth From e4d1d50918889ade08255c1ed29ce901a73e36bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 20 Sep 2018 10:27:30 +0200 Subject: [PATCH 20/88] Make unique headers --- .../src/jwt-authentication-plugin.adoc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index f501126830ae..a9942792c10b 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -125,11 +125,11 @@ claimsMatch ; JSON object of claims (key) that must match a regular expressi |=== -== Editing Authentication Plugin Configuration +== Editing JWT Authentication Plugin Configuration All properties mentioned above can be set or changed using the <>. You can thus start with a simple configuration with only `class` configured and then configure the rest using the API. -=== Set a Property +=== Set a config Property Set properties for the authentication plugin. Each of the configuration keys in the table above can be used as parameter keys for the `set-property` command. @@ -137,7 +137,7 @@ Example: [.dynamic-tabs] -- -[example.tab-pane#v1set-property] +[example.tab-pane#jwt-v1set-property] ==== [.tab-label]*V1 API* @@ -147,7 +147,7 @@ curl --user solr:SolrRocks http://localhost:8983/solr/admin/authentication -H 'C ---- ==== -[example.tab-pane#v2set-property] +[example.tab-pane#jwt-v2set-property] ==== [.tab-label]*V2 API* @@ -160,10 +160,12 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica == Using clients with JWT Auth +[#jwt-soljr] === SolrJ SolrJ does not currently support setting JWT tokens per request. This is planned for a future release. +[#jwt-curl] === cURL To authenticate with Solr with cURL @@ -172,6 +174,6 @@ To authenticate with Solr with cURL The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. Because of this the Admin UI will stop working after enabling this plugin. To circumvent this you can install a browser plugin that adds HTTP headers to requests, and add a valid JWT token to the `Authorization` header by hand. -== Using the Solr Control Script with Basic Auth +== Using the Solr Control Script with JWT Auth The control script (`bin/solr`) does not currently support JWT Auth. \ No newline at end of file From 6014b855ffe5ab9c74a7e16a4ea65d4a80668121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 20 Sep 2018 14:45:46 +0200 Subject: [PATCH 21/88] SOLR-12786: HTML reference guide has invalid links --- solr/solr-ref-guide/src/basic-authentication-plugin.adoc | 2 +- solr/solr-ref-guide/src/common-query-parameters.adoc | 2 +- solr/solr-ref-guide/src/enabling-ssl.adoc | 2 +- solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc | 2 +- solr/solr-ref-guide/src/major-changes-in-solr-7.adoc | 2 +- solr/solr-ref-guide/src/resource-and-plugin-loading.adoc | 2 +- solr/solr-ref-guide/src/solr-control-script-reference.adoc | 4 ++-- solr/solr-ref-guide/src/solr-tutorial.adoc | 2 +- solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc | 2 +- .../src/using-zookeeper-to-manage-configuration-files.adoc | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc index 88e8a0c8d24c..971f59eb3bbb 100644 --- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc @@ -65,7 +65,7 @@ If you are using SolrCloud, you must upload `security.json` to ZooKeeper. You ca bin/solr zk cp file:path_to_local_security.json zk:/security.json -z localhost:9983 ---- -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. === Caveats diff --git a/solr/solr-ref-guide/src/common-query-parameters.adoc b/solr/solr-ref-guide/src/common-query-parameters.adoc index 499401525c0e..5df3ca3e3c25 100644 --- a/solr/solr-ref-guide/src/common-query-parameters.adoc +++ b/solr/solr-ref-guide/src/common-query-parameters.adoc @@ -102,7 +102,7 @@ fq=+popularity:[10 TO *] +section:0 ---- * The document sets from each filter query are cached independently. Thus, concerning the previous examples: use a single `fq` containing two mandatory clauses if those clauses appear together often, and use two separate `fq` parameters if they are relatively independent. (To learn about tuning cache sizes and making sure a filter cache actually exists, see <>.) -* It is also possible to use <> inside the `fq` to cache clauses individually and - among other things - to achieve union of cached filter queries. +* It is also possible to use <> inside the `fq` to cache clauses individually and - among other things - to achieve union of cached filter queries. * As with all parameters: special characters in an URL need to be properly escaped and encoded as hex values. Online tools are available to help you with URL-encoding. For example: http://meyerweb.com/eric/tools/dencoder/. diff --git a/solr/solr-ref-guide/src/enabling-ssl.adoc b/solr/solr-ref-guide/src/enabling-ssl.adoc index 96262bd4e3ec..77e66440d983 100644 --- a/solr/solr-ref-guide/src/enabling-ssl.adoc +++ b/solr/solr-ref-guide/src/enabling-ssl.adoc @@ -175,7 +175,7 @@ If you have set up your ZooKeeper cluster to use a <>) you can omit `-z ` from all of the `bin/solr`/`bin\solr.cmd` commands below. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from all of the `bin/solr`/`bin\solr.cmd` commands below. ==== Create Solr Home Directories for Two Nodes diff --git a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc index ed0b3cb900ad..ee1337b7caee 100644 --- a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc @@ -298,7 +298,7 @@ Once the configuration is complete, you can start Solr with the `bin/solr` scrip bin/solr -c -z server1:2181,server2:2181,server3:2181/solr ---- -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. === Test the Configuration diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc index 9689d276d019..a6b639c8f526 100644 --- a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc +++ b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc @@ -157,7 +157,7 @@ Limited back-compatibility is offered by automatically adding a default instance + This behavior can be also disabled by specifying a `SolrJmxReporter` configuration with a boolean init argument `enabled` set to `false`. For a more fine-grained control users should explicitly specify at least one `SolrJmxReporter` configuration. + -See also the section < Element>>, which describes how to set up Metrics Reporters in `solr.xml`. Note that back-compatibility support may be removed in Solr 8. +See also the section < Element>>, which describes how to set up Metrics Reporters in `solr.xml`. Note that back-compatibility support may be removed in Solr 8. * MBean names and attributes now follow the hierarchical names used in metrics. This is reflected also in `/admin/mbeans` and `/admin/plugins` output, and can be observed in the UI Plugins tab, because now all these APIs get their data from the metrics API. The old (mostly flat) JMX view has been removed. diff --git a/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc b/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc index 04dc84078df6..6efd13532903 100644 --- a/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc +++ b/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc @@ -46,7 +46,7 @@ CAUTION: By default, ZooKeeper's file size limit is 1MB. If your files are large Under standalone Solr, when looking up a plugin or resource to be loaded, Solr's resource loader will first look under the `/conf/` directory. If the plugin or resource is not found, the configured plugin and resource file paths are searched - see the section <> below. -On core load, Solr's resource loader constructs a list of paths (subdirectories and jars), first under <>, and then under directories pointed to by <` directives in SolrConfig>>. +On core load, Solr's resource loader constructs a list of paths (subdirectories and jars), first under <>, and then under directories pointed to by <` directives in SolrConfig>>. When looking up a resource or plugin to be loaded, the paths on the list are searched in the order they were added. diff --git a/solr/solr-ref-guide/src/solr-control-script-reference.adoc b/solr/solr-ref-guide/src/solr-control-script-reference.adoc index 5d5808e49d93..b4ebd34964f3 100644 --- a/solr/solr-ref-guide/src/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/src/solr-control-script-reference.adoc @@ -60,7 +60,7 @@ Start Solr in SolrCloud mode, which will also launch the embedded ZooKeeper inst + This option can be shortened to simply `-c`. + -If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) or pass the -z parameter. +If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) or pass the -z parameter. + For more details, see the section <> below. + @@ -172,7 +172,7 @@ The `-c` and `-cloud` options are equivalent: If you specify a ZooKeeper connection string, such as `-z 192.168.1.4:2181`, then Solr will connect to ZooKeeper and join the cluster. -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from all `bin/solr` commands. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from all `bin/solr` commands. When starting Solr in cloud mode, if you neither define `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` nor specify the `-z` option, then Solr will launch an embedded ZooKeeper server listening on the Solr port + 1000, i.e., if Solr is running on port 8983, then the embedded ZooKeeper will be listening on port 9983. diff --git a/solr/solr-ref-guide/src/solr-tutorial.adoc b/solr/solr-ref-guide/src/solr-tutorial.adoc index 577fc5cdd2f3..34c1f9cce15c 100644 --- a/solr/solr-ref-guide/src/solr-tutorial.adoc +++ b/solr/solr-ref-guide/src/solr-tutorial.adoc @@ -497,7 +497,7 @@ This starts the first node. When it's done start the second node, and tell it ho `./bin/solr start -c -p 7574 -s example/cloud/node2/solr -z localhost:9983` -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. === Create a New Collection diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc index 0b41e21914d1..ddb2002cd394 100644 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc +++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc @@ -55,7 +55,7 @@ generated, which may significantly differ due to the rate limits set by `waitFor indicates the nodes that were lost or added. == Trigger Configuration -Trigger configurations are managed using the <> with the commands `<>`, `<>`, +Trigger configurations are managed using the <> with the commands `<>`, `<>`, `suspend-trigger`, and `resume-trigger`. === Trigger Properties diff --git a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc b/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc index b1b3b5a77e1b..091011eebc6b 100644 --- a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc +++ b/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc @@ -85,4 +85,4 @@ If you for example would like to keep your `solr.xml` in ZooKeeper to avoid havi bin/solr zk cp file:local/file/path/to/solr.xml zk:/solr.xml -z localhost:2181 ---- -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. \ No newline at end of file +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. \ No newline at end of file From 91699c26eab63d22e04e527af4db675cadb1ea2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 20 Sep 2018 16:28:20 +0200 Subject: [PATCH 22/88] Revert "SOLR-12786: HTML reference guide has invalid links" This reverts commit 6014b85 --- solr/solr-ref-guide/src/basic-authentication-plugin.adoc | 2 +- solr/solr-ref-guide/src/common-query-parameters.adoc | 2 +- solr/solr-ref-guide/src/enabling-ssl.adoc | 2 +- solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc | 2 +- solr/solr-ref-guide/src/major-changes-in-solr-7.adoc | 2 +- solr/solr-ref-guide/src/resource-and-plugin-loading.adoc | 2 +- solr/solr-ref-guide/src/solr-control-script-reference.adoc | 4 ++-- solr/solr-ref-guide/src/solr-tutorial.adoc | 2 +- solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc | 2 +- .../src/using-zookeeper-to-manage-configuration-files.adoc | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc index 971f59eb3bbb..88e8a0c8d24c 100644 --- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc @@ -65,7 +65,7 @@ If you are using SolrCloud, you must upload `security.json` to ZooKeeper. You ca bin/solr zk cp file:path_to_local_security.json zk:/security.json -z localhost:9983 ---- -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. === Caveats diff --git a/solr/solr-ref-guide/src/common-query-parameters.adoc b/solr/solr-ref-guide/src/common-query-parameters.adoc index 5df3ca3e3c25..499401525c0e 100644 --- a/solr/solr-ref-guide/src/common-query-parameters.adoc +++ b/solr/solr-ref-guide/src/common-query-parameters.adoc @@ -102,7 +102,7 @@ fq=+popularity:[10 TO *] +section:0 ---- * The document sets from each filter query are cached independently. Thus, concerning the previous examples: use a single `fq` containing two mandatory clauses if those clauses appear together often, and use two separate `fq` parameters if they are relatively independent. (To learn about tuning cache sizes and making sure a filter cache actually exists, see <>.) -* It is also possible to use <> inside the `fq` to cache clauses individually and - among other things - to achieve union of cached filter queries. +* It is also possible to use <> inside the `fq` to cache clauses individually and - among other things - to achieve union of cached filter queries. * As with all parameters: special characters in an URL need to be properly escaped and encoded as hex values. Online tools are available to help you with URL-encoding. For example: http://meyerweb.com/eric/tools/dencoder/. diff --git a/solr/solr-ref-guide/src/enabling-ssl.adoc b/solr/solr-ref-guide/src/enabling-ssl.adoc index 77e66440d983..96262bd4e3ec 100644 --- a/solr/solr-ref-guide/src/enabling-ssl.adoc +++ b/solr/solr-ref-guide/src/enabling-ssl.adoc @@ -175,7 +175,7 @@ If you have set up your ZooKeeper cluster to use a <>) you can omit `-z ` from all of the `bin/solr`/`bin\solr.cmd` commands below. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from all of the `bin/solr`/`bin\solr.cmd` commands below. ==== Create Solr Home Directories for Two Nodes diff --git a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc index ee1337b7caee..ed0b3cb900ad 100644 --- a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc @@ -298,7 +298,7 @@ Once the configuration is complete, you can start Solr with the `bin/solr` scrip bin/solr -c -z server1:2181,server2:2181,server3:2181/solr ---- -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. === Test the Configuration diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc index a6b639c8f526..9689d276d019 100644 --- a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc +++ b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc @@ -157,7 +157,7 @@ Limited back-compatibility is offered by automatically adding a default instance + This behavior can be also disabled by specifying a `SolrJmxReporter` configuration with a boolean init argument `enabled` set to `false`. For a more fine-grained control users should explicitly specify at least one `SolrJmxReporter` configuration. + -See also the section < Element>>, which describes how to set up Metrics Reporters in `solr.xml`. Note that back-compatibility support may be removed in Solr 8. +See also the section < Element>>, which describes how to set up Metrics Reporters in `solr.xml`. Note that back-compatibility support may be removed in Solr 8. * MBean names and attributes now follow the hierarchical names used in metrics. This is reflected also in `/admin/mbeans` and `/admin/plugins` output, and can be observed in the UI Plugins tab, because now all these APIs get their data from the metrics API. The old (mostly flat) JMX view has been removed. diff --git a/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc b/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc index 6efd13532903..04dc84078df6 100644 --- a/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc +++ b/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc @@ -46,7 +46,7 @@ CAUTION: By default, ZooKeeper's file size limit is 1MB. If your files are large Under standalone Solr, when looking up a plugin or resource to be loaded, Solr's resource loader will first look under the `/conf/` directory. If the plugin or resource is not found, the configured plugin and resource file paths are searched - see the section <> below. -On core load, Solr's resource loader constructs a list of paths (subdirectories and jars), first under <>, and then under directories pointed to by <` directives in SolrConfig>>. +On core load, Solr's resource loader constructs a list of paths (subdirectories and jars), first under <>, and then under directories pointed to by <` directives in SolrConfig>>. When looking up a resource or plugin to be loaded, the paths on the list are searched in the order they were added. diff --git a/solr/solr-ref-guide/src/solr-control-script-reference.adoc b/solr/solr-ref-guide/src/solr-control-script-reference.adoc index b4ebd34964f3..5d5808e49d93 100644 --- a/solr/solr-ref-guide/src/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/src/solr-control-script-reference.adoc @@ -60,7 +60,7 @@ Start Solr in SolrCloud mode, which will also launch the embedded ZooKeeper inst + This option can be shortened to simply `-c`. + -If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) or pass the -z parameter. +If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) or pass the -z parameter. + For more details, see the section <> below. + @@ -172,7 +172,7 @@ The `-c` and `-cloud` options are equivalent: If you specify a ZooKeeper connection string, such as `-z 192.168.1.4:2181`, then Solr will connect to ZooKeeper and join the cluster. -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from all `bin/solr` commands. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from all `bin/solr` commands. When starting Solr in cloud mode, if you neither define `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` nor specify the `-z` option, then Solr will launch an embedded ZooKeeper server listening on the Solr port + 1000, i.e., if Solr is running on port 8983, then the embedded ZooKeeper will be listening on port 9983. diff --git a/solr/solr-ref-guide/src/solr-tutorial.adoc b/solr/solr-ref-guide/src/solr-tutorial.adoc index 34c1f9cce15c..577fc5cdd2f3 100644 --- a/solr/solr-ref-guide/src/solr-tutorial.adoc +++ b/solr/solr-ref-guide/src/solr-tutorial.adoc @@ -497,7 +497,7 @@ This starts the first node. When it's done start the second node, and tell it ho `./bin/solr start -c -p 7574 -s example/cloud/node2/solr -z localhost:9983` -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. === Create a New Collection diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc index ddb2002cd394..0b41e21914d1 100644 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc +++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc @@ -55,7 +55,7 @@ generated, which may significantly differ due to the rate limits set by `waitFor indicates the nodes that were lost or added. == Trigger Configuration -Trigger configurations are managed using the <> with the commands `<>`, `<>`, +Trigger configurations are managed using the <> with the commands `<>`, `<>`, `suspend-trigger`, and `resume-trigger`. === Trigger Properties diff --git a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc b/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc index 091011eebc6b..b1b3b5a77e1b 100644 --- a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc +++ b/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc @@ -85,4 +85,4 @@ If you for example would like to keep your `solr.xml` in ZooKeeper to avoid havi bin/solr zk cp file:local/file/path/to/solr.xml zk:/solr.xml -z localhost:2181 ---- -NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. \ No newline at end of file +NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <>) you can omit `-z ` from the above command. \ No newline at end of file From 9cc9eab3db405c57b92d9aaa664aa88bdf0b238a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 24 Sep 2018 13:34:14 +0200 Subject: [PATCH 23/88] Remove unused adminScope and supported_ops --- .../src/java/org/apache/solr/security/JWTAuthPlugin.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 0c277aec9b79..6aa05bf1635e 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -93,7 +93,6 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String PARAM_JWK_CACHE_DURATION = "jwkCacheDur"; private static final String PARAM_CLAIMS_MATCH = "claimsMatch"; private static final String PARAM_SCOPE = "scope"; - private static final String PARAM_ADMIN_SCOPE = "adminScope"; private static final String PARAM_CLIENT_ID = "clientId"; private static final String PARAM_WELL_KNOWN_URL = "wellKnownUrl"; @@ -101,10 +100,9 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String CLAIM_SCOPE = "scope"; private final JwtPkiDelegationInterceptor interceptor = new JwtPkiDelegationInterceptor(); - static final Set supported_ops = ImmutableSet.of("set-user", "delete-user"); private static final Set PROPS = ImmutableSet.of(PARAM_BLOCK_UNKNOWN, PARAM_JWK_URL, PARAM_JWK, PARAM_ISSUER, PARAM_AUDIENCE, PARAM_REQUIRE_SUBJECT, PARAM_PRINCIPAL_CLAIM, PARAM_REQUIRE_EXPIRATIONTIME, PARAM_ALG_WHITELIST, - PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_ADMIN_SCOPE, PARAM_CLIENT_ID, PARAM_WELL_KNOWN_URL); + PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_CLIENT_ID, PARAM_WELL_KNOWN_URL); private JwtConsumer jwtConsumer; private String iss; @@ -116,7 +114,6 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private String principalClaim; private HashMap claimsMatchCompiled; private boolean blockUnknown; - private String adminScope; private HashSet requiredScopes = new HashSet<>(); private String clientId; private long jwkCacheDuration; @@ -178,7 +175,6 @@ public void init(Map pluginConfig) { if (!StringUtils.isEmpty(requiredScopesStr)) { requiredScopes = new HashSet<>(Arrays.asList(requiredScopesStr.split("\\s+"))); } - adminScope = (String) pluginConfig.get(PARAM_ADMIN_SCOPE); Map claimsMatch = (Map) pluginConfig.get(PARAM_CLAIMS_MATCH); claimsMatchCompiled = new HashMap<>(); if (claimsMatch != null) { From 404a2671e4ea7bf21e4b5738758f36e0d92afaea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 24 Sep 2018 13:50:59 +0200 Subject: [PATCH 24/88] Prettify docs table --- .../src/jwt-authentication-plugin.adoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index a9942792c10b..426f55af58fc 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -107,19 +107,19 @@ Let's comment on this config: [%header,format=csv,separator=;] |=== -Key ; Description ; Default +Key ; Description ; Default blockUnknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` -wellKnownUrl ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; +wellKnownUrl ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; clientId ; Client identifier for use with OpenID Connect ; -scope ; Whitespace separate list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; +scope ; Whitespace separate list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; jwkUrl ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided -jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided -iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. -aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it +jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided +iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. +aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it requireSub ; Makes `sub` (subject) mandatory ; `true` requireExp ; Makes `exp` (expiry time) mandatory ; `true` algWhitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; -jwkCacheDur ; Duration of JWK cache in seconds ; `3600` (1 hour) +jwkCacheDur ; Duration of JWK cache in seconds ; `3600` (1 hour) principalClaim ; What claim id to pull principal from ; `sub` claimsMatch ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) |=== From 78b1341f2f902a195d4983041258ff62a30d4bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 24 Sep 2018 13:58:59 +0200 Subject: [PATCH 25/88] Finalise clients chapter --- .../solr-ref-guide/src/jwt-authentication-plugin.adoc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 426f55af58fc..86d38486c4d2 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -163,16 +163,21 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica [#jwt-soljr] === SolrJ -SolrJ does not currently support setting JWT tokens per request. This is planned for a future release. +SolrJ does not currently support supplying JWT tokens per request. [#jwt-curl] === cURL -To authenticate with Solr with cURL +To authenticate with Solr when using the cURL utility, supply a valid JWT access token in an `Authorization` header, as follows (replace xxxxxx.xxxxxx.xxxxxx with your JWT compact token): + +[source,bash] +---- +curl -H "Authorization: Bearer xxxxxx.xxxxxx.xxxxxx" http://localhost:8983/solr/admin/info/system +---- === Admin UI -The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. Because of this the Admin UI will stop working after enabling this plugin. To circumvent this you can install a browser plugin that adds HTTP headers to requests, and add a valid JWT token to the `Authorization` header by hand. +The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. Because of this the Admin UI will stop working after enabling this plugin. To circumvent this you may be able to install a browser plugin that adds HTTP headers to requests, and add manually a valid JWT token to the `Authorization` header. == Using the Solr Control Script with JWT Auth From 06adf2ecc93a1451fb0dca703d64aa13084aacd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 12 Dec 2018 15:07:14 +0100 Subject: [PATCH 26/88] Pull out parts that belong to other JIRAs Changes entry moved to 8.0.0 section --- lucene/ivy-versions.properties | 2 +- solr/CHANGES.txt | 4 ++-- .../solr/handler/component/HttpShardHandler.java | 9 +-------- .../solr/security/PKIAuthenticationPlugin.java | 2 +- .../org/apache/solr/update/SolrCmdDistributor.java | 3 --- .../org/apache/solr/client/solrj/SolrRequest.java | 11 ----------- .../solr/client/solrj/impl/HttpSolrClient.java | 13 +++---------- 7 files changed, 8 insertions(+), 36 deletions(-) diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index d0140c94e56c..52d965dcbe4b 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -308,4 +308,4 @@ org.slf4j.version = 1.7.24 ua.net.nlp.morfologik-ukrainian-search.version = 3.9.0 /ua.net.nlp/morfologik-ukrainian-search = ${ua.net.nlp.morfologik-ukrainian-search.version} -/xerces/xercesImpl = 2.9.1 \ No newline at end of file +/xerces/xercesImpl = 2.9.1 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b39719cd7265..1502c38da56c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -79,6 +79,8 @@ New Features * SOLR-12593: The default configSet now includes an "ignored_*" dynamic field. (David Smiley) +* SOLR-12121: JWT Token authentication plugin (janhoy) + * SOLR-12791: Add Metrics reporting for AuthenticationPlugin (janhoy) Bug Fixes @@ -845,8 +847,6 @@ New Features * SOLR-11913: SolrJ SolrParams now implements Iterable> and also has a stream() method using it for convenience. (David Smiley, Tapan Vaishnav) -* SOLR-12121: JWT Token authentication plugin (janhoy) - * SOLR-11924: Added the ability to listen to changes in the set of active collections in a cloud in the ZkStateReader, through the CloudCollectionsListener. (Houston Putman, Dennis Gove) diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 05988eb9c8f6..a548031f7bd2 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -18,7 +18,6 @@ import java.lang.invoke.MethodHandles; import java.net.ConnectException; -import java.security.Principal; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -57,7 +56,6 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -137,11 +135,7 @@ private List getURLs(String shard) { public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) { // do this outside of the callable for thread safety reasons final List urls = getURLs(shard); - - // If request has a Principal (authenticated user), extract it for passing on to the new shard request - SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo(); - final Principal userPrincipal = requestInfo == null ? null : requestInfo.getReq().getUserPrincipal(); - + Callable task = () -> { ShardResponse srsp = new ShardResponse(); @@ -160,7 +154,6 @@ public void submit(final ShardRequest sreq, final String shard, final Modifiable QueryRequest req = makeQueryRequest(sreq, params, shard); req.setMethod(SolrRequest.METHOD.POST); - req.setUserPrincipal(userPrincipal); // no need to set the response parser as binary is the default // req.setResponseParser(new BinaryResponseParser()); diff --git a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java index a1ef420a9e49..fe1ae7bdaca0 100644 --- a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java @@ -250,7 +250,7 @@ public void process(HttpRequest httpRequest, HttpContext httpContext) throws Htt } @SuppressForbidden(reason = "Needs currentTimeMillis to set current time in header") - protected void setHeader(HttpRequest httpRequest) { + void setHeader(HttpRequest httpRequest) { SolrRequestInfo reqInfo = getRequestInfo(); String usr; if (reqInfo != null) { diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index 48632f98abf9..9536f9dd5108 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -47,7 +47,6 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.Diagnostics; -import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker; import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker; @@ -283,8 +282,6 @@ void addCommit(UpdateRequest ureq, CommitUpdateCommand cmd) { } private void submit(final Req req, boolean isCommit) { - // Copy user principal from the original request to the new update request, for later authentication interceptor use - req.uReq.setUserPrincipal(SolrRequestInfo.getRequestInfo().getReq().getUserPrincipal()); if (req.synchronous) { blockAndDoRetries(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java index 04b94f7aae84..7dbaab90915c 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.io.Serializable; -import java.security.Principal; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -37,16 +36,6 @@ * @since solr 1.3 */ public abstract class SolrRequest implements Serializable { - // This user principal is typically used by Auth plugins during distributed/sharded search - private Principal userPrincipal; - - public void setUserPrincipal(Principal userPrincipal) { - this.userPrincipal = userPrincipal; - } - - public Principal getUserPrincipal() { - return userPrincipal; - } public enum METHOD { GET, diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index 8831448579ef..ad845e828fe4 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -24,7 +24,6 @@ import java.net.ConnectException; import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; -import java.security.Principal; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -253,7 +252,7 @@ public NamedList request(final SolrRequest request, final ResponseParser throws SolrServerException, IOException { HttpRequestBase method = createMethod(request, collection); setBasicAuthHeader(request, method); - return executeMethod(method, request.getUserPrincipal(), processor, isV2ApiRequest(request)); + return executeMethod(method, processor, isV2ApiRequest(request)); } private boolean isV2ApiRequest(final SolrRequest request) { @@ -297,7 +296,7 @@ public HttpUriRequestResponse httpUriRequest(final SolrRequest request, final Re ExecutorService pool = ExecutorUtil.newMDCAwareFixedThreadPool(1, new SolrjNamedThreadFactory("httpUriRequest")); try { MDC.put("HttpSolrClient.url", baseUrl); - mrr.future = pool.submit(() -> executeMethod(method, request.getUserPrincipal(), processor, isV2ApiRequest(request))); + mrr.future = pool.submit(() -> executeMethod(method, processor, isV2ApiRequest(request))); } finally { pool.shutdown(); @@ -518,7 +517,7 @@ private HttpEntityEnclosingRequestBase fillContentStream(SolrRequest request, Co private static final List errPath = Arrays.asList("metadata", "error-class");//Utils.getObjectByPath(err, false,"metadata/error-class") - protected NamedList executeMethod(HttpRequestBase method, Principal userPrincipal, final ResponseParser processor, final boolean isV2Api) throws SolrServerException { + protected NamedList executeMethod(HttpRequestBase method, final ResponseParser processor, final boolean isV2Api) throws SolrServerException { method.addHeader("User-Agent", AGENT); org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = HttpClientUtil.createDefaultRequestConfigBuilder(); @@ -540,12 +539,6 @@ protected NamedList executeMethod(HttpRequestBase method, Principal user try { // Execute the method. HttpClientContext httpClientRequestContext = HttpClientUtil.createNewHttpClientRequestContext(); - if (userPrincipal != null) { - // Normally the context contains a static userToken to enable reuse resources. - // However, if a personal Principal object exists, we use that instead, also as a means - // to transfer authentication information to Auth plugins that wish to intercept the request later - httpClientRequestContext.setUserToken(userPrincipal); - } final HttpResponse response = httpClient.execute(method, httpClientRequestContext); int httpStatus = response.getStatusLine().getStatusCode(); From d35d157b648f89ddd343e03b772e45eaa00dfe14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 12 Dec 2018 15:22:50 +0100 Subject: [PATCH 27/88] New sha1 sum --- solr/licenses/jose4j-0.6.3.jar.sha1 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 solr/licenses/jose4j-0.6.3.jar.sha1 diff --git a/solr/licenses/jose4j-0.6.3.jar.sha1 b/solr/licenses/jose4j-0.6.3.jar.sha1 deleted file mode 100644 index 968ae6dd27e2..000000000000 --- a/solr/licenses/jose4j-0.6.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -05bf092aa7be6fe8894d11f8d2040a2b3b401a14 From f08bb76081869430cd631677948048ac623fcf2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 12 Dec 2018 15:32:27 +0100 Subject: [PATCH 28/88] Fix precommit by specifying charset --- .../apache/solr/security/JWTAuthPluginIntegrationTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index d2f35babc138..03ae5416c30c 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -23,6 +23,7 @@ import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -154,7 +155,7 @@ private Pair get(String url, String token) throws IOException { HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); if (token != null) createConn.setRequestProperty("Authorization", "Bearer " + token); - BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent())); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) createConn.getContent(), StandardCharsets.UTF_8)); String result = br2.lines().collect(Collectors.joining("\n")); int code = createConn.getResponseCode(); createConn.disconnect(); @@ -171,12 +172,12 @@ private Pair post(String url, String json, String token) throws con.setDoOutput(true); OutputStream os = con.getOutputStream(); - os.write(json.getBytes()); + os.write(json.getBytes(StandardCharsets.UTF_8)); os.flush(); os.close(); con.connect(); - BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent())); + BufferedReader br2 = new BufferedReader(new InputStreamReader((InputStream) con.getContent(), StandardCharsets.UTF_8)); String result = br2.lines().collect(Collectors.joining("\n")); int code = con.getResponseCode(); con.disconnect(); From dd783b27ad557c47e1d3652839866ae8006f8b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 12 Dec 2018 21:23:24 +0100 Subject: [PATCH 29/88] Update patch from other combined branch --- lucene/ivy-versions.properties | 2 +- .../org/apache/solr/core/CoreContainer.java | 14 +- .../solr/security/AuthenticationPlugin.java | 1 + .../apache/solr/security/JWTAuthPlugin.java | 91 +++++++--- .../security/BasicAuthIntegrationTest.java | 45 ++++- .../solr/security/JWTAuthPluginTest.java | 16 ++ solr/licenses/jose4j-0.6.4.jar.sha1 | 1 + .../src/jwt-authentication-plugin.adoc | 72 ++++---- solr/webapp/web/index.html | 1 + solr/webapp/web/js/angular/app.js | 16 +- .../web/js/angular/controllers/login.js | 162 ++++++++++++++++++ .../web/js/angular/controllers/unknown.js | 37 ++++ solr/webapp/web/js/angular/services.js | 46 +++++ solr/webapp/web/partials/login.html | 63 +++++++ solr/webapp/web/partials/unknown.html | 23 +++ 15 files changed, 516 insertions(+), 74 deletions(-) create mode 100644 solr/licenses/jose4j-0.6.4.jar.sha1 create mode 100644 solr/webapp/web/js/angular/controllers/unknown.js create mode 100644 solr/webapp/web/partials/unknown.html diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 52d965dcbe4b..d0140c94e56c 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -308,4 +308,4 @@ org.slf4j.version = 1.7.24 ua.net.nlp.morfologik-ukrainian-search.version = 3.9.0 /ua.net.nlp/morfologik-ukrainian-search = ${ua.net.nlp.morfologik-ukrainian-search.version} -/xerces/xercesImpl = 2.9.1 +/xerces/xercesImpl = 2.9.1 \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 3051dbfef55c..d736aad9e0bc 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -431,15 +431,11 @@ public Lookup getAuthSchemeRegistry() { } HttpClientUtil.setHttpClientRequestContextBuilder(httpClientBuilder); - - } else { - if (pkiAuthenticationPlugin != null) { - //this happened due to an authc plugin reload. no need to register the pkiAuthc plugin again - if(pkiAuthenticationPlugin.isInterceptorRegistered()) return; - log.info("PKIAuthenticationPlugin is managing internode requests"); - setupHttpClientForAuthPlugin(pkiAuthenticationPlugin); - pkiAuthenticationPlugin.setInterceptorRegistered(); - } + } + // Always register PKI auth interceptor, which will then delegate the decision of who should secure + // each request to the configured authentication plugin. + if (pkiAuthenticationPlugin != null && !pkiAuthenticationPlugin.isInterceptorRegistered()) { + pkiAuthenticationPlugin.getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder()); } } diff --git a/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java index 48b073d30ee7..d64c32393e63 100644 --- a/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java @@ -40,6 +40,7 @@ public abstract class AuthenticationPlugin implements Closeable, SolrInfoBean, SolrMetricProducer { final public static String AUTHENTICATION_PLUGIN_PROP = "authenticationPlugin"; + final public static String HTTP_HEADER_X_SOLR_AUTHDATA = "X-Solr-AuthData"; // Metrics private Set metricNames = ConcurrentHashMap.newKeySet(); diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 6aa05bf1635e..eceac623fb0d 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -29,6 +29,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.Principal; import java.time.Instant; import java.util.ArrayList; @@ -55,6 +56,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.StringUtils; +import org.apache.solr.common.util.Base64; import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; @@ -73,6 +75,7 @@ import org.jose4j.keys.resolvers.JwksVerificationKeyResolver; import org.jose4j.keys.resolvers.VerificationKeyResolver; import org.jose4j.lang.JoseException; +import org.noggit.JSONUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,8 +96,11 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String PARAM_JWK_CACHE_DURATION = "jwkCacheDur"; private static final String PARAM_CLAIMS_MATCH = "claimsMatch"; private static final String PARAM_SCOPE = "scope"; + private static final String PARAM_ADMINUI_SCOPE = "adminUiScope"; + private static final String PARAM_REDIRECT_URIS = "redirectUris"; private static final String PARAM_CLIENT_ID = "clientId"; private static final String PARAM_WELL_KNOWN_URL = "wellKnownUrl"; + private static final String PARAM_AUTHORIZATION_ENDPOINT = "authorizationEndpoint"; private static final String AUTH_REALM = "solr-jwt"; private static final String CLAIM_SCOPE = "scope"; @@ -102,7 +108,8 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private final JwtPkiDelegationInterceptor interceptor = new JwtPkiDelegationInterceptor(); private static final Set PROPS = ImmutableSet.of(PARAM_BLOCK_UNKNOWN, PARAM_JWK_URL, PARAM_JWK, PARAM_ISSUER, PARAM_AUDIENCE, PARAM_REQUIRE_SUBJECT, PARAM_PRINCIPAL_CLAIM, PARAM_REQUIRE_EXPIRATIONTIME, PARAM_ALG_WHITELIST, - PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_CLIENT_ID, PARAM_WELL_KNOWN_URL); + PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_CLIENT_ID, PARAM_WELL_KNOWN_URL, + PARAM_AUTHORIZATION_ENDPOINT, PARAM_ADMINUI_SCOPE, PARAM_REDIRECT_URIS); private JwtConsumer jwtConsumer; private String iss; @@ -114,7 +121,7 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private String principalClaim; private HashMap claimsMatchCompiled; private boolean blockUnknown; - private HashSet requiredScopes = new HashSet<>(); + private List requiredScopes = new ArrayList<>(); private String clientId; private long jwkCacheDuration; private WellKnownDiscoveryConfig oidcDiscoveryConfig; @@ -122,8 +129,11 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private Map pluginConfig; private Instant lastInitTime = Instant.now(); private CoreContainer coreContainer; + private String authorizationEndpoint; + private String adminUiScope; + private List redirectUris; + - /** * Initialize plugin with core container, this method is chosen by reflection at create time * @param coreContainer instance of core container @@ -146,11 +156,21 @@ public void init(Map pluginConfig) { requireExpirationTime = Boolean.parseBoolean(String.valueOf(pluginConfig.getOrDefault(PARAM_REQUIRE_EXPIRATIONTIME, "true"))); principalClaim = (String) pluginConfig.getOrDefault(PARAM_PRINCIPAL_CLAIM, "sub"); confIdpConfigUrl = (String) pluginConfig.get(PARAM_WELL_KNOWN_URL); + Object redirectUrisObj = pluginConfig.get(PARAM_REDIRECT_URIS); + redirectUris = Collections.emptyList(); + if (redirectUrisObj != null) { + if (redirectUrisObj instanceof String) { + redirectUris = Collections.singletonList((String) redirectUrisObj); + } else if (redirectUrisObj instanceof List) { + redirectUris = (List) redirectUrisObj; + } + } if (confIdpConfigUrl != null) { log.debug("Initializing well-known oidc config from {}", confIdpConfigUrl); oidcDiscoveryConfig = WellKnownDiscoveryConfig.parse(confIdpConfigUrl); iss = oidcDiscoveryConfig.getIssuer(); + authorizationEndpoint = oidcDiscoveryConfig.getAuthorizationEndpoint(); } if (pluginConfig.containsKey(PARAM_ISSUER)) { @@ -159,6 +179,13 @@ public void init(Map pluginConfig) { } iss = (String) pluginConfig.get(PARAM_ISSUER); } + + if (pluginConfig.containsKey(PARAM_AUTHORIZATION_ENDPOINT)) { + if (authorizationEndpoint != null) { + log.debug("Explicitly setting authorizationEndpoint instead of using issuer from well-known config"); + } + authorizationEndpoint = (String) pluginConfig.get(PARAM_AUTHORIZATION_ENDPOINT); + } if (pluginConfig.containsKey(PARAM_AUDIENCE)) { if (clientId != null) { @@ -173,8 +200,20 @@ public void init(Map pluginConfig) { String requiredScopesStr = (String) pluginConfig.get(PARAM_SCOPE); if (!StringUtils.isEmpty(requiredScopesStr)) { - requiredScopes = new HashSet<>(Arrays.asList(requiredScopesStr.split("\\s+"))); + requiredScopes = Arrays.asList(requiredScopesStr.split("\\s+")); + } + + adminUiScope = (String) pluginConfig.get(PARAM_ADMINUI_SCOPE); + if (adminUiScope == null && requiredScopes.size() > 0) { + adminUiScope = requiredScopes.get(0); + log.warn("No adminUiScope given, using first scope in 'scope' list as required scope for accessing Admin UI"); } + + if (adminUiScope == null) { + adminUiScope = "solr"; + log.warn("Warning: No adminUiScope provided, fallback to 'solr' as required scope. If this is not correct, the Admin UI login may not work"); + } + Map claimsMatch = (Map) pluginConfig.get(PARAM_CLAIMS_MATCH); claimsMatchCompiled = new HashMap<>(); if (claimsMatch != null) { @@ -203,7 +242,7 @@ private void initJwk(Map pluginConfig) { PARAM_WELL_KNOWN_URL + ", " + PARAM_JWK_URL + " and " + PARAM_JWK); } if (jwkConfigured == 0) { - log.warn("Initialized JWTAuthPlugin without any JWK config. No requests will succeed"); + log.warn("Initialized JWTAuthPlugin without any JWK config. Requests with jwk header will fail."); } if (oidcDiscoveryConfig != null) { String jwkUrl = oidcDiscoveryConfig.getJwksUrl(); @@ -224,14 +263,15 @@ private void initJwk(Map pluginConfig) { private void setupJwkUrl(String url) { // The HttpsJwks retrieves and caches keys from a the given HTTPS JWKS endpoint. - try { - URL jwkUrl = new URL(url); - if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be an HTTPS url"); - } - } catch (MalformedURLException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be a valid URL"); - } +// NOCOMMIT: Disable https requirement for now +// try { +// URL jwkUrl = new URL(url); +// if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { +// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be an HTTPS url"); +// } +// } catch (MalformedURLException e) { +// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be a valid URL"); +// } HttpsJwks httpsJkws = new HttpsJwks(url); httpsJkws.setDefaultCacheDuration(jwkCacheDuration); verificationKeyResolver = new HttpsJwksVerificationKeyResolver(httpsJkws); @@ -301,7 +341,6 @@ public Principal getUserPrincipal() { return true; case AUTZ_HEADER_PROBLEM: - log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_BAD_REQUEST, BearerWwwAuthErrorCode.invalid_request); return false; @@ -310,7 +349,6 @@ public Principal getUserPrincipal() { case JWT_PARSE_ERROR: case JWT_VALIDATION_EXCEPTION: case PRINCIPAL_MISSING: - log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); if (authResponse.getJwtException() != null) { log.warn("Exception: {}", authResponse.getJwtException().getMessage()); } @@ -318,14 +356,12 @@ public Principal getUserPrincipal() { return false; case SCOPE_MISSING: - log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.insufficient_scope); return false; case NO_AUTZ_HEADER: default: - log.debug("Authentication failed with reason {}, message {}", authResponse.getAuthCode(), authResponse.getErrorMessage()); - authenticationFailure(response, authResponse.getAuthCode().getMsg()); + authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, null); return false; } } @@ -462,10 +498,6 @@ public ValidatingJsonMap getSpec() { return Utils.getSpec("cluster.security.BasicAuth.Commands").getSpec(); } - private void authenticationFailure(HttpServletResponse response, String message) throws IOException { - authenticationFailure(response, message, HttpServletResponse.SC_UNAUTHORIZED, null); - } - /** * Operate the commands on the latest conf and return a new conf object * If there are errors in the commands , throw a SolrException. return a null @@ -504,7 +536,19 @@ private void authenticationFailure(HttpServletResponse response, String message, wwwAuthParams.add("error_description=\"" + message + "\""); } response.addHeader(HttpHeaders.WWW_AUTHENTICATE, org.apache.commons.lang.StringUtils.join(wwwAuthParams, ", ")); + response.addHeader(AuthenticationPlugin.HTTP_HEADER_X_SOLR_AUTHDATA, generateAuthDataHeader()); response.sendError(httpCode, message); + log.info("JWT Authentication attempt failed: {}", message); + } + + protected String generateAuthDataHeader() { + Map data = new HashMap<>(); + data.put(PARAM_AUTHORIZATION_ENDPOINT, authorizationEndpoint); + data.put("client_id", clientId); + data.put("scope", adminUiScope); + data.put("redirect_uris", redirectUris); + String headerJson = JSONUtil.toJSON(data); + return Base64.byteArrayToBase64(headerJson.getBytes(StandardCharsets.UTF_8)); } @@ -598,7 +642,8 @@ public static class WellKnownDiscoveryConfig { public static WellKnownDiscoveryConfig parse(String urlString) { try { URL url = new URL(urlString); - if (!Arrays.asList("https", "file").contains(url.getProtocol())) { + // NOCOMMIT - require HTTPS + if (!Arrays.asList("http", "https", "file").contains(url.getProtocol())) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Well-known config URL must be HTTPS or file"); } return parse(url.openStream()); diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index db47d3721fa1..3f655dfda178 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -30,6 +30,7 @@ import java.util.Random; import java.util.function.Predicate; +import org.apache.commons.io.IOUtils; import com.codahale.metrics.MetricRegistry; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -40,14 +41,17 @@ import org.apache.http.message.BasicHeader; import org.apache.http.util.EntityUtils; import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.RequestWriter.StringPayloadContentWriter; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.SolrCloudAuthTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.DocCollection; @@ -215,14 +219,7 @@ public void testBasicAuth() throws Exception { executeCommand(baseUrl + authzPrefix, cl,"{set-permission : { name : update , role : admin}}", "harry", "HarryIsUberCool"); - SolrInputDocument doc = new SolrInputDocument(); - doc.setField("id","4"); - UpdateRequest update = new UpdateRequest(); - update.setBasicAuthCredentials("harry","HarryIsUberCool"); - update.add(doc); - update.setCommitWithin(100); - cluster.getSolrClient().request(update, COLLECTION); - + addDocument("harry","HarryIsUberCool","id", "4"); executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", "harry", "HarryIsUberCool"); verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "true", 20, "harry", "HarryIsUberCool"); @@ -279,6 +276,34 @@ private void assertNumberOfMetrics(int num) { assertEquals(num, registry0.getMetrics().entrySet().stream().filter(e -> e.getKey().startsWith("SECURITY")).count()); } + private QueryResponse executeQuery(ModifiableSolrParams params, String user, String pass) throws IOException, SolrServerException { + SolrRequest req = new QueryRequest(params); + req.setBasicAuthCredentials(user, pass); + QueryResponse resp = (QueryResponse) req.process(cluster.getSolrClient(), COLLECTION); + assertNull(resp.getException()); + assertEquals(0, resp.getStatus()); + return resp; + } + + private void addDocument(String user, String pass, String... fields) throws IOException, SolrServerException { + SolrInputDocument doc = new SolrInputDocument(); + boolean isKey = true; + String key = null; + for (String field : fields) { + if (isKey) { + key = field; + isKey = false; + } else { + doc.setField(key, field); + } + } + UpdateRequest update = new UpdateRequest(); + update.setBasicAuthCredentials(user, pass); + update.add(doc); + cluster.getSolrClient().request(update, COLLECTION); + update.commit(cluster.getSolrClient(), COLLECTION); + } + public static void executeCommand(String url, HttpClient cl, String payload, String user, String pwd) throws IOException { HttpPost httpPost; @@ -288,7 +313,9 @@ public static void executeCommand(String url, HttpClient cl, String payload, Str httpPost.setEntity(new ByteArrayEntity(payload.getBytes(UTF_8))); httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); r = cl.execute(httpPost); - assertEquals(200, r.getStatusLine().getStatusCode()); + String response = IOUtils.toString(r.getEntity().getContent(), StandardCharsets.UTF_8); + assertEquals("Non-200 response code. Response was " + response, 200, r.getStatusLine().getStatusCode()); + assertFalse("Response contained errors: " + response, response.contains("errorMessages")); Utils.consumeFully(r.getEntity()); } diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index 99f07687f144..ad0935bc5e92 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -32,6 +32,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.Base64; import org.apache.solr.common.util.Utils; import org.jose4j.jwk.RsaJsonWebKey; import org.jose4j.jwk.RsaJwkGenerator; @@ -43,6 +44,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mortbay.util.ajax.JSON; import static org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.AUTZ_HEADER_PROBLEM; import static org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode.NO_AUTZ_HEADER; @@ -387,4 +389,18 @@ public void wellKnownConfigFromString() throws IOException { assertEquals(Arrays.asList("READ", "WRITE", "DELETE", "openid", "scope", "profile", "email", "address", "phone"), config.getScopesSupported()); assertEquals(Arrays.asList("code", "code id_token", "code token", "code id_token token", "token", "id_token", "id_token token"), config.getResponseTypesSupported()); } + + @Test + public void xSolrAuthDataHeader() { + testConfig.put("adminUiScope", "solr:admin"); + testConfig.put("authorizationEndpoint", "http://acmepaymentscorp/oauth/auz/authorize"); + testConfig.put("clientId", "solr-cluster"); + plugin.init(testConfig); + String headerBase64 = plugin.generateAuthDataHeader(); + String headerJson = new String(Base64.base64ToByteArray(headerBase64), StandardCharsets.UTF_8); + Map parsed = (Map) JSON.parse(headerJson); + assertEquals("solr:admin", parsed.get("scope")); + assertEquals("http://acmepaymentscorp/oauth/auz/authorize", parsed.get("authorizationEndpoint")); + assertEquals("solr-cluster", parsed.get("client_id")); + } } \ No newline at end of file diff --git a/solr/licenses/jose4j-0.6.4.jar.sha1 b/solr/licenses/jose4j-0.6.4.jar.sha1 new file mode 100644 index 000000000000..d4a446e04023 --- /dev/null +++ b/solr/licenses/jose4j-0.6.4.jar.sha1 @@ -0,0 +1 @@ +0ee27e0fa2e82f1cce75c70861190730ff174e49 diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 86d38486c4d2..9dd284743c24 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -16,7 +16,7 @@ // specific language governing permissions and limitations // under the License. -Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token] (JWT) based Bearer authentication with the use of the JWTAuthPlugin. This allows Solr to assert that a user is already authenticated with an external https://en.wikipedia.org/wiki/Identity_provider[Identity Provider] by validating that the JWT is digitally signed by the Identity Provider. One possible use case is for https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect] where a JWT formatted https://en.wikipedia.org/wiki/Access_token[access token] . +Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token] (JWT) based Bearer authentication with the use of the JWTAuthPlugin. This allows Solr to assert that a user is already authenticated with an external https://en.wikipedia.org/wiki/Identity_provider[Identity Provider] by validating that the JWT is digitally signed by the Identity Provider. One possible use case is for https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect] which uses JWT formatted https://en.wikipedia.org/wiki/Access_token[access tokens] . == Enable JWT Authentication @@ -35,8 +35,35 @@ The simplest possible `security.json` for registering the plugin without configu } ---- -The plugin will NOT block anonymous traffic in this mode, since the default for `block_unknown` is false. It is then possible to start configuring the plugin using REST calls. +The plugin will NOT block anonymous traffic in this mode, since the default for `block_unknown` is false. It is then possible to start configuring the plugin using REST API calls. +== Configuration parameters + +[%header,format=csv,separator=;] +|=== +Key ; Description ; Default +blockUnknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` +wellKnownUrl ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; (no default) +clientId ; Client identifier for use with OpenID Connect ; (no default value) Required to authenticate with Admin UI +realm ; Name of the authentication realm to echo back in HTTP 401 responses. Will also be displayed in Admin UI login page ; 'solr-jwt' +scope ; Whitespace separated list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; +jwkUrl ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided +jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided +iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. +aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it +requireSub ; Makes `sub` (subject) mandatory ; `true` +requireExp ; Makes `exp` (expiry time) mandatory ; `true` +algWhitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; Default is to allow all algorithms +jwkCacheDur ; Duration of JWK cache in seconds ; `3600` (1 hour) +principalClaim ; What claim id to pull principal from ; '`sub`' +claimsMatch ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) +adminUiScope ; Define what scope is requested when logging in from Admin UI ; If not defined, the first scope from `scope` parameter is used +authorizationEndpoint; The URL for the Id Provider's authorization endpoint ; If `well_known_url` is provided, uses `authorization_endpoint` from there. +redirectUris ; Valid location(s) for redirect after external authentication. Takes a string or array of strings. Must be the base URL of Solr, e.g. https://solr1.example.com:8983/solr/ and must match the list of redirect URIs registered with the Identity Provider beforehand. ; Defaults to empty list, i.e. any node is assumed to be a valid redirect target. +|=== + +== More configuration examples +=== With JWK URL To start enforcing authentication for all users, requiring a valid JWT token in the `Authorization` header, you need to configure the plugin with one or more https://tools.ietf.org/html/rfc7517[JSON Web Key]s (JWK). This is a JSON document containing the key used to sign/encrypt the JWT. It could be a symmetric or asymmetric key. The JWK can either be fetched (and cached) from an external HTTPS endpoint or specified directly in `security.json`. Below is an example of the former: [source,json] @@ -50,7 +77,8 @@ To start enforcing authentication for all users, requiring a valid JWT token in } ---- -The next example shows configuring using https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] with a well-konwn URI. +=== With Admin UI support +The next example shows configuring using https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] with a well-known URI for automatic configuration of many common settings, including ability to use the Admin UI with an OpenID Connect enabled Identity Provider. [source,json] ---- @@ -58,14 +86,17 @@ The next example shows configuring using https://openid.net/specs/openid-connect "authentication": { "class": "solr.JWTAuthPlugin", "blockUnknown": true, - "wellKnownUrl": "https://idp.example.com/.well-known/openid-configuration" + "wellKnownUrl": "https://idp.example.com/.well-known/openid-configuration", + "clientId": "xyz", + "redirectUri": "https://my.solr.server:8983/solr/#/login" } } ---- -In this case, `jwk_url` and `iss` will be automatically configured from the fetched configuration. +In this case, `jwk_url`, `iss` and `authorizationEndpoint` will be automatically configured from the fetched configuration. -A more complex configuration with static JWK: +=== Complex example +Let's look at a more complex configuration, this time with a static embedded JWK: [source,json] ---- @@ -95,7 +126,7 @@ Let's comment on this config: <1> Plugin class <2> Make sure to block anyone without a valid token <3> Here we pass the JWK inline instead of referring to a URL with `jwkUrl` -<4> Set the client id registered with Identiy Provider +<4> Set the client id registered with Identity Provider <5> The issuer claim must match "https://example.com/idp" <6> The audience claim must match "https://example.com/solr" <7> Fetch the user id from another claim than the default `sub` @@ -103,27 +134,6 @@ Let's comment on this config: <9> Require one of the scopes `solr:read`, `solr:write` or `solr:amin` <10> Only accept RSA algorithms for signatures -== Configuration parameters - -[%header,format=csv,separator=;] -|=== -Key ; Description ; Default -blockUnknown ; Set to `false` in order to pass requests from unknown users without a token ; `true` -wellKnownUrl ; URL to an https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] endpoint ; -clientId ; Client identifier for use with OpenID Connect ; -scope ; Whitespace separate list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; -jwkUrl ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided -jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided -iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. -aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it -requireSub ; Makes `sub` (subject) mandatory ; `true` -requireExp ; Makes `exp` (expiry time) mandatory ; `true` -algWhitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; -jwkCacheDur ; Duration of JWK cache in seconds ; `3600` (1 hour) -principalClaim ; What claim id to pull principal from ; `sub` -claimsMatch ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) -|=== - == Editing JWT Authentication Plugin Configuration @@ -177,7 +187,11 @@ curl -H "Authorization: Bearer xxxxxx.xxxxxx.xxxxxx" http://localhost:8983/solr/ === Admin UI -The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. Because of this the Admin UI will stop working after enabling this plugin. To circumvent this you may be able to install a browser plugin that adds HTTP headers to requests, and add manually a valid JWT token to the `Authorization` header. +The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. So logging in to the Admin UI in this context simply means to obtain a valid JWT token which can be passed to Solr. + + + +When this plugin is enabled, users will be redirected to a login page in the Admin UI once they attempt to do a restricted action. The page has a button that user will click and be redirected to the Identity Provider's login page. Once authenticated the user will be redirected back to Solr Admin UI to the last known location. The session will last as long as the JWT token expiry time. There is also a logout menu in the left column where user can explicitly log out. == Using the Solr Control Script with JWT Auth diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html index 23b9dbd6e8e4..5c3112141a36 100644 --- a/solr/webapp/web/index.html +++ b/solr/webapp/web/index.html @@ -85,6 +85,7 @@ + diff --git a/solr/webapp/web/js/angular/app.js b/solr/webapp/web/js/angular/app.js index cb04ba3efb7c..9abacee4697c 100644 --- a/solr/webapp/web/js/angular/app.js +++ b/solr/webapp/web/js/angular/app.js @@ -32,10 +32,18 @@ solrAdminApp.config([ templateUrl: 'partials/index.html', controller: 'IndexController' }). + when('/unknown', { + templateUrl: 'partials/unknown.html', + controller: 'UnknownController' + }). when('/login', { templateUrl: 'partials/login.html', controller: 'LoginController' }). + when('/login/:route', { + templateUrl: 'partials/login.html', + controller: 'LoginController' + }). when('/~logging', { templateUrl: 'partials/logging.html', controller: 'LoggingController' @@ -76,7 +84,7 @@ solrAdminApp.config([ templateUrl: 'partials/cluster_suggestions.html', controller: 'ClusterSuggestionsController' }). - when('/:core', { + when('/:core/core-overview', { templateUrl: 'partials/core_overview.html', controller: 'CoreOverviewController' }). @@ -143,7 +151,8 @@ solrAdminApp.config([ controller: 'SegmentsController' }). otherwise({ - redirectTo: '/' + templateUrl: 'partials/unknown.html', + controller: 'UnknownController' }); }]) .constant('Constants', { @@ -351,7 +360,7 @@ solrAdminApp.config([ $rootScope.$broadcast('connectionStatusInactive'); },2000); } - if (!$location.path().startsWith('/login')) { + if (!$location.path().startsWith('/login') && !$location.path().startsWith('/unknown')) { sessionStorage.removeItem("http401"); sessionStorage.removeItem("auth.state"); sessionStorage.removeItem("auth.statusText"); @@ -379,6 +388,7 @@ solrAdminApp.config([ var headers = rejection.headers(); var wwwAuthHeader = headers['www-authenticate']; sessionStorage.setItem("auth.wwwAuthHeader", wwwAuthHeader); + sessionStorage.setItem("auth.authDataHeader", headers['x-solr-authdata']); sessionStorage.setItem("auth.statusText", rejection.statusText); sessionStorage.setItem("http401", "true"); sessionStorage.removeItem("auth.scheme"); diff --git a/solr/webapp/web/js/angular/controllers/login.js b/solr/webapp/web/js/angular/controllers/login.js index 9935191e1276..672641e73113 100644 --- a/solr/webapp/web/js/angular/controllers/login.js +++ b/solr/webapp/web/js/angular/controllers/login.js @@ -45,6 +45,97 @@ solrAdminApp.controller('LoginController', var supportedSchemes = ['Basic', 'Bearer']; $scope.authSchemeSupported = supportedSchemes.includes(authScheme); + + if (authScheme === 'Bearer') { + // Check for OpenId redirect response + var errorText = ""; + $scope.isCallback = false; + if ($scope.subPath === 'callback') { + $scope.isCallback = true; + var hash = $location.hash(); + var hp = AuthenticationService.decodeHashParams(hash); + var expectedState = sessionStorage.getItem("auth.stateRandom") + "_" + sessionStorage.getItem("auth.location"); + sessionStorage.setItem("auth.state", "error"); + if (hp['access_token'] && hp['token_type'] && hp['state']) { + // Validate state + if (hp['state'] !== expectedState) { + $scope.error = "Problem with auth callback"; + console.log("Expected state param " + expectedState + " but got " + hp['state']); + errorText += "Invalid values in state parameter. "; + } + // Validate token type + if (hp['token_type'].toLowerCase() !== "bearer") { + console.log("Expected token_type param 'bearer', but got " + hp['token_type']); + errorText += "Invalid values in token_type parameter. "; + } + // Unpack access token and validate nonce, get username + var accessToken = hp['access_token'].split("."); + if (accessToken.length === 3) { + var payload = AuthenticationService.decodeJwtPart(accessToken[1]); + if (!payload['nonce'] || payload['nonce'] !== sessionStorage.getItem("auth.nonce")) { + // NOCOMMIT: Accept no nonce for now with demo IdP + console.log("Temporarily disregarding missing nonce"); + // errorText += "Invalid 'nonce' value, possible attack detected. Please log in again. "; + } + + if (errorText === "") { + sessionStorage.setItem("auth.username", payload['sub']); + sessionStorage.setItem("auth.header", "Bearer " + hp['access_token']); + sessionStorage.removeItem("auth.statusText"); + sessionStorage.removeItem("auth.stateRandom"); + sessionStorage.removeItem("auth.wwwAuthHeader"); + console.log("User " + payload['sub'] + " is logged in"); + var redirectTo = sessionStorage.getItem("auth.location"); + console.log("Redirecting to stored location " + redirectTo); + sessionStorage.setItem("auth.state", "authenticated"); + sessionStorage.removeItem("http401"); + $location.path(redirectTo).hash(""); + } + } else { + console.log("Expected JWT compact access_token param but got " + accessToken); + errorText += "Invalid values in access_token parameter. "; + } + if (errorText !== "") { + $scope.error = "Problems with OpenID callback"; + $scope.errorDescription = errorText; + $scope.http401 = "true"; + } + // End callback processing + } else if (hp['error']) { + // The callback had errors + console.log("Error received from idp: " + hp['error']); + alert("Error received from idp: " + hp['error']); + var errorDescriptions = {}; + errorDescriptions['invalid_request'] = "The request is missing a required parameter, includes an invalid parameter value, includes a parameter more than once, or is otherwise malformed."; + errorDescriptions['unauthorized_client'] = "The client is not authorized to request an access token using this method."; + errorDescriptions['access_denied'] = "The resource owner or authorization server denied the request."; + errorDescriptions['unsupported_response_type'] = "The authorization server does not support obtaining an access token using this method."; + errorDescriptions['invalid_scope'] = "The requested scope is invalid, unknown, or malformed."; + errorDescriptions['server_error'] = "The authorization server encountered an unexpected condition that prevented it from fulfilling the request."; + errorDescriptions['temporarily_unavailable'] = "The authorization server is currently unable to handle the request due to a temporary overloading or maintenance of the server."; + $scope.error = "Callback from Id Provider contained error: "; + if (hp['error_description']) { + $scope.errorDescription = decodeURIComponent(hp['error_description']); + } else { + $scope.errorDescription = errorDescriptions[hp['error']]; + } + if (hp['error_uri']) { + $scope.errorDescription += " More information at " + hp['error_uri'] + ". "; + } + if (hp['state'] !== expectedState) { + $scope.errorDescription += "The state parameter returned from ID Provider did not match the one we sent."; + } + sessionStorage.setItem("auth.state", "error"); + } + } + + if (errorText === "" && authParams) { + $scope.error = authParams['error']; + $scope.errorDescription = authParams['error_description']; + $scope.authData = AuthenticationService.getAuthDataHeader(); + } + } + $scope.authScheme = sessionStorage.getItem("auth.scheme"); $scope.authRealm = sessionStorage.getItem("auth.realm"); $scope.wwwAuthHeader = sessionStorage.getItem("auth.wwwAuthHeader"); @@ -65,6 +156,77 @@ solrAdminApp.controller('LoginController', $location.path("/"); }; + $scope.jwtLogin = function () { + var stateRandom = Math.random().toString(36).substr(2); + sessionStorage.setItem("auth.stateRandom", stateRandom); + var authState = stateRandom + "_" + sessionStorage.getItem("auth.location"); + var authNonce = Math.random().toString(36).substr(2) + Math.random().toString(36).substr(2) + Math.random().toString(36).substr(2); + sessionStorage.setItem("auth.nonce", authNonce); + var params = { + "response_type" : "id_token token", + "client_id" : $scope.authData['client_id'], + "redirect_uri" : $window.location.href.split('#')[0], + "scope" : "openid " + $scope.authData['scope'], + "state" : authState, + "nonce" : authNonce + }; + + var endpointBaseUrl = $scope.authData['authorizationEndpoint']; + var loc = endpointBaseUrl + "?" + paramsToString(params); + console.log("Redirecting to " + loc); + sessionStorage.setItem("auth.state", "expectCallback"); + $window.location.href = loc; + + function paramsToString(params) { + var arr = []; + for (var p in params) { + if( params.hasOwnProperty(p) ) { + arr.push(p + "=" + encodeURIComponent(params[p])); + } + } + return arr.join("&"); + } + }; + + $scope.jwtIsLoginNode = function() { + var redirect = $scope.authData ? $scope.authData['redirect_uris'] : undefined; + if (redirect && Array.isArray(redirect) && redirect.length > 0) { + var isLoginNode = false; + redirect.forEach(function(uri) { // Check that current node URL is among the configured callback URIs + if ($window.location.href.startsWith(uri)) isLoginNode = true; + }); + return isLoginNode; + } else { + return true; // no redirect UIRs configured, all nodes are potential login nodes + } + }; + + $scope.jwtFindLoginNode = function() { + var redirect = $scope.authData ? $scope.authData['redirect_uris'] : undefined; + if (redirect && Array.isArray(redirect) && redirect.length > 0) { + var loginNode = redirect[0]; + redirect.forEach(function(uri) { // if current node is in list, return its callback uri + if ($window.location.href.startsWith(uri)) loginNode = uri; + }); + return loginNode; + } else { + return $window.location.href.split('#')[0]; // Return base url of current URL as the url to use + } + }; + + // Redirect to login node if this is not a valid one + $scope.jwtGotoLoginNode = function() { + if (!$scope.jwtIsLoginNode()) { + $window.location.href = $scope.jwtFindLoginNode(); + } + }; + + $scope.jwtLogout = function() { + // reset login status + AuthenticationService.ClearCredentials(); + $location.path("/"); + }; + $scope.isLoggedIn = function() { return (sessionStorage.getItem("auth.username") !== null); }; diff --git a/solr/webapp/web/js/angular/controllers/unknown.js b/solr/webapp/web/js/angular/controllers/unknown.js new file mode 100644 index 000000000000..2d959e6cedbf --- /dev/null +++ b/solr/webapp/web/js/angular/controllers/unknown.js @@ -0,0 +1,37 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/** + * This controller is called whenever no other routes match. + * It is a place to intercept to look for special flows such as authentication callbacks (that do not support fragment in URL). + * Normal action is to redirect to dashboard "/" if no login is in progress + */ +solrAdminApp.controller('UnknownController', + ['$scope', '$window', '$routeParams', '$location', 'Constants', 'AuthenticationService', + function($scope, $window, $routeParams, $location, Constants, AuthenticationService) { + var fragment = $window.location.hash.startsWith("#/") ? $window.location.hash.substring(2) : $window.location.hash; + // Check if the URL is actually a callback from Identiy provider + if (AuthenticationService.isJwtCallback(fragment)) { + console.log("Detected an authentication callback, redirecting to /#/login/callback"); + $location.path("/login/callback").hash(fragment); + } else { + console.log("Redirecting from unknown path " + fragment + " to Dashboard"); + $location.path("/").hash(""); + } + } + ] +); diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js index 8eb148fb3ae1..8c6a7e990a34 100644 --- a/solr/webapp/web/js/angular/services.js +++ b/solr/webapp/web/js/angular/services.js @@ -281,7 +281,53 @@ solrAdminServices.factory('System', sessionStorage.removeItem("auth.username"); sessionStorage.removeItem("auth.wwwAuthHeader"); sessionStorage.removeItem("auth.statusText"); + localStorage.removeItem("auth.stateRandom"); + sessionStorage.removeItem("auth.nonce"); }; + service.getAuthDataHeader = function () { + try { + var header64 = sessionStorage.getItem("auth.authDataHeader"); + var headerJson = base64.decode(header64); + return JSON.parse(headerJson); + } catch (e) { + console.log("WARN: Wrong or missing X-Solr-AuthData header on 401 response " + e); + return null; + } + }; + + service.decodeJwtPart = function (jwtPart) { + try { + return JSON.parse(base64.urldecode(jwtPart)); + } catch (e) { + console.log("WARN: Invalid format of JWT part: " + e); + return {}; + } + }; + + service.isJwtCallback = function (hash) { + var hp = this.decodeHashParams(hash); + // console.log("Decoded hash as " + JSON.stringify(hp, undefined, 2)); // For debugging callbacks + return (hp['access_token'] && hp['token_type'] && hp['state']) || hp['error']; + }; + + service.decodeHashParams = function(hash) { + // access_token, token_type, expires_in, state + if (hash == null || hash.length === 0) { + return {}; + } + var params = {}; + var parts = hash.split("&"); + for (var p in parts) { + var kv = parts[p].split("="); + if (kv.length === 2) { + params[kv[0]] = decodeURIComponent(kv[1]); + } else { + console.log("Invalid callback URI, got parameter " + parts[p] + " but expected key=value"); + } + } + return params; + }; + return service; }]); diff --git a/solr/webapp/web/partials/login.html b/solr/webapp/web/partials/login.html index 10a3caf27c54..3407004767c2 100644 --- a/solr/webapp/web/partials/login.html +++ b/solr/webapp/web/partials/login.html @@ -60,7 +60,70 @@

Basic Authentication

+
+

OpenID Connect (JWT) authentication

+ +
+ Callback from ID Provider received. +

+ There were errors during login with ID Provider. Please try again.
+

+
+
+

+ Solr requires authentication for resource {{authLocation === '/' ? 'Dashboard' : authLocation}}. +

+
+

+ Please log in with your Identity Provider (IdP) for realm {{authRealm}}. +

+

+ Clicking the button below, you will be redirected to the authorization endpoint of the ID provider:
+ {{authData['authorizationEndpoint']}} +

+
+
{{error}}
+
+
+ +
+
+
+
+

+ In order to log in to the identity provider, you need to load this page from the Solr node registered as callback node:
+ {{jwtFindLoginNode()}}
+ After successful login you will be able to navigate to other nodes. +

+

+

+
+ +
+
+

+
+ +
+
+

+ Logged in as user {{authLoggedinUser}}. Realm={{authRealm}}.
+

+
+
+
+ +
+
+
+ +
+

Authentication scheme not supported

diff --git a/solr/webapp/web/partials/unknown.html b/solr/webapp/web/partials/unknown.html new file mode 100644 index 000000000000..51895ab8b58b --- /dev/null +++ b/solr/webapp/web/partials/unknown.html @@ -0,0 +1,23 @@ + +
+ +
+ Oops, this URL is unknown to us, redirecting you back to Dashboard +
+ +
From e11270c0e0820ea9c4dacfe9321c2b8e63cda6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 12 Dec 2018 21:44:26 +0100 Subject: [PATCH 30/88] Re-add line shift --- lucene/ivy-versions.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index d0140c94e56c..52d965dcbe4b 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -308,4 +308,4 @@ org.slf4j.version = 1.7.24 ua.net.nlp.morfologik-ukrainian-search.version = 3.9.0 /ua.net.nlp/morfologik-ukrainian-search = ${ua.net.nlp.morfologik-ukrainian-search.version} -/xerces/xercesImpl = 2.9.1 \ No newline at end of file +/xerces/xercesImpl = 2.9.1 From e627a3d28dea962ad7a883ebbbcd05ebe1cbefb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 12:30:33 +0100 Subject: [PATCH 31/88] Sanitize refguide docs. --- .../src/basic-authentication-plugin.adoc | 2 +- .../src/jwt-authentication-plugin.adoc | 24 ++++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc index f6c872e4a044..e3cd24ff1d21 100644 --- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc @@ -81,7 +81,7 @@ There are a few things to keep in mind when using the Basic authentication plugi * A user who has access to write permissions to `security.json` will be able to modify all the permissions and how users have been assigned permissions. Special care should be taken to only grant access to editing security to appropriate users. * Your network should, of course, be secure. Even with Basic authentication enabled, you should not unnecessarily expose Solr to the outside world. -== Editing Authentication Plugin Configuration +== Editing Basic Authentication Plugin Configuration An Authentication API allows modifying user IDs and passwords. The API provides an endpoint with specific commands to set user details or delete a user. diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 9dd284743c24..41cf22db04fd 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -35,7 +35,7 @@ The simplest possible `security.json` for registering the plugin without configu } ---- -The plugin will NOT block anonymous traffic in this mode, since the default for `block_unknown` is false. It is then possible to start configuring the plugin using REST API calls. +The plugin will NOT block anonymous traffic in this mode, since the default for `blockUnknown` is false. It is then possible to start configuring the plugin using REST API calls. == Configuration parameters @@ -47,10 +47,10 @@ wellKnownUrl ; URL to an https://openid.net/specs/openid-connect-discove clientId ; Client identifier for use with OpenID Connect ; (no default value) Required to authenticate with Admin UI realm ; Name of the authentication realm to echo back in HTTP 401 responses. Will also be displayed in Admin UI login page ; 'solr-jwt' scope ; Whitespace separated list of valid scopes. If configured, the JWT access token MUST contain a `scope` claim with at least one of the listed scopes. Example: `solr:read solr:admin` ; -jwkUrl ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. Not required if `well_known_url` is provided ; Auto configured if `well_known_url` is provided -jwk ; As an alternative to `jwk_url` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `well_known_url` is provided -iss ; Validates that `iss` (issuer) equals this string ; If `well_known_url` is provided, use `issuer` from there. -aud ; Validates that `aud` (audience) equals this string ; If `client_id` is configured, require `aud` to match it +jwkUrl ; An https URL to a https://tools.ietf.org/html/rfc7517[JWK] keys file. ; Auto configured if `wellKnownUrl` is provided +jwk ; As an alternative to `jwkUrl` you may provide a JSON object here containing the public key(s) of the issuer. ; Auto configured if `wellKnownUrl` is provided +iss ; Validates that `iss` (issuer) equals this string ; Auto configured if `wellKnownUrl` is provided +aud ; Validates that `aud` (audience) equals this string ; If `clientId` is configured, require `aud` to match it requireSub ; Makes `sub` (subject) mandatory ; `true` requireExp ; Makes `exp` (expiry time) mandatory ; `true` algWhitelist ; JSON array with algorithms to accept: `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512`, `none ; Default is to allow all algorithms @@ -58,7 +58,7 @@ jwkCacheDur ; Duration of JWK cache in seconds ; principalClaim ; What claim id to pull principal from ; '`sub`' claimsMatch ; JSON object of claims (key) that must match a regular expression (value). Example: `{ "foo" : "A|B" }` will require the `foo` claim to be either "A" or "B". ; (none) adminUiScope ; Define what scope is requested when logging in from Admin UI ; If not defined, the first scope from `scope` parameter is used -authorizationEndpoint; The URL for the Id Provider's authorization endpoint ; If `well_known_url` is provided, uses `authorization_endpoint` from there. +authorizationEndpoint; The URL for the Id Provider's authorization endpoint ; Auto configured if `wellKnownUrl` is provided redirectUris ; Valid location(s) for redirect after external authentication. Takes a string or array of strings. Must be the base URL of Solr, e.g. https://solr1.example.com:8983/solr/ and must match the list of redirect URIs registered with the Identity Provider beforehand. ; Defaults to empty list, i.e. any node is assumed to be a valid redirect target. |=== @@ -93,7 +93,7 @@ The next example shows configuring using https://openid.net/specs/openid-connect } ---- -In this case, `jwk_url`, `iss` and `authorizationEndpoint` will be automatically configured from the fetched configuration. +In this case, `jwkUrl`, `iss` and `authorizationEndpoint` will be automatically configured from the fetched configuration. === Complex example Let's look at a more complex configuration, this time with a static embedded JWK: @@ -131,13 +131,13 @@ Let's comment on this config: <6> The audience claim must match "https://example.com/solr" <7> Fetch the user id from another claim than the default `sub` <8> Require that the `roles` claim is one of "A" or "B" and that the `dept` claim is "IT" -<9> Require one of the scopes `solr:read`, `solr:write` or `solr:amin` +<9> Require one of the scopes `solr:read`, `solr:write` or `solr:admin` <10> Only accept RSA algorithms for signatures == Editing JWT Authentication Plugin Configuration -All properties mentioned above can be set or changed using the <>. You can thus start with a simple configuration with only `class` configured and then configure the rest using the API. +All properties mentioned above can be set or changed using the Config Edit API. You can thus start with a simple configuration with only `class` configured and then configure the rest using the API. === Set a config Property @@ -187,11 +187,7 @@ curl -H "Authorization: Bearer xxxxxx.xxxxxx.xxxxxx" http://localhost:8983/solr/ === Admin UI -The Admin UI runs as a SPA (single page application) in your browser, and talks with the Solr APIs just like any other client. So logging in to the Admin UI in this context simply means to obtain a valid JWT token which can be passed to Solr. - - - -When this plugin is enabled, users will be redirected to a login page in the Admin UI once they attempt to do a restricted action. The page has a button that user will click and be redirected to the Identity Provider's login page. Once authenticated the user will be redirected back to Solr Admin UI to the last known location. The session will last as long as the JWT token expiry time. There is also a logout menu in the left column where user can explicitly log out. +When this plugin is enabled, users will be redirected to a login page in the Admin UI once they attempt to do a restricted action. The page has a button that users will click and be redirected to the Identity Provider's login page. Once authenticated, the user will be redirected back to Solr Admin UI to the last known location. The session will last as long as the JWT token expiry time and is valid for one Solr server only. That means you have to login again when navigating to another Solr node. There is also a logout menu in the left column where user can explicitly log out. == Using the Solr Control Script with JWT Auth From f506da959b2b086a27ae0aeb640080e9f7570b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 12:32:39 +0100 Subject: [PATCH 32/88] Add JWT in list of supported plugins for Admin UI login --- .../src/authentication-and-authorization-plugins.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc index e6df62230357..f0c91216ce31 100644 --- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc +++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc @@ -164,7 +164,8 @@ Whenever an authentication plugin is enabled, authentication is also required fo When authentication is required the Admin UI will presented you with a login dialogue. The authentication plugins currently supported by the Admin UI are: -* `BasicAuthPlugin` +* <> +* <> If your plugin of choice is not supported, you will have to interact with Solr sending HTTP requests instead of through the graphical user interface of the Admin UI. All operations supported by Admin UI can be performed through Solr's RESTful APIs. From e866ffba9e4f6ddc002fb520230bdc43bea56e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 12:38:11 +0100 Subject: [PATCH 33/88] Remove nocommit for nonce --- solr/webapp/web/js/angular/controllers/login.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/solr/webapp/web/js/angular/controllers/login.js b/solr/webapp/web/js/angular/controllers/login.js index 672641e73113..eca491c259b0 100644 --- a/solr/webapp/web/js/angular/controllers/login.js +++ b/solr/webapp/web/js/angular/controllers/login.js @@ -73,9 +73,7 @@ solrAdminApp.controller('LoginController', if (accessToken.length === 3) { var payload = AuthenticationService.decodeJwtPart(accessToken[1]); if (!payload['nonce'] || payload['nonce'] !== sessionStorage.getItem("auth.nonce")) { - // NOCOMMIT: Accept no nonce for now with demo IdP - console.log("Temporarily disregarding missing nonce"); - // errorText += "Invalid 'nonce' value, possible attack detected. Please log in again. "; + errorText += "Invalid 'nonce' value, possible attack detected. Please log in again. "; } if (errorText === "") { From a5863d7d9adbebe1b2760f79a3d566c6df8d102d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 12:41:34 +0100 Subject: [PATCH 34/88] Fix nocommit for V2 API spec docs --- solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index eceac623fb0d..dcc52978f185 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -492,10 +492,9 @@ public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) return builder; } - // NOCOMMIT: v2 api documentation @Override public ValidatingJsonMap getSpec() { - return Utils.getSpec("cluster.security.BasicAuth.Commands").getSpec(); + return Utils.getSpec("cluster.security.JwtAuth.Commands").getSpec(); } /** From 18417724dd713df2ab7bab5e0a7c5cb220da3836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 12:49:17 +0100 Subject: [PATCH 35/88] Harden test, wait for collection creation, move collection name to constant, adapt asserts --- .../JWTAuthPluginIntegrationTest.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 03ae5416c30c..41f0f954cad6 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -55,6 +55,7 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudTestCase { protected static final int NUM_SERVERS = 2; protected static final int NUM_SHARDS = 2; protected static final int REPLICATION_FACTOR = 1; + private static final String COLLECTION = "jwtColl"; private static String jwtTestToken; private static String baseUrl; private static AtomicInteger jwtInterceptCount = new AtomicInteger(); @@ -123,26 +124,27 @@ public void infoRequestWithToken() throws IOException { @Test public void createCollectionUpdateAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed - assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=mycoll&numShards=2", jwtTestToken).second().intValue()); - - // Now update two documents - Pair result = post(baseUrl + "mycoll/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); + assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=" + COLLECTION + "&numShards=2", jwtTestToken).second().intValue()); + cluster.waitForActiveCollection(COLLECTION, 2, 2); + + // Now update three documents + Pair result = post(baseUrl + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(2,2); + verifyInterRequestHeaderCounts(1,1); // First a non distributed query - result = get(baseUrl + "mycoll/query?q=*:*&distrib=false", jwtTestToken); + result = get(baseUrl + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(2,2); + verifyInterRequestHeaderCounts(1,1); // Now do a distributed query, using JWTAUth for inter-node - result = get(baseUrl + "mycoll/query?q=*:*", jwtTestToken); + result = get(baseUrl + COLLECTION + "/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(6,6); + verifyInterRequestHeaderCounts(5,5); // Delete - assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=mycoll", jwtTestToken).second().intValue()); - verifyInterRequestHeaderCounts(6,6); + assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); + verifyInterRequestHeaderCounts(5,5); } private void verifyInterRequestHeaderCounts(int jwt, int pki) { From 145678a6755cfaad63ff716a47c638af5a881feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 13:05:07 +0100 Subject: [PATCH 36/88] Require https for jwk --- solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index dcc52978f185..6b8fd683944b 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -641,8 +641,7 @@ public static class WellKnownDiscoveryConfig { public static WellKnownDiscoveryConfig parse(String urlString) { try { URL url = new URL(urlString); - // NOCOMMIT - require HTTPS - if (!Arrays.asList("http", "https", "file").contains(url.getProtocol())) { + if (!Arrays.asList("https", "file").contains(url.getProtocol())) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Well-known config URL must be HTTPS or file"); } return parse(url.openStream()); From e850052ef2dc8f0f6304bf3055eba80e69f12005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 13:08:37 +0100 Subject: [PATCH 37/88] Require https for jwk --- .../org/apache/solr/security/JWTAuthPlugin.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 6b8fd683944b..97cf9e2ecbd6 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -263,15 +263,14 @@ private void initJwk(Map pluginConfig) { private void setupJwkUrl(String url) { // The HttpsJwks retrieves and caches keys from a the given HTTPS JWKS endpoint. -// NOCOMMIT: Disable https requirement for now -// try { -// URL jwkUrl = new URL(url); -// if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { -// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be an HTTPS url"); -// } -// } catch (MalformedURLException e) { -// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be a valid URL"); -// } + try { + URL jwkUrl = new URL(url); + if (!"https".equalsIgnoreCase(jwkUrl.getProtocol())) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be an HTTPS url"); + } + } catch (MalformedURLException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, PARAM_JWK_URL + " must be a valid URL"); + } HttpsJwks httpsJkws = new HttpsJwks(url); httpsJkws.setDefaultCacheDuration(jwkCacheDuration); verificationKeyResolver = new HttpsJwksVerificationKeyResolver(httpsJkws); From df22ff223382df0aa1dbfdf34791c04ea24a0044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 13:42:36 +0100 Subject: [PATCH 38/88] Implement interceptInternodeRequest and get rid of custom interceptor --- .../apache/solr/security/JWTAuthPlugin.java | 57 +++++-------------- 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 97cf9e2ecbd6..ff0ba580a43f 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -82,7 +82,7 @@ /** * Authenticaion plugin that finds logged in user by validating the signature of a JWT token */ -public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin, SpecProvider, ConfigEditablePlugin { +public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider, ConfigEditablePlugin { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String PARAM_BLOCK_UNKNOWN = "blockUnknown"; private static final String PARAM_JWK_URL = "jwkUrl"; @@ -104,8 +104,8 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements HttpClientBui private static final String AUTH_REALM = "solr-jwt"; private static final String CLAIM_SCOPE = "scope"; + private static final long RETRY_INIT_DELAY_SECONDS = 30; - private final JwtPkiDelegationInterceptor interceptor = new JwtPkiDelegationInterceptor(); private static final Set PROPS = ImmutableSet.of(PARAM_BLOCK_UNKNOWN, PARAM_JWK_URL, PARAM_JWK, PARAM_ISSUER, PARAM_AUDIENCE, PARAM_REQUIRE_SUBJECT, PARAM_PRINCIPAL_CLAIM, PARAM_REQUIRE_EXPIRATIONTIME, PARAM_ALG_WHITELIST, PARAM_JWK_CACHE_DURATION, PARAM_CLAIMS_MATCH, PARAM_SCOPE, PARAM_CLIENT_ID, PARAM_WELL_KNOWN_URL, @@ -306,8 +306,8 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser return true; } // Retry config - if (lastInitTime.plusSeconds(10).isAfter(Instant.now())) { - log.info("Retrying JWTAuthPlugin initialization"); + if (lastInitTime.plusSeconds(RETRY_INIT_DELAY_SECONDS).isAfter(Instant.now())) { + log.info("Retrying JWTAuthPlugin initialization (retry delay={}s)", RETRY_INIT_DELAY_SECONDS); init(pluginConfig); } if (jwtConsumer == null) { @@ -474,21 +474,7 @@ private void initConsumer() { @Override public void close() throws IOException { - HttpClientUtil.removeRequestInterceptor(interceptor); - } - - /** - * Register an interceptor to be able to add our header to inter-node requests - * @param builder any existing builder or null to create a new one - * @return Returns an instance of a SolrHttpClientBuilder to be used for configuring the - * HttpClients for use with SolrJ clients. - * @lucene.experimental - */ - @Override - public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { - // Register interceptor for inter-node requests, that delegates to PKI if JWTPrincipal is not found on http context - HttpClientUtil.addRequestInterceptor(interceptor); - return builder; + jwtConsumer = null; } @Override @@ -690,31 +676,16 @@ public List getResponseTypesSupported() { } } - /** - * The interceptor class that adds correct header or delegates to PKI. - */ - public class JwtPkiDelegationInterceptor implements HttpRequestInterceptor { - private final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Override - public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { - if (context instanceof HttpClientContext) { - HttpClientContext httpClientContext = (HttpClientContext) context; - if (httpClientContext.getUserToken() instanceof JWTPrincipal) { - JWTPrincipal jwtPrincipal = (JWTPrincipal) httpClientContext.getUserToken(); - request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.getToken()); - log.debug("Set JWT header on inter-node request"); - return; - } - } - - if (coreContainer.getPkiAuthenticationPlugin() != null) { - log.debug("Inter-node request delegated from JWTAuthPlugin to PKIAuthenticationPlugin"); - coreContainer.getPkiAuthenticationPlugin().setHeader(request); - } else { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "JWTAuthPlugin wants to delegate inter-node request to PKI, but PKI plugin was not initialized"); + @Override + protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) { + if (httpContext instanceof HttpClientContext) { + HttpClientContext httpClientContext = (HttpClientContext) httpContext; + if (httpClientContext.getUserToken() instanceof JWTPrincipal) { + JWTPrincipal jwtPrincipal = (JWTPrincipal) httpClientContext.getUserToken(); + httpRequest.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.getToken()); + return true; } } + return false; } } From 1ca2b9a25584bd73e014a9c92e537a142b810f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 13:59:20 +0100 Subject: [PATCH 39/88] Validate metric counts --- .../solr/security/JWTAuthPluginIntegrationTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 41f0f954cad6..bcc962a9164e 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -35,7 +35,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.protocol.HttpContext; import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.SolrCloudAuthTestCase; import org.apache.solr.common.util.Pair; import org.jose4j.jwk.PublicJsonWebKey; import org.jose4j.jwk.RsaJsonWebKey; @@ -51,7 +51,7 @@ * Validate that JWT token authentication works in a real cluster. * TODO: Test also using SolrJ as client. But that requires a way to set Authorization header on request */ -public class JWTAuthPluginIntegrationTest extends SolrCloudTestCase { +public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SERVERS = 2; protected static final int NUM_SHARDS = 2; protected static final int REPLICATION_FACTOR = 1; @@ -128,23 +128,30 @@ public void createCollectionUpdateAndQueryDistributed() throws Exception { cluster.waitForActiveCollection(COLLECTION, 2, 2); // Now update three documents + assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); Pair result = post(baseUrl + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(1,1); - + assertAuthMetricsMinimums(3, 0, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); + // First a non distributed query result = get(baseUrl + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(1,1); + assertAuthMetricsMinimums(4, 0, 0, 0, 0, 0); // Now do a distributed query, using JWTAUth for inter-node result = get(baseUrl + COLLECTION + "/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(5,5); + assertAuthMetricsMinimums(5, 0, 0, 0, 0, 0); // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); verifyInterRequestHeaderCounts(5,5); + assertAuthMetricsMinimums(10, 0, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); } private void verifyInterRequestHeaderCounts(int jwt, int pki) { From bc748ea8f29a6761e738bddfdd4b4d309b0fbfef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 14:05:39 +0100 Subject: [PATCH 40/88] Extra wait for nodes in before() --- .../apache/solr/security/JWTAuthPluginIntegrationTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index bcc962a9164e..bf2751c19a71 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -24,6 +24,7 @@ import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -104,9 +105,10 @@ public static void tearDownClass() throws Exception { } @Before - public void before() throws IOException, InterruptedException { + public void before() throws IOException, InterruptedException, TimeoutException { jwtInterceptCount.set(0); pkiInterceptCount.set(0); + cluster.waitForAllNodes(10); } @Test(expected = IOException.class) From 344fae2bcfb30d66cf853f084c403a8e4c5b17ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 14:11:47 +0100 Subject: [PATCH 41/88] Remove need for CoreContainer --- .../org/apache/solr/security/JWTAuthPlugin.java | 17 +++++------------ .../apache/solr/security/JWTAuthPluginTest.java | 6 +++--- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index ff0ba580a43f..64e30ab1a636 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -45,14 +45,10 @@ import java.util.stream.Collectors; import com.google.common.collect.ImmutableSet; -import org.apache.http.HttpException; import org.apache.http.HttpHeaders; import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.protocol.HttpContext; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.StringUtils; @@ -60,7 +56,6 @@ import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; -import org.apache.solr.core.CoreContainer; import org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode; import org.jose4j.jwa.AlgorithmConstraints; import org.jose4j.jwk.HttpsJwks; @@ -128,19 +123,15 @@ public class JWTAuthPlugin extends AuthenticationPlugin implements SpecProvider, private String confIdpConfigUrl; private Map pluginConfig; private Instant lastInitTime = Instant.now(); - private CoreContainer coreContainer; private String authorizationEndpoint; private String adminUiScope; private List redirectUris; /** - * Initialize plugin with core container, this method is chosen by reflection at create time - * @param coreContainer instance of core container + * Initialize plugin */ - public JWTAuthPlugin(CoreContainer coreContainer) { - this.coreContainer = coreContainer; - } + public JWTAuthPlugin() {} @Override public void init(Map pluginConfig) { @@ -296,7 +287,9 @@ JsonWebKeySet parseJwkSet(Map jwkObj) throws JoseException { public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws Exception { HttpServletRequest request = (HttpServletRequest) servletRequest; HttpServletResponse response = (HttpServletResponse) servletResponse; - + + // NOCOMMIT: increment metric counters + String header = request.getHeader(HttpHeaders.AUTHORIZATION); if (jwtConsumer == null) { diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java index ad0935bc5e92..72a908edbb32 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginTest.java @@ -108,7 +108,7 @@ static JwtClaims generateClaims() { public void setUp() throws Exception { super.setUp(); // Create an auth plugin - plugin = new JWTAuthPlugin(null); + plugin = new JWTAuthPlugin(); // Create a JWK config for security.json testJwk = new HashMap<>(); @@ -167,7 +167,7 @@ public void initFromSecurityJSONUrlJwk() throws Exception { public void initWithJwk() { HashMap authConf = new HashMap<>(); authConf.put("jwk", testJwk); - plugin = new JWTAuthPlugin(null); + plugin = new JWTAuthPlugin(); plugin.init(authConf); } @@ -175,7 +175,7 @@ public void initWithJwk() { public void initWithJwkUrl() { HashMap authConf = new HashMap<>(); authConf.put("jwkUrl", "https://127.0.0.1:9999/foo.jwk"); - plugin = new JWTAuthPlugin(null); + plugin = new JWTAuthPlugin(); plugin.init(authConf); } From befd365d43d05dc9482bf2149d706f4af068d251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 14:13:02 +0100 Subject: [PATCH 42/88] Reference SOLR-13070 --- .../org/apache/solr/security/JWTAuthPluginIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index bf2751c19a71..fd852e0c7b5c 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -50,7 +50,7 @@ /** * Validate that JWT token authentication works in a real cluster. - * TODO: Test also using SolrJ as client. But that requires a way to set Authorization header on request + * TODO: Test also using SolrJ as client. But that requires a way to set Authorization header on request, see SOLR-13070 */ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SERVERS = 2; From f6401c1ebd3f08856786050df5c4902a2a7b8e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 14:31:37 +0100 Subject: [PATCH 43/88] Added metrics counting, and testing of metrics Still need to test various error situations, missing token etc --- .../org/apache/solr/security/JWTAuthPlugin.java | 11 +++++++++-- .../security/JWTAuthPluginIntegrationTest.java | 16 ++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 64e30ab1a636..038cc023bf38 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -288,14 +288,13 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser HttpServletRequest request = (HttpServletRequest) servletRequest; HttpServletResponse response = (HttpServletResponse) servletResponse; - // NOCOMMIT: increment metric counters - String header = request.getHeader(HttpHeaders.AUTHORIZATION); if (jwtConsumer == null) { if (header == null && !blockUnknown) { log.info("JWTAuth not configured, but allowing anonymous access since {}==false", PARAM_BLOCK_UNKNOWN); filterChain.doFilter(request, response); + numPassThrough.inc();; return true; } // Retry config @@ -305,6 +304,7 @@ public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse ser } if (jwtConsumer == null) { log.warn("JWTAuth not configured"); + numErrors.mark(); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin not correctly configured"); } } @@ -319,21 +319,25 @@ public Principal getUserPrincipal() { } }; if (!(authResponse.getPrincipal() instanceof JWTPrincipal)) { + numErrors.mark(); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "JWTAuth plugin says AUTHENTICATED but no token extracted"); } if (log.isDebugEnabled()) log.debug("Authentication SUCCESS"); filterChain.doFilter(wrapper, response); + numAuthenticated.inc(); return true; case PASS_THROUGH: if (log.isDebugEnabled()) log.debug("Unknown user, but allow due to {}=false", PARAM_BLOCK_UNKNOWN); filterChain.doFilter(request, response); + numPassThrough.inc(); return true; case AUTZ_HEADER_PROBLEM: authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_BAD_REQUEST, BearerWwwAuthErrorCode.invalid_request); + numErrors.mark(); return false; case CLAIM_MISMATCH: @@ -345,15 +349,18 @@ public Principal getUserPrincipal() { log.warn("Exception: {}", authResponse.getJwtException().getMessage()); } authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.invalid_token); + numWrongCredentials.inc(); return false; case SCOPE_MISSING: authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, BearerWwwAuthErrorCode.insufficient_scope); + numWrongCredentials.inc(); return false; case NO_AUTZ_HEADER: default: authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_UNAUTHORIZED, null); + numMissingCredentials.inc(); return false; } } diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index fd852e0c7b5c..09d438177f34 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -69,8 +69,7 @@ public static void setupClass() throws Exception { .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json")) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) .configure(); - String hostport = cluster.getSolrClient().getClusterStateProvider().getLiveNodes().iterator().next().split("_")[0]; - baseUrl = "http://" + hostport + "/solr/"; + baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); String jwkJSON = "{\n" + " \"kty\": \"RSA\",\n" + @@ -123,6 +122,11 @@ public void infoRequestWithToken() throws IOException { verifyInterRequestHeaderCounts(0,0); } + @Test + public void testMetrics() { + // NOCOMMIT: Metrics tests, with failing requests etc + } + @Test public void createCollectionUpdateAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed @@ -134,25 +138,25 @@ public void createCollectionUpdateAndQueryDistributed() throws Exception { Pair result = post(baseUrl + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(1,1); - assertAuthMetricsMinimums(3, 0, 0, 0, 0, 0); + assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); // First a non distributed query result = get(baseUrl + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(1,1); - assertAuthMetricsMinimums(4, 0, 0, 0, 0, 0); + assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); // Now do a distributed query, using JWTAUth for inter-node result = get(baseUrl + COLLECTION + "/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(5,5); - assertAuthMetricsMinimums(5, 0, 0, 0, 0, 0); + assertAuthMetricsMinimums(5, 5, 0, 0, 0, 0); // Delete assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); verifyInterRequestHeaderCounts(5,5); - assertAuthMetricsMinimums(10, 0, 0, 0, 0, 0); + assertAuthMetricsMinimums(10, 10, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); } From 05500b655b773ac89adacd31eed684b79d7c1b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 13 Dec 2018 14:38:54 +0100 Subject: [PATCH 44/88] Mention OIDC implicit flow in CHANGES --- solr/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a9f6f66ddfac..d6d77d224475 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -83,7 +83,7 @@ New Features The BasicAuth plugin now supports a new parameter 'forwardCredentials', and when set to 'true', user's BasicAuth credentials will be used instead of PKI for client initiated internode requests. (janhoy, noble) -* SOLR-12121: JWT Token authentication plugin (janhoy) +* SOLR-12121: JWT Token authentication plugin with OpenID Connect implicit flow login through Admin UI (janhoy) * SOLR-12791: Add Metrics reporting for AuthenticationPlugin (janhoy) From ab5001bdf1d6c016ca588b87d1bf6c575a82444f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 14 Dec 2018 10:53:24 +0100 Subject: [PATCH 45/88] Fix paths --- .../security/JWTAuthPluginIntegrationTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 09d438177f34..59d9c2a2d759 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -112,12 +112,12 @@ public void before() throws IOException, InterruptedException, TimeoutException @Test(expected = IOException.class) public void infoRequestWithoutToken() throws Exception { - get(baseUrl + "admin/info/system", null); + get(baseUrl + "/admin/info/system", null); } @Test public void infoRequestWithToken() throws IOException { - Pair result = get(baseUrl + "admin/info/system", jwtTestToken); + Pair result = get(baseUrl + "/admin/info/system", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(0,0); } @@ -130,31 +130,31 @@ public void testMetrics() { @Test public void createCollectionUpdateAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed - assertEquals(200, get(baseUrl + "admin/collections?action=CREATE&name=" + COLLECTION + "&numShards=2", jwtTestToken).second().intValue()); + assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + COLLECTION + "&numShards=2", jwtTestToken).second().intValue()); cluster.waitForActiveCollection(COLLECTION, 2, 2); // Now update three documents assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); - Pair result = post(baseUrl + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); + Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(1,1); assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); // First a non distributed query - result = get(baseUrl + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); + result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(1,1); assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); // Now do a distributed query, using JWTAUth for inter-node - result = get(baseUrl + COLLECTION + "/query?q=*:*", jwtTestToken); + result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); verifyInterRequestHeaderCounts(5,5); assertAuthMetricsMinimums(5, 5, 0, 0, 0, 0); // Delete - assertEquals(200, get(baseUrl + "admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); + assertEquals(200, get(baseUrl + "/admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); verifyInterRequestHeaderCounts(5,5); assertAuthMetricsMinimums(10, 10, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); From 0de4cf9adf7f58d3aba62b60c701881929bcd898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 14 Dec 2018 10:53:37 +0100 Subject: [PATCH 46/88] Redact tokens in toString --- solr/core/src/java/org/apache/solr/security/JWTPrincipal.java | 2 +- .../org/apache/solr/security/JWTPrincipalWithUserRoles.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java b/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java index eaa14e4a72d5..737f3fa8e4a8 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java +++ b/solr/core/src/java/org/apache/solr/security/JWTPrincipal.java @@ -81,7 +81,7 @@ public int hashCode() { public String toString() { return "JWTPrincipal{" + "username='" + username + '\'' + - ", token='" + token + '\'' + + ", token='" + "*****" + '\'' + ", claims=" + claims + '}'; } diff --git a/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java b/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java index ecad497929c9..850dc1f89647 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java +++ b/solr/core/src/java/org/apache/solr/security/JWTPrincipalWithUserRoles.java @@ -63,7 +63,7 @@ public int hashCode() { public String toString() { return "JWTPrincipalWithUserRoles{" + "username='" + username + '\'' + - ", token='" + token + '\'' + + ", token='" + "*****" + '\'' + ", claims=" + claims + ", roles=" + roles + '}'; From 6194fc37632983f49f0c2da1995b5a111fbed565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 14 Dec 2018 10:56:03 +0100 Subject: [PATCH 47/88] Remove unused system prop --- .../org/apache/solr/security/JWTAuthPluginIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 59d9c2a2d759..162192008228 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -99,7 +99,6 @@ public static void setupClass() throws Exception { @AfterClass public static void tearDownClass() throws Exception { - System.clearProperty("java.security.auth.login.config"); shutdownCluster(); } From f2198dfd63658439fc26dfc34c13cd5f6ec63c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 14:10:12 +0100 Subject: [PATCH 48/88] Rearrange tests and get them to pass --- .../security/BasicAuthIntegrationTest.java | 71 +-------- .../security/BasicAuthStandaloneTest.java | 2 +- .../JWTAuthPluginIntegrationTest.java | 143 ++++++++++-------- .../solr/cloud/SolrCloudAuthTestCase.java | 82 ++++++++++ 4 files changed, 171 insertions(+), 127 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index 7bf38cec1951..fb4a363f41f7 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -16,9 +16,6 @@ */ package org.apache.solr.security; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Collections.singletonMap; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -27,23 +24,16 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Random; import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; -import org.apache.commons.io.IOUtils; import com.codahale.metrics.MetricRegistry; +import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.message.AbstractHttpMessage; -import org.apache.http.message.BasicHeader; -import org.apache.http.util.EntityUtils; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; @@ -65,9 +55,7 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.Base64; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.util.SolrCLI; import org.junit.After; @@ -76,6 +64,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.singletonMap; + public class BasicAuthIntegrationTest extends SolrCloudAuthTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -164,7 +155,7 @@ public void testBasicAuth() throws Exception { "}"; HttpPost httpPost = new HttpPost(baseUrl + authcPrefix); - setBasicAuthHeader(httpPost, "solr", "SolrRocks"); + setAuthorizationHeader(httpPost, makeBasicAuthHeader("solr", "SolrRocks")); httpPost.setEntity(new ByteArrayEntity(command.getBytes(UTF_8))); httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication.enabled", "true", 20); @@ -342,7 +333,7 @@ public static void executeCommand(String url, HttpClient cl, String payload, Str HttpPost httpPost; HttpResponse r; httpPost = new HttpPost(url); - setBasicAuthHeader(httpPost, user, pwd); + setAuthorizationHeader(httpPost, makeBasicAuthHeader(user, pwd)); httpPost.setEntity(new ByteArrayEntity(payload.getBytes(UTF_8))); httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); r = cl.execute(httpPost); @@ -352,54 +343,6 @@ public static void executeCommand(String url, HttpClient cl, String payload, Str Utils.consumeFully(r.getEntity()); } - public static void verifySecurityStatus(HttpClient cl, String url, String objPath, - Object expected, int count) throws Exception { - verifySecurityStatus(cl, url, objPath, expected, count, null, null); - } - - - public static void verifySecurityStatus(HttpClient cl, String url, String objPath, - Object expected, int count, String user, String pwd) - throws Exception { - boolean success = false; - String s = null; - List hierarchy = StrUtils.splitSmart(objPath, '/'); - for (int i = 0; i < count; i++) { - HttpGet get = new HttpGet(url); - if (user != null) setBasicAuthHeader(get, user, pwd); - HttpResponse rsp = cl.execute(get); - s = EntityUtils.toString(rsp.getEntity()); - Map m = null; - try { - m = (Map) Utils.fromJSONString(s); - } catch (Exception e) { - fail("Invalid json " + s); - } - Utils.consumeFully(rsp.getEntity()); - Object actual = Utils.getObjectByPath(m, true, hierarchy); - if (expected instanceof Predicate) { - Predicate predicate = (Predicate) expected; - if (predicate.test(actual)) { - success = true; - break; - } - } else if (Objects.equals(actual == null ? null : String.valueOf(actual), expected)) { - success = true; - break; - } - Thread.sleep(50); - } - assertTrue("No match for " + objPath + " = " + expected + ", full response = " + s, success); - - } - - public static void setBasicAuthHeader(AbstractHttpMessage httpMsg, String user, String pwd) { - String userPass = user + ":" + pwd; - String encoded = Base64.byteArrayToBase64(userPass.getBytes(UTF_8)); - httpMsg.setHeader(new BasicHeader("Authorization", "Basic " + encoded)); - log.info("Added Basic Auth security Header {}",encoded ); - } - public static Replica getRandomReplica(DocCollection coll, Random random) { ArrayList l = new ArrayList<>(); @@ -412,8 +355,6 @@ public static Replica getRandomReplica(DocCollection coll, Random random) { return l.isEmpty() ? null : l.get(0); } - protected static final Predicate NOT_NULL_PREDICATE = o -> o != null; - //the password is 'SolrRocks' //this could be generated everytime. But , then we will not know if there is any regression protected static final String STD_CONF = "{\n" + diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java index 1cfd68173b86..381a1fb38cfc 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java @@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.solr.security.BasicAuthIntegrationTest.NOT_NULL_PREDICATE; +import static org.apache.solr.cloud.SolrCloudAuthTestCase.NOT_NULL_PREDICATE; import static org.apache.solr.security.BasicAuthIntegrationTest.STD_CONF; import static org.apache.solr.security.BasicAuthIntegrationTest.verifySecurityStatus; diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 162192008228..c8d5a35ee03e 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -24,47 +24,58 @@ import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import org.apache.http.Header; -import org.apache.http.HttpException; +import org.apache.commons.io.IOUtils; import org.apache.http.HttpHeaders; -import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; -import org.apache.http.protocol.HttpContext; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.cloud.SolrCloudAuthTestCase; import org.apache.solr.common.util.Pair; +import org.apache.solr.common.util.Utils; import org.jose4j.jwk.PublicJsonWebKey; import org.jose4j.jwk.RsaJsonWebKey; import org.jose4j.jws.AlgorithmIdentifiers; import org.jose4j.jws.JsonWebSignature; import org.jose4j.jwt.JwtClaims; -import org.junit.AfterClass; +import org.jose4j.lang.JoseException; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Validate that JWT token authentication works in a real cluster. - * TODO: Test also using SolrJ as client. But that requires a way to set Authorization header on request, see SOLR-13070 + *

+ * TODO: + *

    + *
  • Test also using SolrJ as client. But that requires a way to set Authorization header on request, see SOLR-13070
    • + *
  • This is also the reason we use {@link org.apache.solr.SolrTestCaseJ4.SuppressSSL} annotation, since we use HttpUrlConnection
    • + *
+ *

*/ +@SolrTestCaseJ4.SuppressSSL public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { protected static final int NUM_SERVERS = 2; protected static final int NUM_SHARDS = 2; protected static final int REPLICATION_FACTOR = 1; - private static final String COLLECTION = "jwtColl"; - private static String jwtTestToken; - private static String baseUrl; - private static AtomicInteger jwtInterceptCount = new AtomicInteger(); - private static AtomicInteger pkiInterceptCount = new AtomicInteger(); - private static final CountInterceptor interceptor = new CountInterceptor(); - - @BeforeClass - public static void setupClass() throws Exception { + private final String COLLECTION = "jwtColl"; + private String jwtTestToken; + private String baseUrl; + private JsonWebSignature jws; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + configureCluster(NUM_SERVERS)// nodes .withSecurityJson(TEST_PATH().resolve("security").resolve("jwt_plugin_jwk_security.json")) .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) @@ -83,7 +94,7 @@ public static void setupClass() throws Exception { PublicJsonWebKey jwk = RsaJsonWebKey.Factory.newPublicJwk(jwkJSON); JwtClaims claims = JWTAuthPluginTest.generateClaims(); - JsonWebSignature jws = new JsonWebSignature(); + jws = new JsonWebSignature(); jws.setPayload(claims.toJson()); jws.setKey(jwk.getPrivateKey()); jws.setKeyIdHeaderValue(jwk.getKeyId()); @@ -91,22 +102,14 @@ public static void setupClass() throws Exception { jwtTestToken = jws.getCompactSerialization(); - HttpClientUtil.removeRequestInterceptor(interceptor); - HttpClientUtil.addRequestInterceptor(interceptor); - cluster.waitForAllNodes(10); } - @AfterClass - public static void tearDownClass() throws Exception { + @Override + @After + public void tearDown() throws Exception { shutdownCluster(); - } - - @Before - public void before() throws IOException, InterruptedException, TimeoutException { - jwtInterceptCount.set(0); - pkiInterceptCount.set(0); - cluster.waitForAllNodes(10); + super.tearDown(); } @Test(expected = IOException.class) @@ -118,50 +121,62 @@ public void infoRequestWithoutToken() throws Exception { public void infoRequestWithToken() throws IOException { Pair result = get(baseUrl + "/admin/info/system", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(0,0); } @Test - public void testMetrics() { + public void testMetrics() throws Exception { + boolean isUseV2Api = random().nextBoolean(); + String authcPrefix = "/admin/authentication"; + String authzPrefix = "/admin/authorization"; + if(isUseV2Api){ + authcPrefix = "/____v2/cluster/security/authentication"; + authzPrefix = "/____v2/cluster/security/authorization"; + } + String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + CloseableHttpClient cl = HttpClientUtil.createClient(null); + // NOCOMMIT: Metrics tests, with failing requests etc + createCollection(COLLECTION); + + executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: false}}", jws); + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "false", 20, jws); + verifySecurityStatus(cl, baseUrl + "/admin/info/key", "key", NOT_NULL_PREDICATE, 20); + assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); + + // Now update three documents + assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); + Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); + assertEquals(Integer.valueOf(200), result.second()); + assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); } @Test public void createCollectionUpdateAndQueryDistributed() throws Exception { // Admin request will use PKI inter-node auth from Overseer, and succeed - assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + COLLECTION + "&numShards=2", jwtTestToken).second().intValue()); - cluster.waitForActiveCollection(COLLECTION, 2, 2); + createCollection(COLLECTION); // Now update three documents assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(1,1); - assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); + assertAuthMetricsMinimums(2, 2, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); // First a non distributed query result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(1,1); - assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); + assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); - // Now do a distributed query, using JWTAUth for inter-node + // Now do a distributed query, using JWTAuth for inter-node result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - verifyInterRequestHeaderCounts(5,5); - assertAuthMetricsMinimums(5, 5, 0, 0, 0, 0); + assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); // Delete assertEquals(200, get(baseUrl + "/admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); - verifyInterRequestHeaderCounts(5,5); - assertAuthMetricsMinimums(10, 10, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); - } - - private void verifyInterRequestHeaderCounts(int jwt, int pki) { - assertEquals(jwt, jwtInterceptCount.get()); - assertEquals(pki, jwtInterceptCount.get()); + assertAuthMetricsMinimums(5, 5, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(20, 20, 0, 0, 0, 0); } private Pair get(String url, String token) throws IOException { @@ -198,16 +213,22 @@ private Pair post(String url, String json, String token) throws return new Pair<>(result, code); } - private static class CountInterceptor implements HttpRequestInterceptor { - @Override - public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { - Header ah = request.getFirstHeader(HttpHeaders.AUTHORIZATION); - if (ah != null && ah.getValue().startsWith("Bearer")) - jwtInterceptCount.addAndGet(1); + private void createCollection(String collectionName) throws IOException { + assertEquals(200, get(baseUrl + "/admin/collections?action=CREATE&name=" + collectionName + "&numShards=2", jwtTestToken).second().intValue()); + cluster.waitForActiveCollection(collectionName, 2, 2); + } - Header ph = request.getFirstHeader(PKIAuthenticationPlugin.HEADER); - if (ph != null) - pkiInterceptCount.addAndGet(1); - } + private void executeCommand(String url, HttpClient cl, String payload, JsonWebSignature jws) throws IOException, JoseException { + HttpPost httpPost; + HttpResponse r; + httpPost = new HttpPost(url); + setAuthorizationHeader(httpPost, "Bearer " + jws.getCompactSerialization()); + httpPost.setEntity(new ByteArrayEntity(payload.getBytes(UTF_8))); + httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); + r = cl.execute(httpPost); + String response = IOUtils.toString(r.getEntity().getContent(), StandardCharsets.UTF_8); + assertEquals("Non-200 response code. Response was " + response, 200, r.getStatusLine().getStatusCode()); + assertFalse("Response contained errors: " + response, response.contains("errorMessages")); + Utils.consumeFully(r.getEntity()); } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java index 4bcf8b92b471..6a7be0b9fd97 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; +import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Arrays; @@ -24,15 +25,30 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.message.AbstractHttpMessage; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.apache.solr.common.util.Base64; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.common.util.Utils; +import org.jose4j.jws.JsonWebSignature; +import org.jose4j.lang.JoseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Base test class for cloud tests wanting to track authentication metrics. * The assertions provided by this base class require a *minimum* count, not exact count from metrics. @@ -46,6 +62,7 @@ public class SolrCloudAuthTestCase extends SolrCloudTestCase { private static final List AUTH_METRICS_TIMER_KEYS = Collections.singletonList("requestTimes"); private static final String METRICS_PREFIX_PKI = "SECURITY./authentication/pki."; private static final String METRICS_PREFIX = "SECURITY./authentication."; + public static final Predicate NOT_NULL_PREDICATE = o -> o != null; /** * Used to check metric counts for PKI auth @@ -129,4 +146,69 @@ else if (AUTH_METRICS_TIMER_KEYS.contains(key)) else return metrics.stream().mapToLong(l -> ((Counter)l.get(prefix + key)).getCount()).sum(); } + + public static void verifySecurityStatus(HttpClient cl, String url, String objPath, + Object expected, int count) throws Exception { + verifySecurityStatus(cl, url, objPath, expected, count, (String)null); + } + + + public static void verifySecurityStatus(HttpClient cl, String url, String objPath, + Object expected, int count, String user, String pwd) + throws Exception { + verifySecurityStatus(cl, url, objPath, expected, count, makeBasicAuthHeader(user, pwd)); + } + + protected void verifySecurityStatus(HttpClient cl, String url, String objPath, + Object expected, int count, JsonWebSignature jws) throws Exception { + verifySecurityStatus(cl, url, objPath, expected, count, getBearerAuthHeader(jws)); + } + + + private static void verifySecurityStatus(HttpClient cl, String url, String objPath, + Object expected, int count, String authHeader) throws IOException, InterruptedException { + boolean success = false; + String s = null; + List hierarchy = StrUtils.splitSmart(objPath, '/'); + for (int i = 0; i < count; i++) { + HttpGet get = new HttpGet(url); + if (authHeader != null) setAuthorizationHeader(get, authHeader); + HttpResponse rsp = cl.execute(get); + s = EntityUtils.toString(rsp.getEntity()); + Map m = null; + try { + m = (Map) Utils.fromJSONString(s); + } catch (Exception e) { + fail("Invalid json " + s); + } + Utils.consumeFully(rsp.getEntity()); + Object actual = Utils.getObjectByPath(m, true, hierarchy); + if (expected instanceof Predicate) { + Predicate predicate = (Predicate) expected; + if (predicate.test(actual)) { + success = true; + break; + } + } else if (Objects.equals(actual == null ? null : String.valueOf(actual), expected)) { + success = true; + break; + } + Thread.sleep(50); + } + assertTrue("No match for " + objPath + " = " + expected + ", full response = " + s, success); + } + + protected static String makeBasicAuthHeader(String user, String pwd) { + String userPass = user + ":" + pwd; + return "Basic " + Base64.byteArrayToBase64(userPass.getBytes(UTF_8)); + } + + static String getBearerAuthHeader(JsonWebSignature jws) throws JoseException { + return "Bearer " + jws.getCompactSerialization(); + } + + public static void setAuthorizationHeader(AbstractHttpMessage httpMsg, String headerString) { + httpMsg.setHeader(new BasicHeader("Authorization", headerString)); + log.info("Added Authorization Header {}", headerString); + } } From b0d81f7eeb9c1569a08c54a8e8e7692f58f4f440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 15:48:58 +0100 Subject: [PATCH 49/88] Handle JWT parsing error and unresolvable JWK key Support HTTP2 for interception --- .../apache/solr/security/JWTAuthPlugin.java | 22 ++++++- .../JWTAuthPluginIntegrationTest.java | 64 +++++++++++++------ 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java index 038cc023bf38..39aafe256b73 100644 --- a/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/JWTAuthPlugin.java @@ -49,6 +49,7 @@ import org.apache.http.HttpRequest; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.protocol.HttpContext; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.common.SolrException; import org.apache.solr.common.SpecProvider; import org.apache.solr.common.StringUtils; @@ -57,6 +58,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.security.JWTAuthPlugin.JWTAuthenticationResponse.AuthCode; +import org.eclipse.jetty.client.api.Request; import org.jose4j.jwa.AlgorithmConstraints; import org.jose4j.jwk.HttpsJwks; import org.jose4j.jwk.JsonWebKey; @@ -70,6 +72,7 @@ import org.jose4j.keys.resolvers.JwksVerificationKeyResolver; import org.jose4j.keys.resolvers.VerificationKeyResolver; import org.jose4j.lang.JoseException; +import org.jose4j.lang.UnresolvableKeyException; import org.noggit.JSONUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -137,6 +140,7 @@ public JWTAuthPlugin() {} public void init(Map pluginConfig) { List unknownKeys = pluginConfig.keySet().stream().filter(k -> !PROPS.contains(k)).collect(Collectors.toList()); unknownKeys.remove("class"); + unknownKeys.remove(""); if (!unknownKeys.isEmpty()) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid JwtAuth configuration parameter " + unknownKeys); } @@ -336,13 +340,13 @@ public Principal getUserPrincipal() { return true; case AUTZ_HEADER_PROBLEM: + case JWT_PARSE_ERROR: authenticationFailure(response, authResponse.getAuthCode().getMsg(), HttpServletResponse.SC_BAD_REQUEST, BearerWwwAuthErrorCode.invalid_request); numErrors.mark(); return false; case CLAIM_MISMATCH: case JWT_EXPIRED: - case JWT_PARSE_ERROR: case JWT_VALIDATION_EXCEPTION: case PRINCIPAL_MISSING: if (authResponse.getJwtException() != null) { @@ -429,7 +433,10 @@ protected JWTAuthenticationResponse authenticate(String authorizationHeader) { if (e.hasExpired()) { return new JWTAuthenticationResponse(AuthCode.JWT_EXPIRED, "Authentication failed due to expired JWT token. Expired at " + e.getJwtContext().getJwtClaims().getExpirationTime()); } - return new JWTAuthenticationResponse(AuthCode.JWT_VALIDATION_EXCEPTION, e); + if (e.getCause() != null && e.getCause() instanceof UnresolvableKeyException) { + return new JWTAuthenticationResponse(AuthCode.JWT_VALIDATION_EXCEPTION, "Unable to find a suitable verification key for the JWT"); + } + return new JWTAuthenticationResponse(AuthCode.JWT_PARSE_ERROR, e); } } catch (MalformedClaimException e) { return new JWTAuthenticationResponse(AuthCode.JWT_PARSE_ERROR, "Malformed claim, error was: " + e.getMessage()); @@ -688,4 +695,15 @@ protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext } return false; } + + @Override + protected boolean interceptInternodeRequest(Request request) { + Object userToken = request.getAttributes().get(Http2SolrClient.REQ_PRINCIPAL_KEY); + if (userToken instanceof JWTPrincipal) { + JWTPrincipal jwtPrincipal = (JWTPrincipal) userToken; + request.header(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.getToken()); + return true; + } + return false; + } } diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index c8d5a35ee03e..d5ab3dc57000 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -41,6 +41,7 @@ import org.apache.solr.common.util.Utils; import org.jose4j.jwk.PublicJsonWebKey; import org.jose4j.jwk.RsaJsonWebKey; +import org.jose4j.jwk.RsaJwkGenerator; import org.jose4j.jws.AlgorithmIdentifiers; import org.jose4j.jws.JsonWebSignature; import org.jose4j.jwt.JwtClaims; @@ -70,6 +71,7 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { private String jwtTestToken; private String baseUrl; private JsonWebSignature jws; + private String jwtTokenWrongSignature; @Override @Before @@ -101,6 +103,15 @@ public void setUp() throws Exception { jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); jwtTestToken = jws.getCompactSerialization(); + + PublicJsonWebKey jwk2 = RsaJwkGenerator.generateJwk(2048); + jwk2.setKeyId("k2"); + JsonWebSignature jws2 = new JsonWebSignature(); + jws2.setPayload(claims.toJson()); + jws2.setKey(jwk2.getPrivateKey()); + jws2.setKeyIdHeaderValue(jwk2.getKeyId()); + jws2.setAlgorithmHeaderValue(AlgorithmIdentifiers.RSA_USING_SHA256); + jwtTokenWrongSignature = jws2.getCompactSerialization(); cluster.waitForAllNodes(10); } @@ -117,12 +128,6 @@ public void infoRequestWithoutToken() throws Exception { get(baseUrl + "/admin/info/system", null); } - @Test - public void infoRequestWithToken() throws IOException { - Pair result = get(baseUrl + "/admin/info/system", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - } - @Test public void testMetrics() throws Exception { boolean isUseV2Api = random().nextBoolean(); @@ -138,17 +143,29 @@ public void testMetrics() throws Exception { // NOCOMMIT: Metrics tests, with failing requests etc createCollection(COLLECTION); + // Missing token + getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", null); + assertAuthMetricsMinimums(2, 1, 0, 0, 1, 0); executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: false}}", jws); verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "false", 20, jws); + // Pass through verifySecurityStatus(cl, baseUrl + "/admin/info/key", "key", NOT_NULL_PREDICATE, 20); - assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); + // Now succeeds since blockUnknown=false + get(baseUrl + "/" + COLLECTION + "/query?q=*:*", null); + executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", null); + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/blockUnknown", "true", 20, jws); + + assertAuthMetricsMinimums(9, 4, 4, 0, 1, 0); - // Now update three documents - assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); - Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); - assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); + // Wrong Credentials + getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTokenWrongSignature); + assertAuthMetricsMinimums(10, 4, 4, 1, 1, 0); + + // JWT parse error + getAndFail(baseUrl + "/" + COLLECTION + "/query?q=*:*", "foozzz"); + assertAuthMetricsMinimums(11, 4, 4, 1, 1, 1); + + HttpClientUtil.close(cl); } @Test @@ -157,28 +174,36 @@ public void createCollectionUpdateAndQueryDistributed() throws Exception { createCollection(COLLECTION); // Now update three documents + assertAuthMetricsMinimums(1, 1, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(12, 12, 0, 0, 0, 0); Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(2, 2, 0, 0, 0, 0); + assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); assertPkiAuthMetricsMinimums(13, 13, 0, 0, 0, 0); // First a non distributed query result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(3, 3, 0, 0, 0, 0); + assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); // Now do a distributed query, using JWTAuth for inter-node result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); - assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); + assertAuthMetricsMinimums(9, 9, 0, 0, 0, 0); // Delete assertEquals(200, get(baseUrl + "/admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); - assertAuthMetricsMinimums(5, 5, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(20, 20, 0, 0, 0, 0); + assertAuthMetricsMinimums(10, 10, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(15, 15, 0, 0, 0, 0); } + private void getAndFail(String url, String token) throws IOException { + try { + get(url, token); + fail("Request to " + url + " with token " + token + " should have failed"); + } catch(Exception e) {} + } + private Pair get(String url, String token) throws IOException { URL createUrl = new URL(url); HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); @@ -222,7 +247,8 @@ private void executeCommand(String url, HttpClient cl, String payload, JsonWebSi HttpPost httpPost; HttpResponse r; httpPost = new HttpPost(url); - setAuthorizationHeader(httpPost, "Bearer " + jws.getCompactSerialization()); + if (jws != null) + setAuthorizationHeader(httpPost, "Bearer " + jws.getCompactSerialization()); httpPost.setEntity(new ByteArrayEntity(payload.getBytes(UTF_8))); httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); r = cl.execute(httpPost); From 13e41ae2cc240b14cec0a5f5f8e014a37b4f83d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 15:51:07 +0100 Subject: [PATCH 50/88] Remove NOCOMMIT --- .../org/apache/solr/security/JWTAuthPluginIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index d5ab3dc57000..9a5f716346ab 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -140,7 +140,6 @@ public void testMetrics() throws Exception { String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); CloseableHttpClient cl = HttpClientUtil.createClient(null); - // NOCOMMIT: Metrics tests, with failing requests etc createCollection(COLLECTION); // Missing token From 7a9c208cb061f8c3c6a7e6b1557d3d84a242be90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 16:08:12 +0100 Subject: [PATCH 51/88] Precommit --- .../org/apache/solr/security/JWTAuthPluginIntegrationTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 9a5f716346ab..58c7a93ece82 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -54,13 +54,12 @@ /** * Validate that JWT token authentication works in a real cluster. - *

+ * * TODO: *

    *
  • Test also using SolrJ as client. But that requires a way to set Authorization header on request, see SOLR-13070
    • *
  • This is also the reason we use {@link org.apache.solr.SolrTestCaseJ4.SuppressSSL} annotation, since we use HttpUrlConnection
    • *
- *

*/ @SolrTestCaseJ4.SuppressSSL public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { From 02d768e8858db5af7145d87465fdf57afa80c464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 16:17:14 +0100 Subject: [PATCH 52/88] RefGuide fixes, unused import --- .../solr/security/PKIAuthenticationIntegrationTest.java | 1 - solr/solr-ref-guide/src/jwt-authentication-plugin.adoc | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java index eb25b8306af7..bff9766a8340 100644 --- a/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java @@ -37,7 +37,6 @@ import static java.util.Collections.singletonMap; import static org.apache.solr.common.util.Utils.makeMap; -import static org.apache.solr.security.TestAuthorizationFramework.verifySecurityStatus; public class PKIAuthenticationIntegrationTest extends SolrCloudAuthTestCase { diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc index 41cf22db04fd..7849b43db5f7 100644 --- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc +++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc @@ -88,7 +88,7 @@ The next example shows configuring using https://openid.net/specs/openid-connect "blockUnknown": true, "wellKnownUrl": "https://idp.example.com/.well-known/openid-configuration", "clientId": "xyz", - "redirectUri": "https://my.solr.server:8983/solr/#/login" + "redirectUri": "https://my.solr.server:8983/solr/" } } ---- @@ -153,7 +153,7 @@ Example: [source,bash] ---- -curl --user solr:SolrRocks http://localhost:8983/solr/admin/authentication -H 'Content-type:application/json' -d '{"set-property": {"blockUnknown":true, "wellKnownUrl": "https://example.com/.well-knwon/openid-configuration", "scope": "solr:read solr:write"}}' +curl http://localhost:8983/solr/admin/authentication -H 'Content-type:application/json' -H 'Authorization: Bearer xxx.yyy.zzz' -d '{"set-property": {"blockUnknown":true, "wellKnownUrl": "https://example.com/.well-knwon/openid-configuration", "scope": "solr:read solr:write"}}' ---- ==== @@ -163,11 +163,13 @@ curl --user solr:SolrRocks http://localhost:8983/solr/admin/authentication -H 'C [source,bash] ---- -curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentication -H 'Content-type:application/json' -d '{"set-property": {"blockUnknown":true, "wellKnownUrl": "https://example.com/.well-knwon/openid-configuration", "scope": "solr:read solr:write"}}' +curl http://localhost:8983/api/cluster/security/authentication -H 'Content-type:application/json' -H 'Authorization: Bearer xxx.yyy.zzz' -d -d '{"set-property": {"blockUnknown":true, "wellKnownUrl": "https://example.com/.well-knwon/openid-configuration", "scope": "solr:read solr:write"}}' ---- ==== -- +Insert your JWT token in compact `xxx.yyy.zzz` format above to authenticate with Solr once the plugin is active. + == Using clients with JWT Auth [#jwt-soljr] From 16d288b39b527217478009b1a516103f5cba81b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 21 Dec 2018 16:31:37 +0100 Subject: [PATCH 53/88] Javadoc --- .../solr/security/JWTAuthPluginIntegrationTest.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 58c7a93ece82..f14587b01f70 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -54,12 +54,10 @@ /** * Validate that JWT token authentication works in a real cluster. - * - * TODO: - *
    - *
  • Test also using SolrJ as client. But that requires a way to set Authorization header on request, see SOLR-13070
    • - *
  • This is also the reason we use {@link org.apache.solr.SolrTestCaseJ4.SuppressSSL} annotation, since we use HttpUrlConnection
    • - *
+ *

+ * TODO: Test also using SolrJ as client. But that requires a way to set Authorization header on request, see SOLR-13070
+ * TODO: This is also the reason we use {@link org.apache.solr.SolrTestCaseJ4.SuppressSSL} annotation, since we use HttpUrlConnection + *

*/ @SolrTestCaseJ4.SuppressSSL public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { From 465d73676dd793d280ec94e37254d7a96cd8d6c4 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 20 Dec 2018 14:20:50 -0500 Subject: [PATCH 54/88] SOLR-13045: Sim node versioning should start at 0 Prior to this commit, new ZK nodes being simulated by the sim framework were started with a version of -1. This causes problems, since -1 is also coincidentally the flag value used to ignore optimistic concurrency locking and force overwrite values. --- .../autoscaling/OverseerTriggerThread.java | 2 +- .../sim/SimClusterStateProvider.java | 6 ++-- .../sim/SimDistribStateManager.java | 20 ++++++------- .../sim/TestSimDistribStateManager.java | 29 +++++++++++++++++++ .../cloud/autoscaling/AutoScalingConfig.java | 2 +- 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java index 7e36378dafe0..41ae59b4a515 100644 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java +++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java @@ -70,7 +70,7 @@ public class OverseerTriggerThread implements Runnable, SolrCloseable { /* Following variables are only accessed or modified when updateLock is held */ - private int znodeVersion = -1; + private int znodeVersion = 0; private Map activeTriggers = new HashMap<>(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java index 757c0bd3178b..ad0177de3baa 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java @@ -631,10 +631,10 @@ private ClusterState saveClusterState(ClusterState state) throws IOException { byte[] data = Utils.toJSON(state); try { VersionedData oldData = stateManager.getData(ZkStateReader.CLUSTER_STATE); - int version = oldData != null ? oldData.getVersion() : -1; - Assert.assertEquals(clusterStateVersion, version + 1); + int version = oldData != null ? oldData.getVersion() : 0; + Assert.assertEquals(clusterStateVersion, version); stateManager.setData(ZkStateReader.CLUSTER_STATE, data, version); - log.debug("** saved cluster state version " + (version + 1)); + log.debug("** saved cluster state version " + (version)); clusterStateVersion++; } catch (Exception e) { throw new IOException(e); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java index d01928a739fe..9b02b57aa917 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java @@ -72,7 +72,7 @@ public class SimDistribStateManager implements DistribStateManager { public static final class Node { ReentrantLock dataLock = new ReentrantLock(); - private int version = -1; + private int version = 0; private int seq = 0; private final CreateMode mode; private final String clientId; @@ -90,7 +90,11 @@ public static final class Node { this.path = path; this.mode = mode; this.clientId = clientId; + } + Node(Node parent, String name, String path, byte[] data, CreateMode mode, String clientId) { + this(parent, name, path, mode, clientId); + this.data = data; } public void clear() { @@ -311,7 +315,7 @@ private Node traverse(String path, boolean create, CreateMode mode) throws IOExc n = parentNode.children != null ? parentNode.children.get(currentName) : null; if (n == null) { if (create) { - n = createNode(parentNode, mode, currentPath, currentName,true); + n = createNode(parentNode, mode, currentPath, currentName,null, true); } else { break; } @@ -323,7 +327,7 @@ private Node traverse(String path, boolean create, CreateMode mode) throws IOExc return n; } - private Node createNode(Node parentNode, CreateMode mode, StringBuilder fullChildPath, String baseChildName, boolean attachToParent) throws IOException { + private Node createNode(Node parentNode, CreateMode mode, StringBuilder fullChildPath, String baseChildName, byte[] data, boolean attachToParent) throws IOException { String nodeName = baseChildName; if ((parentNode.mode == CreateMode.EPHEMERAL || parentNode.mode == CreateMode.EPHEMERAL_SEQUENTIAL) && (mode == CreateMode.EPHEMERAL || mode == CreateMode.EPHEMERAL_SEQUENTIAL)) { @@ -335,7 +339,7 @@ private Node createNode(Node parentNode, CreateMode mode, StringBuilder fullChil } fullChildPath.append(nodeName); - Node child = new Node(parentNode, nodeName, fullChildPath.toString(), mode, id); + Node child = new Node(parentNode, nodeName, fullChildPath.toString(), data, mode, id); if (attachToParent) { parentNode.setChild(nodeName, child); @@ -480,13 +484,9 @@ public String createData(String path, byte[] data, CreateMode mode) throws Alrea multiLock.lock(); try { String nodeName = elements[elements.length-1]; - Node childNode = createNode(parentNode, mode, parentStringBuilder.append("/"), nodeName, false); - childNode.setData(data, -1); + Node childNode = createNode(parentNode, mode, parentStringBuilder.append("/"), nodeName, data,false); parentNode.setChild(childNode.name, childNode); return childNode.path; - } catch (BadVersionException e) { - // not happening - return null; } finally { multiLock.unlock(); } @@ -586,7 +586,7 @@ public List multi(Iterable ops) throws BadVersionException, NoSuch @Override public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException { Map map = new HashMap<>(); - int version = -1; + int version = 0; try { VersionedData data = getData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, watcher); if (data != null && data.getData() != null && data.getData().length > 0) { diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java index 1f80e237a8fc..731d6e828abd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java @@ -341,6 +341,35 @@ public void testGetSetRemoveData() throws Exception { } } + @Test + public void testNewlyCreatedPathsStartWithVersionZero() throws Exception { + stateManager.makePath("/createdWithoutData"); + VersionedData vd = stateManager.getData("/createdWithoutData", null); + assertEquals(0, vd.getVersion()); + + stateManager.createData("/createdWithData", new String("helloworld").getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT); + vd = stateManager.getData("/createdWithData"); + assertEquals(0, vd.getVersion()); + } + + @Test + public void testModifiedDataNodesGetUpdatedVersion() throws Exception { + stateManager.createData("/createdWithData", new String("foo").getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT); + VersionedData vd = stateManager.getData("/createdWithData"); + assertEquals(0, vd.getVersion()); + + stateManager.setData("/createdWithData", new String("bar").getBytes(StandardCharsets.UTF_8), 0); + vd = stateManager.getData("/createdWithData"); + assertEquals(1, vd.getVersion()); + } + + // This is a little counterintuitive, so probably worth its own testcase so we don't break it accidentally. + @Test + public void testHasDataReturnsTrueForExistingButEmptyNodes() throws Exception { + stateManager.makePath("/nodeWithoutData"); + assertTrue(stateManager.hasData("/nodeWithoutData")); + } + @Test public void testMulti() throws Exception { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java index ccd02eb6b44c..335312ea7e6a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java @@ -311,7 +311,7 @@ public AutoScalingConfig(byte[] utf8) { */ public AutoScalingConfig(Map jsonMap) { this.jsonMap = jsonMap; - int version = -1; + int version = 0; if (jsonMap.containsKey(AutoScalingParams.ZK_VERSION)) { try { version = (Integer)jsonMap.get(AutoScalingParams.ZK_VERSION); From b5c4c6fd28fa8c9dcd608680f6026598e163ed7a Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 21 Dec 2018 13:26:03 -0500 Subject: [PATCH 55/88] SOLR-13080: TermsQParserPlugin automaton method should (must?) sort input --- solr/CHANGES.txt | 3 +++ .../src/java/org/apache/solr/search/TermsQParserPlugin.java | 4 +++- solr/core/src/test/org/apache/solr/search/TestQueryTypes.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ab651f397169..6cd3c06b9d8e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -184,6 +184,9 @@ Bug Fixes * SOLR-13072: Management of markers for nodeLost / nodeAdded events is broken. This bug could have caused some events to be lost if they coincided with an Overseer leader crash. (ab) +* SOLR-13080: The "terms" QParser's "automaton" method semi-required that the input terms/IDs be sorted. This + query parser now does this. Unclear if this is a perf issue or actual bug. (Daniel Lowe, David Smiley) + Improvements ---------------------- diff --git a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java index 45bb13fc310b..805cca33b067 100644 --- a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java @@ -30,6 +30,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.automaton.Automata; @@ -79,7 +80,8 @@ Query makeFilter(String fname, BytesRef[] byteRefs) { automaton { @Override Query makeFilter(String fname, BytesRef[] byteRefs) { - Automaton union = Automata.makeStringUnion(Arrays.asList(byteRefs)); + ArrayUtil.timSort(byteRefs); // same sort algo as TermInSetQuery's choice + Automaton union = Automata.makeStringUnion(Arrays.asList(byteRefs)); // input must be sorted return new AutomatonQuery(new Term(fname), union);//constant scores } }, diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java index 29c9a37a9921..ff05e1a22f8f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java +++ b/solr/core/src/test/org/apache/solr/search/TestQueryTypes.java @@ -109,7 +109,7 @@ public void testQueryTypes() { ); String termsMethod = new String[]{"termsFilter", "booleanQuery", "automaton", "docValuesTermsFilter"}[random().nextInt(4)]; - assertQ(req( "q", "{!terms f=v_s method=" + termsMethod + " }other stuff,wow dude") + assertQ(req( "q", "{!terms f=v_s method=" + termsMethod + " }wow dude,other stuff")//terms reverse sorted to show this works ,"//result[@numFound='2']" ); From 1c2154a93bcb1bf040ee612fcac7824142103aa9 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 26 Dec 2018 09:39:42 -0500 Subject: [PATCH 56/88] SOLR-12535: index time boosts in JSON are no longer accepted --- solr/CHANGES.txt | 6 + .../solr/handler/loader/JsonLoader.java | 118 +++++------------- .../apache/solr/handler/JsonLoaderTest.java | 55 ++------ 3 files changed, 46 insertions(+), 133 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6cd3c06b9d8e..8aba176cd181 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -65,6 +65,10 @@ Upgrade Notes SchemaSimilarityFactory, then LegacyBM25Similarity is automatically selected for 'luceneMatchVersion' < 8.0.0. See also explanation in Reference Guide chapter "Other Schema Elements". +* SOLR-12535: Solr no longer accepts index time boosts in JSON provided to Solr. This used to be provided like so: + {'id':'1', 'val_s':{'value':'foo', 'boost':2.0}} but will now produce an error. A object/map structure will now only + be interpreted as a child document or an atomic update; nothing else. A uniqueKey is currently required on all child + documents to be interpreted as such, though this may change in the future. (David Smiley) New Features ---------------------- @@ -139,6 +143,8 @@ Other Changes * SOLR-13036: Fix retry logic in JettySolrRunner (Gus Heck) +* SOLR-12535: Solr no longer accepts index time boosts in JSON provided to Solr. (David Smiley) + ================== 7.7.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java index 67628e540f0d..5b484a4bcb62 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java @@ -558,94 +558,18 @@ private SolrInputDocument parseDoc(int ev) throws IOException { sdoc.addChildDocument(parseDoc(ev)); } } else { - SolrInputField sif = new SolrInputField(fieldName); - parseFieldValue(sif); - // pulling out the pieces may seem weird, but it's because + ev = parser.nextEvent(); + Object val = parseFieldValue(ev, fieldName); // SolrInputDocument.addField will do the right thing // if the doc already has another value for this field // (ie: repeating fieldname keys) - sdoc.addField(sif.getName(), sif.getValue()); + sdoc.addField(fieldName, val); } } } - private void parseFieldValue(SolrInputField sif) throws IOException { - int ev = parser.nextEvent(); - if (ev == JSONParser.OBJECT_START) { - parseExtendedFieldValue(ev, sif); - } else { - Object val = parseNormalFieldValue(ev, sif); - sif.setValue(val); - } - } - - /** - * A method to either extract an index time boost (deprecated), a map for atomic update, or a child document. - * firstly, a solr document SolrInputDocument constructed. It is then determined whether the document is indeed a childDocument(if it has a unique field). - * If so, it is added. - * Otherwise the document is looped over as a map, and is then parsed as an Atomic Update if that is the case. - * @param ev json parser event - * @param sif input field to add value to. - * @throws IOException in case of parsing exception. - */ - private void parseExtendedFieldValue(int ev, SolrInputField sif) throws IOException { - assert ev == JSONParser.OBJECT_START; - - SolrInputDocument extendedSolrDocument = parseDoc(ev); - - if (isChildDoc(extendedSolrDocument)) { - sif.addValue(extendedSolrDocument); - return; - } - - Object normalFieldValue = null; - Map extendedInfo = null; - - for (SolrInputField entry: extendedSolrDocument) { - Object val = entry.getValue(); - String label = entry.getName(); - if ("boost".equals(label)) { - Object boostVal = val; - if (!(boostVal instanceof Double)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Boost should have number. " - + "Unexpected value: " + boostVal.toString() + "field=" + label); - } - - String message = "Ignoring field boost: " + boostVal.toString() + " as index-time boosts are not supported anymore"; - if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) { - log.warn(message); - } else { - log.debug(message); - } - } else if ("value".equals(label)) { - normalFieldValue = val; - } else { - // If we encounter other unknown map keys, then use a map - if (extendedInfo == null) { - extendedInfo = new HashMap<>(2); - } - // for now, the only extended info will be field values - // we could either store this as an Object or a SolrInputField - extendedInfo.put(label, val); - } - if (extendedInfo != null) { - if (normalFieldValue != null) { - extendedInfo.put("value", normalFieldValue); - } - sif.setValue(extendedInfo); - } else { - sif.setValue(normalFieldValue); - } - } - } - - - private Object parseNormalFieldValue(int ev, SolrInputField sif) throws IOException { - return ev == JSONParser.ARRAY_START ? parseArrayFieldValue(ev, sif): parseSingleFieldValue(ev, sif); - } - - private Object parseSingleFieldValue(int ev, SolrInputField sif) throws IOException { + private Object parseFieldValue(int ev, String fieldName) throws IOException { switch (ev) { case JSONParser.STRING: return parser.getString(); @@ -661,18 +585,16 @@ private Object parseSingleFieldValue(int ev, SolrInputField sif) throws IOExcept parser.getNull(); return null; case JSONParser.ARRAY_START: - return parseArrayFieldValue(ev, sif); + return parseArrayFieldValue(ev, fieldName); case JSONParser.OBJECT_START: - parseExtendedFieldValue(ev, sif); - return sif.getValue(); + return parseObjectFieldValue(ev, fieldName); default: throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing JSON field value. " - + "Unexpected " + JSONParser.getEventString(ev) + " at [" + parser.getPosition() + "], field=" + sif.getName()); + + "Unexpected " + JSONParser.getEventString(ev) + " at [" + parser.getPosition() + "], field=" + fieldName); } } - - private List parseArrayFieldValue(int ev, SolrInputField sif) throws IOException { + private List parseArrayFieldValue(int ev, String fieldName) throws IOException { assert ev == JSONParser.ARRAY_START; ArrayList lst = new ArrayList(2); @@ -681,9 +603,27 @@ private List parseArrayFieldValue(int ev, SolrInputField sif) throws IOE if (ev == JSONParser.ARRAY_END) { return lst; } - Object val = parseSingleFieldValue(ev, sif); - lst.add(val); - sif.setValue(null); + lst.add(parseFieldValue(ev, fieldName)); + } + } + + /** + * Parses this object as either a map for atomic update, or a child document. + */ + private Object parseObjectFieldValue(int ev, String fieldName) throws IOException { + assert ev == JSONParser.OBJECT_START; + + SolrInputDocument extendedSolrDocument = parseDoc(ev); + // is this a partial update or a child doc? + if (isChildDoc(extendedSolrDocument)) { + return extendedSolrDocument; + } else { + //return extendedSolrDocument.toMap(new HashMap<>(extendedSolrDocument.size())); not quite right + Map map = new HashMap<>(extendedSolrDocument.size()); + for (SolrInputField inputField : extendedSolrDocument) { + map.put(inputField.getName(), inputField.getValue()); + } + return map; } } diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java index fdfeae44336e..15330c6dde59 100644 --- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java +++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java @@ -52,15 +52,7 @@ public static void beforeTests() throws Exception { " 'doc': {\n" + " 'bool': true,\n" + " 'f0': 'v0',\n" + - " 'f2': {\n" + - " 'boost': 2.3,\n" + - " 'value': 'test'\n" + - " },\n" + - " 'array': [ 'aaa', 'bbb' ],\n" + - " 'boosted': {\n" + - " 'boost': 6.7,\n" + // make sure we still accept boosts - " 'value': [ 'aaa', 'bbb' ]\n" + - " }\n" + + " 'array': [ 'aaa', 'bbb' ]\n" + " }\n" + "},\n" + "'add': {\n" + @@ -98,19 +90,13 @@ public void testParsing() throws Exception assertEquals( 2, p.addCommands.size() ); AddUpdateCommand add = p.addCommands.get(0); - SolrInputDocument d = add.solrDoc; - SolrInputField f = d.getField( "boosted" ); - assertEquals(2, f.getValues().size()); + assertEquals("SolrInputDocument(fields: [bool=true, f0=v0, array=[aaa, bbb]])", add.solrDoc.toString()); // add = p.addCommands.get(1); - d = add.solrDoc; - f = d.getField( "f1" ); - assertEquals(2, f.getValues().size()); + assertEquals("SolrInputDocument(fields: [f1=[v1, v2], f2=null])", add.solrDoc.toString()); assertEquals(false, add.overwrite); - assertEquals(0, d.getField("f2").getValueCount()); - // parse the commit commands assertEquals( 2, p.commitCommands.size() ); CommitUpdateCommand commit = p.commitCommands.get( 0 ); @@ -235,26 +221,14 @@ public void testFieldValueOrdering() throws Exception { // list checkFieldValueOrdering((pre+ "'f':[45,67,89]" +post) - .replace('\'', '"'), - 1.0F); + .replace('\'', '"') + ); // dup fieldname keys checkFieldValueOrdering((pre+ "'f':45,'f':67,'f':89" +post) - .replace('\'', '"'), - 1.0F); - // extended w/boost - checkFieldValueOrdering((pre+ "'f':{'boost':4.0,'value':[45,67,89]}" +post) - .replace('\'', '"'), - 4.0F); - // dup keys extended w/ multiplicitive boost - checkFieldValueOrdering((pre+ - "'f':{'boost':2.0,'value':[45,67]}," + - "'f':{'boost':2.0,'value':89}" - +post) - .replace('\'', '"'), - 4.0F); - + .replace('\'', '"') + ); } - private void checkFieldValueOrdering(String rawJson, float fBoost) throws Exception { + private void checkFieldValueOrdering(String rawJson) throws Exception { SolrQueryRequest req = req(); SolrQueryResponse rsp = new SolrQueryResponse(); BufferingRequestProcessor p = new BufferingRequestProcessor(null); @@ -265,7 +239,7 @@ private void checkFieldValueOrdering(String rawJson, float fBoost) throws Except SolrInputDocument d = p.addCommands.get(0).solrDoc; assertEquals(2, d.getFieldNames().size()); assertEquals("1", d.getFieldValue("id")); - assertEquals(new Object[] {45L, 67L, 89L} , d.getFieldValues("f").toArray()); + assertArrayEquals(new Object[] {45L, 67L, 89L} , d.getFieldValues("f").toArray()); d = p.addCommands.get(1).solrDoc; assertEquals(1, d.getFieldNames().size()); @@ -520,7 +494,7 @@ private static void assertOnlyValue(String expected, SolrInputDocument doc, Stri assertEquals(Collections.singletonList(expected), doc.getFieldValues(field)); } - public void testExtendedFieldValues() throws Exception { + public void testAtomicUpdateFieldValue() throws Exception { String str = "[{'id':'1', 'val_s':{'add':'foo'}}]".replace('\'', '"'); SolrQueryRequest req = req(); SolrQueryResponse rsp = new SolrQueryResponse(); @@ -533,14 +507,7 @@ public void testExtendedFieldValues() throws Exception { AddUpdateCommand add = p.addCommands.get(0); assertEquals(add.commitWithin, -1); assertEquals(add.overwrite, true); - SolrInputDocument d = add.solrDoc; - - SolrInputField f = d.getField( "id" ); - assertEquals("1", f.getValue()); - - f = d.getField( "val_s" ); - Map map = (Map)f.getValue(); - assertEquals("foo",map.get("add")); + assertEquals("SolrInputDocument(fields: [id=1, val_s={add=foo}])", add.solrDoc.toString()); req.close(); } From 68188e8296089daa255ca1bd179857d928caff25 Mon Sep 17 00:00:00 2001 From: Joel Bernstein Date: Thu, 27 Dec 2018 14:42:03 -0500 Subject: [PATCH 57/88] SOLR-13088: Add zplot Stream Evaluator to plot math expressions in Apache Zeppelin --- .../org/apache/solr/client/solrj/io/Lang.java | 4 +- .../client/solrj/io/stream/ZplotStream.java | 208 ++++++++++++++++++ .../apache/solr/client/solrj/io/TestLang.java | 3 +- .../solrj/io/stream/MathExpressionTest.java | 77 +++++++ 4 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java index 050fa7e1f732..a1a796d1ecfe 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java @@ -92,8 +92,10 @@ public static void register(StreamFactory streamFactory) { .withFunctionName("tuple", TupStream.class) .withFunctionName("sql", SqlStream.class) .withFunctionName("plist", ParallelListStream.class) + .withFunctionName("zplot", ZplotStream.class) - // metrics + + // metrics .withFunctionName("min", MinMetric.class) .withFunctionName("max", MaxMetric.class) .withFunctionName("avg", MeanMetric.class) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java new file mode 100644 index 000000000000..c5280dca78e7 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/ZplotStream.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.io.stream; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.comp.StreamComparator; +import org.apache.solr.client.solrj.io.eval.StreamEvaluator; +import org.apache.solr.client.solrj.io.stream.expr.Explanation; +import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType; +import org.apache.solr.client.solrj.io.stream.expr.Expressible; +import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; + +public class ZplotStream extends TupleStream implements Expressible { + + private static final long serialVersionUID = 1; + private StreamContext streamContext; + private Map letParams = new LinkedHashMap(); + private Iterator out; + + public ZplotStream(StreamExpression expression, StreamFactory factory) throws IOException { + + List namedParams = factory.getNamedOperands(expression); + //Get all the named params + + for(StreamExpressionParameter np : namedParams) { + String name = ((StreamExpressionNamedParameter)np).getName(); + StreamExpressionParameter param = ((StreamExpressionNamedParameter)np).getParameter(); + if(param instanceof StreamExpressionValue) { + String paramValue = ((StreamExpressionValue) param).getValue(); + letParams.put(name, factory.constructPrimitiveObject(paramValue)); + } else if(factory.isEvaluator((StreamExpression)param)) { + StreamEvaluator evaluator = factory.constructEvaluator((StreamExpression) param); + letParams.put(name, evaluator); + } + } + } + + @Override + public StreamExpression toExpression(StreamFactory factory) throws IOException{ + return toExpression(factory, true); + } + + private StreamExpression toExpression(StreamFactory factory, boolean includeStreams) throws IOException { + // function name + StreamExpression expression = new StreamExpression(factory.getFunctionName(this.getClass())); + + return expression; + } + + @Override + public Explanation toExplanation(StreamFactory factory) throws IOException { + + StreamExplanation explanation = new StreamExplanation(getStreamNodeId().toString()); + explanation.setFunctionName(factory.getFunctionName(this.getClass())); + explanation.setImplementingClass(this.getClass().getName()); + explanation.setExpressionType(ExpressionType.STREAM_DECORATOR); + explanation.setExpression(toExpression(factory, false).toString()); + + return explanation; + } + + public void setStreamContext(StreamContext context) { + this.streamContext = context; + } + + public List children() { + List l = new ArrayList(); + return l; + } + + public Tuple read() throws IOException { + if(out.hasNext()) { + return out.next(); + } else { + Map m = new HashMap(); + m.put("EOF", true); + Tuple t = new Tuple(m); + return t; + } + } + + public void close() throws IOException { + } + + public void open() throws IOException { + Map lets = streamContext.getLets(); + Set> entries = letParams.entrySet(); + Map evaluated = new HashMap(); + + //Load up the StreamContext with the data created by the letParams. + int numTuples = -1; + int columns = 0; + boolean table = false; + for(Map.Entry entry : entries) { + ++columns; + + String name = entry.getKey(); + if(name.equals("table")) { + table = true; + } + + Object o = entry.getValue(); + if(o instanceof StreamEvaluator) { + Tuple eTuple = new Tuple(lets); + StreamEvaluator evaluator = (StreamEvaluator)o; + evaluator.setStreamContext(streamContext); + Object eo = evaluator.evaluate(eTuple); + if(eo instanceof List) { + List l = (List)eo; + if(numTuples == -1) { + numTuples = l.size(); + } else { + if(l.size() != numTuples) { + throw new IOException("All lists provided to the zplot function must be the same length."); + } + } + evaluated.put(name, l); + } else if (eo instanceof Tuple) { + evaluated.put(name, eo); + } + } else { + Object eval = lets.get(o); + if(eval instanceof List) { + List l = (List)eval; + if(numTuples == -1) { + numTuples = l.size(); + } else { + if(l.size() != numTuples) { + throw new IOException("All lists provided to the zplot function must be the same length."); + } + } + evaluated.put(name, l); + } else if(eval instanceof Tuple) { + evaluated.put(name, eval); + } + } + } + + if(columns > 1 && table) { + throw new IOException("If the table parameter is set there can only be one parameter."); + } + //Load the values into tuples + + List outTuples = new ArrayList(); + if(!table) { + //Handle the vectors + for (int i = 0; i < numTuples; i++) { + Tuple tuple = new Tuple(new HashMap()); + for (String key : evaluated.keySet()) { + List l = (List) evaluated.get(key); + tuple.put(key, l.get(i)); + } + + outTuples.add(tuple); + } + } else { + //Handle the Tuple and List of Tuples + Object o = evaluated.get("table"); + if(o instanceof List) { + List tuples = (List)o; + outTuples.addAll(tuples); + } else if(o instanceof Tuple) { + outTuples.add((Tuple)o); + } + } + + this.out = outTuples.iterator(); + } + + /** Return the stream sort - ie, the order in which records are returned */ + public StreamComparator getStreamSort(){ + return null; + } + + public int getCost() { + return 0; + } + + +} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java index 3b238c2faa8d..b5b731717e56 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java @@ -73,7 +73,8 @@ public class TestLang extends LuceneTestCase { "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "latlonVectors", "convexHull", "getVertices", "getBaryCenter", "getArea", "getBoundarySize","oscillate", "getAmplitude", "getPhase", "getAngularFrequency", "enclosingDisk", "getCenter", "getRadius", - "getSupportPoints", "pairSort", "log10", "plist", "recip", "pivot", "ltrim", "rtrim", "export"}; + "getSupportPoints", "pairSort", "log10", "plist", "recip", "pivot", "ltrim", "rtrim", "export", + "zplot"}; @Test public void testLang() { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java index 8ac184a592a8..8e973d299ef8 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java @@ -1356,6 +1356,83 @@ public void testMatrix() throws Exception { assertTrue(tuples.get(0).getLong("i")== 2); } + + @Test + public void testZplot() throws Exception { + String cexpr = "let(c=tuple(a=add(1,2), b=add(2,3))," + + " zplot(table=c))"; + + ModifiableSolrParams paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", cexpr); + paramsLoc.set("qt", "/stream"); + String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS; + TupleStream solrStream = new SolrStream(url, paramsLoc); + StreamContext context = new StreamContext(); + solrStream.setStreamContext(context); + List tuples = getTuples(solrStream); + assertTrue(tuples.size() == 1); + Tuple out = tuples.get(0); + + assertEquals(out.getDouble("a").doubleValue(), 3.0, 0.0); + assertEquals(out.getDouble("b").doubleValue(), 5.0, 0.0); + + cexpr = "let(c=list(tuple(a=add(1,2), b=add(2,3)), tuple(a=add(1,3), b=add(2,4)))," + + " zplot(table=c))"; + + paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", cexpr); + paramsLoc.set("qt", "/stream"); + solrStream = new SolrStream(url, paramsLoc); + context = new StreamContext(); + solrStream.setStreamContext(context); + tuples = getTuples(solrStream); + assertTrue(tuples.size() == 2); + out = tuples.get(0); + + assertEquals(out.getDouble("a").doubleValue(), 3.0, 0.0); + assertEquals(out.getDouble("b").doubleValue(), 5.0, 0.0); + + out = tuples.get(1); + + assertEquals(out.getDouble("a").doubleValue(), 4.0, 0.0); + assertEquals(out.getDouble("b").doubleValue(), 6.0, 0.0); + + + cexpr = "let(a=array(1,2,3,4)," + + " b=array(10,11,12,13),"+ + " zplot(x=a, y=b))"; + + paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", cexpr); + paramsLoc.set("qt", "/stream"); + solrStream = new SolrStream(url, paramsLoc); + context = new StreamContext(); + solrStream.setStreamContext(context); + tuples = getTuples(solrStream); + assertTrue(tuples.size() == 4); + out = tuples.get(0); + + assertEquals(out.getDouble("x").doubleValue(), 1.0, 0.0); + assertEquals(out.getDouble("y").doubleValue(), 10.0, 0.0); + + out = tuples.get(1); + + assertEquals(out.getDouble("x").doubleValue(), 2.0, 0.0); + assertEquals(out.getDouble("y").doubleValue(), 11.0, 0.0); + + out = tuples.get(2); + + assertEquals(out.getDouble("x").doubleValue(), 3.0, 0.0); + assertEquals(out.getDouble("y").doubleValue(), 12.0, 0.0); + + out = tuples.get(3); + + assertEquals(out.getDouble("x").doubleValue(), 4.0, 0.0); + assertEquals(out.getDouble("y").doubleValue(), 13.0, 0.0); + + } + + @Test public void testMatrixMath() throws Exception { String cexpr = "let(echo=true, a=matrix(array(1.5, 2.5, 3.5), array(4.5,5.5,6.5)), " + From 0469eff1539fa1c2ddb2c81b73aa63dfc418d99a Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Thu, 27 Dec 2018 18:03:50 -0800 Subject: [PATCH 58/88] SOLR-12028: Catching up with annotations after recent Solr test work --- .../apache/solr/ltr/TestLTROnSolrCloud.java | 2 +- .../solr/cloud/BasicDistributedZkTest.java | 1 + .../cloud/ChaosMonkeyNothingIsSafeTest.java | 3 +-- .../apache/solr/cloud/DeleteReplicaTest.java | 5 +++++ .../solr/cloud/DistribCursorPagingTest.java | 2 +- .../cloud/FullSolrCloudDistribCmdsTest.java | 1 + .../apache/solr/cloud/HttpPartitionTest.java | 5 ++--- .../cloud/LegacyCloudClusterPropTest.java | 1 + .../apache/solr/cloud/MoveReplicaTest.java | 2 ++ .../solr/cloud/MultiThreadedOCPTest.java | 1 + .../solr/cloud/ReplicationFactorTest.java | 2 +- .../solr/cloud/RestartWhileUpdatingTest.java | 1 + .../solr/cloud/TestCloudConsistency.java | 3 +-- .../TestStressCloudBlindAtomicUpdates.java | 2 +- .../cloud/TriLevelCompositeIdRoutingTest.java | 1 + .../solr/cloud/UnloadDistributedZkTest.java | 2 +- .../CollectionsAPIDistributedZkTest.java | 2 +- .../cloud/api/collections/ShardSplitTest.java | 2 +- .../autoscaling/AutoScalingHandlerTest.java | 2 +- .../autoscaling/ComputePlanActionTest.java | 4 ++-- .../MetricTriggerIntegrationTest.java | 2 +- .../SearchRateTriggerIntegrationTest.java | 2 +- .../autoscaling/TriggerIntegrationTest.java | 1 + .../TriggerSetPropertiesIntegrationTest.java | 1 + .../sim/TestSimComputePlanAction.java | 2 +- .../sim/TestSimExecutePlanAction.java | 3 +-- .../autoscaling/sim/TestSimPolicyCloud.java | 1 + .../sim/TestSimTriggerIntegration.java | 1 + .../cloud/cdcr/CdcrOpsAndBoundariesTest.java | 2 +- .../cloud/cdcr/CdcrWithNodesRestartsTest.java | 2 +- .../hdfs/HdfsBasicDistributedZk2Test.java | 3 +-- .../HdfsChaosMonkeyNothingIsSafeTest.java | 3 +-- .../hdfs/HdfsChaosMonkeySafeLeaderTest.java | 3 +-- .../solr/cloud/hdfs/HdfsSyncSliceTest.java | 3 +-- ...fsTlogReplayBufferedWhileIndexingTest.java | 3 +-- .../solr/cloud/hdfs/StressHdfsTest.java | 2 +- .../solr/handler/TestReplicationHandler.java | 3 +-- .../handler/TestSystemCollAutoCreate.java | 1 + .../admin/ZookeeperStatusHandlerTest.java | 2 +- .../CustomHighlightComponentTest.java | 2 +- .../DistributedDebugComponentTest.java | 2 +- .../solr/index/hdfs/CheckHdfsIndexTest.java | 3 +-- .../apache/solr/search/TestRecoveryHdfs.java | 3 +-- .../solr/search/TestStressRecovery.java | 4 ++-- .../security/BasicAuthIntegrationTest.java | 1 + .../store/blockcache/BlockDirectoryTest.java | 3 +-- .../TestDocTermOrdsUninvertLimit.java | 4 ++-- .../apache/solr/update/TestHdfsUpdateLog.java | 3 +-- .../update/TestInPlaceUpdatesDistrib.java | 1 + .../solr/client/solrj/SolrExceptionTest.java | 2 +- .../solrj/beans/TestDocumentObjectBinder.java | 4 ++-- .../embedded/LargeVolumeBinaryJettyTest.java | 3 +-- .../solrj/embedded/LargeVolumeJettyTest.java | 3 +-- .../impl/CloudSolrClientBuilderTest.java | 12 ++++++------ .../CloudSolrClientMultiConstructorTest.java | 6 +++--- ...ConcurrentUpdateSolrClientBuilderTest.java | 2 +- .../client/solrj/impl/HttpClientUtilTest.java | 6 +++--- .../solrj/io/stream/MathExpressionTest.java | 2 +- .../solrj/io/stream/StreamDecoratorTest.java | 19 ++++++++++--------- .../StreamExpressionToExpessionTest.java | 4 ++-- .../StreamExpressionToExplanationTest.java | 4 ++-- .../request/TestCollectionAdminRequest.java | 10 +++++----- .../solrj/request/TestUpdateRequestCodec.java | 6 +++--- .../solrj/request/TestV1toV2ApiMapper.java | 6 +++--- .../solrj/response/QueryResponseTest.java | 8 ++++---- .../response/TestDelegationTokenResponse.java | 4 ++-- .../solr/common/TestToleratedUpdateError.java | 2 +- .../solr/common/cloud/SolrZkClientTest.java | 1 + .../cloud/TestCloudCollectionsListeners.java | 6 +++--- .../cloud/TestCollectionStateWatchers.java | 1 + .../solr/common/util/NamedListTest.java | 4 ++-- .../solr/common/util/TestFastInputStream.java | 2 +- .../solr/common/util/TestNamedListCodec.java | 8 ++++---- 73 files changed, 121 insertions(+), 114 deletions(-) diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java index 85563e68a2eb..0d000ce8923c 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java @@ -73,7 +73,7 @@ public void tearDown() throws Exception { @Test // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testSimpleQuery() throws Exception { // will randomly pick a configuration with [1..5] shards and [1..3] replicas diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index c95ae85675f0..a34170accf3d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -184,6 +184,7 @@ protected void setDistributedParams(ModifiableSolrParams params) { @Test @ShardsFixed(num = 4) + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { // setLoggingLevel(null); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 24d5217ff0a8..aceb9d6e2f2d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Set; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrQuery; @@ -140,7 +139,7 @@ protected CloudSolrClient createCloudClient(String defaultCollection, int socket @Test //05-Jul-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018 - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void test() throws Exception { // None of the operations used here are particularly costly, so this should work. // Using this low timeout will also help us catch index stalling. diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java index b3186c2be416..9e62a387f9d1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java @@ -88,6 +88,7 @@ public void tearDown() throws Exception { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void deleteLiveReplicaTest() throws Exception { final String collectionName = "delLiveColl"; @@ -176,6 +177,7 @@ public void deleteReplicaByCount() throws Exception { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void deleteReplicaByCountForAllShards() throws Exception { final String collectionName = "deleteByCountNew"; @@ -249,6 +251,7 @@ private void deleteReplicaFromClusterState(String legacyCloud) throws Exception @Test @Slow + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void raceConditionOnDeleteAndRegisterReplica() throws Exception { raceConditionOnDeleteAndRegisterReplica("false"); CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, null).process(cluster.getSolrClient()); @@ -256,11 +259,13 @@ public void raceConditionOnDeleteAndRegisterReplica() throws Exception { @Test @Slow + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void raceConditionOnDeleteAndRegisterReplicaLegacy() throws Exception { raceConditionOnDeleteAndRegisterReplica("true"); CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, null).process(cluster.getSolrClient()); } + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void raceConditionOnDeleteAndRegisterReplica(String legacyCloud) throws Exception { CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, legacyCloud).process(cluster.getSolrClient()); diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java index 59be08a2f095..9e0289ec9569 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java @@ -76,7 +76,7 @@ protected String getCloudSolrConfig() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 public void test() throws Exception { boolean testFinished = false; try { diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java index c7cc9e42c22a..f42e987a17eb 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java @@ -69,6 +69,7 @@ public FullSolrCloudDistribCmdsTest() { @Test @ShardsFixed(num = 6) // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { handle.clear(); handle.put("timestamp", SKIPVAL); diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java index b8a6048d130e..c7c0e34488ef 100644 --- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java @@ -32,7 +32,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.JSONTestUtil; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; @@ -73,7 +72,7 @@ @Slow @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18 public class HttpPartitionTest extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -133,7 +132,7 @@ public JettySolrRunner createJetty(File solrHome, String dataDir, } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") public void test() throws Exception { waitForThingsToLevelOut(30000); diff --git a/solr/core/src/test/org/apache/solr/cloud/LegacyCloudClusterPropTest.java b/solr/core/src/test/org/apache/solr/cloud/LegacyCloudClusterPropTest.java index 0c631e4158b4..ed017937bcdf 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LegacyCloudClusterPropTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LegacyCloudClusterPropTest.java @@ -73,6 +73,7 @@ public void afterTest() throws Exception { @Test //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testCreateCollectionSwitchLegacyCloud() throws Exception { createAndTest("legacyTrue", true); createAndTest("legacyFalse", false); diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index 56b0b458d14f..a8797b6ccc8a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -97,6 +97,7 @@ public void afterTest() throws Exception { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove; log.info("total_jettys: " + cluster.getJettySolrRunners().size()); @@ -239,6 +240,7 @@ public void test() throws Exception { @Test // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 17-Mar-2018 This JIRA is fixed, but this test still fails //17-Aug-2018 commented @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testFailedMove() throws Exception { String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove; int REPLICATION = 2; diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java index a688897d37ac..0a3efa3ac094 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java @@ -62,6 +62,7 @@ public MultiThreadedOCPTest() { @Test // commented 20-July-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { testParallelCollectionAPICalls(); testTaskExclusivity(); diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java index 9feadfe56b97..6fe472c69e68 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java @@ -77,7 +77,7 @@ public JettySolrRunner createJetty(File solrHome, String dataDir, } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 public void test() throws Exception { log.info("replication factor test running"); waitForThingsToLevelOut(30000); diff --git a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java index f33e01fe996d..eb64f6dd4329 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java @@ -75,6 +75,7 @@ public static void afterRestartWhileUpdatingTest() { @Test //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { handle.clear(); handle.put("timestamp", SKIPVAL); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java index 66ebb2c2201a..a7419256db1c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.JSONTestUtil; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.cloud.SocketProxy; @@ -99,7 +98,7 @@ public void testOutOfSyncReplicasCannotBecomeLeader() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testOutOfSyncReplicasCannotBecomeLeaderAfterRestart() throws Exception { testOutOfSyncReplicasCannotBecomeLeader(true); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java index 366d578a86ea..89c0eaf0e6fa 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java @@ -188,7 +188,7 @@ private void startTestInjection() { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") public void test_dv() throws Exception { String field = "long_dv"; checkExpectedSchemaField(map("name", field, diff --git a/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java b/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java index 2710eafc9755..b78a765f66c9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/TriLevelCompositeIdRoutingTest.java @@ -57,6 +57,7 @@ public TriLevelCompositeIdRoutingTest() { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { boolean testFinished = false; try { diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 3e0e71ae2e46..943774828ea9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -65,7 +65,7 @@ public UnloadDistributedZkTest() { @Test //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void test() throws Exception { testCoreUnloadAndLeaders(); // long diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java index 42a2e4092cb3..42d05f090949 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java @@ -419,7 +419,7 @@ public void testCreateNodeSet() throws Exception { @Test //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018 SOLR-12028 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018 SOLR-12028 public void testCollectionsAPI() throws Exception { // create new collections rapid fire diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java index 6098ed861558..97512a87e29f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java @@ -281,7 +281,7 @@ private int assertConsistentReplicas(Slice shard) throws SolrServerException, IO */ @Test //05-Jul-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018 public void testSplitAfterFailedSplit() throws Exception { waitForThingsToLevelOut(15); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java index 296fde74ff78..e8e796d29d55 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java @@ -657,7 +657,7 @@ public void testPolicyAndPreferences() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 public void testReadApi() throws Exception { CloudSolrClient solrClient = cluster.getSolrClient(); // first trigger diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java index 8b5efd7eaac6..574b25fe0606 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java @@ -252,7 +252,7 @@ static String getNodeStateProviderState() { } - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testNodeWithMultipleReplicasLost() throws Exception { // start 3 more nodes cluster.startJettySolrRunner(); @@ -525,7 +525,7 @@ public void testNodeAddedTriggerWithAddReplicaPreferredOp_1Shard() throws Except } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard() throws Exception { String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard"; int numShards = 2; diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java index 13e04cc659d2..07b93b90a29c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java @@ -82,7 +82,7 @@ public static void setupCluster() throws Exception { @Test // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testMetricTrigger() throws Exception { String collectionName = "testMetricTrigger"; CloudSolrClient solrClient = cluster.getSolrClient(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java index f5abc31c2b44..08b45ca6eaa2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java @@ -275,7 +275,7 @@ public void testAboveSearchRate() throws Exception { @Test //17-Aug-2018 commented @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018 public void testBelowSearchRate() throws Exception { CloudSolrClient solrClient = cluster.getSolrClient(); String COLL1 = "belowRate_collection"; diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java index 19db39815cca..e5fe1b061a02 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java @@ -172,6 +172,7 @@ private void deleteChildrenRecursively(String path) throws Exception { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testTriggerThrottling() throws Exception { // for this test we want to create two triggers so we must assert that the actions were created twice actionInitCalled = new CountDownLatch(2); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java index 9f3cb856b7bd..c36d5e459780 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java @@ -64,6 +64,7 @@ public static void setupCluster() throws Exception { * Test that we can add/remove triggers to a scheduler, and change the config on the fly, and still get * expected behavior */ + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testSetProperties() throws Exception { final JettySolrRunner runner = cluster.getJettySolrRunner(0); final SolrResourceLoader resourceLoader = runner.getCoreContainer().getResourceLoader(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java index ff08a3712481..f82c5fe98940 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java @@ -256,7 +256,7 @@ public void testNodeWithMultipleReplicasLost() throws Exception { @Test //17-Aug-2018 commented @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testNodeAdded() throws Exception { AssertingTriggerAction.expectedNode = null; SolrClient solrClient = cluster.simGetSolrClient(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java index 37f1007d165e..e42510cd2343 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java @@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; @@ -82,7 +81,7 @@ public void printState() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018 public void testExecute() throws Exception { SolrClient solrClient = cluster.simGetSolrClient(); String collectionName = "testExecute"; diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java index 185c36e58c28..9b68d36f6198 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java @@ -112,6 +112,7 @@ public void testDataProviderPerReplicaDetails() throws Exception { } + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testCreateCollectionAddReplica() throws Exception { SolrClient solrClient = cluster.simGetSolrClient(); String nodeId = cluster.getSimClusterStateProvider().simGetRandomNode(); diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java index 968ed2815da9..8dab189d9a4f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java @@ -338,6 +338,7 @@ public void testNodeLostTriggerRestoreState() throws Exception { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testNodeAddedTriggerRestoreState() throws Exception { // for this test we want to update the trigger so we must assert that the actions were created twice actionInitCalled = new CountDownLatch(2); diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java index 6c116ea29d79..41da87d14f5f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java @@ -64,7 +64,7 @@ public void after() throws Exception { * Check the ops statistics. */ @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testOps() throws Exception { createCollections(); diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrWithNodesRestartsTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrWithNodesRestartsTest.java index 4888eb744e51..1db9b4f917e0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrWithNodesRestartsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrWithNodesRestartsTest.java @@ -204,7 +204,7 @@ public void testReplicationAfterRestart() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testReplicationAfterLeaderChange() throws Exception { createCollections(); CdcrTestsUtil.cdcrStart(sourceSolrClient); diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java index 8810d005f4ab..0c7081b3f394 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.BasicDistributedZk2Test; import org.apache.solr.util.BadHdfsThreadsFilter; @@ -35,7 +34,7 @@ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) // commented 20-July-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018 -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 public class HdfsBasicDistributedZk2Test extends BasicDistributedZk2Test { private static MiniDFSCluster dfsCluster; diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java index 76667981e0de..076c678f6398 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase.BadApple; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest; import org.apache.solr.util.BadHdfsThreadsFilter; @@ -34,7 +33,7 @@ @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028, https://issues.apache.org/jira/browse/SOLR-10191") +// commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028, https://issues.apache.org/jira/browse/SOLR-10191") public class HdfsChaosMonkeyNothingIsSafeTest extends ChaosMonkeyNothingIsSafeTest { private static MiniDFSCluster dfsCluster; diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java index 0c03885a93a2..517d93262be1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.ChaosMonkeySafeLeaderTest; import org.apache.solr.util.BadHdfsThreadsFilter; @@ -34,7 +33,7 @@ @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public class HdfsChaosMonkeySafeLeaderTest extends ChaosMonkeySafeLeaderTest { private static MiniDFSCluster dfsCluster; diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java index 5387d1ec5769..49998472f46b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.SyncSliceTest; import org.apache.solr.util.BadHdfsThreadsFilter; @@ -34,7 +33,7 @@ @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 public class HdfsSyncSliceTest extends SyncSliceTest { private static MiniDFSCluster dfsCluster; diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java index d5015a3e31c1..4986090b5b64 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest; import org.apache.solr.util.BadHdfsThreadsFilter; @@ -32,7 +31,7 @@ @Slow @Nightly // 12-Jun-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java index 77d3410d1a44..b8aa53a12db3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java @@ -61,7 +61,7 @@ @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018 @Nightly public class StressHdfsTest extends BasicDistributedZkTest { diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index f2b0b398874a..541092439765 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -43,7 +43,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Constants; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.util.TestUtil; import org.apache.solr.BaseDistributedSearchTestCase; @@ -92,7 +91,7 @@ @Slow @SuppressSSL // Currently unknown why SSL does not work with this test // commented 20-July-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 public class TestReplicationHandler extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java b/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java index cadda583d834..ccc2e51c75ab 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java +++ b/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java @@ -22,6 +22,7 @@ import org.apache.solr.common.cloud.DocCollection; public class TestSystemCollAutoCreate extends AbstractFullDistribZkTestBase { + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testAutoCreate() throws Exception { TestBlobHandler.checkBlobPost(cloudJettys.get(0).jetty.getBaseUrl().toExternalForm(), cloudClient); DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system"); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java index b75873fece48..3e9dec641da5 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java @@ -63,7 +63,7 @@ public void tearDown() throws Exception { NOTE: We do not currently test with multiple zookeepers, but the only difference is that there are multiple "details" objects and mode is "ensemble"... */ @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018 public void monitorZookeeper() throws IOException, SolrServerException, InterruptedException, ExecutionException, TimeoutException { URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); HttpSolrClient solr = new HttpSolrClient.Builder(baseUrl.toString()).build(); diff --git a/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java index 69edab87bc4e..0dd9ee5dc293 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java @@ -126,7 +126,7 @@ public static void setupCluster() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void test() throws Exception { // determine custom search handler name (the exact name should not matter) diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java index 245e3e07d2b0..f1daf395e31c 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java @@ -344,7 +344,7 @@ private void verifyDebugSections(SolrQuery query, SolrClient client) throws Solr } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testCompareWithNonDistributedRequest() throws SolrServerException, IOException { SolrQuery query = new SolrQuery(); query.setQuery("id:1 OR id:2"); diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java index d4f2307c47a4..d1d3f722b5f4 100644 --- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java +++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.lucene.index.BaseTestCheckIndex; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.cloud.AbstractFullDistribZkTestBase; @@ -43,7 +42,7 @@ @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase { private static MiniDFSCluster dfsCluster; private static Path path; diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java index df244bed8194..75f301281c2f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java @@ -41,7 +41,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.util.IOUtils; @@ -66,7 +65,7 @@ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) // TODO: longer term this should be combined with TestRecovery somehow ?? -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 public class TestRecoveryHdfs extends SolrTestCaseJ4 { // means that we've seen the leader and have version info (i.e. we are a non-leader replica) diff --git a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java index 61d808f781cd..40053ab34524 100644 --- a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java +++ b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java @@ -68,8 +68,8 @@ public void afterClass() { // This version simulates updates coming from the leader and sometimes being reordered // and tests the ability to buffer updates and apply them later @Test -// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018 + // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018 public void testStressRecovery() throws Exception { assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS); diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index fb4a363f41f7..6261e245fa57 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -91,6 +91,7 @@ public void tearDownCluster() throws Exception { @Test //commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testBasicAuth() throws Exception { boolean isUseV2Api = random().nextBoolean(); String authcPrefix = "/admin/authentication"; diff --git a/solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java b/solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java index 6f9f7fc80d23..bba7cc5ac4a8 100644 --- a/solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java +++ b/solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java @@ -30,13 +30,12 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.MergeInfo; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.junit.After; import org.junit.Before; import org.junit.Test; -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 public class BlockDirectoryTest extends SolrTestCaseJ4 { private static class MapperCache implements Cache { diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java index 4868d46be0ba..278b2c51e8bd 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java @@ -38,8 +38,8 @@ public class TestDocTermOrdsUninvertLimit extends LuceneTestCase { * New limit is 2^31, which is not very realistic to unit-test. */ @SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"}) @Nightly -// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testTriggerUnInvertLimit() throws IOException { final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java index 25528d11bebd..62bd1d8bc636 100644 --- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java +++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java @@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.util.IOUtils; @@ -38,7 +37,7 @@ @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 public class TestHdfsUpdateLog extends SolrTestCaseJ4 { private static MiniDFSCluster dfsCluster; diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java index 72dae068e912..52770175aee9 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java @@ -125,6 +125,7 @@ public TestInPlaceUpdatesDistrib() throws Exception { @SuppressWarnings("unchecked") //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void test() throws Exception { waitForRecoveriesToFinish(true); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java index c3dec85db56b..70721604bc82 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExceptionTest.java @@ -32,7 +32,7 @@ public class SolrExceptionTest extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSolrException() throws Throwable { // test a connection to a solr server that probably doesn't exist // this is a very simple test and most of the test should be considered verified diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/beans/TestDocumentObjectBinder.java b/solr/solrj/src/test/org/apache/solr/client/solrj/beans/TestDocumentObjectBinder.java index 327a034dcccc..94c77d25f5dc 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/beans/TestDocumentObjectBinder.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/beans/TestDocumentObjectBinder.java @@ -37,7 +37,7 @@ public class TestDocumentObjectBinder extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSimple() throws Exception { DocumentObjectBinder binder = new DocumentObjectBinder(); XMLResponseParser parser = new XMLResponseParser(); @@ -82,7 +82,7 @@ public void testSingleVal4Array() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testDynamicFieldBinding() { DocumentObjectBinder binder = new DocumentObjectBinder(); XMLResponseParser parser = new XMLResponseParser(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeBinaryJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeBinaryJettyTest.java index ebe2693d70d9..5b5bd111b691 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeBinaryJettyTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeBinaryJettyTest.java @@ -16,7 +16,6 @@ */ package org.apache.solr.client.solrj.embedded; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.LargeVolumeTestBase; import org.junit.BeforeClass; @@ -24,7 +23,7 @@ * @see org.apache.solr.client.solrj.impl.BinaryRequestWriter * @see org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec */ -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public class LargeVolumeBinaryJettyTest extends LargeVolumeTestBase { @BeforeClass public static void beforeTest() throws Exception { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java index 5c7f36ae2882..e7cb58f4d1f4 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java @@ -16,12 +16,11 @@ */ package org.apache.solr.client.solrj.embedded; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.LargeVolumeTestBase; import org.junit.BeforeClass; // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 +// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public class LargeVolumeJettyTest extends LargeVolumeTestBase { @BeforeClass public static void beforeTest() throws Exception { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java index 833d4b3603c3..c8729f506a59 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java @@ -40,7 +40,7 @@ public void testNoZkHostSpecified() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSingleZkHostSpecified() throws IOException { try(CloudSolrClient createdClient = new Builder(Collections.singletonList(ANY_ZK_HOST), Optional.of(ANY_CHROOT)) .build()) { @@ -51,7 +51,7 @@ public void testSingleZkHostSpecified() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSeveralZkHostsSpecifiedSingly() throws IOException { final List zkHostList = new ArrayList<>(); zkHostList.add(ANY_ZK_HOST); zkHostList.add(ANY_OTHER_ZK_HOST); @@ -65,7 +65,7 @@ public void testSeveralZkHostsSpecifiedSingly() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSeveralZkHostsSpecifiedTogether() throws IOException { final ArrayList zkHosts = new ArrayList(); zkHosts.add(ANY_ZK_HOST); @@ -79,7 +79,7 @@ public void testSeveralZkHostsSpecifiedTogether() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testByDefaultConfiguresClientToSendUpdatesOnlyToShardLeaders() throws IOException { try(CloudSolrClient createdClient = new Builder(Collections.singletonList(ANY_ZK_HOST), Optional.of(ANY_CHROOT)).build()) { assertTrue(createdClient.isUpdatesToLeaders() == true); @@ -87,7 +87,7 @@ public void testByDefaultConfiguresClientToSendUpdatesOnlyToShardLeaders() throw } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testIsDirectUpdatesToLeadersOnlyDefault() throws IOException { try(CloudSolrClient createdClient = new Builder(Collections.singletonList(ANY_ZK_HOST), Optional.of(ANY_CHROOT)).build()) { assertFalse(createdClient.isDirectUpdatesToLeadersOnly()); @@ -95,7 +95,7 @@ public void testIsDirectUpdatesToLeadersOnlyDefault() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void test0Timeouts() throws IOException { try(CloudSolrClient createdClient = new Builder(Collections.singletonList(ANY_ZK_HOST), Optional.empty()) .withSocketTimeout(0) diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientMultiConstructorTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientMultiConstructorTest.java index d92305366ad4..3e6a22a47aac 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientMultiConstructorTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientMultiConstructorTest.java @@ -37,7 +37,7 @@ public class CloudSolrClientMultiConstructorTest extends LuceneTestCase { Collection hosts; @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testZkConnectionStringSetterWithValidChroot() throws IOException { boolean setOrList = random().nextBoolean(); int numOfZKServers = TestUtil.nextInt(random(), 1, 5); @@ -76,7 +76,7 @@ public void testZkConnectionStringSetterWithValidChroot() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testZkConnectionStringConstructorWithValidChroot() throws IOException { int numOfZKServers = TestUtil.nextInt(random(), 1, 5); boolean withChroot = random().nextBoolean(); @@ -104,7 +104,7 @@ public void testZkConnectionStringConstructorWithValidChroot() throws IOExceptio } @Test(expected = IllegalArgumentException.class) - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testBadChroot() { final List zkHosts = new ArrayList<>(); zkHosts.add("host1:2181"); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java index 65c2f1f11372..30769da10dd1 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java @@ -32,7 +32,7 @@ public void testRejectsMissingBaseSolrUrl() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testMissingQueueSize() { try (ConcurrentUpdateSolrClient client = new Builder("someurl").build()){ // Do nothing as we just need to test that the only mandatory parameter for building the client diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java index 8eaba22cd30f..4e3ae44805f4 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClientUtilTest.java @@ -46,7 +46,7 @@ public void resetHttpClientBuilder() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSSLSystemProperties() throws IOException { assertNotNull("HTTPS scheme could not be created using system defaults", @@ -85,7 +85,7 @@ private void assertSSLHostnameVerifier(Class expecte } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testToBooleanDefaultIfNull() throws Exception { assertFalse(HttpClientUtil.toBooleanDefaultIfNull(Boolean.FALSE, true)); assertTrue(HttpClientUtil.toBooleanDefaultIfNull(Boolean.TRUE, false)); @@ -94,7 +94,7 @@ public void testToBooleanDefaultIfNull() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testToBooleanObject() throws Exception { assertEquals(Boolean.TRUE, HttpClientUtil.toBooleanObject("true")); assertEquals(Boolean.TRUE, HttpClientUtil.toBooleanObject("TRUE")); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java index 8e973d299ef8..1e3c40ba792a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java @@ -1760,7 +1760,7 @@ public void testProbabilityRange() throws Exception { @Test // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 public void testDistributions() throws Exception { String cexpr = "let(a=normalDistribution(10, 2), " + "b=sample(a, 250), " + diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 997561caf1b9..560cb685ff0e 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -662,7 +662,7 @@ public void testHavingStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelHavingStream() throws Exception { SolrClientCache solrClientCache = new SolrClientCache(); @@ -873,7 +873,7 @@ public void testFetchStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelFetchStream() throws Exception { new UpdateRequest() @@ -1385,7 +1385,7 @@ public void testParallelShuffleStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelReducerStream() throws Exception { new UpdateRequest() @@ -1518,7 +1518,7 @@ public void testParallelRankStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelMergeStream() throws Exception { new UpdateRequest() @@ -1570,7 +1570,7 @@ public void testParallelMergeStream() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testParallelRollupStream() throws Exception { new UpdateRequest() @@ -2326,7 +2326,7 @@ public void testPriorityStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelPriorityStream() throws Exception { Assume.assumeTrue(!useAlias); @@ -2495,7 +2495,7 @@ public void testUpdateStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelUpdateStream() throws Exception { CollectionAdminRequest.createCollection("parallelDestinationCollection", "conf", 2, 1).process(cluster.getSolrClient()); @@ -2594,7 +2594,7 @@ public void testParallelUpdateStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testParallelDaemonUpdateStream() throws Exception { CollectionAdminRequest.createCollection("parallelDestinationCollection1", "conf", 2, 1).process(cluster.getSolrClient()); @@ -2981,6 +2981,7 @@ public void testCommitStream() throws Exception { } @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testParallelCommitStream() throws Exception { CollectionAdminRequest.createCollection("parallelDestinationCollection", "conf", 2, 1).process(cluster.getSolrClient()); @@ -3512,7 +3513,7 @@ public void testStream() throws Exception { } @Test - @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 + // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 public void testExecutorStream() throws Exception { CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); cluster.waitForActiveCollection("workQueue", 2, 2); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java index 3dd426176b17..3fb601411f3a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java @@ -107,7 +107,7 @@ public void testSelectStream() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testDaemonStream() throws Exception { String expressionString; @@ -220,7 +220,7 @@ public void testReducerStream() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testUpdateStream() throws Exception { StreamExpression expression = StreamExpressionParser.parse("update(" + "collection2, " diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java index d1ceebc7a08f..b7307e446399 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java @@ -88,7 +88,7 @@ public void testSelectStream() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testDaemonStream() throws Exception { // Basic test try (DaemonStream stream = new DaemonStream(StreamExpressionParser.parse("daemon(search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"), id=\"blah\", runInterval=\"1000\", queueSize=\"100\")"), factory)) { @@ -175,7 +175,7 @@ public void testReducerStream() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testUpdateStream() throws Exception { StreamExpression expression = StreamExpressionParser.parse("update(" + "collection2, " diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java index 5c62c535cd52..a554aea75ff4 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCollectionAdminRequest.java @@ -28,7 +28,7 @@ public class TestCollectionAdminRequest extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testInvalidCollectionNameRejectedWhenCreatingCollection() { final SolrException e = expectThrows(SolrException.class, () -> { CollectionAdminRequest.createCollection("invalid$collection@name", null, 1, 1); @@ -40,7 +40,7 @@ public void testInvalidCollectionNameRejectedWhenCreatingCollection() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testInvalidShardNamesRejectedWhenCreatingImplicitCollection() { final SolrException e = expectThrows(SolrException.class, () -> { CollectionAdminRequest.createCollectionWithImplicitRouter("fine", "fine", "invalid$shard@name",1,0,0); @@ -52,7 +52,7 @@ public void testInvalidShardNamesRejectedWhenCreatingImplicitCollection() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testInvalidShardNamesRejectedWhenCallingSetShards() { CollectionAdminRequest.Create request = CollectionAdminRequest.createCollectionWithImplicitRouter("fine",null,"fine",1); final SolrException e = expectThrows(SolrException.class, () -> { @@ -65,7 +65,7 @@ public void testInvalidShardNamesRejectedWhenCallingSetShards() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testInvalidAliasNameRejectedWhenCreatingAlias() { final SolrException e = expectThrows(SolrException.class, () -> { CreateAlias createAliasRequest = CollectionAdminRequest.createAlias("invalid$alias@name","ignored"); @@ -77,7 +77,7 @@ public void testInvalidAliasNameRejectedWhenCreatingAlias() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testInvalidShardNameRejectedWhenCreatingShard() { final SolrException e = expectThrows(SolrException.class, () -> { CreateShard createShardRequest = CollectionAdminRequest.createShard("ignored","invalid$shard@name"); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java index 2db72ec1c98c..3aa3d12455ea 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestUpdateRequestCodec.java @@ -44,7 +44,7 @@ public class TestUpdateRequestCodec extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void simple() throws IOException { UpdateRequest updateRequest = new UpdateRequest(); updateRequest.deleteById("*:*"); @@ -111,7 +111,7 @@ public void update(SolrInputDocument document, UpdateRequest req, Integer commit } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testIteratable() throws IOException { final List values = new ArrayList<>(); values.add("iterItem1"); @@ -162,7 +162,7 @@ public void update(SolrInputDocument document, UpdateRequest req, Integer commit @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testBackCompat4_5() throws IOException { UpdateRequest updateRequest = new UpdateRequest(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV1toV2ApiMapper.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV1toV2ApiMapper.java index b5ff3f0c1cbd..398f262b4c24 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV1toV2ApiMapper.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV1toV2ApiMapper.java @@ -31,7 +31,7 @@ public class TestV1toV2ApiMapper extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testCreate() throws IOException { Create cmd = CollectionAdminRequest .createCollection("mycoll", "conf1", 3, 2) @@ -49,7 +49,7 @@ public void testCreate() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testAddReplica() throws IOException { CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard("mycoll", "shard1"); V2Request v2r = V1toV2ApiMapper.convert(addReplica).build(); @@ -60,7 +60,7 @@ public void testAddReplica() throws IOException { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSetCollectionProperty() throws IOException { CollectionAdminRequest.CollectionProp collectionProp = CollectionAdminRequest.setCollectionProperty("mycoll", "prop", "value"); V2Request v2r = V1toV2ApiMapper.convert(collectionProp).build(); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java index 69b2b9f1c133..94b17366fdb3 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java @@ -41,7 +41,7 @@ @Limit(bytes=20000) public class QueryResponseTest extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testRangeFacets() throws Exception { XMLResponseParser parser = new XMLResponseParser(); NamedList response = null; @@ -103,7 +103,7 @@ public void testRangeFacets() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testGroupResponse() throws Exception { XMLResponseParser parser = new XMLResponseParser(); NamedList response = null; @@ -209,7 +209,7 @@ public void testGroupResponse() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSimpleGroupResponse() throws Exception { XMLResponseParser parser = new XMLResponseParser(); NamedList response = null; @@ -255,7 +255,7 @@ public void testSimpleGroupResponse() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testIntervalFacetsResponse() throws Exception { XMLResponseParser parser = new XMLResponseParser(); try(SolrResourceLoader loader = new SolrResourceLoader()) { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java index 2f83c3d90eab..c1f99481bec2 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java @@ -62,7 +62,7 @@ private String getMapJson(String key, Object value) { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testGetResponse() throws Exception { DelegationTokenRequest.Get getRequest = new DelegationTokenRequest.Get(); DelegationTokenResponse.Get getResponse = new DelegationTokenResponse.Get(); @@ -98,7 +98,7 @@ public void testGetResponse() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testRenewResponse() throws Exception { DelegationTokenRequest.Renew renewRequest = new DelegationTokenRequest.Renew("token"); DelegationTokenResponse.Renew renewResponse = new DelegationTokenResponse.Renew(); diff --git a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java index 2c4c53082db8..031b608d651a 100644 --- a/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java +++ b/solr/solrj/src/test/org/apache/solr/common/TestToleratedUpdateError.java @@ -51,7 +51,7 @@ public void testParseMetadataErrorHandling() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testParseMapErrorChecking() { SimpleOrderedMap bogus = new SimpleOrderedMap(); try { diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/SolrZkClientTest.java b/solr/solrj/src/test/org/apache/solr/common/cloud/SolrZkClientTest.java index 126d35b5dcdb..7056de04566e 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/SolrZkClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/SolrZkClientTest.java @@ -109,6 +109,7 @@ public void tearDown() throws Exception { @Test + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testSimpleUpdateACLs() throws KeeperException, InterruptedException { assertTrue("Initial create was in secure mode; please check the test", canRead(defaultClient, PATH)); assertTrue("Credentialed client should always be able to read", canRead(credentialsClient, PATH)); diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCloudCollectionsListeners.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCloudCollectionsListeners.java index d302341a361b..1ef806e3696f 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCloudCollectionsListeners.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCloudCollectionsListeners.java @@ -69,7 +69,7 @@ public void afterTest() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 public void testSimpleCloudCollectionsListener() throws Exception { CloudSolrClient client = cluster.getSolrClient(); @@ -130,7 +130,7 @@ public void testSimpleCloudCollectionsListener() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 public void testCollectionDeletion() throws Exception { CloudSolrClient client = cluster.getSolrClient(); @@ -195,7 +195,7 @@ public void testCollectionDeletion() throws Exception { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 public void testWatchesWorkForBothStateFormats() throws Exception { CloudSolrClient client = cluster.getSolrClient(); diff --git a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java index d063970bc31e..b5a0a8b71f7d 100644 --- a/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java +++ b/solr/solrj/src/test/org/apache/solr/common/cloud/TestCollectionStateWatchers.java @@ -201,6 +201,7 @@ public void testWaitForStateChecksCurrentState() throws Exception { @Test // commented 20-July-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testCanWaitForNonexistantCollection() throws Exception { Future future = waitInBackground("delayed", MAX_WAIT_TIMEOUT, TimeUnit.SECONDS, diff --git a/solr/solrj/src/test/org/apache/solr/common/util/NamedListTest.java b/solr/solrj/src/test/org/apache/solr/common/util/NamedListTest.java index 8a87f489c980..9e103206d65c 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/NamedListTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/NamedListTest.java @@ -65,7 +65,7 @@ public void testRemoveAll() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testRemoveArgs() { NamedList nl = new NamedList<>(); nl.add("key1", "value1-1"); @@ -191,7 +191,7 @@ public void testRecursive() { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testShallowMap() { NamedList nl = new NamedList(); nl.add("key1", "Val1"); diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestFastInputStream.java b/solr/solrj/src/test/org/apache/solr/common/util/TestFastInputStream.java index 863de183fa66..42697ac2f94d 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestFastInputStream.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestFastInputStream.java @@ -31,7 +31,7 @@ */ public class TestFastInputStream extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testgzip() throws Exception { ByteArrayOutputStream b = new ByteArrayOutputStream(); FastOutputStream fos = new FastOutputStream(b); diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestNamedListCodec.java b/solr/solrj/src/test/org/apache/solr/common/util/TestNamedListCodec.java index ce01c09765b1..d53889e7ce7a 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/TestNamedListCodec.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestNamedListCodec.java @@ -32,7 +32,7 @@ public class TestNamedListCodec extends LuceneTestCase { @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testSimple() throws Exception{ NamedList nl = new NamedList(); @@ -96,7 +96,7 @@ public void testSimple() throws Exception{ } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testIterator() throws Exception{ NamedList nl = new NamedList(); @@ -137,7 +137,7 @@ public void testIterator() throws Exception{ } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testIterable() throws Exception { @@ -253,7 +253,7 @@ public Object makeRandom(int lev) { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 + // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void testRandom() throws Exception { // Random r = random; // let's keep it deterministic since just the wrong From a247f50246d13e6d5b12cf743346f733177e480e Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Fri, 28 Dec 2018 10:55:25 +0100 Subject: [PATCH 59/88] LUCENE-8624: int overflow in ByteBuffersDataOutput.size(). --- lucene/CHANGES.txt | 3 +++ .../lucene/store/ByteBuffersDataOutput.java | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 069bb7287f5a..242a45f41215 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -240,6 +240,9 @@ Bug fixes: * LUCENE-8603: Fix the inversion of right ids for additional nouns in the Korean user dictionary. (Yoo Jeongin via Jim Ferenczi) +* LUCENE-8624: int overflow in ByteBuffersDataOutput.size(). (Mulugeta Mammo, + Dawid Weiss) + New Features * LUCENE-8026: ExitableDirectoryReader may now time out queries that run on diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java index 8840f213b539..9bdddd89d5e1 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataOutput.java @@ -248,6 +248,9 @@ public ByteBuffersDataInput toDataInput() { /** * Return a contiguous array with the current content written to the output. The returned * array is always a copy (can be mutated). + * + * If the {@link #size()} of the underlying buffers exceeds maximum size of Java array, an + * {@link RuntimeException} will be thrown. */ public byte[] toArrayCopy() { if (blocks.size() == 0) { @@ -257,6 +260,10 @@ public byte[] toArrayCopy() { // We could try to detect single-block, array-based ByteBuffer here // and use Arrays.copyOfRange, but I don't think it's worth the extra // instance checks. + long size = size(); + if (size > Integer.MAX_VALUE) { + throw new RuntimeException("Data exceeds maximum size of a single byte array: " + size); + } byte [] arr = new byte[Math.toIntExact(size())]; int offset = 0; @@ -288,8 +295,8 @@ public long size() { long size = 0; int blockCount = blocks.size(); if (blockCount >= 1) { - int fullBlockSize = (blockCount - 1) * blockSize(); - int lastBlockSize = blocks.getLast().position(); + long fullBlockSize = (blockCount - 1L) * blockSize(); + long lastBlockSize = blocks.getLast().position(); size = fullBlockSize + lastBlockSize; } return size; @@ -486,10 +493,10 @@ private static int computeBlockSizeBitsFor(long bytes) { * A consumer-based UTF16-UTF8 encoder (writes the input string in smaller buffers.). */ private static int UTF16toUTF8(final CharSequence s, - final int offset, - final int length, - byte[] buf, - IntConsumer bufferFlusher) { + final int offset, + final int length, + byte[] buf, + IntConsumer bufferFlusher) { int utf8Len = 0; int j = 0; for (int i = offset, end = offset + length; i < end; i++) { From 932f38ea86ce29b3344fa90f93161e0560ecabf7 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Fri, 28 Dec 2018 10:54:16 +0100 Subject: [PATCH 60/88] LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList --- lucene/CHANGES.txt | 3 ++ .../lucene/store/ByteBuffersDataInput.java | 10 ++-- .../store/TestByteBuffersDataInput.java | 49 +++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 242a45f41215..7ce6f4643244 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -243,6 +243,9 @@ Bug fixes: * LUCENE-8624: int overflow in ByteBuffersDataOutput.size(). (Mulugeta Mammo, Dawid Weiss) +* LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList. (Mulugeta Mammo, + Dawid Weiss) + New Features * LUCENE-8026: ExitableDirectoryReader may now time out queries that run on diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java index 9b4f4af41f0d..bc6cec5753a7 100644 --- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java @@ -290,17 +290,17 @@ private static List sliceBufferList(List buffers, long o if (buffers.size() == 1) { ByteBuffer cloned = buffers.get(0).asReadOnlyBuffer(); cloned.position(Math.toIntExact(cloned.position() + offset)); - cloned.limit(Math.toIntExact(length + cloned.position())); + cloned.limit(Math.toIntExact(cloned.position() + length)); return Arrays.asList(cloned); } else { long absStart = buffers.get(0).position() + offset; - long absEnd = Math.toIntExact(absStart + length); + long absEnd = absStart + length; int blockBytes = ByteBuffersDataInput.determineBlockPage(buffers); int blockBits = Integer.numberOfTrailingZeros(blockBytes); - int blockMask = (1 << blockBits) - 1; + long blockMask = (1L << blockBits) - 1; - int endOffset = (int) absEnd & blockMask; + int endOffset = Math.toIntExact(absEnd & blockMask); ArrayList cloned = buffers.subList(Math.toIntExact(absStart / blockBytes), @@ -313,7 +313,7 @@ private static List sliceBufferList(List buffers, long o cloned.add(ByteBuffer.allocate(0)); } - cloned.get(0).position((int) absStart & blockMask); + cloned.get(0).position(Math.toIntExact(absStart & blockMask)); cloned.get(cloned.size() - 1).limit(endOffset); return cloned; } diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java index 5d3d7f60dcde..20e3bd2f35e1 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java @@ -21,6 +21,7 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import org.apache.lucene.util.ArrayUtil; @@ -203,4 +204,52 @@ public void testEofOnArrayReadPastBufferSize() throws Exception { in.readBytes(ByteBuffer.allocate(100), 100); }); } + + // https://issues.apache.org/jira/browse/LUCENE-8625 + @Test + public void testSlicingLargeBuffers() throws IOException { + // Simulate a "large" (> 4GB) input by duplicating + // buffers with the same content. + int MB = 1024 * 1024; + byte [] pageBytes = randomBytesOfLength(4 * MB); + ByteBuffer page = ByteBuffer.wrap(pageBytes); + + // Add some head shift on the first buffer. + final int shift = randomIntBetween(0, pageBytes.length / 2); + + final long simulatedLength = + randomLongBetween(0, 2018) + 4L * Integer.MAX_VALUE; + + List buffers = new ArrayList<>(); + long remaining = simulatedLength + shift; + while (remaining > 0) { + ByteBuffer bb = page.duplicate(); + if (bb.remaining() > remaining) { + bb.limit(Math.toIntExact(bb.position() + remaining)); + } + buffers.add(bb); + remaining -= bb.remaining(); + } + buffers.get(0).position(shift); + + ByteBuffersDataInput dst = new ByteBuffersDataInput(buffers); + assertEquals(simulatedLength, dst.size()); + + final long max = dst.size(); + long offset = 0; + for (; offset < max; offset += randomIntBetween(MB, 4 * MB)) { + assertEquals(0, dst.slice(offset, 0).size()); + assertEquals(1, dst.slice(offset, 1).size()); + + long window = Math.min(max - offset, 1024); + ByteBuffersDataInput slice = dst.slice(offset, window); + assertEquals(window, slice.size()); + + // Sanity check of the content against original pages. + for (int i = 0; i < window; i++) { + byte expected = pageBytes[(int) ((shift + offset + i) % pageBytes.length)]; + assertEquals(expected, slice.readByte(i)); + } + } + } } From b30262e69b88b0ac8a7e0cf4563ae896ac746ead Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Fri, 28 Dec 2018 12:23:53 +0000 Subject: [PATCH 61/88] SOLR-12973: Admin UI Nodes view support for replica* replica names. (Daniel Collins, Christine Poerschke, janhoy) --- solr/CHANGES.txt | 2 ++ solr/webapp/web/js/angular/controllers/cloud.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8aba176cd181..6e568aaea1d7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -213,6 +213,8 @@ Improvements * SOLR-12885: BinaryResponseWriter (javabin format) should directly copy from BytesRef to output (noble) +* SOLR-12973: Admin UI "Nodes" view support for replica* replica names. (Daniel Collins, Christine Poerschke, janhoy) + Other Changes ---------------------- diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js index 119f00c631de..d6730d518ec7 100644 --- a/solr/webapp/web/js/angular/controllers/cloud.js +++ b/solr/webapp/web/js/angular/controllers/cloud.js @@ -404,7 +404,7 @@ var nodesSubController = function($scope, Collections, System, Metrics) { if (cores) { for (coreId in cores) { var core = cores[coreId]; - var keyName = "solr.core." + core['core'].replace(/(.*?)_(shard(\d+_?)+)_(replica_?[ntp]?\d+)/, '\$1.\$2.\$4'); + var keyName = "solr.core." + core['core'].replace(/(.*?)_(shard(\d+_?)+)_(replica.*?)/, '\$1.\$2.\$4'); var nodeMetric = m.metrics[keyName]; var size = nodeMetric['INDEX.sizeInBytes']; size = (typeof size !== 'undefined') ? size : 0; From f3076a4f82959a97400d506f70393a13b1fbd31f Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Mon, 31 Dec 2018 11:05:31 +0000 Subject: [PATCH 62/88] SOLR-13096: rename TestRankQueryPlugin to RankQueryTestPlugin --- .../solr/collection1/conf/solrconfig-plugcollector.xml | 2 +- .../{TestRankQueryPlugin.java => RankQueryTestPlugin.java} | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) rename solr/core/src/test/org/apache/solr/search/{TestRankQueryPlugin.java => RankQueryTestPlugin.java} (99%) diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml index 56f8a01de748..b60bd6aa6a64 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml @@ -457,7 +457,7 @@ based HashBitset. --> prefix-${solr.test.sys.prop2}-suffix - + diff --git a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java b/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java similarity index 99% rename from solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java rename to solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java index a678110a7441..bc3839771bc3 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java @@ -64,11 +64,9 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; -import org.junit.Ignore; -@Ignore -public class TestRankQueryPlugin extends QParserPlugin { +public class RankQueryTestPlugin extends QParserPlugin { public QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest req) { From e7ab82b45afe9d12c6c0c7ce8201663eb644242f Mon Sep 17 00:00:00 2001 From: Gus Heck Date: Mon, 31 Dec 2018 07:54:56 -0500 Subject: [PATCH 63/88] SOLR-13086 improve error message in DocumentObjectBinder --- solr/CHANGES.txt | 5 +++++ .../apache/solr/client/solrj/beans/DocumentObjectBinder.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6e568aaea1d7..4e3a11deca39 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -145,6 +145,8 @@ Other Changes * SOLR-12535: Solr no longer accepts index time boosts in JSON provided to Solr. (David Smiley) +* SOLR-13086: Improve the error message reported by DocumentObjectBinder when a setter is not found (Gus Heck) + ================== 7.7.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. @@ -230,6 +232,9 @@ Other Changes * SOLR-12727: Upgrade ZooKeeper dependency to 3.4.13 (Kevin Risden, Erick Erickson, Cao Manh Dat) +* SOLR-13086: Improve the error message reported by DocumentObjectBinder when a setter is not found (Gus Heck) + + ================== 7.6.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java b/solr/solrj/src/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java index e2e65bb89e5a..2e7809323d6f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java @@ -232,7 +232,8 @@ private void storeType() { } else { Class[] params = setter.getParameterTypes(); if (params.length != 1) { - throw new BindingException("Invalid setter method. Must have one and only one parameter"); + throw new BindingException("Invalid setter method (" + setter + + "). A setter must have one and only one parameter but we found " + params.length + " parameters."); } type = params[0]; } From c210caf454e0e54ea84ea10dd6d304a0e5823f4d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 1 Jan 2019 17:50:20 +0000 Subject: [PATCH 64/88] LUCENE-8612: Add Intervals.extend() --- lucene/CHANGES.txt | 4 + .../intervals/DifferenceIntervalFunction.java | 101 --------------- .../intervals/ExtendedIntervalIterator.java | 118 ++++++++++++++++++ .../intervals/ExtendedIntervalsSource.java | 83 ++++++++++++ .../search/intervals/IntervalFunction.java | 24 +++- .../search/intervals/IntervalIterator.java | 6 +- .../search/intervals/IntervalMatches.java | 4 +- .../lucene/search/intervals/Intervals.java | 24 +++- .../search/intervals/TestIntervalQuery.java | 6 + .../search/intervals/TestIntervals.java | 32 +++++ 10 files changed, 290 insertions(+), 112 deletions(-) create mode 100644 lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalIterator.java create mode 100644 lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalsSource.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 7ce6f4643244..e8820cb03250 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -105,6 +105,10 @@ API Changes * LUCENE-8609: Remove IndexWriter#numDocs() and IndexWriter#maxDoc() in favor of IndexWriter#getDocStats(). (Simon Willnauer) +* LUCENE-8612: Intervals.extend() treats an interval as if it covered a wider + span than it actually does, allowing users to force minimum gaps between + intervals in a phrase. (Alan Woodward) + Changes in Runtime Behavior * LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java index def1d03d6fac..8479716305f4 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java @@ -18,7 +18,6 @@ package org.apache.lucene.search.intervals; import java.io.IOException; -import java.util.Objects; /** * A function that takes two interval iterators and combines them to produce a third, @@ -160,106 +159,6 @@ public int nextInterval() throws IOException { } } - /** - * Filters the minuend iterator so that only intervals that do not occur within a set number - * of positions of intervals from the subtrahend iterator are returned - */ - static class NotWithinFunction extends DifferenceIntervalFunction { - - private final int positions; - - NotWithinFunction(int positions) { - this.positions = positions; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NotWithinFunction that = (NotWithinFunction) o; - return positions == that.positions; - } - - @Override - public String toString() { - return "NOTWITHIN/" + positions; - } - - @Override - public int hashCode() { - return Objects.hash(positions); - } - - @Override - public IntervalIterator apply(IntervalIterator minuend, IntervalIterator subtrahend) { - IntervalIterator notWithin = new IntervalIterator() { - - @Override - public int docID() { - return subtrahend.docID(); - } - - @Override - public int nextDoc() throws IOException { - positioned = false; - return subtrahend.nextDoc(); - } - - @Override - public int advance(int target) throws IOException { - positioned = false; - return subtrahend.advance(target); - } - - @Override - public long cost() { - return subtrahend.cost(); - } - - boolean positioned = false; - - @Override - public int start() { - if (positioned == false) - return -1; - int start = subtrahend.start(); - return Math.max(0, start - positions); - } - - @Override - public int end() { - if (positioned == false) - return -1; - int end = subtrahend.end(); - int newEnd = end + positions; - if (newEnd < 0) // check for overflow - return Integer.MAX_VALUE; - return newEnd; - } - - @Override - public int gaps() { - throw new UnsupportedOperationException(); - } - - @Override - public int nextInterval() throws IOException { - if (positioned == false) { - positioned = true; - } - return subtrahend.nextInterval(); - } - - @Override - public float matchCost() { - return subtrahend.matchCost(); - } - - }; - return NON_OVERLAPPING.apply(minuend, notWithin); - } - } - private static class NotContainingIterator extends RelativeIterator { private NotContainingIterator(IntervalIterator minuend, IntervalIterator subtrahend) { diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalIterator.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalIterator.java new file mode 100644 index 000000000000..843b113a8025 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalIterator.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.intervals; + +import java.io.IOException; + +/** + * Wraps an IntervalIterator and extends the bounds of its intervals + * + * Useful for specifying gaps in an ordered iterator; if you want to match + * `a b [2 spaces] c`, you can search for phrase(a, extended(b, 0, 2), c) + * + * An interval with prefix bounds extended by n will skip over matches that + * appear in positions lower than n + */ +class ExtendedIntervalIterator extends IntervalIterator { + + private final IntervalIterator in; + private final int before; + private final int after; + + private boolean positioned; + + /** + * Create a new ExtendedIntervalIterator + * @param in the iterator to wrap + * @param before the number of positions to extend before the delegated interval + * @param after the number of positions to extend beyond the delegated interval + */ + ExtendedIntervalIterator(IntervalIterator in, int before, int after) { + this.in = in; + this.before = before; + this.after = after; + } + + @Override + public int start() { + if (positioned == false) { + return -1; + } + int start = in.start(); + if (start == NO_MORE_INTERVALS) { + return NO_MORE_INTERVALS; + } + return Math.max(0, start - before); + } + + @Override + public int end() { + if (positioned == false) { + return -1; + } + int end = in.end(); + if (end == NO_MORE_INTERVALS) { + return NO_MORE_INTERVALS; + } + end += after; + if (end < 0 || end == NO_MORE_INTERVALS) { + // overflow + end = NO_MORE_INTERVALS - 1; + } + return end; + } + + @Override + public int gaps() { + return in.gaps(); + } + + @Override + public int nextInterval() throws IOException { + positioned = true; + in.nextInterval(); + return start(); + } + + @Override + public float matchCost() { + return in.matchCost(); + } + + @Override + public int docID() { + return in.docID(); + } + + @Override + public int nextDoc() throws IOException { + positioned = false; + return in.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + positioned = false; + return in.advance(target); + } + + @Override + public long cost() { + return in.cost(); + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalsSource.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalsSource.java new file mode 100644 index 000000000000..d4e3bfa56930 --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/ExtendedIntervalsSource.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.search.intervals; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchesIterator; + +class ExtendedIntervalsSource extends IntervalsSource { + + final IntervalsSource source; + private final int before; + private final int after; + + ExtendedIntervalsSource(IntervalsSource source, int before, int after) { + this.source = source; + this.before = before; + this.after = after; + } + + @Override + public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException { + IntervalIterator in = source.intervals(field, ctx); + if (in == null) { + return null; + } + return new ExtendedIntervalIterator(in, before, after); + } + + @Override + public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException { + MatchesIterator in = source.matches(field, ctx, doc); + if (in == null) { + return null; + } + IntervalIterator wrapped = new ExtendedIntervalIterator(IntervalMatches.wrapMatches(in, doc), before, after); + return IntervalMatches.asMatches(wrapped, in, doc); + } + + @Override + public void extractTerms(String field, Set terms) { + source.extractTerms(field, terms); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExtendedIntervalsSource that = (ExtendedIntervalsSource) o; + return before == that.before && + after == that.after && + Objects.equals(source, that.source); + } + + @Override + public int hashCode() { + return Objects.hash(source, before, after); + } + + @Override + public String toString() { + return "EXTEND(" + source + "," + before + "," + after + ")"; + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java index 1a5eab63607e..9460d8dd6579 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java @@ -75,19 +75,19 @@ public int gaps() { @Override public int nextInterval() throws IOException { if (subIterators.get(0).nextInterval() == IntervalIterator.NO_MORE_INTERVALS) - return IntervalIterator.NO_MORE_INTERVALS; + return start = end = IntervalIterator.NO_MORE_INTERVALS; int i = 1; while (i < subIterators.size()) { while (subIterators.get(i).start() <= subIterators.get(i - 1).end()) { if (subIterators.get(i).nextInterval() == IntervalIterator.NO_MORE_INTERVALS) - return IntervalIterator.NO_MORE_INTERVALS; + return start = end = IntervalIterator.NO_MORE_INTERVALS; } if (subIterators.get(i).start() == subIterators.get(i - 1).end() + 1) { i = i + 1; } else { if (subIterators.get(0).nextInterval() == IntervalIterator.NO_MORE_INTERVALS) - return IntervalIterator.NO_MORE_INTERVALS; + return start = end = IntervalIterator.NO_MORE_INTERVALS; i = 1; } } @@ -150,6 +150,9 @@ public int nextInterval() throws IOException { i++; } start = subIterators.get(0).start(); + if (start == NO_MORE_INTERVALS) { + return end = NO_MORE_INTERVALS; + } firstEnd = subIterators.get(0).end(); end = subIterators.get(subIterators.size() - 1).end(); b = subIterators.get(subIterators.size() - 1).start(); @@ -248,7 +251,7 @@ public int nextInterval() throws IOException { if (allowOverlaps == false) { while (hasOverlaps(it)) { if (it.nextInterval() == IntervalIterator.NO_MORE_INTERVALS) - return IntervalIterator.NO_MORE_INTERVALS; + return start = end = IntervalIterator.NO_MORE_INTERVALS; } } queue.add(it); @@ -256,7 +259,7 @@ public int nextInterval() throws IOException { } } if (this.queue.size() < subIterators.length) - return IntervalIterator.NO_MORE_INTERVALS; + return start = end = IntervalIterator.NO_MORE_INTERVALS; // then, minimize it do { start = queue.top().start(); @@ -408,11 +411,17 @@ public IntervalIterator apply(List iterators) { @Override public int start() { + if (bpos == false) { + return NO_MORE_INTERVALS; + } return a.start(); } @Override public int end() { + if (bpos == false) { + return NO_MORE_INTERVALS; + } return a.end(); } @@ -427,12 +436,15 @@ public int nextInterval() throws IOException { return IntervalIterator.NO_MORE_INTERVALS; while (a.nextInterval() != IntervalIterator.NO_MORE_INTERVALS) { while (b.end() < a.end()) { - if (b.nextInterval() == IntervalIterator.NO_MORE_INTERVALS) + if (b.nextInterval() == IntervalIterator.NO_MORE_INTERVALS) { + bpos = false; return IntervalIterator.NO_MORE_INTERVALS; + } } if (b.start() <= a.start()) return a.start(); } + bpos = false; return IntervalIterator.NO_MORE_INTERVALS; } diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java index f819aab1bc2e..305f56cf42b6 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java @@ -48,14 +48,16 @@ public abstract class IntervalIterator extends DocIdSetIterator { /** * The start of the current interval * - * Returns -1 if {@link #nextInterval()} has not yet been called + * Returns -1 if {@link #nextInterval()} has not yet been called and {@link #NO_MORE_INTERVALS} + * once the iterator is exhausted. */ public abstract int start(); /** * The end of the current interval * - * Returns -1 if {@link #nextInterval()} has not yet been called + * Returns -1 if {@link #nextInterval()} has not yet been called and {@link #NO_MORE_INTERVALS} + * once the iterator is exhausted. */ public abstract int end(); diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java index a28f6e47821b..24424d21591e 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java @@ -49,12 +49,12 @@ public boolean next() throws IOException { @Override public int startPosition() { - return source.startPosition(); + return iterator.start(); } @Override public int endPosition() { - return source.endPosition(); + return iterator.end(); } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java index b0a482943a0b..a98adbdc7a36 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java @@ -103,6 +103,27 @@ protected boolean accept(IntervalIterator it) { }; } + /** + * Create an {@link IntervalsSource} that wraps another source, extending its + * intervals by a number of positions before and after. + * + * This can be useful for adding defined gaps in a block query; for example, + * to find 'a b [2 arbitrary terms] c', you can call: + * 
+   *   Intervals.phrase(Intervals.term("a"), Intervals.extend(Intervals.term("b"), 0, 2), Intervals.term("c"));
+   *
+ * + * Note that calling {@link IntervalIterator#gaps()} on iterators returned by this source + * delegates directly to the wrapped iterator, and does not include the extensions. + * + * @param source the source to extend + * @param before how many positions to extend before the delegated interval + * @param after how many positions to extend after the delegated interval + */ + public static IntervalsSource extend(IntervalsSource source, int before, int after) { + return new ExtendedIntervalsSource(source, before, after); + } + /** * Create an ordered {@link IntervalsSource} * @@ -162,7 +183,8 @@ public static IntervalsSource nonOverlapping(IntervalsSource minuend, IntervalsS * @param subtrahend the {@link IntervalsSource} to filter by */ public static IntervalsSource notWithin(IntervalsSource minuend, int positions, IntervalsSource subtrahend) { - return new DifferenceIntervalsSource(minuend, subtrahend, new DifferenceIntervalFunction.NotWithinFunction(positions)); + return new DifferenceIntervalsSource(minuend, Intervals.extend(subtrahend, positions, positions), + DifferenceIntervalFunction.NON_OVERLAPPING); } /** diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java index 18c69a7d5867..61106052e1a3 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java @@ -188,4 +188,10 @@ public void testUnordered() throws IOException { ); checkHits(q, new int[]{3}); } + + public void testDefinedGaps() throws IOException { + Query q = new IntervalQuery(field, + Intervals.phrase(Intervals.term("w1"), Intervals.extend(Intervals.term("w2"), 1, 0))); + checkHits(q, new int[]{ 1, 2, 5 }); + } } diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java index 030855041820..139cea985c9f 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java +++ b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java @@ -124,6 +124,8 @@ private void checkIntervals(IntervalsSource source, String field, int expectedMa i += 2; } assertEquals("Wrong number of endpoints in doc " + id, expected[id].length, i); + assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.start()); + assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.end()); if (i > 0) matchedDocs++; } @@ -504,4 +506,34 @@ public void testNestedMaxGaps() throws IOException { assertMatch(mi, 4, 8, 12, 26); } + public void testDefinedGaps() throws IOException { + IntervalsSource source = Intervals.phrase( + Intervals.term("pease"), + Intervals.extend(Intervals.term("cold"), 1, 1), + Intervals.term("porridge") + ); + checkIntervals(source, "field1", 3, new int[][]{ + {}, + { 3, 7 }, + { 0, 4 }, + {}, + { 3, 7 }, + {} + }); + + MatchesIterator mi = getMatches(source, 1, "field1"); + assertMatch(mi, 3, 7, 20, 55); + MatchesIterator sub = mi.getSubMatches(); + assertNotNull(sub); + assertMatch(sub, 3, 3, 20, 25); + assertMatch(sub, 4, 6, 35, 39); + assertMatch(sub, 7, 7, 47, 55); + + source = Intervals.extend(Intervals.term("w1"), 5, Integer.MAX_VALUE); + checkIntervals(source, "field2", 1, new int[][]{ + {}, {}, {}, {}, {}, + { 0, Integer.MAX_VALUE - 1, 0, Integer.MAX_VALUE - 1, 5, Integer.MAX_VALUE - 1 } + }); + } + } From 313f5b52453a3f2607dcb2f2775032738f565cd8 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Wed, 2 Jan 2019 11:59:00 +0530 Subject: [PATCH 65/88] SOLR-13082: A trigger that creates trigger events more frequently than the cool down period can starve other triggers. This is mitigated to some extent by randomly choosing the trigger to resume after cool down. It is recommended that scheduled triggers not be used for very frequent operations to avoid this problem. --- solr/CHANGES.txt | 5 +++++ .../cloud/autoscaling/ScheduledTriggers.java | 21 +++++++++++++++++-- .../src/solrcloud-autoscaling-triggers.adoc | 12 +++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4e3a11deca39..4d4e641f8dca 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -195,6 +195,11 @@ Bug Fixes * SOLR-13080: The "terms" QParser's "automaton" method semi-required that the input terms/IDs be sorted. This query parser now does this. Unclear if this is a perf issue or actual bug. (Daniel Lowe, David Smiley) +* SOLR-13082: A trigger that creates trigger events more frequently than the cool down period can starve other triggers. + This is mitigated to some extent by randomly choosing the trigger to resume after cool down. It is recommended that + scheduled triggers not be used for very frequent operations to avoid this problem. + (ab, shalin) + Improvements ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java index cad463b0a5b2..05a8020aacd5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java +++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -88,6 +89,18 @@ public class ScheduledTriggers implements Closeable { DEFAULT_PROPERTIES.put(ACTION_THROTTLE_PERIOD_SECONDS, DEFAULT_ACTION_THROTTLE_PERIOD_SECONDS); } + protected static final Random RANDOM; + static { + // We try to make things reproducible in the context of our tests by initializing the random instance + // based on the current seed + String seed = System.getProperty("tests.seed"); + if (seed == null) { + RANDOM = new Random(); + } else { + RANDOM = new Random(seed.hashCode()); + } + } + private final Map scheduledTriggerWrappers = new ConcurrentHashMap<>(); /** @@ -381,9 +394,13 @@ public synchronized void pauseTriggers() { * @lucene.internal */ public synchronized void resumeTriggers(long afterDelayMillis) { - scheduledTriggerWrappers.forEach((s, triggerWrapper) -> { + List> entries = new ArrayList<>(scheduledTriggerWrappers.entrySet()); + Collections.shuffle(entries, RANDOM); + entries.forEach(e -> { + String key = e.getKey(); + TriggerWrapper triggerWrapper = e.getValue(); if (triggerWrapper.scheduledFuture.isCancelled()) { - log.debug("Resuming trigger: {} after {}ms", s, afterDelayMillis); + log.debug("Resuming trigger: {} after {}ms", key, afterDelayMillis); triggerWrapper.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(triggerWrapper, afterDelayMillis, cloudManager.getTimeSource().convertDelay(TimeUnit.SECONDS, triggerDelay.get(), TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); } diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc index d09153705344..cd89d7541cb5 100644 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc +++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc @@ -527,3 +527,15 @@ This trigger applies the `every` date math expression on the `startTime` or the Apart from the common event properties described in the <> section, the trigger adds an additional `actualEventTime` event property which has the actual event time as opposed to the scheduled time. For example, if the scheduled time was `2018-01-31T15:30:00Z` and grace time was `+15MINUTES` then an event may be fired at `2018-01-31T15:45:00Z`. Such an event will have `eventTime` as `2018-01-31T15:30:00Z`, the scheduled time, but the `actualEventTime` property will have a value of `2018-01-31T15:45:00Z`, the actual time. + +.Frequently scheduled events and trigger starvation +[CAUTION] +==== +Be cautious with scheduled triggers that are set to run as or more frequently than the trigger cooldown period (defaults to 5 seconds). + +Solr pauses all triggers for a cooldown period after a trigger fires so that the system has some time to stabilize. An aggressive scheduled trigger can starve all other triggers from +ever executing if a new scheduled event is ready as soon as the cooldown period is over. The same starvation scenario can happen to the scheduled trigger as well. + +Solr randomizes the order in which the triggers are resumed after the cooldown period to mitigate this problem. However, it is recommended that scheduled triggers +are not used with low `every` values and an external scheduling process such as cron be used for such cases instead. +==== \ No newline at end of file From 2a3580e698138267489100e1eca84df63be02857 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 2 Jan 2019 09:37:10 +0100 Subject: [PATCH 66/88] LUCENE-8627: Fix SearchAfter#testQueries to always count the number of hits accurately. --- .../apache/lucene/search/TestSearchAfter.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java b/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java index 4d95c875f0ab..2ce677c84e6f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java @@ -216,14 +216,24 @@ void assertQuery(Query query, Sort sort) throws Exception { if (VERBOSE) { System.out.println("\nassertQuery " + (iter++) + ": query=" + query + " sort=" + sort + " pageSize=" + pageSize); } - final boolean doScores = random().nextBoolean(); + final boolean doScores; + final TopDocsCollector allCollector; if (sort == null) { - all = searcher.search(query, maxDoc); + allCollector = TopScoreDocCollector.create(maxDoc, null, Integer.MAX_VALUE); + doScores = false; } else if (sort == Sort.RELEVANCE) { - all = searcher.search(query, maxDoc, sort, true); + allCollector = TopFieldCollector.create(sort, maxDoc, Integer.MAX_VALUE); + doScores = true; } else { - all = searcher.search(query, maxDoc, sort, doScores); + allCollector = TopFieldCollector.create(sort, maxDoc, Integer.MAX_VALUE); + doScores = random().nextBoolean(); } + searcher.search(query, allCollector); + all = allCollector.topDocs(); + if (doScores) { + TopFieldCollector.populateScores(all.scoreDocs, searcher, query); + } + if (VERBOSE) { System.out.println(" all.totalHits.value=" + all.totalHits.value); int upto = 0; @@ -235,21 +245,28 @@ void assertQuery(Query query, Sort sort) throws Exception { ScoreDoc lastBottom = null; while (pageStart < all.totalHits.value) { TopDocs paged; + final TopDocsCollector pagedCollector; if (sort == null) { if (VERBOSE) { System.out.println(" iter lastBottom=" + lastBottom); } - paged = searcher.searchAfter(lastBottom, query, pageSize); + pagedCollector = TopScoreDocCollector.create(pageSize, lastBottom, Integer.MAX_VALUE); } else { if (VERBOSE) { System.out.println(" iter lastBottom=" + lastBottom); } if (sort == Sort.RELEVANCE) { - paged = searcher.searchAfter(lastBottom, query, pageSize, sort, true); + pagedCollector = TopFieldCollector.create(sort, pageSize, (FieldDoc) lastBottom, Integer.MAX_VALUE); } else { - paged = searcher.searchAfter(lastBottom, query, pageSize, sort, doScores); + pagedCollector = TopFieldCollector.create(sort, pageSize, (FieldDoc) lastBottom, Integer.MAX_VALUE); } } + searcher.search(query, pagedCollector); + paged = pagedCollector.topDocs(); + if (doScores) { + TopFieldCollector.populateScores(paged.scoreDocs, searcher, query); + } + if (VERBOSE) { System.out.println(" " + paged.scoreDocs.length + " hits on page"); } From 30a6a61f18d97403743f2fcbbf67bc0f39ea0b8b Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Wed, 2 Jan 2019 19:44:03 +1100 Subject: [PATCH 67/88] SOLR-12514: Rule-base Authorization plugin skips authorization if querying node does not have collection replica --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/servlet/HttpSolrCall.java | 1 + .../TestSolrCloudWithSecureImpersonation.java | 1 + .../solr/security/BasicAuthIntegrationTest.java | 15 ++++++++++++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4d4e641f8dca..2c900a37b656 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -200,6 +200,8 @@ Bug Fixes scheduled triggers not be used for very frequent operations to avoid this problem. (ab, shalin) +* SOLR-12514: Rule-base Authorization plugin skips authorization if querying node does not have collection replica (noble) + Improvements ---------------------- diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index bb4432c58c8b..b8332448e60d 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -496,6 +496,7 @@ public Action call() throws IOException { handleAdminRequest(); return RETURN; case REMOTEQUERY: + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, new SolrQueryResponse())); remoteQuery(coreUrl + path, resp); return RETURN; case PROCESS: diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java index a149b33ab4a8..1f737993112e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java @@ -312,6 +312,7 @@ public void testProxyNullProxyUser() throws Exception { } @Test + @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13098") public void testForwarding() throws Exception { String collectionName = "forwardingCollection"; miniCluster.uploadConfigSet(TEST_PATH().resolve("collection1/conf"), "conf1"); diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index 6261e245fa57..409b476e014f 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -91,7 +91,7 @@ public void tearDownCluster() throws Exception { @Test //commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 +// @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testBasicAuth() throws Exception { boolean isUseV2Api = random().nextBoolean(); String authcPrefix = "/admin/authentication"; @@ -230,6 +230,19 @@ public void testBasicAuth() throws Exception { del.setCommitWithin(10); del.process(cluster.getSolrClient(), COLLECTION); + //Test for SOLR-12514. Create a new jetty . This jetty does not have the collection. + //Make a request to that jetty and it should fail + JettySolrRunner aNewJetty = cluster.startJettySolrRunner(); + try { + del = new UpdateRequest().deleteByQuery("*:*"); + del.process(aNewJetty.newClient(), COLLECTION); + fail("This should not have succeeded without credentials"); + } catch (HttpSolrClient.RemoteSolrException e) { + assertTrue(e.getMessage().contains("Unauthorized request")); + } finally { + cluster.stopJettySolrRunner(aNewJetty); + } + addDocument("harry","HarryIsUberCool","id", "4"); executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", "harry", "HarryIsUberCool"); From 493a72da443b8004157ecc3ff3bc0cc3397322b1 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Wed, 2 Jan 2019 19:49:40 +1100 Subject: [PATCH 68/88] SOLR-12514: Rule-base Authorization plugin skips authorization if querying node does not have collection replica --- .../test/org/apache/solr/security/BasicAuthIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index 409b476e014f..960f99403303 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -91,7 +91,7 @@ public void tearDownCluster() throws Exception { @Test //commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 -// @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 + @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 public void testBasicAuth() throws Exception { boolean isUseV2Api = random().nextBoolean(); String authcPrefix = "/admin/authentication"; From 7870e1ac50ff5641c2abb374341538ebe51537b1 Mon Sep 17 00:00:00 2001 From: Andrzej Bialecki Date: Wed, 2 Jan 2019 16:55:16 +0100 Subject: [PATCH 69/88] SOLR-13050: Fix another test that could accidentally kill the .system leader node. Improve fallback in SystemLogListener when target collection is not present. --- .../solr/cloud/autoscaling/SystemLogListener.java | 15 +++++++++++---- .../admin/AutoscalingHistoryHandlerTest.java | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java index c6f0e685ca87..09b0865ec90c 100644 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java +++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java @@ -36,6 +36,8 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; @@ -81,6 +83,12 @@ public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { try { + ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); + DocCollection coll = clusterState.getCollectionOrNull(collection); + if (coll == null) { + log.debug("Collection {} missing, skip sending event {}", collection, event); + return; + } SolrInputDocument doc = new SolrInputDocument(); doc.addField(CommonParams.TYPE, DOC_TYPE); doc.addField(SOURCE_FIELD, SOURCE); @@ -118,11 +126,10 @@ public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String cloudManager.request(req); } catch (Exception e) { if ((e instanceof SolrException) && e.getMessage().contains("Collection not found")) { - // relatively benign - log.info("Collection " + collection + " does not exist, disabling logging."); - enabled = false; + // relatively benign but log this - collection still existed when we started + log.info("Collection {} missing, skip sending event {}", collection, event); } else { - log.warn("Exception sending event to collection " + collection, e); + log.warn("Exception sending event. Collection: {}, event: {}, exception: {}", collection, event, e); } } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java index ddc6f38c160f..2a44f463d7f8 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java @@ -18,6 +18,7 @@ import java.lang.invoke.MethodHandles; import java.util.Collection; +import java.util.Collections; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -336,10 +337,24 @@ public void testHistory() throws Exception { String overseerLeader = (String) overSeerStatus.get("leader"); ClusterState state = cluster.getSolrClient().getZkStateReader().getClusterState(); DocCollection coll = state.getCollection(COLL_NAME); + DocCollection system = state.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL); + Set systemLeaderNodes; + if (system != null) { + systemLeaderNodes = system.getReplicas().stream() + .filter(r -> r.getBool("leader", false)) + .map(r -> r.getNodeName()) + .collect(Collectors.toSet()); + } else { + systemLeaderNodes = Collections.emptySet(); + } String nodeToKill = null; for (Replica r : coll.getReplicas()) { if (r.isActive(state.getLiveNodes()) && !r.getNodeName().equals(overseerLeader)) { + if (systemLeaderNodes.contains(r.getNodeName())) { + log.info("--skipping .system leader replica {}", r); + continue; + } nodeToKill = r.getNodeName(); break; } From 43b51d2f22f35ff857967bde0523e51d2844c59b Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Wed, 2 Jan 2019 07:19:49 -0500 Subject: [PATCH 70/88] SOLR-13090: Add sysprop override for maxBooleanClauses --- .../clustering/solr/collection1/conf/solrconfig.xml | 2 +- .../dihextras/solr/collection1/conf/dataimport-solrconfig.xml | 2 +- .../dih/solr/collection1/conf/contentstream-solrconfig.xml | 2 +- .../collection1/conf/dataimport-nodatasource-solrconfig.xml | 2 +- .../dih/solr/collection1/conf/dataimport-solrconfig.xml | 2 +- .../extraction/solr/collection1/conf/solrconfig.xml | 2 +- .../src/test-files/configsets/collection1/conf/solrconfig.xml | 2 +- .../solr/collection1/conf/solrconfig-analytics-query.xml | 2 +- .../solr/collection1/conf/solrconfig-cache-enable-disable.xml | 2 +- .../solr/collection1/conf/solrconfig-collapseqparser.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-elevate.xml | 2 +- .../test-files/solr/collection1/conf/solrconfig-minhash.xml | 2 +- .../solr/collection1/conf/solrconfig-plugcollector.xml | 2 +- solr/core/src/test-files/solr/collection1/conf/solrconfig.xml | 2 +- .../test-files/solr/configsets/_default/conf/solrconfig.xml | 2 +- solr/core/src/test-files/solr/crazy-path-to-config.xml | 2 +- solr/example/example-DIH/solr/db/conf/solrconfig.xml | 2 +- solr/example/example-DIH/solr/mail/conf/solrconfig.xml | 2 +- solr/example/example-DIH/solr/solr/conf/solrconfig.xml | 2 +- solr/example/files/conf/solrconfig.xml | 2 +- solr/server/solr/configsets/_default/conf/solrconfig.xml | 2 +- .../sample_techproducts_configs/conf/solrconfig.xml | 2 +- solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc | 4 +++- 23 files changed, 25 insertions(+), 23 deletions(-) diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml index 5ff42541ed48..50714a7e74cb 100644 --- a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml @@ -70,7 +70,7 @@ - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024} - 1024 + ${solr.max.booleanClauses:1024}