Skip to content

Commit

Permalink
[cleanup] Consolidate certs in broker (and some proxy) tests (#20353)
Browse files Browse the repository at this point in the history 
(cherry picked from commit d565c95)
  • Loading branch information
@michaeljmarshall @lhotari
michaeljmarshall authored and lhotari committed Jan 26, 2024
1 parent d510b1d commit e1982bc
Show file tree
Hide file tree
Showing 21 changed files with 84 additions and 628 deletions.
9 changes: 3 additions & 6 deletions pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,6 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest {

private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class);

private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";

private MockedPulsarService mockPulsarSetup;

private PulsarService otherPulsar;
Expand Down Expand Up @@ -188,8 +185,8 @@ private void applyDefaultConfig() {
conf.setLoadBalancerEnabled(true);
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setMessageExpiryCheckIntervalInMinutes(1);
conf.setSubscriptionExpiryCheckIntervalInMinutes(1);
conf.setBrokerDeleteInactiveTopicsEnabled(false);
Expand All @@ -206,7 +203,7 @@ private void setupConfigAndStart(java.util.function.Consumer<ServiceConfiguratio

bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());

adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
.serviceHttpUrl(brokerUrlTls.toString()).build());

// create otherbroker to test redirect on calls that need
Expand Down
9 changes: 3 additions & 6 deletions pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,6 @@ public class V1_AdminApiTest extends MockedPulsarServiceBaseTest {

private static final Logger LOG = LoggerFactory.getLogger(V1_AdminApiTest.class);

private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";

private MockedPulsarService mockPulsarSetup;

private PulsarService otherPulsar;
Expand All @@ -147,15 +144,15 @@ public void setup() throws Exception {
conf.setLoadBalancerEnabled(true);
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setNumExecutorThreadPoolSize(5);

super.internalSetup();

bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());

adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(CA_CERT_FILE_PATH)
.serviceHttpUrl(brokerUrlTls.toString()).build());

// create otherbroker to test redirect on calls that need
Expand Down
41 changes: 18 additions & 23 deletions pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,6 @@
@Test(groups = "broker")
public class BrokerServiceTest extends BrokerTestBase {

private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";

@BeforeClass
@Override
protected void setup() throws Exception {
Expand Down Expand Up @@ -681,8 +676,8 @@ public void testTlsEnabled() throws Exception {
conf.setAuthenticationEnabled(false);
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setNumExecutorThreadPoolSize(5);
restartBroker();

Expand Down Expand Up @@ -736,7 +731,7 @@ public void testTlsEnabled() throws Exception {
// Case 4: Access with TLS (Use trusted certificates)
try {
pulsarClient = PulsarClient.builder().serviceUrl(brokerUrlTls.toString()).enableTls(true)
.allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
.allowTlsInsecureConnection(false).tlsTrustCertsFilePath(BROKER_CERT_FILE_PATH)
.statsInterval(0, TimeUnit.SECONDS)
.operationTimeout(1000, TimeUnit.MILLISECONDS).build();

Expand All @@ -760,8 +755,8 @@ public void testTlsEnabledWithoutNonTlsServicePorts() throws Exception {
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePort(Optional.empty());
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setNumExecutorThreadPoolSize(5);
restartBroker();

Expand Down Expand Up @@ -797,15 +792,15 @@ public void testTlsAuthAllowInsecure() throws Exception {
conf.setAuthenticationProviders(providers);
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setTlsAllowInsecureConnection(true);
conf.setNumExecutorThreadPoolSize(5);
restartBroker();

Map<String, String> authParams = new HashMap<>();
authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));

PulsarClient pulsarClient = null;

Expand Down Expand Up @@ -860,15 +855,15 @@ public void testTlsAuthDisallowInsecure() throws Exception {
conf.setAuthenticationProviders(providers);
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setTlsAllowInsecureConnection(false);
conf.setNumExecutorThreadPoolSize(5);
restartBroker();

Map<String, String> authParams = new HashMap<>();
authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));

PulsarClient pulsarClient = null;

Expand Down Expand Up @@ -922,16 +917,16 @@ public void testTlsAuthUseTrustCert() throws Exception {
conf.setAuthenticationProviders(providers);
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setTlsAllowInsecureConnection(false);
conf.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
conf.setNumExecutorThreadPoolSize(5);
restartBroker();

Map<String, String> authParams = new HashMap<>();
authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
authParams.put("tlsCertFile", getTlsFileForClient("admin.cert"));
authParams.put("tlsKeyFile", getTlsFileForClient("admin.key-pk8"));

PulsarClient pulsarClient = null;

Expand Down
30 changes: 19 additions & 11 deletions pulsar-broker/src/test/java/org/apache/pulsar/broker/web/WebServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.apache.pulsar.common.policies.data.TenantInfo;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.common.util.SecurityUtility;
import org.apache.pulsar.utils.ResourceUtils;
import org.asynchttpclient.AsyncHttpClient;
import org.asynchttpclient.BoundRequestBuilder;
import org.asynchttpclient.DefaultAsyncHttpClient;
Expand All @@ -84,10 +85,17 @@ public class WebServiceTest {
private PulsarService pulsar;
private String BROKER_LOOKUP_URL;
private String BROKER_LOOKUP_URL_TLS;
private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";

private final static String CA_CERT_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");
private final static String BROKER_CERT_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
private final static String BROKER_KEY_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
private final static String CLIENT_CERT_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
private final static String CLIENT_KEY_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");


@Test
Expand Down Expand Up @@ -351,8 +359,8 @@ private String makeHttpRequest(boolean useTls, boolean useAuth) throws Exception
if (useTls) {
KeyManager[] keyManagers = null;
if (useAuth) {
Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(CLIENT_CERT_FILE_PATH);
PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(CLIENT_KEY_FILE_PATH);

KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
ks.load(null, null);
Expand Down Expand Up @@ -403,10 +411,10 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut
config.setAuthenticationProviders(providers);
config.setAuthorizationEnabled(false);
config.setSuperUserRoles(roles);
config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
config.setTlsAllowInsecureConnection(allowInsecure);
config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
config.setTlsTrustCertsFilePath(allowInsecure ? "" : CA_CERT_FILE_PATH);
config.setClusterName("local");
config.setAdvertisedAddress("localhost"); // TLS certificate expects localhost
config.setMetadataStoreUrl("zk:localhost:2181");
Expand All @@ -433,8 +441,8 @@ private void setupEnv(boolean enableFilter, boolean enableTls, boolean enableAut
serviceUrl = BROKER_URL_BASE_TLS;

Map<String, String> authParams = new HashMap<>();
authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
authParams.put("tlsCertFile", CLIENT_CERT_FILE_PATH);
authParams.put("tlsKeyFile", CLIENT_KEY_FILE_PATH);

adminBuilder.authentication(AuthenticationTls.class.getName(), authParams).allowTlsInsecureConnection(true);
}
Expand Down
44 changes: 14 additions & 30 deletions pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import com.google.common.util.concurrent.MoreExecutors;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import io.prometheus.client.CollectorRegistry;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -43,10 +42,6 @@
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
Expand All @@ -64,10 +59,7 @@
import java.util.stream.Collectors;
import javax.naming.AuthenticationException;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import lombok.Cleanup;
import org.apache.pulsar.broker.BrokerTestUtil;
import org.apache.pulsar.broker.PulsarService;
Expand Down Expand Up @@ -438,10 +430,6 @@ public void testPartitionTopicLookup() throws Exception {
@Test
public void testWebserviceServiceTls() throws Exception {
log.info("-- Starting {} test --", methodName);
final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";

/**** start broker-2 ****/
ServiceConfiguration conf2 = new ServiceConfiguration();
Expand All @@ -454,12 +442,15 @@ public void testWebserviceServiceTls() throws Exception {
conf2.setWebServicePort(Optional.of(0));
conf2.setWebServicePortTls(Optional.of(0));
conf2.setAdvertisedAddress("localhost");
conf2.setTlsAllowInsecureConnection(true);
conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
conf2.setTlsRequireTrustedClientCertOnConnect(true);
conf2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf2.setClusterName(conf.getClusterName());
conf2.setMetadataStoreUrl("zk:localhost:2181");
conf2.setConfigurationMetadataStoreUrl("zk:localhost:3181");
// Not in use, and because TLS is not configured, it will fail to start
conf2.setSystemTopicEnabled(false);

@Cleanup
PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf2);
Expand All @@ -468,10 +459,13 @@ public void testWebserviceServiceTls() throws Exception {
// restart broker1 with tls enabled
conf.setBrokerServicePortTls(Optional.of(0));
conf.setWebServicePortTls(Optional.of(0));
conf.setTlsAllowInsecureConnection(true);
conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH);
conf.setTlsRequireTrustedClientCertOnConnect(true);
conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH);
conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH);
conf.setNumExecutorThreadPoolSize(5);
// Not in use, and because TLS is not configured, it will fail to start
conf.setSystemTopicEnabled(false);
stopBroker();
startBroker();
pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
Expand Down Expand Up @@ -505,18 +499,8 @@ public void testWebserviceServiceTls() throws Exception {
final String lookupResourceUrl = "/lookup/v2/topic/persistent/my-property/my-ns/my-topic1";

// set client cert_key file
KeyManager[] keyManagers = null;
Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
ks.load(null, null);
ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(ks, "".toCharArray());
keyManagers = kmf.getKeyManagers();
TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
SSLContext sslCtx = SSLContext.getInstance("TLS");
sslCtx.init(keyManagers, trustManagers, new SecureRandom());
SSLContext sslCtx = SecurityUtility.createSslContext(false, CA_CERT_FILE_PATH,
getTlsFileForClient("admin.cert"), getTlsFileForClient("admin.key-pk8"), "");
HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());

// hit broker2 url
Expand Down
16 changes: 11 additions & 5 deletions pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig;
import org.apache.pulsar.functions.sink.PulsarSink;
import org.apache.pulsar.functions.worker.service.WorkerServiceLoader;
import org.apache.pulsar.utils.ResourceUtils;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
import org.awaitility.Awaitility;
import org.testng.annotations.AfterMethod;
Expand All @@ -66,11 +67,16 @@ public class PulsarFunctionTlsTest {

protected static final int BROKER_COUNT = 2;

private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/broker-cert.pem";
private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem";
private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem";
private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem";
private static final String CA_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem";
private final String TLS_SERVER_CERT_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem");
private final String TLS_SERVER_KEY_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem");
private final String TLS_CLIENT_CERT_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.cert.pem");
private final String TLS_CLIENT_KEY_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/client-keys/admin.key-pk8.pem");
private final String CA_CERT_FILE_PATH =
ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem");

LocalBookkeeperEnsemble bkEnsemble;
protected PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT];
Expand Down
Loading

0 comments on commit e1982bc

Please sign in to comment.