make server_urls dynamically reloadable

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Features
  * Add support for Spring's JMS flavor - instrumenting `org.springframework.jms.listener.SessionAwareMessageListener`
  * Add support to legacy ApacheHttpClient APIs (which adds support to Axis2 configured to use ApacheHttpClient)
+ * Added support for setting `server_urls` dynamically via properties file [#723](https://github.com/elastic/apm-agent-java/issues/723)
 
 ## Bug Fixes
  * Some JMS Consumers and Producers are filtered due to class name filtering in instrumentation matching

apm-agent-core/src/main/java/co/elastic/apm/agent/report/ApmServerClient.java

@@ -27,6 +27,7 @@
 import co.elastic.apm.agent.util.VersionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.stagemonitor.configuration.ConfigurationOption;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -64,20 +65,29 @@ public class ApmServerClient {
     private static final Logger logger = LoggerFactory.getLogger(ApmServerClient.class);
     private static final String USER_AGENT = "elasticapm-java/" + VersionUtils.getAgentVersion();
     private final ReporterConfiguration reporterConfiguration;
-    private final List<URL> serverUrls;
+    private volatile List<URL> serverUrls;
     private final AtomicInteger errorCount = new AtomicInteger();
 
     public ApmServerClient(ReporterConfiguration reporterConfiguration) {
-        this(reporterConfiguration, shuffleUrls(reporterConfiguration));
+        this(reporterConfiguration, shuffleUrls(reporterConfiguration.getServerUrls()));
     }
 
-    public ApmServerClient(ReporterConfiguration reporterConfiguration, List<URL> serverUrls) {
+    public ApmServerClient(ReporterConfiguration reporterConfiguration, List<URL> shuffledUrls) {
         this.reporterConfiguration = reporterConfiguration;
-        this.serverUrls = Collections.unmodifiableList(serverUrls);
+        this.reporterConfiguration.getServerUrlsOption().addChangeListener(new ConfigurationOption.ChangeListener<List<URL>>() {
+            @Override
+            public void onChange(ConfigurationOption<?> configurationOption, List<URL> oldValue, List<URL> newValue) {
+                logger.debug("server_urls override with value = ({}).", newValue);
+                if (newValue != null && !newValue.isEmpty()) {
+                    serverUrls = shuffleUrls(newValue);
+                    errorCount.set(0);
+                }
+            }
+        });
+        this.serverUrls = Collections.unmodifiableList(shuffledUrls);
     }
 
-    private static List<URL> shuffleUrls(ReporterConfiguration reporterConfiguration) {
-        List<URL> serverUrls = new ArrayList<>(reporterConfiguration.getServerUrls());
+    private static List<URL> shuffleUrls(List<URL> serverUrls) {
         // shuffling the URL list helps to distribute the load across the apm servers
         // when there are multiple agents, they should not all start connecting to the same apm server
         Collections.shuffle(serverUrls);
@@ -140,6 +150,7 @@ private URL appendPath(URL serverUrl, String apmServerPath) throws MalformedURLE
      * the error count is not incremented.
      * This avoids concurrent requests from incrementing the error multiple times due to only one failing server.
      * </p>
+     *
      * @param expectedErrorCount the error count that is expected by the current thread
      * @return the new expected error count
      */
@@ -240,6 +251,10 @@ int getErrorCount() {
         return errorCount.get();
     }
 
+    List<URL> getServerUrls() {
+        return this.serverUrls;
+    }
+
     public interface ConnectionHandler<T> {
         @Nullable
         T withConnection(HttpURLConnection connection) throws IOException;

apm-agent-core/src/main/java/co/elastic/apm/agent/report/ReporterConfiguration.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -68,7 +68,7 @@ public class ReporterConfiguration extends ConfigurationOptionProvider {
             "If outgoing HTTP traffic has to go through a proxy," +
             "you can use the Java system properties `http.proxyHost` and `http.proxyPort` to set that up.\n" +
             "See also [Java's proxy documentation](https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html) for more information.")
-        .dynamic(false)
+        .dynamic(true)
         .buildWithDefault(Collections.singletonList(UrlValueConverter.INSTANCE.convert("http://localhost:8200")));
 
     private final ConfigurationOption<TimeDuration> serverTimeout = TimeDurationValueConverter.durationOption("s")
@@ -212,4 +212,9 @@ public long getMetricsIntervalMs() {
     public List<WildcardMatcher> getDisableMetrics() {
         return disableMetrics.get();
     }
+
+    public ConfigurationOption<List<URL>> getServerUrlsOption() {
+        return this.serverUrl;
+    }
+
 }

apm-agent-core/src/test/java/co/elastic/apm/agent/report/ApmServerClientTest.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,17 +24,24 @@
  */
 package co.elastic.apm.agent.report;
 
+import co.elastic.apm.agent.configuration.SpyConfiguration;
+import co.elastic.apm.agent.impl.ElasticApmTracer;
+import co.elastic.apm.agent.impl.ElasticApmTracerBuilder;
 import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
 import com.github.tomakehurst.wiremock.junit.WireMockRule;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.mockito.Mockito;
+import org.stagemonitor.configuration.ConfigurationRegistry;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLConnection;
+import java.util.Arrays;
 import java.util.List;
 
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
@@ -52,17 +59,27 @@ public class ApmServerClientTest {
     @Rule
     public WireMockRule apmServer2 = new WireMockRule(WireMockConfiguration.wireMockConfig().dynamicPort());
     private ApmServerClient apmServerClient;
+    private ConfigurationRegistry config;
+    private ElasticApmTracer tracer;
+    private ReporterConfiguration reporterConfiguration;
 
     @Before
     public void setUp() throws MalformedURLException {
+        URL url1 = new URL("http", "localhost", apmServer1.port(), "/");
+        URL url2 = new URL("http", "localhost", apmServer2.port(), "/");
+        config = SpyConfiguration.createSpyConfig();
+        tracer = new ElasticApmTracerBuilder()
+            .configurationRegistry(config)
+            .build();
+        reporterConfiguration = tracer.getConfig(ReporterConfiguration.class);
+
+        Mockito.when(reporterConfiguration.getServerUrls()).thenReturn(Arrays.asList(url1, url2));
+
         apmServer1.stubFor(get(urlEqualTo("/test")).willReturn(notFound()));
         apmServer1.stubFor(get(urlEqualTo("/not-found")).willReturn(notFound()));
         apmServer2.stubFor(get(urlEqualTo("/test")).willReturn(ok("hello from server 2")));
         apmServer2.stubFor(get(urlEqualTo("/not-found")).willReturn(notFound()));
-        apmServerClient = new ApmServerClient(new ReporterConfiguration(), List.of(
-            new URL("http", "localhost", apmServer1.port(), "/"),
-            new URL("http", "localhost", apmServer2.port(), "/")
-        ));
+        apmServerClient = new ApmServerClient(reporterConfiguration, tracer.getConfig(ReporterConfiguration.class).getServerUrls());
     }
 
     @Test
@@ -148,4 +165,14 @@ public void testSimulateConcurrentConnectionError() {
         apmServerClient.incrementAndGetErrorCount(0);
         assertThat(apmServerClient.getErrorCount()).isOne();
     }
+
+    @Test
+    public void testGetServerUrlsVerifyThatServerUrlsWillBeReloaded() throws IOException {
+        URL tempUrl = new URL("http", "localhost", 9999, "");
+        config.save("server_urls", tempUrl.toString(), SpyConfiguration.CONFIG_SOURCE_NAME);
+
+        List<URL> updatedServerUrls = apmServerClient.getServerUrls();
+
+        assertThat(updatedServerUrls).isEqualTo(Arrays.asList(tempUrl));
+    }
 }

docs/configuration.asciidoc

@@ -882,7 +882,7 @@ See also [Java's proxy documentation](https://docs.oracle.com/javase/8/docs/tech
 [options="header"]
 |============
 | Default                          | Type                | Dynamic
-| `http://localhost:8200` | List | false
+| `http://localhost:8200` | List | true
 |============
 
 
@@ -1633,7 +1633,7 @@ The default unit for this option is `ms`
 # If outgoing HTTP traffic has to go through a proxy,you can use the Java system properties `http.proxyHost` and `http.proxyPort` to set that up.
 # See also [Java's proxy documentation](https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html) for more information.
 #
-# This setting can not be changed at runtime. Changes require a restart of the application.
+# This setting can be changed at runtime
 # Type: comma separated list
 # Default value: http://localhost:8200
 #