Skip to content

Commit

Permalink
Fix intermittent issue on OciMetricsSupportTest (#6151)
Browse files Browse the repository at this point in the history
* Fix intermittent issue on OciMetricsSupportTest

RCA is described here: #6112 (comment)

Changes include the following:
1. In OciMetricsSupportTest.testEndpoint, extend the amount of validation time to 10 seconds for checking that the metric endpoint has been restored. Intermittently, a race condition exist where the validation happens before the endpoint is restored.
2. Modify all countdownLatch to be locally defined in the test methods rather than being a static variable, which is causing chain reaction failure to other tests if a previous test fails because they share the same countdownLatch.
3. Always check that countDownLatch.await() is verified to have completed or otherwise, assert a failure.
4. Remove the use of fixed port when starting a WebServer.
5. Reset postingEndPoint to its original value before each test, so @RepeatedTest can be used in the future for debugging purposes.
6. Apply Helidon Code Style on both OciMetricsSupportTest and OciMetricsCdiExtensionTest. This would include making the tests's class and methods package local  rather than public, rearranging variable fields order based on whether they are static, final, etc.
7. Note that OciMetricsCdiExtensionTest only involves Code Style change and removal of delay method which is never used, so logic in that test class will be the same as before. Only OciMetricsSupportTest contain significant change to resolve the issue reported.
8. Fail the OciMetricsCdiExtensionTest if enabled OCI Metrics validation times out on countDownLatch.await()
  • Loading branch information
klustria committed Feb 11, 2023
1 parent 234bca5 commit bcf0022
Show file tree
Hide file tree
Showing 2 changed files with 166 additions and 107 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,38 +15,63 @@
*/
package io.helidon.integrations.oci.metrics.cdi;

import com.oracle.bmc.Region;
import com.oracle.bmc.monitoring.Monitoring;
import com.oracle.bmc.monitoring.MonitoringPaginators;
import com.oracle.bmc.monitoring.MonitoringWaiters;
import com.oracle.bmc.monitoring.model.MetricDataDetails;
import com.oracle.bmc.monitoring.model.PostMetricDataDetails;
import com.oracle.bmc.monitoring.requests.*;
import com.oracle.bmc.monitoring.responses.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import io.helidon.config.Config;
import io.helidon.integrations.oci.metrics.OciMetricsSupport;
import io.helidon.metrics.api.RegistryFactory;
import io.helidon.microprofile.config.ConfigCdiExtension;
import io.helidon.microprofile.server.ServerCdiExtension;
import io.helidon.microprofile.server.JaxRsCdiExtension;
import io.helidon.microprofile.server.ServerCdiExtension;
import io.helidon.microprofile.tests.junit5.AddBean;
import io.helidon.microprofile.tests.junit5.AddConfig;
import io.helidon.microprofile.tests.junit5.AddExtension;
import io.helidon.microprofile.tests.junit5.DisableDiscovery;
import io.helidon.microprofile.tests.junit5.HelidonTest;

import com.oracle.bmc.Region;
import com.oracle.bmc.monitoring.Monitoring;
import com.oracle.bmc.monitoring.MonitoringPaginators;
import com.oracle.bmc.monitoring.MonitoringWaiters;
import com.oracle.bmc.monitoring.model.MetricDataDetails;
import com.oracle.bmc.monitoring.model.PostMetricDataDetails;
import com.oracle.bmc.monitoring.requests.ChangeAlarmCompartmentRequest;
import com.oracle.bmc.monitoring.requests.CreateAlarmRequest;
import com.oracle.bmc.monitoring.requests.DeleteAlarmRequest;
import com.oracle.bmc.monitoring.requests.GetAlarmHistoryRequest;
import com.oracle.bmc.monitoring.requests.GetAlarmRequest;
import com.oracle.bmc.monitoring.requests.ListAlarmsRequest;
import com.oracle.bmc.monitoring.requests.ListAlarmsStatusRequest;
import com.oracle.bmc.monitoring.requests.ListMetricsRequest;
import com.oracle.bmc.monitoring.requests.PostMetricDataRequest;
import com.oracle.bmc.monitoring.requests.RemoveAlarmSuppressionRequest;
import com.oracle.bmc.monitoring.requests.RetrieveDimensionStatesRequest;
import com.oracle.bmc.monitoring.requests.SummarizeMetricsDataRequest;
import com.oracle.bmc.monitoring.requests.UpdateAlarmRequest;
import com.oracle.bmc.monitoring.responses.ChangeAlarmCompartmentResponse;
import com.oracle.bmc.monitoring.responses.CreateAlarmResponse;
import com.oracle.bmc.monitoring.responses.DeleteAlarmResponse;
import com.oracle.bmc.monitoring.responses.GetAlarmHistoryResponse;
import com.oracle.bmc.monitoring.responses.GetAlarmResponse;
import com.oracle.bmc.monitoring.responses.ListAlarmsResponse;
import com.oracle.bmc.monitoring.responses.ListAlarmsStatusResponse;
import com.oracle.bmc.monitoring.responses.ListMetricsResponse;
import com.oracle.bmc.monitoring.responses.PostMetricDataResponse;
import com.oracle.bmc.monitoring.responses.RemoveAlarmSuppressionResponse;
import com.oracle.bmc.monitoring.responses.RetrieveDimensionStatesResponse;
import com.oracle.bmc.monitoring.responses.SummarizeMetricsDataResponse;
import com.oracle.bmc.monitoring.responses.UpdateAlarmResponse;

import org.eclipse.microprofile.metrics.MetricRegistry;

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.fail;

@HelidonTest(resetPerTest = true)
@AddBean(OciMetricsCdiExtensionTest.MockMonitoring.class)
Expand All @@ -64,41 +89,40 @@
@AddConfig(key = "ocimetrics.namespace",
value = OciMetricsCdiExtensionTest.MetricDataDetailsOCIParams.namespace)
@AddConfig(key = "ocimetrics.resourceGroup",
value = OciMetricsCdiExtensionTest.MetricDataDetailsOCIParams.resourceGroup)
value = OciMetricsCdiExtensionTest.MetricDataDetailsOCIParams.resourceGroup)
@AddConfig(key = "ocimetrics.initialDelay", value = "1")
@AddConfig(key = "ocimetrics.delay", value = "2")
public class OciMetricsCdiExtensionTest {
private final RegistryFactory rf = RegistryFactory.getInstance();
private final MetricRegistry baseMetricRegistry = rf.getRegistry(MetricRegistry.Type.BASE);
private final MetricRegistry vendorMetricRegistry = rf.getRegistry(MetricRegistry.Type.VENDOR);
private final MetricRegistry appMetricRegistry = rf.getRegistry(MetricRegistry.Type.APPLICATION);
class OciMetricsCdiExtensionTest {
private static volatile int testMetricCount = 0;
// Use countDownLatch1 to signal the test that results to be asserted has been retrieved
private static CountDownLatch countDownLatch1 = new CountDownLatch(1);
private static CountDownLatch countDownLatch = new CountDownLatch(1);
private static PostMetricDataDetails postMetricDataDetails;
private static boolean activateOciMetricsSupportIsInvoked;
private final RegistryFactory rf = RegistryFactory.getInstance();
private final MetricRegistry appMetricRegistry = rf.getRegistry(MetricRegistry.Type.APPLICATION);
private final MetricRegistry baseMetricRegistry = rf.getRegistry(MetricRegistry.Type.BASE);
private final MetricRegistry vendorMetricRegistry = rf.getRegistry(MetricRegistry.Type.VENDOR);

@AfterEach
private void resetState() {
void resetState() {
postMetricDataDetails = null;
activateOciMetricsSupportIsInvoked = false;
countDownLatch1 = new CountDownLatch(1);
countDownLatch = new CountDownLatch(1);
}

@Test
@AddConfig(key = "ocimetrics.enabled", value = "true")
public void testEnableOciMetrics() throws InterruptedException {
void testEnableOciMetrics() throws InterruptedException {
validateOciMetricsSupport(true);
}

@Test
public void testEnableOciMetricsWithoutConfig() throws InterruptedException {
void testEnableOciMetricsWithoutConfig() throws InterruptedException {
validateOciMetricsSupport(true);
}

@Test
@AddConfig(key = "ocimetrics.enabled", value = "false")
public void testDisableOciMetrics() throws InterruptedException {
void testDisableOciMetrics() throws InterruptedException {
validateOciMetricsSupport(false);
}

Expand All @@ -107,7 +131,12 @@ private void validateOciMetricsSupport(boolean enabled) throws InterruptedExcept
vendorMetricRegistry.counter("vendorDummyCounter").inc();
appMetricRegistry.counter("appDummyCounter").inc();
// Wait for signal from metric update that testMetricCount has been retrieved
countDownLatch1.await(3, TimeUnit.SECONDS);
if (!countDownLatch.await(3, TimeUnit.SECONDS)) {
// If Oci Metrics is enabled, this means that countdown() of CountDownLatch was never triggered, and hence should fail
if (enabled) {
fail("CountDownLatch timed out");
}
}

if (enabled) {
assertThat(activateOciMetricsSupportIsInvoked, is(true));
Expand All @@ -126,56 +155,80 @@ private void validateOciMetricsSupport(boolean enabled) throws InterruptedExcept
}
}

interface MetricDataDetailsOCIParams {
String compartmentId = "dummy.compartmentId";
String namespace = "dummy-namespace";
String resourceGroup = "dummy_resourceGroup";
}

static class MockMonitoring implements Monitoring {
@Override
public void setEndpoint(String s) {}
public String getEndpoint() {
return "http://www.DummyEndpoint.com";
}

@Override
public String getEndpoint() {return "http://www.DummyEndpoint.com";}
public void setEndpoint(String s) {
}

@Override
public void setRegion(Region region) {}
public void setRegion(Region region) {
}

@Override
public void setRegion(String s) {}
public void setRegion(String s) {
}

@Override
public void refreshClient() {}
public void refreshClient() {
}

@Override
public ChangeAlarmCompartmentResponse changeAlarmCompartment(ChangeAlarmCompartmentRequest changeAlarmCompartmentRequest) {
return null;
}

@Override
public CreateAlarmResponse createAlarm(CreateAlarmRequest createAlarmRequest) {return null;}
public CreateAlarmResponse createAlarm(CreateAlarmRequest createAlarmRequest) {
return null;
}

@Override
public DeleteAlarmResponse deleteAlarm(DeleteAlarmRequest deleteAlarmRequest) {return null;}
public DeleteAlarmResponse deleteAlarm(DeleteAlarmRequest deleteAlarmRequest) {
return null;
}

@Override
public GetAlarmResponse getAlarm(GetAlarmRequest getAlarmRequest) {return null;}
public GetAlarmResponse getAlarm(GetAlarmRequest getAlarmRequest) {
return null;
}

@Override
public GetAlarmHistoryResponse getAlarmHistory(GetAlarmHistoryRequest getAlarmHistoryRequest) {return null;}
public GetAlarmHistoryResponse getAlarmHistory(GetAlarmHistoryRequest getAlarmHistoryRequest) {
return null;
}

@Override
public ListAlarmsResponse listAlarms(ListAlarmsRequest listAlarmsRequest) {return null;}
public ListAlarmsResponse listAlarms(ListAlarmsRequest listAlarmsRequest) {
return null;
}

@Override
public ListAlarmsStatusResponse listAlarmsStatus(ListAlarmsStatusRequest listAlarmsStatusRequest) {
return null;
}

@Override
public ListMetricsResponse listMetrics(ListMetricsRequest listMetricsRequest) {return null;}
public ListMetricsResponse listMetrics(ListMetricsRequest listMetricsRequest) {
return null;
}

@Override
public PostMetricDataResponse postMetricData(PostMetricDataRequest postMetricDataRequest) {
postMetricDataDetails = postMetricDataRequest.getPostMetricDataDetails();
testMetricCount = postMetricDataDetails.getMetricData().size();
// Give signal that testMetricCount was retrieved
countDownLatch1.countDown();
countDownLatch.countDown();
return PostMetricDataResponse.builder()
.__httpStatusCode__(200)
.build();
Expand All @@ -197,16 +250,23 @@ public SummarizeMetricsDataResponse summarizeMetricsData(SummarizeMetricsDataReq
}

@Override
public UpdateAlarmResponse updateAlarm(UpdateAlarmRequest updateAlarmRequest) {return null;}
public UpdateAlarmResponse updateAlarm(UpdateAlarmRequest updateAlarmRequest) {
return null;
}

@Override
public MonitoringWaiters getWaiters() {return null;}
public MonitoringWaiters getWaiters() {
return null;
}

@Override
public MonitoringPaginators getPaginators() {return null;}
public MonitoringPaginators getPaginators() {
return null;
}

@Override
public void close() throws Exception {}
public void close() throws Exception {
}
}

static class MockOciMetricsBean extends OciMetricsBean {
Expand All @@ -217,17 +277,4 @@ void activateOciMetricsSupport(Config config, OciMetricsSupport.Builder builder)
super.activateOciMetricsSupport(config, builder);
}
}

public interface MetricDataDetailsOCIParams {
String compartmentId = "dummy.compartmentId";
String namespace = "dummy-namespace";
String resourceGroup = "dummy_resourceGroup";
}

private static void delay(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException ignore) {
}
}
}

0 comments on commit bcf0022

Please sign in to comment.