Run verification after catchup completes

CHANGELOG.md

@@ -10,6 +10,7 @@ on [Keep a CHANGELOG](http://keepachangelog.com/). This project adheres to
 the running of PrePublisherTimerBean and PublisherTimerBean respectively
 - Error message of event linking verification now gives more accurate error messages
 based on whether the problem is in published_event or processed_event
+- Catchup now runs verification on completion and will fail catchup if the verification fails
 
 ## [2.2.1] - 2019-10-18
 ### Fixed

event-store-management/event-store-management-core/src/main/java/uk/gov/justice/services/eventstore/management/catchup/observers/CatchupCompletionEventFirer.java

@@ -0,0 +1,81 @@
+package uk.gov.justice.services.eventstore.management.catchup.observers;
+
+import static java.lang.String.format;
+import static uk.gov.justice.services.jmx.api.domain.CommandState.COMMAND_COMPLETE;
+import static uk.gov.justice.services.jmx.api.domain.CommandState.COMMAND_FAILED;
+
+import uk.gov.justice.services.common.util.UtcClock;
+import uk.gov.justice.services.eventstore.management.catchup.state.CatchupError;
+import uk.gov.justice.services.eventstore.management.validation.commands.VerificationCommandResult;
+import uk.gov.justice.services.jmx.api.command.CatchupCommand;
+import uk.gov.justice.services.jmx.state.events.SystemCommandStateChangedEvent;
+
+import java.time.ZonedDateTime;
+import java.util.List;
+import java.util.UUID;
+
+import javax.enterprise.event.Event;
+import javax.inject.Inject;
+
+import org.slf4j.Logger;
+
+public class CatchupCompletionEventFirer {
+
+    @Inject
+    private Event<SystemCommandStateChangedEvent> systemCommandStateChangedEventFirer;
+
+    @Inject
+    private UtcClock clock;
+
+    @Inject
+    private Logger logger;
+
+    public void completeSuccessfully(final UUID commandId, final CatchupCommand catchupCommand) {
+        final ZonedDateTime completedAt = clock.now();
+        final String message = format("%s successfully completed with 0 errors at %s", catchupCommand.getName(), completedAt);
+        logger.info(message);
+
+        systemCommandStateChangedEventFirer.fire(new SystemCommandStateChangedEvent(
+                commandId,
+                catchupCommand,
+                COMMAND_COMPLETE,
+                completedAt,
+                message
+        ));
+    }
+
+    public void failCatchup(final UUID commandId, final CatchupCommand catchupCommand, final List<CatchupError> errors) {
+        final ZonedDateTime completedAt = clock.now();
+        final String message = format("%s failed with %d errors at %s", catchupCommand.getName(), errors.size(), completedAt);
+        logger.error(message);
+
+        systemCommandStateChangedEventFirer.fire(new SystemCommandStateChangedEvent(
+                commandId,
+                catchupCommand,
+                COMMAND_FAILED,
+                clock.now(),
+                message
+        ));
+    }
+
+    public void failVerification(
+            final UUID commandId,
+            final CatchupCommand catchupCommand,
+            final VerificationCommandResult verificationCommandResult) {
+
+        final String message = format(
+                "%s run successfully but failed verification: %s",
+                catchupCommand.getName(),
+                verificationCommandResult.getMessage());
+
+        logger.error(message);
+
+        systemCommandStateChangedEventFirer.fire(new SystemCommandStateChangedEvent(
+                commandId,
+                catchupCommand,
+                COMMAND_FAILED,
+                clock.now(),
+                message
+        ));
+    }
+}

event-store-management/event-store-management-core/src/main/java/uk/gov/justice/services/eventstore/management/catchup/observers/CatchupProcessCompleter.java

@@ -1,74 +1,43 @@
 package uk.gov.justice.services.eventstore.management.catchup.observers;
 
-import static java.lang.String.format;
 import static uk.gov.justice.services.jmx.api.domain.CommandState.COMMAND_COMPLETE;
-import static uk.gov.justice.services.jmx.api.domain.CommandState.COMMAND_FAILED;
 
-import uk.gov.justice.services.common.util.UtcClock;
 import uk.gov.justice.services.eventstore.management.catchup.state.CatchupError;
 import uk.gov.justice.services.eventstore.management.catchup.state.CatchupErrorStateManager;
+import uk.gov.justice.services.eventstore.management.validation.commands.VerificationCommandResult;
+import uk.gov.justice.services.eventstore.management.validation.process.CatchupVerificationProcess;
 import uk.gov.justice.services.jmx.api.command.CatchupCommand;
-import uk.gov.justice.services.jmx.state.events.SystemCommandStateChangedEvent;
 
-import java.time.ZonedDateTime;
 import java.util.List;
 import java.util.UUID;
 
-import javax.enterprise.event.Event;
 import javax.inject.Inject;
 
-import org.slf4j.Logger;
-
 public class CatchupProcessCompleter {
 
     @Inject
     private CatchupErrorStateManager catchupErrorStateManager;
 
     @Inject
-    private UtcClock clock;
-
-    @Inject
-    private Event<SystemCommandStateChangedEvent> systemCommandStateChangedEventFirer;
+    private CatchupVerificationProcess catchupVerificationProcess;
 
     @Inject
-    private Logger logger;
+    private CatchupCompletionEventFirer catchupCompletionEventFirer;
 
     public void handleCatchupComplete(final UUID commandId, final CatchupCommand catchupCommand) {
 
         final List<CatchupError> errors = catchupErrorStateManager.getErrors(catchupCommand);
 
         if (errors.isEmpty()) {
-            completeSuccessfully(commandId, catchupCommand);
+            final VerificationCommandResult verificationCommandResult = catchupVerificationProcess.runVerification(commandId);
+
+            if (verificationCommandResult.getCommandState() == COMMAND_COMPLETE) {
+                catchupCompletionEventFirer.completeSuccessfully(commandId, catchupCommand);
+            } else {
+                catchupCompletionEventFirer.failVerification(commandId, catchupCommand, verificationCommandResult);
+            }
         } else {
-            fail(commandId, catchupCommand, errors);
+            catchupCompletionEventFirer.failCatchup(commandId, catchupCommand, errors);
         }
     }
-
-    private void completeSuccessfully(final UUID commandId, final CatchupCommand catchupCommand) {
-        final ZonedDateTime completedAt = clock.now();
-        final String message = format("%s successfully completed with 0 errors at %s", catchupCommand.getName(), completedAt);
-        logger.info(message);
-
-        systemCommandStateChangedEventFirer.fire(new SystemCommandStateChangedEvent(
-                commandId,
-                catchupCommand,
-                COMMAND_COMPLETE,
-                completedAt,
-                message
-        ));
-    }
-
-    private void fail(final UUID commandId, final CatchupCommand catchupCommand, final List<CatchupError> errors) {
-        final ZonedDateTime completedAt = clock.now();
-        final String message = format("%s failed with %d errors at %s", catchupCommand.getName(), errors.size(), completedAt);
-        logger.error(message);
-
-        systemCommandStateChangedEventFirer.fire(new SystemCommandStateChangedEvent(
-                commandId,
-                catchupCommand,
-                COMMAND_FAILED,
-                clock.now(),
-                message
-        ));
-    }
 }

event-store-management/event-store-management-core/src/test/java/uk/gov/justice/services/eventstore/management/catchup/observers/CatchupCompletionEventFirerTest.java

@@ -0,0 +1,132 @@
+package uk.gov.justice.services.eventstore.management.catchup.observers;
+
+import static java.time.ZoneOffset.UTC;
+import static java.util.Arrays.asList;
+import static java.util.UUID.randomUUID;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static uk.gov.justice.services.jmx.api.domain.CommandState.COMMAND_COMPLETE;
+import static uk.gov.justice.services.jmx.api.domain.CommandState.COMMAND_FAILED;
+
+import uk.gov.justice.services.common.util.UtcClock;
+import uk.gov.justice.services.eventstore.management.catchup.state.CatchupError;
+import uk.gov.justice.services.eventstore.management.validation.commands.VerificationCommandResult;
+import uk.gov.justice.services.jmx.api.command.CatchupCommand;
+import uk.gov.justice.services.jmx.api.command.EventCatchupCommand;
+import uk.gov.justice.services.jmx.state.events.SystemCommandStateChangedEvent;
+
+import java.time.ZonedDateTime;
+import java.util.List;
+import java.util.UUID;
+
+import javax.enterprise.event.Event;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.slf4j.Logger;
+
+
+@RunWith(MockitoJUnitRunner.class)
+public class CatchupCompletionEventFirerTest {
+
+    @Mock
+    private Event<SystemCommandStateChangedEvent> systemCommandStateChangedEventFirer;
+
+    @Mock
+    private UtcClock clock;
+
+    @Mock
+    private Logger logger;
+
+    @InjectMocks
+    private CatchupCompletionEventFirer catchupCompletionEventFirer;
+
+    @Captor
+    private ArgumentCaptor<SystemCommandStateChangedEvent> systemCommandStateChangedEventCaptor;
+
+    @Test
+    public void shouldCompleteSuccessfully() throws Exception {
+        final UUID commandId = randomUUID();
+        final CatchupCommand catchupCommand = new EventCatchupCommand();
+
+        final ZonedDateTime completedAt = ZonedDateTime.of(2016, 10, 10, 23, 23, 23, 0, UTC);
+
+        when(clock.now()).thenReturn(completedAt);
+
+        catchupCompletionEventFirer.completeSuccessfully(commandId, catchupCommand);
+
+        verify(systemCommandStateChangedEventFirer).fire(systemCommandStateChangedEventCaptor.capture());
+
+        final SystemCommandStateChangedEvent stateChangedEvent = systemCommandStateChangedEventCaptor.getValue();
+
+        assertThat(stateChangedEvent.getCommandId(), is(commandId));
+        assertThat(stateChangedEvent.getCommandState(), is(COMMAND_COMPLETE));
+        assertThat(stateChangedEvent.getSystemCommand(), is(catchupCommand));
+        assertThat(stateChangedEvent.getStatusChangedAt(), is(completedAt));
+        assertThat(stateChangedEvent.getMessage(), is("CATCHUP successfully completed with 0 errors at 2016-10-10T23:23:23Z"));
+
+        verify(logger).info("CATCHUP successfully completed with 0 errors at 2016-10-10T23:23:23Z");
+    }
+
+    @Test
+    public void shouldFailCatchup() throws Exception {
+        final UUID commandId = randomUUID();
+        final CatchupCommand catchupCommand = new EventCatchupCommand();
+
+        final ZonedDateTime completedAt = ZonedDateTime.of(2016, 10, 10, 23, 23, 23, 0, UTC);
+
+        final List<CatchupError> errors = asList(mock(CatchupError.class), mock(CatchupError.class));
+        when(clock.now()).thenReturn(completedAt);
+
+        catchupCompletionEventFirer.failCatchup(commandId, catchupCommand, errors);
+
+        verify(systemCommandStateChangedEventFirer).fire(systemCommandStateChangedEventCaptor.capture());
+
+        final SystemCommandStateChangedEvent stateChangedEvent = systemCommandStateChangedEventCaptor.getValue();
+
+        assertThat(stateChangedEvent.getCommandId(), is(commandId));
+        assertThat(stateChangedEvent.getCommandState(), is(COMMAND_FAILED));
+        assertThat(stateChangedEvent.getSystemCommand(), is(catchupCommand));
+        assertThat(stateChangedEvent.getStatusChangedAt(), is(completedAt));
+        assertThat(stateChangedEvent.getMessage(), is("CATCHUP failed with 2 errors at 2016-10-10T23:23:23Z"));
+
+        verify(logger).error("CATCHUP failed with 2 errors at 2016-10-10T23:23:23Z");
+    }
+
+    @Test
+    public void shouldFailVerification() throws Exception {
+
+        final UUID commandId = randomUUID();
+        final CatchupCommand catchupCommand = new EventCatchupCommand();
+        final String errorMessage = "verification failed with 23 errors";
+
+        final VerificationCommandResult verificationCommandResult = mock(VerificationCommandResult.class);
+
+        final ZonedDateTime completedAt = ZonedDateTime.of(2016, 10, 10, 23, 23, 23, 0, UTC);
+
+        when(clock.now()).thenReturn(completedAt);
+        when(verificationCommandResult.getMessage()).thenReturn(errorMessage);
+
+        catchupCompletionEventFirer.failVerification(commandId, catchupCommand, verificationCommandResult);
+
+        verify(systemCommandStateChangedEventFirer).fire(systemCommandStateChangedEventCaptor.capture());
+
+        final SystemCommandStateChangedEvent stateChangedEvent = systemCommandStateChangedEventCaptor.getValue();
+
+        assertThat(stateChangedEvent.getCommandId(), is(commandId));
+        assertThat(stateChangedEvent.getCommandState(), is(COMMAND_FAILED));
+        assertThat(stateChangedEvent.getSystemCommand(), is(catchupCommand));
+        assertThat(stateChangedEvent.getStatusChangedAt(), is(completedAt));
+        assertThat(stateChangedEvent.getMessage(), is("CATCHUP run successfully but failed verification: verification failed with 23 errors"));
+
+        verify(logger).error("CATCHUP run successfully but failed verification: verification failed with 23 errors");
+    }
+}