[WIP] Add control options to override and/or validate index file

ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java

@@ -12,7 +12,6 @@
  * Configuration used to run the migration.
  */
 public class MigrationConfig {
-
   private String migrationPath = "dbmigration";
   private String migrationInitPath = "dbinit";
   private String metaTable = "db_migration";
@@ -67,6 +66,8 @@ public class MigrationConfig {
   private Properties properties;
   private boolean earlyChecksumMode;
   private boolean fastMode;
+  private boolean forceLocal = false;
+  private boolean forceLocalCheck = false;
 
   /**
    * Return the name of the migration table.
@@ -457,6 +458,38 @@ public void setMinVersionFailMessage(String minVersionFailMessage) {
     this.minVersionFailMessage = minVersionFailMessage;
   }
 
+  /**
+   * Determines, if the local files should be used, although if there is an index file present.
+   */
+  public boolean isForceLocal() {
+    return forceLocal;
+  }
+
+  /**
+   * If this option is set, migrations are read from local resources and validated against the idx file.
+   */
+  public void setForceLocal(boolean forceLocal) {
+    this.forceLocal = forceLocal;
+  }
+
+  /**
+   * Determines, if the local files should be checked against the index.
+   */
+  public boolean isForceLocalCheck() {
+    return forceLocalCheck;
+  }
+
+  /**
+   * Determines, if local files should be checked. This means, the checksums of local files are validated against
+   * the ones in the index file. If this option is set, a MigrationException is thrown, when the index file does not
+   * match to local resources. (Note: Setting this option implies <code>forceLocal</code>. If this option is set to false,
+   * and <code>forceLocal = true</code>, then index mismatches will be logged as warning only)
+   */
+  public void setForceLocalCheck(boolean forceLocalCheck) {
+    this.forceLocalCheck = forceLocalCheck;
+  }
+
+
   /**
    * Load configuration from standard properties.
    */
@@ -483,6 +516,8 @@ public void load(Properties props) {
     runPlaceholders = property("placeholders", runPlaceholders);
     minVersion = property("minVersion", minVersion);
     minVersionFailMessage = property("minVersionFailMessage", minVersionFailMessage);
+    forceLocal = property("forceLocal", forceLocal);
+    forceLocalCheck = property("forceLocalCheck", forceLocalCheck);
 
     String patchInsertOn = property("patchInsertOn");
     if (patchInsertOn != null) {

ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java

@@ -1,6 +1,7 @@
 package io.ebean.migration;
 
 import java.util.Arrays;
+import java.util.Objects;
 
 import static java.lang.System.Logger.Level.*;
 
@@ -265,4 +266,19 @@ public String type() {
   public String getType() {
     return type();
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    MigrationVersion that = (MigrationVersion) o;
+    return Arrays.equals(ordering, that.ordering) && Objects.equals(comment, that.comment);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = Objects.hash(comment);
+    result = 31 * result + Arrays.hashCode(ordering);
+    return result;
+  }
 }

ebean-migration/src/main/java/io/ebean/migration/runner/LocalDdlMigrationResource.java

@@ -37,4 +37,8 @@ private String missingOpensMessage() {
     return "NPE reading DB migration content at [" + location + "] Probably missing an 'opens dbmigration;' in module-info.java";
   }
 
+  @Override
+  int checksum() {
+    return Checksum.calculate(content());
+  }
 }

ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResource.java

@@ -91,4 +91,9 @@ public String type() {
   void setInitType() {
     this.type = MigrationVersion.BOOTINIT_TYPE;
   }
+
+  /**
+   * The checksum of this resource (without parameter replacement).
+   */
+  abstract int checksum();
 }

ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java

@@ -4,16 +4,21 @@
 import io.avaje.classpath.scanner.core.Scanner;
 import io.ebean.migration.JdbcMigration;
 import io.ebean.migration.MigrationConfig;
+import io.ebean.migration.MigrationException;
 import io.ebean.migration.MigrationVersion;
 
 import java.io.*;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
 import java.util.function.Predicate;
 
 import static java.lang.System.Logger.Level.DEBUG;
+import static java.lang.System.Logger.Level.WARNING;
 
 /**
  * Loads the DB migration resources and sorts them into execution order.
@@ -47,14 +52,57 @@ boolean readInitResources() {
    * Read all the migration resources (SQL scripts) returning true if there are versions.
    */
   boolean readResources() {
+
     if (readFromIndex()) {
       // automatically enable earlyChecksumMode when using index file with pre-computed checksums
       migrationConfig.setEarlyChecksumMode(true);
+      if (migrationConfig.isForceLocal() || migrationConfig.isForceLocalCheck()) {
+        return useLocalAndCheck(migrationConfig.isForceLocalCheck());
+      }
       return true;
     }
     return readResourcesForPath(migrationConfig.getMigrationPath());
   }
 
+  private boolean useLocalAndCheck(boolean fail) {
+    // throw away, what we have read from index.
+    Map<MigrationVersion, LocalMigrationResource> index = new LinkedHashMap<>();
+    for (LocalMigrationResource version : versions) {
+      index.put(version.version(), version);
+    }
+    versions.clear();
+    // and re-read the local resources
+    if (!readResourcesForPath(migrationConfig.getMigrationPath())) {
+      if (fail) {
+        throw new MigrationException("Could not read local resources");
+      } else {
+        log.log(WARNING, "Could not read local resources");
+        return false;
+      }
+    }
+
+    StringJoiner errors = new StringJoiner("; ");
+    for (LocalMigrationResource version : versions) {
+      LocalMigrationResource refVersion = index.remove(version.version);
+      if (refVersion == null) {
+        errors.add("'" + version + "' not in index file");
+      } else if (refVersion.checksum() != version.checksum()) {
+        errors.add("'" + version + "' checksum mismatch (index " + refVersion.checksum() + ", local " + version.checksum() + ")");
+      }
+    }
+    for (LocalMigrationResource refVersion : index.values()) {
+      errors.add("'" + refVersion + "' not in local migrations");
+    }
+    if (errors.length() != 0) {
+      if (fail) {
+        throw new MigrationException("Index validation failed: " + errors);
+      } else {
+        log.log(WARNING, "Index validation failed: {0}", errors);
+      }
+    }
+    return true;
+  }
+
   private boolean readFromIndex() {
     final var base = "/" + migrationConfig.getMigrationPath() + "/";
     final var basePlatform = migrationConfig.getBasePlatform();

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java

@@ -302,7 +302,7 @@ private boolean runMigration(LocalMigrationResource local, MigrationMetaRow exis
     int checksum;
     int checksum2 = 0;
     if (local instanceof LocalUriMigrationResource) {
-      checksum = ((LocalUriMigrationResource)local).checksum();
+      checksum = local.checksum();
       checksum2 = patchLegacyChecksums ? AUTO_PATCH_CHECKSUM : 0;
       script = convertScript(local.content());
     } else if (local instanceof LocalDdlMigrationResource) {
@@ -312,7 +312,7 @@ private boolean runMigration(LocalMigrationResource local, MigrationMetaRow exis
       checksum = Checksum.calculate(earlyChecksumMode ? content : script);
       checksum2 = patchLegacyChecksums ? Checksum.calculate(script) : 0;
     } else {
-      checksum = ((LocalJdbcMigrationResource) local).checksum();
+      checksum = local.checksum();
     }
 
     if (existing == null && patchInsertMigration(local, checksum)) {

ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_FastCheckTest.java

@@ -9,6 +9,7 @@
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 public class MigrationRunner_FastCheckTest {
 
@@ -97,4 +98,47 @@ public void autoEnableEarlyMode() {
     assertThat(config.isEarlyChecksumMode()).isTrue();
   }
 
+  @Test
+  public void checkIndex_valid() {
+
+    MigrationConfig config = createMigrationConfig();
+
+    config.setPlatform("h2");
+    config.setMigrationPath("indexB_check_valid");
+    config.setForceLocalCheck(true);
+    config.setDbUrl("jdbc:h2:mem:checkindex_valid");
+    config.setRunPlaceholderMap(Map.of("my_table_name", "bar"));
+
+    MigrationRunner runner = new MigrationRunner(config);
+    runner.run();
+
+  }
+
+  @Test
+  public void checkIndex_invalid() {
+
+    MigrationConfig config = createMigrationConfig();
+
+    config.setPlatform("h2");
+    config.setMigrationPath("indexB_check_invalid");
+    config.setForceLocalCheck(true);
+    config.setDbUrl("jdbc:h2:mem:checkindex_invalid");
+    config.setRunPlaceholderMap(Map.of("my_table_name", "bar"));
+
+    MigrationRunner runner = new MigrationRunner(config);
+    assertThatThrownBy(runner::checkState)
+      .isInstanceOf(MigrationException.class)
+      .hasMessageContaining("'1.2' checksum mismatch (index -123456, local -212580746)")
+      .hasMessageContaining("'1.3' not in index file");
+
+    // switch to forceLocal only. This means, we get a warning about index validations on the console,
+    // but the local files are used
+    config.setForceLocalCheck(false);
+    config.setForceLocal(true);
+
+    List<MigrationResource> state = runner.checkState();
+    assertThat(state).hasSize(4);
+    // 1.3 not mentioned in the idx.
+    assertThat(state.get(3).location()).isEqualTo("indexB_check_invalid/1.3.sql");
+  }
 }

ebean-migration/src/test/resources/indexB_check_invalid/1.0__initial.sql

@@ -0,0 +1,6 @@
+-- apply changes
+create table ${my_table_name} (
+  id                            integer generated by default as identity not null,
+  name                          varchar(255),
+  constraint pk_${my_table_name} primary key (id)
+);

ebean-migration/src/test/resources/indexB_check_invalid/1.1.sql

@@ -0,0 +1 @@
+insert into ${my_table_name} (name) values ('hi');

ebean-migration/src/test/resources/indexB_check_invalid/1.2.sql

@@ -0,0 +1 @@
+create table foo (acol varchar(10));

ebean-migration/src/test/resources/indexB_check_invalid/1.3.sql

@@ -0,0 +1 @@
+create table bazz (acol varchar(10));

ebean-migration/src/test/resources/indexB_check_invalid/idx_h2.migrations

@@ -0,0 +1,5 @@
+-354472720,      1.0__initial.sql
+-443571620,      1.1.sql
+-123456,         1.2.sql
+
+

ebean-migration/src/test/resources/indexB_check_valid/1.0__initial.sql

@@ -0,0 +1,6 @@
+-- apply changes
+create table ${my_table_name} (
+  id                            integer generated by default as identity not null,
+  name                          varchar(255),
+  constraint pk_${my_table_name} primary key (id)
+);

ebean-migration/src/test/resources/indexB_check_valid/1.1.sql

@@ -0,0 +1 @@
+insert into ${my_table_name} (name) values ('hi');

ebean-migration/src/test/resources/indexB_check_valid/1.2.sql

@@ -0,0 +1 @@
+create table foo (acol varchar(10));

ebean-migration/src/test/resources/indexB_check_valid/1.3.sql

@@ -0,0 +1 @@
+create table bazz (acol varchar(10));

ebean-migration/src/test/resources/indexB_check_valid/idx_h2.migrations

@@ -0,0 +1,5 @@
+-354472720,      1.0__initial.sql
+-443571620,      1.1.sql
+-212580746,      1.2.sql
+-1779180902,     1.3.sql
+