DAT-13285 - Fix CheckSum generation issues + improvement (#3914)

* Compute checksum method updated

* Checksum logic updated for visitors and changes.

* GenerateCheckSum field filter logic updated, plus refactoring on generateCheckSum logic from createProcedure change

* generateCheckSum code refactoring done for Create Procedure, updated same method logic for CreateView change, plus some minor refactorings.

* Fixed unit tests.

* Added createProcedure and changeSet unit tests

* Added back procedure text null check.

* Null check added on normalizing stream constructor method.

* Correction to previous commmit on null check added on normalizing stream constructor

* CURRENT_CHECKSUM_ALGORITHM_VERSION updated from 8 to 9.

* Update checksum test assertions to match with algorithm version change

* Review comments applied.

* Update encoding configuration for CreateProcedure change.

* CreateProcedureChange test name updated.

* Update calculateCheckSum command tests to upgrade expected checksum which changed after upgrading algorithm version.

* CreateProcedure and CreateView tests added, plus minor refactorings.

* Merge

* Updates checksum for tests.

* Exclude fields for PRO change types to avoid recalculating checksum when there are not significant changes.

* Added a generic generateCheckSum method to use it across some different change types extending from CreateProcedureChange.

* Add missing parameter

* - Update change objects (CreateProcedure, CreateView and SQLFile) to make use properties updates when using a SQL file
- Added unit tests for changes made.

* - Support added to exclude change set attribute fields not needed when computing checksum.
- Extract expanding stream content in to a separate component.

* - Undo fields exclusion from generateCheckSum (AbstractSQLChange).
- PropertyExpandingStream refactoring.

* - New checksum unit tests added for some more change attributes for CreateProcedure and CreateView.
- Minor refactoring done on tests.

* Undo db change property changes recently done from CreateProcedure change.

* DAT-14642 - Prevents Liquibase from incorrectly attempting to redeploy changesets  during checksum upgrade (#4193)

* Makes sure that we reset ChangeLogHistoryServiceFactory after upgradingCheckSums.

* Rollback DropAllCommand.java changes.

* Trigger Build

* Rollback changes

* Makes ReleaseLocks Command checksum upgrade consistent with ListLocks

* Fix null md5sum issues during checksum upgrades

* When updating checksums use the stored file path instead of the generated one as we changed the way that we store paths during the years

* Fix validate command

* Fix unexpected-changesets command

* Adding unit tests for PR 4210 .

---------

Co-authored-by: Sayali M <sayali@Sayalis-MacBook-Pro>

---------

Co-authored-by: Daniel Mallorga <dmallorga@liquibase.com>
Co-authored-by: Daniel Mallorga <75833793+MalloD12@users.noreply.github.com>
Co-authored-by: filipe <filipe@caelum>
Co-authored-by: Sayali M <sayali@Sayalis-MacBook-Pro>
Co-authored-by: suryaaki2 <80348493+suryaaki2@users.noreply.github.com>

liquibase-integration-tests/src/test/java/liquibase/statementexecute/MarkChangeSetRanExecuteTest.java

@@ -35,85 +35,85 @@ public void generateSql_insert() throws Exception {
         assertCorrect("insert into [databasechangelog] ([id], [author], [filename], [dateexecuted], " +
                         "[orderexecuted], [md5sum], [description], [comments], [exectype], [contexts], [labels], " +
                         "[liquibase], [deployment_id]) values ('a', 'b', 'c', getdate(), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 MSSQLDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
-                        "('a', 'b', 'c', systimestamp, 1, '8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', " +
+                        "('a', 'b', 'c', systimestamp, 1, '9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', " +
                         "'executed', 'e', null, '" + version + "', null)",
                 OracleDatabase.class);
         assertCorrect("insert into [databasechangelog] ([id], [author], [filename], [dateexecuted], " +
                         "[orderexecuted], [md5sum], [description], [comments], [exectype], [contexts], [labels], " +
                         "[liquibase], [deployment_id]) values ('a', 'b', 'c', getdate(), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 SybaseDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', " +
-                        "current year to fraction(5), 1, '8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', " +
+                        "current year to fraction(5), 1, '9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', " +
                         "'executed', " +
                         "'e', null, '" + version + "', null)",
                 InformixDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', current timestamp, 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 DB2Database.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', current_timestamp, 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 FirebirdDatabase.class, DerbyDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', now, 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 HsqlDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', now(), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 SybaseASADatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, `description`, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', now(), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 MySQLDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, `description`, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', now(), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 MariaDBDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', now(), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 PostgresDatabase.class, H2Database.class, CockroachDatabase.class, EnterpriseDBDatabase.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', date('now'), 1, " +
-                        "'8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
+                        "'9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, '" + version + "'," +
                         " null)",
                 Ingres9Database.class);
         assertCorrect("insert into databasechangelog (id, author, filename, dateexecuted, orderexecuted, " +
                         "md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) values " +
                         "('a', 'b', 'c', current_timestamp::timestamp_ntz, 1," +
-                        " '8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, " +
+                        " '9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, " +
                         "'" + version + "', null)",
                 SnowflakeDatabase.class);
         assertCorrectOnRest("insert into databasechangelog (id, author, filename, dateexecuted, " +
                 "orderexecuted, md5sum, description, comments, exectype, contexts, labels, liquibase, deployment_id) " +
                 "values ('a', 'b', 'c', " +
-                "current timestamp, 1, '8:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, " +
+                "current timestamp, 1, '9:d41d8cd98f00b204e9800998ecf8427e', 'empty', '', 'executed', 'e', null, " +
                 "'" + version + "', null)");
     }
 
@@ -124,39 +124,39 @@ public void generateSql_update() throws Exception {
         this.statementUnderTest = new MarkChangeSetRanStatement(new ChangeSet("a", "b", false, false, "c", "e", "f",
                 null), ChangeSet.ExecType.RERAN);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = getdate(), [deployment_id] = null, [exectype] " +
-                        "= 'reran', [labels] = null, [md5sum] = '8:d41d8cd98f00b204e9800998ecf8427e', [orderexecuted] = 1 where [id] =" +
+                        "= 'reran', [labels] = null, [md5sum] = '9:d41d8cd98f00b204e9800998ecf8427e', [orderexecuted] = 1 where [id] =" +
                         " 'a' and" +
                         " [author] = 'b' and [filename] = 'c'",
                 MSSQLDatabase.class);
         assertCorrect("update databasechangelog set comments = '', contexts = 'e', dateexecuted = systimestamp, deployment_id = null, exectype = " +
-                        "'reran', labels = null, md5sum = '8:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id = 'a' and" +
+                        "'reran', labels = null, md5sum = '9:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id = 'a' and" +
                         " author " +
                         "= 'b' and filename = 'c'",
                 OracleDatabase.class);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = getdate(), [deployment_id] = null, [exectype] " +
-                "= 'reran', [labels] = null, [md5sum] = '8:d41d8cd98f00b204e9800998ecf8427e', [orderexecuted] = 1 where [id] = 'a' and" +
+                "= 'reran', [labels] = null, [md5sum] = '9:d41d8cd98f00b204e9800998ecf8427e', [orderexecuted] = 1 where [id] = 'a' and" +
                 " [author] = 'b' and [filename] = 'c'", SybaseDatabase.class);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = current year to fraction(5), deployment_id = " +
-                "null, exectype = 'reran', [labels] = null, md5sum = '8:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id " +
+                "null, exectype = 'reran', [labels] = null, md5sum = '9:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id " +
                 "= 'a' and author = 'b' and filename = 'c'", InformixDatabase.class);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = current timestamp, deployment_id = null, " +
-                        "exectype = 'reran', [labels] = null, md5sum = '8:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where " +
+                        "exectype = 'reran', [labels] = null, md5sum = '9:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where " +
                         "id = 'a' and author = 'b' and filename = 'c'",
                 DB2Database.class);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = current_timestamp, deployment_id = null, " +
-                        "exectype = 'reran', [labels] = null, md5sum = '8:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where " +
+                        "exectype = 'reran', [labels] = null, md5sum = '9:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where " +
                         "id = 'a' and author = 'b' and filename = 'c'",
                 FirebirdDatabase.class,
                 DerbyDatabase.class);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = NOW(), deployment_id = null, exectype = " +
-                        "'reran', [labels] = null, md5sum = '8:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id = 'a' and" +
+                        "'reran', [labels] = null, md5sum = '9:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id = 'a' and" +
                         " author = 'b' and filename = 'c'",
                 SybaseASADatabase.class);
         assertCorrect("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = NOW(), deployment_id = null, exectype = " +
-                        "'reran', [labels] = null, md5sum = '8:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id = 'a' and" +
+                        "'reran', [labels] = null, md5sum = '9:d41d8cd98f00b204e9800998ecf8427e', orderexecuted = 1 where id = 'a' and" +
                         " author = 'b' and filename = 'c'",
                 MySQLDatabase.class, MariaDBDatabase.class, HsqlDatabase.class, PostgresDatabase.class, H2Database.class, CockroachDatabase.class);
         assertCorrectOnRest("update [databasechangelog] set [comments] = '', [contexts] = 'e', [dateexecuted] = NOW(), [deployment_id] = null, [exectype] = 'reran', [labels] = null, [md5sum] = " +
-                "'8:d41d8cd98f00b204e9800998ecf8427e', [orderexecuted] = 1 where id = 'a' and author = 'b' and filename = 'c'");
+                "'9:d41d8cd98f00b204e9800998ecf8427e', [orderexecuted] = 1 where id = 'a' and author = 'b' and filename = 'c'");
     }
 }

liquibase-integration-tests/src/test/resources/liquibase/extension/testing/command/calculateChecksum.test.groovy

@@ -40,7 +40,7 @@ Optional Args:
         ]
 
         expectedResults = [
-                checksumResult   : "8:b6084e5d5f46b534bbbe18a0d35d34e0"
+                checksumResult   : "9:10de8cd690aed1d88d837cbe555d1684"
         ]
     }
 

liquibase-integration-tests/src/test/resources/liquibase/extension/testing/command/updateCountSql.test.groovy

@@ -62,7 +62,7 @@ Optional Args:
 --             They aren't used to compute the changeSet MD5Sum, so you can update them whenever you want without causing problems.
 CREATE TABLE PUBLIC.person (id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL, firstname VARCHAR(50), lastname VARCHAR(50) NOT NULL, CONSTRAINT PK_PERSON PRIMARY KEY (id));
 
-INSERT INTO PUBLIC.DATABASECHANGELOG (ID, AUTHOR, FILENAME, DATEEXECUTED, ORDEREXECUTED, MD5SUM, DESCRIPTION, COMMENTS, EXECTYPE, CONTEXTS, LABELS, LIQUIBASE, DEPLOYMENT_ID) VALUES ('1', 'nvoxland', 'changelogs/h2/complete/simple.changelog.xml', NOW(), 1, '8:b6084e5d5f46b534bbbe18a0d35d34e0', 'createTable tableName=person', 'You can add comments to changeSets.
+INSERT INTO PUBLIC.DATABASECHANGELOG (ID, AUTHOR, FILENAME, DATEEXECUTED, ORDEREXECUTED, MD5SUM, DESCRIPTION, COMMENTS, EXECTYPE, CONTEXTS, LABELS, LIQUIBASE, DEPLOYMENT_ID) VALUES ('1', 'nvoxland', 'changelogs/h2/complete/simple.changelog.xml', NOW(), 1, '9:10de8cd690aed1d88d837cbe555d1684', 'createTable tableName=person', 'You can add comments to changeSets.
             They can even be multiple lines if you would like.
             They aren''t used to compute the changeSet MD5Sum, so you can update them whenever you want without causing problems.', 'EXECUTED', NULL, NULL, 'DEV'
 """

liquibase-standard/src/main/java/liquibase/Liquibase.java

@@ -1397,6 +1397,7 @@ public void reportUnexpectedChangeSets(boolean verbose, Contexts contexts, Label
                                            Writer out) throws LiquibaseException {
         changeLogParameters.setContexts(contexts);
         changeLogParameters.setLabels(labelExpression);
+        checkLiquibaseTables(false, getDatabaseChangeLog(true), null, null);
 
         try {
             Collection<RanChangeSet> unexpectedChangeSets = listUnexpectedChangeSets(contexts, labelExpression);
@@ -1540,6 +1541,7 @@ public DiffResult diff(Database referenceDatabase, Database targetDatabase, Comp
      */
     public void validate() throws LiquibaseException {
         DatabaseChangeLog changeLog = getDatabaseChangeLog(true);
+        checkLiquibaseTables(false, changeLog, null, null);
         if (changeLog != null) {
             changeLog.validate(database);
         }

liquibase-standard/src/main/java/liquibase/change/AbstractChange.java

@@ -489,7 +489,19 @@ public boolean supportsRollback(Database database) {
      */
     @Override
     public CheckSum generateCheckSum() {
-        return CheckSum.compute(new StringChangeLogSerializer().serialize(this, false));
+        return CheckSum.compute(new StringChangeLogSerializer(new StringChangeLogSerializer.FieldFilter() {
+            @Override
+            public boolean include(Object obj, String field, Object value) {
+                if(Arrays.stream(getExcludedFieldFilters()).anyMatch(filter -> filter.equals(field))) {
+                    return false;
+                }
+                return super.include(obj, field, value);
+            }
+        }).serialize(this, false));
+    }
+
+    public String[] getExcludedFieldFilters() {
+        return new String[0];
     }
 
     /*
@@ -818,4 +830,5 @@ public String getDescription() {
 
         return description;
     }
+
 }