issue #2694 updated schema docs merged with main

Signed-off-by: Robin Arnold <robin.arnold@ibm.com>

.github/workflows/rebuild.yml

@@ -26,6 +26,10 @@ jobs:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
         cache: 'maven'
+    - name: Patch release script
+      run: |
+        # only needed until we get past FHIR 4.10.2
+        sed -i '/^mvn -T2C test .* -f fhir-parent -P "${BUILD_PROFILES}"$/ s/$/ -Dtest=!com.ibm.fhir.ig.us.spl.ExamplesValidationTest/' build/release/bin/20_test/1_code_coverage.sh
     - name: Build and Release
       env:
         BASE: origin/${{ github['base_ref'] }}

fhir-audit/src/main/java/com/ibm/fhir/audit/mapper/impl/AuditEventMapper.java

@@ -442,16 +442,12 @@ private List<CodeableConcept> purposeOfEvent(AuditLogEntry entry) {
         // @formatter:on
     }
 
-    /*
-     * in the future, we could
-     * enhance this part to log the alternative outcomes.
-     * This applies to the next two methods: outcomeDesc, outcome
+    /**
+     * @implNote This applies to the next two methods: outcomeDesc, outcome
+     * Previously, it had pending as an option, however it should now be success/failure in all cases.
      */
     private com.ibm.fhir.model.type.String outcomeDesc(AuditLogEntry entry) {
-        if (entry.getContext().getEndTime() == null ||
-                entry.getContext().getStartTime().equalsIgnoreCase(entry.getContext().getEndTime())) {
-            return string("pending");
-        } else if (entry.getContext().getApiParameters().getStatus() < 400) {
+        if (entry.getContext().getApiParameters().getStatus() < 400) {
             return string("success");
         } else if (entry.getContext().getApiParameters().getStatus() < 500) {
             return string("minor failure");

fhir-audit/src/main/java/com/ibm/fhir/audit/mapper/impl/CADFMapper.java

@@ -133,10 +133,11 @@ public CadfEvent createCadfEvent(AuditLogEntry logEntry) throws IllegalStateExce
             fhirContext.setLocation(logEntry.getLocation());
             fhirContext.setDescription(logEntry.getDescription());
 
-            if (logEntry.getContext().getEndTime() == null ||
-                    logEntry.getContext().getStartTime().equalsIgnoreCase(logEntry.getContext().getEndTime())) {
-                cadfEventOutCome = Outcome.pending;
-            } else if (logEntry.getContext().getApiParameters().getStatus() < 400) {
+            /*
+             * @implNote This applies to the next two methods: outcomeDesc, outcome
+             * Previously, it had pending as an option, however it should now be success/failure in all cases.
+             */
+            if (logEntry.getContext().getApiParameters().getStatus() < 400) {
                 cadfEventOutCome = Outcome.success;
             } else {
                 cadfEventOutCome = Outcome.failure;

fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2020, 2021
+ * (C) Copyright IBM Corp. 2020, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -753,13 +753,13 @@ public void bootstrapDb() {
             // If our schema is already at the latest version, we can skip a lot of processing
             SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, schemaName,
                 FhirBucketSchemaVersion.getLatestSchemaVersion().vid());
-            if (svm.isLatestSchema()) {
-                logger.info("Already at latest version; skipping update for: '" + schemaName + "'");
-            } else {
+            if (svm.isSchemaOld()) {
                 buildSchema();
 
                 // Update the whole schema version
                 svm.updateSchemaVersion();
+            } else {
+                logger.info("Already at latest version; skipping update for: '" + schemaName + "'");
             }
         } finally {
             leaseManager.cancelLease();

fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2021
+ * (C) Copyright IBM Corp. 2021, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -8,8 +8,6 @@
 
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.Optional;
-import java.util.stream.Stream;
 
 import com.ibm.fhir.database.utils.api.IConnectionProvider;
 import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
@@ -90,15 +88,28 @@ public void updateSchemaVersionId(int versionId) {
      * table
      */
     public void updateSchemaVersion() {
-        updateSchemaVersionId(this.latestCodeVersion);
+        // The schema version should never regress, so make sure we don't change
+        // anything if this code is older than what's currently in the database
+        if (latestCodeVersion > getVersionForSchema()) {
+            updateSchemaVersionId(this.latestCodeVersion);
+        }
+    }
+
+    /**
+     * Returns true if the current schema version recorded in WHOLE_SCHEMA_VERSION is older
+     * the last version in FhirSchemaVersion and therefore needs to be updated
+     * @return
+     */
+    public boolean isSchemaOld() {
+        return getVersionForSchema() < latestCodeVersion;
     }
 
     /**
-     * Returns true if the current schema version recorded in WHOLE_SCHEMA_VERSION is the
-     * latest FhirSchemaVersion
+     * Returns true if the current schema version recorded in WHOLE_SCHEMA_VERSION 
+     * exactly matches the last version in FhirSchemaVersion
      * @return
      */
-    public boolean isLatestSchema() {
+    public boolean isSchemaVersionMatch() {
         return getVersionForSchema() == latestCodeVersion;
     }
 }

fhir-database-utils/src/test/java/com/ibm/fhir/database/utils/schema/DerbySchemaVersionsTest.java

@@ -1,12 +1,13 @@
 /*
- * (C) Copyright IBM Corp. 2021
+ * (C) Copyright IBM Corp. 2021, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
 package com.ibm.fhir.database.utils.schema;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 
 import org.testng.annotations.Test;
@@ -16,7 +17,6 @@
 import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider;
 import com.ibm.fhir.database.utils.derby.DerbyMaster;
 import com.ibm.fhir.database.utils.pool.PoolConnectionProvider;
-import com.ibm.fhir.database.utils.schema.SchemaVersionsManager;
 import com.ibm.fhir.database.utils.transaction.SimpleTransactionProvider;
 import com.ibm.fhir.database.utils.version.CreateWholeSchemaVersion;
 
@@ -53,7 +53,64 @@ public void test() throws Exception {
             svm.updateSchemaVersion();
             assertEquals(svm.getVersionForSchema(), 5);
 
-            assertTrue(svm.isLatestSchema());
+            assertFalse(svm.isSchemaOld());
+            assertTrue(svm.isSchemaVersionMatch());
        }
     }
+
+    /**
+     * Make sure that we don't regress the schema version
+     * @throws Exception
+     */
+    @Test(dependsOnMethods = "test")
+    public void testSchemaVersionRegression() throws Exception {
+        String dbPath = TARGET_DIR + "fhirdb";
+        try (DerbyMaster db = new DerbyMaster(dbPath)) {
+
+            // Create the WHOLE_SCHEMA_VERSION table
+            db.runWithAdapter(adapter -> CreateWholeSchemaVersion.createTableIfNeeded(SCHEMA_NAME, adapter));
+
+            IConnectionProvider cp = new DerbyConnectionProvider(db, null);
+            PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, 10);
+            ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool);
+
+            // Pretend to be an older version of code
+            SchemaVersionsManager svmVersion3 = new SchemaVersionsManager(db.getTranslator(), connectionPool, transactionProvider, SCHEMA_NAME,
+                3);
+
+            // Apply the update (should be a NOP because we don't allow regression)
+            svmVersion3.updateSchemaVersion();
+
+            // The schema version should still be 5
+            assertEquals(svmVersion3.getVersionForSchema(), 5);
+            assertFalse(svmVersion3.isSchemaVersionMatch());
+        }
+    }
+
+    /**
+     * Make sure we don't apply changes if the schema is newer than
+     * the latest code
+     * @throws Exception
+     */
+    @Test(dependsOnMethods = "test")
+    public void testSchemaVersionCurrent() throws Exception {
+        String dbPath = TARGET_DIR + "fhirdb";
+        try (DerbyMaster db = new DerbyMaster(dbPath)) {
+
+            // Create the WHOLE_SCHEMA_VERSION table
+            db.runWithAdapter(adapter -> CreateWholeSchemaVersion.createTableIfNeeded(SCHEMA_NAME, adapter));
+
+            IConnectionProvider cp = new DerbyConnectionProvider(db, null);
+            PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, 10);
+            ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool);
+
+            // Pretend to be an older version of code
+            SchemaVersionsManager svm = new SchemaVersionsManager(db.getTranslator(), connectionPool, transactionProvider, SCHEMA_NAME,
+                3);
+
+            // The current schema version exceeds this code, so there's no need to apply any updates
+            assertFalse(svm.isSchemaOld());
+            assertFalse(svm.isSchemaVersionMatch());
+        }
+    }
 }

fhir-parent/pom.xml

@@ -343,12 +343,12 @@
             <dependency>
                 <groupId>org.apache.httpcomponents</groupId>
                 <artifactId>httpcore</artifactId>
-                <version>4.4.14</version>
+                <version>4.4.15</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.httpcomponents</groupId>
                 <artifactId>httpcore-nio</artifactId>
-                <version>4.4.14</version>
+                <version>4.4.15</version>
             </dependency>
             <dependency>
                 <groupId>commons-beanutils</groupId>
@@ -467,7 +467,7 @@
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protoc</artifactId>
-                <version>3.19.1</version>
+                <version>3.19.3</version>
                 <type>pom</type>
             </dependency>
             <dependency>

fhir-persistence-schema/README.md

@@ -12,4 +12,4 @@ The following guides contain detailed descriptions on usage, design and the mult
 * [Schema Design](https://github.com/IBM/FHIR/tree/master/fhir-persistence-schema/docs/SchemaDesign.md)
 * [Db2 Multi-tenancy](https://github.com/IBM/FHIR/tree/master/fhir-persistence-schema/docs/DB2MultiTenancy.md)
 
-FHIR® is the registered trademark of HL7 and is used with the permission of HL7.
+FHIR® is the registered trademark of HL7 and is used with the permission of HL7.

fhir-persistence-schema/docs/SchemaToolUsageGuide.md

@@ -18,7 +18,7 @@ The schema tool generates the following object types that require management:
 For details on the schema design, refer to the [Schema Design](https://github.com/IBM/FHIR/tree/master/fhir-persistence-schema/docs/SchemaDesign.md) document.
 
 ----------------------------------------------------------------
-# Database Support
+## Database Support
 
 
 | Database   | Version   | Support |
@@ -564,6 +564,7 @@ and grants permission to the username|
 |--vacuum-cost-limit|costLimit|The Vacuum cost limit to set|
 |--skip-allocate-if-tenant-exists||Skips allocating a tenant if it already exists|
 |--force-unused-table-removal||Forces the removal of unused tables - DomainResource, Resource|
+| --force || Do not skip schema update process when the whole-schema-version matches.|
 |--prop|name=value|name=value that is passed in on the commandline|
 |--pool-size|poolSize|poolsize used with the database actions|
 |--drop-schema-oauth||drop the db schema used by liberty's oauth/openid connect features|

fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019, 2021
+ * (C) Copyright IBM Corp. 2019, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -23,6 +23,7 @@
 import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_FHIR;
 import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_OAUTH;
 import static com.ibm.fhir.schema.app.menu.Menu.DROP_TENANT;
+import static com.ibm.fhir.schema.app.menu.Menu.FORCE;
 import static com.ibm.fhir.schema.app.menu.Menu.FORCE_UNUSED_TABLE_REMOVAL;
 import static com.ibm.fhir.schema.app.menu.Menu.FREEZE_TENANT;
 import static com.ibm.fhir.schema.app.menu.Menu.GRANT_TO;
@@ -258,6 +259,9 @@ public class Main {
     // Forces the removal of tables if data exists
     private boolean forceUnusedTableRemoval = false;
 
+    // Force schema update even if whole-schema-version is current
+    private boolean force = false;
+
     // Tenant Key Output or Input File
     private String tenantKeyFileName;
     private TenantKeyFileUtil tenantKeyFileUtil = new TenantKeyFileUtil();
@@ -501,9 +505,7 @@ protected void updateFhirSchema() {
             // If our schema is already at the latest version, we can skip a lot of processing
             SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName,
                 FhirSchemaVersion.getLatestFhirSchemaVersion().vid());
-            if (svm.isLatestSchema()) {
-                logger.info("Already at latest version; skipping update for: '" + targetSchemaName + "'");
-            } else {
+            if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) {
                 // Build/update the FHIR-related tables as well as the stored procedures
                 PhysicalDataModel pdm = new PhysicalDataModel();
                 buildFhirDataSchemaModel(pdm);
@@ -541,6 +543,11 @@ protected void updateFhirSchema() {
                     // Finally, update the whole schema version
                     svm.updateSchemaVersion();
                 }
+            } else if (this.force) {
+                logger.info("Cannot force when schema is ahead of this version; skipping update for: '" + targetSchemaName + "'");
+                this.exitStatus = EXIT_BAD_ARGS;
+            } else {
+                logger.info("Schema is up-to-date; skipping update for: '" + targetSchemaName + "'");
             }
         } finally {
             leaseManager.cancelLease();
@@ -568,9 +575,7 @@ protected void updateOauthSchema() {
             // If our schema is already at the latest version, we can skip a lot of processing
             SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName,
                 FhirSchemaVersion.getLatestFhirSchemaVersion().vid());
-            if (svm.isLatestSchema()) {
-                logger.info("Already at latest version; skipping update for: '" + targetSchemaName + "'");
-            } else {
+            if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) {
                 PhysicalDataModel pdm = new PhysicalDataModel();
                 buildOAuthSchemaModel(pdm);
                 updateSchema(pdm);
@@ -584,6 +589,11 @@ protected void updateOauthSchema() {
                     // Mark the schema as up-to-date
                     svm.updateSchemaVersion();
                 }
+            } else if (this.force) {
+                logger.info("Cannot force when schema is ahead of this version; skipping update for: '" + targetSchemaName + "'");
+                this.exitStatus = EXIT_BAD_ARGS;
+            } else {
+                logger.info("Schema is current; skipping update for: '" + targetSchemaName + "'");
             }
         } finally {
             leaseManager.cancelLease();
@@ -611,9 +621,7 @@ protected void updateJavaBatchSchema() {
             // If our schema is already at the latest version, we can skip a lot of processing
             SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName,
                 FhirSchemaVersion.getLatestFhirSchemaVersion().vid());
-            if (svm.isLatestSchema()) {
-                logger.info("Already at latest version; skipping update for: '" + targetSchemaName + "'");
-            } else {
+            if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) {
                 PhysicalDataModel pdm = new PhysicalDataModel();
                 buildJavaBatchSchemaModel(pdm);
                 updateSchema(pdm);
@@ -627,6 +635,11 @@ protected void updateJavaBatchSchema() {
                     // Mark the schema as up-to-date
                     svm.updateSchemaVersion();
                 }
+            } else if (this.force) {
+                logger.info("Cannot force when schema is ahead of this version; skipping update for: '" + targetSchemaName + "'");
+                this.exitStatus = EXIT_BAD_ARGS;
+            } else {
+                logger.info("Schema is current; skipping update for: '" + targetSchemaName + "'");
             }
         } finally {
             leaseManager.cancelLease();
@@ -2061,6 +2074,9 @@ protected void parseArgs(String[] args) {
             case FORCE_UNUSED_TABLE_REMOVAL:
                 forceUnusedTableRemoval = true;
                 break;
+            case FORCE:
+                force = true;
+                break;
             default:
                 throw new IllegalArgumentException("Invalid argument: '" + arg + "'");
             }

fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2021
+ * (C) Copyright IBM Corp. 2021, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -55,6 +55,7 @@ public class Menu {
     public static final String CREATE_SCHEMA_FHIR = "--create-schema-fhir";
     public static final String CREATE_SCHEMA_BATCH = "--create-schema-batch";
     public static final String CREATE_SCHEMA_OAUTH = "--create-schema-oauth";
+    public static final String FORCE = "--force";
     public static final String HELP = "--help";
 
     public Menu() {
@@ -91,6 +92,7 @@ public enum HelpMenu {
         MI_VACUUM_TRESHOLD(VACUUM_TRESHOLD, "threshold", "The threshold value to alter to 'threshold'"),
         MI_VACUUM_COST_LIMIT(VACUUM_COST_LIMIT, "costLimit", "The Vacuum cost limit to set"),
         MI_SKIP_ALLOCATE_IF_TENANT_EXISTS(SKIP_ALLOCATE_IF_TENANT_EXISTS, "", "Skips allocating a tenant if it already exists"),
+        MI_FORCE(FORCE, "", "Do not skip schema update process when the whole-schema-version matches."),
         MI_FORCE_UNUSED_TABLE_REMOVAL(FORCE_UNUSED_TABLE_REMOVAL, "", "Forces the removal of unused tables - DomainResource, Resource"),
         MI_PROP(PROP, "name=value", "name=value that is passed in on the commandline"),
         MI_POOL_SIZE(POOL_SIZE, "poolSize", "poolsize used with the database actions"),