feat: syncing status for all organizations

src/database/migrations/20231020201652-OrgSyncStatus.js

@@ -0,0 +1,23 @@
+'use strict';
+
+export default {
+  async up(queryInterface, Sequelize) {
+    await Promise.all(
+      ['organizations'].map((table) => {
+        queryInterface.addColumn(table, 'synced', {
+          type: Sequelize.BOOLEAN,
+          allowNull: true,
+          defaultValue: false,
+        });
+      }),
+    );
+  },
+
+  async down(queryInterface) {
+    await Promise.all(
+      ['organizations'].map((table) => {
+        queryInterface.removeColumn(table, 'synced');
+      }),
+    );
+  },
+};

src/database/migrations/index.js

@@ -30,6 +30,7 @@ import PopulateUnitsFTS from './20220808192709-populate-units-fts';
 import ResetDBForNewSingletons from './20220816155101-reset-db-for-new-singletons';
 import AddIsTransferColumn from './20220825124702-add-isTransfer-column';
 import AddOrgMetadata from './20220831023546-add-org-metadata';
+import OrgSyncStatus from './20231020201652-OrgSyncStatus';
 
 export const migrations = [
   {
@@ -164,4 +165,8 @@ export const migrations = [
     migration: AddOrgMetadata,
     name: '20220831023546-add-org-metadata',
   },
+  {
+    migration: OrgSyncStatus,
+    name: '20231020201652-OrgSyncStatus',
+  },
 ];

src/middleware.js

@@ -6,14 +6,14 @@ import bodyParser from 'body-parser';
 import cors from 'cors';
 import { V1Router } from './routes/v1';
 import { getConfig } from './utils/config-loader';
-import { logger } from './config/logger.cjs';
 import {
   assertChiaNetworkMatchInConfiguration,
   assertDataLayerAvailable,
   assertWalletIsAvailable,
 } from './utils/data-assertions';
 import packageJson from '../package.json' assert { type: 'json' };
 import datalayer from './datalayer';
+import { Organization } from './models';
 
 const { CADT_API_KEY, READ_ONLY, IS_GOVERNANCE_BODY, USE_SIMULATOR } =
   getConfig().APP;
@@ -24,6 +24,8 @@ const headerKeys = Object.freeze({
   DATA_MODEL_VERION_HEADER_KEY: 'x-datamodel-version',
   GOVERNANCE_BODY_HEADER_KEY: 'x-governance-body',
   WALLET_SYNCED: 'x-wallet-synced',
+  HOME_ORGANIZATION_SYNCED: 'x-home-org-synced',
+  ALL_DATA_SYNCED: 'x-data-synced',
 });
 
 const app = express();
@@ -88,9 +90,6 @@ app.use(function (req, res, next) {
 });
 
 app.use(function (req, res, next) {
-  logger.info(
-    `Setting header x-api-verion to package.json version: ${packageJson.version}`,
-  );
   const version = packageJson.version;
   res.setHeader(headerKeys.API_VERSION_HEADER_KEY, version);
 
@@ -100,6 +99,33 @@ app.use(function (req, res, next) {
   next();
 });
 
+app.use(async function (req, res, next) {
+  // If the home organization is syncing, then we treat all requests as read-only
+  const homeOrg = await Organization.getHomeOrg();
+
+  if (req.method !== 'GET' && !homeOrg.synced) {
+    res.status(400).json({
+      message:
+        'Your organization data is still resyncing, please try again after it completes',
+      success: false,
+    });
+  } else if (homeOrg.synced) {
+    res.setHeader(headerKeys.HOME_ORGANIZATION_SYNCED, true);
+  } else {
+    res.setHeader(headerKeys.HOME_ORGANIZATION_SYNCED, false);
+  }
+
+  next();
+});
+
+app.use(async function (req, res, next) {
+  const orgMap = await Organization.getOrgsMap();
+  const notSynced = Object.keys(orgMap).find((key) => !orgMap[key].synced);
+
+  res.setHeader(headerKeys.ALL_DATA_SYNCED, !notSynced);
+  next();
+});
+
 app.use(async function (req, res, next) {
   if (USE_SIMULATOR) {
     res.setHeader(headerKeys.WALLET_SYNCED, true);

src/models/organizations/organizations.model.js

@@ -33,6 +33,7 @@ class Organization extends Model {
         'registryId',
         'fileStoreId',
         'metadata',
+        'synced',
       ],
       where: { isHome: true },
       raw: true,
@@ -52,13 +53,15 @@ class Organization extends Model {
       delete myOrganization.metadata;
     }
 
+    myOrganization.synced = myOrganization.synced === 1;
+
     if (myOrganization && includeAddress) {
       myOrganization.xchAddress = await datalayer.getPublicAddress();
       myOrganization.fileStoreSubscribed = true;
       return myOrganization;
     }
 
-    return undefined;
+    return myOrganization;
   }
 
   static async getOrgsMap() {
@@ -69,6 +72,7 @@ class Organization extends Model {
         'icon',
         'isHome',
         'subscribed',
+        'synced',
         'fileStoreSubscribed',
       ],
     });
@@ -256,8 +260,7 @@ class Organization extends Model {
     });
   }
 
-  // eslint-disable-next-line
-  static importOrganization = async (orgUid) => {
+  static async importOrganization(orgUid) {
     try {
       console.log('Importing organization ' + orgUid);
       const orgData = await datalayer.getSubscribedStoreData(orgUid);
@@ -310,10 +313,9 @@ class Organization extends Model {
     } catch (error) {
       logger.info(error.message);
     }
-  };
+  }
 
-  // eslint-disable-next-line
-  static subscribeToOrganization = async (orgUid) => {
+  static async subscribeToOrganization(orgUid) {
     const exists = await Organization.findOne({ where: { orgUid } });
     if (exists) {
       await Organization.update({ subscribed: true }, { where: { orgUid } });
@@ -322,14 +324,13 @@ class Organization extends Model {
         'Can not subscribe, please import this organization first',
       );
     }
-  };
+  }
 
-  // eslint-disable-next-line
-  static unsubscribeToOrganization = async (orgUid) => {
+  static async unsubscribeToOrganization(orgUid) {
     await Organization.update({ subscribed: false }, { orgUid });
-  };
+  }
 
-  static syncOrganizationMeta = async () => {
+  static async syncOrganizationMeta() {
     try {
       const allSubscribedOrganizations = await Organization.findAll({
         subscribed: true,
@@ -389,9 +390,9 @@ class Organization extends Model {
     } catch (error) {
       logger.info(error.message);
     }
-  };
+  }
 
-  static subscribeToDefaultOrganizations = async () => {
+  static async subscribeToDefaultOrganizations() {
     try {
       const defaultOrgs = await getDefaultOrganizationList();
       if (!Array.isArray(defaultOrgs)) {
@@ -414,9 +415,9 @@ class Organization extends Model {
     } catch (error) {
       logger.info(error);
     }
-  };
+  }
 
-  static editOrgMeta = async ({ name, icon }) => {
+  static async editOrgMeta({ name, icon }) {
     const myOrganization = await Organization.getHomeOrg();
 
     const payload = {};
@@ -430,20 +431,20 @@ class Organization extends Model {
     }
 
     await datalayer.upsertDataLayer(myOrganization.orgUid, payload);
-  };
+  }
 
-  static addMetadata = async (payload) => {
+  static async addMetadata(payload) {
     const myOrganization = await Organization.getHomeOrg();
 
     // Prefix keys with "meta_"
     const metadata = _.mapKeys(payload, (_value, key) => `meta_${key}`);
 
     await datalayer.upsertDataLayer(myOrganization.orgUid, metadata);
-  };
+  }
 
-  static removeMirror = async (storeId, coinId) => {
+  static async removeMirror(storeId, coinId) {
     datalayer.removeMirror(storeId, coinId);
-  };
+  }
 }
 
 Organization.init(ModelTypes, {

src/models/organizations/organizations.modeltypes.cjs

@@ -33,6 +33,11 @@ module.exports = {
     allowNull: true,
     defaultValue: '{}',
   },
+  synced: {
+    type: Sequelize.BOOLEAN,
+    allowNull: true,
+    defaultValue: false,
+  },
   createdAt: Sequelize.DATE,
   updatedAt: Sequelize.DATE,
 };

src/tasks/sync-audit-table.js

@@ -55,6 +55,11 @@ const task = new Task('sync-audit', async () => {
           metaKey: 'migratedToNewSync',
           metaValue: 'true',
         });
+
+        await Organization.upsert({
+          synced: false,
+        });
+
         logger.info(`Migration Complete`);
       }
     }
@@ -155,6 +160,8 @@ const syncOrganizationAudit = async (organization) => {
   try {
     logger.info(`Syncing Registry: ${_.get(organization, 'name')}`);
     let afterCommitCallbacks = [];
+
+    const homeOrg = await Organization.getHomeOrg();
     const rootHistory = await datalayer.getRootHistory(organization.registryId);
 
     let lastRootSaved;
@@ -198,7 +205,7 @@ const syncOrganizationAudit = async (organization) => {
 
       // Destroy existing records for this singleton
       // On a fresh db this does nothing, but when the audit table
-      // is reset this will ensure that this organizations regsitry data is
+      // is reset this will ensure that this organizations registry data is
       // cleaned up on both the local db and mirror db and ready to resync
       await Promise.all(
         Object.keys(ModelKeys).map(async (modelKey) => {
@@ -213,7 +220,15 @@ const syncOrganizationAudit = async (organization) => {
       return;
     }
 
-    if (historyIndex === rootHistory.length) {
+    const isSynced = rootHistory[rootHistory.length - 1].root_hash === rootHash;
+
+    await Organization.update(
+      { synced: isSynced },
+      { where: { orgUid: organization.orgUid } },
+    );
+
+    if (isSynced) {
+      logger.info(`No new data to sync for ${organization.name}`);
       return;
     }
 
@@ -254,8 +269,6 @@ const syncOrganizationAudit = async (organization) => {
       return typeOrder[a.type] - typeOrder[b.type];
     });
 
-    const homeOrg = await Organization.getHomeOrg();
-
     const updateTransaction = async (transaction, mirrorTransaction) => {
       for (const diff of kvDiff) {
         const key = decodeHex(diff.key);