Sm/metadata-preview

.circleci/config.yml

@@ -13,6 +13,16 @@ jobs:
       - run: yarn build
       - run: yarn test:registry
 
+  registry-check-preview:
+    description: Checks registry against next version of metadataCoverageReport, reporting results to slack
+    docker:
+      - image: node:lts
+    steps:
+      - checkout
+      - run: yarn install
+      - run: yarn build
+      - run: yarn metadata:preview
+
   external-nut:
     description: Runs NUTs from other (external) repos by cloning them.  Substitutes a dependency for the current pull request.  For example, you're testing a PR to a library and want to test a plugin in another repo that uses the library.
 
@@ -102,8 +112,22 @@ jobs:
             echo "Environment Variables:"
             env
             NODE_OPTIONS=--max-old-space-size=8192 yarn test:nuts
+
 workflows:
   version: 2
+  registry-check-preview:
+    triggers:
+      - schedule:
+          # weekly on Monday morning
+          cron: 30 3 * * 1
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - registry-check-preview:
+          # required for slack webhook
+          context: salesforce-cli
   registry-check:
     triggers:
       - schedule:

package.json

@@ -91,6 +91,7 @@
     "local:install": "./scripts/localInstall.js install",
     "local:link": "./scripts/localInstall.js link",
     "local:unlink": "./scripts/localInstall.js unlink",
+    "metadata:preview": "npx ts-node scripts/update-registry/preview.ts",
     "prepack": "sf-prepack",
     "pretest": "sf-compile-test",
     "repl": "node --inspect ./scripts/repl.js",

scripts/update-registry/preview.ts

@@ -0,0 +1,61 @@
+import { CoverageObject } from '../../src/registry/types';
+import got from 'got';
+import { getMissingTypes } from '../../test/utils/getMissingTypes';
+import { registry } from '../../src';
+
+(async () => {
+  const currentApiVersion = (
+    JSON.parse((await got(`https://mdcoverage.secure.force.com/services/apexrest/report`)).body) as {
+      versions: { selected: number };
+    }
+  ).versions.selected;
+
+  const nextCoverage = JSON.parse(
+    (await got(`https://na${currentApiVersion - 8}.test1.pc-rnd.salesforce.com/mdcoverage/api.jsp`)).body
+  ) as CoverageObject;
+
+  const missingTypes = getMissingTypes(nextCoverage, registry).map((type) => type[0]);
+
+  console.log(`There are ${missingTypes.length} new types for v${nextCoverage.apiVersion} not in the registry.`);
+
+  const typesByFeature = new Map<string, string[]>();
+  missingTypes.map((t) => {
+    const featureLabel = nextCoverage.types[t].orgShapes.developer.features?.join(' & ') ?? 'NO FEATURE REQUIRED';
+    typesByFeature.set(featureLabel, [...(typesByFeature.get(featureLabel) ?? []), t]);
+  });
+  console.log(typesByFeature);
+  const formattedTypes = Array.from(typesByFeature, ([feature, types]) => `*${feature}*\n - ${types.join('\n - ')}`);
+
+  const json = {
+    blocks: [
+      {
+        type: 'header',
+        text: {
+          type: 'plain_text',
+          text: `v${nextCoverage.apiVersion} Metadata Preview`,
+        },
+      },
+      {
+        type: 'section',
+        text: {
+          type: 'plain_text',
+          text: `There are ${missingTypes.length} new types not in the registry, organized by required features (if any).`,
+        },
+      },
+      {
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text: formattedTypes.join('\n\n'),
+        },
+      },
+    ],
+  };
+  try {
+    await got.post(process.env.DEFAULT_SLACK_WEBHOOK, {
+      json,
+    });
+  } catch (e) {
+    console.error(e);
+  }
+})();

scripts/update-registry/update2.ts

@@ -103,7 +103,7 @@ const getMissingTypesAsDescribeResult = (missingTypes: [string, CoverageObjectTy
 
 const updateProjectScratchDef = (missingTypes: [string, CoverageObjectType][]) => {
   const scratchDefSummary = deepmerge.all(
-    [{}].concat(missingTypes.map(([key, missingType]) => JSON.parse(missingType.scratchDefinitions.developer)))
+    [{}].concat(missingTypes.map(([key, missingType]) => missingType.orgShapes.developer))
   ) as {
     features: string[];
   };

src/client/deployStrategies/containerDeploy.ts

@@ -5,6 +5,8 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 import { readFileSync } from 'graceful-fs';
+import { sleep } from '@salesforce/kit';
+
 import { deployTypes } from '../toolingApi';
 import { DeployError } from '../../errors';
 import {
@@ -106,7 +108,7 @@ export class ContainerDeploy extends BaseDeploy {
     let containerStatus: ContainerAsyncRequest;
     do {
       if (count > 0) {
-        await this.sleep(100);
+        await sleep(100);
       }
       containerStatus = (await this.connection.tooling.retrieve(
         ContainerDeploy.CONTAINER_ASYNC_REQUEST,
@@ -117,10 +119,6 @@ export class ContainerDeploy extends BaseDeploy {
     return containerStatus;
   }
 
-  private sleep(ms: number): Promise<number> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
   private buildSourceDeployResult(containerRequest: ContainerAsyncRequest): SourceDeployResult {
     const componentDeployment: ComponentDeployment = {
       component: this.component,

src/registry/nonSupportedTypes.ts

@@ -26,8 +26,6 @@ export const metadataTypes = [
 
   // things that don't show up in describe so far
   'PicklistValue', // only existed in v37, so it's hard to describe!
-  'FieldRestrictionRule', // not in describe for devorg.  ScratchDef might need feature 'EMPLOYEEEXPERIENCE' but it doesn't say that
-  'AppointmentSchedulingPolicy', // not in describe?
   'AppointmentAssignmentPolicy', // not in describe?
   'WorkflowFlowAction', // not in describe
   'AdvAcctForecastDimSource', // not in describe
@@ -40,21 +38,15 @@ export const metadataTypes = [
 ];
 
 export const hasUnsupportedFeatures = (type: CoverageObjectType): boolean => {
-  if (!type.scratchDefinitions?.developer) {
+  if (!type.orgShapes?.developer) {
     return true;
   }
-  const scratchDef = JSON.parse(type.scratchDefinitions.developer) as {
-    features?: string[];
-    settings?: {
-      [key: string]: unknown;
-    };
-  };
+
   if (
-    scratchDef.features &&
-    scratchDef.features.length > 0 &&
-    features.some((feature) => scratchDef.features.includes(feature))
+    type.orgShapes.developer.features?.length &&
+    features.some((feature) => type.orgShapes?.developer.features.includes(feature))
   ) {
     return true;
   }
-  return scratchDef.settings && settings.some((setting) => scratchDef.settings[setting]);
+  return type.orgShapes?.developer.settings && settings.some((setting) => type.orgShapes?.developer.settings[setting]);
 };

src/registry/types.ts

@@ -178,20 +178,23 @@ export const enum TransformerStrategy {
   NonDecomposed = 'nonDecomposed',
 }
 
+interface Channel {
+  exposed: boolean;
+}
 /**
  * Subset of an item from the Metadata Coverage Report
  */
 export interface CoverageObjectType {
-  scratchDefinitions: {
-    professional: string;
-    group: string;
-    enterprise: string;
-    developer: string;
+  orgShapes: {
+    developer: {
+      features?: string[];
+      settings?: Record<string, Record<string, unknown>>;
+    };
   };
   channels: {
-    metadataApi: boolean;
-    sourceTracking: boolean;
-    toolingApi: boolean;
+    metadataApi: Channel;
+    sourceTracking: Channel;
+    toolingApi: Channel;
   };
 }
 
@@ -202,9 +205,7 @@ export interface CoverageObject {
   types: {
     [key: string]: CoverageObjectType;
   };
-  versions: {
-    selected: number;
-    max: number;
-    min: number;
-  };
+  // only exists on the test1 instances flavor of coverage report
+  apiVersion: number;
+  release: string;
 }

test/registry/registryValidation.test.ts

@@ -7,7 +7,7 @@
 import { expect } from 'chai';
 import { MetadataRegistry } from '../../src';
 import { registry as defaultRegistry } from '../../src/registry/registry';
-import { MetadataType } from '../../src/registry/types';
+import { MetadataType, TransformerStrategy, DecompositionStrategy } from '../../src/registry/types';
 
 describe('Registry Validation', () => {
   const registry = defaultRegistry as MetadataRegistry;
@@ -194,4 +194,108 @@ describe('Registry Validation', () => {
         });
     });
   });
+
+  describe('top level required properties', () => {
+    describe('all have names and directoryName', () => {
+      Object.entries(registry.types).forEach(([key, type]) => {
+        it(`${type.id} has a name`, () => {
+          expect(type.name).to.be.a('string');
+        });
+        it(`${type.id} has a directoryName`, () => {
+          expect(type.directoryName).to.be.a('string');
+        });
+      });
+    });
+  });
+
+  describe('valid strategies', () => {
+    const typesWithStrategies = Object.values(registry.types).filter((type) => type.strategies);
+
+    // there isn't an enum for this in Types, to the known are hardcoded here
+    describe('valid, known adapters', () => {
+      typesWithStrategies.forEach((type) => {
+        it(`${type.id} has a valid adapter`, () => {
+          expect(['default', 'mixedContent', 'bundle', 'matchingContentFile', 'decomposed']).includes(
+            type.strategies.adapter
+          );
+        });
+      });
+    });
+
+    describe('adapter = matchingContentFile => no other strategy properties', () => {
+      typesWithStrategies
+        .filter((t) => t.strategies.adapter === 'matchingContentFile')
+        .forEach((type) => {
+          it(`${type.id} has no other strategy properties`, () => {
+            expect(type.strategies.decomposition).to.be.undefined;
+            expect(type.strategies.recomposition).to.be.undefined;
+            expect(type.strategies.transformer).to.be.undefined;
+          });
+        });
+    });
+
+    describe('adapter = bundle => no other strategy properties', () => {
+      typesWithStrategies
+        .filter((t) => t.strategies.adapter === 'bundle')
+        .forEach((type) => {
+          it(`${type.id} has no other strategy properties`, () => {
+            expect(type.strategies.decomposition).to.be.undefined;
+            expect(type.strategies.recomposition).to.be.undefined;
+            expect(type.strategies.transformer).to.be.undefined;
+          });
+        });
+    });
+
+    describe('adapter = decomposed => has transformer and decomposition props', () => {
+      typesWithStrategies
+        .filter((t) => t.strategies.adapter === 'decomposed')
+        .forEach((type) => {
+          it(`${type.id} has expected properties`, () => {
+            expect(type.strategies.decomposition).to.be.a('string');
+            expect(
+              [DecompositionStrategy.FolderPerType.valueOf(), DecompositionStrategy.TopLevel.valueOf()].includes(
+                type.strategies.decomposition
+              )
+            ).to.be.true;
+            expect(type.strategies.transformer).to.be.a('string');
+            expect(
+              [
+                TransformerStrategy.Standard.valueOf(),
+                TransformerStrategy.Decomposed.valueOf(),
+                TransformerStrategy.StaticResource.valueOf(),
+                TransformerStrategy.NonDecomposed.valueOf(),
+              ].includes(type.strategies.transformer)
+            ).to.be.true;
+            expect(type.strategies.recomposition).to.be.undefined;
+          });
+        });
+    });
+    it('no standard types specified in registry', () => {
+      expect(typesWithStrategies.filter((t) => t.strategies.transformer === 'standard')).to.have.length(0);
+    });
+    describe('adapter = mixedContent => has no decomposition/recomposition props', () => {
+      typesWithStrategies
+        .filter((t) => t.strategies.adapter === 'mixedContent')
+        .forEach((type) => {
+          it(`${type.id} has expected properties`, () => {
+            expect(type.strategies.decomposition).to.be.undefined;
+            expect(type.strategies.recomposition).to.be.undefined;
+            type.strategies.transformer
+              ? expect(type.strategies.transformer).to.be.a('string')
+              : expect(type.strategies.transformer).to.be.undefined;
+          });
+        });
+    });
+  });
+
+  describe('folders', () => {
+    const folderTypes = Object.values(registry.types).filter((type) => type.inFolder);
+
+    folderTypes.forEach((type) => {
+      it(`${type.name} has a valid folderType in the registry`, () => {
+        expect(type.folderType).to.not.be.undefined;
+        expect(registry.types[type.folderType]).to.be.an('object');
+      });
+    });
+  });
 });

test/utils/getMissingTypes.ts

@@ -14,7 +14,7 @@ export const getMissingTypes = (
 ): Array<[string, CoverageObjectType]> => {
   const metadataApiTypesFromCoverage = Object.entries(metadataCoverage.types).filter(
     ([key, value]) =>
-      value.channels.metadataApi && // if it's not in the mdapi, we don't worry about the registry
+      value.channels.metadataApi.exposed && // if it's not in the mdapi, we don't worry about the registry
       !metadataTypes.includes(key) && // types we should ignore, see the imported file for explanations
       !key.endsWith('Settings') && // individual settings shouldn't be in the registry
       !hasUnsupportedFeatures(value) // we don't support these types