feat: improve query performance for manage permissions

libs/features/manage-permissions/src/usePermissionRecords.tsx

@@ -1,7 +1,7 @@
 import { logger } from '@jetstream/shared/client-logger';
-import { describeSObject, queryAll, queryAllUsingOffset } from '@jetstream/shared/data';
+import { describeSObject, queryAll, queryAllUsingCursor } from '@jetstream/shared/data';
 import { tracker } from '@jetstream/shared/ui-utils';
-import { getErrorMessage, groupByFlat, splitArrayToMaxSize } from '@jetstream/shared/utils';
+import { ConcurrencyLimiter, createConcurrencyLimiter, getErrorMessage, groupByFlat } from '@jetstream/shared/utils';
 import {
   EntityParticlePermissionsRecord,
   FieldPermissionDefinitionMap,
@@ -17,6 +17,7 @@ import {
 import { useCallback, useEffect, useRef, useState } from 'react';
 import {
   getFieldDefinitionKey,
+  getPermissionableFieldObjectChunks,
   getQueryForAllPermissionableFields,
   getQueryForFieldPermissions,
   getQueryObjectPermissions,
@@ -68,24 +69,36 @@ export function usePermissionRecords(selectedOrg: SalesforceOrgUi, sobjects: str
       if (hasError) {
         setHasError(false);
       }
+      // Every request below shares this limiter so that a large selection cannot flood Salesforce
+      const limit = createConcurrencyLimiter(QUERY_CONCURRENCY);
       // query all data and transform into state maps
       const output = await Promise.all([
-        describeSObject(selectedOrg, 'FieldPermissions'),
-        queryAndCombineResults<EntityParticlePermissionsRecord>(selectedOrg, getQueryForAllPermissionableFields(sobjects), true, true),
-        queryAndCombineResults<ObjectPermissionRecord>(selectedOrg, getQueryObjectPermissions(sobjects, permSetIds, profilePermSetIds)),
-        queryAndCombineResults<FieldPermissionRecord>(selectedOrg, getQueryForFieldPermissions(sobjects, permSetIds, profilePermSetIds)),
+        limit(() => describeSObject(selectedOrg, 'FieldPermissions')),
+        queryPermissionableFields(limit, selectedOrg, sobjects),
+        queryAndCombineResults<ObjectPermissionRecord>(
+          limit,
+          selectedOrg,
+          getQueryObjectPermissions(sobjects, permSetIds, profilePermSetIds),
+        ),
+        queryAndCombineResults<FieldPermissionRecord>(
+          limit,
+          selectedOrg,
+          getQueryForFieldPermissions(sobjects, permSetIds, profilePermSetIds),
+        ),
         queryAndCombineResults<TabVisibilityPermissionRecord>(
+          limit,
           selectedOrg,
           getQueryTabVisibilityPermissions(sobjects, permSetIds, profilePermSetIds),
         ).then((record) => record.map((item) => ({ ...item, Name: item.Name.replace('standard-', '') }))),
-        queryAndCombineResults<TabDefinitionRecord>(selectedOrg, getQueryTabDefinition(sobjects), false, true).then((tabs) =>
+        queryAndCombineResults<TabDefinitionRecord>(limit, selectedOrg, getQueryTabDefinition(sobjects), true).then((tabs) =>
           groupByFlat(tabs, 'SobjectName'),
         ),
         // System permissions are `Permissions*` columns on the PermissionSet record itself; describe to
         // learn which are settable in this org, then query their current values by permission set id.
-        describeSObject(selectedOrg, 'PermissionSet').then((describeResult) => {
+        limit(() => describeSObject(selectedOrg, 'PermissionSet')).then((describeResult) => {
           const systemPermissionFields = getSystemPermissionFieldsFromDescribe(describeResult.data.fields);
           return queryAndCombineResults<SystemPermissionSetRecord>(
+            limit,
             selectedOrg,
             getQuerySystemPermissions(
               [...permSetIds, ...profilePermSetIds],
@@ -198,30 +211,53 @@ function getSystemPermissionMap(
 }
 
 /**
- * Number of queries to run concurrently. Offset-paged queries (EntityParticle) are split into many small batches to stay
- * under Salesforce's OFFSET cap, so we run them in bounded waves rather than sequentially to keep large selections fast.
+ * Maximum requests in flight at once for a single load. Every query group shares one limiter, since limiting each
+ * group on its own would still let the groups fan out in parallel and multiply into a request flood.
  */
 const QUERY_CONCURRENCY = 5;
 
+/** Runs each item through `runItem` under the shared limiter and flattens the records they return. */
+async function queryChunks<TItem, TRecord>(
+  limit: ConcurrencyLimiter,
+  items: TItem[],
+  runItem: (item: TItem) => Promise<TRecord[]>,
+): Promise<TRecord[]> {
+  // Results are keyed/grouped downstream, so ordering across items does not matter
+  const results = await Promise.all(items.map((item) => limit(() => runItem(item))));
+  return results.flat();
+}
+
 // This could be eligible to pull into generic method for expanded use
 async function queryAndCombineResults<T>(
+  limit: ConcurrencyLimiter,
   selectedOrg: SalesforceOrgUi,
   queries: string[],
-  useOffset = false,
   isTooling = false,
 ): Promise<T[]> {
-  const runQuery = (currQuery: string) =>
-    useOffset ? queryAllUsingOffset<T>(selectedOrg, currQuery, isTooling) : queryAll<T>(selectedOrg, currQuery, isTooling);
-
-  const output: T[] = [];
-  // Results are keyed/grouped downstream, so ordering does not matter - run each wave concurrently
-  for (const wave of splitArrayToMaxSize(queries, QUERY_CONCURRENCY)) {
-    const waveResults = await Promise.all(wave.map(runQuery));
-    for (const { queryResults } of waveResults) {
-      output.push(...queryResults.records);
-    }
-  }
-  return output;
+  return queryChunks(limit, queries, (currQuery) =>
+    queryAll<T>(selectedOrg, currQuery, isTooling).then(({ queryResults }) => queryResults.records),
+  );
+}
+
+/**
+ * EntityParticle neither supports queryMore nor an OFFSET beyond 2000, so each chunk of objects is paged using a
+ * DurableId cursor. That removes the depth ceiling, letting one query cover many objects instead of a handful.
+ *
+ * Each chunk holds a single concurrency slot for all of its pages, since those pages must run in sequence.
+ */
+async function queryPermissionableFields(
+  limit: ConcurrencyLimiter,
+  selectedOrg: SalesforceOrgUi,
+  sobjects: string[],
+): Promise<EntityParticlePermissionsRecord[]> {
+  return queryChunks(limit, getPermissionableFieldObjectChunks(sobjects), (chunk) =>
+    queryAllUsingCursor<EntityParticlePermissionsRecord>(
+      selectedOrg,
+      (afterDurableId) => getQueryForAllPermissionableFields(chunk, afterDurableId),
+      ({ DurableId }) => DurableId,
+      true,
+    ).then(({ queryResults }) => queryResults.records),
+  );
 }
 
 function getAllFieldsByObject(fields: EntityParticlePermissionsRecord[]): Record<string, string[]> {

libs/features/manage-permissions/src/utils/__tests__/permission-manager-permissionable-fields-query.spec.ts

@@ -0,0 +1,57 @@
+import { getPermissionableFieldObjectChunks, getQueryForAllPermissionableFields } from '../permission-manager-utils';
+
+describe('getQueryForAllPermissionableFields', () => {
+  it('should order by DurableId first so that cursor paging can resume where the prior page ended', () => {
+    const query = getQueryForAllPermissionableFields(['Account']);
+    expect(query).toContain('ORDER BY DurableId ASC, EntityDefinitionId ASC, QualifiedApiName ASC');
+  });
+
+  it('should filter to permissionable, non-component particles for the provided objects', () => {
+    const query = getQueryForAllPermissionableFields(['Account', 'Contact']);
+    expect(query).toContain("EntityDefinition.QualifiedApiName IN ('Account', 'Contact')");
+    expect(query).toContain('IsPermissionable = TRUE');
+    expect(query).toContain('IsComponent = FALSE');
+  });
+
+  it('should omit the cursor filter for the first page', () => {
+    expect(getQueryForAllPermissionableFields(['Account'])).not.toContain('DurableId >');
+    expect(getQueryForAllPermissionableFields(['Account'], null)).not.toContain('DurableId >');
+    expect(getQueryForAllPermissionableFields(['Account'], undefined)).not.toContain('DurableId >');
+  });
+
+  it('should resume after the provided cursor while keeping all other filters', () => {
+    const query = getQueryForAllPermissionableFields(['Account', 'Contact'], 'Account.AccountNumber');
+    expect(query).toContain("DurableId > 'Account.AccountNumber'");
+    expect(query).toContain("EntityDefinition.QualifiedApiName IN ('Account', 'Contact')");
+    expect(query).toContain('IsPermissionable = TRUE');
+    expect(query).toContain('IsComponent = FALSE');
+    // Every filter must be ANDed together, otherwise the cursor would widen rather than narrow results
+    expect(query).not.toContain(' OR ');
+  });
+
+  it('should select the fields the permission table depends on, including the cursor field', () => {
+    const query = getQueryForAllPermissionableFields(['Account']);
+    ['QualifiedApiName', 'Label', 'DataType', 'DurableId', 'EntityDefinition.QualifiedApiName', 'IsPermissionable'].forEach((field) => {
+      expect(query).toContain(field);
+    });
+  });
+});
+
+describe('getPermissionableFieldObjectChunks', () => {
+  it('should keep a typical selection in a single query', () => {
+    const sobjects = Array.from({ length: 100 }, (_, i) => `Object_${i}__c`);
+    expect(getPermissionableFieldObjectChunks(sobjects)).toHaveLength(1);
+  });
+
+  it('should split selections that exceed the max objects per query', () => {
+    const sobjects = Array.from({ length: 250 }, (_, i) => `Object_${i}__c`);
+    const chunks = getPermissionableFieldObjectChunks(sobjects);
+
+    expect(chunks).toHaveLength(3);
+    expect(chunks.flat()).toEqual(sobjects);
+  });
+
+  it('should handle an empty selection', () => {
+    expect(getPermissionableFieldObjectChunks([]).flat()).toEqual([]);
+  });
+});

libs/features/manage-permissions/src/utils/permission-manager-utils.ts

@@ -9,6 +9,7 @@ import {
   FieldPermissionDefinitionMap,
   FieldPermissionRecord,
   FieldPermissionRecordForSave,
+  Maybe,
   ObjectPermissionDefinitionMap,
   ObjectPermissionRecord,
   ObjectPermissionRecordForSave,
@@ -34,15 +35,16 @@ import {
   TabVisibilityPermissionRecord,
   TabVisibilityPermissionRecordForSave,
 } from '@jetstream/types';
-import { Query, WhereClause, composeQuery, getField } from '@jetstreamapp/soql-parser-js';
+import { Condition, Query, WhereClause, composeQuery, getField } from '@jetstreamapp/soql-parser-js';
 
 const MAX_OBJ_IN_QUERY = 100;
 
 /**
- * EntityParticle does not support queryMore, so we page it using OFFSET, which Salesforce caps at 2000 for this object.
- * Keep the objects really small to avoid hitting the 2000 limit
+ * EntityParticle does not support queryMore and Salesforce caps OFFSET at 2000 for this object, so it is paged
+ * with a DurableId cursor instead (see `queryAllUsingCursor`). Cursor paging has no depth ceiling, which lets many
+ * objects share one query - the prior OFFSET approach had to keep chunks tiny to stay under the 2000 limit.
  */
-const MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY = 2;
+const MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY = MAX_OBJ_IN_QUERY;
 
 export function filterPermissionsSobjects(sobject: DescribeGlobalSObjectResult | null) {
   if (!sobject) {
@@ -694,69 +696,95 @@ export function permissionsHaveError<T extends PermissionDefinitionMap>(permissi
 }
 
 /**
- * Gets query for all permissionable fields
- * EntityParticle
- * @param allSobjects
- * @returns query for all permissionable fields
+ * Splits the selected objects into groups that can each be fetched with one cursor-paged EntityParticle query.
  */
-export function getQueryForAllPermissionableFields(allSobjects: string[]): string[] {
-  const queries = splitArrayToMaxSize(allSobjects, MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY).map((sobjects) => {
-    return composeQuery({
-      fields: [
-        getField('QualifiedApiName'),
-        getField('Label'),
-        getField('DataType'),
-        getField('DurableId'),
-        getField('EntityDefinition.QualifiedApiName'),
-        getField('FieldDefinitionId'),
-        getField('NamespacePrefix'),
-        getField('IsCompound'),
-        getField('IsCreatable'),
-        getField('IsUpdatable'),
-        getField('IsPermissionable'),
-      ],
-      sObject: 'EntityParticle',
-      where: {
-        left: {
-          field: 'EntityDefinition.QualifiedApiName',
-          operator: 'IN',
-          value: sobjects,
-          literalType: 'STRING',
-        },
-        operator: 'AND',
-        right: {
-          left: {
-            field: 'IsPermissionable',
-            operator: '=',
-            value: 'TRUE',
-            literalType: 'BOOLEAN',
-          },
-          operator: 'AND',
-          right: {
-            left: {
-              field: 'IsComponent',
-              operator: '=',
-              value: 'FALSE',
-              literalType: 'BOOLEAN',
-            },
-          },
-        },
-      },
-      orderBy: [
-        {
-          // EntityDefinition.QualifiedApiName is not supported in order by
-          field: 'EntityDefinitionId',
-          order: 'ASC',
-        },
-        {
-          field: 'QualifiedApiName',
-          order: 'ASC',
-        },
-      ],
+export function getPermissionableFieldObjectChunks(allSobjects: string[]): string[][] {
+  return splitArrayToMaxSize(allSobjects, MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY);
+}
+
+/**
+ * Gets one page of the query for all permissionable fields (EntityParticle).
+ *
+ * Results are ordered by DurableId so that `afterDurableId` can resume where the prior page ended.
+ * DurableId is unique and formatted as `{EntityDefinitionId}.{FieldApiName}`, so this ordering also keeps
+ * fields grouped by object and alphabetized within each object.
+ *
+ * @param sobjects a single chunk from {@link getPermissionableFieldObjectChunks}
+ * @param afterDurableId DurableId of the last record from the prior page, if any
+ */
+export function getQueryForAllPermissionableFields(sobjects: string[], afterDurableId?: Maybe<string>): string {
+  const conditions: Condition[] = [
+    {
+      field: 'EntityDefinition.QualifiedApiName',
+      operator: 'IN',
+      value: sobjects,
+      literalType: 'STRING',
+    },
+    {
+      field: 'IsPermissionable',
+      operator: '=',
+      value: 'TRUE',
+      literalType: 'BOOLEAN',
+    },
+    {
+      field: 'IsComponent',
+      operator: '=',
+      value: 'FALSE',
+      literalType: 'BOOLEAN',
+    },
+  ];
+
+  if (afterDurableId) {
+    conditions.push({
+      field: 'DurableId',
+      operator: '>',
+      value: afterDurableId,
+      literalType: 'STRING',
     });
+  }
+
+  const query = composeQuery({
+    fields: [
+      getField('QualifiedApiName'),
+      getField('Label'),
+      getField('DataType'),
+      getField('DurableId'),
+      getField('EntityDefinition.QualifiedApiName'),
+      getField('FieldDefinitionId'),
+      getField('NamespacePrefix'),
+      getField('IsCompound'),
+      getField('IsCreatable'),
+      getField('IsUpdatable'),
+      getField('IsPermissionable'),
+    ],
+    sObject: 'EntityParticle',
+    where: joinConditionsWithAnd(conditions),
+    orderBy: [
+      {
+        field: 'DurableId',
+        order: 'ASC',
+      },
+      {
+        // EntityDefinition.QualifiedApiName is not supported in order by
+        field: 'EntityDefinitionId',
+        order: 'ASC',
+      },
+      {
+        field: 'QualifiedApiName',
+        order: 'ASC',
+      },
+    ],
   });
-  logger.log('getFieldPermissionQueries()', queries);
-  return queries;
+  logger.log('getQueryForAllPermissionableFields()', query);
+  return query;
+}
+
+/**
+ * Chains conditions together with AND, since soql-parser-js models a where clause as a nested linked list.
+ */
+function joinConditionsWithAnd(conditions: Condition[]): WhereClause {
+  const [lastCondition, ...remainingInReverse] = [...conditions].reverse();
+  return remainingInReverse.reduce<WhereClause>((right, left) => ({ left, operator: 'AND', right }), { left: lastCondition });
 }
 
 /**