diff --git a/libs/features/manage-permissions/src/usePermissionRecords.tsx b/libs/features/manage-permissions/src/usePermissionRecords.tsx index 248330175..0ffbe1cdb 100644 --- a/libs/features/manage-permissions/src/usePermissionRecords.tsx +++ b/libs/features/manage-permissions/src/usePermissionRecords.tsx @@ -1,7 +1,7 @@ import { logger } from '@jetstream/shared/client-logger'; -import { describeSObject, queryAll, queryAllUsingOffset } from '@jetstream/shared/data'; +import { describeSObject, queryAll, queryAllUsingCursor } from '@jetstream/shared/data'; import { tracker } from '@jetstream/shared/ui-utils'; -import { getErrorMessage, groupByFlat, splitArrayToMaxSize } from '@jetstream/shared/utils'; +import { ConcurrencyLimiter, createConcurrencyLimiter, getErrorMessage, groupByFlat } from '@jetstream/shared/utils'; import { EntityParticlePermissionsRecord, FieldPermissionDefinitionMap, @@ -17,6 +17,7 @@ import { import { useCallback, useEffect, useRef, useState } from 'react'; import { getFieldDefinitionKey, + getPermissionableFieldObjectChunks, getQueryForAllPermissionableFields, getQueryForFieldPermissions, getQueryObjectPermissions, @@ -68,24 +69,36 @@ export function usePermissionRecords(selectedOrg: SalesforceOrgUi, sobjects: str if (hasError) { setHasError(false); } + // Every request below shares this limiter so that a large selection cannot flood Salesforce + const limit = createConcurrencyLimiter(QUERY_CONCURRENCY); // query all data and transform into state maps const output = await Promise.all([ - describeSObject(selectedOrg, 'FieldPermissions'), - queryAndCombineResults(selectedOrg, getQueryForAllPermissionableFields(sobjects), true, true), - queryAndCombineResults(selectedOrg, getQueryObjectPermissions(sobjects, permSetIds, profilePermSetIds)), - queryAndCombineResults(selectedOrg, getQueryForFieldPermissions(sobjects, permSetIds, profilePermSetIds)), + limit(() => describeSObject(selectedOrg, 'FieldPermissions')), + queryPermissionableFields(limit, selectedOrg, sobjects), + queryAndCombineResults( + limit, + selectedOrg, + getQueryObjectPermissions(sobjects, permSetIds, profilePermSetIds), + ), + queryAndCombineResults( + limit, + selectedOrg, + getQueryForFieldPermissions(sobjects, permSetIds, profilePermSetIds), + ), queryAndCombineResults( + limit, selectedOrg, getQueryTabVisibilityPermissions(sobjects, permSetIds, profilePermSetIds), ).then((record) => record.map((item) => ({ ...item, Name: item.Name.replace('standard-', '') }))), - queryAndCombineResults(selectedOrg, getQueryTabDefinition(sobjects), false, true).then((tabs) => + queryAndCombineResults(limit, selectedOrg, getQueryTabDefinition(sobjects), true).then((tabs) => groupByFlat(tabs, 'SobjectName'), ), // System permissions are `Permissions*` columns on the PermissionSet record itself; describe to // learn which are settable in this org, then query their current values by permission set id. - describeSObject(selectedOrg, 'PermissionSet').then((describeResult) => { + limit(() => describeSObject(selectedOrg, 'PermissionSet')).then((describeResult) => { const systemPermissionFields = getSystemPermissionFieldsFromDescribe(describeResult.data.fields); return queryAndCombineResults( + limit, selectedOrg, getQuerySystemPermissions( [...permSetIds, ...profilePermSetIds], @@ -198,30 +211,53 @@ function getSystemPermissionMap( } /** - * Number of queries to run concurrently. Offset-paged queries (EntityParticle) are split into many small batches to stay - * under Salesforce's OFFSET cap, so we run them in bounded waves rather than sequentially to keep large selections fast. + * Maximum requests in flight at once for a single load. Every query group shares one limiter, since limiting each + * group on its own would still let the groups fan out in parallel and multiply into a request flood. */ const QUERY_CONCURRENCY = 5; +/** Runs each item through `runItem` under the shared limiter and flattens the records they return. */ +async function queryChunks( + limit: ConcurrencyLimiter, + items: TItem[], + runItem: (item: TItem) => Promise, +): Promise { + // Results are keyed/grouped downstream, so ordering across items does not matter + const results = await Promise.all(items.map((item) => limit(() => runItem(item)))); + return results.flat(); +} + // This could be eligible to pull into generic method for expanded use async function queryAndCombineResults( + limit: ConcurrencyLimiter, selectedOrg: SalesforceOrgUi, queries: string[], - useOffset = false, isTooling = false, ): Promise { - const runQuery = (currQuery: string) => - useOffset ? queryAllUsingOffset(selectedOrg, currQuery, isTooling) : queryAll(selectedOrg, currQuery, isTooling); - - const output: T[] = []; - // Results are keyed/grouped downstream, so ordering does not matter - run each wave concurrently - for (const wave of splitArrayToMaxSize(queries, QUERY_CONCURRENCY)) { - const waveResults = await Promise.all(wave.map(runQuery)); - for (const { queryResults } of waveResults) { - output.push(...queryResults.records); - } - } - return output; + return queryChunks(limit, queries, (currQuery) => + queryAll(selectedOrg, currQuery, isTooling).then(({ queryResults }) => queryResults.records), + ); +} + +/** + * EntityParticle neither supports queryMore nor an OFFSET beyond 2000, so each chunk of objects is paged using a + * DurableId cursor. That removes the depth ceiling, letting one query cover many objects instead of a handful. + * + * Each chunk holds a single concurrency slot for all of its pages, since those pages must run in sequence. + */ +async function queryPermissionableFields( + limit: ConcurrencyLimiter, + selectedOrg: SalesforceOrgUi, + sobjects: string[], +): Promise { + return queryChunks(limit, getPermissionableFieldObjectChunks(sobjects), (chunk) => + queryAllUsingCursor( + selectedOrg, + (afterDurableId) => getQueryForAllPermissionableFields(chunk, afterDurableId), + ({ DurableId }) => DurableId, + true, + ).then(({ queryResults }) => queryResults.records), + ); } function getAllFieldsByObject(fields: EntityParticlePermissionsRecord[]): Record { diff --git a/libs/features/manage-permissions/src/utils/__tests__/permission-manager-permissionable-fields-query.spec.ts b/libs/features/manage-permissions/src/utils/__tests__/permission-manager-permissionable-fields-query.spec.ts new file mode 100644 index 000000000..47d86c562 --- /dev/null +++ b/libs/features/manage-permissions/src/utils/__tests__/permission-manager-permissionable-fields-query.spec.ts @@ -0,0 +1,57 @@ +import { getPermissionableFieldObjectChunks, getQueryForAllPermissionableFields } from '../permission-manager-utils'; + +describe('getQueryForAllPermissionableFields', () => { + it('should order by DurableId first so that cursor paging can resume where the prior page ended', () => { + const query = getQueryForAllPermissionableFields(['Account']); + expect(query).toContain('ORDER BY DurableId ASC, EntityDefinitionId ASC, QualifiedApiName ASC'); + }); + + it('should filter to permissionable, non-component particles for the provided objects', () => { + const query = getQueryForAllPermissionableFields(['Account', 'Contact']); + expect(query).toContain("EntityDefinition.QualifiedApiName IN ('Account', 'Contact')"); + expect(query).toContain('IsPermissionable = TRUE'); + expect(query).toContain('IsComponent = FALSE'); + }); + + it('should omit the cursor filter for the first page', () => { + expect(getQueryForAllPermissionableFields(['Account'])).not.toContain('DurableId >'); + expect(getQueryForAllPermissionableFields(['Account'], null)).not.toContain('DurableId >'); + expect(getQueryForAllPermissionableFields(['Account'], undefined)).not.toContain('DurableId >'); + }); + + it('should resume after the provided cursor while keeping all other filters', () => { + const query = getQueryForAllPermissionableFields(['Account', 'Contact'], 'Account.AccountNumber'); + expect(query).toContain("DurableId > 'Account.AccountNumber'"); + expect(query).toContain("EntityDefinition.QualifiedApiName IN ('Account', 'Contact')"); + expect(query).toContain('IsPermissionable = TRUE'); + expect(query).toContain('IsComponent = FALSE'); + // Every filter must be ANDed together, otherwise the cursor would widen rather than narrow results + expect(query).not.toContain(' OR '); + }); + + it('should select the fields the permission table depends on, including the cursor field', () => { + const query = getQueryForAllPermissionableFields(['Account']); + ['QualifiedApiName', 'Label', 'DataType', 'DurableId', 'EntityDefinition.QualifiedApiName', 'IsPermissionable'].forEach((field) => { + expect(query).toContain(field); + }); + }); +}); + +describe('getPermissionableFieldObjectChunks', () => { + it('should keep a typical selection in a single query', () => { + const sobjects = Array.from({ length: 100 }, (_, i) => `Object_${i}__c`); + expect(getPermissionableFieldObjectChunks(sobjects)).toHaveLength(1); + }); + + it('should split selections that exceed the max objects per query', () => { + const sobjects = Array.from({ length: 250 }, (_, i) => `Object_${i}__c`); + const chunks = getPermissionableFieldObjectChunks(sobjects); + + expect(chunks).toHaveLength(3); + expect(chunks.flat()).toEqual(sobjects); + }); + + it('should handle an empty selection', () => { + expect(getPermissionableFieldObjectChunks([]).flat()).toEqual([]); + }); +}); diff --git a/libs/features/manage-permissions/src/utils/permission-manager-utils.ts b/libs/features/manage-permissions/src/utils/permission-manager-utils.ts index 420b10417..240c1d8f9 100644 --- a/libs/features/manage-permissions/src/utils/permission-manager-utils.ts +++ b/libs/features/manage-permissions/src/utils/permission-manager-utils.ts @@ -9,6 +9,7 @@ import { FieldPermissionDefinitionMap, FieldPermissionRecord, FieldPermissionRecordForSave, + Maybe, ObjectPermissionDefinitionMap, ObjectPermissionRecord, ObjectPermissionRecordForSave, @@ -34,15 +35,16 @@ import { TabVisibilityPermissionRecord, TabVisibilityPermissionRecordForSave, } from '@jetstream/types'; -import { Query, WhereClause, composeQuery, getField } from '@jetstreamapp/soql-parser-js'; +import { Condition, Query, WhereClause, composeQuery, getField } from '@jetstreamapp/soql-parser-js'; const MAX_OBJ_IN_QUERY = 100; /** - * EntityParticle does not support queryMore, so we page it using OFFSET, which Salesforce caps at 2000 for this object. - * Keep the objects really small to avoid hitting the 2000 limit + * EntityParticle does not support queryMore and Salesforce caps OFFSET at 2000 for this object, so it is paged + * with a DurableId cursor instead (see `queryAllUsingCursor`). Cursor paging has no depth ceiling, which lets many + * objects share one query - the prior OFFSET approach had to keep chunks tiny to stay under the 2000 limit. */ -const MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY = 2; +const MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY = MAX_OBJ_IN_QUERY; export function filterPermissionsSobjects(sobject: DescribeGlobalSObjectResult | null) { if (!sobject) { @@ -694,69 +696,95 @@ export function permissionsHaveError(permissi } /** - * Gets query for all permissionable fields - * EntityParticle - * @param allSobjects - * @returns query for all permissionable fields + * Splits the selected objects into groups that can each be fetched with one cursor-paged EntityParticle query. */ -export function getQueryForAllPermissionableFields(allSobjects: string[]): string[] { - const queries = splitArrayToMaxSize(allSobjects, MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY).map((sobjects) => { - return composeQuery({ - fields: [ - getField('QualifiedApiName'), - getField('Label'), - getField('DataType'), - getField('DurableId'), - getField('EntityDefinition.QualifiedApiName'), - getField('FieldDefinitionId'), - getField('NamespacePrefix'), - getField('IsCompound'), - getField('IsCreatable'), - getField('IsUpdatable'), - getField('IsPermissionable'), - ], - sObject: 'EntityParticle', - where: { - left: { - field: 'EntityDefinition.QualifiedApiName', - operator: 'IN', - value: sobjects, - literalType: 'STRING', - }, - operator: 'AND', - right: { - left: { - field: 'IsPermissionable', - operator: '=', - value: 'TRUE', - literalType: 'BOOLEAN', - }, - operator: 'AND', - right: { - left: { - field: 'IsComponent', - operator: '=', - value: 'FALSE', - literalType: 'BOOLEAN', - }, - }, - }, - }, - orderBy: [ - { - // EntityDefinition.QualifiedApiName is not supported in order by - field: 'EntityDefinitionId', - order: 'ASC', - }, - { - field: 'QualifiedApiName', - order: 'ASC', - }, - ], +export function getPermissionableFieldObjectChunks(allSobjects: string[]): string[][] { + return splitArrayToMaxSize(allSobjects, MAX_OBJ_IN_PERMISSIONABLE_FIELDS_QUERY); +} + +/** + * Gets one page of the query for all permissionable fields (EntityParticle). + * + * Results are ordered by DurableId so that `afterDurableId` can resume where the prior page ended. + * DurableId is unique and formatted as `{EntityDefinitionId}.{FieldApiName}`, so this ordering also keeps + * fields grouped by object and alphabetized within each object. + * + * @param sobjects a single chunk from {@link getPermissionableFieldObjectChunks} + * @param afterDurableId DurableId of the last record from the prior page, if any + */ +export function getQueryForAllPermissionableFields(sobjects: string[], afterDurableId?: Maybe): string { + const conditions: Condition[] = [ + { + field: 'EntityDefinition.QualifiedApiName', + operator: 'IN', + value: sobjects, + literalType: 'STRING', + }, + { + field: 'IsPermissionable', + operator: '=', + value: 'TRUE', + literalType: 'BOOLEAN', + }, + { + field: 'IsComponent', + operator: '=', + value: 'FALSE', + literalType: 'BOOLEAN', + }, + ]; + + if (afterDurableId) { + conditions.push({ + field: 'DurableId', + operator: '>', + value: afterDurableId, + literalType: 'STRING', }); + } + + const query = composeQuery({ + fields: [ + getField('QualifiedApiName'), + getField('Label'), + getField('DataType'), + getField('DurableId'), + getField('EntityDefinition.QualifiedApiName'), + getField('FieldDefinitionId'), + getField('NamespacePrefix'), + getField('IsCompound'), + getField('IsCreatable'), + getField('IsUpdatable'), + getField('IsPermissionable'), + ], + sObject: 'EntityParticle', + where: joinConditionsWithAnd(conditions), + orderBy: [ + { + field: 'DurableId', + order: 'ASC', + }, + { + // EntityDefinition.QualifiedApiName is not supported in order by + field: 'EntityDefinitionId', + order: 'ASC', + }, + { + field: 'QualifiedApiName', + order: 'ASC', + }, + ], }); - logger.log('getFieldPermissionQueries()', queries); - return queries; + logger.log('getQueryForAllPermissionableFields()', query); + return query; +} + +/** + * Chains conditions together with AND, since soql-parser-js models a where clause as a nested linked list. + */ +function joinConditionsWithAnd(conditions: Condition[]): WhereClause { + const [lastCondition, ...remainingInReverse] = [...conditions].reverse(); + return remainingInReverse.reduce((right, left) => ({ left, operator: 'AND', right }), { left: lastCondition }); } /** diff --git a/libs/shared/data/src/lib/client-data.ts b/libs/shared/data/src/lib/client-data.ts index 2de17a808..27b76337b 100644 --- a/libs/shared/data/src/lib/client-data.ts +++ b/libs/shared/data/src/lib/client-data.ts @@ -960,6 +960,54 @@ export async function queryAllUsingOffset( return results; } +/** + * Query all records by paging on a strictly increasing unique cursor field ("keyset pagination"). + * + * Some objects (e.g. EntityParticle) do not support queryMore and cap OFFSET at 2000, which forces + * {@link queryAllUsingOffset} callers to split work into tiny chunks to stay under that ceiling. + * Paging on a cursor has no such ceiling, so a caller can pull far more data per query. + * + * `getQuery` must return a query sorted ascending by the same field `getCursorValue` reads, and must + * not include its own LIMIT. + * + * @param getQuery builds the query, resuming after `afterCursor` when provided + * @param getCursorValue reads the cursor value from the last record of a page + */ +export async function queryAllUsingCursor( + selectedOrg: SalesforceOrgUi, + getQuery: (afterCursor: Maybe) => string, + getCursorValue: (record: T) => string, + isTooling = false, + pageSize = 2000, +): Promise> { + const results = await query(selectedOrg, `${getQuery(null)} LIMIT ${pageSize}`, isTooling); + const records = results.queryResults.records; + + // A full page means there may be more records, resume after the last record we received + let lastPageSize = records.length; + let cursor: Maybe = lastPageSize ? getCursorValue(records[lastPageSize - 1]) : null; + while (lastPageSize === pageSize) { + const { queryResults } = await query(selectedOrg, `${getQuery(cursor)} LIMIT ${pageSize}`, isTooling); + lastPageSize = queryResults.records.length; + if (!lastPageSize) { + break; + } + // A query that ignores the cursor would page forever, discard the page instead of duplicating records + const nextCursor = getCursorValue(queryResults.records[lastPageSize - 1]); + if (nextCursor === cursor) { + logger.warn('[queryAllUsingCursor] Cursor did not advance, stopping pagination'); + break; + } + cursor = nextCursor; + records.push(...queryResults.records); + } + + results.queryResults.records = records; + results.queryResults.totalSize = records.length; + results.queryResults.done = true; + return results; +} + /** * Query more using OFFSET clause with query cache * diff --git a/libs/shared/utils/src/lib/__tests__/create-concurrency-limiter.spec.ts b/libs/shared/utils/src/lib/__tests__/create-concurrency-limiter.spec.ts new file mode 100644 index 000000000..c6784f854 --- /dev/null +++ b/libs/shared/utils/src/lib/__tests__/create-concurrency-limiter.spec.ts @@ -0,0 +1,115 @@ +import { createConcurrencyLimiter } from '../utils'; + +/** Task that resolves only when told to, so tests can hold slots open deterministically. */ +function createControllableTask() { + let resolveTask!: (value: string) => void; + let rejectTask!: (reason: Error) => void; + const promise = new Promise((resolve, reject) => { + resolveTask = resolve; + rejectTask = reject; + }); + return { run: () => promise, resolve: resolveTask, reject: rejectTask }; +} + +/** Lets any already-resolved promises settle so queued tasks can start. */ +const flushMicrotasks = () => new Promise((resolve) => setTimeout(resolve, 0)); + +describe('createConcurrencyLimiter', () => { + it('should run tasks immediately while under the limit', async () => { + const limit = createConcurrencyLimiter(3); + let running = 0; + let maxRunning = 0; + + await Promise.all( + Array.from({ length: 3 }, () => + limit(async () => { + running++; + maxRunning = Math.max(maxRunning, running); + await flushMicrotasks(); + running--; + }), + ), + ); + + expect(maxRunning).toBe(3); + }); + + it('should never exceed the limit and should still run every task', async () => { + const limit = createConcurrencyLimiter(5); + let running = 0; + let maxRunning = 0; + let completed = 0; + + await Promise.all( + Array.from({ length: 26 }, () => + limit(async () => { + running++; + maxRunning = Math.max(maxRunning, running); + await flushMicrotasks(); + running--; + completed++; + }), + ), + ); + + expect(maxRunning).toBe(5); + expect(completed).toBe(26); + }); + + it('should hold queued tasks until a slot frees up', async () => { + const limit = createConcurrencyLimiter(2); + const first = createControllableTask(); + const second = createControllableTask(); + let thirdStarted = false; + + limit(first.run); + limit(second.run); + const thirdResult = limit(async () => { + thirdStarted = true; + return 'third'; + }); + + await flushMicrotasks(); + expect(thirdStarted).toBe(false); + + first.resolve('first'); + await flushMicrotasks(); + expect(thirdStarted).toBe(true); + await expect(thirdResult).resolves.toBe('third'); + + second.resolve('second'); + }); + + it('should release the slot when a task rejects so later tasks are not stranded', async () => { + const limit = createConcurrencyLimiter(1); + const failing = limit(() => Promise.reject(new Error('boom'))); + + await expect(failing).rejects.toThrow('boom'); + await expect(limit(() => Promise.resolve('ran anyway'))).resolves.toBe('ran anyway'); + }); + + it('should return each task result to its own caller', async () => { + const limit = createConcurrencyLimiter(2); + const results = await Promise.all([1, 2, 3, 4, 5].map((value) => limit(() => Promise.resolve(value * 10)))); + expect(results).toEqual([10, 20, 30, 40, 50]); + }); + + it('should treat a limit below one as a limit of one rather than deadlocking', async () => { + const limit = createConcurrencyLimiter(0); + let running = 0; + let maxRunning = 0; + + await Promise.all( + Array.from({ length: 3 }, () => + limit(async () => { + running++; + maxRunning = Math.max(maxRunning, running); + await flushMicrotasks(); + running--; + }), + ), + ); + + expect(maxRunning).toBe(1); + }); +}); diff --git a/libs/shared/utils/src/lib/utils.ts b/libs/shared/utils/src/lib/utils.ts index 2f32148d0..9c43efe75 100644 --- a/libs/shared/utils/src/lib/utils.ts +++ b/libs/shared/utils/src/lib/utils.ts @@ -237,6 +237,33 @@ export function flattenRecord(record: SalesforceRecord, fields: string[], flattO }, {}); } +export type ConcurrencyLimiter = (task: () => Promise) => Promise; + +/** + * Creates a function that runs at most `maxConcurrent` tasks at a time and queues the rest. + * + * Limiting each group of work on its own does not limit the total - a caller that fans out several groups at once + * should share a single limiter across all of them, otherwise the individual limits multiply. + */ +export function createConcurrencyLimiter(maxConcurrent: number): ConcurrencyLimiter { + const limit = Math.max(1, maxConcurrent); + const waitingForSlot: (() => void)[] = []; + let runningCount = 0; + + return async function limitConcurrency(task: () => Promise): Promise { + if (runningCount >= limit) { + await new Promise((resolve) => waitingForSlot.push(resolve)); + } + runningCount++; + try { + return await task(); + } finally { + runningCount--; + waitingForSlot.shift()?.(); + } + }; +} + export function splitArrayToMaxSize(items: T[], maxSize: number): T[][] { if (!maxSize || maxSize < 1) { throw new Error('maxSize must be greater than 0');