fix: wait until all records are emitted

bin/dev

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+
+const oclif = require('@oclif/core');
+
+const path = require('path');
+const project = path.join(__dirname, '..', 'tsconfig.json');
+
+// In dev mode -> use ts-node and dev plugins
+process.env.NODE_ENV = 'development';
+
+require('ts-node').register({ project });
+
+// In dev mode, always show stack traces
+oclif.settings.debug = true;
+
+// Start the CLI
+oclif.run().then(oclif.flush).catch(oclif.Errors.handle);

bin/dev.cmd

@@ -0,0 +1,3 @@
+@echo off
+
+node "%~dp0\dev" %*

package.json

@@ -86,7 +86,7 @@
   "dependencies": {
     "@oclif/core": "^1.6.4",
     "@salesforce/command": "^5.1.3",
-    "@salesforce/core": "^3.19.0",
+    "@salesforce/core": "^3.21.1",
     "@salesforce/ts-types": "^1.5.20",
     "@types/fs-extra": "^9.0.13",
     "chalk": "^4.1.0",
@@ -103,6 +103,7 @@
     "@salesforce/plugin-command-reference": "^1.3.0",
     "@salesforce/prettier-config": "^0.0.2",
     "@salesforce/ts-sinon": "^1.3.15",
+    "@types/shelljs": "^0.8.10",
     "@types/chai-as-promised": "^7.1.3",
     "@types/graceful-fs": "^4.1.5",
     "@types/mkdirp": "^1.0.1",
@@ -127,6 +128,7 @@
     "oclif": "^2.6.3",
     "prettier": "^2.4.1",
     "pretty-quick": "^3.1.0",
+    "shelljs": "^0.8.3",
     "shx": "^0.3.3",
     "sinon": "10.0.0",
     "ts-node": "^10.4.0",

src/commands/force/data/soql/query.ts

@@ -9,7 +9,7 @@ import * as os from 'os';
 import { flags, FlagsConfig } from '@salesforce/command';
 import { CliUx } from '@oclif/core';
 import { Connection, Logger, Messages, SfdxConfigAggregator } from '@salesforce/core';
-import { QueryOptions, Record } from 'jsforce';
+import { QueryOptions, QueryResult, Record } from 'jsforce';
 import {
   AnyJson,
   ensureJsonArray,
@@ -35,44 +35,49 @@ const commonMessages = Messages.loadMessages('@salesforce/plugin-data', 'message
  * Will collect all records and the column metadata of the query
  */
 export class SoqlQuery {
-  public async runSoqlQuery(connection: Connection, query: string, logger: Logger): Promise<SoqlQueryResult> {
-    const config: SfdxConfigAggregator = await SfdxConfigAggregator.create();
-
-    let columns: Field[] = [];
+  public async runSoqlQuery(
+    connection: Connection,
+    query: string,
+    logger: Logger,
+    configAgg: SfdxConfigAggregator
+  ): Promise<SoqlQueryResult> {
     logger.debug('running query');
 
-    // take the limit from the config, then default 10,000
+    // take the limit from the config, then default 50,000
     const queryOpts: Partial<QueryOptions> = {
       autoFetch: true,
-      maxFetch: (config.getInfo('maxQueryLimit').value as number) || 10000,
+      maxFetch: (configAgg.getInfo('maxQueryLimit').value as number) ?? 50000,
     };
 
-    const records: Record[] = [];
-
-    const result = await connection.query(query, queryOpts).on('record', (rec) => records.push(rec));
-
-    const totalSize = getNumber(result, 'totalSize', 0);
+    const result: QueryResult<Record> = await new Promise((resolve, reject) => {
+      const records: Record[] = [];
+      const res = connection
+        .query(query)
+        .on('record', (rec) => records.push(rec))
+        .on('error', (err) => reject(err))
+        .on('end', () => {
+          resolve({
+            done: true,
+            totalSize: getNumber(res, 'totalSize', 0),
+            records,
+          });
+        })
+        .run(queryOpts);
+    });
 
-    if (records.length && totalSize > records.length) {
+    if (result.records.length && result.totalSize > result.records.length) {
       CliUx.ux.warn(
-        `The query result is missing ${totalSize - records.length} records due to a ${
+        `The query result is missing ${result.totalSize - result.records.length} records due to a ${
           queryOpts.maxFetch
-        } record limit. Increase the number of records returned by setting the config value "maxQueryLimit" or the environment variable "SFDX_MAX_QUERY_LIMIT" to ${totalSize} or greater than ${
-          queryOpts.maxFetch
-        }.`
+        } record limit. Increase the number of records returned by setting the config value "maxQueryLimit" or the environment variable "SFDX_MAX_QUERY_LIMIT" to ${
+          result.totalSize
+        } or greater than ${queryOpts.maxFetch}.`
       );
     }
 
-    logger.debug(`Query complete with ${totalSize} records returned`);
-    if (totalSize) {
-      logger.debug('fetching columns for query');
-      columns = await this.retrieveColumns(connection, query);
-    }
+    logger.debug(`Query complete with ${result.totalSize} records returned`);
 
-    result.records = records;
-    // remove nextRecordsUrl and force done to true
-    delete result.nextRecordsUrl;
-    result.done = true;
+    const columns = result.totalSize ? await this.retrieveColumns(connection, query, logger) : [];
 
     return {
       query,
@@ -91,7 +96,8 @@ export class SoqlQuery {
    * @param query
    */
 
-  public async retrieveColumns(connection: Connection, query: string): Promise<Field[]> {
+  public async retrieveColumns(connection: Connection, query: string, logger?: Logger): Promise<Field[]> {
+    logger?.debug('fetching columns for query');
     // eslint-disable-next-line no-underscore-dangle
     const columnUrl = `${connection._baseUrl()}/query?q=${encodeURIComponent(query)}&columns=true`;
     const results = toJsonMap(await connection.request<Record>(columnUrl));
@@ -214,7 +220,12 @@ export class DataSoqlQueryCommand extends DataCommand {
       if (this.flags.resultformat !== 'json') this.ux.startSpinner(messages.getMessage('queryRunningMessage'));
       const query = new SoqlQuery();
       const conn = this.getConnection();
-      const queryResult: SoqlQueryResult = await query.runSoqlQuery(conn as Connection, this.flags.query, this.logger);
+      const queryResult: SoqlQueryResult = await query.runSoqlQuery(
+        conn as Connection,
+        this.flags.query,
+        this.logger,
+        this.configAggregator
+      );
       const results = {
         ...queryResult,
       };

test/commands/force/data/soql/query/dataSoqlQuery.nut.ts

@@ -5,6 +5,8 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 import * as path from 'path';
+import * as shell from 'shelljs';
+import { isArray, AnyJson, ensureString } from '@salesforce/ts-types';
 import { expect } from 'chai';
 import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit';
 import { Dictionary, getString } from '@salesforce/ts-types';
@@ -48,25 +50,65 @@ function runQuery(query: string, options: QueryOptions = { json: true, ensureExi
 
 describe('data:soql:query command', () => {
   let testSession: TestSession;
+  let hubOrgUsername: string;
 
   before(async () => {
     testSession = await TestSession.create({
       setupCommands: [
         'sfdx force:org:create -f config/project-scratch-def.json --setdefaultusername --wait 10 --durationdays 1',
         'sfdx force:source:push',
+        'sfdx config:get defaultdevhubusername --json',
       ],
       project: { sourceDir: path.join('test', 'test-files', 'data-project') },
     });
     // Import data to the default org.
     execCmd(`force:data:tree:import --plan ${path.join('.', 'data', 'accounts-contacts-plan.json')}`, {
       ensureExitCode: 0,
     });
+
+    // get default devhub username
+    if (isArray<AnyJson>(testSession.setup)) {
+      hubOrgUsername = ensureString(
+        (testSession.setup[2] as { result: [{ key: string; value: string }] }).result.find(
+          (config) => config.key === 'defaultdevhubusername'
+        )?.value
+      );
+    }
   });
 
   after(async () => {
     await testSession?.clean();
   });
 
+  describe('data:soql:query respects maxQueryLimit config', () => {
+    it('should return 1 account record', () => {
+      // set maxQueryLimit to 1 globally
+      shell.exec('sfdx config:set maxQueryLimit=1 -g', { silent: true });
+
+      const result = runQuery('SELECT Id, Name, Phone FROM Account', { json: true }) as QueryResult;
+
+      expect(result.records.length).to.equal(1);
+      verifyRecordFields(result?.records[0], ['Id', 'Name', 'Phone', 'attributes']);
+    });
+
+    it('should return 3756 ScratchOrgInfo records', () => {
+      //
+      // set maxQueryLimit to 3756 globally
+      shell.exec('sfdx config:set maxQueryLimit=3756 -g', { silent: true });
+
+      const soqlQuery = 'SELECT Id FROM ScratchOrgInfo';
+      const queryCmd = `force:data:soql:query --query "${soqlQuery}" --json --targetusername ${hubOrgUsername}`;
+      const results = execCmd<QueryResult>(queryCmd, { ensureExitCode: 0 });
+
+      const queryResult: QueryResult = results.jsonOutput?.result ?? { done: false, records: [], totalSize: 0 };
+      expect(queryResult).to.have.property('totalSize').to.be.greaterThan(0);
+      expect(queryResult).to.have.property('done', true);
+      expect(queryResult).to.have.property('records').to.not.have.lengthOf(0);
+      expect(queryResult.records.length).to.equal(3756);
+      verifyRecordFields(queryResult?.records[0], ['Id', 'attributes']);
+    });
+  });
+
   describe('data:soql:query verify query errors', () => {
     it('should error with invalid soql', () => {
       const result = runQuery('SELECT', { ensureExitCode: 1, json: false }) as string;

test/soqlQuery.test.ts

@@ -9,7 +9,7 @@
 import * as chai from 'chai';
 import { expect } from 'chai';
 import * as chaiAsPromised from 'chai-as-promised';
-import { Logger } from '@salesforce/core';
+import { Logger, SfdxConfigAggregator } from '@salesforce/core';
 import { QueryResult } from 'jsforce';
 import sinon = require('sinon');
 import { SoqlQuery } from '../src/commands/force/data/soql/query';
@@ -30,18 +30,20 @@ describe('soqlQuery tests', () => {
   });
 
   it.skip('should handle a simple query with all records returned in single call', async () => {
+    const configAgg = await SfdxConfigAggregator.create();
     sandbox
       .stub(fakeConnection, 'request')
       .resolves({ columnMetadata: queryFieldsExemplars.simpleQuery.columnMetadata });
     querySpy = sandbox
       .stub(fakeConnection, 'query')
       .resolves(soqlQueryExemplars.simpleQuery.queryResult as unknown as QueryResult<any>);
     const soqlQuery = new SoqlQuery();
-    const results = await soqlQuery.runSoqlQuery(fakeConnection, 'SELECT id, name FROM Contact', logger);
+    const results = await soqlQuery.runSoqlQuery(fakeConnection, 'SELECT id, name FROM Contact', logger, configAgg);
     sinon.assert.calledOnce(querySpy);
     expect(results).to.be.deep.equal(soqlQueryExemplars.simpleQuery.soqlQueryResult);
   });
   it.skip('should handle a query with a subquery', async () => {
+    const configAgg = await SfdxConfigAggregator.create();
     sandbox.stub(fakeConnection, 'request').resolves({ columnMetadata: queryFieldsExemplars.subquery.columnMetadata });
     querySpy = sandbox
       .stub(fakeConnection, 'query')
@@ -50,19 +52,22 @@ describe('soqlQuery tests', () => {
     const results = await soqlQuery.runSoqlQuery(
       fakeConnection,
       'SELECT Name, ( SELECT LastName FROM Contacts ) FROM Account',
-      logger
+      logger,
+      configAgg
     );
     sinon.assert.calledOnce(querySpy);
     expect(results).to.be.deep.equal(soqlQueryExemplars.subQuery.soqlQueryResult);
   });
   it.skip('should handle empty query', async () => {
+    const configAgg = await SfdxConfigAggregator.create();
     requestSpy = sandbox.stub(fakeConnection, 'request');
     querySpy = sandbox.stub(fakeConnection, 'query').resolves(soqlQueryExemplars.emptyQuery.queryResult);
     const soqlQuery = new SoqlQuery();
     const results = await soqlQuery.runSoqlQuery(
       fakeConnection,
       "SELECT Name FROM Contact where name = 'some nonexistent name'",
-      logger
+      logger,
+      configAgg
     );
     sinon.assert.calledOnce(querySpy);
     sinon.assert.notCalled(requestSpy);

yarn.lock

@@ -1104,10 +1104,10 @@
     semver "^7.3.5"
     ts-retry-promise "^0.6.0"
 
-"@salesforce/core@^3.19.0":
-  version "3.19.1"
-  resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.19.1.tgz#bd154d2676a7ab320b4b3596e5bb61635c52a3fe"
-  integrity sha512-f2Up1N9dVFv4vEKxK97CrPu6IoYl56IynH6kv07/ZO/f4F6JTxpa27e88ZDmPiZw5eBzURhCWCYqpwk5ev0YBg==
+"@salesforce/core@^3.19.0", "@salesforce/core@^3.21.1":
+  version "3.21.1"
+  resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.21.1.tgz#3e51454d6e5f5fbf523e0372378210b8cf85f60f"
+  integrity sha512-TdOhTeXrfUhRsqqwQKQsA410uBvHsRQiD66wtC0lmUGHRBNxUYEmK6+zbMvVLyPLo6Db4GOPuZ/3DV839fIVBg==
   dependencies:
     "@salesforce/bunyan" "^2.0.0"
     "@salesforce/kit" "^1.5.41"
@@ -1353,7 +1353,7 @@
   dependencies:
     "@types/node" "*"
 
-"@types/glob@^7.1.1":
+"@types/glob@*", "@types/glob@^7.1.1":
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb"
   integrity sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==
@@ -1442,6 +1442,14 @@
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
   integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ==
 
+"@types/shelljs@^0.8.10":
+  version "0.8.11"
+  resolved "https://registry.yarnpkg.com/@types/shelljs/-/shelljs-0.8.11.tgz#17a5696c825974e96828e96e89585d685646fcb8"
+  integrity sha512-x9yaMvEh5BEaZKeVQC4vp3l+QoFj3BXcd4aYfuKSzIIyihjdVARAadYy3SMNIz0WCCdS2vB9JL/U6GQk5PaxQw==
+  dependencies:
+    "@types/glob" "*"
+    "@types/node" "*"
+
 "@types/sinon@*", "@types/sinon@10.0.11":
   version "10.0.11"
   resolved "https://registry.yarnpkg.com/@types/sinon/-/sinon-10.0.11.tgz#8245827b05d3fc57a6601bd35aee1f7ad330fc42"
@@ -6997,7 +7005,7 @@ shebang-regex@^3.0.0:
   resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
   integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
 
-shelljs@^0.8.4, shelljs@^0.8.5, shelljs@~0.8.4:
+shelljs@^0.8.3, shelljs@^0.8.4, shelljs@^0.8.5, shelljs@~0.8.4:
   version "0.8.5"
   resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c"
   integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==