chore: revamp build setup + bump/prune deps

.github/workflows/externalNut.yml

@@ -89,8 +89,8 @@ jobs:
         run: |
           cd ./jsforce
           npm install
-          ## skip webpack, we are only interested in types and node build
-          npm run clean && npm run build:types && npm run build:lib
+          ## skip webpack, we are only interested in the node build
+          npm run clean && npm run build:node:cjs
           yarn link
       - name: link jsforce into plugin
         if: ${{ steps.cache-nodemodules.outputs.cache-hit != 'true' }}

.github/workflows/manualRelease.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   docs:
     needs: release
-    if: github.ref_name === '2.0'
+    if: github.ref_name === '3.0'
     uses: ./.github/workflows/publishDocs.yml
   release:
     runs-on: ubuntu-latest
@@ -33,7 +33,7 @@ jobs:
           output-file: false
           pre-commit: ./scripts/build.js
           pre-release: true
-          pre-release-identifier: 'beta'
+          pre-release-identifier: 'next'
           # always do the release, even if there are no semantic commits
           skip-on-empty: false
           tag-prefix: ''

.github/workflows/onRelease.yml

@@ -31,6 +31,6 @@ jobs:
       - name: Build
         run: npm run clean && npm run build
       - name: Publish
-        run: npm publish --tag beta
+        run: npm publish --tag next
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/publishDocs.yml

@@ -1,7 +1,6 @@
 name: publish-docs
 
-on:
-  workflow_dispatch:
+on: workflow_call
 
 jobs:
   publish:
@@ -26,14 +25,14 @@ jobs:
         run: |
           rm -rf docs
           git worktree prune
-          git fetch origin v2-docs:v2-docs
-          git worktree add docs v2-docs
+          git fetch origin v3-docs:v3-docs
+          git worktree add docs v3-docs
           npx typedoc --out docs/tmp
           cp -r docs/tmp/* docs/tmp/..
           rm -rf docs/tmp
       - name: send to git
         run: |
           cd docs
           git add .
-          git commit -m 'docs: publishing v2-docs [skip ci]' --no-verify
-          git push origin v2-docs --no-verify
+          git commit -m 'docs: publishing v3-docs [skip ci]' --no-verify
+          git push origin v3-docs --no-verify

.github/workflows/tests.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: lts/*
-
+      - uses: google/wireit@setup-github-actions-caching/v1
       - name: Install dependencies
         run: npm install
 
@@ -36,18 +36,30 @@ jobs:
         run: |
           npx zx scripts/org-setup.mjs
 
-      - name: Run browser tests
+      - name: Run browser tests with 3 attempts
+        uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
+        with:
+          max_attempts: 3
+          timeout_minutes: 30
+          retry_wait_seconds: 60
+          command: |
+            SF_USERNAME=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.username') \
+            SF_PASSWORD=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.password') \
+            SF_LOGIN_URL=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.instanceUrl') \
+            DISPLAY=:99 CHROME_BIN=$(which chrome) \
+            npm run test:browser-ci
+          new_command_on_retry: |
+            SF_USERNAME=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.username') \
+            SF_PASSWORD=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.password') \
+            SF_LOGIN_URL=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.instanceUrl') \
+            DISPLAY=:99 CHROME_BIN=$(which chrome) \
+            npm run test:browser-ci:retry
+          retry_on: error
         env:
           SF_OAUTH2_CLIENT_ID: ${{ secrets.SF_OAUTH2_CLIENT_ID }}
           SF_OAUTH2_CLIENT_SECRET: ${{ secrets.SF_OAUTH2_CLIENT_SECRET }}
           SF_OAUTH2_REDIRECT_URI: ${{ secrets.SF_OAUTH2_REDIRECT_URI }}
           SF_AJAX_PROXY_URL: ${{ vars.SF_AJAX_PROXY_URL }}
-        run: |
-          SF_USERNAME=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.username') \
-          SF_PASSWORD=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.password') \
-          SF_LOGIN_URL=$(sf org display user --target-org jsforce-test-org --json | jq -r '.result.instanceUrl') \
-          DISPLAY=:99 CHROME_BIN=$(which chrome) \
-          npm run test:browser-ci
 
   lint-and-typecheck:
     runs-on: ubuntu-latest
@@ -57,6 +69,8 @@ jobs:
         with:
           node-version: lts/*
 
+      - uses: google/wireit@setup-github-actions-caching/v1
+
       - name: Lockfile lint
         run: |
           npm install -g lockfile-lint
@@ -69,7 +83,8 @@ jobs:
       - name: Lint and Type Check
         run: |
           npm run lint
-          npm run build:types
+          # TODO: typecheck fails if there is no build, see if tests are importing from lib
+          npm run build:node:cjs
           npm run typecheck
 
   test-node-linux:
@@ -84,6 +99,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node_version }}
+      - uses: google/wireit@setup-github-actions-caching/v1
 
       - name: Install dependencies
         run: npm install
@@ -151,6 +167,7 @@ jobs:
  #      - uses: actions/setup-node@v3
  #        with:
  #          node-version: ${{ matrix.node_version }}
+ #      - uses: google/wireit@setup-github-actions-caching/v1
  #
  #      - name: Install dependencies
  #        run: npm install

.gitignore

@@ -7,6 +7,10 @@ module
 .env
 .env*
 
+.wireit
+*.tsbuildinfo
+.eslintcache
+
 
 # Dependency directories
 node_modules

DEVELOPING.md

@@ -0,0 +1,66 @@
+# Getting started
+
+Clone the project and `cd` into it:
+```
+git clone git@github.com:jsforce/jsforce.git
+cd jsforce
+```
+
+then install dependencies and build:
+```
+npm install
+npm run build
+```
+
+`npm run build` will generate all the distributions for node and browsers:
+* Node CJS build (dir: `lib`)
+* Browser ESM build (dir: `browser`)
+* Browser bundles (dir: `dist`)
+
+# Type-checking
+
+Babel doesn't perform type-checking when transpiling TS to JS, run `npm run typecheck` to validate the code and tests with the TypeScript compiler. 
+
+# Linting
+Run `npm run lint` to lint the project with ESLint.
+
+# Tests
+jsforce has mostly E2E tests, written in TS (file name ends with `test.ts`, use `.ts` for test utils) and are located under the `test` folder.
+Keep in mind these tests are executed in a browser and node env, so you have to be careful not to use browser/node-specific code or they will fail, or write node-only tests like this:
+https://github.com/jsforce/jsforce/blob/7c8e6a130a149cc7f310e6628f786f94a98c91b9/test/bulk2.test.ts#L136
+
+We run them on latest Chrome and Node (`Current`, `LTS` and `Maintenance` releases) on CI.
+
+## Scratch org setup for testing
+
+> [!NOTE] 
+> The Salesforce CLI is required for this, install instructions:
+>
+> https://developer.salesforce.com/docs/atlas.en-us.sfdx_setup.meta/sfdx_setup/sfdx_setup_install_cli.htm
+
+E2E tests need an org to run tests against, if you have a dev hub and the `sf` CLI you can create a scratch org for testing with the `org-setup` script:
+
+1) Set a default dev hub globally: `sf config set target-dev-hub <hub-username> --global`.
+2) Run `npx zx scripts/org-setup.mjs`, if it finishes successfully you'll see the following output:
+```
+Run tests using this scratch org by appending SF_LOGIN_URL=<login-url> SF_USERNAME=<username> SF_PASSWORD=<password>
+```
+
+Set these 3 env vars in you current shell session, tests will use them to establish a connection to the org.
+
+## Running tests on the browser
+Due to the Same-origin policy we can't make API requests to Salesforce without a proxy server (REST API supports CORS but jsforce also uses the SOAP API which doesn't), to start the proxy server and run the tests:
+
+1) Set the proxy URL env var: `SF_AJAX_PROXY_URL="http://localhost:3123/proxy"`. If you want to use a different port, use the `PORT` env var and update the URL.
+2) Run `npm run test:browser-ci`
+
+This will start a proxy server ([`jsforce-ajax-proxy`](https://github.com/jsforce/jsforce-ajax-proxy)), wait until it is ready and then run `karma start` to launch Chrome and run the tests.
+If you host your own proxy server or want to use a 3rd party one set it via `SF_AJAX_PROXY_URL` and run `npm run test:browser` (will only run `karma start`). 
+
+We use [`karma`](https://karma-runner.github.io/latest/index.html) as the test runner and [`jasmine`](https://github.com/jasmine/jasmine/) for testing.
+`karma` is set up to use `webpack` + `babel` ([`babel-loader`](https://github.com/babel/babel-loader)) to transpile and load TS tests in the browser.
+
+## Running tests on node
+Run all tests by running the `npm run test:node` script or pass it a file path if you want to run a specific one, e.g. `npm run test:node -- test/cache.test.ts`.
+
+[`jest`](https://jestjs.io/) is used both as the test runner and testing framework.

README.md

@@ -2,8 +2,6 @@
 
 Salesforce API Library for JavaScript applications (both on web browser and Node.js)
 
-[![CircleCI](https://circleci.com/gh/jsforce/jsforce.svg?style=svg)](https://circleci.com/gh/jsforce/jsforce)
-
 ## Overview
 
 JSforce (f.k.a. Node-Salesforce) is an isomorphic JavaScript Library utilizing Salesforce's API: It works both in the browser and with Node.js.
@@ -44,23 +42,8 @@ See [license](LICENSE) (MIT License).
 
 If you have any questions first file it on [issues](https://github.com/jsforce/jsforce/issues) before contacting authors via e-mail.
 
-## Tests
-
-In order to run tests you will need a [Salesforce Developer Org](https://developer.salesforce.com/signup)
-
-You will also need to install the JsforceTestSuite package, which can be done by running:
-
-    SF_USERNAME=myusername SF_PASSWORD=password+securityToken ./test/bin/org-setup
-
-You may need to run this more then once if you encounter timeouts or dropped connections/
-
-Finally, to run the tests simply do:
-
-    SF_USERNAME=myusername SF_PASSWORD=password+securityToken npm run test:node
-
-    SF_USERNAME=myusername SF_PASSWORD=password+securityToken npm run test:browser
-
-Alternatively, if you are using `sfdx` and have a default DevHub set up, you can create a scratch org with all the test setup ready by running `./test/bin/org-setup-sfdx.sh` (it's a bash script and probably won't help most windows users)
+## How to build/run tests:
+See [DEVELOPING.md](./DEVELOPING.md)
 
 ## Contributions
 

babel.config.js

@@ -3,59 +3,44 @@ module.exports = {
     [
       '@babel/preset-env',
       {
-        useBuiltIns: 'usage',
-        corejs: 3,
-        targets: { node: '8' },
+        targets: { node: 18 },
       },
     ],
     '@babel/preset-typescript',
   ],
   plugins: [
-    '@babel/plugin-proposal-optional-chaining',
-    '@babel/plugin-proposal-nullish-coalescing-operator',
-    '@babel/plugin-proposal-class-properties',
     [
       '@babel/plugin-transform-runtime',
       {
-        corejs: 3,
+        corejs: "3",
       },
     ],
   ],
   env: {
-    module: {
-      presets: [
-        [
-          '@babel/preset-env',
-          {
-            modules: false,
-            useBuiltIns: 'usage',
-            corejs: 3,
-            targets: { node: '8' },
-          },
-        ],
-        '@babel/preset-typescript',
-      ],
-    },
     browser: {
       presets: [
         [
           '@babel/preset-env',
           {
             modules: false,
             useBuiltIns: 'usage',
-            corejs: 3,
-            targets: { browsers: 'last 2 versions, ie 11, not dead' },
+            corejs: "3.33",
+            targets: { browsers: 'last 2 versions, not dead, > 0.2%' },
           },
         ],
         '@babel/preset-typescript',
       ],
     },
+    // This env config is used by jest for TS:
+    // https://jestjs.io/docs/getting-started#using-babel
+    //
+    // for type-check run `tsc` before running tests.
     test: {
       presets: [
         [
           '@babel/preset-env',
           {
-            targets: { node: '8' },
+            targets: { node: 18 },
           },
         ],
         '@babel/preset-typescript',

jest.setup.js

@@ -1,2 +1,3 @@
 /* global jest */
 jest.setTimeout(120000);
+jest.retryTimes(3, {logErrorsBeforeRetry: true})

karma.conf.js

@@ -1,16 +1,40 @@
 // Karma configuration
 
+const { readFileSync } = require('fs');
 const webpackConfig = require('./webpack.config.test');
 
 // Set BABEL_ENV to use proper preset config
 process.env.NODE_ENV = 'test';
 process.env.BABEL_ENV = 'test';
 
 module.exports = function (config) {
+
+  let specsToRetry;
+  if (config.retryFailed) {
+    // Loads names of failed specs and prepares them for use in a regex.
+    specsToRetry = readFileSync(
+      'test/karma/karma-failed-tests.txt',
+      'utf-8'
+    )
+      .replace(/[|\\{}()[\]^$+*?.]/g, '\\$&')
+      .replace(/-/g, '\\x2d')
+      .split('\n')
+      .join('|');
+  }
+
   config.set({
     // base path that will be used to resolve all patterns (eg. files, exclude)
     basePath: '',
 
+    plugins: [
+      'karma-webpack',
+      'karma-jasmine',
+      'karma-sourcemap-loader',
+      'karma-jasmine-html-reporter',
+      'karma-chrome-launcher',
+      require('./test/karma/karma-failed-tests-reporter.js'),
+    ],
+
     // frameworks to use
     // available frameworks: https://npmjs.org/browse/keyword/karma-adapter
     frameworks: ['jasmine'],
@@ -30,7 +54,11 @@ module.exports = function (config) {
     // test results reporter to use
     // possible values: 'dots', 'progress'
     // available reporters: https://npmjs.org/browse/keyword/karma-reporter
-    reporters: ['kjhtml', 'progress'],
+    reporters: ['kjhtml', 'progress','failed-tests'],
+
+    failedTestsReporter: {
+      outputFile: 'test/karma/karma-failed-tests.txt',
+    },
 
     // web server port
     port: 9876,
@@ -62,6 +90,10 @@ module.exports = function (config) {
 
     // Jasmine Setting
     client: {
+      args: [
+        specsToRetry && '--grep',
+        specsToRetry && `/${specsToRetry}/`,
+      ].filter(Boolean),
       jasmine: {
         random: false,
         timeoutInterval: 120000,