From ea04879032a355cb7bcf3dddb9424d33277a2a68 Mon Sep 17 00:00:00 2001 From: Tim Jones Date: Tue, 12 Sep 2023 23:31:52 -0500 Subject: [PATCH] docs and updates --- package.json | 2 +- releasenotesv10.md | 85 ++++++++++++++++++++++++------------- src/index.js | 2 +- test/backgroundErrorTest.js | 4 +- test/hooks.js | 2 +- test/migrationTest.js | 4 +- test/multiMasterTest.js | 4 +- test/opsTest.js | 6 +-- test/scheduleTest.js | 6 +-- types.d.ts | 3 +- 10 files changed, 73 insertions(+), 45 deletions(-) diff --git a/package.json b/package.json index 00f6bdb6..455463d9 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Queueing jobs in Node.js using PostgreSQL like a boss", "main": "./src/index.js", "engines": { - "node": ">=16" + "node": ">=18" }, "dependencies": { "cron-parser": "^4.0.0", diff --git a/releasenotesv10.md b/releasenotesv10.md index 45582463..6fae34fc 100644 --- a/releasenotesv10.md +++ b/releasenotesv10.md @@ -1,41 +1,68 @@ -## v10 +v10 is the largest semver major release of pg-boss in years. The API changes included below are ordered by significance. -* Postgres 12 and Node 18 required - -* Created policy queues. Each queue is partitioned into dedicated storage (via postgres declarative partitioning) +## Database changes +PostgreSQL 12 is now the minimum supported version. If you upgrade and run `start()`, the database will automatically be upgraded. However, this release requires rebuilding almost all of the job table indexes, which may require a bit of downtime depending on the size of your queues. If this is a concern, you may extract the migration script via `getMigrationPlans()` and run it against a backup to get an estimate on downtime. -* cascade configuration for send() and insert() from policy queue and then global settings in the constructor +If the standard auto-migration isn't desired, consider alternatives, such as running a new v10 schema side by side of a v9 schema until the v9 queues are drained. -* Introduce dead letter queue config - * Removes completion jobs and onComplete config - * Allows retries in dlq, since they become just like any other queue +## API changes -* Add primary key to archive - * allows replication of database for read-replica and/or HA use cases - * Existing archive table will be renamed to archive_backup and kept until the next release of pgboss +* MAJOR: **Job retries are now opt-out instead of opt-in.** The default `retryLimit` is now 2 retries. This will cause an issue for any job handlers that aren't idempotent. Consider setting `retryLimit=0` on these queues if needed. -* Allow instances to connect without trying to migrate to latest version (instances that should be able to process jobs, but not have access to schema changes or upgrades) +* MAJOR: **Policy queues.** Queues can now be optionally created using `createQueue()` with a new set of storage policies. Each policy will store jobs in dedicated partition tables (courtesy of Postgres's declarative partitioning). Additionally, these queues can store default retry and retention policies that will be auto-applied to all new jobs (see below). + + * **`standard`** (default): Standard queues are the default queue policy, which supports all existing features. This will provision a dedicated job partition for all jobs with this name. + * **`short`**: Short queues only allow 1 item to be queued (in created state), which replaces the previous `sendSingleton()` and `sendOnce()` functions. + * **`singleton`**: Singleton queues only allow 1 item to be active, which replaces the previous `fetch()` option `enforceSingletonQueueActiveLimit`. + * **`stately`**: Stately queues are a combination of `short` and `singleton`, only allowing 1 job to be queued and 1 job active. -New constructor option -```js - migrate: false -``` -* Update existing constructor options for maintenance and scheduling: -```js - supervise: true, - schedule: true -``` -* consolidate failed states: expired => failed +* MAJOR: **Dead letter queues replace completion jobs.** Failed jobs will be added to optional dead letter queues after exhausting all retries. This is preferred over completion jobs to gain retry support via `work()`. Additionally, dead letter queues only make a copy of the job if it fails, instead of filling up the job table with numerous, mostly unneeded completion jobs. + * `onComplete` option in `send()` and `insert()` has been removed + * `onComplete()`, `offComplete()`, and `fetchCompleted()` have been removed + * `deadLetter` option added to `send()` and `insert()` and `createQueue()` +* MAJOR: Dropped the following API functions in favor of policy queues + * `sendOnce()` + * `sendSingleton()` -* Add priority option to work and fetch to bypass priority sorting +* MAJOR: Postgres 12 is now the minimum required version +* MAJOR: Node 18 is now the minimum required version +* MINOR: `send()` and `insert()` cascade configuration from policy queues (if they exist) and then global settings in the constructor. Use the following table to help identify which settings are inherited and when. + + | Setting | API | Queue | Constructor | + | - | - | - | - | + | `retryLimit` | * | - [x] | - [x] | + | `retryDelay` | * | - [x] | - [x] | + | `retryBackoff` | * | - [x] | - [x] | + | `expireInSeconds` | * | - [x] | - [x] | + | `expireInMinutes` | `send()`, `createQueue()` | - [x] | - [x] | + | `expireInHours` | `send()`, `createQueue()` | - [x] | - [x] | + | `retentionSeconds` | `send()`, `createQueue()` | - [x] | - [x] | + | `retentionMinutes` | `send()`, `createQueue()` | - [x] | - [x] | + | `retentionHours` | `send()`, `createQueue()` | - [x] | - [x] | + | `retentionDays` | `send()`, `createQueue()` | - [x] | - [x] | + | `deadLetter` | * | - [x] | - [ ] | -* Add manual maintenance API for one-off upgrade API without processing queues -```js - await boss.maintain() -``` +* MINOR: Added primary key to job archive to support replication use cases such as read replicas or high availability standbys. + * Existing archive table will be renamed to archive_backup and kept until the next release of pgboss, at which event it will be removed. This is only to make sure the automatic schema migration is fast. If you no longer need the jobs in archive and it's blocking you from replication, you can run the following to drop it. + + ```sql + DROP TABLE archive_backup + ``` - ## TODO +* MINOR: Added a new constructor option, `migrate:false`, to block an instance from attempting to migrate to the latest database schema version. This is useful if the configured credentials don't have schema modification privileges or complete control of when and how migrations are run is required. + +* MINOR: `noSupervisor` and `noScheduling` were renamed to a more intuitive naming convention. + * If using `noSupervisor: true` to disable mainteance, instead use `supervise: false` + * If using `noScheduling: true` to disable scheduled cron jobs, use `schedule: false` -* Add peek API for running TOP N queries against job tables \ No newline at end of file +* MINOR: The `expired` failed state has been consolidated into `failed` for simplicity. + +* MINOR: Added `priority:false` option to `work()` and `fetch()` to opt out of priority sorting during job fetching. If a queue is very large and not using the priority feature, this may help job fetch performance. + +* MINOR: Added a manual maintenance API if desired: `maintain()`. + +* MINOR: `stop()` will now wait for the default graceful stop timeout (30s) before resolving its promise. The `stopped` event will still emit. If you want to the original behavior, set the new `wait` option to `false`. + +* MINOR: Added `id` property as an option to `send()` for pre-assigning the job id. Previously, only `insert()` supported pre-assignment. \ No newline at end of file diff --git a/src/index.js b/src/index.js index c3abd9ab..cc8dcc23 100644 --- a/src/index.js +++ b/src/index.js @@ -123,7 +123,7 @@ class PgBoss extends EventEmitter { return } - let { destroy = false, graceful = true, timeout = 30000, wait = false } = options + let { destroy = false, graceful = true, timeout = 30000, wait = true } = options timeout = Math.max(timeout, 1000) diff --git a/test/backgroundErrorTest.js b/test/backgroundErrorTest.js index 41ebbc62..f17291f0 100644 --- a/test/backgroundErrorTest.js +++ b/test/backgroundErrorTest.js @@ -109,7 +109,7 @@ describe('background processing error handling', function () { await boss.start() - await boss.stop() + await boss.stop({ wait: false }) await delay(1000) @@ -127,7 +127,7 @@ describe('background processing error handling', function () { await boss.start() try { - await boss.stop() + await boss.stop({ wait: false }) assert(false) } catch (err) { assert(true) diff --git a/test/hooks.js b/test/hooks.js index 0a67410a..e25e6462 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -26,7 +26,7 @@ async function afterEach () { const { boss } = this.currentTest if (boss) { - await boss.stop({ wait: true, timeout: 2000 }) + await boss.stop({ timeout: 2000 }) } await helper.dropSchema(config.schema) diff --git a/test/migrationTest.js b/test/migrationTest.js index 9b69a6b1..81350296 100644 --- a/test/migrationTest.js +++ b/test/migrationTest.js @@ -136,7 +136,7 @@ describe('migration', function () { } catch (error) { assert(error.message.includes('wat')) } finally { - await boss1.stop({ graceful: false }) + await boss1.stop({ graceful: false, wait: false }) } const version1 = await contractor.version() @@ -154,7 +154,7 @@ describe('migration', function () { assert.strictEqual(version2, currentSchemaVersion) - await boss2.stop({ graceful: false }) + await boss2.stop({ graceful: false, wait: false }) }) it('should not install if migrate option is false', async function () { diff --git a/test/multiMasterTest.js b/test/multiMasterTest.js index 2364bd69..c714fa71 100644 --- a/test/multiMasterTest.js +++ b/test/multiMasterTest.js @@ -21,7 +21,7 @@ describe('multi-master', function () { } catch (err) { assert(false, err.message) } finally { - await pMap(instances, i => i.stop({ graceful: false })) + await pMap(instances, i => i.stop({ graceful: false, wait: false })) } }) @@ -58,7 +58,7 @@ describe('multi-master', function () { } catch (err) { assert(false) } finally { - await pMap(instances, i => i.stop({ graceful: false })) + await pMap(instances, i => i.stop({ graceful: false, wait: false })) } }) }) diff --git a/test/opsTest.js b/test/opsTest.js index 8d56423d..20616e8e 100644 --- a/test/opsTest.js +++ b/test/opsTest.js @@ -38,19 +38,19 @@ describe('ops', function () { it('should force stop', async function () { const boss = this.test.boss = await helper.start({ ...this.test.bossConfig }) - await boss.stop({ graceful: false }) + await boss.stop({ graceful: false, wait: false }) }) it('should destroy the connection pool', async function () { const boss = this.test.boss = await helper.start({ ...this.test.bossConfig }) - await boss.stop({ destroy: true, graceful: false }) + await boss.stop({ destroy: true, graceful: false, wait: false }) assert(boss.db.pool.totalCount === 0) }) it('should destroy the connection pool gracefully', async function () { const boss = this.test.boss = await helper.start({ ...this.test.bossConfig }) - await boss.stop({ destroy: true }) + await boss.stop({ destroy: true, wait: false }) await new Promise((resolve) => { boss.on('stopped', () => resolve()) }) diff --git a/test/scheduleTest.js b/test/scheduleTest.js index 475107df..88f99832 100644 --- a/test/scheduleTest.js +++ b/test/scheduleTest.js @@ -109,7 +109,7 @@ describe('schedule', function () { await boss.schedule(queue, '* * * * *') - await boss.stop() + await boss.stop({ wait: false }) boss = await helper.start({ ...this.test.bossConfig, cronWorkerIntervalSeconds: 1, schedule: true }) @@ -119,7 +119,7 @@ describe('schedule', function () { assert(job) - await boss.stop() + await boss.stop({ wait: false }) }) it('should remove previously scheduled job', async function () { @@ -135,7 +135,7 @@ describe('schedule', function () { await boss.unschedule(queue) - await boss.stop({ graceful: false }) + await boss.stop({ graceful: false, wait: false }) const db = await helper.getDb() await db.executeSql(plans.clearStorage(this.test.bossConfig.schema)) diff --git a/types.d.ts b/types.d.ts index ac3df340..07695b47 100644 --- a/types.d.ts +++ b/types.d.ts @@ -77,6 +77,7 @@ declare namespace PgBoss { } interface JobOptions { + id?: string, priority?: number; startAfter?: number | string | Date; singletonKey?: string; @@ -93,7 +94,7 @@ declare namespace PgBoss { type InsertOptions = ConnectionOptions; - type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & CompletionOptions & ConnectionOptions; + type SendOptions = JobOptions & ExpirationOptions & RetentionOptions & RetryOptions & ConnectionOptions; type ScheduleOptions = SendOptions & { tz?: string }