fix: change payload.jobs.run and bin script to only run jobs from default queue by default, adds support for allQueues argument (#12799)

By default, calling `payload.jobs.run()` will incorrectly run all jobs
from all queues. The `npx payload jobs:run` bin script behaves the same
way.

The `payload-jobs/run` endpoint runs jobs from the `default` queue,
which is the correct behavior.

This PR does the following:
- Change `payload.jobs.run()` to only runs jobs from the `default` queue
by default
- Change `npx payload jobs:run` bin script to only runs jobs from the
`default` queue by default
- Add new allQueues / --all-queues arguments/queryparams/flags for the
local API, rest API and bin script to allow you to run all jobs from all
queues
- Clarify the docs

Author: AlessioGr

docs/jobs-queue/queues.mdx

@@ -82,6 +82,12 @@ await fetch('/api/payload-jobs/run?limit=100&queue=nightly', {
 
 This endpoint is automatically mounted for you and is helpful in conjunction with serverless platforms like Vercel, where you might want to use Vercel Cron to invoke a serverless function that executes your jobs.
 
+**Query Parameters:**
+
+- `limit`: The maximum number of jobs to run in this invocation (default: 10).
+- `queue`: The name of the queue to run jobs from. If not specified, jobs will be run from the `default` queue.
+- `allQueues`: If set to `true`, all jobs from all queues will be run. This will ignore the `queue` parameter.
+
 **Vercel Cron Example**
 
 If you're deploying on Vercel, you can add a `vercel.json` file in the root of your project that configures Vercel Cron to invoke the `run` endpoint on a cron schedule.
@@ -139,11 +145,15 @@ If you want to process jobs programmatically from your server-side code, you can
 **Run all jobs:**
 
 ```ts
+// Run all jobs from the `default` queue - default limit is 10
 const results = await payload.jobs.run()
 
 // You can customize the queue name and limit by passing them as arguments:
 await payload.jobs.run({ queue: 'nightly', limit: 100 })
 
+// Run all jobs from all queues:
+await payload.jobs.run({ allQueues: true })
+
 // You can provide a where clause to filter the jobs that should be run:
 await payload.jobs.run({
   where: { 'input.message': { equals: 'secret' } },
@@ -160,10 +170,22 @@ const results = await payload.jobs.runByID({
 
 ### Bin script
 
-Finally, you can process jobs via the bin script that comes with Payload out of the box.
+Finally, you can process jobs via the bin script that comes with Payload out of the box. By default, this script will run jobs from the `default` queue, with a limit of 10 jobs per invocation:
+
+```sh
+npx payload jobs:run
+```
+
+You can override the default queue and limit by passing the `--queue` and `--limit` flags:
+
+```sh
+npx payload jobs:run --queue myQueue --limit 15
+```
+
+If you want to run all jobs from all queues, you can pass the `--all-queues` flag:
 
 ```sh
-npx payload jobs:run --queue default --limit 10
+npx payload jobs:run --all-queues
 ```
 
 In addition, the bin script allows you to pass a `--cron` flag to the `jobs:run` command to run the jobs on a scheduled, cron basis:

packages/payload/src/bin/index.ts

@@ -110,10 +110,12 @@ export const bin = async () => {
     const payload = await getPayload({ config })
     const limit = args.limit ? parseInt(args.limit, 10) : undefined
     const queue = args.queue ? args.queue : undefined
+    const allQueues = !!args.allQueues
 
     if (args.cron) {
       new Cron(args.cron, async () => {
         await payload.jobs.run({
+          allQueues,
           limit,
           queue,
         })
@@ -124,6 +126,7 @@ export const bin = async () => {
       return
     } else {
       await payload.jobs.run({
+        allQueues,
         limit,
         queue,
       })

packages/payload/src/queues/localAPI.ts

@@ -98,6 +98,18 @@ export const getJobsLocalAPI = (payload: Payload) => ({
   },
 
   run: async (args?: {
+    /**
+     * If you want to run jobs from all queues, set this to true.
+     * If you set this to true, the `queue` property will be ignored.
+     *
+     * @default false
+     */
+    allQueues?: boolean
+    /**
+     * The maximum number of jobs to run in this invocation
+     *
+     * @default 10
+     */
     limit?: number
     overrideAccess?: boolean
     /**
@@ -106,6 +118,11 @@ export const getJobsLocalAPI = (payload: Payload) => ({
      * FIFO would equal `createdAt` and LIFO would equal `-createdAt`.
      */
     processingOrder?: Sort
+    /**
+     * If you want to run jobs from a specific queue, set this to the queue name.
+     *
+     * @default jobs from the `default` queue will be executed.
+     */
     queue?: string
     req?: PayloadRequest
     /**
@@ -118,6 +135,7 @@ export const getJobsLocalAPI = (payload: Payload) => ({
     const newReq: PayloadRequest = args?.req ?? (await createLocalReq({}, payload))
 
     return await runJobs({
+      allQueues: args?.allQueues,
       limit: args?.limit,
       overrideAccess: args?.overrideAccess !== false,
       processingOrder: args?.processingOrder,

packages/payload/src/queues/operations/runJobs/index.ts

@@ -18,10 +18,22 @@ import { runJob } from './runJob/index.js'
 import { runJSONJob } from './runJSONJob/index.js'
 
 export type RunJobsArgs = {
+  /**
+   * If you want to run jobs from all queues, set this to true.
+   * If you set this to true, the `queue` property will be ignored.
+   *
+   * @default false
+   */
+  allQueues?: boolean
   /**
    * ID of the job to run
    */
   id?: number | string
+  /**
+   * The maximum number of jobs to run in this invocation
+   *
+   * @default 10
+   */
   limit?: number
   overrideAccess?: boolean
   /**
@@ -32,6 +44,11 @@ export type RunJobsArgs = {
    * @default all jobs for all queues will be executed in FIFO order.
    */
   processingOrder?: Sort
+  /**
+   * If you want to run jobs from a specific queue, set this to the queue name.
+   *
+   * @default jobs from the `default` queue will be executed.
+   */
   queue?: string
   req: PayloadRequest
   /**
@@ -57,10 +74,11 @@ export type RunJobsResult = {
 export const runJobs = async (args: RunJobsArgs): Promise<RunJobsResult> => {
   const {
     id,
+    allQueues = false,
     limit = 10,
     overrideAccess,
     processingOrder,
-    queue,
+    queue = 'default',
     req,
     sequential,
     where: whereFromProps,
@@ -106,10 +124,10 @@ export const runJobs = async (args: RunJobsArgs): Promise<RunJobsResult> => {
     ],
   }
 
-  if (queue) {
+  if (allQueues !== true) {
     where.and?.push({
       queue: {
-        equals: queue,
+        equals: queue ?? 'default',
       },
     })
   }
@@ -146,7 +164,12 @@ export const runJobs = async (args: RunJobsArgs): Promise<RunJobsResult> => {
     if (typeof processingOrderConfig === 'function') {
       defaultProcessingOrder = await processingOrderConfig(args)
     } else if (typeof processingOrderConfig === 'object' && !Array.isArray(processingOrderConfig)) {
-      if (queue && processingOrderConfig.queues && processingOrderConfig.queues[queue]) {
+      if (
+        !allQueues &&
+        queue &&
+        processingOrderConfig.queues &&
+        processingOrderConfig.queues[queue]
+      ) {
         defaultProcessingOrder = processingOrderConfig.queues[queue]
       } else if (processingOrderConfig.default) {
         defaultProcessingOrder = processingOrderConfig.default

packages/payload/src/queues/restEndpointRun.ts

@@ -39,23 +39,27 @@ export const runJobsEndpoint: Endpoint = {
       )
     }
 
-    const { limit, queue } = req.query
+    const { allQueues, limit, queue } = req.query as {
+      allQueues?: boolean
+      limit?: number
+      queue?: string
+    }
 
     const runJobsArgs: RunJobsArgs = {
-      queue: 'default',
+      queue,
       req,
       // We are checking access above, so we can override it here
       overrideAccess: true,
     }
 
-    if (typeof queue === 'string') {
-      runJobsArgs.queue = queue
-    }
-
     if (typeof limit !== 'undefined') {
       runJobsArgs.limit = Number(limit)
     }
 
+    if (allQueues && !(typeof allQueues === 'string' && allQueues === 'false')) {
+      runJobsArgs.allQueues = true
+    }
+
     let noJobsRemaining = false
     let remainingJobsFromQueried = 0
     try {