scripts.js

/**
 * Includes all the scripts needed by the queue and jobs.
 */

'use strict';

const _ = require('lodash');

const scripts = {
  isJobInList(client, listKey, jobId) {
    return client.isJobInList([listKey, jobId]).then(result => {
      return result === 1;
    });
  },

  addJob(client, queue, job, opts) {
    const queueKeys = queue.keys;
    let keys = [
      queueKeys.wait,
      queueKeys.paused,
      queueKeys['meta-paused'],
      queueKeys.id,
      queueKeys.delayed,
      queueKeys.priority
    ];

    const args = [
      queueKeys[''],
      _.isUndefined(opts.customJobId) ? '' : opts.customJobId,
      job.name,
      job.data,
      job.opts,
      job.timestamp,
      job.delay,
      job.delay ? job.timestamp + job.delay : 0,
      opts.priority || 0,
      opts.lifo ? 'RPUSH' : 'LPUSH',
      queue.token
    ];
    keys = keys.concat(args);
    return client.addJob(keys);
  },

  pause(queue, pause) {
    let src = 'wait',
      dst = 'paused';
    if (!pause) {
      src = 'paused';
      dst = 'wait';
    }

    const keys = _.map(
      [src, dst, 'meta-paused', pause ? 'paused' : 'resumed'],
      name => {
        return queue.toKey(name);
      }
    );

    return queue.client.pause(keys.concat([pause ? 'paused' : 'resumed']));
  },

  moveToActive(queue, jobId) {
    const queueKeys = queue.keys;
    const keys = [queueKeys.wait, queueKeys.active, queueKeys.priority];

    keys[3] = keys[1] + '@' + queue.token;
    keys[4] = queueKeys.stalled;
    keys[5] = queueKeys.limiter;
    keys[6] = queueKeys.delayed;
    keys[7] = queueKeys.drained;

    const args = [
      queueKeys[''],
      queue.token,
      queue.settings.lockDuration,
      Date.now(),
      jobId
    ];

    if (queue.limiter) {
      args.push(
        queue.limiter.max,
        queue.limiter.duration,
        !!queue.limiter.bounceBack
      );
    }
    return queue.client.moveToActive(keys.concat(args)).then(raw2jobData);
  },

  updateProgress(job, progress) {
    const queue = job.queue;
    const keys = [job.id, 'progress'].map(name => {
      return queue.toKey(name);
    });

    const progressJson = JSON.stringify(progress);
    return queue.client
      .updateProgress(keys, [progressJson, job.id + ',' + progressJson])
      .then(() => {
        queue.emit('progress', job, progress);
      });
  },

  moveToFinishedArgs(
    job,
    val,
    propVal,
    shouldRemove,
    target,
    ignoreLock,
    notFetch
  ) {
    const queue = job.queue;
    const queueKeys = queue.keys;

    const keys = [
      queueKeys.active,
      queueKeys[target],
      queue.toKey(job.id),
      queueKeys.wait,
      queueKeys.priority,
      queueKeys.active + '@' + queue.token
    ];

    if (typeof shouldRemove === 'boolean') {
      shouldRemove = shouldRemove ? '1' : '0';
    } else if (typeof shouldRemove === 'number') {
      shouldRemove = `${shouldRemove + 1}`;
    }

    const args = [
      job.id,
      job.finishedOn,
      propVal,
      _.isUndefined(val) ? 'null' : val,
      ignoreLock ? '0' : queue.token,
      shouldRemove,
      JSON.stringify({ jobId: job.id, val: val }),
      notFetch || queue.paused || queue.closing || queue.limiter ? 0 : 1,
      queueKeys[''],
      queue.settings.lockDuration,
      queue.token
    ];

    return keys.concat(args);
  },

  moveToFinished(
    job,
    val,
    propVal,
    shouldRemove,
    target,
    ignoreLock,
    notFetch = false
  ) {
    const args = scripts.moveToFinishedArgs(
      job,
      val,
      propVal,
      shouldRemove,
      target,
      ignoreLock,
      notFetch,
      job.queue.toKey('')
    );
    return job.queue.client.moveToFinished(args).then(result => {
      if (result < 0) {
        throw scripts.finishedErrors(result, job.id, 'finished');
      } else if (result) {
        return raw2jobData(result);
      }
      return 0;
    });
  },

  finishedErrors(code, jobId, command) {
    switch (code) {
      case -1:
        return new Error('Missing key for job ' + jobId + ' ' + command);
      case -2:
        return new Error('Missing lock for job ' + jobId + ' ' + command);
    }
  },

  // TODO: add a retention argument for completed and finished jobs (in time).
  moveToCompleted(
    job,
    returnvalue,
    removeOnComplete,
    ignoreLock,
    notFetch = false
  ) {
    return scripts.moveToFinished(
      job,
      returnvalue,
      'returnvalue',
      removeOnComplete,
      'completed',
      ignoreLock,
      notFetch
    );
  },

  moveToFailedArgs(job, failedReason, removeOnFailed, ignoreLock) {
    return scripts.moveToFinishedArgs(
      job,
      failedReason,
      'failedReason',
      removeOnFailed,
      'failed',
      ignoreLock,
      true
    );
  },

  moveToFailed(job, failedReason, removeOnFailed, ignoreLock) {
    const args = scripts.moveToFailedArgs(
      job,
      failedReason,
      removeOnFailed,
      ignoreLock
    );
    return scripts.moveToFinished(args);
  },

  isFinished(job) {
    const keys = _.map(['completed', 'failed'], key => {
      return job.queue.toKey(key);
    });

    return job.queue.client.isFinished(keys.concat([job.id]));
  },

  moveToDelayedArgs(queue, jobId, timestamp, ignoreLock) {
    //
    // Bake in the job id first 12 bits into the timestamp
    // to guarantee correct execution order of delayed jobs
    // (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
    //
    // WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
    //
    timestamp = _.isUndefined(timestamp) ? 0 : timestamp;

    timestamp = +timestamp || 0;
    timestamp = timestamp < 0 ? 0 : timestamp;
    if (timestamp > 0) {
      timestamp = timestamp * 0x1000 + (jobId & 0xfff);
    }

    const keys = _.map(['active', 'delayed', jobId], name => {
      return queue.toKey(name);
    });
    return keys.concat([
      JSON.stringify(timestamp),
      jobId,
      ignoreLock ? '0' : queue.token
    ]);
  },

  moveToDelayed(queue, jobId, timestamp, ignoreLock) {
    const args = scripts.moveToDelayedArgs(queue, jobId, timestamp, ignoreLock);
    return queue.client.moveToDelayed(args).then(result => {
      switch (result) {
        case -1:
          throw new Error(
            'Missing Job ' +
              jobId +
              ' when trying to move from active to delayed'
          );
        case -2:
          throw new Error(
            'Job ' +
              jobId +
              ' was locked when trying to move from active to delayed'
          );
      }
    });
  },

  remove(queue, jobId) {
    const keys = _.map(
      [
        'active',
        'wait',
        'delayed',
        'paused',
        'completed',
        'failed',
        'priority',
        jobId,
        jobId + ':logs'
      ],
      name => {
        return queue.toKey(name);
      }
    );

    return queue.client.removeJob(keys.concat([jobId, queue.token]));
  },

  async removeWithPattern(queue, pattern) {
    const keys = _.map(
      [
        'active',
        'wait',
        'delayed',
        'paused',
        'completed',
        'failed',
        'priority'
      ],
      name => {
        return queue.toKey(name);
      }
    );

    const allRemoved = [];
    let cursor = '0',
      removed;
    do {
      [cursor, removed] = await queue.client.removeJobs(
        keys.concat([queue.toKey(''), pattern, cursor])
      );
      allRemoved.push.apply(allRemoved, removed);
    } while (cursor !== '0');

    return allRemoved;
  },

  extendLock(queue, jobId) {
    return queue.client.extendLock([
      queue.toKey(jobId) + ':lock',
      queue.keys.stalled,
      queue.token,
      queue.settings.lockDuration,
      jobId
    ]);
  },

  releaseLock(queue, jobId) {
    return queue.client.releaseLock([
      queue.toKey(jobId) + ':lock',
      queue.token
    ]);
  },

  takeLock(queue, job) {
    return queue.client.takeLock([
      job.lockKey(),
      queue.token,
      queue.settings.lockDuration
    ]);
  },

  /**
    It checks if the job in the top of the delay set should be moved back to the
    top of the  wait queue (so that it will be processed as soon as possible)
  */
  updateDelaySet(queue, delayedTimestamp) {
    const keys = [
      queue.keys.delayed,
      queue.keys.active,
      queue.keys.wait,
      queue.keys.priority,
      queue.keys.paused,
      queue.keys['meta-paused']
    ];

    const args = [queue.toKey(''), delayedTimestamp, queue.token];
    return queue.client.updateDelaySet(keys.concat(args));
  },

  promote(queue, jobId) {
    const keys = [queue.keys.delayed, queue.keys.wait, queue.keys.priority];

    const args = [queue.toKey(''), jobId, queue.token];

    return queue.client.promote(keys.concat(args));
  },

  /**
   * Looks for unlocked jobs in the active queue.
   *
   *    The job was being worked on, but the worker process died and it failed to renew the lock.
   *    We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
   *    back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
   *    (e.g. if the job handler keeps crashing), we limit the number stalled job recoveries to settings.maxStalledCount.
   */
  moveUnlockedJobsToWait(queue) {
    const keys = [
      queue.keys.stalled,
      queue.keys.wait,
      queue.keys.active,
      queue.keys.failed,
      queue.keys['stalled-check'],
      queue.keys['meta-paused'],
      queue.keys.paused
    ];
    const args = [
      queue.settings.maxStalledCount,
      queue.toKey(''),
      Date.now(),
      queue.settings.stalledInterval
    ];
    return queue.client.moveStalledJobsToWait(keys.concat(args));
  },

  cleanJobsInSet(queue, set, ts, limit) {
    return queue.client.cleanJobsInSet([
      queue.toKey(set),
      queue.toKey(''),
      ts,
      limit || 0,
      set
    ]);
  },

  retryJobArgs(job, ignoreLock) {
    const queue = job.queue;
    const jobId = job.id;

    const keys = _.map(['active', 'wait', jobId], name => {
      return queue.toKey(name);
    });

    const pushCmd = (job.opts.lifo ? 'R' : 'L') + 'PUSH';

    return keys.concat([pushCmd, jobId, ignoreLock ? '0' : job.queue.token]);
  },

  /**
   * Attempts to reprocess a job
   *
   * @param {Job} job
   * @param {Object} options
   * @param {String} options.state The expected job state. If the job is not found
   * on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
   *
   * @return {Promise<Number>} Returns a promise that evaluates to a return code:
   * 1 means the operation was a success
   * 0 means the job does not exist
   * -1 means the job is currently locked and can't be retried.
   * -2 means the job was not found in the expected set
   */
  reprocessJob(job, options) {
    const queue = job.queue;

    const keys = [
      queue.toKey(job.id),
      queue.toKey(job.id) + ':lock',
      queue.toKey(options.state),
      queue.toKey('wait')
    ];

    const args = [job.id, (job.opts.lifo ? 'R' : 'L') + 'PUSH', queue.token];

    return queue.client.reprocessJob(keys.concat(args));
  }
};

module.exports = scripts;

function array2obj(arr) {
  const obj = {};
  for (let i = 0; i < arr.length; i += 2) {
    obj[arr[i]] = arr[i + 1];
  }
  return obj;
}

function raw2jobData(raw) {
  if (raw) {
    const jobData = raw[0];
    if (jobData.length) {
      const job = array2obj(jobData);
      return [job, raw[1]];
    }
  }
  return [];
}