Automatically regenerate media sizes

[jmikrut]

This is a commonly asked topic and it just came up in Discord today, so I figured I would share a quick script that the team and I use for our Payload projects when we need something like WP's "Regenerate Thumbnails" plugin. Basically if you update your imageSizes and want to automatically re-run the image transformations for each one of your uploads, this can be super handy.

const payload = require('payload');
const path = require('path');

require('dotenv').config();

const { PAYLOAD_SECRET_KEY, MONGO_URL } = process.env;

payload.init({
  secret: PAYLOAD_SECRET_KEY,
  mongoURL: MONGO_URL,
  local: true,
});

const regenerateMediaSizes = async () => {
  try {
    const media = await payload.find({
      collection: 'media',
      depth: 0,
      limit: 300,
    });
    await Promise.all(media.docs.map(async (mediaDoc) => {
      try {
        const staticDir = path.resolve(__dirname, '../media');

        await payload.update({
          collection: 'media',
          id: mediaDoc.id,
          data: mediaDoc,
          filePath: `${staticDir}/${mediaDoc.filename}`,
          overwriteExistingFiles: true,
        });

        console.log(`Media ${mediaDoc.alt || mediaDoc.id} regenerated successfully`);
      } catch (err) {
        console.error(`Media ${mediaDoc.alt || mediaDoc.id} failed to regenerate`);
        console.error(err);
      }
    }));
  } catch (err) {
    console.log('Unable to find documents with payload');
    console.error(err);
    process.exit(0);
  }

  console.log('Media size regeneration completed!');
  process.exit(0);
};

regenerateMediaSizes();

If you save this file as regenerateMediaSizes.js, then all you need to do to run it is node ./regenerateMediaSizes.js and boom.

Related: #434

[fabioquarantini]

How to delete old unused thumbnails?

[ChristopherNowlan]

I had to move payload init function into the regenerateMediaSizes async function. Because my config file was within the src directory I had to update the command to be cross-env PAYLOAD_CONFIG_PATH=dist/payload.config.js node ./regenerateMediaSizes.js

const payload = require('payload');
const path = require('path');

require('dotenv').config();

const { PAYLOAD_SECRET_KEY, MONGO_URL } = process.env;

const regenerateMediaSizes = async () => {
  try {

  await payload.init({
        secret: PAYLOAD_SECRET_KEY,
        mongoURL: MONGODB_URL,
        local: true,
    });

    const media = await payload.find({
      collection: 'media',
      depth: 0,
      limit: 300,
    });

    await Promise.all(media.docs.map(async (mediaDoc) => {
      try {
        const staticDir = path.resolve(__dirname, '../media');

        await payload.update({
          collection: 'media',
          id: mediaDoc.id,
          data: mediaDoc,
          filePath: `${staticDir}/${mediaDoc.filename}`,
          overwriteExistingFiles: true,
        });

        console.log(`Media ${mediaDoc.alt || mediaDoc.id} regenerated successfully`);
      } catch (err) {
        console.error(`Media ${mediaDoc.alt || mediaDoc.id} failed to regenerate`);
        console.error(err);
      }
    }));
  } catch (err) {
    console.log('Unable to find documents with payload');
    console.error(err);
    process.exit(0);
  }

  console.log('Media size regeneration completed!');
  process.exit(0);
};

regenerateMediaSizes();

[andriasang]

@jmikrut : Is this script compatible with the cloud storage plugin? I removed the "filePath" section of the update call and it seems that every image in my collection is being iterated over but the sizes area is just empty objects:

sizes: { w400: {}, w768: {}, w1024: {} },

[arieltonglet]

@andriasang I've managed to make a version of this script that works with plugin-cloud-storage – building on top of @ChristopherNowlan script and with a huge help from @jmikrut on Discord.

The main point to get it working is: get the media url, fetch it, and pass it as a File in a file prop instead of filePath

Important to note as well: Check if you have the latest config file in your dist folder, to get the updated imageSizes. To make sure of that, I added a build to the script. Its more of a local development problem than a production one, but just in case:

yarn build && cross-env PAYLOAD_CONFIG_PATH=dist/payload.config.js node ./regenerateMediaSizes.js

const payload = require("payload");
const path = require("path");

require("dotenv").config();

const { PAYLOAD_SECRET, MONGODB_URI } = process.env;

const regenerateMediaSizes = async () => {
  try {
    await payload.init({
      secret: PAYLOAD_SECRET,
      mongoURL: MONGODB_URI,
      local: true,
    });

    const media = await payload.find({
      collection: "media",
      depth: 0,
      limit: 300,
    });

    await Promise.all(
      media.docs.map(async (mediaDoc) => {
        try {
          await fetch(mediaDoc.url)
            .then((response) => response.blob())
            .then(async (blob) => {
              const arrayBuffer = await blob.arrayBuffer();
              const buffer = await Buffer.from(arrayBuffer, "binary");

              const file = {
                data: buffer,
                mimetype: blob.type,
                name: mediaDoc.filename,
                size: mediaDoc.filesize,
              };

              await payload.update({
                collection: "media",
                id: mediaDoc.id,
                data: mediaDoc,
                file,
                overwriteExistingFiles: true,
                contentType: blob.type,
              });

              console.log(
                `Media ${mediaDoc.id} (${mediaDoc.filename}) regenerated successfully`
              );
            });
        } catch (err) {
          console.error(
            `Media ${mediaDoc.id} (${mediaDoc.filename}) failed to regenerate`
          );
          console.error(err);
        }
      })
    );
  } catch (err) {
    console.log("Unable to find documents with payload");
    console.error(err);
    process.exit(0);
  }

  console.log("Media size regeneration completed!");
  process.exit(0);
};

regenerateMediaSizes();

[linobino1]

When I use @jmikrut's script*, images get lost in a random way. Often it works as expected, but sometimes the original file is removed during the process.

* adapted to work with payload 1.6.27 async payload.init()

[richadr]

try with overwriteExistingFiles: false

[johnrisby]

Hi, is there a problem running this under the current Payload (I'm on 1.15.7) ? I'm getting errors about not finding the collection. I've spent a while trying various things but still not working. If it does work, can someone explain where it should be run from etc please. Thanks

[danpii]

@ChristopherNowlan thanks for your help, really appreciated! Yes, I am running locally. I tried your recommendation, but I still get the same error:

Unable to find documents with payload
/Users/danielpanagiotopulos/Documents/_Work/WebStorm/playgrounds/payload-test/payload-0/src/payload.config.ts:1
import path from 'path'
^^^^^^

SyntaxError: Cannot use import statement outside a module

[ChristopherNowlan]

No worries. Change to
/dist/payload.config.js

[ChristopherNowlan]

Because I got the same issue when I tried to run it through src/payload.config.ts

[danpii]

@ChristopherNowlan Awesome! That worked! I had always run the scipt directly, but now with a npm command and /dist/payload.config.js it's running! Was that the reason why you have 'media' at root level?

For future me, and maybe others (@johnrisby @isamardzija ) here a little how to, this is how it works for me now:
Tested with Payload v2.11.0 on localhost

• media folder and regenerateMediaSizes.js on payload root level (same where package.json lies)
• use regenerateMediaSizes.js from @ChristopherNowlan - see his comment
• run yarn build
• add this npm command to "scripts" in package.json
  • "regenerateMediaSizes": "cross-env PAYLOAD_CONFIG_PATH=dist/payload.config.js node ./regenerateMediaSizes.js"
• then: npm run regenerateMediaSizes

info: when you change 'imageSizes' in your Media collection you have to run yarn build again

[ChristopherNowlan]

@danpii you could add the regenerate script to end of build script. That way it runs every time you build.

[58bits]

Here's a version based on @arieltonglet 's excellent script - also for plugin-cloud-storage .

In this version only one image is processed at a time. For concurrency (with rate limits) and a larger number of files we'd use https://github.com/sindresorhus/p-limit

This is a Typescript file, that can be placed anywhere, but must be run via ts-node in the root of the Payload project (at the same level as the .env file)

For example... ts-node src/scripts/regenerateMediaSizes.ts

/* eslint-disable @typescript-eslint/ban-ts-comment */
// @ts-nocheck

import payload from 'payload'

// eslint-disable-next-line
require('dotenv').config()

const { PAYLOAD_SECRET, DATABASE_URI } = process.env

const regenerateMediaSizes = async (): Promise<void> => {
  await payload.init({
    secret: PAYLOAD_SECRET,
    mongoURL: DATABASE_URI,
    local: true
  })

  const media = await payload.find({
    collection: 'media',
    depth: 0,
    limit: 100
  })

  if (media != null && media.totalDocs > 0) {
    payload.logger.info(`Found ${media.totalDocs} media files.`)
    for (let index = 0; index < media.docs.length; index++) {
      const mediaDoc = media.docs[index]
      try {
        await fetch(mediaDoc.url)
          .then(async (response) => await response.blob())
          .then(async (blob) => {
            const arrayBuffer = await blob.arrayBuffer()
            const buffer = Buffer.from(arrayBuffer, 'binary')

            const file = {
              data: buffer,
              mimetype: blob.type,
              name: mediaDoc.filename,
              size: mediaDoc.filesize
            }

            await payload.update({
              collection: 'media',
              id: mediaDoc.id,
              data: mediaDoc,
              file,
              overwriteExistingFiles: true,
              contentType: blob.type
            })

            payload.logger.info(
              `Media ${mediaDoc.id} (${mediaDoc.filename}) regenerated successfully`
            )
          })
      } catch (err) {
        payload.logger.error(`Media ${mediaDoc.id} (${mediaDoc.filename}) failed to regenerate`)
        console.error(err)
      }
    }
  } else {
    payload.logger.info('No media files found.')
  }

  payload.logger.info('Done!')
  process.exit(0)
}

void regenerateMediaSizes()

[tsemachh]

For Payload 3 we used:

import { getPayload } from 'payload'
import * as dotenv from 'dotenv'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { importConfig } from 'payload/node'
dotenv.config() // Load environment variables from .env

const __dirname: string = path.dirname(fileURLToPath(import.meta.url))
const payloadConfigPath: string = path.join(__dirname, '../payload.config.ts')
const awaitedConfig = await importConfig(payloadConfigPath)
const payload = await getPayload({ config: awaitedConfig })

// eslint-disable-next-line
import 'dotenv/config'

const regenerateMediaSizes = async (): Promise<void> => {
  const media = await payload.find({
    collection: 'media',
    depth: 0,
    limit: 500,
  })
  if (media != null && media.totalDocs > 0) {
    payload.logger.info(`Found ${media.totalDocs} media files.`)
    for (let index = 0; index < media.docs.length; index++) {
      const mediaDoc = media.docs[index]
      const req = {
        headers: new Map(), // Or use any object literal
      } as PayloadRequestWithData
      req.headers.set('origin', process.env.PAYLOAD_URL)
      try {
        await payload.update({
          collection: 'media',
          id: mediaDoc.id,
          data: mediaDoc,
          overwriteExistingFiles: true,
          req,
        })

        payload.logger.info(
          `Media ${mediaDoc.id} (${mediaDoc.filename}) successfully and new copy created.`,
        )
      } catch (err) {
        payload.logger.error(`Media ${mediaDoc.id} (${mediaDoc.filename}) failed to regenerate`)
        console.error(err)
      }
    }
  } else {
    payload.logger.info('No media files found.')
  }

  payload.logger.info('Done!')
  process.exit(0)
}

void regenerateMediaSizes()