Skip to content
Permalink
Browse files

πŸ› Fixed 3.0 migration on SQLite with many posts

refs TryGhost#11270

- Fixed 3.0/11-update-posts-html migration which failed in scenario when more than 999 posts with posts_meta relation were present
- The issue was originally spotted here: TryGhost#11270 (comment)
- The main problem is in the `SELECT` statement which is generated for `findAll` method in Bookshelf which creates `WHERE IN(post_ids_here)` statement with all posts in the database
- Using knex directly as that's a preferred way to write migrations (does not depend on the model layer)
  • Loading branch information...
gargol committed Oct 29, 2019
1 parent d370a4e commit 0441e07a366a89496aa5762f004620b824b0f1b5
Showing with 36 additions and 8 deletions.
  1. +36 βˆ’8 core/server/data/migrations/versions/3.0/11-update-posts-html.js
@@ -1,30 +1,31 @@
const _ = require('lodash');
const Promise = require('bluebird');
const htmlToText = require('html-to-text');
const common = require('../../../../lib/common');
const models = require('../../../../models');
const converters = require('../../../../lib/mobiledoc/converters');

module.exports.config = {
transaction: true
};

module.exports.up = (options) => {
const postAllColumns = ['id', 'html', 'mobiledoc'];
const columns = ['id', 'html', 'mobiledoc', 'plaintext'];

let localOptions = _.merge({
context: {internal: true},
migrating: true
}, options);

common.logging.info('Starting re-generation of posts html.');

return models.Post.findAll(_.merge({columns: postAllColumns}, localOptions))
.then(function (posts) {
return Promise.map(posts.models, function (post) {
return localOptions
.transacting('posts')
.select(columns)
.then((posts) => {
return Promise.map(posts, function (post) {
let mobiledoc;

try {
mobiledoc = JSON.parse(post.get('mobiledoc') || null);
mobiledoc = JSON.parse(post.mobiledoc || null);

if (!mobiledoc) {
common.logging.warn(`No mobiledoc for ${post.id}. Skipping.`);
@@ -37,7 +38,34 @@ module.exports.up = (options) => {

const html = converters.mobiledocConverter.render(mobiledoc);

return models.Post.edit({html}, _.merge({id: post.id}, localOptions));
const updatedAttrs = {
html: html,
mobiledoc: JSON.stringify(mobiledoc)
};

// NOTE: block comes straight from the Post model (https://github.com/TryGhost/Ghost/blob/3.0.0/core/server/models/post.js#L416)
if (html !== post.html || !post.plaintext) {
const plaintext = htmlToText.fromString(post.html, {
wordwrap: 80,
ignoreImage: true,
hideLinkHrefIfSameAsText: true,
preserveNewlines: true,
returnDomByDefault: true,
uppercaseHeadings: false
});

// CASE: html is e.g. <p></p>
// @NOTE: Otherwise we will always update the resource to `plaintext: ''` and Bookshelf thinks that this
// value was modified.
if (plaintext || plaintext !== post.plaintext) {
updatedAttrs.plaintext = plaintext;
}
}

return localOptions
.transacting('posts')
.where('id', '=', post.id)
.update(updatedAttrs);
}, {concurrency: 100});
})
.then(() => {

0 comments on commit 0441e07

Please sign in to comment.
You can’t perform that action at this time.