-
-
Notifications
You must be signed in to change notification settings - Fork 10.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🏗 Extracted post metadata in new post_meta table (#11102)
NOTE: The post metadata table split is purely an internal optimization for v3 and doesn't require or expect any external actions including related API usage in v3 We keep running into issues adding new fields to the post table because there are too many fields making the post table "too wide". We have also hit MySQL limitations in how many bytes can be in a row (64kb) with post table. In v3, we decided to split the 8 post fields (meta, twitter and og) used for meta data into a posts_meta table as these 8 fields are all "problem" `varchar` fields and make sense logically grouped together. The API layer is unaffected by the split as input/output serializers ensure the data flow works the same way as it was in v2. Only thing to note is json export in v3 will have slightly different structure with posts meta fields as separate. - Creates new post_meta schema/table with 8 fields (2 meta_* , 3 twitter_* and 3 og_*) - Update relations between post and post_meta table - Update input/output serializers to keep existing API behavior - Avoids new entry in post_meta table for post where all meta fields are null - Keeps the current fields API param behavior - Handles migration of existing posts to new table structure - Updates importer/exporter to work seamlessly with table changes
- Loading branch information
1 parent
378ebe6
commit 8ec12d9
Showing
30 changed files
with
515 additions
and
46 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
core/server/data/migrations/versions/3.0.0/2-add-posts-meta-table.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| const common = require('../../../../lib/common'); | ||
| const commands = require('../../../schema').commands; | ||
| const table = 'posts_meta'; | ||
| const message1 = `Adding table: ${table}`; | ||
| const message2 = `Dropping table: ${table}`; | ||
|
|
||
| module.exports.up = (options) => { | ||
| const connection = options.connection; | ||
|
|
||
| return connection.schema.hasTable(table) | ||
| .then(function (exists) { | ||
| if (exists) { | ||
| common.logging.warn(message1); | ||
| return; | ||
| } | ||
|
|
||
| common.logging.info(message1); | ||
| return commands.createTable(table, connection); | ||
| }); | ||
| }; | ||
|
|
||
| module.exports.down = (options) => { | ||
| const connection = options.connection; | ||
|
|
||
| return connection.schema.hasTable(table) | ||
| .then(function (exists) { | ||
| if (!exists) { | ||
| common.logging.warn(message2); | ||
| return; | ||
| } | ||
|
|
||
| common.logging.info(message2); | ||
| return commands.deleteTable(table, connection); | ||
| }); | ||
| }; |
78 changes: 78 additions & 0 deletions
78
core/server/data/migrations/versions/3.0.0/3-populate-posts-meta-table.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| const postsMetaSchema = require('../../../schema').tables.posts_meta; | ||
| const ObjectId = require('bson-objectid'); | ||
| const _ = require('lodash'); | ||
| const models = require('../../../../models'); | ||
| const common = require('../../../../lib/common'); | ||
|
|
||
| module.exports.config = { | ||
| transaction: true | ||
| }; | ||
|
|
||
| module.exports.up = (options) => { | ||
| const localOptions = _.merge({ | ||
| context: {internal: true}, | ||
| migrating: true | ||
| }, options); | ||
| const metaAttrs = _.keys(postsMetaSchema); | ||
|
|
||
| return models.Posts | ||
| .forge() | ||
| .query((qb) => { | ||
| // We only want to add entries in new table for posts which have any metadata | ||
| qb.whereNotNull('meta_title'); | ||
| qb.orWhereNotNull('meta_description'); | ||
| qb.orWhereNotNull('twitter_title'); | ||
| qb.orWhereNotNull('twitter_description'); | ||
| qb.orWhereNotNull('twitter_image'); | ||
| qb.orWhereNotNull('og_description'); | ||
| qb.orWhereNotNull('og_title'); | ||
| qb.orWhereNotNull('og_image'); | ||
| }) | ||
| .fetch(localOptions) | ||
| .then(({models: posts}) => { | ||
| if (posts.length > 0) { | ||
| common.logging.info(`Adding ${posts.length} entries to posts_meta`); | ||
| let postsMetaEntries = _.map(posts, (post) => { | ||
| let postsMetaEntry = metaAttrs.reduce(function (obj, entry) { | ||
| return Object.assign(obj, { | ||
| [entry]: post.get(entry) | ||
| }); | ||
| }, {}); | ||
| postsMetaEntry.post_id = post.get('id'); | ||
| postsMetaEntry.id = ObjectId.generate(); | ||
| return postsMetaEntry; | ||
| }); | ||
| return localOptions.transacting('posts_meta').insert(postsMetaEntries); | ||
| } else { | ||
| common.logging.info('Skipping populating posts_meta table: found 0 posts with metadata'); | ||
| return Promise.resolve(); | ||
| } | ||
| }); | ||
| }; | ||
|
|
||
| module.exports.down = function (options) { | ||
| let localOptions = _.merge({ | ||
| context: {internal: true}, | ||
| migrating: true | ||
| }, options); | ||
| const metaAttrs = _.keys(_.omit(postsMetaSchema, ['id', 'post_id'])); | ||
|
|
||
| return models.PostsMeta | ||
| .findAll(localOptions) | ||
| .then(({models: postsMeta}) => { | ||
| if (postsMeta.length > 0) { | ||
| common.logging.info(`Adding metadata for ${postsMeta.length} posts from posts_meta table`); | ||
| return Promise.map(postsMeta, (postsMeta) => { | ||
| let data = metaAttrs.reduce(function (obj, entry) { | ||
| return Object.assign(obj, { | ||
| [entry]: postsMeta.get(entry) | ||
| }); | ||
| }, {}); | ||
| return localOptions.transacting('posts').where({id: postsMeta.get('post_id')}).update(data); | ||
| }); | ||
| } else { | ||
| common.logging.info('Skipping populating meta fields from posts_meta: found 0 entries'); | ||
| return Promise.resolve(); | ||
| } | ||
| }); | ||
| }; |
Oops, something went wrong.