-
Notifications
You must be signed in to change notification settings - Fork 48
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Enable config of multiple redis instances. * no message * no message * no message * Provide a utility fn to populate the sampleStore from db. If the sampleStore index keys are not present, populate the sampleStore from the db on server start. * no message * no message * no message * no message * no message * no message * no message * no message * Per Pallavi’s comments * no message * no message * per Shriram comment * no message * Aspects from db (not just from samples) * set/reset initial feature enablement state * no message * Store the lowercase aspect name of each sample in its subject set * Define env var PERSIST_REDIS_SAMPLE_STORE_MILLISECONDS. Define a new clock job using PERSIST_REDIS_SAMPLE_STORE_MILLISECONDS. If ENABLE_REDIS_SAMPLE_STORE env var is true, this new job will truncate the db samples table and insert all the samples from the redis sample store into the table. * no message * no message * In the test, after populating redis, delete/change a sample from postgres db, then persist and assert that we get all samples back.
- Loading branch information
1 parent
64e340a
commit e570d43
Showing
7 changed files
with
292 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /** | ||
| * Copyright (c) 2017, salesforce.com, inc. | ||
| * All rights reserved. | ||
| * Licensed under the BSD 3-Clause license. | ||
| * For full license text, see LICENSE.txt file in the repo root or | ||
| * https://opensource.org/licenses/BSD-3-Clause | ||
| */ | ||
|
|
||
| /** | ||
| * ./cache/sampleStorePersist.js | ||
| * | ||
| * Functions for saving the redis sample store to the db. | ||
| */ | ||
| 'use strict'; // eslint-disable-line strict | ||
| const featureToggles = require('feature-toggles'); | ||
| const Sample = require('../db').Sample; | ||
| const redisClient = require('./redisCache').client.sampleStore; | ||
| const samsto = require('./sampleStore'); | ||
| const constants = samsto.constants; | ||
|
|
||
| /** | ||
| * Truncate the sample table in the db and persist all the samples from redis | ||
| * into the empty table. | ||
| * | ||
| * @returns {Promise} which resolves to true upon complete if redis sample | ||
| * store feature is enabled, or false on error or if feature is disabled. | ||
| */ | ||
| function persist() { | ||
| if (!featureToggles.isFeatureEnabled(constants.featureName)) { | ||
| return Promise.resolve(false); | ||
| } | ||
|
|
||
| return Sample.destroy({ truncate: true, force: true }) | ||
| .then(() => redisClient.smembersAsync(constants.indexKey.sample)) | ||
| .then((keys) => keys.map((key) => ['hgetall', key])) | ||
| .then((cmds) => redisClient.batch(cmds).execAsync()) | ||
| .then((res) => { | ||
| const samplesToCreate = res.map((sample) => { | ||
| sample.relatedLinks = JSON.parse(sample.relatedLinks); | ||
| return sample; | ||
| }); | ||
| return Sample.bulkCreate(samplesToCreate); | ||
| }) | ||
| .then(() => console.log('persisted redis sample store to db')) | ||
| .then(() => true) | ||
| .catch((err) => { | ||
| // NO-OP | ||
| console.error(err); // eslint-disable-line no-console | ||
| Promise.resolve(false); | ||
| }); | ||
| } // persist | ||
|
|
||
| module.exports = { | ||
| persist, | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /** | ||
| * Copyright (c) 2017, salesforce.com, inc. | ||
| * All rights reserved. | ||
| * Licensed under the BSD 3-Clause license. | ||
| * For full license text, see LICENSE.txt file in the repo root or | ||
| * https://opensource.org/licenses/BSD-3-Clause | ||
| */ | ||
|
|
||
| /** | ||
| * clock/scheduledJobs/persistSampleStoreJob.js | ||
| * | ||
| * Executes the process to save the redis sample store to postgres db. If | ||
| * worker process is enabled, enqueues a job, otherwise just executes directly | ||
| * in this process. | ||
| */ | ||
| const featureToggles = require('feature-toggles'); | ||
| const sampleStore = require('../../cache/sampleStore'); | ||
| const sampleStorePersist = require('../../cache/sampleStorePersist'); | ||
| const jwr = '../../jobQueue/jobWrapper'; | ||
| const jse = '../../jobQueue/setup'; | ||
|
|
||
| /** | ||
| * Execute the call to check for sample timeouts. | ||
| * | ||
| * @returns {Promise} | ||
| */ | ||
| function execute() { | ||
| return sampleStorePersist.persist(); | ||
| } // execute | ||
|
|
||
| module.exports = { | ||
| enqueue() { | ||
| if (featureToggles.isFeatureEnabled(sampleStore.constants.featureName)) { | ||
| const jobWrapper = require(jwr); // eslint-disable-line global-require | ||
| const jobType = | ||
| require(jse).jobType; // eslint-disable-line global-require | ||
| jobWrapper.createJob(jobType.PERSIST_SAMPLE_STORE, | ||
| { reqStartTime: Date.now() }); | ||
| return Promise.resolve(true); | ||
| } | ||
|
|
||
| // If not using worker process, execute directly; | ||
| return execute(); | ||
| }, | ||
|
|
||
| execute, | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| /** | ||
| * Copyright (c) 2017, salesforce.com, inc. | ||
| * All rights reserved. | ||
| * Licensed under the BSD 3-Clause license. | ||
| * For full license text, see LICENSE.txt file in the repo root or | ||
| * https://opensource.org/licenses/BSD-3-Clause | ||
| */ | ||
|
|
||
| /** | ||
| * tests/cache/persist.js | ||
| */ | ||
| 'use strict'; // eslint-disable-line strict | ||
| const sampleStore = require('../../cache/sampleStore'); | ||
| const samstoinit = require('../../cache/sampleStoreInit'); | ||
| const samstopersist = require('../../cache/sampleStorePersist'); | ||
| const redisClient = require('../../cache/redisCache').client.sampleStore; | ||
| const featureToggles = require('feature-toggles'); | ||
| const expect = require('chai').expect; | ||
| const tu = require('../testUtils'); | ||
| const u = require('./utils'); | ||
| const Sample = tu.db.Sample; | ||
| const Aspect = tu.db.Aspect; | ||
| const Subject = tu.db.Subject; | ||
| const initialFeatureState = featureToggles | ||
| .isFeatureEnabled(sampleStore.constants.featureName); | ||
|
|
||
| describe('persist sample store back to db', () => { | ||
| let a1; | ||
| let a2; | ||
| let a3; | ||
| let s1; | ||
| let s2; | ||
| let s3; | ||
|
|
||
| before((done) => { | ||
| tu.toggleOverride(sampleStore.constants.featureName, true); | ||
| Aspect.create({ | ||
| isPublished: true, | ||
| name: `${tu.namePrefix}Aspect1`, | ||
| timeout: '30s', | ||
| valueType: 'NUMERIC', | ||
| criticalRange: [0, 1], | ||
| relatedLinks: [ | ||
| { name: 'Google', value: 'http://www.google.com' }, | ||
| { name: 'Yahoo', value: 'http://www.yahoo.com' }, | ||
| ], | ||
| }) | ||
| .then((created) => (a1 = created)) | ||
| .then(() => Aspect.create({ | ||
| isPublished: true, | ||
| name: `${tu.namePrefix}Aspect2`, | ||
| timeout: '10m', | ||
| valueType: 'BOOLEAN', | ||
| okRange: [10, 100], | ||
| })) | ||
| .then((created) => (a2 = created)) | ||
| .then(() => Aspect.create({ | ||
| isPublished: true, | ||
| name: `${tu.namePrefix}Aspect3`, | ||
| timeout: '10m', | ||
| valueType: 'BOOLEAN', | ||
| okRange: [10, 100], | ||
| })) | ||
| .then((created) => (a3 = created)) | ||
| .then(() => Subject.create({ | ||
| isPublished: true, | ||
| name: `${tu.namePrefix}Subject1`, | ||
| })) | ||
| .then((created) => (s1 = created)) | ||
| .then(() => Subject.create({ | ||
| isPublished: true, | ||
| name: `${tu.namePrefix}Subject2`, | ||
| parentId: s1.id, | ||
| })) | ||
| .then((created) => (s2 = created)) | ||
| .then(() => Subject.create({ | ||
| isPublished: true, | ||
| name: `${tu.namePrefix}Subject3`, | ||
| parentId: s1.id, | ||
| })) | ||
| .then((created) => (s3 = created)) | ||
| .then(() => Sample.create({ | ||
| subjectId: s2.id, | ||
| aspectId: a1.id, | ||
| value: '0', | ||
| relatedLinks: [ | ||
| { name: 'Salesforce', value: 'http://www.salesforce.com' }, | ||
| ] | ||
| })) | ||
| .then(() => Sample.create({ | ||
| subjectId: s2.id, | ||
| aspectId: a2.id, | ||
| value: '50', | ||
| relatedLinks: [ | ||
| { name: 'Salesforce', value: 'http://www.salesforce.com' }, | ||
| ] | ||
| })) | ||
| .then(() => Sample.create({ | ||
| subjectId: s3.id, | ||
| aspectId: a1.id, | ||
| value: '5', | ||
| relatedLinks: [ | ||
| { name: 'Salesforce', value: 'http://www.salesforce.com' }, | ||
| ] | ||
| })) | ||
| .then(() => done()) | ||
| .catch(done); | ||
| }); | ||
|
|
||
| after((done) => { | ||
| u.forceDelete(done) | ||
| .then(() => redisClient.flushallAsync()) | ||
| .then(() => tu.toggleOverride(sampleStore.constants.featureName, | ||
| initialFeatureState)) | ||
| .then(() => done()) | ||
| .catch(done); | ||
| }); | ||
|
|
||
| it('ok', (done) => { | ||
| samstoinit.eradicate() | ||
| .then(() => samstoinit.populate()) | ||
| .then(() => redisClient.scardAsync(sampleStore.constants.indexKey.sample)) | ||
| .then((res) => expect(res).to.eql(3)) | ||
| .then(() => Sample.destroy({ | ||
| where: { | ||
| name: `${tu.namePrefix}Subject1.${tu.namePrefix}Subject2` + | ||
| `|${tu.namePrefix}Aspect1`, | ||
| }, | ||
| })) | ||
| .then(() => samstopersist.persist()) | ||
| .then(() => Sample.findAll()) | ||
| .then((res) => expect(res.length).to.eql(3)) | ||
| .then(() => done()) | ||
| .catch(done); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| /** | ||
| * Copyright (c) 2017, salesforce.com, inc. | ||
| * All rights reserved. | ||
| * Licensed under the BSD 3-Clause license. | ||
| * For full license text, see LICENSE.txt file in the repo root or | ||
| * https://opensource.org/licenses/BSD-3-Clause | ||
| */ | ||
|
|
||
| /** | ||
| * tests/clock/persistSampleStoreJob.js | ||
| */ | ||
| const expect = require('chai').expect; | ||
| const j = require('../../clock/scheduledJobs/persistSampleStoreJob'); | ||
| const tu = require('../testUtils'); | ||
| const featureToggles = require('feature-toggles'); | ||
| const sampleStore = require('../../cache/sampleStore'); | ||
| const initialFeatureState = featureToggles | ||
| .isFeatureEnabled(sampleStore.constants.featureName); | ||
|
|
||
| describe('persistSampleStoreJob', () => { | ||
| before(() => tu.toggleOverride(sampleStore.constants.featureName, false)); | ||
| after(() => tu.toggleOverride(sampleStore.constants.featureName, | ||
| initialFeatureState)); | ||
| it('ok, feature not enabled', (done) => { | ||
| j.execute() | ||
| .then((res) => expect(res).to.be.false) | ||
| .then(() => done()) | ||
| .catch(done); | ||
| }); | ||
| }); | ||
|
|
||
| describe('persistSampleStoreJob', () => { | ||
| before(() => tu.toggleOverride(sampleStore.constants.featureName, true)); | ||
| after(() => tu.toggleOverride(sampleStore.constants.featureName, | ||
| initialFeatureState)); | ||
| it('ok, feature enabled', (done) => { | ||
| j.execute() | ||
| .then((res) => expect(res).to.be.true) | ||
| .then(() => done()) | ||
| .catch(done); | ||
| }); | ||
| }); |