feat(posts): Keep the cache warm – pull once every hour.

Per 1646ace.
randytarampi · Aug 18, 2018 · 41729f1 · 41729f1
1 parent a0d9c10
commit 41729f1
Show file tree

Hide file tree

Showing 11 changed files with 106 additions and 4 deletions.
diff --git a/packages/posts/photos/cachePhotos.js b/packages/posts/photos/cachePhotos.js
@@ -0,0 +1,26 @@
+import _ from "lodash";
+import logger from "../lib/logger";
+import {initializePhotoSources} from "./photoSources";
+
+const cachePhotos = photoSearchParams => {
+    return initializePhotoSources()
+        .then(photoSources => {
+            return Promise.all(
+                photoSources.map((photoSource) => {
+                    return photoSource.getServicePosts(photoSearchParams)
+                        .catch((error) => {
+                            logger.error(error);
+                            return [];
+                        });
+                })
+            );
+        })
+        .then(_.flatten)
+        .then(flattenedPhotos => {
+            return _.sortBy(flattenedPhotos, [
+                post => -1 * (post.dateCreated ? post.dateCreated.valueOf() : post.datePublished ? post.datePublished.valueOf() : 0)
+            ]);
+        });
+};
+
+export default cachePhotos;
diff --git a/packages/posts/serverless.yml b/packages/posts/serverless.yml
@@ -103,6 +103,20 @@ custom:
     prewarm: true
 
 functions:
+  cacheWords:
+    warmup: true
+    handler: serverless/handler/initializedCacheWords.default
+    events:
+    - schedule: rate(1 hour)
+    onError:
+      Ref: LambdaDeadLetterQueue
+  cachePhotos:
+    warmup: true
+    handler: serverless/handler/initializedCachePhotos.default
+    events:
+    - schedule: rate(1 hour)
+    onError:
+      Ref: LambdaDeadLetterQueue
   getPosts:
     warmup: true
     handler: serverless/handler/initializedGetPosts.default

diff --git a/packages/posts/serverless/handler/cachePhotos.js b/packages/posts/serverless/handler/cachePhotos.js
@@ -0,0 +1,13 @@
+import cachePhotos from "../../photos/cachePhotos";
+import configureEnvironment from "../util/configureEnvironment";
+import responseBuilder from "../util/responseBuilder";
+import returnErrorResponse from "../util/returnErrorResponse";
+
+export default (event, context, callback) => {
+    configureEnvironment()
+        .then(() => {
+            return cachePhotos()
+                .then(sortedPhotos => callback(null, responseBuilder(sortedPhotos)));
+        })
+        .catch(returnErrorResponse(callback));
+};
diff --git a/packages/posts/serverless/handler/cacheWords.js b/packages/posts/serverless/handler/cacheWords.js
@@ -0,0 +1,13 @@
+import cacheWords from "../../words/cacheWords";
+import configureEnvironment from "../util/configureEnvironment";
+import responseBuilder from "../util/responseBuilder";
+import returnErrorResponse from "../util/returnErrorResponse";
+
+export default (event, context, callback) => {
+    configureEnvironment()
+        .then(() => {
+            return cacheWords()
+                .then(sortedWords => callback(null, responseBuilder(sortedWords)));
+        })
+        .catch(returnErrorResponse(callback));
+};
diff --git a/packages/posts/serverless/handler/getPhotos.js b/packages/posts/serverless/handler/getPhotos.js
@@ -11,7 +11,7 @@ export default (event, context, callback) => {
 
     configureEnvironment()
         .then(() => {
-            const searchParams = new SearchParams({
+            const searchParams = event.queryStringParameters && new SearchParams({
                 page: parseInt(event.queryStringParameters.page, 10),
                 perPage: parseInt(event.queryStringParameters.perPage, 10)
             });

diff --git a/packages/posts/serverless/handler/initializedCachePhotos.js b/packages/posts/serverless/handler/initializedCachePhotos.js
@@ -0,0 +1,3 @@
+require("../util/configureEnvironment");
+
+module.exports = require("./cachePhotos");
diff --git a/packages/posts/serverless/handler/initializedCacheWords.js b/packages/posts/serverless/handler/initializedCacheWords.js
@@ -0,0 +1,3 @@
+require("../util/configureEnvironment");
+
+module.exports = require("./cacheWords");
diff --git a/packages/posts/test/unit/words/s3/wordSource.js b/packages/posts/test/unit/words/s3/wordSource.js
@@ -181,6 +181,7 @@ describe("S3WordSource", function () {
                     sinon.assert.calledOnce(stubServiceClient.listObjects);
                     sinon.assert.calledWith(stubServiceClient.listObjects, {
                         Bucket: process.env.S3_BUCKET_NAME,
+                        Marker: "0",
                         MaxKeys: stubParams.perPage
                     });
                 });

diff --git a/packages/posts/words/cacheWords.js b/packages/posts/words/cacheWords.js
@@ -0,0 +1,26 @@
+import _ from "lodash";
+import logger from "../lib/logger";
+import {initializeWordSources} from "./wordSources";
+
+const cacheWords = wordSearchParams => {
+    return initializeWordSources()
+        .then(wordSources => {
+            return Promise.all(
+                wordSources.map((wordSource) => {
+                    return wordSource.getServicePosts(wordSearchParams)
+                        .catch((error) => {
+                            logger.error(error);
+                            return [];
+                        });
+                })
+            );
+        })
+        .then(_.flatten)
+        .then(flattenedWords => {
+            return _.sortBy(flattenedWords, [
+                post => -1 * (post.dateCreated ? post.dateCreated.valueOf() : post.datePublished ? post.datePublished.valueOf() : 0)
+            ]);
+        });
+};
+
+export default cacheWords;
diff --git a/packages/posts/words/s3/wordSource.js b/packages/posts/words/s3/wordSource.js
@@ -2,6 +2,7 @@ import {Post} from "@randy.tarampi/js";
 import Aws from "aws-sdk";
 import jsyaml from "js-yaml";
 import WordSource from "../wordSource";
+import SearchParams from "../../photos/searchParams";
 
 class S3WordSource extends WordSource {
     constructor(dataClient, cacheClient) {
@@ -15,7 +16,9 @@ class S3WordSource extends WordSource {
         return process.env.S3_BUCKET_NAME;
     }
 
-    postsGetter(params) {
+    postsGetter(params = {}) {
+        params = params instanceof SearchParams ? params : new SearchParams(params);
+
         const options = {
             Bucket: process.env.S3_BUCKET_NAME,
             MaxKeys: params.perPage || 20

diff --git a/packages/posts/words/tumblr/wordSource.js b/packages/posts/words/tumblr/wordSource.js
@@ -20,15 +20,15 @@ class TumblrWordSource extends WordSource {
         return process.env.TUMBLR_API_KEY && process.env.TUMBLR_API_SECRET;
     }
 
-    async postsGetter(params) {
+    async postsGetter(params = {}) {
         params = params instanceof SearchParams ? params : new SearchParams(params);
         params.type = "text";
 
         return this.client.blogPosts(process.env.TUMBLR_USER_NAME, params.Tumblr)
             .then(response => response.posts.map(postJson => this.jsonToPost(postJson, response.blog)));
     }
 
-    async postGetter(id, params) {
+    async postGetter(id, params = {}) {
         params = params instanceof SearchParams ? params : new SearchParams(params);
 
         return this.client.blogPosts(process.env.TUMBLR_USER_NAME, Object.assign({id}, params.Tumblr))