From 891533ca5c69c4ba2e179720bf920eb4bfd38a73 Mon Sep 17 00:00:00 2001 From: Julien Blin Date: Thu, 14 Jun 2018 18:13:49 -0400 Subject: [PATCH] Added s# cache implementation. --- src/services/aws/cache.ts | 146 ++++++++++++++++++ src/services/aws/key-value-repository.ts | 17 +- src/services/aws/s3-client.ts | 14 ++ src/services/index.ts | 1 + test/unit/services/aws/cache-test.ts | 115 ++++++++++++++ .../services/aws/key-value-repository-test.ts | 3 +- 6 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 src/services/aws/cache.ts create mode 100644 src/services/aws/s3-client.ts create mode 100644 test/unit/services/aws/cache-test.ts diff --git a/src/services/aws/cache.ts b/src/services/aws/cache.ts new file mode 100644 index 0000000..355eed5 --- /dev/null +++ b/src/services/aws/cache.ts @@ -0,0 +1,146 @@ +import { S3 } from "aws-sdk"; +import * as HttpStatusCodes from "http-status-codes"; +import { randomStr } from "../../core/utils"; +import { Cache } from "../cache"; +import { checkHealth, CheckHealth } from "../health-check"; +import { S3Client } from "./s3-client"; + +/** Options fro S3Cache */ +export interface S3CacheOptions { + /** S3 bucket name. */ + bucket: string | Promise; + + /** The content type for the files. */ + contentType?: string; + + /** Base path to use in the bucket. */ + path?: string; + + /** S3 client to use. */ + s3?: S3Client; + + /** The server side encryption to use */ + serverSideEncryption?: string; + + /** Custom deserializer. */ + deserialize?(text: string): CacheItem; + + /** Custom serializer. */ + serialize?(value: CacheItem): string; +} + +/** + * Cache that uses S3 as a backing store. + */ +export class S3Cache implements Cache, CheckHealth { + + /** Options resolved with default values. */ + private readonly options: Required; + + public constructor( + { + bucket, + contentType = "application/json", + path = "", + s3 = new S3({ maxRetries: 3 }), + serverSideEncryption = "", + deserialize = (text: string) => JSON.parse(text), + serialize = (value: T) => JSON.stringify(value), + }: S3CacheOptions) { + this.options = { + bucket, + contentType, + deserialize, + path: path.endsWith("/") ? path.slice(0, -1) : path, + s3, + serialize, + serverSideEncryption, + }; + } + + public async checkHealth() { + return checkHealth( + "S3CacheOptions", + `${await this.options.bucket}/${this.options.path}`, + async () => { + const testKey = randomStr(); + await this.set(testKey, { testKey }); + await this.delete(testKey); + }); + } + + /** Delete the value associated with the key */ + public async delete(key: string): Promise { + await this.options.s3.deleteObject({ + Bucket: await this.options.bucket, + Key: this.getKey(key), + }).promise(); + } + + /** Get the value associated with the key */ + public async get(key: string): Promise { + try { + const response = await this.options.s3.getObject({ + Bucket: await this.options.bucket, + Key: this.getKey(key), + }).promise(); + if (!response.Body) { + return undefined; + } + + const cacheItem = this.options.deserialize(response.Body.toString()); + if (cacheItem.expiresAt && cacheItem.expiresAt < (new Date().getTime() / 1000)) { + await this.delete(key); + return undefined; + } + + return cacheItem.item; + + } catch (error) { + if (error.statusCode === HttpStatusCodes.NOT_FOUND) { + return undefined; + } + + throw error; + } + } + + public async getOrFetch(key: string, fetch: () => Promise, useCache = true, ttl?: number): Promise { + if (ttl === 0) { + return fetch(); + } + + if (useCache) { + const cachedValue = await this.get(key); + if (cachedValue !== undefined) { return cachedValue; } + } + + const value = await fetch(); + await this.set(key, value, ttl); + + return value; + } + + /** Set the value associated with the key */ + public async set(key: string, value: T, ttl?: number): Promise { + const item: CacheItem = { + expiresAt: ttl ? (new Date().getTime() / 1000) + ttl : undefined, + item: value, + }; + await this.options.s3.putObject({ + Body: this.options.serialize(item), + Bucket: await this.options.bucket, + ContentType: this.options.contentType, + Key: this.getKey(key), + ServerSideEncryption: this.options.serverSideEncryption ? this.options.serverSideEncryption : undefined, + }).promise(); + } + + /** Computes the path + key. */ + private getKey(key: string) { return `${this.options.path}/${key}`; } +} + +export interface CacheItem { + expiresAt?: number; + item: T; +} diff --git a/src/services/aws/key-value-repository.ts b/src/services/aws/key-value-repository.ts index a556409..c90a766 100644 --- a/src/services/aws/key-value-repository.ts +++ b/src/services/aws/key-value-repository.ts @@ -1,20 +1,9 @@ -import { AWSError, S3 } from "aws-sdk"; -import { PromiseResult } from "aws-sdk/lib/request"; +import { S3 } from "aws-sdk"; import * as HttpStatusCodes from "http-status-codes"; import { randomStr } from "../../core/utils"; import { checkHealth, CheckHealth } from "../health-check"; import { KeyValueRepository } from "../key-value-repository"; - -export interface S3Client { - deleteObject(params: S3.Types.DeleteObjectRequest) - : { promise(): Promise> }; - - getObject(params: S3.Types.GetObjectRequest) - : { promise(): Promise> }; - - putObject(params: S3.Types.PutObjectRequest) - : { promise(): Promise> }; -} +import { S3Client } from "./s3-client"; export interface S3KeyValueRepositoryOptions { /** S3 bucket name. */ @@ -29,7 +18,7 @@ export interface S3KeyValueRepositoryOptions { /** S3 client to use. */ s3?: S3Client; - /** The serverside encryption to use */ + /** The server side encryption to use */ serverSideEncryption?: string; /** Custom deserializer. */ diff --git a/src/services/aws/s3-client.ts b/src/services/aws/s3-client.ts new file mode 100644 index 0000000..964dbf7 --- /dev/null +++ b/src/services/aws/s3-client.ts @@ -0,0 +1,14 @@ +import { AWSError, S3 } from "aws-sdk"; +import { PromiseResult } from "aws-sdk/lib/request"; + +export interface S3Client { + deleteObject(params: S3.Types.DeleteObjectRequest): { + promise(): Promise>; + }; + getObject(params: S3.Types.GetObjectRequest): { + promise(): Promise>; + }; + putObject(params: S3.Types.PutObjectRequest): { + promise(): Promise>; + }; +} diff --git a/src/services/index.ts b/src/services/index.ts index f70d33d..5a8ef6f 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,3 +1,4 @@ +export * from "./aws/cache"; export * from "./aws/config"; export * from "./aws/function-proxy"; export * from "./aws/key-value-repository"; diff --git a/test/unit/services/aws/cache-test.ts b/test/unit/services/aws/cache-test.ts new file mode 100644 index 0000000..022c664 --- /dev/null +++ b/test/unit/services/aws/cache-test.ts @@ -0,0 +1,115 @@ +import { AWSError, Response, S3 } from "aws-sdk"; +import { expect } from "chai"; +import * as HttpStatusCodes from "http-status-codes"; +import { randomStr } from "../../../../src/core/utils"; +import { S3Cache } from "../../../../src/services/aws/cache"; +import { S3Client } from "../../../../src/services/aws/s3-client"; +import { InMemoryCache } from "../../../../src/services/cache"; + +class S3ClientStub implements S3Client { + + private readonly innerCache = new InMemoryCache({ defaultTtl: 3600 }); + + public deleteObject(params: S3.Types.DeleteObjectRequest) { + return { + promise: async () => { + await this.innerCache.delete(params.Key); + + return { + $response: {} as Response, + }; + }, + }; + } + + public getObject(params: S3.Types.GetObjectRequest) { + return { + promise: async () => { + const object = await this.innerCache.get(params.Key); + if (!object) { + throw { + statusCode: HttpStatusCodes.NOT_FOUND, + } as AWSError; + } + return ({ + $response: {} as Response, + Body: object, + }); + }, + }; + } + + public putObject(params: S3.Types.PutObjectRequest) { + return { + promise: async () => { + await this.innerCache.set(params.Key, params.Body); + + return { + $response: {} as Response, + }; + }, + }; + } +} + +describe("S3Cache", () => { + + it("should fetch when ttl is 0", async () => { + const cache = new S3Cache({ bucket: randomStr(), s3: new S3ClientStub() }); + let fetched = false; + const result = await cache.getOrFetch( + "key", + async () => { + fetched = true; + + return 0; + }, + true, + 0); + + expect(result).equal(0); + expect(fetched).be.true; + }); + + it("should fetch and cache", async () => { + const cache = new S3Cache({ bucket: randomStr(), s3: new S3ClientStub() }); + let fetched = 0; + let result = await cache.getOrFetch("key", async () => { + ++fetched; + + return 0; }); + + expect(result).equal(0); + expect(fetched).equal(1); + result = await cache.getOrFetch("key", async () => { + ++fetched; + + return 0; }); + + expect(result).equal(0); + expect(fetched).equal(1); + }); + + it("should fetch when useCache is false.", async () => { + const cache = new S3Cache({ bucket: randomStr(), s3: new S3ClientStub() }); + let fetched = 0; + let result = await cache.getOrFetch("key", async () => { + ++fetched; + + return 0; }); + + expect(result).equal(0); + expect(fetched).equal(1); + result = await cache.getOrFetch( + "key", + async () => { + ++fetched; + + return 0; }, + false); + + expect(result).equal(0); + expect(fetched).equal(2); + }); + +}); diff --git a/test/unit/services/aws/key-value-repository-test.ts b/test/unit/services/aws/key-value-repository-test.ts index 89f4ba2..12f224f 100644 --- a/test/unit/services/aws/key-value-repository-test.ts +++ b/test/unit/services/aws/key-value-repository-test.ts @@ -2,7 +2,8 @@ import { AWSError, Response, S3 } from "aws-sdk"; import { expect } from "chai"; import * as HttpStatusCodes from "http-status-codes"; import { randomStr } from "../../../../src/core/utils"; -import { S3Client, S3KeyValueRepository } from "../../../../src/services/aws/key-value-repository"; +import { S3KeyValueRepository } from "../../../../src/services/aws/key-value-repository"; +import { S3Client } from "../../../../src/services/aws/s3-client"; import { HealthCheckStatus } from "../../../../src/services/health-check"; class S3ClientStub implements S3Client {