From 993c3631268f493ee93408d29ef7fb8fcb49b7dc Mon Sep 17 00:00:00 2001 From: Jeremy Rifkin <51220084+jeremy-rifkin@users.noreply.github.com> Date: Wed, 28 Dec 2022 10:19:04 -0700 Subject: [PATCH] Tsify lib/storage (#4499) * Tsify lib/storage * Add @types/request * Fix get/post types * Better type solution --- lib/storage/{_all.js => _all.ts} | 0 lib/storage/{base.js => base.ts} | 43 ++++++++++++---------------- lib/storage/{index.js => index.ts} | 0 lib/storage/{local.js => local.ts} | 8 ++++-- lib/storage/{null.js => null.ts} | 0 lib/storage/{remote.js => remote.ts} | 22 +++++++++++--- lib/storage/{s3.js => s3.ts} | 10 +++++-- 7 files changed, 49 insertions(+), 34 deletions(-) rename lib/storage/{_all.js => _all.ts} (100%) rename lib/storage/{base.js => base.ts} (83%) rename lib/storage/{index.js => index.ts} (100%) rename lib/storage/{local.js => local.ts} (96%) rename lib/storage/{null.js => null.ts} (100%) rename lib/storage/{remote.js => remote.ts} (76%) rename lib/storage/{s3.js => s3.ts} (95%) diff --git a/lib/storage/_all.js b/lib/storage/_all.ts similarity index 100% rename from lib/storage/_all.js rename to lib/storage/_all.ts diff --git a/lib/storage/base.js b/lib/storage/base.ts similarity index 83% rename from lib/storage/base.js rename to lib/storage/base.ts index 774799190c4..0eefcc6f44e 100644 --- a/lib/storage/base.js +++ b/lib/storage/base.ts @@ -22,11 +22,18 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -import profanities from 'profanities'; +import * as express from 'express'; import {logger} from '../logger'; +import {CompilerProps} from '../properties'; import * as utils from '../utils'; +// When it's import profanities from 'profanities'; ts says "Cannot find module 'profanities' or its corresponding type +// declarations." +// Updating profanities to v3 requires ESM modules +// eslint-disable-next-line @typescript-eslint/no-var-requires, unicorn/prefer-module +const profanities = require('profanities'); + const FILE_HASH_VERSION = 'Compiler Explorer Config Hasher 2'; /* How long a string to check for possible unusable hashes (Profanities or confusing text) Note that a Hash might end up being longer than this! @@ -34,23 +41,17 @@ Note that a Hash might end up being longer than this! const USABLE_HASH_CHECK_LENGTH = 9; // Quite generous const MAX_TRIES = 4; -export class StorageBase { - constructor(httpRootDir, compilerProps) { - this.compilerProps = compilerProps; - this.httpRootDir = httpRootDir; - } +export abstract class StorageBase { + constructor(protected readonly httpRootDir: string, protected readonly compilerProps: CompilerProps) {} /** * Encode a buffer as a URL-safe string. - * - * @param {Buffer} buffer - * @returns {string} */ - static encodeBuffer(buffer) { + static encodeBuffer(buffer: Buffer): string { return utils.base32Encode(buffer); } - static isCleanText(text) { + static isCleanText(text: string) { const lowercased = text.toLowerCase(); return !profanities.some(badWord => lowercased.includes(badWord)); } @@ -80,7 +81,7 @@ export class StorageBase { return {config, configHash}; } - static configFor(req) { + static configFor(req: express.Request) { if (req.body.config) { return req.body.config; } else if (req.body.sessions) { @@ -89,7 +90,7 @@ export class StorageBase { return null; } - handler(req, res) { + handler(req: express.Request, res: express.Response) { // Get the desired config and check for profanities in its hash const origConfig = StorageBase.configFor(req); if (!origConfig) { @@ -128,19 +129,11 @@ export class StorageBase { }); } - async storeItem(item) { - throw new Error(`Trying to store item from base storage: ${item}`); - } + abstract storeItem(item, req: express.Request): Promise; - async findUniqueSubhash(hash) { - throw new Error(`Trying to find unique subhash from base storage ${hash}`); - } + abstract findUniqueSubhash(hash: string): Promise; - async expandId(id) { - throw new Error(`Trying to expand from base storage ${id}`); - } + abstract expandId(id): Promise; - async incrementViewCount(id) { - throw new Error(`Trying to increment view count from base storage ${id}`); - } + abstract incrementViewCount(id): Promise; } diff --git a/lib/storage/index.js b/lib/storage/index.ts similarity index 100% rename from lib/storage/index.js rename to lib/storage/index.ts diff --git a/lib/storage/local.js b/lib/storage/local.ts similarity index 96% rename from lib/storage/local.js rename to lib/storage/local.ts index 0bc2ea3f7a6..8f30d0b26e1 100644 --- a/lib/storage/local.js +++ b/lib/storage/local.ts @@ -38,6 +38,8 @@ export class StorageLocal extends StorageBase { return 'local'; } + protected readonly storageFolder: string; + constructor(httpRootDir, compilerProps) { super(httpRootDir, compilerProps); this.storageFolder = path.normalize(compilerProps.ceProps('localStorageFolder', './lib/storage/data/')); @@ -57,18 +59,18 @@ export class StorageLocal extends StorageBase { return item; } - async findUniqueSubhash(hash) { + async findUniqueSubhash(hash: string) { logger.info(`Finding local unique subhash for ${hash}`); // This currently works on a hardcoded, local directory. try { const files = await fs.readdir(this.storageFolder); - let prefix = hash.substring(0, MIN_STORED_ID_LENGTH); + const prefix = hash.substring(0, MIN_STORED_ID_LENGTH); const filenames = _.chain(files) .filter(filename => filename.startsWith(prefix)) .sort() .value(); for (let i = MIN_STORED_ID_LENGTH; i < hash.length - 1; i++) { - let subHash = hash.substring(0, i); + const subHash = hash.substring(0, i); // Check if the current subHash is present in the array const index = _.indexOf(filenames, subHash, true); if (index === -1) { diff --git a/lib/storage/null.js b/lib/storage/null.ts similarity index 100% rename from lib/storage/null.js rename to lib/storage/null.ts diff --git a/lib/storage/remote.js b/lib/storage/remote.ts similarity index 76% rename from lib/storage/remote.js rename to lib/storage/remote.ts index 261ec86f3e1..5a720d01dd4 100644 --- a/lib/storage/remote.js +++ b/lib/storage/remote.ts @@ -24,6 +24,7 @@ import {promisify} from 'util'; +import * as express from 'express'; import request from 'request'; import {logger} from '../logger'; @@ -35,6 +36,10 @@ export class StorageRemote extends StorageBase { return 'remote'; } + protected readonly baseUrl: string; + protected readonly get: (uri: string, options?: request.CoreOptions) => Promise; + protected readonly post: (uri: string, options?: request.CoreOptions) => Promise; + constructor(httpRootDir, compilerProps) { super(httpRootDir, compilerProps); @@ -44,18 +49,23 @@ export class StorageRemote extends StorageBase { baseUrl: this.baseUrl, }); - this.get = promisify(req.get); - this.post = promisify(req.post); + // Workaround for ts type shenanigans with defaulting to the last overload + this.get = promisify((uri: string, options?: request.CoreOptions, callback?: request.RequestCallback) => + req.get(uri, options, callback), + ); + this.post = promisify((uri: string, options?: request.CoreOptions, callback?: request.RequestCallback) => + req.post(uri, options, callback), + ); } - async handler(req, res) { + override async handler(req: express.Request, res: express.Response) { let resp; try { resp = await this.post('/api/shortener', { json: true, body: req.body, }); - } catch (err) { + } catch (err: any) { logger.error(err); res.status(500); res.end(err.message); @@ -87,4 +97,8 @@ export class StorageRemote extends StorageBase { } async incrementViewCount() {} + + async findUniqueSubhash() {} + + async storeItem() {} } diff --git a/lib/storage/s3.js b/lib/storage/s3.ts similarity index 95% rename from lib/storage/s3.js rename to lib/storage/s3.ts index 1cacbd5468e..96bff4d80db 100644 --- a/lib/storage/s3.js +++ b/lib/storage/s3.ts @@ -27,6 +27,7 @@ import assert from 'assert'; import AWS from 'aws-sdk'; import _ from 'underscore'; +import {unwrap} from '../assert'; import {logger} from '../logger'; import {S3Bucket} from '../s3-handler'; import {anonymizeIp} from '../utils'; @@ -58,6 +59,11 @@ export class StorageS3 extends StorageBase { return 's3'; } + protected readonly prefix: string; + protected readonly table: string; + protected readonly s3: S3Bucket; + protected readonly dynamoDb: AWS.DynamoDB; + constructor(httpRootDir, compilerProps, awsProps) { super(httpRootDir, compilerProps); const region = awsProps('region'); @@ -122,7 +128,7 @@ export class StorageS3 extends StorageBase { const subHashes = _.chain(data.Items).pluck('unique_subhash').pluck('S').value(); const fullHashes = _.chain(data.Items).pluck('full_hash').pluck('S').value(); for (let i = MIN_STORED_ID_LENGTH; i < hash.length - 1; i++) { - let subHash = hash.substring(0, i); + const subHash = hash.substring(0, i); // Check if the current base is present in the subHashes array const index = _.indexOf(subHashes, subHash, true); if (index === -1) { @@ -168,7 +174,7 @@ export class StorageS3 extends StorageBase { const attributes = item.Item; if (!attributes) throw new Error(`ID ${id} not present in links table`); - const result = await this.s3.get(attributes.full_hash.S, this.prefix); + const result = await this.s3.get(unwrap(attributes.full_hash.S), this.prefix); // If we're here, we are pretty confident there is a match. But never hurts to double check if (!result.hit) throw new Error(`ID ${id} not present in storage`); const metadata = attributes.named_metadata ? attributes.named_metadata.M : null;