Tsify lib/storage (#4499)

* Tsify lib/storage

* Add @types/request

* Fix get/post types

* Better type solution

lib/storage/base.js → lib/storage/base.ts

@@ -22,35 +22,36 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
-import profanities from 'profanities';
+import * as express from 'express';
 
 import {logger} from '../logger';
+import {CompilerProps} from '../properties';
 import * as utils from '../utils';
 
+// When it's import profanities from 'profanities'; ts says "Cannot find module 'profanities' or its corresponding type
+// declarations."
+// Updating profanities to v3 requires ESM modules
+// eslint-disable-next-line @typescript-eslint/no-var-requires, unicorn/prefer-module
+const profanities = require('profanities');
+
 const FILE_HASH_VERSION = 'Compiler Explorer Config Hasher 2';
 /* How long a string to check for possible unusable hashes (Profanities or confusing text)
 Note that a Hash might end up being longer than this!
  */
 const USABLE_HASH_CHECK_LENGTH = 9; // Quite generous
 const MAX_TRIES = 4;
 
-export class StorageBase {
-    constructor(httpRootDir, compilerProps) {
-        this.compilerProps = compilerProps;
-        this.httpRootDir = httpRootDir;
-    }
+export abstract class StorageBase {
+    constructor(protected readonly httpRootDir: string, protected readonly compilerProps: CompilerProps) {}
 
     /**
      * Encode a buffer as a URL-safe string.
-     *
-     * @param {Buffer} buffer
-     * @returns {string}
      */
-    static encodeBuffer(buffer) {
+    static encodeBuffer(buffer: Buffer): string {
         return utils.base32Encode(buffer);
     }
 
-    static isCleanText(text) {
+    static isCleanText(text: string) {
         const lowercased = text.toLowerCase();
         return !profanities.some(badWord => lowercased.includes(badWord));
     }
@@ -80,7 +81,7 @@ export class StorageBase {
         return {config, configHash};
     }
 
-    static configFor(req) {
+    static configFor(req: express.Request) {
         if (req.body.config) {
             return req.body.config;
         } else if (req.body.sessions) {
@@ -89,7 +90,7 @@ export class StorageBase {
         return null;
     }
 
-    handler(req, res) {
+    handler(req: express.Request, res: express.Response) {
         // Get the desired config and check for profanities in its hash
         const origConfig = StorageBase.configFor(req);
         if (!origConfig) {
@@ -128,19 +129,11 @@ export class StorageBase {
             });
     }
 
-    async storeItem(item) {
-        throw new Error(`Trying to store item from base storage: ${item}`);
-    }
+    abstract storeItem(item, req: express.Request): Promise<any>;
 
-    async findUniqueSubhash(hash) {
-        throw new Error(`Trying to find unique subhash from base storage ${hash}`);
-    }
+    abstract findUniqueSubhash(hash: string): Promise<any>;
 
-    async expandId(id) {
-        throw new Error(`Trying to expand from base storage ${id}`);
-    }
+    abstract expandId(id): Promise<any>;
 
-    async incrementViewCount(id) {
-        throw new Error(`Trying to increment view count from base storage ${id}`);
-    }
+    abstract incrementViewCount(id): Promise<any>;
 }

lib/storage/local.js → lib/storage/local.ts

@@ -38,6 +38,8 @@ export class StorageLocal extends StorageBase {
         return 'local';
     }
 
+    protected readonly storageFolder: string;
+
     constructor(httpRootDir, compilerProps) {
         super(httpRootDir, compilerProps);
         this.storageFolder = path.normalize(compilerProps.ceProps('localStorageFolder', './lib/storage/data/'));
@@ -57,18 +59,18 @@ export class StorageLocal extends StorageBase {
         return item;
     }
 
-    async findUniqueSubhash(hash) {
+    async findUniqueSubhash(hash: string) {
         logger.info(`Finding local unique subhash for ${hash}`);
         // This currently works on a hardcoded, local directory.
         try {
             const files = await fs.readdir(this.storageFolder);
-            let prefix = hash.substring(0, MIN_STORED_ID_LENGTH);
+            const prefix = hash.substring(0, MIN_STORED_ID_LENGTH);
             const filenames = _.chain(files)
                 .filter(filename => filename.startsWith(prefix))
                 .sort()
                 .value();
             for (let i = MIN_STORED_ID_LENGTH; i < hash.length - 1; i++) {
-                let subHash = hash.substring(0, i);
+                const subHash = hash.substring(0, i);
                 // Check if the current subHash is present in the array
                 const index = _.indexOf(filenames, subHash, true);
                 if (index === -1) {

lib/storage/remote.js → lib/storage/remote.ts

@@ -24,6 +24,7 @@
 
 import {promisify} from 'util';
 
+import * as express from 'express';
 import request from 'request';
 
 import {logger} from '../logger';
@@ -35,6 +36,10 @@ export class StorageRemote extends StorageBase {
         return 'remote';
     }
 
+    protected readonly baseUrl: string;
+    protected readonly get: (uri: string, options?: request.CoreOptions) => Promise<request.Response>;
+    protected readonly post: (uri: string, options?: request.CoreOptions) => Promise<request.Response>;
+
     constructor(httpRootDir, compilerProps) {
         super(httpRootDir, compilerProps);
 
@@ -44,18 +49,23 @@ export class StorageRemote extends StorageBase {
             baseUrl: this.baseUrl,
         });
 
-        this.get = promisify(req.get);
-        this.post = promisify(req.post);
+        // Workaround for ts type shenanigans with defaulting to the last overload
+        this.get = promisify((uri: string, options?: request.CoreOptions, callback?: request.RequestCallback) =>
+            req.get(uri, options, callback),
+        );
+        this.post = promisify((uri: string, options?: request.CoreOptions, callback?: request.RequestCallback) =>
+            req.post(uri, options, callback),
+        );
     }
 
-    async handler(req, res) {
+    override async handler(req: express.Request, res: express.Response) {
         let resp;
         try {
             resp = await this.post('/api/shortener', {
                 json: true,
                 body: req.body,
             });
-        } catch (err) {
+        } catch (err: any) {
             logger.error(err);
             res.status(500);
             res.end(err.message);
@@ -87,4 +97,8 @@ export class StorageRemote extends StorageBase {
     }
 
     async incrementViewCount() {}
+
+    async findUniqueSubhash() {}
+
+    async storeItem() {}
 }

lib/storage/s3.js → lib/storage/s3.ts

@@ -27,6 +27,7 @@ import assert from 'assert';
 import AWS from 'aws-sdk';
 import _ from 'underscore';
 
+import {unwrap} from '../assert';
 import {logger} from '../logger';
 import {S3Bucket} from '../s3-handler';
 import {anonymizeIp} from '../utils';
@@ -58,6 +59,11 @@ export class StorageS3 extends StorageBase {
         return 's3';
     }
 
+    protected readonly prefix: string;
+    protected readonly table: string;
+    protected readonly s3: S3Bucket;
+    protected readonly dynamoDb: AWS.DynamoDB;
+
     constructor(httpRootDir, compilerProps, awsProps) {
         super(httpRootDir, compilerProps);
         const region = awsProps('region');
@@ -122,7 +128,7 @@ export class StorageS3 extends StorageBase {
         const subHashes = _.chain(data.Items).pluck('unique_subhash').pluck('S').value();
         const fullHashes = _.chain(data.Items).pluck('full_hash').pluck('S').value();
         for (let i = MIN_STORED_ID_LENGTH; i < hash.length - 1; i++) {
-            let subHash = hash.substring(0, i);
+            const subHash = hash.substring(0, i);
             // Check if the current base is present in the subHashes array
             const index = _.indexOf(subHashes, subHash, true);
             if (index === -1) {
@@ -168,7 +174,7 @@ export class StorageS3 extends StorageBase {
 
         const attributes = item.Item;
         if (!attributes) throw new Error(`ID ${id} not present in links table`);
-        const result = await this.s3.get(attributes.full_hash.S, this.prefix);
+        const result = await this.s3.get(unwrap(attributes.full_hash.S), this.prefix);
         // If we're here, we are pretty confident there is a match. But never hurts to double check
         if (!result.hit) throw new Error(`ID ${id} not present in storage`);
         const metadata = attributes.named_metadata ? attributes.named_metadata.M : null;