feat(Agents): Merges xapi-agents service.

package.json

@@ -38,22 +38,36 @@
     "npm": ">3.0.0"
   },
   "dependencies": {
-    "@learninglocker/xapi-agents": "4.4.3",
+    "@azure/storage-blob": "^10.3.0",
+    "@google-cloud/storage": "^1.5.2",
+    "@learninglocker/xapi-validation": "^2.1.10",
     "accept-language-parser": "^1.5.0",
+    "atob": "^2.0.3",
+    "aws-sdk": "^2.205.0",
+    "bluebird": "3.5.0",
     "boolean": "^0.2.0",
+    "btoa": "^1.1.2",
     "dotenv": "^5.0.0",
     "express": "^4.14.1",
+    "fs-extra": "^5.0.0",
+    "http-status-codes": "^1.3.0",
     "install": "^0.13.0",
     "ioredis": "^4.14.0",
-    "jscommons": "^2.3.3",
+    "jscommons": "^2.3.0",
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.4",
+    "mime-types": "2.1.17",
     "mongodb": "^3.0.2",
+    "node-fetch": "^2.0.0",
     "object-hash": "^1.3.1",
     "query-string": "^6.8.2",
     "redis": "^2.8.0",
+    "rulr": "^4.0.0",
+    "sha1": "^1.1.1",
     "source-map-support": "^0.5.0",
-    "string-to-stream": "^2.0.0"
+    "stream-to-string": "^1.2.0",
+    "string-to-stream": "^2.0.0",
+    "uuid": "^3.0.1"
   },
   "devDependencies": {
     "@ht2-labs/semantic-release": "1.1.90",
@@ -72,6 +86,7 @@
     "@types/mocha": "5.2.7",
     "@types/mongodb": "3.1.10",
     "@types/node": "9.6.51",
+    "@types/node-fetch": "2.5.0",
     "@types/object-hash": "1.3.0",
     "@types/redis": "2.8.6",
     "@types/source-map-support": "0.5.0",

src/apps/activities/expressPresenter/index.ts

@@ -1,7 +1,6 @@
 import { Router } from 'express';
 import mixinCors from 'jscommons/dist/expressPresenter/mixins/cors';
 import mixinHelmet from 'jscommons/dist/expressPresenter/mixins/helmet';
-import mixinMorgan from 'jscommons/dist/expressPresenter/mixins/morgan';
 import Config from './Config';
 import deleteProfile from './deleteProfile';
 import getProfiles from './getProfiles';
@@ -13,7 +12,6 @@ export default (config: Config): Router => {
 
   router.use(mixinCors());
   router.use(mixinHelmet());
-  router.use(mixinMorgan(config.morganDirectory));
 
   router.delete('', deleteProfile(config));
   router.get('', getProfiles(config));

src/apps/activities/repo/storage/factory.ts

@@ -5,7 +5,7 @@ import {
   StorageURL,
 } from '@azure/storage-blob';
 import Storage from '@google-cloud/storage';
-import { S3 } from 'aws-sdk';
+import S3 from 'aws-sdk/clients/s3';
 import azureStorageRepo from '../../azureStorageRepo';
 import googleStorageRepo from '../../googleStorageRepo';
 import localStorageRepo from '../../localStorageRepo';
@@ -18,7 +18,7 @@ export default (factoryConfig: FactoryConfig): Repo => {
     case 's3':
       return s3StorageRepo({
         bucketName: factoryConfig.s3.bucketName,
-        client: new S3(factoryConfig.s3.awsConfig),
+        client: new S3(factoryConfig.s3.awsConfig) as any,
         subFolder: factoryConfig.s3.subFolder,
       });
     case 'google':

src/apps/activities/repoFactory/index.ts

@@ -68,7 +68,7 @@ const getStorageRepo = (): StorageRepo => {
     case 's3':
       return s3StorageRepo({
         bucketName: config.s3StorageRepo.bucketName,
-        client: new S3(config.s3StorageRepo.awsConfig),
+        client: new S3(config.s3StorageRepo.awsConfig) as any,
         subFolder: config.storageSubFolders.activities,
       });
     case 'google':

src/apps/agents/app.ts

@@ -1,10 +1,10 @@
 /* tslint:disable:max-file-line-count */
-import presenterFactory from '@learninglocker/xapi-agents/dist/expressPresenter';
-import serviceFactory from '@learninglocker/xapi-agents/dist/service';
-import enTranslator from '@learninglocker/xapi-agents/dist/translatorFactory/en';
 import { Router } from 'express';
 import AppConfig from './AppConfig';
+import presenterFactory from './expressPresenter';
 import repoFactory from './repo/factory';
+import serviceFactory from './service';
+import enTranslator from './translatorFactory/en';
 
 export default (appConfig: AppConfig): Router => {
   const translator = enTranslator;

src/apps/agents/azureStorageRepo/Config.ts

@@ -0,0 +1,6 @@
+import { ContainerURL } from '@azure/storage-blob';
+
+export default interface Config {
+  readonly containerUrl: ContainerURL;
+  readonly subFolder: string;
+}

src/apps/agents/azureStorageRepo/clearRepo.ts

@@ -0,0 +1,27 @@
+import {
+  Aborter,
+  BlobURL,
+  Models,
+} from '@azure/storage-blob';
+import Config from './Config';
+
+export default (config: Config) => {
+  return async (): Promise<void> => {
+    // tslint:disable-next-line:no-let
+    let marker;
+    do {
+      const listBlobsResponse: Models.ContainerListBlobFlatSegmentResponse =
+        await config.containerUrl.listBlobFlatSegment(Aborter.none, marker);
+      marker = listBlobsResponse.nextMarker;
+      const deletePromises = listBlobsResponse.segment.blobItems.map(
+        async (blobItem: Models.BlobItem) => {
+          const blobUrl = BlobURL.fromContainerURL(config.containerUrl, blobItem.name);
+          if (blobItem.name.startsWith(config.subFolder)) {
+            await blobUrl.delete(Aborter.none);
+          }
+        },
+      );
+      await Promise.all(deletePromises);
+    } while (marker !== '');
+  };
+};

src/apps/agents/azureStorageRepo/deleteProfileContent.ts

@@ -0,0 +1,18 @@
+import {
+  Aborter,
+  BlobURL,
+} from '@azure/storage-blob';
+import DeleteProfileContentOptions from '../repoFactory/options/DeleteProfileContentOptions';
+import getStorageDir from '../utils/getStorageDir';
+import Config from './Config';
+
+ export default (config: Config) => {
+  return async (opts: DeleteProfileContentOptions): Promise<void> => {
+    const profileDir = getStorageDir({ subfolder: config.subFolder, lrs_id: opts.lrs_id });
+    const filePath = `${profileDir}/${opts.key}`;
+
+     const blobUrl = BlobURL.fromContainerURL(config.containerUrl, filePath);
+
+     await blobUrl.delete(Aborter.none);
+  };
+};

src/apps/agents/azureStorageRepo/getProfileContent.ts

@@ -0,0 +1,20 @@
+import { Aborter, BlobURL } from '@azure/storage-blob';
+import GetProfileContentOptions from '../repoFactory/options/GetProfileContentOptions';
+import GetProfileContentResult from '../repoFactory/results/GetProfileContentResult';
+import getStorageDir from '../utils/getStorageDir';
+import Config from './Config';
+
+ export default (config: Config) => {
+  return async (opts: GetProfileContentOptions): Promise<GetProfileContentResult> => {
+    const profileDir = getStorageDir({ subfolder: config.subFolder, lrs_id: opts.lrs_id });
+    const filePath = `${profileDir}/${opts.key}`;
+
+     const blobUrl = BlobURL.fromContainerURL(config.containerUrl, filePath);
+    const content = (await blobUrl.download(Aborter.none, 0)).readableStreamBody;
+    if (content === undefined) {
+      throw new Error('Blob not found');
+    }
+
+    return { content };
+  };
+};

src/apps/agents/azureStorageRepo/index.ts

@@ -0,0 +1,17 @@
+import StorageRepo from '../repoFactory/StorageRepo';
+import clearRepo from './clearRepo';
+import Config from './Config';
+import deleteProfileContent from './deleteProfileContent';
+import getProfileContent from './getProfileContent';
+import storeProfileContent from './storeProfileContent';
+
+ export default (config: Config): StorageRepo => {
+  return {
+    clearRepo: clearRepo(config),
+    deleteProfileContent: deleteProfileContent(config),
+    getProfileContent: getProfileContent(config),
+    migrate: async () => Promise.resolve(),
+    rollback: async () => Promise.resolve(),
+    storeProfileContent: storeProfileContent(config),
+  };
+};

src/apps/agents/azureStorageRepo/storeProfileContent.ts

@@ -0,0 +1,48 @@
+import {
+  Aborter,
+  BlobURL,
+  BlockBlobURL,
+  uploadStreamToBlockBlob,
+} from '@azure/storage-blob';
+import { Readable } from 'stream';
+import StoreProfileContentOptions from '../repoFactory/options/StoreProfileContentOptions';
+import getStorageDir from '../utils/getStorageDir';
+import Config from './Config';
+
+const BYTES_IN_KILOBYTES = 1024;
+const KILOBYTES_IN_MEGABYTES = 1024;
+const FOUR = 4;
+
+// https://github.com/Azure/azure-storage-js/blob/master/blob/samples/highlevel.sample.js
+const BUFFER_SIZE = FOUR * KILOBYTES_IN_MEGABYTES * BYTES_IN_KILOBYTES; // 4MB
+const MAX_BUFFERS = 20;
+
+export default (config: Config) => {
+  return async (opts: StoreProfileContentOptions): Promise<void> => {
+    return new Promise<void>(async (resolve, reject) => {
+      const profileDir = getStorageDir({
+        subfolder: config.subFolder,
+        lrs_id: opts.lrs_id,
+      });
+      const filePath = `${profileDir}/${opts.key}`;
+
+      const blobUrl = BlobURL.fromContainerURL(config.containerUrl, filePath);
+      const blockBlobUrl = BlockBlobURL.fromBlobURL(blobUrl);
+
+      opts.content.on('error', reject);
+
+      try {
+        await uploadStreamToBlockBlob(
+          Aborter.none,
+          opts.content as Readable,
+          blockBlobUrl,
+          BUFFER_SIZE,
+          MAX_BUFFERS,
+        );
+      } catch (err) {
+        reject(err);
+      }
+      resolve();
+    });
+  };
+};

src/apps/agents/errors/Conflict.ts

@@ -0,0 +1,4 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {}

src/apps/agents/errors/ExpiredClientError.ts

@@ -0,0 +1,8 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {
+  constructor() {
+    super();
+  }
+}

src/apps/agents/errors/IfMatch.ts

@@ -0,0 +1,4 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {}

src/apps/agents/errors/IfNoneMatch.ts

@@ -0,0 +1,4 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {}

src/apps/agents/errors/InvalidMethod.ts

@@ -0,0 +1,8 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {
+  constructor(public method: string) {
+    super();
+  }
+}

src/apps/agents/errors/JsonSyntaxError.ts

@@ -0,0 +1,8 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {
+  constructor(public path: string[]) {
+    super();
+  }
+}

src/apps/agents/errors/MaxEtags.ts

@@ -0,0 +1,4 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {}

src/apps/agents/errors/MissingEtags.ts

@@ -0,0 +1,4 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {}

src/apps/agents/errors/NonJsonObject.ts

@@ -0,0 +1,4 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {}

src/apps/agents/errors/UntrustedClientError.ts

@@ -0,0 +1,8 @@
+/* tslint:disable:no-class */
+import BaseError from 'jscommons/dist/errors/BaseError';
+
+export default class extends BaseError {
+  constructor() {
+    super();
+  }
+}

src/apps/agents/expressPresenter/Config.ts

@@ -0,0 +1,12 @@
+import CommonExpressConfig from 'jscommons/dist/expressPresenter/Config';
+import Tracker from 'jscommons/dist/tracker/Tracker';
+import Service from '../serviceFactory/Service';
+import Translator from '../translatorFactory/Translator';
+
+interface Config extends CommonExpressConfig {
+  readonly service: Service;
+  readonly translator: Translator;
+  readonly tracker: Promise<Tracker>;
+}
+
+export default Config;

src/apps/agents/expressPresenter/deleteProfile.ts

@@ -0,0 +1,12 @@
+import { Request, Response } from 'express';
+import Config from './Config';
+import catchErrors from './utils/catchErrors';
+import deleteProfileWithService from './utils/deleteProfileWithService';
+
+export default (config: Config) => {
+  return catchErrors(config, async (req: Request, res: Response): Promise<void> => {
+    const query = req.query;
+    const headers = req.headers;
+    return deleteProfileWithService({ config, res, query, headers });
+  });
+};

src/apps/agents/expressPresenter/getFullAgent.ts

@@ -0,0 +1,20 @@
+import { Request, Response } from 'express';
+import { OK } from 'http-status-codes';
+import { xapiHeaderVersion } from '../utils/constants';
+import Config from './Config';
+import catchErrors from './utils/catchErrors';
+import getAgent from './utils/getAgent';
+import getClient from './utils/getClient';
+import validateVersionHeader from './utils/validateVersionHeader';
+
+export default (config: Config) => {
+  return catchErrors(config, async (req: Request, res: Response): Promise<void> => {
+    const client = await getClient(config, req.header('Authorization'));
+    validateVersionHeader(req.header('X-Experience-API-Version'));
+    const agent = getAgent(req.query.agent);
+    const result = await config.service.getFullAgent({ client, agent });
+    res.status(OK);
+    res.setHeader('X-Experience-API-Version', xapiHeaderVersion);
+    res.json(result);
+  });
+};

src/apps/agents/expressPresenter/getProfiles.ts

@@ -0,0 +1,12 @@
+import { Request, Response } from 'express';
+import Config from './Config';
+import catchErrors from './utils/catchErrors';
+import getWithService from './utils/getWithService';
+
+export default (config: Config) => {
+  return catchErrors(config, async (req: Request, res: Response): Promise<void> => {
+    const query = req.query;
+    const headers = req.headers;
+    return getWithService({ config, res, query, headers });
+  });
+};

src/apps/agents/expressPresenter/index.ts

@@ -0,0 +1,24 @@
+import { Router } from 'express';
+import mixinCors from 'jscommons/dist/expressPresenter/mixins/cors';
+import mixinHelmet from 'jscommons/dist/expressPresenter/mixins/helmet';
+import Config from './Config';
+import deleteProfile from './deleteProfile';
+import getFullAgent from './getFullAgent';
+import getProfiles from './getProfiles';
+import postProfile from './postProfile';
+import putProfile from './putProfile';
+
+export default (config: Config): Router => {
+  const router = Router();
+
+  router.use(mixinCors());
+  router.use(mixinHelmet());
+
+  router.delete('/profile', deleteProfile(config));
+  router.get('/profile', getProfiles(config));
+  router.put('/profile', putProfile(config));
+  router.post('/profile', postProfile(config));
+  router.get('', getFullAgent(config));
+
+  return router;
+};