add support for enc parameter for POST: /runs

config.js

@@ -23,5 +23,14 @@ exports = module.exports = {
     PASS: process.env.AMQP_PASS || 'codingblocks',
     HOST: process.env.AMQP_HOST || 'localhost',
     PORT: process.env.AMQP_PORT || 5672
+  },
+
+  S3: {
+    endpoint: process.env.S3_ENDPOINT || 'localhost',
+    port: process.env.S3_PORT || 9000,
+    ssl: process.env.S3_SSL || false,
+    accessKey: process.env.S3_ACCESS_KEY || '',
+    secretKey: process.env.S3_SECRET_KEY || '',
+    bucket: process.env.S3_BUCKET || 'judge-submissions'
   }
 }

migrations/001-addOutputsToSubmission.sql

@@ -0,0 +1 @@
+alter table submissions add column outputs varchar[];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "judge-api",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Judge API",
   "main": "dist/server.js",
   "repository": "https://github.com/coding-blocks/judge-api",
@@ -11,17 +11,21 @@
     "@types/amqplib": "^0.5.4",
     "amqplib": "^0.5.2",
     "apidoc": "^0.17.6",
+    "axios": "^0.18.0",
     "base-64": "^0.1.0",
     "debug": "^4.0.0",
     "express": "^4.16.2",
+    "minio": "^7.0.3",
     "pg": "^7.4.3",
     "pg-hstore": "^2.3.2",
-    "sequelize": "^4.22.6"
+    "sequelize": "^4.22.6",
+    "uuid": "^3.3.2"
   },
   "devDependencies": {
     "@types/chai": "^4.0.4",
     "@types/debug": "^0.0.30",
     "@types/express": "^4.0.39",
+    "@types/minio": "^7.0.1",
     "@types/mocha": "^5.0.0",
     "@types/request": "^2.0.8",
     "@types/sequelize": "^4.0.79",

src/db/models.ts

@@ -42,7 +42,8 @@ const Submissions = db.define('submissions', {
   },
   start_time: Sequelize.DATE,
   end_time: Sequelize.DATE,
-  results: Sequelize.ARRAY(Sequelize.INTEGER)
+  results: Sequelize.ARRAY(Sequelize.INTEGER),
+  outputs: Sequelize.ARRAY(Sequelize.STRING)
 }, {
   paranoid: true, // We do not want to lose any submission data
   timestamps: false // Start and end times are already logged
@@ -53,6 +54,7 @@ export type SubmissionAttributes = {
   start_time: Date
   end_time?: Date
   results?: Array<number>
+  outputs?: Array<string>
 }
 
 const ApiKeys = db.define('apikeys', {

src/routes/api/run.ts

@@ -1,15 +1,21 @@
 import {Response, Router, Request} from 'express'
-import {SubmissionAttributes, Submissions} from '../../db/models'
+import axios from 'axios'
+
+import {SubmissionAttributes, Submissions, db} from '../../db/models'
 import {RunJob, queueJob, successListener} from '../../rabbitmq/jobqueue'
 import {isInvalidRunRequest} from '../../validators/SubmissionValidators'
+import {upload} from '../../utils/s3'
+import {normalizeRunJob} from '../../utils'
 import config = require('../../../config')
 
 const route: Router = Router()
 
 export type RunRequestBody = {
   source: string, //Base64 encoded
   lang: string,
-  stdin: string
+  stdin: string,
+  mode: string,
+  callback?: string
 }
 export interface RunRequest extends Request {
   body: RunRequestBody
@@ -21,7 +27,77 @@ export interface RunResponse {
   stderr: string
 }
 
-const runPool: {[x: number]: Response} = {}
+export type RunPoolElement = {
+  mode: string,
+  res: Response,
+  callback?: string
+}
+
+const runPool: {[x: number]: RunPoolElement} = {}
+
+const handleTimeoutForSubmission = function (submissionId: number) {
+  const job = runPool[submissionId]
+  const errorResponse = {
+    id: submissionId,
+    code: 408,
+    message: "Compile/Run timed out",
+  }
+
+  switch (job.mode) {
+    case 'sync': 
+      job.res.status(408).json(errorResponse)
+      break;
+    case 'callback':
+      axios.post(job.callback, errorResponse)
+  }
+}
+
+const handleSuccessForSubmission = function (result: RunResponse) {
+  const job = runPool[result.id]
+  switch (job.mode) {
+    case 'sync':
+      job.res.status(200).json(result)
+      break;
+    case 'callback':
+      // send a post request to callback 
+      (async () => {
+        // 1. upload the result to s3 and get the url
+        const {url} = await upload(result)
+
+        // 2. save the url in db
+        await Submissions.update({
+          outputs: [url]
+        }, {
+          where: {
+            id: result.id
+          }
+        })
+
+        // make the callback request
+        await axios.post(job.callback, {id: result.id, outputs: [url]})
+      })()
+      break;
+  }
+}
+
+/**
+ * Returns a runPoolElement for request
+ */
+const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoolElement {
+  switch (body.mode) {
+    case 'sync':
+      return ({
+        mode: 'sync',
+        res
+      })
+    case 'callback':
+      return ({
+        mode: 'callback',
+        res,
+        callback: body.callback
+      })
+  }
+}
 
 /**
  * @api {post} /runs POST /runs
@@ -33,6 +109,9 @@ const runPool: {[x: number]: Response} = {}
  * @apiParam {String(Base64)} source source code to run (encoded in base64)
  * @apiParam {Enum} lang Language of code to execute
  * @apiParam {String(Base64)} input [Optional] stdin input for the program (encoded in base64)
+ * @apiParam {Enum} mode [Optional] mode for request. Default = `sync`, see: https://github.com/coding-blocks/judge-api/issues/16
+ * @apiParam {String)} callback [Optional] callback url for request. Required for `mode = callback`
+ * @apiParam {String)} enc [Optional] Encoding type for stdin and source. Can be `url`|`base64`. Default = 'base64' 
  *
  * @apiUse AvailableLangs
  *
@@ -41,14 +120,23 @@ const runPool: {[x: number]: Response} = {}
  * @apiSuccess {String(Base64)} stderr Output of stderr of execution (encoded in base64)
  * @apiSuccess {Number} statuscode Result of operation
  *
- * @apiSuccessExample {JSON} Success-Response:
+ * @apiSuccessExample {JSON} Success-Response(mode=sync):
  *  HTTP/1.1 200 OK
  *  {
  *    "id": 10,
  *    "statuscode": 0,
  *    "stdout": "NA0KMg0KMw=="
  *    "stderr": "VHlwZUVycm9y"
  *  }
+ *  @apiSuccessExample {JSON} Success-Response(mode=callback):
+ *  HTTP/1.1 200 OK
+ * 
+ *  @apiSuccessExample {JSON} Body for Callback(mode=callback):
+ *  HTTP/1.1 200 OK
+ *  {
+ *    "id": 10,
+ *    "outputs": ["http://localhost/judge-submissions/file.json"]
+ *  }
  */
 route.post('/', (req, res, next) => {
   const invalidRequest = isInvalidRunRequest(req)
@@ -62,27 +150,32 @@ route.post('/', (req, res, next) => {
   Submissions.create(<SubmissionAttributes>{
     lang: req.body.lang,
     start_time: new Date()
-  }).then((submission: SubmissionAttributes) => {
+  }).then(async (submission: SubmissionAttributes) => {
 
-    let queued = queueJob(<RunJob>{
+    const job: RunJob = await normalizeRunJob({
       id: submission.id,
       source: req.body.source,
       lang: req.body.lang,
       stdin: req.body.stdin
-    })
+    }, req.body.enc)
+
+    let queued = queueJob(job)
+
     // Put into pool and wait for judge-worker to respond
-    runPool[submission.id] = res
+    runPool[submission.id] = getRunPoolElement(req.body, res)
+
     setTimeout(() => {
       if (runPool[submission.id]) {
-        runPool[submission.id].status(408).json({
-          id: submission.id,
-          code: 408,
-          message: "Compile/Run timed out",
-        })
+        handleTimeoutForSubmission(submission.id)
         delete runPool[submission.id]
       }
     }, config.RUN.TIMEOUT)
 
+    switch (req.body.mode) {
+      case 'callback':
+        res.sendStatus(200)
+    }
+
   }).catch(err => {
     res.status(501).json({
       code: 501,
@@ -97,7 +190,7 @@ route.post('/', (req, res, next) => {
  */
 successListener.on('success', (result: RunResponse) => {
   if (runPool[result.id]) {
-    runPool[result.id].status(200).json(result)
+    handleSuccessForSubmission(result)
     delete runPool[result.id]
   }
   Submissions.update({

src/utils/index.ts

@@ -0,0 +1,21 @@
+import {RunJob} from '../rabbitmq/jobqueue'
+import {download} from './s3'
+
+/**
+ * Normalizes the Run Job (effectively normalizes source and stdin to base64 immediate values)
+ * @param {RunJob} job
+ * @param {string} enc (Default = 'base64')
+ * @returns {Promise<RunJob>} the normalized runJob
+ */
+export const normalizeRunJob = async function (job: RunJob, enc:string = 'base64') : Promise<RunJob> {
+  switch (enc) {
+    case 'url':
+      const [source, stdin] = await Promise.all([download(job.source), download(job.stdin)])
+      return {
+        ...job,
+        source,
+        stdin
+      }
+    default: return job
+  }
+}

src/utils/s3.ts

@@ -0,0 +1,48 @@
+import Minio = require('minio')
+import v4 =  require('uuid/v4')
+import axios from 'axios'
+import config = require('../../config')
+
+const client = new Minio.Client({
+  endPoint: config.S3.endpoint,
+  port: config.S3.port,
+  useSSL: config.S3.ssl,
+  accessKey: config.S3.accessKey,
+  secretKey: config.S3.secretKey,
+})
+
+export type savedFile = {
+  etag: string,
+  url: string
+}
+
+export const urlForFilename = (bucket: string, filename: string) : string => `http${config.S3.ssl ? 's': ''}://${config.S3.endpoint}/${bucket}/${filename}`
+
+/**
+ * Uploads an object to s3 encoded as json
+ * @param {object} object
+ * @param {string} filename The filename (Default = randomized)
+ * @param {string} bucket The bucket name (Default = picked from config.json)
+ * @returns {Promise<savedFile>} The etag and url for the file saved
+ */
+export const upload = function (object:object, filename:string = v4() + '.json' ,bucket:string = config.S3.bucket) : Promise<savedFile> {
+  return new Promise((resolve, reject) => {
+    client.putObject(bucket, filename, JSON.stringify(object), function(err, etag) {
+      if (err) return reject(err)
+      resolve({etag, url: urlForFilename(bucket, filename) })
+    })
+  })
+}
+
+/**
+ * Downloads a file from url and encodes it 
+ * @param {string} url
+ * @param {string} enc (Default = 'base64')
+ * @returns {Promise<string>} the downloaded file encoded as specified
+ */
+export const download = async function (url: string, enc: string = 'base64') : Promise<string> {
+  if (!url) return ''
+
+  const {data} = await axios.get(url)
+  return Buffer.from(data).toString(enc)
+}

src/validators/SubmissionValidators.ts

@@ -12,6 +12,15 @@ export function isInvalidRunRequest(req: Request): Error | boolean {
   if (!req.body.stdin) {
     req.body.stdin = ''
   }
+  if (!req.body.mode) {
+    req.body.mode = 'sync'
+  }
+  if (!['sync', 'callback'].includes(req.body.mode)) {
+    return new Error('Mode must be one of sync, callback')
+  }
+  if (req.body.mode === 'callback' && !req.body.callback) {
+    return new Error('Must specify a callback for callback mode')
+  }
 
   return false
 }

tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "module": "commonjs",
-    "target": "es2015",
+    "target": "es2016",
     "moduleResolution": "node",
     "allowSyntheticDefaultImports": true,
     "sourceMap": true,