From 9dc4e19f294b43e7c18cc097d69b3ffcf0f88960 Mon Sep 17 00:00:00 2001 From: Abhishek Gupta Date: Tue, 15 Jan 2019 17:38:43 +0530 Subject: [PATCH 1/6] add support for mode parameter for POST /runs --- package.json | 1 + src/routes/api/run.ts | 80 +++++++++++++++++++++++--- src/validators/SubmissionValidators.ts | 9 +++ tsconfig.json | 2 +- yarn.lock | 17 +++++- 5 files changed, 98 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 7da3e7d..4fb0f4d 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@types/amqplib": "^0.5.4", "amqplib": "^0.5.2", "apidoc": "^0.17.6", + "axios": "^0.18.0", "base-64": "^0.1.0", "debug": "^4.0.0", "express": "^4.16.2", diff --git a/src/routes/api/run.ts b/src/routes/api/run.ts index 804f590..d0f7eac 100644 --- a/src/routes/api/run.ts +++ b/src/routes/api/run.ts @@ -1,4 +1,6 @@ import {Response, Router, Request} from 'express' +import axios from 'axios' + import {SubmissionAttributes, Submissions} from '../../db/models' import {RunJob, queueJob, successListener} from '../../rabbitmq/jobqueue' import {isInvalidRunRequest} from '../../validators/SubmissionValidators' @@ -9,7 +11,9 @@ const route: Router = Router() export type RunRequestBody = { source: string, //Base64 encoded lang: string, - stdin: string + stdin: string, + mode: string, + callback?: string } export interface RunRequest extends Request { body: RunRequestBody @@ -21,7 +25,62 @@ export interface RunResponse { stderr: string } -const runPool: {[x: number]: Response} = {} +export type RunPoolElement = { + mode: string, + res: Response, + callback?: string +} + +const runPool: {[x: number]: RunPoolElement} = {} + +const handleTimeoutForSubmission = function (submissionId: number) { + const job = runPool[submissionId] + const errorResponse = { + id: submissionId, + code: 408, + message: "Compile/Run timed out", + } + + switch (job.mode) { + case 'sync': + job.res.status(408).json(errorResponse) + break; + case 'callback': + axios.post(job.callback, errorResponse) + } +} + +const handleSuccessForSubmission = function (result: RunResponse) { + const job = runPool[result.id] + switch (job.mode) { + case 'sync': + job.res.status(200).json(result) + break; + case 'callback': + // send a post request to callback + axios.post(job.callback, result) + break; + } +} + +/** + * Returns a runPoolElement for request + */ +const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoolElement { + switch (body.mode) { + case 'sync': + return ({ + mode: 'sync', + res + }) + case 'callback': + return ({ + mode: 'callback', + res, + callback: body.callback + }) + } +} /** * @api {post} /runs POST /runs @@ -70,19 +129,22 @@ route.post('/', (req, res, next) => { lang: req.body.lang, stdin: req.body.stdin }) + // Put into pool and wait for judge-worker to respond - runPool[submission.id] = res + runPool[submission.id] = getRunPoolElement(req.body, res) + setTimeout(() => { if (runPool[submission.id]) { - runPool[submission.id].status(408).json({ - id: submission.id, - code: 408, - message: "Compile/Run timed out", - }) + handleTimeoutForSubmission(submission.id) delete runPool[submission.id] } }, config.RUN.TIMEOUT) + switch (req.body.mode) { + case 'callback': + res.sendStatus(200) + } + }).catch(err => { res.status(501).json({ code: 501, @@ -97,7 +159,7 @@ route.post('/', (req, res, next) => { */ successListener.on('success', (result: RunResponse) => { if (runPool[result.id]) { - runPool[result.id].status(200).json(result) + handleSuccessForSubmission(result) delete runPool[result.id] } Submissions.update({ diff --git a/src/validators/SubmissionValidators.ts b/src/validators/SubmissionValidators.ts index 5979baa..bc5c764 100644 --- a/src/validators/SubmissionValidators.ts +++ b/src/validators/SubmissionValidators.ts @@ -12,6 +12,15 @@ export function isInvalidRunRequest(req: Request): Error | boolean { if (!req.body.stdin) { req.body.stdin = '' } + if (!req.body.mode) { + req.body.mode = 'sync' + } + if (!['sync', 'callback'].includes(req.body.mode)) { + return new Error('Mode must be one of sync, callback') + } + if (req.body.mode === 'callback' && !req.body.callback) { + return new Error('Must specify a callback for callback mode') + } return false } \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index ea1964a..dbd72f1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "module": "commonjs", - "target": "es2015", + "target": "es2016", "moduleResolution": "node", "allowSyntheticDefaultImports": true, "sourceMap": true, diff --git a/yarn.lock b/yarn.lock index 83541a6..f74e59d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -365,6 +365,14 @@ aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" +axios@^0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102" + integrity sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI= + dependencies: + follow-redirects "^1.3.0" + is-buffer "^1.1.5" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -721,7 +729,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: dependencies: ms "2.0.0" -debug@3.1.0, debug@^3.0.0, debug@^3.1.0: +debug@3.1.0, debug@=3.1.0, debug@^3.0.0, debug@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: @@ -1026,6 +1034,13 @@ find-up@^2.1.0: dependencies: locate-path "^2.0.0" +follow-redirects@^1.3.0: + version "1.6.1" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.6.1.tgz#514973c44b5757368bad8bddfe52f81f015c94cb" + integrity sha512-t2JCjbzxQpWvbhts3l6SH1DKzSrx8a+SsaVf4h6bG4kOXUuPYS/kg2Lr4gQSb7eemaHqJkOThF1BGyjlUkO1GQ== + dependencies: + debug "=3.1.0" + for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" From 50816db17ca0819cb3ea41013cb02fbd308566d9 Mon Sep 17 00:00:00 2001 From: Abhishek Gupta Date: Tue, 15 Jan 2019 20:42:15 +0530 Subject: [PATCH 2/6] the dawgs --- src/routes/api/run.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/routes/api/run.ts b/src/routes/api/run.ts index d0f7eac..48744e0 100644 --- a/src/routes/api/run.ts +++ b/src/routes/api/run.ts @@ -92,6 +92,8 @@ const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoo * @apiParam {String(Base64)} source source code to run (encoded in base64) * @apiParam {Enum} lang Language of code to execute * @apiParam {String(Base64)} input [Optional] stdin input for the program (encoded in base64) + * @apiParam {Enum} mode [Optional] mode for request. Default = `sync`, see: https://github.com/coding-blocks/judge-api/issues/16 + * @apiParam {String)} callback [Optional] callback url for request. Required for `mode = callback` * * @apiUse AvailableLangs * @@ -100,7 +102,7 @@ const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoo * @apiSuccess {String(Base64)} stderr Output of stderr of execution (encoded in base64) * @apiSuccess {Number} statuscode Result of operation * - * @apiSuccessExample {JSON} Success-Response: + * @apiSuccessExample {JSON} Success-Response(mode=sync): * HTTP/1.1 200 OK * { * "id": 10, @@ -108,6 +110,8 @@ const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoo * "stdout": "NA0KMg0KMw==" * "stderr": "VHlwZUVycm9y" * } + * @apiSuccessExample {JSON} Success-Response(mode=callback): + * HTTP/1.1 200 OK */ route.post('/', (req, res, next) => { const invalidRequest = isInvalidRunRequest(req) From c2bc622d9c4a541ec0bbb11e0663a45ae0008fff Mon Sep 17 00:00:00 2001 From: Abhishek Gupta Date: Wed, 16 Jan 2019 17:55:00 +0530 Subject: [PATCH 3/6] upload result to s3 for callback mode (POST: /runs) --- config.js | 9 ++ migrations/001-addOutputsToSubmission.sql | 2 + package.json | 7 +- src/db/models.ts | 4 +- src/routes/api/run.ts | 27 +++- src/utils/s3.ts | 27 ++++ yarn.lock | 178 ++++++++++++++++++++-- 7 files changed, 239 insertions(+), 15 deletions(-) create mode 100644 migrations/001-addOutputsToSubmission.sql create mode 100644 src/utils/s3.ts diff --git a/config.js b/config.js index 5931c63..0f64ed1 100644 --- a/config.js +++ b/config.js @@ -23,5 +23,14 @@ exports = module.exports = { PASS: process.env.AMQP_PASS || 'codingblocks', HOST: process.env.AMQP_HOST || 'localhost', PORT: process.env.AMQP_PORT || 5672 + }, + + S3: { + endpoint: process.env.S3_ENDPOINT || 'localhost', + port: process.env.S3_PORT || 9000, + ssl: process.env.S3_SSL || false, + accessKey: process.env.S3_ACCESS_KEY || '', + secretKey: process.env.S3_SECRET_KEY || '', + bucket: process.env.S3_BUCKET || 'judge-submissions' } } \ No newline at end of file diff --git a/migrations/001-addOutputsToSubmission.sql b/migrations/001-addOutputsToSubmission.sql new file mode 100644 index 0000000..7e09578 --- /dev/null +++ b/migrations/001-addOutputsToSubmission.sql @@ -0,0 +1,2 @@ +alter table submissions + add column outputs varchar[]; diff --git a/package.json b/package.json index 4fb0f4d..c211f03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "judge-api", - "version": "1.0.0", + "version": "1.1.0", "description": "Judge API", "main": "dist/server.js", "repository": "https://github.com/coding-blocks/judge-api", @@ -15,14 +15,17 @@ "base-64": "^0.1.0", "debug": "^4.0.0", "express": "^4.16.2", + "minio": "^7.0.3", "pg": "^7.4.3", "pg-hstore": "^2.3.2", - "sequelize": "^4.22.6" + "sequelize": "^4.22.6", + "uuid": "^3.3.2" }, "devDependencies": { "@types/chai": "^4.0.4", "@types/debug": "^0.0.30", "@types/express": "^4.0.39", + "@types/minio": "^7.0.1", "@types/mocha": "^5.0.0", "@types/request": "^2.0.8", "@types/sequelize": "^4.0.79", diff --git a/src/db/models.ts b/src/db/models.ts index ca6ba6f..348cd29 100644 --- a/src/db/models.ts +++ b/src/db/models.ts @@ -42,7 +42,8 @@ const Submissions = db.define('submissions', { }, start_time: Sequelize.DATE, end_time: Sequelize.DATE, - results: Sequelize.ARRAY(Sequelize.INTEGER) + results: Sequelize.ARRAY(Sequelize.INTEGER), + outputs: Sequelize.ARRAY(Sequelize.STRING) }, { paranoid: true, // We do not want to lose any submission data timestamps: false // Start and end times are already logged @@ -53,6 +54,7 @@ export type SubmissionAttributes = { start_time: Date end_time?: Date results?: Array + outputs?: Array } const ApiKeys = db.define('apikeys', { diff --git a/src/routes/api/run.ts b/src/routes/api/run.ts index 48744e0..0703f8a 100644 --- a/src/routes/api/run.ts +++ b/src/routes/api/run.ts @@ -1,9 +1,10 @@ import {Response, Router, Request} from 'express' import axios from 'axios' -import {SubmissionAttributes, Submissions} from '../../db/models' +import {SubmissionAttributes, Submissions, db} from '../../db/models' import {RunJob, queueJob, successListener} from '../../rabbitmq/jobqueue' import {isInvalidRunRequest} from '../../validators/SubmissionValidators' +import {upload} from '../../utils/s3' import config = require('../../../config') const route: Router = Router() @@ -58,7 +59,22 @@ const handleSuccessForSubmission = function (result: RunResponse) { break; case 'callback': // send a post request to callback - axios.post(job.callback, result) + (async () => { + // 1. upload the result to s3 and get the url + const {url} = await upload(result) + + // 2. save the url in db + await Submissions.update({ + outputs: [url] + }, { + where: { + id: result.id + } + }) + + // make the callback request + await axios.post(job.callback, {id: result.id, outputs: [url]}) + })() break; } } @@ -112,6 +128,13 @@ const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoo * } * @apiSuccessExample {JSON} Success-Response(mode=callback): * HTTP/1.1 200 OK + * + * @apiSuccessExample {JSON} Body for Callback(mode=callback): + * HTTP/1.1 200 OK + * { + * "id": 10, + * "outputs": ["http://localhost/judge-submissions/file.json"] + * } */ route.post('/', (req, res, next) => { const invalidRequest = isInvalidRunRequest(req) diff --git a/src/utils/s3.ts b/src/utils/s3.ts new file mode 100644 index 0000000..ac38d8c --- /dev/null +++ b/src/utils/s3.ts @@ -0,0 +1,27 @@ +import Minio = require('minio') +import v4 = require('uuid/v4') +import config = require('../../config') + +const client = new Minio.Client({ + endPoint: config.S3.endpoint, + port: config.S3.port, + useSSL: config.S3.ssl, + accessKey: config.S3.accessKey, + secretKey: config.S3.secretKey, +}) + +export type savedFile = { + etag: string, + url: string +} + +export const urlForFilename = (bucket: string, filename: string) : string => `http${config.S3.ssl ? 's': ''}://${config.S3.endpoint}/${bucket}/${filename}` + +export const upload = function (object:object, filename:string = v4() + '.json' ,bucket:string = config.S3.bucket) : Promise { + return new Promise((resolve, reject) => { + client.putObject(bucket, filename, JSON.stringify(object), function(err, etag) { + if (err) return reject(err) + resolve({etag, url: urlForFilename(bucket, filename) }) + }) + }) +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index f74e59d..db0dc65 100644 --- a/yarn.lock +++ b/yarn.lock @@ -156,6 +156,13 @@ version "2.0.0" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b" +"@types/minio@^7.0.1": + version "7.0.1" + resolved "https://registry.yarnpkg.com/@types/minio/-/minio-7.0.1.tgz#98e6348c747eb618c88412f6f297dd82f9801e9a" + integrity sha512-PnhsyzE7/n3Ag7pgExXB2eoTaeh8cUmNylEa++2xm/t/vUaOueGWkCq1HRCUWUxeQlFQvBADmpgASrSyEooSQQ== + dependencies: + "@types/node" "*" + "@types/mocha@^5.0.0": version "5.2.5" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-5.2.5.tgz#8a4accfc403c124a0bafe8a9fc61a05ec1032073" @@ -340,7 +347,7 @@ assign-symbols@^1.0.0: resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= -async@^1.4.0: +async@^1.4.0, async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" @@ -406,6 +413,15 @@ bitsyntax@~0.0.4: dependencies: buffer-more-ints "0.0.2" +block-stream2@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/block-stream2/-/block-stream2-1.1.0.tgz#c738e3a91ba977ebb5e1fef431e13ca11d8639e2" + integrity sha1-xzjjqRupd+u14f70MeE8oR2GOeI= + dependencies: + defined "^1.0.0" + inherits "^2.0.1" + readable-stream "^2.0.4" + bluebird@^3.4.6: version "3.5.1" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" @@ -656,6 +672,16 @@ concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" +concat-stream@^1.4.8: + version "1.6.2" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" + integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== + dependencies: + buffer-from "^1.0.0" + inherits "^2.0.3" + readable-stream "^2.2.2" + typedarray "^0.0.6" + content-disposition@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" @@ -785,6 +811,11 @@ define-property@^2.0.2: is-descriptor "^1.0.2" isobject "^3.0.1" +defined@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" + integrity sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM= + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -834,6 +865,11 @@ error-ex@^1.2.0: dependencies: is-arrayish "^0.2.1" +es6-error@^2.0.2: + version "2.1.1" + resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-2.1.1.tgz#91384301ec5ed1c9a7247d1128247216f03547cd" + integrity sha1-kThDAexe0cmnJH0RKCRyFvA1R80= + escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" @@ -1288,7 +1324,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.3, inherits@~2.0.1: +inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -1460,7 +1496,7 @@ isarray@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" -isarray@1.0.0: +isarray@1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -1561,6 +1597,11 @@ json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" +json-stream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-stream/-/json-stream-1.0.0.tgz#1a3854e28d2bbeeab31cc7ddf683d2ddc5652708" + integrity sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg= + json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -1641,15 +1682,15 @@ locate-path@^2.0.0: p-locate "^2.0.0" path-exists "^3.0.0" -lodash@^4.17.1, lodash@~4.17.4: - version "4.17.4" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" - -lodash@^4.17.5: +lodash@^4.14.2, lodash@^4.17.5: version "4.17.11" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== +lodash@^4.17.1, lodash@~4.17.4: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" @@ -1773,12 +1814,24 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" +mime-db@~1.37.0: + version "1.37.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" + integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== + mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: mime-db "~1.30.0" +mime-types@^2.1.14: + version "2.1.21" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" + integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== + dependencies: + mime-db "~1.37.0" + mime@1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" @@ -1805,6 +1858,25 @@ minimist@~0.0.1: version "0.0.10" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" +minio@^7.0.3: + version "7.0.3" + resolved "https://registry.yarnpkg.com/minio/-/minio-7.0.3.tgz#3530cfd8bad1774c6d0bfa9ebc98de82b793b49c" + integrity sha512-VgFn6Z+iFMd6HiW7GcCpAg5H8WN/cq5E1MLSlqv0CxwRObRTmmUSBHXMr4+vAn9qZDkKpC+gQwE52mLkuBNzMQ== + dependencies: + async "^1.5.2" + block-stream2 "^1.0.0" + concat-stream "^1.4.8" + es6-error "^2.0.2" + json-stream "^1.0.0" + lodash "^4.14.2" + mime-types "^2.1.14" + mkdirp "^0.5.1" + querystring "0.2.0" + through2 "^0.6.5" + uuid "^3.1.0" + xml "^1.0.0" + xml2js "^0.4.15" + mixin-deep@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.1.tgz#a49e7268dce1a0d9698e45326c5626df3543d0fe" @@ -2184,6 +2256,11 @@ preserve@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b" +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw== + proxy-addr@~2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.2.tgz#6571504f47bb988ec8180253f85dd7e14952bdec" @@ -2203,6 +2280,11 @@ qs@6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + randomatic@^1.1.3: version "1.1.7" resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-1.1.7.tgz#c7abe9cc8b87c0baa876b19fde83fd464797e38c" @@ -2247,6 +2329,29 @@ read-pkg@^1.0.0: isarray "0.0.1" string_decoder "~0.10.x" +"readable-stream@>=1.0.33-1 <1.1.0-0": + version "1.0.34" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" + integrity sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@^2.0.4, readable-stream@^2.2.2: + version "2.3.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + regex-cache@^0.4.2: version "0.4.4" resolved "https://registry.yarnpkg.com/regex-cache/-/regex-cache-0.4.4.tgz#75bdc58a2a1496cec48a12835bc54c8d562336dd" @@ -2345,7 +2450,7 @@ safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" -safe-buffer@~5.1.1: +safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== @@ -2357,6 +2462,11 @@ safe-regex@^1.1.0: dependencies: ret "~0.1.10" +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + "semver@2 || 3 || 4 || 5", semver@^5.0.1: version "5.4.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" @@ -2648,6 +2758,13 @@ string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + stringstream@~0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2720,6 +2837,14 @@ test-exclude@^4.2.0: read-pkg-up "^1.0.1" require-main-filename "^1.0.1" +through2@^0.6.5: + version "0.6.5" + resolved "https://registry.yarnpkg.com/through2/-/through2-0.6.5.tgz#41ab9c67b29d57209071410e1d7a7a968cd3ad48" + integrity sha1-QaucZ7KdVyCQcUEOHXp6lozTrUg= + dependencies: + readable-stream ">=1.0.33-1 <1.1.0-0" + xtend ">=4.0.0 <4.1.0-0" + through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -2803,6 +2928,11 @@ type-is@~1.6.15: media-typer "0.3.0" mime-types "~2.1.15" +typedarray@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= + typescript@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.6.1.tgz#ef39cdea27abac0b500242d6726ab90e0c846631" @@ -2868,6 +2998,11 @@ use@^3.1.0: resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" @@ -2876,6 +3011,11 @@ uuid@^3.0.0, uuid@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" +uuid@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + validate-npm-package-license@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc" @@ -2964,7 +3104,25 @@ write-file-atomic@^1.1.4: imurmurhash "^0.1.4" slide "^1.1.5" -xtend@^4.0.0: +xml2js@^0.4.15: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" + integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + +"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" From a9fdd03780fc4d70583709e178bc8d0abce5a85c Mon Sep 17 00:00:00 2001 From: Abhishek Gupta Date: Wed, 16 Jan 2019 18:47:00 +0530 Subject: [PATCH 4/6] add support for enc parameter for run --- src/routes/api/run.ts | 9 ++++++--- src/utils/index.ts | 21 +++++++++++++++++++++ src/utils/s3.ts | 21 +++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 src/utils/index.ts diff --git a/src/routes/api/run.ts b/src/routes/api/run.ts index 0703f8a..944603b 100644 --- a/src/routes/api/run.ts +++ b/src/routes/api/run.ts @@ -5,6 +5,7 @@ import {SubmissionAttributes, Submissions, db} from '../../db/models' import {RunJob, queueJob, successListener} from '../../rabbitmq/jobqueue' import {isInvalidRunRequest} from '../../validators/SubmissionValidators' import {upload} from '../../utils/s3' +import {normalizeRunJob} from '../../utils' import config = require('../../../config') const route: Router = Router() @@ -148,14 +149,16 @@ route.post('/', (req, res, next) => { Submissions.create({ lang: req.body.lang, start_time: new Date() - }).then((submission: SubmissionAttributes) => { + }).then(async (submission: SubmissionAttributes) => { - let queued = queueJob({ + const job: RunJob = await normalizeRunJob({ id: submission.id, source: req.body.source, lang: req.body.lang, stdin: req.body.stdin - }) + }, req.body.enc) + + let queued = queueJob(job) // Put into pool and wait for judge-worker to respond runPool[submission.id] = getRunPoolElement(req.body, res) diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 0000000..92cac30 --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1,21 @@ +import {RunJob} from '../rabbitmq/jobqueue' +import {download} from './s3' + +/** + * Normalizes the Run Job (effectively normalizes source and stdin to base64 immediate values) + * @param {RunJob} job + * @param {string} enc (Default = 'base64') + * @returns {Promise} the normalized runJob + */ +export const normalizeRunJob = async function (job: RunJob, enc:string = 'base64') : Promise { + switch (enc) { + case 'url': + const [source, stdin] = await Promise.all([download(job.source), download(job.stdin)]) + return { + ...job, + source, + stdin + } + default: return job + } +} \ No newline at end of file diff --git a/src/utils/s3.ts b/src/utils/s3.ts index ac38d8c..6ccd6c4 100644 --- a/src/utils/s3.ts +++ b/src/utils/s3.ts @@ -1,5 +1,6 @@ import Minio = require('minio') import v4 = require('uuid/v4') +import axios from 'axios' import config = require('../../config') const client = new Minio.Client({ @@ -17,6 +18,13 @@ export type savedFile = { export const urlForFilename = (bucket: string, filename: string) : string => `http${config.S3.ssl ? 's': ''}://${config.S3.endpoint}/${bucket}/${filename}` +/** + * Uploads an object to s3 encoded as json + * @param {object} object + * @param {string} filename The filename (Default = randomized) + * @param {string} bucket The bucket name (Default = picked from config.json) + * @returns {Promise} The etag and url for the file saved + */ export const upload = function (object:object, filename:string = v4() + '.json' ,bucket:string = config.S3.bucket) : Promise { return new Promise((resolve, reject) => { client.putObject(bucket, filename, JSON.stringify(object), function(err, etag) { @@ -24,4 +32,17 @@ export const upload = function (object:object, filename:string = v4() + '.json' resolve({etag, url: urlForFilename(bucket, filename) }) }) }) +} + +/** + * Downloads a file from url and encodes it + * @param {string} url + * @param {string} enc (Default = 'base64') + * @returns {Promise} the downloaded file encoded as specified + */ +export const download = async function (url: string, enc: string = 'base64') : Promise { + if (!url) return '' + + const {data} = await axios.get(url) + return Buffer.from(data).toString(enc) } \ No newline at end of file From 356e2bd2a002146e2f144c65a2a939f5bd57003a Mon Sep 17 00:00:00 2001 From: Abhishek Gupta Date: Wed, 16 Jan 2019 18:52:02 +0530 Subject: [PATCH 5/6] update the docs --- src/routes/api/run.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routes/api/run.ts b/src/routes/api/run.ts index 944603b..b1d81b8 100644 --- a/src/routes/api/run.ts +++ b/src/routes/api/run.ts @@ -111,6 +111,7 @@ const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoo * @apiParam {String(Base64)} input [Optional] stdin input for the program (encoded in base64) * @apiParam {Enum} mode [Optional] mode for request. Default = `sync`, see: https://github.com/coding-blocks/judge-api/issues/16 * @apiParam {String)} callback [Optional] callback url for request. Required for `mode = callback` + * @apiParam {String)} enc [Optional] Encoding type for stdin and source. Can be `url`|`base64`. Default = 'base64' * * @apiUse AvailableLangs * From 93c97beb0befdc5a4e217339e9d0a746d9ba74cb Mon Sep 17 00:00:00 2001 From: Arnav Gupta Date: Thu, 24 Jan 2019 18:53:06 +0530 Subject: [PATCH 6/6] Update 001-addOutputsToSubmission.sql --- migrations/001-addOutputsToSubmission.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/migrations/001-addOutputsToSubmission.sql b/migrations/001-addOutputsToSubmission.sql index 7e09578..b3a5763 100644 --- a/migrations/001-addOutputsToSubmission.sql +++ b/migrations/001-addOutputsToSubmission.sql @@ -1,2 +1 @@ -alter table submissions - add column outputs varchar[]; +alter table submissions add column outputs varchar[];