Fix timeouts not being caught correctly

src/__tests__/examples/chap1.ts

@@ -17,15 +17,20 @@ const validStudentPartial =
   `const f = i => i < 3 ? 1 : f(i-1) + f(i-2);`
 
 const invalidStudentRuntime =
-  `const f = i => f(i+1);`
+  `const f = i => f(j+1);`
 
 const invalidStudentSyntax =
   `const f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2)`
 
+// Does not compute fast enough to exceed max call stacks, relies on timeout
+const invalidStudentTimeout =
+   `const f = i => i < -3 ? 0 : f(i+1) + f(i+2);`
+
 export const student: Student = {
   invalid: {
     runtime: invalidStudentRuntime,
-    syntax: invalidStudentSyntax
+    syntax: invalidStudentSyntax,
+    timeout: invalidStudentTimeout
   },
   valid: {
     correct: validStudentCorrect,

src/__tests__/examples/types.ts

@@ -11,6 +11,7 @@ export type Student = {
 type InvalidPrograms<Program> = {
   runtime: Program
   syntax: Program
+  timeout?: Program
 }
 
 type ValidPrograms = {

src/__tests__/test_chapt1.ts

@@ -47,7 +47,14 @@ test('grader OK, student runtimeError', async () => {
       location: 'student'
     }))
   })
-}, 30000)
+})
+
+test('grader OK, student timeoutError', async () => {
+  const results = await runAll(makeAwsEvent(grader.valid, student.invalid.timeout))
+  results.map(result => {
+    expect(result.resultType).toBe('timeout')
+  })
+}, 10000)
 
 test('grader OK, student syntaxError', async () => {
   const results = await runAll(makeAwsEvent(grader.valid, student.invalid.syntax))

src/index.ts

@@ -1,6 +1,8 @@
-import { createContext, runInContext } from 'js-slang'
+import { createContext, runInContext, Result as SourceResult } from 'js-slang'
 import { SourceError } from 'js-slang/dist/types'
 
+const TIMEOUT_DURATION = 20000
+
 type AwsEvent = {
   graderPrograms: string[]
   studentProgram: string
@@ -21,31 +23,41 @@ export type Library = {
   globals: Array<string[]>
 }
 
-type Output = OutputPass | OutputError
+/**
+ * Output is the 'refined' version of a Result.
+ *   OutputError - program raises a js-slang SourceError
+ *   OutputPass - program raises no errors
+ *   OutputTimeout - program has run pass the TIMEOUT_DURATION
+ */
+type Output = OutputError | OutputPass | OutputTimeout
+
+type OutputError = {
+  errors: Array<{
+    errorType: 'runtime' | 'syntax'
+    line?: number
+    location?: string
+  }>
+  resultType: 'error'
+}
 
 type OutputPass = {
   grade: number
   resultType: 'pass'
 }
 
-type OutputError = {
-  errors: Array<{
-    errorType: 'syntax' | 'runtime'
-    line: number
-    location: string
-  }>
-  resultType: 'error'
+type OutputTimeout = {
+  resultType: 'timeout'
 }
 
-export const run = async (chap: number, stdPrg: string, gdrPrg: string): Promise<Output> => {
-  const context = createContext<{}>(chap)
-  const program = stdPrg + '\n' + gdrPrg
-  const result = await runInContext(program, context, { scheduler: 'preemptive' })
-  if (result.status == 'finished') {
-    return { resultType: "pass", grade: result.value } as OutputPass
-  } else {
-    return parseError(context.errors, stdPrg, gdrPrg)
-  }
+/**
+ * Result is the 'raw' result of the js-slang interpreter running a
+ * student/grader program. It will be transformed into a more 'refined' Output
+ * to be returned to a backend.
+ */
+type Result = SourceResult | TimeoutResult
+
+type TimeoutResult = {
+  status: 'timeout'
 }
 
 export const runAll = async (event: AwsEvent): Promise<Output[]> => {
@@ -57,14 +69,40 @@ export const runAll = async (event: AwsEvent): Promise<Output[]> => {
   return results
 }
 
+const run = async (chap: number, stdPrg: string, gdrPrg: string): Promise<Output> => {
+  const context = createContext<{}>(chap)
+  const program = stdPrg + '\n' + gdrPrg
+  const result = await catchTimeouts(runInContext(
+    program, context, { scheduler: 'preemptive' }
+  ))
+  if (result.status === 'finished') {
+    return { resultType: "pass", grade: result.value } as OutputPass
+  } else if (result.status === 'error') {
+    return parseError(context.errors, stdPrg, gdrPrg)
+  } else {
+    return { resultType: 'timeout' } // from timeout/1 in catchTimeouts/1
+  }
+}
+
+const catchTimeouts = (slangPromise: Promise<Result>): Promise<Result> => {
+  const timeoutDuration = process.env.NODE_ENV !== 'test'
+    ? TIMEOUT_DURATION
+    : 1000 // facilitate testing
+  return Promise.race([slangPromise, timeout(timeoutDuration)])
+}
+
+const timeout = (msDuration: number): Promise<TimeoutResult> => (
+  new Promise(resolve => setTimeout(resolve, msDuration, { status: 'timeout' }))
+)
+
 /**
  * Transforms the given SourceErrors and student, grader programs into an output
  * of @type {OutputError}.
  * @param sourceErrors Non-empty array of SourceErrors.
  * @param stdProg Student program.
  * @param grdProg Grader program.
  */
-export const parseError = (
+const parseError = (
   sourceErrors: Array<SourceError>,
   stdProg: string,
   grdProg: string
@@ -89,4 +127,4 @@ export const parseError = (
  * Count the number of lines in a given string.
  * @param lines String to count number of lines of.
  */
-export const numLines = (lines: string) => lines.split("\n").length
+const numLines = (lines: string) => lines.split("\n").length