From 682e352d182abbc8d8987132e8cb14c17a4c6451 Mon Sep 17 00:00:00 2001 From: Seohyun Lee Date: Tue, 4 Jan 2022 13:27:44 +0900 Subject: [PATCH] feat: support subtask judge --- res/test.html | 38 +++++---- src/judge.ts | 2 +- src/runner/common.ts | 157 +++++++++++++++++++++++------------ src/runner/index.ts | 18 ++-- src/runner/languages/text.ts | 19 +++-- src/runner/util.ts | 10 +++ src/types/request.ts | 17 +++- src/types/response.ts | 11 +-- 8 files changed, 182 insertions(+), 90 deletions(-) diff --git a/res/test.html b/res/test.html index 81d4aa3..dd5dc4d 100644 --- a/res/test.html +++ b/res/test.html @@ -254,15 +254,15 @@ setCard( res.data.uid, 1, - `${res.data.reason}`, + res.data.resultCode, `

#${pmp[res.data.uid]}

UID

${res.data.uid}

Result

-

${reasonString[res.data.reason]}

+

${res.data.reason}

Scoring

-

${res.data.result}

+

${JSON.stringify(res.data.result)}

Message

${res.data.message}

Run Info

@@ -275,13 +275,13 @@

Run Info

setCard( res.data.uid, res.data.progress, - res.data.reason, + res.data.resultCode, `

#${pmp[res.data.uid]}

UID

${res.data.uid}

State

-

${reasonString[res.data.reason]}

+

${res.data.reason}

` ) if (res.type === 'JUDGE_STATUS') { @@ -295,7 +295,7 @@

#${i}

UID

${res.data.uid}

State

-

${reasonString[res.data.reason]}

+

${res.data.reason}

` ) } @@ -383,14 +383,24 @@

State

source: document.getElementById('source').value, }, ], - dataSet: { - data: [ - { input: '1 1', output: '2' }, - { input: '10 1', output: '11' }, - { input: '5 1', output: '6' }, - { input: '1 3', output: '4' }, - ], - }, + dataSet: [ + { + scoringType: 'QUANTIZED', + data: [ + { input: '1 1', output: '2' }, + { input: '10 1', output: '11' }, + { input: '5 1', output: '6' }, + { input: '1 3', output: '4' }, + ], + }, + { + scoringType: 'QUANTIZED', + data: [ + { input: '15 1', output: '16' }, + { input: '17 21', output: '38' }, + ], + }, + ], timeLimit: 2000, memoryLimit: 1024, }), diff --git a/src/judge.ts b/src/judge.ts index 1c17b94..907cc3d 100644 --- a/src/judge.ts +++ b/src/judge.ts @@ -39,7 +39,7 @@ export function requestJudge(problem: JudgeRequest) { sendMessage(WebSocketResponseType.JUDGE_PROGRESS, { uid: problem.uid, progress: 0, - reason: 'PD', + resultCode: 'PD', }) while (judgeList.length < MultiJudgeCount && waitList.length) judge(waitList.shift() as string) diff --git a/src/runner/common.ts b/src/runner/common.ts index 146bd03..960feea 100644 --- a/src/runner/common.ts +++ b/src/runner/common.ts @@ -1,4 +1,4 @@ -import { JudgeRequest } from '../types/request' +import { JudgeRequest, ScoringType } from '../types/request' import { sendMessage } from '../socket' import { JudgeResult, @@ -13,6 +13,7 @@ import { clearTempEnv, ExecuteResult, getTmpPath, + representativeResult, } from './util' export default function commonJudge( @@ -21,20 +22,19 @@ export default function commonJudge( getExePath: (path: string) => string ) { return new Promise(async (resolve) => { - let result = Array(data.dataSet.data.length).fill(0), - judgeResult = 'AC' as JudgeResultCode, - message = '' - sendMessage(WebSocketResponseType.JUDGE_PROGRESS, { uid: data.uid, progress: 0, - reason: 'CP', + resultCode: 'CP', }) const tmpPath = initTempEnv(data.uid, data.source) - let maxMemoryUsage = 0, - maxTimeUsage = 0 + let message = '' + const result: (number | number[])[] = [], + judgeResult: JudgeResultCode[] = [], + maxMemoryUsage: number[] = [], + maxTimeUsage: number[] = [] if (build) { const buildResult = await build(tmpPath) @@ -43,10 +43,18 @@ export default function commonJudge( clearTempEnv(data.uid) resolve({ uid: data.uid, - result, - reason: 'CE', - time: 0, - memory: 0, + result: data.dataSet.map((subtask) => { + switch (subtask.scoringType) { + case ScoringType.QUANTIZED: + return 0 + case ScoringType.PROPORTIONAL: + return Array(subtask.data.length).fill(0) + } + }), + resultCode: 'CE', + reason: Array(data.dataSet.length).fill('CE'), + time: Array(data.dataSet.length).fill(0), + memory: Array(data.dataSet.length).fill(0), message: buildResult.stderr .replaceAll(getTmpPath(data.uid), '~') .replaceAll(`/p-${data.uid}`, ''), @@ -58,57 +66,100 @@ export default function commonJudge( sendMessage(WebSocketResponseType.JUDGE_PROGRESS, { uid: data.uid, progress: 0, - reason: 'RUN', + resultCode: 'RUN', }) - for (const i in data.dataSet.data) { - const { code, stdout, stderr, resultType } = await executeJudge( - data, - getExePath(tmpPath), - data.dataSet.data[i].input - ) - if (code) { - let errorMsg = '' - switch (resultType) { - case ResultType.normal: - errorMsg = stderr.split('\n').slice(0, -2).join('\n') - if (judgeResult === 'AC') - judgeResult = 'RE' as JudgeResultCode - break - case ResultType.timeLimitExceeded: - errorMsg = stderr - judgeResult = 'TLE' as JudgeResultCode + for (const subtaskI in data.dataSet) { + const subtask = data.dataSet[subtaskI] + + let subtaskResult = Array(subtask.data.length).fill(0), + subtaskJudgeResult = [] as JudgeResultCode[], + subtaskMaxMemoryUsage = 0, + subtaskMaxTimeUsage = 0 + + for (const i in subtask.data) { + const { code, stdout, stderr, resultType } = await executeJudge( + data, + getExePath(tmpPath), + subtask.data[i].input + ) + if (code) { + let errorMsg = '' + switch (resultType) { + case ResultType.normal: + errorMsg = stderr + .split('\n') + .slice(0, -2) + .join('\n') + subtaskJudgeResult.push('RE') + break + case ResultType.timeLimitExceeded: + errorMsg = stderr + subtaskJudgeResult.push('TLE') + } + if (!message) + message = errorMsg.replaceAll(getTmpPath(data.uid), '~') + if (subtask.scoringType === ScoringType.QUANTIZED) break + } else { + let info = '', + err = stderr.split('\n') + while (!info.includes('|') && err.length) + info = err.pop() || '' + const timeUsage = + parseFloat(info.split('m ')[1].split('s')[0]) * 1000 + const memUsage = parseInt(info.split('|')[1]) + subtaskMaxTimeUsage = Math.max( + subtaskMaxTimeUsage, + timeUsage + ) + subtaskMaxMemoryUsage = Math.max( + subtaskMaxMemoryUsage, + memUsage + ) + if (timeUsage > data.timeLimit) { + subtaskJudgeResult.push('TLE') + if (subtask.scoringType === ScoringType.QUANTIZED) break + } else if (isSame(stdout, subtask.data[i].output)) { + subtaskJudgeResult.push('AC') + subtaskResult[i as any] = 1 + } else { + subtaskJudgeResult.push('WA') + if (subtask.scoringType === ScoringType.QUANTIZED) break + } } - if (!message) - message = errorMsg.replaceAll(getTmpPath(data.uid), '~') - } else { - let info = '', - err = stderr.split('\n') - while (!info.includes('|') && err.length) info = err.pop() || '' - const timeUsage = - parseFloat(info.split('m ')[1].split('s')[0]) * 1000 - const memUsage = parseInt(info.split('|')[1]) - maxTimeUsage = Math.max(maxTimeUsage, timeUsage) - maxMemoryUsage = Math.max(maxMemoryUsage, memUsage) - if (timeUsage > data.timeLimit) - judgeResult = 'TLE' as JudgeResultCode - else if (isSame(stdout, data.dataSet.data[i].output)) - result[i as any] = 1 - else if (judgeResult === 'AC') - judgeResult = 'WA' as JudgeResultCode + sendMessage(WebSocketResponseType.JUDGE_PROGRESS, { + uid: data.uid, + progress: + (parseInt(i) + 1) / subtask.data.length / subtask.data + + parseInt(subtaskI) / data.dataSet.length, + resultCode: 'RUN', + }) + } + + subtaskMaxTimeUsage = Math.min( + Math.round(subtaskMaxTimeUsage), + data.timeLimit + ) + subtaskMaxMemoryUsage = Math.min( + Math.round(subtaskMaxMemoryUsage), + data.memoryLimit + ) + + if (subtask.scoringType === ScoringType.QUANTIZED) { + subtaskResult = subtaskResult.slice(-1)[0] } - sendMessage(WebSocketResponseType.JUDGE_PROGRESS, { - uid: data.uid, - progress: (parseInt(i) + 1) / data.dataSet.data.length, - reason: 'RUN', - }) + result.push(subtaskResult) + judgeResult.push(representativeResult(subtaskJudgeResult)) + maxTimeUsage.push(subtaskMaxTimeUsage) + maxMemoryUsage.push(subtaskMaxMemoryUsage) } clearTempEnv(data.uid) resolve({ uid: data.uid, result, + resultCode: representativeResult(judgeResult), reason: judgeResult, - time: Math.min(Math.round(maxTimeUsage), data.timeLimit), + time: maxTimeUsage, memory: maxMemoryUsage, message, }) diff --git a/src/runner/index.ts b/src/runner/index.ts index 651e95d..77f1c25 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -1,4 +1,4 @@ -import { JudgeRequest, JudgeType } from '../types/request' +import { JudgeRequest, JudgeType, ScoringType } from '../types/request' import { JudgeResult } from '../types/response' import * as fs from 'fs' import * as path from 'path' @@ -57,10 +57,18 @@ export default async function (data: JudgeRequest): Promise { } return { uid: data.uid, - result: Array(data.dataSet.data.length).fill(0), - reason: 'CE', + result: data.dataSet.map((subtask) => { + switch (subtask.scoringType) { + case ScoringType.QUANTIZED: + return 0 + case ScoringType.PROPORTIONAL: + return Array(subtask.data.length).fill(0) + } + }), + resultCode: 'CE', + reason: Array(data.dataSet.length).fill('CE'), message: 'Unknown judge type', - time: 0, - memory: 0, + time: Array(data.dataSet.length).fill(0), + memory: Array(data.dataSet.length).fill(0), } } diff --git a/src/runner/languages/text.ts b/src/runner/languages/text.ts index 9ffb4d1..ffc3716 100644 --- a/src/runner/languages/text.ts +++ b/src/runner/languages/text.ts @@ -12,28 +12,31 @@ export function judge( data: JudgeRequest ) { return new Promise((resolve) => { - let match = Array(data.dataSet.data.length).fill(false) + let match = Array(data.dataSet.length).fill(0) for (const s in data.source) { - for (const i in data.dataSet.data) { + for (const i in data.dataSet) { if ( - isSame(data.source[s].source, data.dataSet.data[i].output) + isSame( + data.source[s].source, + data.dataSet[i].data[0].output + ) ) { match[i] = true } } sendMessage(WebSocketResponseType.JUDGE_PROGRESS, { uid: data.uid, - progress: (s as unknown as number) / data.source.length, + progress: (parseInt(s) + 1) / data.source.length, reason: 'RUN', }) } resolve({ uid: data.uid, result: match, - reason: - match.reduce((a, b) => a + b, 0) === match.length ? 'AC' : 'WA', - time: 0, - memory: 0, + resultCode: 'WA', + reason: match.map((m) => (m ? 'AC' : 'WA')), + time: Array(data.dataSet.length).fill(0), + memory: Array(data.dataSet.length).fill(0), }) }) } diff --git a/src/runner/util.ts b/src/runner/util.ts index f22855a..474c520 100644 --- a/src/runner/util.ts +++ b/src/runner/util.ts @@ -1,6 +1,7 @@ import { execSync, spawn } from 'child_process' import fs from 'fs' import { JudgeRequest, SourceFile } from '../types/request' +import { JudgeResultCode } from '../types/response' export const enum ResultType { normal, @@ -145,3 +146,12 @@ export function executeJudge( { input, timeout: data.timeLimit || 0, cwd: getTmpPath(data.uid) } ) } + +export function representativeResult(results: JudgeResultCode[]) { + if (results.includes('CE')) return 'CE' + if (results.includes('RE')) return 'RE' + if (results.includes('TLE')) return 'TLE' + if (results.includes('MLE')) return 'MLE' + if (results.includes('WA')) return 'WA' + return 'AC' +} diff --git a/src/types/request.ts b/src/types/request.ts index ec8e280..bde9cd8 100644 --- a/src/types/request.ts +++ b/src/types/request.ts @@ -8,6 +8,11 @@ export interface WebSocketRequest { data?: any } +export const enum ScoringType { + PROPORTIONAL = 'PROPORTIONAL', + QUANTIZED = 'QUANTIZED', +} + export const enum JudgeType { CommonJudge = 'CommonJudge', OutputOnly = 'OutputOnly', @@ -38,13 +43,17 @@ export interface DataSet { data: any } -export interface OutputOnly extends DataSet { +export interface SubTask extends DataSet { + scoringType: ScoringType +} + +export interface OutputOnly extends SubTask { data: { output: string }[] } -export interface CommonDataSet extends DataSet { +export interface CommonDataSet extends SubTask { data: { input: string output: string @@ -54,13 +63,13 @@ export interface CommonDataSet extends DataSet { export interface JudgeRequest< TJudgeType extends JudgeType = JudgeType, TJudgeSourceType extends JudgeSourceType = JudgeSourceType, - TDataSet extends DataSet = DataSet + TSubTask extends SubTask = SubTask > { uid: string language: TJudgeSourceType judgeType: TJudgeType source: SourceFile[] - dataSet: TDataSet + dataSet: TSubTask[] timeLimit: number memoryLimit: number } diff --git a/src/types/response.ts b/src/types/response.ts index 2352c58..8cde695 100644 --- a/src/types/response.ts +++ b/src/types/response.ts @@ -1,4 +1,4 @@ -export type JudgeResultCode = 'AC' | 'WA' | 'RTE' | 'TLE' | 'MLE' | 'OLE' | 'CE' +export type JudgeResultCode = 'AC' | 'WA' | 'RE' | 'TLE' | 'MLE' | 'CE' export type JudgeStatusCode = JudgeResultCode | 'PD' | 'CP' | 'RUN' export const enum WebSocketResponseType { @@ -18,10 +18,11 @@ export interface WebSocketResponse { export interface JudgeResult { uid: string - result: number[] - reason: JudgeResultCode - time: number - memory: number + result: (number[] | number)[] + resultCode: JudgeResultCode + reason: JudgeResultCode[] + time: number[] + memory: number[] example?: { output: string no: number