Skip to content

Commit

Permalink
core: dump config.yaml to database (for #65)
Browse files Browse the repository at this point in the history
  • Loading branch information
undefined-moe committed Apr 10, 2021
1 parent 3171277 commit dafa80f
Show file tree
Hide file tree
Showing 9 changed files with 152 additions and 147 deletions.
118 changes: 19 additions & 99 deletions packages/hydrojudge/src/cases.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import fs from 'fs-extra';
import path from 'path';
import yaml from 'js-yaml';
import { Dictionary, sum } from 'lodash';
import { sum } from 'lodash';
import readYamlCases, { convertIniConfig } from '@hydrooj/utils/lib/cases';
import { changeErrorType } from '@hydrooj/utils/lib/utils';
import { FormatError, SystemError } from './error';
import { parseTimeMS, parseMemoryMB, ensureFile } from './utils';
import { getConfig } from './config';
Expand Down Expand Up @@ -193,102 +195,6 @@ async function readAutoCases(folder, { next }, cfg) {
return config;
}

export async function readYamlCases(folder: string, cfg: Dictionary<any> = {}, args) {
const config: any = {
checker_type: 'default',
count: 0,
subtasks: [],
judge_extra_files: [],
user_extra_files: [],
};
const next = args.next;
const checkFile = ensureFile(folder);
config.checker_type = cfg.checker_type || 'default';
if (cfg.checker) config.checker = checkFile(cfg.checker, 'Cannot find checker {0}.');
if (cfg.interactor) config.interactor = checkFile(cfg.interactor, 'Cannot find interactor {0}.');
if (cfg.judge_extra_files) {
if (typeof cfg.judge_extra_files === 'string') {
config.judge_extra_files = [checkFile(cfg.judge_extra_files, 'Cannot find judge extra file {0}.')];
} else if (cfg.judge_extra_files instanceof Array) {
for (const file of cfg.judge_extra_files) {
config.judge_extra_files.push(checkFile(file, 'Cannot find judge extra file {0}.'));
}
} else throw new FormatError('Invalid judge_extra_files config.');
}
if (cfg.user_extra_files) {
if (typeof cfg.user_extra_files === 'string') {
config.user_extra_files = [checkFile(cfg.user_extra_files, 'Cannot find user extra file {0}.')];
} else if (cfg.user_extra_files instanceof Array) {
for (const file of cfg.user_extra_files) {
config.user_extra_files.push(checkFile(file, 'Cannot find user extra file {0}.'));
}
} else throw new FormatError('Invalid user_extra_files config.');
}
if (cfg.outputs) {
config.type = 'submit_answer';
} else if (cfg.cases) {
config.subtasks = [{
score: parseInt(cfg.score, 10) || Math.floor(100 / cfg.cases.length),
time_limit_ms: parseTimeMS(cfg.time || '1s'),
memory_limit_mb: parseMemoryMB(cfg.memory || '512m'),
cases: [],
type: 'sum',
}];
for (const c of cfg.cases) {
config.count++;
config.subtasks[0].cases.push({
input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
id: config.count,
});
}
} else if (cfg.subtasks) {
for (const subtask of cfg.subtasks) {
const cases = [];
for (const c of subtask.cases) {
config.count++;
cases.push({
input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
id: config.count,
});
}
config.subtasks.push({
score: parseInt(subtask.score, 10),
if: subtask.if || [],
cases,
time_limit_ms: parseTimeMS(subtask.time || cfg.time || '1s'),
memory_limit_mb: parseMemoryMB(subtask.memory || cfg.memory || '512m'),
});
}
} else {
const c = await readAutoCases(folder, { next }, cfg);
config.subtasks = c.subtasks;
config.count = c.count;
}
if (config.type === 'submit_answer' && !cfg.outputs) throw new FormatError('outputs config not found');
return Object.assign(cfg, config);
}

function convertIniConfig(ini: string) {
const f = ini.split('\n');
const count = parseInt(f[0], 10);
const res = { subtasks: [] };
for (let i = 1; i <= count; i++) {
if (!f[i] || !f[i].trim()) throw new FormatError('Testcada count incorrect.');
const [input, output, time, score, memory] = f[i].split('|');
const cur = {
cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }],
score: parseInt(score, 10),
time: `${time}s`,
memory: '512m',
};
if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`;
res.subtasks.push(cur);
}
return res;
}

function isValidConfig(config) {
if (config.count > (getConfig('testcases_max') || 100)) {
throw new FormatError('Too many testcases. Cancelled.');
Expand All @@ -309,9 +215,23 @@ export default async function readCases(folder: string, cfg: Record<string, any>
} else if (fs.existsSync(ymlConfig)) {
config = { ...yaml.load(fs.readFileSync(ymlConfig).toString()) as object, ...cfg };
} else if (fs.existsSync(iniConfig)) {
config = { ...convertIniConfig(fs.readFileSync(iniConfig).toString()), ...cfg };
try {
config = { ...convertIniConfig(fs.readFileSync(iniConfig).toString()), ...cfg };
} catch (e) {
throw changeErrorType(e, FormatError);
}
} else config = cfg;
const result = await readYamlCases(folder, config, args);
let result;
try {
result = await readYamlCases(config, ensureFile(folder));
} catch (e) {
throw changeErrorType(e, FormatError);
}
if (!(result.outputs || result.subtasks.length)) {
const c = await readAutoCases(folder, args, cfg);
config.subtasks = c.subtasks;
config.count = c.count;
}
isValidConfig(result);
return result;
}
3 changes: 1 addition & 2 deletions packages/hydrooj/src/entry/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ const logger = new Logger('common', true);
export const builtinLib = [
'jwt', 'download', 'i18n', 'mail', 'useragent',
'crypto', 'misc', 'paginate', 'hash.hydro', 'rank',
'validator', 'ui', 'testdata.convert.ini', 'testdataConfig', 'difficulty',
'content',
'validator', 'ui', 'testdataConfig', 'difficulty', 'content',
];

export const builtinModel = [
Expand Down
2 changes: 1 addition & 1 deletion packages/hydrooj/src/entry/master.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ export async function load(call: Entry) {
endPoint, accessKey, secretKey, bucket, region, endPointForUser, endPointForJudge,
};
const storage = require('../service/storage');
storage.start(sopts);
await storage.start(sopts);
require('../service/monitor');
for (const i of builtinModel) require(`../model/${i}`);
const scripts = require('../upgrade').default;
Expand Down
1 change: 0 additions & 1 deletion packages/hydrooj/src/interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,6 @@ export interface Lib extends Record<string, any> {
rank: typeof import('./lib/rank'),
rating: typeof import('./lib/rating'),
testdataConfig: typeof import('./lib/testdataConfig'),
'testdata.convert.ini': typeof import('./lib/testdata.convert.ini'),
useragent: typeof import('./lib/useragent'),
validator: typeof import('./lib/validator'),
template?: any,
Expand Down
22 changes: 0 additions & 22 deletions packages/hydrooj/src/lib/testdata.convert.ini.ts

This file was deleted.

22 changes: 3 additions & 19 deletions packages/hydrooj/src/lib/testdataConfig.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,7 @@
import Zip from 'adm-zip';
import readYamlCases from '@hydrooj/utils/lib/cases';
import { load } from 'js-yaml';
import type { ProblemConfig } from '../interface';

export async function readConfig(file: string | Buffer) {
const data = new Zip(file);
const entries = data.getEntries();
for (const entry of entries) {
if (entry.name.toLowerCase() === 'config.yaml') {
return entry.getData().toString();
}
if (entry.name.toLowerCase() === 'config.ini') {
const ini = entry.getData().toString();
const conv = global.Hydro.lib['testdata.convert.ini'];
return conv(ini);
}
}
return '';
}

interface ParseResult {
count: number,
memoryMax: number,
Expand All @@ -37,7 +21,7 @@ export async function parseConfig(config: string | ProblemConfig) {
let cfg: ProblemConfig = {};
if (typeof config === 'string') {
// TODO should validate here?
cfg = load(config) as ProblemConfig;
cfg = readYamlCases(load(config) as Record<string, any>) as ProblemConfig;
} else cfg = config;
if (cfg.cases) {
for (const c of cfg.cases) {
Expand All @@ -57,4 +41,4 @@ export async function parseConfig(config: string | ProblemConfig) {
}
}

global.Hydro.lib.testdataConfig = { readConfig, parseConfig };
global.Hydro.lib.testdataConfig = { parseConfig };
33 changes: 30 additions & 3 deletions packages/hydrooj/src/upgrade.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,21 @@ import { FilterQuery, ObjectID } from 'mongodb';
import AdmZip from 'adm-zip';
import Queue from 'p-queue';
import yaml from 'js-yaml';
import { convertIniConfig } from '@hydrooj/utils/lib/cases';
import { Progress } from './ui';
import { Rdoc } from './interface';
import { Logger } from './logger';
import { streamToBuffer } from './utils';
import { iterateAllDomain, iterateAllProblem } from './pipelineUtils';
import gridfs from './service/gridfs';
import storage from './service/storage';
import db from './service/db';
import difficultyAlgorithm from './lib/difficulty';
import problem from './model/problem';
import domain from './model/domain';
import * as document from './model/document';
import * as system from './model/system';
import difficultyAlgorithm from './lib/difficulty';
import { STATUS } from './model/builtin';
import RecordModel from './model/record';
import { Rdoc } from './interface';

const logger = new Logger('upgrade');
type UpgradeScript = void | (() => Promise<boolean | void>);
Expand Down Expand Up @@ -262,6 +262,33 @@ const scripts: UpgradeScript[] = [
}
return true;
},
async function _20_21() {
await iterateAllProblem([], async (pdoc) => {
let config: string;
try {
const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yaml`);
config = (await streamToBuffer(file)).toString('utf-8');
logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yaml`);
} catch (e) {
try {
const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.yml`);
config = (await streamToBuffer(file)).toString('utf-8');
logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.yml`);
} catch (err) {
try {
const file = await storage.get(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/config.ini`);
config = yaml.dump(convertIniConfig((await streamToBuffer(file)).toString('utf-8')));
logger.info(`Loaded config for ${pdoc.domainId}/${pdoc.docId} from config.ini`);
} catch (error) {
logger.warn('Config for %s/%s(%s) not found', pdoc.domainId, pdoc.docId, pdoc.pid || pdoc.docId);
// no config found
}
}
}
if (config) await problem.edit(pdoc.domainId, pdoc.docId, { config });
});
return true;
},
];

export default scripts;
92 changes: 92 additions & 0 deletions packages/utils/lib/cases.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import { parseTimeMS, parseMemoryMB } from './utils';

// eslint-disable-next-line @typescript-eslint/no-unused-vars
export default async function readYamlCases(cfg: Record<string, any> = {}, checkFile = (s: string, errMsg: string) => s) {
const config: any = {
checker_type: 'default',
count: 0,
subtasks: [],
judge_extra_files: [],
user_extra_files: [],
};
config.checker_type = cfg.checker_type || 'default';
if (cfg.checker) config.checker = checkFile(cfg.checker, 'Cannot find checker {0}.');
if (cfg.interactor) config.interactor = checkFile(cfg.interactor, 'Cannot find interactor {0}.');
if (cfg.judge_extra_files) {
if (typeof cfg.judge_extra_files === 'string') {
config.judge_extra_files = [checkFile(cfg.judge_extra_files, 'Cannot find judge extra file {0}.')];
} else if (cfg.judge_extra_files instanceof Array) {
for (const file of cfg.judge_extra_files) {
config.judge_extra_files.push(checkFile(file, 'Cannot find judge extra file {0}.'));
}
} else throw new Error('Invalid judge_extra_files config.');
}
if (cfg.user_extra_files) {
if (typeof cfg.user_extra_files === 'string') {
config.user_extra_files = [checkFile(cfg.user_extra_files, 'Cannot find user extra file {0}.')];
} else if (cfg.user_extra_files instanceof Array) {
for (const file of cfg.user_extra_files) {
config.user_extra_files.push(checkFile(file, 'Cannot find user extra file {0}.'));
}
} else throw new Error('Invalid user_extra_files config.');
}
if (cfg.outputs) {
config.type = 'submit_answer';
} else if (cfg.cases) {
config.subtasks = [{
score: parseInt(cfg.score, 10) || Math.floor(100 / cfg.cases.length),
time_limit_ms: parseTimeMS(cfg.time || '1s'),
memory_limit_mb: parseMemoryMB(cfg.memory || '512m'),
cases: [],
type: 'sum',
}];
for (const c of cfg.cases) {
config.count++;
config.subtasks[0].cases.push({
input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
id: config.count,
});
}
} else if (cfg.subtasks) {
for (const subtask of cfg.subtasks) {
const cases = [];
for (const c of subtask.cases) {
config.count++;
cases.push({
input: c.input ? checkFile(c.input, 'Cannot find input file {0}.') : '/dev/null',
output: c.output ? checkFile(c.output, 'Cannot find output file {0}.') : '/dev/null',
id: config.count,
});
}
config.subtasks.push({
score: parseInt(subtask.score, 10),
if: subtask.if || [],
cases,
time_limit_ms: parseTimeMS(subtask.time || cfg.time || '1s'),
memory_limit_mb: parseMemoryMB(subtask.memory || cfg.memory || '512m'),
});
}
}
if (config.type === 'submit_answer' && !cfg.outputs) throw new Error('outputs config not found');
return Object.assign(cfg, config);
}

export function convertIniConfig(ini: string) {
const f = ini.split('\n');
const count = parseInt(f[0], 10);
const res = { subtasks: [] };
for (let i = 1; i <= count; i++) {
if (!f[i] || !f[i].trim()) throw new Error('Testcada count incorrect.');
const [input, output, time, score, memory] = f[i].split('|');
const cur = {
cases: [{ input: `input/${input.toLowerCase()}`, output: `output/${output.toLowerCase()}` }],
score: parseInt(score, 10),
time: `${time}s`,
memory: '512m',
};
if (!Number.isNaN(parseInt(memory, 10))) cur.memory = `${Math.floor(parseInt(memory, 10) / 1024)}m`;
res.subtasks.push(cur);
}
return res;
}
6 changes: 6 additions & 0 deletions packages/utils/lib/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,12 @@ export function formatSeconds(_seconds = '0') {
);
}

export function changeErrorType(err: Error, Err: any) {
const e = new Err(err.message);
e.stack = err.stack;
return e;
}

export async function retry(func: Function, ...args: any[]): Promise<any>;
export async function retry(times: number, func: Function, ...args: any[]): Promise<any>;
// eslint-disable-next-line consistent-return
Expand Down

0 comments on commit dafa80f

Please sign in to comment.