Skip to content
This repository has been archived by the owner on Feb 25, 2023. It is now read-only.

Commit

Permalink
Dictionary validate updates (#2137)
Browse files Browse the repository at this point in the history 
* Reuse JsomSchema instance

* Install ajv

* Add support for using ajv as a JSON schema validator

* Update usage
  • Loading branch information
@toasted-nutbread
toasted-nutbread committed May 17, 2022
1 parent 84c9231 commit 5a72303
Show file tree
Hide file tree
Showing 5 changed files with 264 additions and 126 deletions.
63 changes: 40 additions & 23 deletions dev/dictionary-validate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,7 @@ const fs = require('fs');
const path = require('path');
const {performance} = require('perf_hooks');
const {JSZip} = require('./util');
const {VM} = require('./vm');

const vm = new VM();
vm.execute([
'js/core.js',
'js/general/cache-map.js',
'js/data/json-schema.js'
]);
const JsonSchema = vm.get('JsonSchema');
const {createJsonSchema} = require('./schema-validate');


function readSchema(relativeFileName) {
Expand All @@ -37,7 +29,14 @@ function readSchema(relativeFileName) {
}


async function validateDictionaryBanks(zip, fileNameFormat, schema) {
async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) {
let jsonSchema;
try {
jsonSchema = createJsonSchema(mode, schema);
} catch (e) {
e.message += `\n(in file ${fileNameFormat})}`;
throw e;
}
let index = 1;
while (true) {
const fileName = fileNameFormat.replace(/\?/, index);
Expand All @@ -46,28 +45,40 @@ async function validateDictionaryBanks(zip, fileNameFormat, schema) {
if (!file) { break; }

const data = JSON.parse(await file.async('string'));
new JsonSchema(schema).validate(data);
try {
jsonSchema.validate(data);
} catch (e) {
e.message += `\n(in file ${fileName})}`;
throw e;
}

++index;
}
}

async function validateDictionary(archive, schemas) {
const indexFile = archive.files['index.json'];
async function validateDictionary(mode, archive, schemas) {
const fileName = 'index.json';
const indexFile = archive.files[fileName];
if (!indexFile) {
throw new Error('No dictionary index found in archive');
}

const index = JSON.parse(await indexFile.async('string'));
const version = index.format || index.version;

new JsonSchema(schemas.index).validate(index);
try {
const jsonSchema = createJsonSchema(mode, schemas.index);
jsonSchema.validate(index);
} catch (e) {
e.message += `\n(in file ${fileName})}`;
throw e;
}

await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
await validateDictionaryBanks(archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
await validateDictionaryBanks(archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
await validateDictionaryBanks(archive, 'tag_bank_?.json', schemas.tagBankV3);
await validateDictionaryBanks(mode, archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
await validateDictionaryBanks(mode, archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
await validateDictionaryBanks(mode, archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
await validateDictionaryBanks(mode, archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3);
}

function getSchemas() {
Expand All @@ -84,7 +95,7 @@ function getSchemas() {
}


async function testDictionaryFiles(dictionaryFileNames) {
async function testDictionaryFiles(mode, dictionaryFileNames) {
const schemas = getSchemas();

for (const dictionaryFileName of dictionaryFileNames) {
Expand All @@ -93,7 +104,7 @@ async function testDictionaryFiles(dictionaryFileNames) {
console.log(`Validating ${dictionaryFileName}...`);
const source = fs.readFileSync(dictionaryFileName);
const archive = await JSZip.loadAsync(source);
await validateDictionary(archive, schemas);
await validateDictionary(mode, archive, schemas);
const end = performance.now();
console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
} catch (e) {
Expand All @@ -110,12 +121,18 @@ async function main() {
if (dictionaryFileNames.length === 0) {
console.log([
'Usage:',
' node dictionary-validate <dictionary-file-names>...'
' node dictionary-validate [--ajv] <dictionary-file-names>...'
].join('\n'));
return;
}

await testDictionaryFiles(dictionaryFileNames);
let mode = null;
if (dictionaryFileNames[0] === '--ajv') {
mode = 'ajv';
dictionaryFileNames.splice(0, 1);
}

await testDictionaryFiles(mode, dictionaryFileNames);
}


Expand Down
41 changes: 39 additions & 2 deletions dev/schema-validate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,49 @@ vm.execute([
]);
const JsonSchema = vm.get('JsonSchema');

class JsonSchemaAjv {
constructor(schema) {
const Ajv = require('ajv');
const ajv = new Ajv({
meta: false,
strictTuples: false,
allowUnionTypes: true
});
ajv.addMetaSchema(require('ajv/dist/refs/json-schema-draft-07.json'));
this._validate = ajv.compile(schema);
}

validate(data) {
if (this._validate(data)) { return; }
const {errors} = this._validate(data);
const message = errors.map((e) => e.toString()).join('\n');
throw new Error(message);
}
}

function createJsonSchema(mode, schema) {
switch (mode) {
case 'ajv': return new JsonSchemaAjv(schema);
default: return new JsonSchema(schema);
}
}

function main() {
const args = process.argv.slice(2);
if (args.length < 2) {
console.log([
'Usage:',
' node schema-validate <schema-file-name> <data-file-names>...'
' node schema-validate [--ajv] <schema-file-name> <data-file-names>...'
].join('\n'));
return;
}

let mode = null;
if (args[0] === '--ajv') {
mode = 'ajv';
args.splice(0, 1);
}

const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'});
const schema = JSON.parse(schemaSource);

Expand All @@ -47,7 +79,7 @@ function main() {
console.log(`Validating ${dataFileName}...`);
const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'});
const data = JSON.parse(dataSource);
new JsonSchema(schema).validate(data);
createJsonSchema(mode, schema).validate(data);
const end = performance.now();
console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
} catch (e) {
Expand All @@ -60,3 +92,8 @@ function main() {


if (require.main === module) { main(); }


module.exports = {
createJsonSchema
};
Loading

0 comments on commit 5a72303

Please sign in to comment.