Skip to content
This repository has been archived by the owner on Feb 25, 2023. It is now read-only.

Dictionary validate updates #2137

Merged
merged 4 commits into from
May 17, 2022
Merged

Dictionary validate updates #2137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1d7c4e
Reuse JsomSchema instance
toasted-nutbread May 16, 2022
18bc301
Install ajv
toasted-nutbread May 16, 2022
d7434b4
Add support for using ajv as a JSON schema validator
toasted-nutbread May 16, 2022
48cd698
Update usage
toasted-nutbread May 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 40 additions & 23 deletions dev/dictionary-validate.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,7 @@ const fs = require('fs');
const path = require('path');
const {performance} = require('perf_hooks');
const {JSZip} = require('./util');
const {VM} = require('./vm');

const vm = new VM();
vm.execute([
'js/core.js',
'js/general/cache-map.js',
'js/data/json-schema.js'
]);
const JsonSchema = vm.get('JsonSchema');
const {createJsonSchema} = require('./schema-validate');


function readSchema(relativeFileName) {
Expand All @@ -37,7 +29,14 @@ function readSchema(relativeFileName) {
}


async function validateDictionaryBanks(zip, fileNameFormat, schema) {
async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) {
let jsonSchema;
try {
jsonSchema = createJsonSchema(mode, schema);
} catch (e) {
e.message += `\n(in file ${fileNameFormat})}`;
throw e;
}
let index = 1;
while (true) {
const fileName = fileNameFormat.replace(/\?/, index);
Expand All @@ -46,28 +45,40 @@ async function validateDictionaryBanks(zip, fileNameFormat, schema) {
if (!file) { break; }

const data = JSON.parse(await file.async('string'));
new JsonSchema(schema).validate(data);
try {
jsonSchema.validate(data);
} catch (e) {
e.message += `\n(in file ${fileName})}`;
throw e;
}

++index;
}
}

async function validateDictionary(archive, schemas) {
const indexFile = archive.files['index.json'];
async function validateDictionary(mode, archive, schemas) {
const fileName = 'index.json';
const indexFile = archive.files[fileName];
if (!indexFile) {
throw new Error('No dictionary index found in archive');
}

const index = JSON.parse(await indexFile.async('string'));
const version = index.format || index.version;

new JsonSchema(schemas.index).validate(index);
try {
const jsonSchema = createJsonSchema(mode, schemas.index);
jsonSchema.validate(index);
} catch (e) {
e.message += `\n(in file ${fileName})}`;
throw e;
}

await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
await validateDictionaryBanks(archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
await validateDictionaryBanks(archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
await validateDictionaryBanks(archive, 'tag_bank_?.json', schemas.tagBankV3);
await validateDictionaryBanks(mode, archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
await validateDictionaryBanks(mode, archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
await validateDictionaryBanks(mode, archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
await validateDictionaryBanks(mode, archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3);
}

function getSchemas() {
Expand All @@ -84,7 +95,7 @@ function getSchemas() {
}


async function testDictionaryFiles(dictionaryFileNames) {
async function testDictionaryFiles(mode, dictionaryFileNames) {
const schemas = getSchemas();

for (const dictionaryFileName of dictionaryFileNames) {
Expand All @@ -93,7 +104,7 @@ async function testDictionaryFiles(dictionaryFileNames) {
console.log(`Validating ${dictionaryFileName}...`);
const source = fs.readFileSync(dictionaryFileName);
const archive = await JSZip.loadAsync(source);
await validateDictionary(archive, schemas);
await validateDictionary(mode, archive, schemas);
const end = performance.now();
console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
} catch (e) {
Expand All @@ -110,12 +121,18 @@ async function main() {
if (dictionaryFileNames.length === 0) {
console.log([
'Usage:',
' node dictionary-validate <dictionary-file-names>...'
' node dictionary-validate [--ajv] <dictionary-file-names>...'
].join('\n'));
return;
}

await testDictionaryFiles(dictionaryFileNames);
let mode = null;
if (dictionaryFileNames[0] === '--ajv') {
mode = 'ajv';
dictionaryFileNames.splice(0, 1);
}

await testDictionaryFiles(mode, dictionaryFileNames);
}


Expand Down
41 changes: 39 additions & 2 deletions dev/schema-validate.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,49 @@ vm.execute([
]);
const JsonSchema = vm.get('JsonSchema');

class JsonSchemaAjv {
constructor(schema) {
const Ajv = require('ajv');
const ajv = new Ajv({
meta: false,
strictTuples: false,
allowUnionTypes: true
});
ajv.addMetaSchema(require('ajv/dist/refs/json-schema-draft-07.json'));
this._validate = ajv.compile(schema);
}

validate(data) {
if (this._validate(data)) { return; }
const {errors} = this._validate(data);
const message = errors.map((e) => e.toString()).join('\n');
throw new Error(message);
}
}

function createJsonSchema(mode, schema) {
switch (mode) {
case 'ajv': return new JsonSchemaAjv(schema);
default: return new JsonSchema(schema);
}
}

function main() {
const args = process.argv.slice(2);
if (args.length < 2) {
console.log([
'Usage:',
' node schema-validate <schema-file-name> <data-file-names>...'
' node schema-validate [--ajv] <schema-file-name> <data-file-names>...'
].join('\n'));
return;
}

let mode = null;
if (args[0] === '--ajv') {
mode = 'ajv';
args.splice(0, 1);
}

const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'});
const schema = JSON.parse(schemaSource);

Expand All @@ -47,7 +79,7 @@ function main() {
console.log(`Validating ${dataFileName}...`);
const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'});
const data = JSON.parse(dataSource);
new JsonSchema(schema).validate(data);
createJsonSchema(mode, schema).validate(data);
const end = performance.now();
console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
} catch (e) {
Expand All @@ -60,3 +92,8 @@ function main() {


if (require.main === module) { main(); }


module.exports = {
createJsonSchema
};
Loading