From c80d5a0960b1b9297810f9689c9c3791e06abb98 Mon Sep 17 00:00:00 2001 From: shnyder Date: Thu, 28 Jan 2021 02:14:10 +0100 Subject: [PATCH 1/8] first draft for PR --- src/cli/args.ts | 6 ++++++ src/cli/internal/main.ts | 13 ++++++++++--- src/triples/reader.ts | 28 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/cli/args.ts b/src/cli/args.ts index ff1295f..920cba5 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -19,6 +19,7 @@ import {ArgumentParser} from 'argparse'; export interface Options { /** HTTPS URL to an .nt file defining a custom ontology. */ ontology: string; + file: string; verbose: boolean; deprecated: boolean; context: string; @@ -78,6 +79,11 @@ export function ParseFlags(args?: string[]): Options { metavar: 'https://url.to/schema.nt', dest: 'ontology', }); + parser.add_argument('--file', { + default: undefined, + help: 'file path to a .nt file, for using a local ontology file', + dest: 'file', + }); const deprecated = parser.add_mutually_exclusive_group({required: false}); deprecated.add_argument('--deprecated', { diff --git a/src/cli/internal/main.ts b/src/cli/internal/main.ts index 110d41f..226cd23 100644 --- a/src/cli/internal/main.ts +++ b/src/cli/internal/main.ts @@ -16,7 +16,7 @@ import {Log, SetOptions} from '../../logging'; import {WriteDeclarations} from '../../transform/transform'; -import {load} from '../../triples/reader'; +import {load, loadFile} from '../../triples/reader'; import {Context} from '../../ts/context'; import {ParseFlags} from '../args'; @@ -26,9 +26,16 @@ export async function main(args?: string[]) { SetOptions(options); const ontologyUrl = options.ontology; - Log(`Loading Ontology from URL: ${ontologyUrl}`); + const filePath = options.file; + let result = ''; + if (ontologyUrl && ontologyUrl.startsWith('https://')) { + Log(`Loading Ontology from URL: ${ontologyUrl}`); - const result = load(ontologyUrl); + result = load(ontologyUrl); + } else if (filePath) { + Log(`Loading Ontology from path: ${filePath}`); + result = loadFile(filePath); + } const context = Context.Parse(options.context); await WriteDeclarations(result, options.deprecated, context, write); } diff --git a/src/triples/reader.ts b/src/triples/reader.ts index 6ee5f29..764cb14 100644 --- a/src/triples/reader.ts +++ b/src/triples/reader.ts @@ -14,6 +14,8 @@ * limitations under the License. */ import https from 'https'; +import fs from 'fs'; +import readline from 'readline'; import {Observable, Subscriber, TeardownLogic} from 'rxjs'; import {Log} from '../logging'; @@ -75,6 +77,32 @@ export function load(url: string): Observable { }); } +/** + * does the same as load(), but for a local file + */ +export function loadFile(path: string): Observable { + return new Observable(subscriber => { + handleFile(path, subscriber); + }); +} + +function handleFile( + path: string, + subscriber: Subscriber +): TeardownLogic { + const rl = readline.createInterface({ + input: fs.createReadStream(path), + crlfDelay: Infinity, + }); + + rl.on('line', (line: string) => { + subscriber.next(line); + }); + rl.on('close', () => { + subscriber.complete(); + }); +} + function handleUrl(url: string, subscriber: Subscriber): TeardownLogic { https .get(url, response => { From fd6d647faa597eae8a33ab4ed8594222d6c9906c Mon Sep 17 00:00:00 2001 From: shnyder Date: Thu, 28 Jan 2021 02:21:28 +0100 Subject: [PATCH 2/8] fixed some bugs for ts --- src/cli/internal/main.ts | 6 +++++- src/triples/reader.ts | 14 +++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/cli/internal/main.ts b/src/cli/internal/main.ts index 226cd23..96366f3 100644 --- a/src/cli/internal/main.ts +++ b/src/cli/internal/main.ts @@ -27,7 +27,7 @@ export async function main(args?: string[]) { const ontologyUrl = options.ontology; const filePath = options.file; - let result = ''; + let result; if (ontologyUrl && ontologyUrl.startsWith('https://')) { Log(`Loading Ontology from URL: ${ontologyUrl}`); @@ -36,6 +36,10 @@ export async function main(args?: string[]) { Log(`Loading Ontology from path: ${filePath}`); result = loadFile(filePath); } + if (!result) { + Log(`Ontology could not be read`); + return; + } const context = Context.Parse(options.context); await WriteDeclarations(result, options.deprecated, context, write); } diff --git a/src/triples/reader.ts b/src/triples/reader.ts index 764cb14..afca9a5 100644 --- a/src/triples/reader.ts +++ b/src/triples/reader.ts @@ -95,10 +95,22 @@ function handleFile( crlfDelay: Infinity, }); + const data: string[] = []; + rl.on('line', (line: string) => { - subscriber.next(line); + data.push(line); }); + rl.on('close', () => { + try { + const triples = toTripleStrings(data); + for (const triple of process(triples)) { + subscriber.next(triple); + } + } catch (error) { + Log(`Caught Error on end: ${error}`); + subscriber.error(error); + } subscriber.complete(); }); } From f5eac31a2967a7a11310fdbe68f6ec3e072081ea Mon Sep 17 00:00:00 2001 From: shnyder Date: Thu, 28 Jan 2021 19:24:10 +0100 Subject: [PATCH 3/8] added requested changes --- src/cli/args.ts | 2 +- src/cli/internal/main.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/cli/args.ts b/src/cli/args.ts index 920cba5..1094ce7 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -19,7 +19,7 @@ import {ArgumentParser} from 'argparse'; export interface Options { /** HTTPS URL to an .nt file defining a custom ontology. */ ontology: string; - file: string; + file: string | undefined; verbose: boolean; deprecated: boolean; context: string; diff --git a/src/cli/internal/main.ts b/src/cli/internal/main.ts index 96366f3..d6d512d 100644 --- a/src/cli/internal/main.ts +++ b/src/cli/internal/main.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import {Observable} from 'rxjs'; +import {Triple} from '../..'; import {Log, SetOptions} from '../../logging'; import {WriteDeclarations} from '../../transform/transform'; import {load, loadFile} from '../../triples/reader'; @@ -27,14 +29,15 @@ export async function main(args?: string[]) { const ontologyUrl = options.ontology; const filePath = options.file; - let result; - if (ontologyUrl && ontologyUrl.startsWith('https://')) { - Log(`Loading Ontology from URL: ${ontologyUrl}`); + let result: Observable; - result = load(ontologyUrl); - } else if (filePath) { + if (filePath) { Log(`Loading Ontology from path: ${filePath}`); result = loadFile(filePath); + } else { + Log(`Loading Ontology from URL: ${ontologyUrl}`); + + result = load(ontologyUrl); } if (!result) { Log(`Ontology could not be read`); From efdc70b45b6c0fb235b651177bf6b9f1fc418119 Mon Sep 17 00:00:00 2001 From: shnyder Date: Sun, 7 Feb 2021 21:02:51 +0100 Subject: [PATCH 4/8] added arg-test --- test/cli/args_test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/cli/args_test.ts b/test/cli/args_test.ts index 4d7630a..808718f 100644 --- a/test/cli/args_test.ts +++ b/test/cli/args_test.ts @@ -24,6 +24,7 @@ describe('ParseFlags', () => { expect(options.context).toBe('https://schema.org'); expect(options.deprecated).toBe(true); expect(options.verbose).toBe(false); + expect(options.file).toBeUndefined(); expect(options.ontology).toBe( 'https://schema.org/version/latest/schemaorg-current-https.nt' @@ -37,6 +38,13 @@ describe('ParseFlags', () => { expect(options.ontology).toBe('https://google.com/foo'); }); + it('custom file', () => { + const options = ParseFlags(['--file', './ontology.nt'])!; + expect(options).not.toBeUndefined(); + + expect(options.file).toBe('./ontology.nt'); + }); + describe('deprecated fields', () => { let mockExit: jest.MockInstance< ReturnType, From 4ec13091b3047b54727948888b949d872fb7eef4 Mon Sep 17 00:00:00 2001 From: shnyder Date: Tue, 9 Feb 2021 01:48:21 +0100 Subject: [PATCH 5/8] first working version of test with fs-mocking --- test/triples/reader_test.ts | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/triples/reader_test.ts b/test/triples/reader_test.ts index 92845f4..7e92103 100644 --- a/test/triples/reader_test.ts +++ b/test/triples/reader_test.ts @@ -17,9 +17,10 @@ import {ClientRequest, IncomingMessage} from 'http'; import https from 'https'; import {toArray} from 'rxjs/operators'; -import {PassThrough, Writable} from 'stream'; +import {PassThrough, Readable, Writable} from 'stream'; -import {load} from '../../src/triples/reader'; +import fs from 'fs'; +import {load, loadFile} from '../../src/triples/reader'; import {Triple} from '../../src/triples/triple'; import {SchemaString, UrlNode} from '../../src/triples/types'; import {flush} from '../helpers/async'; @@ -434,6 +435,29 @@ describe('load', () => { ]); }); }); + describe('local file', () => { + let readStreamCreatorFn: jest.SpyInstance; + beforeEach(() => { + const mockFileLine = ` .\n`; + const mockedStream = Readable.from([mockFileLine]); + readStreamCreatorFn = jest + .spyOn(fs, 'createReadStream') + //@ts-ignore + .mockImplementation(path => mockedStream); + }); + it('loads a file from the correct path', async () => { + const mockFilePath = './ontology.nt'; + + const fileTriples = loadFile(mockFilePath).toPromise(); + + expect(readStreamCreatorFn).toBeCalledWith(mockFilePath); + await expect(fileTriples).resolves.toEqual({ + Subject: UrlNode.Parse('https://schema.org/Person'), + Predicate: UrlNode.Parse('https://schema.org/knowsAbout'), + Object: UrlNode.Parse('https://schema.org/Event')!, + }); + }); + }); }); }); From 57061cf6e89ed1ebcd1f78a4f9689fa409412286 Mon Sep 17 00:00:00 2001 From: shnyder Date: Sun, 7 Mar 2021 05:22:01 +0100 Subject: [PATCH 6/8] test for file syntax fail --- test/triples/reader_test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/triples/reader_test.ts b/test/triples/reader_test.ts index 7e92103..14cc8bf 100644 --- a/test/triples/reader_test.ts +++ b/test/triples/reader_test.ts @@ -445,6 +445,15 @@ describe('load', () => { //@ts-ignore .mockImplementation(path => mockedStream); }); + it('fails loading a file (containing .nt syntax errors)', async () => { + const failingMockPath = './bad-ontology.nt'; + const failingMockLine = ` failingMockedStream); + + const fileTriples = loadFile(failingMockPath).toPromise(); + await expect(fileTriples).rejects.toThrow('Unexpected'); + }); it('loads a file from the correct path', async () => { const mockFilePath = './ontology.nt'; From a0c9dd0eab1131ebd52eec196a83a25171df1e0e Mon Sep 17 00:00:00 2001 From: shnyder Date: Sun, 7 Mar 2021 07:07:05 +0100 Subject: [PATCH 7/8] tests for logging messages for filepath --- src/cli/internal/main.ts | 5 ---- test/cli/args_logmessages_test.ts | 49 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 test/cli/args_logmessages_test.ts diff --git a/src/cli/internal/main.ts b/src/cli/internal/main.ts index d6d512d..3781287 100644 --- a/src/cli/internal/main.ts +++ b/src/cli/internal/main.ts @@ -36,13 +36,8 @@ export async function main(args?: string[]) { result = loadFile(filePath); } else { Log(`Loading Ontology from URL: ${ontologyUrl}`); - result = load(ontologyUrl); } - if (!result) { - Log(`Ontology could not be read`); - return; - } const context = Context.Parse(options.context); await WriteDeclarations(result, options.deprecated, context, write); } diff --git a/test/cli/args_logmessages_test.ts b/test/cli/args_logmessages_test.ts new file mode 100644 index 0000000..ee932ef --- /dev/null +++ b/test/cli/args_logmessages_test.ts @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import {Readable} from 'stream'; +import fs from 'fs'; +import {main} from '../../src/cli/internal/main'; +import * as Logging from '../../src/logging'; +import * as Transform from '../../src/transform/transform'; + +describe('main Args logs', () => { + let readStreamCreatorFn: jest.SpyInstance; + beforeEach(() => { + const mockFileLine = ` .\n`; + const mockedStream = Readable.from([mockFileLine]); + readStreamCreatorFn = jest + .spyOn(fs, 'createReadStream') + //@ts-ignore + .mockImplementation(path => mockedStream); + }); + it(`the path it is loading from`, async () => { + let logs = ['']; + // log messages get caught for checking assert: + jest + .spyOn(Logging, 'Log') + .mockImplementation((msg: string) => void logs.push(msg)); + // but doesn't write the output .ts-file: + jest + .spyOn(Transform, 'WriteDeclarations') + .mockImplementation(async (...args) => {}); + await main(['--file', `ontology-file.nt`, `--verbose`]); + expect(logs.join('')).toMatchInlineSnapshot( + `"Loading Ontology from path: ontology-file.nt"` + ); + }); +}); From 69b31718e91068ff748d5f061db610b1da0ae2b7 Mon Sep 17 00:00:00 2001 From: shnyder Date: Sun, 7 Mar 2021 07:11:43 +0100 Subject: [PATCH 8/8] let to const --- test/cli/args_logmessages_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/args_logmessages_test.ts b/test/cli/args_logmessages_test.ts index ee932ef..bbbe349 100644 --- a/test/cli/args_logmessages_test.ts +++ b/test/cli/args_logmessages_test.ts @@ -32,7 +32,7 @@ describe('main Args logs', () => { .mockImplementation(path => mockedStream); }); it(`the path it is loading from`, async () => { - let logs = ['']; + const logs = ['']; // log messages get caught for checking assert: jest .spyOn(Logging, 'Log')