diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index c4a2f0f..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: "{build} - {branch}" -skip_tags: true -skip_branch_with_pr: true - -environment: - matrix: - - nodejs_version: "9" - - nodejs_version: "8" - - nodejs_version: "7" - - nodejs_version: "6" - -install: - - ps: Install-Product node $env:nodejs_version - - npm install - -test_script: - - npm test - -build: off diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..cbd4d25 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": ["@smartive/eslint-config"] +} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..972e2cb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,23 @@ +name: Release npm package + +on: + push: + branches: + - master + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@main + - uses: actions/setup-node@v3 + with: + node-version: '18.x' + - run: npm install + - run: npm run build + - name: semantic release + uses: cycjimmy/semantic-release-action@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3887cf7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Unit Tests +on: + push: + branches: + - master + pull_request: + branches: [master, develop] +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [16.x, 18.x] + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Run Tests + run: | + npm install + npm test diff --git a/.gitignore b/.gitignore index 030065e..0ebe65c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ node_modules .node_repl_history # Typescript stuff -build +dist coverage package-lock.json diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..65c08dc --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1 @@ +"@smartive/prettier-config" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f831765..0000000 --- a/.travis.yml +++ /dev/null @@ -1,41 +0,0 @@ -language: node_js - -stages: - - name: test - if: tag IS blank - - name: deploy - if: branch = master AND type != pull_request - -notifications: - email: false - -jobs: - include: - - stage: test - node_js: '9' - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: test - node_js: '8' - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: test - node_js: '7' - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: test - node_js: '6' - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: deploy - node_js: '9' - script: npm run typedoc - deploy: - provider: pages - skip_cleanup: true - github_token: $GH_TOKEN - local_dir: ./docs - - stage: deploy - node_js: '9' - before_script: npm run build - script: npm run semantic-release diff --git a/config/tsconfig.base.json b/config/tsconfig.base.json index 9b2c17d..c54e0e2 100644 --- a/config/tsconfig.base.json +++ b/config/tsconfig.base.json @@ -4,7 +4,7 @@ "module": "commonjs", "moduleResolution": "node", "removeComments": false, - "outDir": "../build", + "outDir": "../dist", "rootDir": "../src", "declaration": true, "sourceMap": false, @@ -24,6 +24,6 @@ ], "exclude": [ "../node_modules", - "../build" + "../dist" ] } diff --git a/config/tsconfig.build.json b/config/tsconfig.build.json index 4bd0edc..3ee7a6c 100644 --- a/config/tsconfig.build.json +++ b/config/tsconfig.build.json @@ -1,6 +1,3 @@ { - "extends": "./tsconfig.base.json", - "compilerOptions": { - "outDir": "../" - } + "extends": "./tsconfig.base.json" } diff --git a/jest.json b/jest.json index 8e751c3..a33ba5c 100644 --- a/jest.json +++ b/jest.json @@ -1,19 +1,10 @@ { - "collectCoverage": true, - "mapCoverage": true, - "transform": { - "^.+\\.tsx?$": "/node_modules/ts-jest/preprocessor.js" - }, - "testMatch": [ - "**/test/**/*.spec.ts" - ], - "testPathIgnorePatterns": [ - "/node_modules/" - ], - "moduleFileExtensions": [ - "ts", - "tsx", - "js", - "json" - ] + "collectCoverage": true, + "mapCoverage": true, + "transform": { + "^.+\\.tsx?$": "ts-jest" + }, + "testMatch": ["**/test/**/*.spec.ts"], + "testPathIgnorePatterns": ["/node_modules/"], + "moduleFileExtensions": ["ts", "tsx", "js", "json"] } diff --git a/package.json b/package.json index d8a9e98..cca3527 100644 --- a/package.json +++ b/package.json @@ -2,17 +2,20 @@ "name": "proc-that-rest-extractor", "version": "0.0.0-development", "description": "extractor for proc-that which loads results from REST apis.", - "main": "index.js", - "typings": "index.d.js", + "main": "dist/index.js", + "typings": "dist/index.d.js", "scripts": { - "clean": "del-cli ./build ./coverage", + "clean": "del-cli ./dist ./coverage", "build": "npm run clean && tsc -p ./config/tsconfig.build.json", "develop": "npm run clean && tsc -p .", - "lint": "tslint -c ./tslint.json -p ./config/tsconfig.build.json", + "lint": "npm run lint:ts && npm run prettier", + "lint:fix": "npm run lint:ts:fix && npm run prettier:fix", + "lint:ts": "eslint --max-warnings=-1", + "lint:ts:fix": "eslint --max-warnings=-1 --fix", + "prettier": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\"", + "prettier:fix": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\" --write", "test": "npm run lint && npm run clean && jest -c ./jest.json", - "test:watch": "npm run clean && jest -c ./jest.json --watch", - "typedoc": "del-cli ./docs && typedoc --ignoreCompilerErrors --out ./docs --mode file --tsconfig ./config/tsconfig.build.json ./src/", - "semantic-release": "semantic-release" + "test:watch": "npm run clean && jest -c ./jest.json --watch" }, "keywords": [ "etl", @@ -22,7 +25,7 @@ "rest" ], "engines": { - "node": ">=6" + "node": ">=16.8" }, "repository": { "type": "git", @@ -32,22 +35,22 @@ "author": "Christoph Bühler ", "license": "MIT", "devDependencies": { - "@smartive/tslint-config": "^2.0.0", - "@types/jest": "^22.0.1", - "del-cli": "^1.1.0", - "jest": "^22.1.1", - "semantic-release": "^12.2.2", - "ts-jest": "^22.0.1", - "tslint": "^5.9.1", - "tsutils": "^2.18.0", - "typedoc": "^0.9.0", - "typescript": "^2.6.2" + "@smartive/eslint-config": "^3.1.1", + "@smartive/prettier-config": "^3.0.0", + "@types/jest": "^29.2.4", + "del-cli": "^5.0.0", + "eslint": "^8.30.0", + "jest": "^29.3.1", + "prettier": "^2.8.1", + "ts-jest": "^29.0.3", + "tsutils": "^3.21.0", + "typescript": "^4.9.4" }, "dependencies": { - "proc-that": "^1.0.2", - "restler": "^3.4.0", - "@types/node": "^9.3.0", - "rxjs": "^5.5.6", - "tslib": "^1.8.1" + "@types/node": "^18.11.17", + "proc-that": "^2.0.0", + "rxjs": "^7.8.0", + "tslib": "^2.4.1", + "undici": "^5.14.0" } } diff --git a/src/RestExtractor.ts b/src/RestExtractor.ts index 013c41b..60ba5b4 100644 --- a/src/RestExtractor.ts +++ b/src/RestExtractor.ts @@ -1,89 +1,38 @@ import { Extractor } from 'proc-that'; import { Observable, Observer } from 'rxjs'; +import { fetch } from 'undici'; -export enum RestExtractorMethod { - Get, - Post, - Put, -} - -type MethodOptions = { - method?: string; -}; - -export type RestExtractorOptions = { - query?: any; - data?: string | any; - parser?: any; - encoding?: string; - decoding?: string; - headers?: { [name: string]: string }; - username?: string; - password?: string; - accessToken?: string; - multipart?: any; - client?: any; - followRedirects?: boolean; - timeout?: number; - rejectUnauthorized?: boolean; - agent?: any; -}; - -/** - * - */ export class RestExtractor implements Extractor { - private rest: any = require('restler'); + constructor( + private url: string, + private resultSelector: (obj: any) => any = (o) => o, + private init: Parameters[1] = {} + ) {} - /** - * - * @param url - * @param method - * @param resultSelector - * @param {number} [timeout=120000] Request timeout in milliseconds - */ - constructor( - private url: string, - private method: RestExtractorMethod = RestExtractorMethod.Get, - private resultSelector: (obj: any) => any = o => o, - private restlerOptions: RestExtractorOptions = {}, - ) { } + public read(): Observable { + return new Observable((observer: Observer) => { + fetch(this.url, this.init) + .then(async (response) => { + try { + if (!response.ok) { + return observer.error(`Request failed with status ${response.status}: ${await response.text()}`); + } - public read(): Observable { - return Observable.create((observer: Observer) => { - const options: MethodOptions & RestExtractorOptions = this.restlerOptions; - options.method = this.getUrlMethod(); - this.rest - .request(this.url, options) - .on('error', (err) => { - observer.error(err); - }) - .on('complete', (result) => { - try { - let json = typeof result === 'string' ? JSON.parse(result) : result; - json = this.resultSelector(json); - if (json instanceof Array || json.constructor === Array) { - json.forEach(element => observer.next(element)); - } else { - observer.next(json); - } - } catch (e) { - observer.error(e); - } finally { - observer.complete(); - } - }); + const data = this.resultSelector(await response.json()); + if (data instanceof Array || data.constructor === Array) { + data.forEach((element) => observer.next(element)); + } else { + observer.next(data); + } + } catch (e) { + observer.error(e); + } finally { + observer.complete(); + } + }) + .catch((err) => { + observer.error(err); }); - } - - private getUrlMethod(): string { - switch (this.method) { - case RestExtractorMethod.Post: - return 'post'; - case RestExtractorMethod.Put: - return 'put'; - default: - return 'get'; - } - } + }); + } } diff --git a/test/RestExtractor.spec.ts b/test/RestExtractor.spec.ts index 50af857..75d40e2 100644 --- a/test/RestExtractor.spec.ts +++ b/test/RestExtractor.spec.ts @@ -1,196 +1,199 @@ -import { RestExtractor, RestExtractorMethod } from '../src'; -import { EventEmitter } from 'events'; - -abstract class Mock { - public request(url: string, opt: any): EventEmitter { - let emitter = new EventEmitter(); - setTimeout(() => { - try { - let data = this.data(); - emitter.emit('complete', data); - } catch (e) { - emitter.emit('error', e); - } - }, 100); - return emitter; - } - - protected abstract data(): any; -} - -class ArrayMock extends Mock { - protected data(): any { - return [ - { objId: 1 }, - { objId: 2 }, - { objId: 3 }, - { objId: 4 }, - { objId: 5 } - ]; - } -} - -class ObjectMock extends Mock { - protected data(): any { - return { objId: 5 }; - } -} - -class ErrorMock extends Mock { - protected data(): any { - throw new Error('test'); - } -} - -class StringMock extends Mock { - protected data(): any { - return '{"objId":5}'; - } -} - -class StringErrorMock extends Mock { - protected data(): any { - return '{objId:5}'; - } -} - -class SelectorMock extends Mock { - protected data(): any { - return { - data: [{ objId: 5 }] - }; - } -} +import { Interceptable, MockAgent, setGlobalDispatcher } from 'undici'; +import { RestExtractor } from '../src'; -describe('RestExtractor', () => { +const arrayMock = [{ objId: 1 }, { objId: 2 }, { objId: 3 }, { objId: 4 }, { objId: 5 }]; - let extractor: RestExtractor; +const objectMock = { objId: 5 }; - beforeEach(() => { - extractor = new RestExtractor('url'); - }); +const errorMock = new Error('test'); - it('should get an array of objects', done => { - let spy = jest.fn(); - (extractor as any).rest = new ArrayMock(); - - extractor - .read() - .subscribe(spy, err => { - done(err); - }, () => { - try { - expect(spy.mock.calls.length).toBe(5); - expect(spy.mock.calls[0][0]).toMatchSnapshot(); - done(); - } catch (e) { - done(e); - } - }); - }); +const stringMock = '{"objId":5}'; - it('should get single object', done => { - let spy = jest.fn(); - (extractor as any).rest = new ObjectMock(); - - extractor - .read() - .subscribe(spy, err => { - done(err); - }, () => { - try { - expect(spy.mock.calls.length).toBe(1); - expect(spy.mock.calls[0][0]).toMatchSnapshot(); - done(); - } catch (e) { - done(e); - } - }); - }); +const stringErrorMock = '{objId:5}'; - it('should call error', done => { - (extractor as any).rest = new ErrorMock(); +const selectorMock = { + data: [{ objId: 5 }], +}; - extractor - .read() - .subscribe(null, err => { - done(); - }, () => { - done(new Error('did not throw')); - }); +describe('RestExtractor', () => { + let extractor: RestExtractor; + let agent: MockAgent; + let mockPool: Interceptable; + + beforeEach(() => { + extractor = new RestExtractor('https://my.example.com'); + + agent = new MockAgent(); + agent.disableNetConnect(); + + setGlobalDispatcher(agent); + mockPool = agent.get('https://my.example.com'); + }); + + it('should get an array of objects', (done) => { + const spy = jest.fn(); + mockPool.intercept({ path: '/' }).reply(200, arrayMock); + + extractor.read().subscribe({ + next: spy, + error: (err) => { + done(err); + }, + complete: () => { + try { + expect(spy.mock.calls.length).toBe(5); + expect(spy.mock.calls[0][0]).toMatchSnapshot(); + done(); + } catch (e) { + done(e); + } + }, }); - - it('should parse string to an object', done => { - let spy = jest.fn(); - (extractor as any).rest = new StringMock(); - - extractor - .read() - .subscribe(spy, err => { - done(err); - }, () => { - try { - expect(spy.mock.calls.length).toBe(1); - expect(spy.mock.calls[0][0]).toMatchSnapshot(); - done(); - } catch (e) { - done(e); - } - }); + }); + + it('should get single object', (done) => { + const spy = jest.fn(); + mockPool.intercept({ path: '/' }).reply(200, objectMock); + + extractor.read().subscribe({ + next: spy, + error: (err) => { + done(err); + }, + complete: () => { + try { + expect(spy.mock.calls.length).toBe(1); + expect(spy.mock.calls[0][0]).toMatchSnapshot(); + done(); + } catch (e) { + done(e); + } + }, }); - - it('should call error on an invalid parse', done => { - (extractor as any).rest = new StringErrorMock(); - - extractor - .read() - .subscribe(null, err => { - done(); - }, () => { - done(new Error('did not throw')); - }); + }); + + it('should call error', (done) => { + mockPool.intercept({ path: '/' }).replyWithError(errorMock); + + extractor.read().subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error('did not throw')); + }, }); - - it('should use resultSelector correctly', done => { - let spy = jest.fn(); - extractor = new RestExtractor('url', RestExtractorMethod.Get, o => o.data); - (extractor as any).rest = new SelectorMock(); - - extractor - .read() - .subscribe(spy, err => { - done(err); - }, () => { - try { - expect(spy.mock.calls.length).toBe(1); - expect(spy.mock.calls[0][0]).toMatchSnapshot(); - done(); - } catch (e) { - done(e); - } - }); + }); + + it('should call error on 500 status code', (done) => { + mockPool.intercept({ path: '/' }).reply(500, 'server error'); + + extractor.read().subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error('did not throw')); + }, }); - - it('should set request timeout', done => { - extractor = new RestExtractor('url', RestExtractorMethod.Get, undefined, { - timeout: 42 - }); - (extractor as any).rest = new SelectorMock(); - - (extractor as any).rest.request = jest.fn((extractor as any).rest.request); - - extractor - .read() - .subscribe(() => { }, err => { - done(err); - }, () => { - try { - expect((extractor as any).rest.request.mock.calls[0]).toMatchSnapshot(); - done(); - } catch (e) { - done(e); - } - }); + }); + + it('should call error on 404 not found', (done) => { + mockPool.intercept({ path: '/' }).reply(404, 'not found'); + + extractor.read().subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error('did not throw')); + }, + }); + }); + + it('should parse string to an object', (done) => { + const spy = jest.fn(); + mockPool.intercept({ path: '/' }).reply(200, stringMock); + + extractor.read().subscribe({ + next: spy, + error: (err) => { + done(err); + }, + complete: () => { + try { + expect(spy.mock.calls.length).toBe(1); + expect(spy.mock.calls[0][0]).toMatchSnapshot(); + done(); + } catch (e) { + done(e); + } + }, + }); + }); + + it('should call error on an invalid parse', (done) => { + mockPool.intercept({ path: '/' }).reply(200, stringErrorMock); + + extractor.read().subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error('did not throw')); + }, + }); + }); + + it('should use resultSelector correctly', (done) => { + const spy = jest.fn(); + extractor = new RestExtractor('https://my.example.com/url', (o) => o.data); + mockPool.intercept({ path: '/url' }).reply(200, selectorMock); + + extractor.read().subscribe({ + next: spy, + error: (err) => { + done(err); + }, + complete: () => { + try { + expect(spy.mock.calls.length).toBe(1); + expect(spy.mock.calls[0][0]).toMatchSnapshot(); + done(); + } catch (e) { + done(e); + } + }, + }); + }); + + it('should set request options', (done) => { + const spy = jest.fn(); + extractor = new RestExtractor('https://my.example.com/url', undefined, { + headers: { + 'x-test': 'test', + authorization: 'Bearer 123', + }, + }); + mockPool.intercept({ path: '/url' }).reply(200, (req) => { + expect(req.headers).toMatchSnapshot(); + return selectorMock; }); + extractor.read().subscribe({ + next: spy, + error: (err) => { + done(err); + }, + complete: () => { + try { + expect(spy.mock.calls.length).toBe(1); + expect(spy.mock.calls[0][0]).toMatchSnapshot(); + done(); + } catch (e) { + done(e); + } + }, + }); + }); }); diff --git a/test/__snapshots__/RestExtractor.spec.ts.snap b/test/__snapshots__/RestExtractor.spec.ts.snap index f45d381..62c4e4e 100644 --- a/test/__snapshots__/RestExtractor.spec.ts.snap +++ b/test/__snapshots__/RestExtractor.spec.ts.snap @@ -1,35 +1,47 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`RestExtractor should get an array of objects 1`] = ` -Object { +{ "objId": 1, } `; exports[`RestExtractor should get single object 1`] = ` -Object { +{ "objId": 5, } `; exports[`RestExtractor should parse string to an object 1`] = ` -Object { +{ "objId": 5, } `; -exports[`RestExtractor should set request timeout 1`] = ` -Array [ - "url", - Object { - "method": "get", - "timeout": 42, - }, -] +exports[`RestExtractor should set request options 1`] = ` +{ + "accept": "*/*", + "accept-encoding": "br, gzip, deflate", + "accept-language": "*", + "authorization": "Bearer 123", + "sec-fetch-mode": "cors", + "user-agent": "undici", + "x-test": "test", +} +`; + +exports[`RestExtractor should set request options 2`] = ` +{ + "data": [ + { + "objId": 5, + }, + ], +} `; exports[`RestExtractor should use resultSelector correctly 1`] = ` -Object { +{ "objId": 5, } `; diff --git a/tslint.json b/tslint.json deleted file mode 100644 index d5c5f71..0000000 --- a/tslint.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "@smartive/tslint-config", - "rules": { - "ter-indent": [ - true, - 4, - { - "SwitchCase": 1 - } - ] - } -}