Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 0 additions & 19 deletions .appveyor.yml

This file was deleted.

3 changes: 3 additions & 0 deletions .eslintrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"extends": ["@smartive/eslint-config"]
}
23 changes: 23 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: Release npm package

on:
push:
branches:
- master

jobs:
release:
name: Release
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@main
- uses: actions/setup-node@v3
with:
node-version: '18.x'
- run: npm install
- run: npm run build
- name: semantic release
uses: cycjimmy/semantic-release-action@v3
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
23 changes: 23 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: Unit Tests
on:
push:
branches:
- master
pull_request:
branches: [master, develop]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [16.x, 18.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Run Tests
run: |
npm install
npm test
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ node_modules
.node_repl_history

# Typescript stuff
build
dist
coverage

package-lock.json
1 change: 1 addition & 0 deletions .prettierrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"@smartive/prettier-config"
41 changes: 0 additions & 41 deletions .travis.yml

This file was deleted.

4 changes: 2 additions & 2 deletions config/tsconfig.base.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"module": "commonjs",
"moduleResolution": "node",
"removeComments": false,
"outDir": "../build",
"outDir": "../dist",
"rootDir": "../src",
"declaration": true,
"sourceMap": false,
Expand All @@ -24,6 +24,6 @@
],
"exclude": [
"../node_modules",
"../build"
"../dist"
]
}
5 changes: 1 addition & 4 deletions config/tsconfig.build.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
{
"extends": "./tsconfig.base.json",
"compilerOptions": {
"outDir": "../"
}
"extends": "./tsconfig.base.json"
}
25 changes: 8 additions & 17 deletions jest.json
Original file line number Diff line number Diff line change
@@ -1,19 +1,10 @@
{
"collectCoverage": true,
"mapCoverage": true,
"transform": {
"^.+\\.tsx?$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
},
"testMatch": [
"**/test/**/*.spec.ts"
],
"testPathIgnorePatterns": [
"/node_modules/"
],
"moduleFileExtensions": [
"ts",
"tsx",
"js",
"json"
]
"collectCoverage": true,
"mapCoverage": true,
"transform": {
"^.+\\.tsx?$": "ts-jest"
},
"testMatch": ["**/test/**/*.spec.ts"],
"testPathIgnorePatterns": ["/node_modules/"],
"moduleFileExtensions": ["ts", "tsx", "js", "json"]
}
49 changes: 26 additions & 23 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,20 @@
"name": "proc-that-rest-extractor",
"version": "0.0.0-development",
"description": "extractor for proc-that which loads results from REST apis.",
"main": "index.js",
"typings": "index.d.js",
"main": "dist/index.js",
"typings": "dist/index.d.js",
"scripts": {
"clean": "del-cli ./build ./coverage",
"clean": "del-cli ./dist ./coverage",
"build": "npm run clean && tsc -p ./config/tsconfig.build.json",
"develop": "npm run clean && tsc -p .",
"lint": "tslint -c ./tslint.json -p ./config/tsconfig.build.json",
"lint": "npm run lint:ts && npm run prettier",
"lint:fix": "npm run lint:ts:fix && npm run prettier:fix",
"lint:ts": "eslint --max-warnings=-1",
"lint:ts:fix": "eslint --max-warnings=-1 --fix",
"prettier": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\"",
"prettier:fix": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\" --write",
"test": "npm run lint && npm run clean && jest -c ./jest.json",
"test:watch": "npm run clean && jest -c ./jest.json --watch",
"typedoc": "del-cli ./docs && typedoc --ignoreCompilerErrors --out ./docs --mode file --tsconfig ./config/tsconfig.build.json ./src/",
"semantic-release": "semantic-release"
"test:watch": "npm run clean && jest -c ./jest.json --watch"
},
"keywords": [
"etl",
Expand All @@ -22,7 +25,7 @@
"rest"
],
"engines": {
"node": ">=6"
"node": ">=16.8"
},
"repository": {
"type": "git",
Expand All @@ -32,22 +35,22 @@
"author": "Christoph Bühler <christoph.buehler@bluewin.ch>",
"license": "MIT",
"devDependencies": {
"@smartive/tslint-config": "^2.0.0",
"@types/jest": "^22.0.1",
"del-cli": "^1.1.0",
"jest": "^22.1.1",
"semantic-release": "^12.2.2",
"ts-jest": "^22.0.1",
"tslint": "^5.9.1",
"tsutils": "^2.18.0",
"typedoc": "^0.9.0",
"typescript": "^2.6.2"
"@smartive/eslint-config": "^3.1.1",
"@smartive/prettier-config": "^3.0.0",
"@types/jest": "^29.2.4",
"del-cli": "^5.0.0",
"eslint": "^8.30.0",
"jest": "^29.3.1",
"prettier": "^2.8.1",
"ts-jest": "^29.0.3",
"tsutils": "^3.21.0",
"typescript": "^4.9.4"
},
"dependencies": {
"proc-that": "^1.0.2",
"restler": "^3.4.0",
"@types/node": "^9.3.0",
"rxjs": "^5.5.6",
"tslib": "^1.8.1"
"@types/node": "^18.11.17",
"proc-that": "^2.0.0",
"rxjs": "^7.8.0",
"tslib": "^2.4.1",
"undici": "^5.14.0"
}
}
111 changes: 30 additions & 81 deletions src/RestExtractor.ts
Original file line number Diff line number Diff line change
@@ -1,89 +1,38 @@
import { Extractor } from 'proc-that';
import { Observable, Observer } from 'rxjs';
import { fetch } from 'undici';

export enum RestExtractorMethod {
Get,
Post,
Put,
}

type MethodOptions = {
method?: string;
};

export type RestExtractorOptions = {
query?: any;
data?: string | any;
parser?: any;
encoding?: string;
decoding?: string;
headers?: { [name: string]: string };
username?: string;
password?: string;
accessToken?: string;
multipart?: any;
client?: any;
followRedirects?: boolean;
timeout?: number;
rejectUnauthorized?: boolean;
agent?: any;
};

/**
*
*/
export class RestExtractor implements Extractor {
private rest: any = require('restler');
constructor(
private url: string,
private resultSelector: (obj: any) => any = (o) => o,
private init: Parameters<typeof fetch>[1] = {}
) {}

/**
*
* @param url
* @param method
* @param resultSelector
* @param {number} [timeout=120000] Request timeout in milliseconds
*/
constructor(
private url: string,
private method: RestExtractorMethod = RestExtractorMethod.Get,
private resultSelector: (obj: any) => any = o => o,
private restlerOptions: RestExtractorOptions = {},
) { }
public read(): Observable<any> {
return new Observable((observer: Observer<any>) => {
fetch(this.url, this.init)
.then(async (response) => {
try {
if (!response.ok) {
return observer.error(`Request failed with status ${response.status}: ${await response.text()}`);
}

public read(): Observable<any> {
return Observable.create((observer: Observer<any>) => {
const options: MethodOptions & RestExtractorOptions = this.restlerOptions;
options.method = this.getUrlMethod();
this.rest
.request(this.url, options)
.on('error', (err) => {
observer.error(err);
})
.on('complete', (result) => {
try {
let json = typeof result === 'string' ? JSON.parse(result) : result;
json = this.resultSelector(json);
if (json instanceof Array || json.constructor === Array) {
json.forEach(element => observer.next(element));
} else {
observer.next(json);
}
} catch (e) {
observer.error(e);
} finally {
observer.complete();
}
});
const data = this.resultSelector(await response.json());
if (data instanceof Array || data.constructor === Array) {
data.forEach((element) => observer.next(element));
} else {
observer.next(data);
}
} catch (e) {
observer.error(e);
} finally {
observer.complete();
}
})
.catch((err) => {
observer.error(err);
});
}

private getUrlMethod(): string {
switch (this.method) {
case RestExtractorMethod.Post:
return 'post';
case RestExtractorMethod.Put:
return 'put';
default:
return 'get';
}
}
});
}
}
Loading