diff --git a/README.md b/README.md index ee8d958..b2d4448 100755 --- a/README.md +++ b/README.md @@ -15,32 +15,63 @@ To read & normalize RSS/ATOM/JSON feed data. - [Give it a try!](https://demos.pwshub.com/feed-reader) - [Example FaaS](https://readfeed.deta.dev/?url=https://news.google.com/rss) -### Usage +## Install & Usage + +### Node.js + +```bash +npm i feed-reader + +# pnpm +pnpm i feed-reader + +# yarn +yarn add feed-reader +``` ```js +// es6 module import { read } from 'feed-reader' -// with CommonJS environments -// const { extract } = require('feed-reader') -// or specify exactly path to cjs variant -// const { read } = require('feed-reader/dist/cjs/feed-reader.js') +// CommonJS +const { read } = require('feed-reader') -const url = 'https://news.google.com/rss' +// or specify exactly path to CommonJS variant +const { read } = require('feed-reader/dist/cjs/feed-reader.js') +``` -read(url).then((feed) => { - console.log(feed) -}).catch((err) => { - console.log(err) -}) +### Deno + +```ts +import { read } from 'https://esm.sh/feed-reader' ``` +### Browser + +```js +import { read } from 'https://unpkg.com/feed-reader@latest/dist/feed-reader.esm.js' +``` + +Please check [the examples](https://github.com/ndaidong/feed-reader/tree/main/examples) for reference. + + ## APIs -### `read(String url [, Object options])` +### `read()` Load and extract feed data from given RSS/ATOM/JSON source. Return a Promise object. -#### `url` *required* +#### Syntax + +```js +read(String url) +read(String url, Object options) +read(String url, Object options, Object fetchOptions) +``` + +#### Parameters + +##### `url` *required* URL of a valid feed source @@ -50,7 +81,37 @@ Feed content must be accessible and conform one of the following standards: - [ATOM Feed](https://datatracker.ietf.org/doc/html/rfc5023) - [JSON Feed](https://www.jsonfeed.org/version/1.1/) -#### `options` *optional* +For example: + +```js +import { read } from 'feed-reader' + +read('https://news.google.com/atom').then(result => console.log(result)) +``` + +Without any options, the result should have the following structure: + +```js +{ + title: String, + link: String, + description: String, + generator: String, + language: String, + published: ISO Date String, + entries: Array[ + { + title: String, + link: String, + description: String, + published: ISO Datetime String + }, + // ... + ] +} +``` + +##### `options` *optional* Object with all or several of the following properties: @@ -62,67 +123,63 @@ Object with all or several of the following properties: Note that when `normalization` is set to `false`, other options will take no effect to the last output. - -Example: +For example: ```js -import { - read -} from 'feed-reader' - -const getFeedData = async (url) => { - try { - console.log(`Get feed data from ${url}`) - const result = await read(url) - // result may be feed data or null - console.log(result) - return result - } catch (err) { - console.trace(err) - } -} +import { read } from 'feed-reader' + +read('https://news.google.com/atom', { + useISODateFormat: false +}) -getFeedData('https://news.google.com/rss') -getFeedData('https://news.google.com/atom') -getFeedData('https://adactio.com/journal/feed.json') +read('https://news.google.com/rss', { + useISODateFormat: false, + includeOptionalElements: true +}) ``` -### Deno +##### `fetchOptions` *optional* -```ts -import { read } from 'https://esm.sh/feed-reader' +You can use this param to set request headers to fetch. + +For example: + +```js +import { read } from 'feed-reader' -(async () => { - const data = await read('https://news.google.com/rss') - console.log(data) -})(); +const url = 'https://news.google.com/rss' +read(url, null, { + headers: { + 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' + } +}) ``` -View [more examples](https://github.com/ndaidong/feed-reader/tree/main/examples). +You can also specify a proxy endpoint to load remote content, instead of fetching directly. +For example: -With default options, feed data object retuned by `read()` method should look like below: +```js +import { read } from 'feed-reader' -```json -{ - "title": "Top stories - Google News", - "link": "https://news.google.com/atom?hl=en-US&gl=US&ceid=US%3Aen", - "description": "Google News", - "generator": "NFE/5.0", - "language": "", - "published": "2021-12-23T15:01:12.000Z", - "entries": [ - { - "title": "Lone suspect in Waukesha parade crash to appear in court today, as Wisconsin reels from tragedy that left 5 dead and dozens more injured - CNN", - "link": "https://news.google.com/__i/rss/rd/articles/CBMiTmh0dHBzOi8vd3d3LmNubi5jb20vMjAyMS8xMS8yMy91cy93YXVrZXNoYS1jYXItcGFyYWRlLWNyb3dkLXR1ZXNkYXkvaW5kZXguaHRtbNIBUmh0dHBzOi8vYW1wLmNubi5jb20vY25uLzIwMjEvMTEvMjMvdXMvd2F1a2VzaGEtY2FyLXBhcmFkZS1jcm93ZC10dWVzZGF5L2luZGV4Lmh0bWw?oc=5", - "description": "Lone suspect in Waukesha parade crash to appear in court today, as Wisconsin reels from tragedy that left 5 dead and dozens more injured    CNN Waukesha Christmas parade attack: 5 dead, 48 injured, Darrell Brooks named as...", - "published": "2021-12-21T22:30:00.000Z" - }, - // ... - ] -} +const url = 'https://news.google.com/rss' + +read(url, null, { + headers: { + 'user-agent': 'Opera/9.60 (Windows NT 6.0; U; en) Presto/2.1.1' + }, + proxy: { + target: 'https://your-secret-proxy.io/loadXml?url=', + headers: { + 'Proxy-Authorization': 'Bearer YWxhZGRpbjpvcGVuc2VzYW1l...' + } + } +}) ``` +Passing requests to proxy is useful while running `feed-reader` on browser. View `examples/browser-feed-reader` as reference example. + + ## Quick evaluation ```bash @@ -133,7 +190,6 @@ npm install node eval.js --url=https://news.google.com/rss --normalization=y --useISODateFormat=y --includeEntryContent=n --includeOptionalElements=n ``` - ## License The MIT License (MIT) diff --git a/dist/cjs/feed-reader.js b/dist/cjs/feed-reader.js index 88ac976..3f1c1c0 100644 --- a/dist/cjs/feed-reader.js +++ b/dist/cjs/feed-reader.js @@ -1,4 +1,4 @@ -// feed-reader@6.1.0, by @ndaidong - built with esbuild at 2022-09-20T06:33:56.898Z - published under MIT license +// feed-reader@6.1.1, by @ndaidong - built with esbuild at 2022-09-22T06:03:19.517Z - published under MIT license var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; @@ -4748,12 +4748,24 @@ var purify = (url) => { // src/utils/retrieve.js var import_cross_fetch = __toESM(require_node_ponyfill(), 1); -var retrieve_default = async (url) => { - const res = await (0, import_cross_fetch.default)(url, { - headers: { - "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0" - } +var profetch = async (url, proxy = {}) => { + const { + target, + headers = {} + } = proxy; + const res = await (0, import_cross_fetch.default)(target + encodeURIComponent(url), { + headers }); + return res; +}; +var retrieve_default = async (url, options = {}) => { + const { + headers = { + "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0" + }, + proxy = null + } = options; + const res = proxy ? await profetch(url, proxy) : await (0, import_cross_fetch.default)(url, { headers }); const status = res.status; if (status >= 400) { throw new Error(`Request failed with error code ${status}`); @@ -5171,11 +5183,11 @@ var parseAtomFeed_default = (data, options = {}) => { }; // src/main.js -var read = async (url, options = {}) => { +var read = async (url, options = {}, fetchOptions = {}) => { if (!isValid(url)) { throw new Error("Input param must be a valid URL"); } - const data = await retrieve_default(url); + const data = await retrieve_default(url, fetchOptions); if (!data.text && !data.json) { throw new Error(`Failed to load content from "${url}"`); } diff --git a/dist/cjs/index.d.ts b/dist/cjs/index.d.ts index f3c0eef..dbbd3a0 100755 --- a/dist/cjs/index.d.ts +++ b/dist/cjs/index.d.ts @@ -1,20 +1,25 @@ // Type definitions export interface FeedEntry { - link?: string; - title?: string; - description?: string; - published?: Date; + link?: string; + title?: string; + description?: string; + published?: Date; } export interface FeedData { - link?: string; - title?: string; - description?: string; - generator?: string; - language?: string; - published?: Date; - entries?: Array; + link?: string; + title?: string; + description?: string; + generator?: string; + language?: string; + published?: Date; + entries?: Array; +} + +export interface ProxyConfig { + target?: string; + headers?: string[]; } export interface ReaderOptions { @@ -22,27 +27,40 @@ export interface ReaderOptions { * normalize feed data or keep original * default: true */ - normalization?: Boolean; + normalization?: boolean; /** * include full content of feed entry if present * default: false */ - includeEntryContent?: Boolean; + includeEntryContent?: boolean; /** * include optional elements if any * default: false */ - includeOptionalElements?: Boolean; + includeOptionalElements?: boolean; /** * convert datetime to ISO format * default: true */ - useISODateFormat?: Boolean; + useISODateFormat?: boolean; /** * to truncate description * default: 210 */ - descriptionMaxLen?: number; + descriptionMaxLen?: number; +} + +export interface FetchOptions { + /** + * list of request headers + * default: null + */ + headers?: string[]; + /** + * the values to configure proxy + * default: null + */ + proxy?: ProxyConfig; } -export function read(url: string, options?: ReaderOptions): Promise; +export function read(url: string, options?: ReaderOptions, fetchOptions?: FetchOptions): Promise; diff --git a/dist/cjs/package.json b/dist/cjs/package.json index fef40b4..8d50468 100644 --- a/dist/cjs/package.json +++ b/dist/cjs/package.json @@ -1,5 +1,5 @@ { "name": "feed-reader", - "version": "6.1.0", + "version": "6.1.1", "main": "./feed-reader.js" } \ No newline at end of file diff --git a/dist/feed-reader.esm.js b/dist/feed-reader.esm.js index 97ef467..8be5d92 100644 --- a/dist/feed-reader.esm.js +++ b/dist/feed-reader.esm.js @@ -1,4 +1,4 @@ -// feed-reader@6.1.0, by @ndaidong - built with esbuild at 2022-09-20T06:33:56.898Z - published under MIT license +// feed-reader@6.1.1, by @ndaidong - built with esbuild at 2022-09-22T06:03:19.517Z - published under MIT license var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; @@ -1869,12 +1869,24 @@ var purify = (url) => { var cross_fetch_default = fetch; // src/utils/retrieve.js -var retrieve_default = async (url) => { - const res = await cross_fetch_default(url, { - headers: { - "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0" - } +var profetch = async (url, proxy = {}) => { + const { + target, + headers = {} + } = proxy; + const res = await cross_fetch_default(target + encodeURIComponent(url), { + headers }); + return res; +}; +var retrieve_default = async (url, options = {}) => { + const { + headers = { + "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0" + }, + proxy = null + } = options; + const res = proxy ? await profetch(url, proxy) : await cross_fetch_default(url, { headers }); const status = res.status; if (status >= 400) { throw new Error(`Request failed with error code ${status}`); @@ -2292,11 +2304,11 @@ var parseAtomFeed_default = (data, options = {}) => { }; // src/main.js -var read = async (url, options = {}) => { +var read = async (url, options = {}, fetchOptions = {}) => { if (!isValid(url)) { throw new Error("Input param must be a valid URL"); } - const data = await retrieve_default(url); + const data = await retrieve_default(url, fetchOptions); if (!data.text && !data.json) { throw new Error(`Failed to load content from "${url}"`); } diff --git a/examples/browser-feed-reader/.gitignore b/examples/browser-feed-reader/.gitignore new file mode 100644 index 0000000..48012a9 --- /dev/null +++ b/examples/browser-feed-reader/.gitignore @@ -0,0 +1,17 @@ +# Logs +logs +*.log +*.debug + +# Runtime data +*.pid +*.seed + +node_modules +coverage +.nyc_output + +.DS_Store +yarn.lock +coverage.lcov +pnpm-lock.yaml diff --git a/examples/browser-feed-reader/README.md b/examples/browser-feed-reader/README.md new file mode 100644 index 0000000..6364796 --- /dev/null +++ b/examples/browser-feed-reader/README.md @@ -0,0 +1,30 @@ +# browser-feed-reader + +This demo shows how to use feed-reader at client side, with or without proxy. + +To install: + +```bash +npm i + +# or pnpm, yarn +``` + +Start server: + +```bash +npm start +``` + +Open `http://localhost:3103/` to test. + +Basically `feed-reader` only works at server side. + +However there are some noble publishers those enable `Access-Control-Allow-Origin` on their service. +For example with feed resource from [Washington Post](https://feeds.washingtonpost.com/rss/business/technology), [Decrypt](https://decrypt.co/feed) or [FeedBlitz](https://feeds.feedblitz.com/baeldung/cs&x=1) we can read from browser. + +Another ideal environment to run `feed-reader` directly is browser extensions. + +With the remaining cases, we need a proxy layer to bypass CORS policy. + +--- diff --git a/examples/browser-feed-reader/package.json b/examples/browser-feed-reader/package.json new file mode 100644 index 0000000..bf5bda1 --- /dev/null +++ b/examples/browser-feed-reader/package.json @@ -0,0 +1,12 @@ +{ + "name": "browser-feed-reader", + "version": "1.0.0", + "type": "module", + "scripts": { + "start": "node server" + }, + "dependencies": { + "express": "^4.18.1", + "got": "^12.5.0" + } +} diff --git a/examples/browser-feed-reader/public/chota.min.css b/examples/browser-feed-reader/public/chota.min.css new file mode 100644 index 0000000..b445236 --- /dev/null +++ b/examples/browser-feed-reader/public/chota.min.css @@ -0,0 +1 @@ +/*! chota.css v0.7.2 | MIT License | github.com/jenil/chota */:root{--bg-color:#fff;--bg-secondary-color:#f3f3f6;--color-primary:#14854f;--color-lightGrey:#d2d6dd;--color-grey:#747681;--color-darkGrey:#3f4144;--color-error:#d43939;--color-success:#28bd14;--grid-maxWidth:120rem;--grid-gutter:2rem;--font-size:1.6rem;--font-color:#333;--font-family-sans:-apple-system,BlinkMacSystemFont,Avenir,"Avenir Next","Segoe UI","Roboto","Oxygen","Ubuntu","Cantarell","Fira Sans","Droid Sans","Helvetica Neue",sans-serif;--font-family-mono:monaco,"Consolas","Lucida Console",monospace}html{-webkit-box-sizing:border-box;box-sizing:border-box;font-size:62.5%;line-height:1.15;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}*,:after,:before{-webkit-box-sizing:inherit;box-sizing:inherit}body{background-color:var(--bg-color);line-height:1.6;font-size:var(--font-size);color:var(--font-color);font-family:Segoe UI,Helvetica Neue,sans-serif;font-family:var(--font-family-sans);margin:0;padding:0}h1,h2,h3,h4,h5,h6{font-weight:500;margin:.35em 0 .7em}h1{font-size:2em}h2{font-size:1.75em}h3{font-size:1.5em}h4{font-size:1.25em}h5{font-size:1em}h6{font-size:.85em}a{color:var(--color-primary);text-decoration:none}a:hover:not(.button){opacity:.75}button{font-family:inherit}p{margin-top:0}blockquote{background-color:var(--bg-secondary-color);padding:1.5rem 2rem;border-left:3px solid var(--color-lightGrey)}dl dt{font-weight:700}hr{background-color:var(--color-lightGrey);height:1px;margin:1rem 0}hr,table{border:none}table{width:100%;border-collapse:collapse;border-spacing:0;text-align:left}table.striped tr:nth-of-type(2n){background-color:var(--bg-secondary-color)}td,th{vertical-align:middle;padding:1.2rem .4rem}thead{border-bottom:2px solid var(--color-lightGrey)}tfoot{border-top:2px solid var(--color-lightGrey)}code,kbd,pre,samp,tt{font-family:var(--font-family-mono)}code,kbd{font-size:90%;white-space:pre-wrap;border-radius:4px;padding:.2em .4em;color:var(--color-error)}code,kbd,pre{background-color:var(--bg-secondary-color)}pre{font-size:1em;padding:1rem;overflow-x:auto}pre code{background:none;padding:0}abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted}img{max-width:100%}fieldset{border:1px solid var(--color-lightGrey)}iframe{border:0}.container{max-width:var(--grid-maxWidth);margin:0 auto;width:96%;padding:0 calc(var(--grid-gutter)/2)}.row{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start;margin-left:calc(var(--grid-gutter)/-2);margin-right:calc(var(--grid-gutter)/-2)}.row,.row.reverse{-webkit-box-orient:horizontal}.row.reverse{-webkit-box-direction:reverse;-ms-flex-direction:row-reverse;flex-direction:row-reverse}.col{-webkit-box-flex:1;-ms-flex:1;flex:1}.col,[class*=" col-"],[class^=col-]{margin:0 calc(var(--grid-gutter)/2) calc(var(--grid-gutter)/2)}.col-1{-ms-flex:0 0 calc(8.33333% - var(--grid-gutter));flex:0 0 calc(8.33333% - var(--grid-gutter));max-width:calc(8.33333% - var(--grid-gutter))}.col-1,.col-2{-webkit-box-flex:0}.col-2{-ms-flex:0 0 calc(16.66667% - var(--grid-gutter));flex:0 0 calc(16.66667% - var(--grid-gutter));max-width:calc(16.66667% - var(--grid-gutter))}.col-3{-ms-flex:0 0 calc(25% - var(--grid-gutter));flex:0 0 calc(25% - var(--grid-gutter));max-width:calc(25% - var(--grid-gutter))}.col-3,.col-4{-webkit-box-flex:0}.col-4{-ms-flex:0 0 calc(33.33333% - var(--grid-gutter));flex:0 0 calc(33.33333% - var(--grid-gutter));max-width:calc(33.33333% - var(--grid-gutter))}.col-5{-ms-flex:0 0 calc(41.66667% - var(--grid-gutter));flex:0 0 calc(41.66667% - var(--grid-gutter));max-width:calc(41.66667% - var(--grid-gutter))}.col-5,.col-6{-webkit-box-flex:0}.col-6{-ms-flex:0 0 calc(50% - var(--grid-gutter));flex:0 0 calc(50% - var(--grid-gutter));max-width:calc(50% - var(--grid-gutter))}.col-7{-ms-flex:0 0 calc(58.33333% - var(--grid-gutter));flex:0 0 calc(58.33333% - var(--grid-gutter));max-width:calc(58.33333% - var(--grid-gutter))}.col-7,.col-8{-webkit-box-flex:0}.col-8{-ms-flex:0 0 calc(66.66667% - var(--grid-gutter));flex:0 0 calc(66.66667% - var(--grid-gutter));max-width:calc(66.66667% - var(--grid-gutter))}.col-9{-ms-flex:0 0 calc(75% - var(--grid-gutter));flex:0 0 calc(75% - var(--grid-gutter));max-width:calc(75% - var(--grid-gutter))}.col-9,.col-10{-webkit-box-flex:0}.col-10{-ms-flex:0 0 calc(83.33333% - var(--grid-gutter));flex:0 0 calc(83.33333% - var(--grid-gutter));max-width:calc(83.33333% - var(--grid-gutter))}.col-11{-ms-flex:0 0 calc(91.66667% - var(--grid-gutter));flex:0 0 calc(91.66667% - var(--grid-gutter));max-width:calc(91.66667% - var(--grid-gutter))}.col-11,.col-12{-webkit-box-flex:0}.col-12{-ms-flex:0 0 calc(100% - var(--grid-gutter));flex:0 0 calc(100% - var(--grid-gutter));max-width:calc(100% - var(--grid-gutter))}@media screen and (max-width:599px){.container{width:100%}.col,[class*=col-],[class^=col-]{-webkit-box-flex:0;-ms-flex:0 1 100%;flex:0 1 100%;max-width:100%}}@media screen and (min-width:900px){.col-1-md{-webkit-box-flex:0;-ms-flex:0 0 calc(8.33333% - var(--grid-gutter));flex:0 0 calc(8.33333% - var(--grid-gutter));max-width:calc(8.33333% - var(--grid-gutter))}.col-2-md{-webkit-box-flex:0;-ms-flex:0 0 calc(16.66667% - var(--grid-gutter));flex:0 0 calc(16.66667% - var(--grid-gutter));max-width:calc(16.66667% - var(--grid-gutter))}.col-3-md{-webkit-box-flex:0;-ms-flex:0 0 calc(25% - var(--grid-gutter));flex:0 0 calc(25% - var(--grid-gutter));max-width:calc(25% - var(--grid-gutter))}.col-4-md{-webkit-box-flex:0;-ms-flex:0 0 calc(33.33333% - var(--grid-gutter));flex:0 0 calc(33.33333% - var(--grid-gutter));max-width:calc(33.33333% - var(--grid-gutter))}.col-5-md{-webkit-box-flex:0;-ms-flex:0 0 calc(41.66667% - var(--grid-gutter));flex:0 0 calc(41.66667% - var(--grid-gutter));max-width:calc(41.66667% - var(--grid-gutter))}.col-6-md{-webkit-box-flex:0;-ms-flex:0 0 calc(50% - var(--grid-gutter));flex:0 0 calc(50% - var(--grid-gutter));max-width:calc(50% - var(--grid-gutter))}.col-7-md{-webkit-box-flex:0;-ms-flex:0 0 calc(58.33333% - var(--grid-gutter));flex:0 0 calc(58.33333% - var(--grid-gutter));max-width:calc(58.33333% - var(--grid-gutter))}.col-8-md{-webkit-box-flex:0;-ms-flex:0 0 calc(66.66667% - var(--grid-gutter));flex:0 0 calc(66.66667% - var(--grid-gutter));max-width:calc(66.66667% - var(--grid-gutter))}.col-9-md{-webkit-box-flex:0;-ms-flex:0 0 calc(75% - var(--grid-gutter));flex:0 0 calc(75% - var(--grid-gutter));max-width:calc(75% - var(--grid-gutter))}.col-10-md{-webkit-box-flex:0;-ms-flex:0 0 calc(83.33333% - var(--grid-gutter));flex:0 0 calc(83.33333% - var(--grid-gutter));max-width:calc(83.33333% - var(--grid-gutter))}.col-11-md{-webkit-box-flex:0;-ms-flex:0 0 calc(91.66667% - var(--grid-gutter));flex:0 0 calc(91.66667% - var(--grid-gutter));max-width:calc(91.66667% - var(--grid-gutter))}.col-12-md{-webkit-box-flex:0;-ms-flex:0 0 calc(100% - var(--grid-gutter));flex:0 0 calc(100% - var(--grid-gutter));max-width:calc(100% - var(--grid-gutter))}}@media screen and (min-width:1200px){.col-1-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(8.33333% - var(--grid-gutter));flex:0 0 calc(8.33333% - var(--grid-gutter));max-width:calc(8.33333% - var(--grid-gutter))}.col-2-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(16.66667% - var(--grid-gutter));flex:0 0 calc(16.66667% - var(--grid-gutter));max-width:calc(16.66667% - var(--grid-gutter))}.col-3-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(25% - var(--grid-gutter));flex:0 0 calc(25% - var(--grid-gutter));max-width:calc(25% - var(--grid-gutter))}.col-4-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(33.33333% - var(--grid-gutter));flex:0 0 calc(33.33333% - var(--grid-gutter));max-width:calc(33.33333% - var(--grid-gutter))}.col-5-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(41.66667% - var(--grid-gutter));flex:0 0 calc(41.66667% - var(--grid-gutter));max-width:calc(41.66667% - var(--grid-gutter))}.col-6-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(50% - var(--grid-gutter));flex:0 0 calc(50% - var(--grid-gutter));max-width:calc(50% - var(--grid-gutter))}.col-7-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(58.33333% - var(--grid-gutter));flex:0 0 calc(58.33333% - var(--grid-gutter));max-width:calc(58.33333% - var(--grid-gutter))}.col-8-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(66.66667% - var(--grid-gutter));flex:0 0 calc(66.66667% - var(--grid-gutter));max-width:calc(66.66667% - var(--grid-gutter))}.col-9-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(75% - var(--grid-gutter));flex:0 0 calc(75% - var(--grid-gutter));max-width:calc(75% - var(--grid-gutter))}.col-10-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(83.33333% - var(--grid-gutter));flex:0 0 calc(83.33333% - var(--grid-gutter));max-width:calc(83.33333% - var(--grid-gutter))}.col-11-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(91.66667% - var(--grid-gutter));flex:0 0 calc(91.66667% - var(--grid-gutter));max-width:calc(91.66667% - var(--grid-gutter))}.col-12-lg{-webkit-box-flex:0;-ms-flex:0 0 calc(100% - var(--grid-gutter));flex:0 0 calc(100% - var(--grid-gutter));max-width:calc(100% - var(--grid-gutter))}}fieldset{padding:.5rem 2rem}legend{text-transform:uppercase;font-size:.8em;letter-spacing:.1rem}input:not([type=checkbox]):not([type=radio]):not([type=submit]):not([type=color]):not([type=button]):not([type=reset]),select,textarea,textarea[type=text]{font-family:inherit;padding:.8rem 1rem;border-radius:4px;border:1px solid var(--color-lightGrey);font-size:1em;-webkit-transition:all .2s ease;transition:all .2s ease;display:block;width:100%}input:not([type=checkbox]):not([type=radio]):not([type=submit]):not([type=color]):not([type=button]):not([type=reset]):not(:disabled):hover,select:hover,textarea:hover,textarea[type=text]:hover{border-color:var(--color-grey)}input:not([type=checkbox]):not([type=radio]):not([type=submit]):not([type=color]):not([type=button]):not([type=reset]):focus,select:focus,textarea:focus,textarea[type=text]:focus{outline:none;border-color:var(--color-primary);-webkit-box-shadow:0 0 1px var(--color-primary);box-shadow:0 0 1px var(--color-primary)}input.error:not([type=checkbox]):not([type=radio]):not([type=submit]):not([type=color]):not([type=button]):not([type=reset]),textarea.error{border-color:var(--color-error)}input.success:not([type=checkbox]):not([type=radio]):not([type=submit]):not([type=color]):not([type=button]):not([type=reset]),textarea.success{border-color:var(--color-success)}select{-webkit-appearance:none;background:#f3f3f6 no-repeat 100%;background-size:1ex;background-origin:content-box;background-image:url("data:image/svg+xml;utf8,")}[type=checkbox],[type=radio]{width:1.6rem;height:1.6rem}.button,[type=button],[type=reset],[type=submit],button{padding:1rem 2.5rem;color:var(--color-darkGrey);background:var(--color-lightGrey);border-radius:4px;border:1px solid transparent;font-size:var(--font-size);line-height:1;text-align:center;-webkit-transition:opacity .2s ease;transition:opacity .2s ease;text-decoration:none;-webkit-transform:scale(1);transform:scale(1);display:inline-block;cursor:pointer}.grouped{display:-webkit-box;display:-ms-flexbox;display:flex}.grouped>:not(:last-child){margin-right:16px}.grouped.gapless>*{margin:0 0 0 -1px!important;border-radius:0!important}.grouped.gapless>:first-child{margin:0!important;border-radius:4px 0 0 4px!important}.grouped.gapless>:last-child{border-radius:0 4px 4px 0!important}.button+.button{margin-left:1rem}.button:hover,[type=button]:hover,[type=reset]:hover,[type=submit]:hover,button:hover{opacity:.8}.button:active,[type=button]:active,[type=reset]:active,[type=submit]:active,button:active{-webkit-transform:scale(.98);transform:scale(.98)}button:disabled,button:disabled:hover,input:disabled,input:disabled:hover{opacity:.4;cursor:not-allowed}.button.dark,.button.error,.button.primary,.button.secondary,.button.success,[type=submit]{color:#fff;z-index:1;background-color:#000;background-color:var(--color-primary)}.button.secondary{background-color:var(--color-grey)}.button.dark{background-color:var(--color-darkGrey)}.button.error{background-color:var(--color-error)}.button.success{background-color:var(--color-success)}.button.outline{background-color:transparent;border-color:var(--color-lightGrey)}.button.outline.primary{border-color:var(--color-primary);color:var(--color-primary)}.button.outline.secondary{border-color:var(--color-grey);color:var(--color-grey)}.button.outline.dark{border-color:var(--color-darkGrey);color:var(--color-darkGrey)}.button.clear{background-color:transparent;border-color:transparent;color:var(--color-primary)}.button.icon{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.button.icon>img{margin-left:2px}.button.icon-only{padding:1rem}::-webkit-input-placeholder{color:#bdbfc4}::-moz-placeholder{color:#bdbfc4}:-ms-input-placeholder{color:#bdbfc4}::-ms-input-placeholder{color:#bdbfc4}::placeholder{color:#bdbfc4}.nav{display:-webkit-box;display:-ms-flexbox;display:flex;min-height:5rem;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch}.nav img{max-height:3rem}.nav-center,.nav-left,.nav-right,.nav>.container{display:-webkit-box;display:-ms-flexbox;display:flex}.nav-center,.nav-left,.nav-right{-webkit-box-flex:1;-ms-flex:1;flex:1}.nav-left{-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.nav-right{-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.nav-center{-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}@media screen and (max-width:480px){.nav,.nav>.container{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.nav-center,.nav-left,.nav-right{-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}}.nav .brand,.nav a{text-decoration:none;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding:1rem 2rem;color:var(--color-darkGrey)}.nav .active:not(.button){color:#000;color:var(--color-primary)}.nav .brand{font-size:1.75em;padding-top:0;padding-bottom:0}.nav .brand img{padding-right:1rem}.nav .button{margin:auto 1rem}.card{padding:1rem 2rem;border-radius:4px;background:var(--bg-color);-webkit-box-shadow:0 1px 3px var(--color-grey);box-shadow:0 1px 3px var(--color-grey)}.card p:last-child{margin:0}.card header>*{margin-top:0;margin-bottom:1rem}.tabs{display:-webkit-box;display:-ms-flexbox;display:flex}.tabs a{text-decoration:none}.tabs>.dropdown>summary,.tabs>a{padding:1rem 2rem;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;color:var(--color-darkGrey);border-bottom:2px solid var(--color-lightGrey);text-align:center}.tabs>a.active,.tabs>a:hover{opacity:1;border-bottom:2px solid var(--color-darkGrey)}.tabs>a.active{border-color:var(--color-primary)}.tabs.is-full a{-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto}.tag{display:inline-block;border:1px solid var(--color-lightGrey);text-transform:uppercase;color:var(--color-grey);padding:.5rem;line-height:1;letter-spacing:.5px}.tag.is-small{padding:.4rem;font-size:.75em}.tag.is-large{padding:.7rem;font-size:1.125em}.tag+.tag{margin-left:1rem}details.dropdown{position:relative;display:inline-block}details.dropdown>:last-child{position:absolute;left:0;white-space:nowrap}.bg-primary{background-color:var(--color-primary)!important}.bg-light{background-color:var(--color-lightGrey)!important}.bg-dark{background-color:var(--color-darkGrey)!important}.bg-grey{background-color:var(--color-grey)!important}.bg-error{background-color:var(--color-error)!important}.bg-success{background-color:var(--color-success)!important}.bd-primary{border:1px solid var(--color-primary)!important}.bd-light{border:1px solid var(--color-lightGrey)!important}.bd-dark{border:1px solid var(--color-darkGrey)!important}.bd-grey{border:1px solid var(--color-grey)!important}.bd-error{border:1px solid var(--color-error)!important}.bd-success{border:1px solid var(--color-success)!important}.text-primary{color:var(--color-primary)!important}.text-light{color:var(--color-lightGrey)!important}.text-dark{color:var(--color-darkGrey)!important}.text-grey{color:var(--color-grey)!important}.text-error{color:var(--color-error)!important}.text-success{color:var(--color-success)!important}.text-white{color:#fff!important}.pull-right{float:right!important}.pull-left{float:left!important}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.text-justify{text-align:justify}.text-uppercase{text-transform:uppercase}.text-lowercase{text-transform:lowercase}.text-capitalize{text-transform:capitalize}.is-full-screen{width:100%;min-height:100vh}.is-full-width{width:100%!important}.is-vertical-align{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.is-center,.is-horizontal-align{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.is-center{-webkit-box-align:center;-ms-flex-align:center;align-items:center}.is-right{-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.is-left,.is-right{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.is-left{-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.is-fixed{position:fixed;width:100%}.is-paddingless{padding:0!important}.is-marginless{margin:0!important}.is-pointer{cursor:pointer!important}.is-rounded{border-radius:100%}.clearfix{content:"";display:table;clear:both}.is-hidden{display:none!important}@media screen and (max-width:599px){.hide-xs{display:none!important}}@media screen and (min-width:600px) and (max-width:899px){.hide-sm{display:none!important}}@media screen and (min-width:900px) and (max-width:1199px){.hide-md{display:none!important}}@media screen and (min-width:1200px){.hide-lg{display:none!important}}@media print{.hide-pr{display:none!important}} \ No newline at end of file diff --git a/examples/browser-feed-reader/public/index.html b/examples/browser-feed-reader/public/index.html new file mode 100644 index 0000000..bc758a1 --- /dev/null +++ b/examples/browser-feed-reader/public/index.html @@ -0,0 +1,88 @@ + + + + Example feed-reader + + + + +
+
+
+

feed-reader on browser

+
+
+
+ enter feed url +

+ +

+

+ + +

+
+
+
+ Result +

+ +

+
+
+
+ + + diff --git a/examples/browser-feed-reader/server.js b/examples/browser-feed-reader/server.js new file mode 100644 index 0000000..b9d6bed --- /dev/null +++ b/examples/browser-feed-reader/server.js @@ -0,0 +1,30 @@ +// server + +import got from 'got' +import express from 'express' + +const app = express() + +const loadRemoteFeed = async (url) => { + try { + const headers = { + 'Accept-Charset': 'utf-8' + } + const data = await got(url, { headers }).text() + return data + } catch (err) { + return err.message + } +} + +app.get('/proxy/getxml', async (req, res) => { + const url = req.query.url + const xml = await loadRemoteFeed(url) + return res.type('text/xml').send(xml) +}) + +app.use(express.static('public')) + +app.listen(3103, () => { + console.log('Server is running at http://localhost:3103') +}) diff --git a/index.d.ts b/index.d.ts index f3c0eef..dbbd3a0 100755 --- a/index.d.ts +++ b/index.d.ts @@ -1,20 +1,25 @@ // Type definitions export interface FeedEntry { - link?: string; - title?: string; - description?: string; - published?: Date; + link?: string; + title?: string; + description?: string; + published?: Date; } export interface FeedData { - link?: string; - title?: string; - description?: string; - generator?: string; - language?: string; - published?: Date; - entries?: Array; + link?: string; + title?: string; + description?: string; + generator?: string; + language?: string; + published?: Date; + entries?: Array; +} + +export interface ProxyConfig { + target?: string; + headers?: string[]; } export interface ReaderOptions { @@ -22,27 +27,40 @@ export interface ReaderOptions { * normalize feed data or keep original * default: true */ - normalization?: Boolean; + normalization?: boolean; /** * include full content of feed entry if present * default: false */ - includeEntryContent?: Boolean; + includeEntryContent?: boolean; /** * include optional elements if any * default: false */ - includeOptionalElements?: Boolean; + includeOptionalElements?: boolean; /** * convert datetime to ISO format * default: true */ - useISODateFormat?: Boolean; + useISODateFormat?: boolean; /** * to truncate description * default: 210 */ - descriptionMaxLen?: number; + descriptionMaxLen?: number; +} + +export interface FetchOptions { + /** + * list of request headers + * default: null + */ + headers?: string[]; + /** + * the values to configure proxy + * default: null + */ + proxy?: ProxyConfig; } -export function read(url: string, options?: ReaderOptions): Promise; +export function read(url: string, options?: ReaderOptions, fetchOptions?: FetchOptions): Promise; diff --git a/package.json b/package.json index 72ef5e6..d2ba8f8 100755 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "6.1.0", + "version": "6.1.1", "name": "feed-reader", "description": "To read and normalize RSS/ATOM/JSON feed data", "homepage": "https://www.npmjs.com/package/feed-reader", diff --git a/src/main.js b/src/main.js index 9e053db..655506e 100755 --- a/src/main.js +++ b/src/main.js @@ -11,11 +11,11 @@ import parseJsonFeed from './utils/parseJsonFeed.js' import parseRssFeed from './utils/parseRssFeed.js' import parseAtomFeed from './utils/parseAtomFeed.js' -export const read = async (url, options = {}) => { +export const read = async (url, options = {}, fetchOptions = {}) => { if (!isValidUrl(url)) { throw new Error('Input param must be a valid URL') } - const data = await retrieve(url) + const data = await retrieve(url, fetchOptions) if (!data.text && !data.json) { throw new Error(`Failed to load content from "${url}"`) } diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js index bbe4d50..5d7a0e0 100755 --- a/src/utils/retrieve.js +++ b/src/utils/retrieve.js @@ -2,12 +2,27 @@ import fetch from 'cross-fetch' -export default async (url) => { - const res = await fetch(url, { - headers: { - 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0' - } +const profetch = async (url, proxy = {}) => { + const { + target, + headers = {} + } = proxy + const res = await fetch(target + encodeURIComponent(url), { + headers }) + return res +} + +export default async (url, options = {}) => { + const { + headers = { + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0' + }, + proxy = null + } = options + + const res = proxy ? await profetch(url, proxy) : await fetch(url, { headers }) + const status = res.status if (status >= 400) { throw new Error(`Request failed with error code ${status}`) diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js index feba471..ee54a88 100755 --- a/src/utils/retrieve.test.js +++ b/src/utils/retrieve.test.js @@ -51,4 +51,26 @@ describe('test retrieve() method', () => { expect(result.type).toEqual('xml') expect(result.text).toBe('
this is content
') }) + + test('test retrieve using proxy', async () => { + const url = 'https://some.where/good/source-with-proxy' + const { baseUrl, path } = parseUrl(url) + nock(baseUrl).get(path).reply(200, 'something bad', { + 'Content-Type': 'bad/thing' + }) + nock('https://proxy-server.com') + .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-proxy') + .reply(200, 'this is xml', { + 'Content-Type': 'text/xml' + }) + + const result = await retrieve(url, { + proxy: { + target: 'https://proxy-server.com/api/proxy?url=' + } + }) + expect(result.type).toEqual('xml') + expect(result.text).toEqual('this is xml') + nock.cleanAll() + }) })