From fb2a3ca9aa60269c9aed9f982df68306a1ae8e12 Mon Sep 17 00:00:00 2001 From: Henry Wang Date: Fri, 28 Jun 2019 05:03:06 +0100 Subject: [PATCH 01/38] =?UTF-8?q?doc:=20=E6=B7=BB=E5=8A=A0=E9=83=A8?= =?UTF-8?q?=E5=88=86=E7=BC=BA=E5=A4=B1=E7=9A=84=E7=8E=AF=E5=A2=83=E5=8F=98?= =?UTF-8?q?=E9=87=8F=20(#2508)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/install/README.md | 4 +++ docs/install/README.md | 74 +++++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 3186d753e59a..7c3c91bd8a31 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -272,6 +272,10 @@ Use environment variables is recommended to avoid conflicts during upgrade. `LISTEN_INADDR_ANY`: open up for external access, default to `1` +`REQUEST_RETRY`: retries allowed for failed requests, default to `2` + +`DEBUG_INFO`: display route information on homepage for debugging purpose, default to `true` + `TITLE_LENGTH_LIMIT`: limit the length of feed title generated in bytes, an English alphabet counts as 1 byte, the rest such as Chinese, Japanese, Korean or Arabic counts as 2 bytes by design, default to `100` `REDIS_URL`: Redis target address（invalid when `CACHE_TYPE` is set to memory）, default to `redis://localhost:6379/` diff --git a/docs/install/README.md b/docs/install/README.md index 969346ea759c..18cb7ecedcb3 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -4,7 +4,7 @@ sidebar: auto # 部署 -部署 RSSHub 需要基本的计算机编程常识, 如果您在部署过程中遇到无法解决的问题请到 [issues](https://github.com/DIYgod/RSSHub/issues) 寻找类似的问题或 [向我们提问](https://github.com/DIYgod/RSSHub/issues/new/choose), 我们会尽快给您答复 +部署 RSSHub 需要基本的计算机编程常识，如果您在部署过程中遇到无法解决的问题请到 [issues](https://github.com/DIYgod/RSSHub/issues) 寻找类似的问题或 [向我们提问](https://github.com/DIYgod/RSSHub/issues/new/choose)，我们会尽快给您答复 部署涉及到以下基本编程常识: @@ -73,7 +73,7 @@ $ docker pull diygod/rsshub $ docker run -d --name rsshub -p 1200:1200 diygod/rsshub ``` -在浏览器中打开 [http://127.0.0.1:1200/](http://127.0.0.1:1200/), enjoy it! ✅ +在浏览器中打开 [http://127.0.0.1:1200/](http://127.0.0.1:1200/)，enjoy it! ✅ 您可以使用下面的命令来关闭 RSSHub @@ -94,9 +94,9 @@ $ docker rm rsshub ### 添加配置 -配置运行在 docker 中的 RSSHub, 最便利的方法是使用 docker 环境变量 +配置运行在 docker 中的 RSSHub，最便利的方法是使用 docker 环境变量 -以设置缓存时间为 1 小时举例, 只需要在运行时增加参数: `-e CACHE_EXPIRE=3600` +以设置缓存时间为 1 小时举例，只需要在运行时增加参数: `-e CACHE_EXPIRE=3600` ```bash $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS_TOKEN=example diygod/rsshub @@ -108,7 +108,7 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS ## 手动部署 -部署 `RSSHub` 最直接的方式, 您可以按照以下步骤将 `RSSHub` 部署在您的电脑、服务器或者其他任何地方 +部署 `RSSHub` 最直接的方式，您可以按照以下步骤将 `RSSHub` 部署在您的电脑、服务器或者其他任何地方 ### 安装 @@ -119,7 +119,7 @@ $ git clone https://github.com/DIYgod/RSSHub.git $ cd RSSHub ``` -下载完成后, 需要安装依赖 +下载完成后，需要安装依赖 使用 `npm` @@ -133,7 +133,7 @@ $ npm install $ yarn ``` -由于众所周知的原因, 在中国使用 `npm` 下载依赖十分缓慢, 建议挂一个代理或者考虑使用 [NPM 镜像](https://npm.taobao.org/) +由于众所周知的原因，在中国使用 `npm` 下载依赖十分缓慢，建议挂一个代理或者考虑使用 [NPM 镜像](https://npm.taobao.org/) 然后在 `RSSHub` 文件夹中运行下面的命令就可以启动 @@ -153,7 +153,7 @@ $ yarn start $ pm2 start lib/index.js --name rsshub ``` -在浏览器中打开 [http://127.0.0.1:1200/](http://127.0.0.1:1200/), enjoy it! ✅ +在浏览器中打开 [http://127.0.0.1:1200/](http://127.0.0.1:1200/)，enjoy it! ✅ 详细使用说明参照 [指南](https://docs.rsshub.app/)，替换所有路由例子中的 `https://rsshub.app/` 为 `http://localhost:1200` 即可正常使用 @@ -194,19 +194,19 @@ $ git pull [Before you begin](https://cloud.google.com/appengine/docs/flexible/nodejs/quickstart) -按照这里的引导完成 GCP 账号设置, 创建 GCP 项目, 创建 App Engine 项目, 开通付费功能（必须）, 安装 git 与 gcloud 工具. 并完成 gcloud 工具的初始化, 初始化具体方式[请查看这个链接](https://cloud.google.com/sdk/gcloud/?hl=zh-CN). 如果你不打算在本地调试本项目, 可以不安装 Node.js 环境 +按照这里的引导完成 GCP 账号设置，创建 GCP 项目，创建 App Engine 项目，开通付费功能（必须），安装 git 与 gcloud 工具。并完成 gcloud 工具的初始化，初始化具体方式[请查看这个链接](https://cloud.google.com/sdk/gcloud/?hl=zh-CN)。如果你不打算在本地调试本项目，可以不安装 Node.js 环境。 -请注意, GAE 免费用量不支持 Flexible Environment, 部署至 Flexible Environment 前请确认收费标准 +请注意，GAE 免费用量不支持 Flexible Environment，部署至 Flexible Environment 前请确认收费标准。 -Node.JS 的 standard environment 仍在测试中, 您可能会在部署或使用中遇到某些不可预期的问题 +Node.JS 的 standard environment 仍在测试中，您可能会在部署或使用中遇到某些不可预期的问题。 -运行 `git clone https://github.com/DIYgod/RSSHub.git` 拉取本项目的最新版本 +运行 `git clone https://github.com/DIYgod/RSSHub.git` 拉取本项目的最新版本。 ### app.yaml 配置 #### 部署至 Flexible Environment -在 RSSHub 项目根目录下建立一个 app.yaml 文件, 内容示例如下: +在 RSSHub 项目根目录下建立一个 app.yaml 文件，内容示例如下： ```yaml # [START app_yaml] @@ -219,7 +219,7 @@ env: flex # https://cloud.google.com/appengine/docs/flexible/nodejs/configuring-your-app-with-app-yaml manual_scaling: instances: 1 -# 以下是 app engine 资源配置, 可以自行修改, 硬盘最低为 10G +# 以下是 app engine 资源配置，可以自行修改，硬盘最低为 10G resources: cpu: 1 memory_gb: 0.5 @@ -228,7 +228,7 @@ network: forwarded_ports: - 80:1200 - 443:1200 -# 以下是环境配置示例, 具体可配置项见本文档配置章节 +# 以下是环境配置示例，具体可配置项见本文档配置章节 env_variables: CACHE_EXPIRE: '300' # [END app_yaml] @@ -236,7 +236,7 @@ env_variables: #### 部署至 standard environment -在 RSSHub 项目根目录下建立一个 app.yaml 文件, 内容示例如下: +在 RSSHub 项目根目录下建立一个 app.yaml 文件，内容示例如下： ```yaml # [START app_yaml] @@ -246,7 +246,7 @@ network: forwarded_ports: - 80:1200 - 443:1200 -# 以下是环境配置示例, 具体可配置项见本文档配置章节 +# 以下是环境配置示例，具体可配置项见本文档配置章节 env_variables: CACHE_EXPIRE: '300' # [END app_yaml] @@ -260,9 +260,9 @@ env_variables: gcloud app deploy ``` -进行项目部署, 如果您需要变更 app.yaml 文件名称或者变更部署的项目 ID 或者指定版本号等, 请参考[这个链接](https://cloud.google.com/appengine/docs/flexible/nodejs/testing-and-deploying-your-app)的"Deploying a service" 部分 +进行项目部署，如果您需要变更 app.yaml 文件名称或者变更部署的项目 ID 或者指定版本号等，请参考 [Deploying a service](https://cloud.google.com/appengine/docs/flexible/nodejs/testing-and-deploying-your-app#deploying_a_service_2)。 -部署完成后可访问您的 Google App Engine URL 查看部署情况 +部署完成后可访问您的 Google App Engine URL 查看部署情况。 ## 部署到 arm32v7 设备（树莓派） @@ -289,7 +289,7 @@ $ cd RSSHub $ docker build -f ./Dockerfile.arm32v7 -t rsshub:arm32v7 . ``` -puppeteer 本身不会下载 chrome-arm，需要在 `lib/config.js` 中的 `puppeteerWSEndpoint`中设置相应的远程 Chrome Websocket 地址, 以启用相应路由 +puppeteer 本身不会下载 chrome-arm，需要在 `lib/config.js` 中的 `puppeteerWSEndpoint`中设置相应的远程 Chrome Websocket 地址，以启用相应路由 TO DO: 暂时还没有找到合适的 `chrome websocket` arm32v7 镜像 @@ -312,13 +312,13 @@ $ docker run -d --name rsshub -p 1200:1200 rsshub:arm32v7 RSSHub 支持 `memory` 和 `redis` 两种缓存方式 -`CACHE_TYPE`: 缓存类型, 可为 `memory` 和 `redis`, 设为空可以禁止缓存, 默认为 `memory` +`CACHE_TYPE`: 缓存类型，可为 `memory` 和 `redis`，设为空可以禁止缓存，默认为 `memory` -`CACHE_EXPIRE`: 路由缓存过期时间, 单位为秒, 默认 `5 * 60` +`CACHE_EXPIRE`: 路由缓存过期时间，单位为秒，默认 `5 * 60` -`CACHE_CONTENT_EXPIRE`: 内容缓存过期时间，每次访问会重新计算过期时间，单位为秒, 默认 `1 * 60 * 60` +`CACHE_CONTENT_EXPIRE`: 内容缓存过期时间，每次访问会重新计算过期时间，单位为秒，默认 `1 * 60 * 60` -`REDIS_URL`: Redis 连接地址（redis 缓存类型时有效）, 默认为 `redis://localhost:6379/` +`REDIS_URL`: Redis 连接地址（redis 缓存类型时有效），默认为 `redis://localhost:6379/` `REDIS_PASSWORD`: Redis 连接密码（redis 缓存类型时有效） @@ -326,7 +326,7 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 部分路由反爬严格，可以配置使用代理抓取 -`PROXY_PROTOCOL`: 使用代理, 支持 socks, http, https +`PROXY_PROTOCOL`: 使用代理，支持 socks，http，https `PROXY_HOST`: 代理服务器域名或 IP @@ -340,29 +340,33 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 `protected_route.js` 内的路由将启用 HTTP Basic Authentication 认证 -支持该认证协议的阅读器, 在添加源地址时, 需要在源地址前添加认证信息, 例如：http://usernam3:passw0rd@127.0.0.1:1200/protected/rsshub/rss +支持该认证协议的阅读器，在添加源地址时，需要在源地址前添加认证信息，例如：http://usernam3:passw0rd@127.0.0.1:1200/protected/rsshub/rss -`HTTP_BASIC_AUTH_NAME`: Http basic authentication 用户名, 默认为 `usernam3`, 请务必修改 +`HTTP_BASIC_AUTH_NAME`: Http basic authentication 用户名，默认为 `usernam3`，请务必修改 -`HTTP_BASIC_AUTH_PASS`: Http basic authentication 密码, 默认为 `passw0rd`, 请务必修改 +`HTTP_BASIC_AUTH_PASS`: Http basic authentication 密码，默认为 `passw0rd`，请务必修改 ### 访问控制配置 -配置黑名单和白名单, 支持 IP 和路由, 设置多项时用英文逗号 `,` 隔开, 同时设置黑名单和白名单时仅白名单有效 +配置黑名单和白名单，支持 IP 和路由，设置多项时用英文逗号 `,` 隔开，同时设置黑名单和白名单时仅白名单有效 - `BLACKLIST`: 黑名单 -- `WHITELIST`: 白名单, 设置白名单后黑名单无效 +- `WHITELIST`: 白名单，设置白名单后黑名单无效 ### 其他应用配置 -`PORT`: 监听端口, 默认为 `1200` +`PORT`: 监听端口，默认为 `1200` -`SOCKET`: 监听 Unix Socket, 默认为 `null` +`SOCKET`: 监听 Unix Socket，默认 `null` -`LISTEN_INADDR_ANY`: 是否允许公网连接, 默认 `1` +`LISTEN_INADDR_ANY`: 是否允许公网连接，默认 `1` -`TITLE_LENGTH_LIMIT`: 限制输出标题的字节长度, 一个英文字符的长度为 1 字节, 部分语言如中文, 日文, 韩文或阿拉伯文等, 统一算作 2 字节, 默认 `100` +`TITLE_LENGTH_LIMIT`: 限制输出标题的字节长度，一个英文字符的长度为 1 字节，部分语言如中文，日文，韩文或阿拉伯文等，统一算作 2 字节，默认 `100` + +`REQUEST_RETRY`: 请求失败重试次数，默认 `2` + +`DEBUG_INFO`: 是否在首页显示路由信息，默认 `true` `LOGGER_LEVEL`: 指明输出到 console 和日志文件的日志的最大[等级](https://github.com/winstonjs/winston#logging-levels)，默认 `info` @@ -404,4 +408,4 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 - 语雀 全部路由: [注册地址](https://www.yuque.com/register) - - `YUQUE_TOKEN`: 语雀 Token，[获取地址](https://www.yuque.com/settings/tokens) 。语雀接口做了访问频率限制，为保证正常访问建议配置 Token，详见[语雀开发者文档](https://www.yuque.com/yuque/developer/api#5b3a1535)。 + - `YUQUE_TOKEN`: 语雀 Token，[获取地址](https://www.yuque.com/settings/tokens)。语雀接口做了访问频率限制，为保证正常访问建议配置 Token，详见[语雀开发者文档](https://www.yuque.com/yuque/developer/api#5b3a1535)。 From 5d9b7ebe37229634dcb42c1847a438cb2b7632bc Mon Sep 17 00:00:00 2001 From: Cloud Date: Fri, 28 Jun 2019 13:00:06 +0800 Subject: [PATCH 02/38] feat: add changba (#2482) --- docs/social-media.md | 6 ++++ lib/router.js | 3 ++ lib/routes/changba/user.js | 71 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 lib/routes/changba/user.js diff --git a/docs/social-media.md b/docs/social-media.md index d15d6bfafd49..bcacf90a7993 100644 --- a/docs/social-media.md +++ b/docs/social-media.md @@ -408,6 +408,12 @@ pageClass: routes +## 唱吧 + +### 用户 + + + ## 豆瓣 ### 正在上映的电影 diff --git a/lib/router.js b/lib/router.js index bfb0d0007279..72a8edf65c89 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1446,4 +1446,7 @@ router.get('/csrc/fashenwei', require('./routes/csrc/fashenwei')); // LWN.net Alerts router.get('/lwn/alerts/:distributor', require('./routes/lwn/alerts')); +// 唱吧 +router.get('/changba/:userid', require('./routes/changba/user')); + module.exports = router; diff --git a/lib/routes/changba/user.js b/lib/routes/changba/user.js new file mode 100644 index 000000000000..fd3f8b9fe59b --- /dev/null +++ b/lib/routes/changba/user.js @@ -0,0 +1,71 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const userid = ctx.params.userid; + const url = `http://changba.com/u/${userid}`; + const response = await got({ + method: 'get', + url: url, + headers: { + 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', + }, + }); + const data = response.data; + const $ = cheerio.load(data); + const list = $('.user-work .work-info').get(); + const author = $('div.user-main-info > span.txt-info > a.uname').text(); + const authorimg = $('div.user-main-info > .poster > img').attr('data-src'); + + const ProcessFeed = (data) => { + const $ = cheerio.load(data); + const description = $('.user-title').html() + '
' + $.html('audio'); + const mp3 = $('audio').attr('src'); + return { description, mp3 }; + }; + + const items = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const link = $('a').attr('href'); + + const cache = await ctx.cache.get(link); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const response = await got({ + method: 'get', + url: link, + headers: { + Referer: url, + 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', + }, + }); + + const result = ProcessFeed(response.data); + + const single = { + title: $('.work-title').text(), + description: result.description, + link: link, + author: author, + itunes_item_image: authorimg, + enclosure_url: result.mp3, + enclosure_type: 'audio/mpeg', + }; + ctx.cache.set(link, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title: $('title').text(), + link: url, + description: $('meta[name="description"]').attr('content') || $('title').text(), + item: items, + image: authorimg, + itunes_author: author, + itunes_category: '唱吧', + }; +}; From 3225415e9581c913490541cc791f7209fd2f0a1b Mon Sep 17 00:00:00 2001 From: Cloud Date: Fri, 28 Jun 2019 13:00:46 +0800 Subject: [PATCH 03/38] fix: konachan safe mode (#2509) fix #2506 --- lib/routes/konachan/post_popular_recent.js | 63 +++++++++++----------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/lib/routes/konachan/post_popular_recent.js b/lib/routes/konachan/post_popular_recent.js index a8a3be9c3706..f52848eb06d3 100644 --- a/lib/routes/konachan/post_popular_recent.js +++ b/lib/routes/konachan/post_popular_recent.js @@ -4,6 +4,7 @@ module.exports = async (ctx) => { const { period = '1d' } = ctx.params; const baseUrl = ctx.path.startsWith('/konachan.net') ? 'https://konachan.net' : 'https://konachan.com'; + const safemode = ctx.path.startsWith('/konachan.net'); const response = await got({ method: 'get', @@ -28,37 +29,39 @@ module.exports = async (ctx) => { ctx.state.data = { title: `${title} - Konachan Anime Wallpapers`, link: `${baseUrl}/post/popular_recent`, - item: posts.map((post) => { - const content = (url) => { - if (url.startsWith('//')) { - url = 'https:' + url; - } - let result = ``; - if (post.source) { - result += `source`; - } - if (post.parent_id) { - result += `parent`; - } - return result; - }; + item: posts + .filter((post) => !(safemode && post.rating != 's')) + .map((post) => { + const content = (url) => { + if (url.startsWith('//')) { + url = 'https:' + url; + } + let result = ``; + if (post.source) { + result += `source`; + } + if (post.parent_id) { + result += `parent`; + } + return result; + }; - const created_at = post.created_at * 1e3; + const created_at = post.created_at * 1e3; - return { - title: post.tags, - id: `${ctx.path}#${post.id}`, - guid: `${ctx.path}#${post.id}`, - link: `${baseUrl}/post/show/${post.id}`, - author: post.author, - published: new Date(created_at).toISOString(), - pubDate: new Date(created_at).toUTCString(), - description: content(post.sample_url), - summary: content(post.sample_url), - content: { html: content(post.file_url) }, - image: post.file_url, - category: post.tags.split(/\s+/), - }; - }), + return { + title: post.tags, + id: `${ctx.path}#${post.id}`, + guid: `${ctx.path}#${post.id}`, + link: `${baseUrl}/post/show/${post.id}`, + author: post.author, + published: new Date(created_at).toISOString(), + pubDate: new Date(created_at).toUTCString(), + description: content(post.sample_url), + summary: content(post.sample_url), + content: { html: content(post.file_url) }, + image: post.file_url, + category: post.tags.split(/\s+/), + }; + }), }; }; From d5a6d5be38b7960f99cee3e5f0f39738a5e3acf6 Mon Sep 17 00:00:00 2001 From: alizeegod Date: Fri, 28 Jun 2019 17:56:08 +0800 Subject: [PATCH 04/38] =?UTF-8?q?feat:=20=E6=8E=8C=E4=B8=8A=E8=8B=B1?= =?UTF-8?q?=E9=9B=84=E8=81=94=E7=9B=9F-=E6=8E=A8=E8=8D=90=E6=96=B0?= =?UTF-8?q?=E9=97=BB=20(#2511)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/game.md | 6 +++++ lib/router.js | 3 +++ lib/routes/lolapp/recommend.js | 48 ++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 lib/routes/lolapp/recommend.js diff --git a/docs/game.md b/docs/game.md index 67c783d9207b..a4a07771ea6c 100644 --- a/docs/game.md +++ b/docs/game.md @@ -253,6 +253,12 @@ Example: `https://store.steampowered.com/search/?specials=1&term=atelier` 中的 +## 掌上英雄联盟 + +### 推荐 + + + ## きららファンタジア｜奇拉拉幻想曲 ### 公告 diff --git a/lib/router.js b/lib/router.js index 72a8edf65c89..8a64bf6292b6 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1449,4 +1449,7 @@ router.get('/lwn/alerts/:distributor', require('./routes/lwn/alerts')); // 唱吧 router.get('/changba/:userid', require('./routes/changba/user')); +// 掌上英雄联盟 +router.get('/lolapp/recommend', require('./routes/lolapp/recommend')); + module.exports = router; diff --git a/lib/routes/lolapp/recommend.js b/lib/routes/lolapp/recommend.js new file mode 100644 index 000000000000..15b7687d680e --- /dev/null +++ b/lib/routes/lolapp/recommend.js @@ -0,0 +1,48 @@ +const got = require('@/utils/got'); + +const sourceTimezoneOffset = -8; +module.exports = async (ctx) => { + const cid = 12; + const areaid = 2; + + const url = 'http://qt.qq.com/lua/lol_news/recommend_refresh?cid=' + cid + '&plat=ios&version=9880&areaid=' + areaid; + const res = await got.get(url); + const articles = (res.data || {}).update_list || []; + const out = await Promise.all( + articles.map(async (article) => { + const link = article.article_url; + const cache = await ctx.cache.get(link); + if (cache) { + return JSON.parse(cache); + } + + const guid = article.content_id; + const title = article.title; + const time = new Date(article.publication_date); + time.setTime(time.getTime() + (sourceTimezoneOffset - time.getTimezoneOffset() / 60) * 60 * 60 * 1000); + const pubDate = time.toUTCString(); + + // const detailJsonLink = article.detail_json; + // const detailRes = await got.get(detailJsonLink); + const description = article.image_url_big; + + const item = { + title, + description, + pubDate, + link, + guid, + }; + + ctx.cache.set(link, JSON.stringify(item)); + + return item; + }) + ); + + ctx.state.data = { + title: '掌上英雄联盟 - 推荐', + link: 'http://lol.qq.com/app/index.html', + item: out, + }; +}; From 20c8c989a73709dfaee3efd1b6124911ceaac9b3 Mon Sep 17 00:00:00 2001 From: zhangzhxb520 Date: Mon, 1 Jul 2019 11:10:24 +0800 Subject: [PATCH 05/38] =?UTF-8?q?feat:=20=E6=9E=81=E5=AE=A2=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E7=9B=B8=E5=85=B3=E5=86=85=E5=AE=B9=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=85=A8=E6=96=87=E8=BE=93=E5=87=BA=20(#2507)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加极客时间|极客新闻 RSS * 极客新闻RSS - 调整md文档格式 * 极客时间相关内容添加全文输出 * 极客时间全文输出 - 将缓存时间MaxAge参数值置空 --- lib/routes/geektime/column.js | 52 +++++++++++++++++++++++++++++++---- lib/routes/geektime/news.js | 52 +++++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/lib/routes/geektime/column.js b/lib/routes/geektime/column.js index d81961dbbcb3..c99ca2d46063 100644 --- a/lib/routes/geektime/column.js +++ b/lib/routes/geektime/column.js @@ -33,15 +33,55 @@ module.exports = async (ctx) => { const articles = latest_response.data.data.list; + const out = await Promise.all( + articles.map(async (item) => { + const title = item.article_title; + const pubDate = new Date(item.article_ctime * 1000).toUTCString(); + const link = `https://time.geekbang.org/column/article/${item.id}`; + const description = item.article_summary; + + const single = { + title: title, + pubDate: pubDate, + link: link, + description: description, + }; + + if (item.id !== undefined) { + const value = ctx.cache.get(item.id); + if (value) { + single.description = value; + } else { + try { + const article_response = await got({ + method: 'post', + url: 'https://time.geekbang.org/serv/v1/article', + headers: { + Referer: link, + }, + json: true, + data: { + id: item.id, + include_neighbors: true, + }, + }); + + single.description = article_response.data.data.article_content; + ctx.cache.set(item.id, single.description); + } catch (err) { + single.description = description; + } + } + } + return Promise.resolve(single); + }) + ); + + out.reverse(); ctx.state.data = { title: intro_data.column_title, link: `https://time.geekbang.org/column/intro/${column_id}`, description: intro_data.column_subtitle, - item: articles.map((item) => ({ - title: item.article_title, - description: item.article_summary, - pubDate: new Date(item.article_ctime * 1000).toUTCString(), - link: `https://time.geekbang.org/column/article/${item.id}`, - })), + item: out, }; }; diff --git a/lib/routes/geektime/news.js b/lib/routes/geektime/news.js index 62cb9c7a531b..7e577f391b79 100644 --- a/lib/routes/geektime/news.js +++ b/lib/routes/geektime/news.js @@ -33,15 +33,55 @@ module.exports = async (ctx) => { const news = news_response.data.data.list; + const out = await Promise.all( + news.map(async (item) => { + const title = item.article_title; + const pubDate = new Date(item.article_ctime * 1000).toUTCString(); + const link = `https://time.geekbang.org/column/article/${item.id}`; + const description = item.article_summary; + + const single = { + title: title, + pubDate: pubDate, + link: link, + description: description, + }; + + if (item.id !== undefined) { + const value = ctx.cache.get(item.id); + if (value) { + single.description = value; + } else { + try { + const article_response = await got({ + method: 'post', + url: 'https://time.geekbang.org/serv/v1/article', + headers: { + Referer: link, + }, + json: true, + data: { + id: item.id, + include_neighbors: true, + }, + }); + + single.description = article_response.data.data.article_content; + ctx.cache.set(item.id, single.description); + } catch (err) { + single.description = description; + } + } + } + return Promise.resolve(single); + }) + ); + + out.reverse(); ctx.state.data = { title: intro_data.column_title, link: `https://time.geekbang.org/column/intro/${column_id}`, description: intro_data.column_subtitle, - item: news.map((item) => ({ - title: item.article_title, - description: item.article_summary, - pubDate: new Date(item.article_ctime * 1000).toUTCString(), - link: `https://time.geekbang.org/column/article/${item.id}`, - })), + item: out, }; }; From 8a6af565291bdf3f0dd7a4c7e1e597d1adf817dd Mon Sep 17 00:00:00 2001 From: Chenyang Shi Date: Mon, 1 Jul 2019 11:11:10 +0800 Subject: [PATCH 06/38] fix: bilibili url (#2521) --- lib/routes/bilibili/partion-ranking.js | 2 +- lib/routes/bilibili/video.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/routes/bilibili/partion-ranking.js b/lib/routes/bilibili/partion-ranking.js index 8bacba35bbfe..029479a38d39 100644 --- a/lib/routes/bilibili/partion-ranking.js +++ b/lib/routes/bilibili/partion-ranking.js @@ -40,7 +40,7 @@ module.exports = async (ctx) => { item = { title: `${item.title} - ${item.author}`, - description: `${item.description}
Tags:${item.tag}`, + description: `${item.description}
Tags:${item.tag}`, pubDate: new Date(item.pubdate).toUTCString(), link: `https://www.bilibili.com/video/av${item.id}`, }; diff --git a/lib/routes/bilibili/video.js b/lib/routes/bilibili/video.js index 0b994c6d1501..208b6498bce6 100644 --- a/lib/routes/bilibili/video.js +++ b/lib/routes/bilibili/video.js @@ -23,7 +23,7 @@ module.exports = async (ctx) => { data.data.vlist && data.data.vlist.map((item) => ({ title: item.title, - description: `${item.description}
`, + description: `${item.description}
`, pubDate: new Date(item.created * 1000).toUTCString(), link: `https://www.bilibili.com/video/av${item.aid}`, })), From 0281592839addb4159a3e2d2d0915301ecd37467 Mon Sep 17 00:00:00 2001 From: Cloud Date: Mon, 1 Jul 2019 11:11:48 +0800 Subject: [PATCH 07/38] feat: add zreading fulltext (#2522) * feat: add zreding fulltext * feat: add zreading fulltext * feat: add zreading fulltext * feat: add zreading fulltext --- docs/reading.md | 6 ++++ lib/router.js | 3 ++ lib/routes/zreading/home.js | 56 +++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 lib/routes/zreading/home.js diff --git a/docs/reading.md b/docs/reading.md index 25d14cc08b25..0c013eabef68 100644 --- a/docs/reading.md +++ b/docs/reading.md @@ -123,3 +123,9 @@ pageClass: routes ### 章节 + +## 左岸读书 + +### 主页 + + diff --git a/lib/router.js b/lib/router.js index 8a64bf6292b6..76552e897b7b 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1452,4 +1452,7 @@ router.get('/changba/:userid', require('./routes/changba/user')); // 掌上英雄联盟 router.get('/lolapp/recommend', require('./routes/lolapp/recommend')); +// 左岸读书 +router.get('/zreading', require('./routes/zreading/home')); + module.exports = router; diff --git a/lib/routes/zreading/home.js b/lib/routes/zreading/home.js new file mode 100644 index 000000000000..0ecbe8a34d7f --- /dev/null +++ b/lib/routes/zreading/home.js @@ -0,0 +1,56 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const parser = require('@/utils/rss-parser'); + +module.exports = async (ctx) => { + const feed = await parser.parseURL('http://www.zreading.cn/feed'); + + const ProcessFeed = (data) => { + const $ = cheerio.load(data); + + $('script').remove(); + $('.adsbygoogle').remove(); + + // 还原图片地址 + $('img').each((index, elem) => { + const $elem = $(elem); + $elem.attr('referrerpolicy', 'no-referrer'); + }); + + // 提取内容 + return $('.grap').html(); + }; + + const items = await Promise.all( + feed.items.map(async (item) => { + const cache = await ctx.cache.get(item.link); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const response = await got({ + method: 'get', + url: item.link, + }); + + const description = ProcessFeed(response.data); + + const single = { + title: item.title, + description, + pubDate: item.pubDate, + link: item.link, + author: item.author, + }; + ctx.cache.set(item.link, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title: feed.title, + link: feed.link, + description: feed.description, + item: items, + }; +}; From 1d81a4a9ce8ebad75f53c0f6dfaec06a38dd2e90 Mon Sep 17 00:00:00 2001 From: Cloud Date: Mon, 1 Jul 2019 11:24:38 +0800 Subject: [PATCH 08/38] feat: add wechat uread channel (#2515) * feat: add wechat uread channel * feat: add wechat uread channel * feat: add wechat uread channel * Update social-media.md --- docs/social-media.md | 4 +++ lib/router.js | 1 + lib/routes/tencent/wechat/uread.js | 51 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 lib/routes/tencent/wechat/uread.js diff --git a/docs/social-media.md b/docs/social-media.md index bcacf90a7993..cb141f7b5aa8 100644 --- a/docs/social-media.md +++ b/docs/social-media.md @@ -680,6 +680,10 @@ pageClass: routes ::: +### 公众号 (优读来源) + + + ### 公众平台系统公告栏目 diff --git a/lib/router.js b/lib/router.js index 76552e897b7b..85c0f9af874e 100644 --- a/lib/router.js +++ b/lib/router.js @@ -447,6 +447,7 @@ router.get('/wechat/csm/:id', require('./routes/tencent/wechat/csm')); router.get('/wechat/announce', require('./routes/tencent/wechat/announce')); router.get('/wechat/miniprogram/plugins', require('./routes/tencent/wechat/miniprogram/plugins')); router.get('/wechat/tgchannel/:id', require('./routes/tencent/wechat/tgchannel')); +router.get('/wechat/uread/:userid', require('./routes/tencent/wechat/uread')); // All the Flight Deals router.get('/atfd/:locations/:nearby?', require('./routes/atfd/index')); diff --git a/lib/routes/tencent/wechat/uread.js b/lib/routes/tencent/wechat/uread.js new file mode 100644 index 000000000000..8283342ebe34 --- /dev/null +++ b/lib/routes/tencent/wechat/uread.js @@ -0,0 +1,51 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const userid = ctx.params.userid; + const url = `http://119.29.146.143:8080/reading/subscription/account/recent?uid=${userid}`; + const response = await got({ + method: 'get', + url: url, + }); + const data = response.data; + + const ProcessFeed = (data) => { + const $ = cheerio.load(data); + return $('.rich_media_content').html(); + }; + + const items = await Promise.all( + data.data.map(async (item) => { + const link = item.url; + + const cache = await ctx.cache.get(link); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const response = await got({ + method: 'get', + url: link, + }); + + const description = ProcessFeed(response.data); + + const single = { + title: item.title, + description, + link: link, + author: item.official_account, + pubDate: item.publish_time, + }; + ctx.cache.set(link, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title: `优读 - ${userid}`, + link: 'https://uread.ai/', + item: items, + }; +}; From 68dd8c570aff0c6751ce87fc2b8974422b84b4d4 Mon Sep 17 00:00:00 2001 From: ruleroller Date: Tue, 2 Jul 2019 11:00:29 +0800 Subject: [PATCH 09/38] =?UTF-8?q?feat:=20Matters-=E6=9C=80=E6=96=B0?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=E5=A2=9E=E5=8A=A0author=E5=AD=97=E6=AE=B5=20?= =?UTF-8?q?(#2520)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update cxxww.js * feat: Matters-最新排名增加author字段 * feat: Matters全部路由加上author * 整理格式 --- lib/routes/matters/author.js | 4 ++++ lib/routes/matters/hot.js | 4 ++++ lib/routes/matters/latest.js | 1 + lib/routes/matters/tags.js | 4 ++++ lib/routes/matters/topics.js | 4 ++++ 5 files changed, 17 insertions(+) diff --git a/lib/routes/matters/author.js b/lib/routes/matters/author.js index 0eeffdda886a..cbe6fca30dad 100644 --- a/lib/routes/matters/author.js +++ b/lib/routes/matters/author.js @@ -16,6 +16,9 @@ module.exports = async (ctx) => { const $ = cheerio.load(item); const time = $('time').attr('datetime'); const title = $('h2').text(); + const author = $('div.content a') + .attr('href') + .replace(/\/@(.*?)\/.*/, '$1'); const postfix = encodeURI($('div.content a').attr('href')); const address = `https://matters.news${postfix}`; const cache = await ctx.cache.get(address); @@ -25,6 +28,7 @@ module.exports = async (ctx) => { const single = { title, pubDate: new Date(time).toUTCString(), + author, link: address, guid: address, }; diff --git a/lib/routes/matters/hot.js b/lib/routes/matters/hot.js index 93bafa02fe02..4496856b4b23 100644 --- a/lib/routes/matters/hot.js +++ b/lib/routes/matters/hot.js @@ -16,6 +16,9 @@ module.exports = async (ctx) => { const $ = cheerio.load(item); const time = $('time').attr('datetime'); const title = $('h2').text(); + const author = $('div.content a') + .attr('href') + .replace(/\/@(.*?)\/.*/, '$1'); const partial = encodeURI($('div.content a').attr('href')); const completeUrl = `https://matters.news${partial}`; const cache = await ctx.cache.get(completeUrl); @@ -25,6 +28,7 @@ module.exports = async (ctx) => { const single = { title, pubDate: new Date(time).toUTCString(), + author, link: completeUrl, guid: completeUrl, }; diff --git a/lib/routes/matters/latest.js b/lib/routes/matters/latest.js index 31c74d3ddd8b..04fa36cb7d38 100644 --- a/lib/routes/matters/latest.js +++ b/lib/routes/matters/latest.js @@ -32,6 +32,7 @@ module.exports = async (ctx) => { title: node.title, link, description: article, + author: node.author.userName, }; ctx.cache.set(link, JSON.stringify(single)); diff --git a/lib/routes/matters/tags.js b/lib/routes/matters/tags.js index f4b987c5820c..08edd65f6a7f 100644 --- a/lib/routes/matters/tags.js +++ b/lib/routes/matters/tags.js @@ -16,6 +16,9 @@ module.exports = async (ctx) => { const $ = cheerio.load(item); const time = $('time').attr('datetime'); const title = $('h2').text(); + const author = $('div.content a') + .attr('href') + .replace(/\/@(.*?)\/.*/, '$1'); const postfix = encodeURI($('div.content a').attr('href')); const address = `https://matters.news${postfix}`; const cache = await ctx.cache.get(address); @@ -25,6 +28,7 @@ module.exports = async (ctx) => { const single = { title, pubDate: new Date(time).toUTCString(), + author, link: address, guid: address, }; diff --git a/lib/routes/matters/topics.js b/lib/routes/matters/topics.js index 57e634d527ed..f7e8973c6b4b 100644 --- a/lib/routes/matters/topics.js +++ b/lib/routes/matters/topics.js @@ -16,6 +16,9 @@ module.exports = async (ctx) => { const $ = cheerio.load(item); const time = $('time').attr('datetime'); const title = $('h2').text(); + const author = $('div.content a') + .attr('href') + .replace(/\/@(.*?)\/.*/, '$1'); const postfix = encodeURI($('div.content a').attr('href')); const address = `https://matters.news${postfix}`; const cache = await ctx.cache.get(address); @@ -25,6 +28,7 @@ module.exports = async (ctx) => { const single = { title, pubDate: new Date(time).toUTCString(), + author, link: address, guid: address, }; From 030f0fb83e9defad1dc83b65b581014a050ded3c Mon Sep 17 00:00:00 2001 From: Cloud Date: Tue, 2 Jul 2019 11:19:23 +0800 Subject: [PATCH 10/38] feat: add tprtc (#2528) * feat: add tprtc * feat: add tprtc * feat: add tprtc * Update other.md * limit 20 * limit 20 --- docs/other.md | 14 +++++++ lib/router.js | 5 +++ lib/routes/tprtc/cqzr.js | 80 ++++++++++++++++++++++++++++++++++++++ lib/routes/tprtc/news.js | 60 ++++++++++++++++++++++++++++ lib/routes/tprtc/qyzc.js | 84 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 243 insertions(+) create mode 100644 lib/routes/tprtc/cqzr.js create mode 100644 lib/routes/tprtc/news.js create mode 100644 lib/routes/tprtc/qyzc.js diff --git a/docs/other.md b/docs/other.md index 47030a9b99c6..c674113c73a8 100644 --- a/docs/other.md +++ b/docs/other.md @@ -799,6 +799,20 @@ type 为 all 时，category 参数不支持 cost 和 free +## 天津产权交易中心 + +### 产权转让 + + + +### 企业资产转让 + + + +### 新闻动态 + + + ## 推酷 ### 周刊 diff --git a/lib/router.js b/lib/router.js index 85c0f9af874e..833a29feb11e 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1456,4 +1456,9 @@ router.get('/lolapp/recommend', require('./routes/lolapp/recommend')); // 左岸读书 router.get('/zreading', require('./routes/zreading/home')); +// 天津产权交易中心 +router.get('/tprtc/cqzr', require('./routes/tprtc/cqzr')); +router.get('/tprtc/qyzc', require('./routes/tprtc/qyzc')); +router.get('/tprtc/news', require('./routes/tprtc/news')); + module.exports = router; diff --git a/lib/routes/tprtc/cqzr.js b/lib/routes/tprtc/cqzr.js new file mode 100644 index 000000000000..49da1e5f9f9c --- /dev/null +++ b/lib/routes/tprtc/cqzr.js @@ -0,0 +1,80 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const url = require('url'); + +module.exports = async (ctx) => { + const response = await got({ + method: 'get', + url: 'http://item.tprtc.com/tjcqp/cgq/search/search_li?xxplfs=&zrdj=0&nzrbl=0&hy=&szdq=&sortTag=0&pageNo=1&pageSize=20&keyWord=', + headers: { + Referer: 'http://item.tprtc.com/tjcqp/cgq/search/index?plfs=2', + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3793.0 Safari/537.36', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + }); + + const data = response.data; + + const $ = cheerio.load(data); + const list = $('li').get(); + + const ProcessFeed = async (link) => { + const response = await got({ + method: 'get', + url: link, + headers: { + Referer: 'http://item.tprtc.com/tjcqp/cgq/search/index?plfs=2', + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3793.0 Safari/537.36', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + }); + const $ = cheerio.load(response.data); + + // 删除没用的tab指示 + $('.bd_detail_main .bd_detail_tab').remove(); + + // 文章样式 + const style = ` + `; + + // 提取内容 + const description = $('.bd_detail_main').html() + style; + + return description; + }; + + const host = 'http://item.tprtc.com'; + const items = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const $title = $('.property_list_title>a'); + const itemUrl = url.resolve(host, $title.attr('href')); + + const cache = await ctx.cache.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const description = await ProcessFeed(itemUrl); + + const single = { + title: $('.property_list_title>span').text() + ' ' + $title.text(), + link: itemUrl, + description, + }; + + ctx.cache.set(itemUrl, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title: '产权转让-天津产权交易中心', + link: 'http://item.tprtc.com/tjcqp/cgq/search/index?plfs=2', + description: '天津产权交易中心-产权转让', + item: items, + }; +}; diff --git a/lib/routes/tprtc/news.js b/lib/routes/tprtc/news.js new file mode 100644 index 000000000000..c6e100ce1a43 --- /dev/null +++ b/lib/routes/tprtc/news.js @@ -0,0 +1,60 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const response = await got({ + method: 'get', + url: 'http://www.tprtc.com/InformationChannel/index.jhtml', + headers: { + Referer: 'http://www.tprtc.com', + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3793.0 Safari/537.36', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + }); + + const data = response.data; + + const $ = cheerio.load(data); + const list = $('.c1-bline').get(); + + const ProcessFeed = async (link) => { + const response = await got.get(link); + const $ = cheerio.load(response.data); + + return $('.content').html(); + }; + + const items = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const $title = $('.f-left>a:last-child'); + const itemUrl = $title.attr('href'); + + const cache = await ctx.cache.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const description = await ProcessFeed(itemUrl); + + const single = { + title: $('a.red').text() + ' ' + $title.attr('title'), + link: itemUrl, + description, + author: $('a.red').text(), + }; + + ctx.cache.set(itemUrl, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title: '新闻动态-天津产权交易中心', + link: 'http://www.tprtc.com/InformationChannel/index.jhtml', + description: '天津产权交易中心-新闻动态', + item: items, + }; +}; diff --git a/lib/routes/tprtc/qyzc.js b/lib/routes/tprtc/qyzc.js new file mode 100644 index 000000000000..a8ccad5745fe --- /dev/null +++ b/lib/routes/tprtc/qyzc.js @@ -0,0 +1,84 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const url = require('url'); + +module.exports = async (ctx) => { + const response = await got({ + method: 'get', + url: 'http://item.tprtc.com/tjcqp/s/jyzx/qyzc_li?plfs=&gpjg=&szdq=&sortTag=0&pageSize=20&pageNo=1&keyWord=', + headers: { + Referer: 'http://item.tprtc.com/tjcqp/s/jyzx/qyzc', + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3793.0 Safari/537.36', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + }); + + const data = response.data; + + const $ = cheerio.load(data); + const list = $('li').get(); + + const ProcessFeed = async (link) => { + const response = await got({ + method: 'get', + url: link, + headers: { + Referer: 'http://item.tprtc.com/tjcqp/s/jyzx/qyzc', + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3793.0 Safari/537.36', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + }, + }); + const $ = cheerio.load(response.data); + + // 删除没用的tab指示 + $('.bd_detail_left .bd_detail_scroll').remove(); + $('.bd_detail_left .property_tab').remove(); + $('.bd_detail_left .bd_detail_follow').remove(); + $('.bd_detail_left [id^="clob_formweb_value_"]').remove(); + + // 文章样式 + const style = ` + `; + + // 提取内容 + const description = $('.bd_detail_left').html() + style; + + return description; + }; + + const host = 'http://item.tprtc.com'; + const items = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + + const $title = $('.bdlist_title>a'); + const itemUrl = url.resolve(host, $title.attr('href')); + + const cache = await ctx.cache.get(itemUrl); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const description = await ProcessFeed(itemUrl); + + const single = { + title: $title.text(), + link: itemUrl, + description, + author: $('.bdlist_time').text(), + }; + + ctx.cache.set(itemUrl, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + + ctx.state.data = { + title: '企业资产转让-天津产权交易中心', + link: 'http://item.tprtc.com/tjcqp/s/jyzx/qyzc', + description: '天津产权交易中心-企业资产转让', + item: items, + }; +}; From 31b7f32b065884814b1de8e4ce6df9387c75ba18 Mon Sep 17 00:00:00 2001 From: alizeegod Date: Tue, 2 Jul 2019 11:20:48 +0800 Subject: [PATCH 11/38] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E5=B1=95=E7=A4=BA=20(#2530)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [feature] 添加掌上英雄联盟-推荐新闻 * [feature] 修改图片展示 * [feature] 修改图片展示 * [feature] 防跨域 --- lib/routes/lolapp/recommend.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/routes/lolapp/recommend.js b/lib/routes/lolapp/recommend.js index 15b7687d680e..1e6c582017fd 100644 --- a/lib/routes/lolapp/recommend.js +++ b/lib/routes/lolapp/recommend.js @@ -22,9 +22,7 @@ module.exports = async (ctx) => { time.setTime(time.getTime() + (sourceTimezoneOffset - time.getTimezoneOffset() / 60) * 60 * 60 * 1000); const pubDate = time.toUTCString(); - // const detailJsonLink = article.detail_json; - // const detailRes = await got.get(detailJsonLink); - const description = article.image_url_big; + const description = ''; const item = { title, From 01c22cee071161dcfa45697688d7de47ecee9712 Mon Sep 17 00:00:00 2001 From: SettingDust Date: Wed, 3 Jul 2019 11:56:52 +0800 Subject: [PATCH 12/38] feature: ceic (#2524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix the scripts run in Windows * Feature: 中国地震台 * 添加item数量限制 * 添加类型检查 * Add doc * Revert --- docs/forecast.md | 28 +++++++++------ lib/router.js | 3 ++ lib/routes/earthquake/ceic.js | 65 ++++++++++++++++++++++++++++++++++ lib/routes/earthquake/index.js | 6 +--- package.json | 2 +- 5 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 lib/routes/earthquake/ceic.js diff --git a/docs/forecast.md b/docs/forecast.md index 60cc20eaf03a..33f0e1821c93 100644 --- a/docs/forecast.md +++ b/docs/forecast.md @@ -4,6 +4,24 @@ pageClass: routes # 预报预警 +## 地震速报 + +### 中国地震局 + + + +可通过全局过滤参数订阅您感兴趣的地区. + + + +### 中国地震台 + + + +可通过全局过滤参数订阅您感兴趣的地区. + + + ## 停电通知 获取未来一天的停电通知 @@ -70,16 +88,6 @@ pageClass: routes -## 中国地震局 - -### 地震速报 - - - -可通过全局过滤参数订阅您感兴趣的地区. - - - ## 中央气象台 ### 全国气象预警 diff --git a/lib/router.js b/lib/router.js index 833a29feb11e..4943eca0f6ae 100644 --- a/lib/router.js +++ b/lib/router.js @@ -484,6 +484,9 @@ router.get('/mafengwo/note/:type', require('./routes/mafengwo/note')); // 中国地震局震情速递（与地震台网同步更新） router.get('/earthquake/:region?', require('./routes/earthquake')); +// 中国地震台网 +router.get('/earthquake/ceic/:type', require('./routes/earthquake/ceic')); + // 笔趣阁 router.get('/biquge/novel/latestchapter/:id', require('./routes/novel/biquge')); diff --git a/lib/routes/earthquake/ceic.js b/lib/routes/earthquake/ceic.js new file mode 100644 index 000000000000..814f0cdb273e --- /dev/null +++ b/lib/routes/earthquake/ceic.js @@ -0,0 +1,65 @@ +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + let type = Number(ctx.params.type); + type = type ? type : 1; + const baseUrl = 'http://www.ceic.ac.cn'; + const api = `${baseUrl}/ajax/speedsearch?num=${type}`; + const mappings = { + O_TIME: '发震时刻(UTC+8)', + LOCATION_C: '参考位置', + M: '震级(M)', + EPI_LAT: '纬度(°)', + EPI_LON: '经度(°)', + EPI_DEPTH: '深度(千米)', + SAVE_TIME: '录入时间', + }; + + const typeMappings = { + 1: '最近24小时地震信息', + 2: '最近48小时地震信息', + 3: '最近7天地震信息', + 4: '最近30天地震信息', + 5: '最近一年3.0级以上地震信息', + 6: '最近一年地震信息', + 7: '最近一年3.0级以下地震', + 8: '最近一年4.0级以上地震信息', + 9: '最近一年5.0级以上地震信息', + 0: '最近一年6.0级以上地震信息', + }; + + // 因为item数量限制20，所以对于RSS来说为无用类型，防止浪费资源 + const disabledType = [3, 4, 6]; + + if (disabledType.includes(type)) { + type = 1; + } + const typeName = typeMappings[type]; + + const response = await got(api); + const data = response.data; + let json = JSON.parse(data.substring(1, data.length - 1)).shuju; + if (json.length > 20) { + json = json.slice(0, 20); + } + + ctx.state.data = { + title: typeName, + link: `${baseUrl}/speedsearch`, + item: json.map((entity) => { + const contentBuilder = []; + const { NEW_DID } = entity; + for (const mappingsKey in mappings) { + contentBuilder.push(`${mappings[mappingsKey]} ${entity[mappingsKey]}`); + } + + return { + title: `${entity.LOCATION_C}发生${entity.M}级地震`, + link: `${baseUrl}/${NEW_DID}.html`, + pubDate: new Date(entity.O_TIME).toUTCString(), + description: contentBuilder.join('
'), + guid: NEW_DID, + }; + }), + }; +}; diff --git a/lib/routes/earthquake/index.js b/lib/routes/earthquake/index.js index 5c5a6ce9927b..6f76cd1d5dad 100644 --- a/lib/routes/earthquake/index.js +++ b/lib/routes/earthquake/index.js @@ -19,13 +19,9 @@ module.exports = async (ctx) => { const out = await Promise.all( data.map(async (item) => { - const id = item.id; - const epicenter = item.epicenter; + const { id, epicenter, latitudes, longitudes, depth } = item; const date = item.orig_time.slice(0, -2); const num = item.num_mag; - const latitudes = item.latitudes; - const longitudes = item.longitudes; - const depth = item.depth; const description = `北京时间${date}，${epicenter}（纬度${latitudes}度，经度${longitudes}度）发生${num}级地震，震源深度${depth}千米`; const single = { diff --git a/package.json b/package.json index 122fa38088a8..95fbc3c4a10f 100644 --- a/package.json +++ b/package.json @@ -118,4 +118,4 @@ "_moduleAliases": { "@": "./lib" } -} \ No newline at end of file +} From 9487b780cc8b0703e681ee7f89bc16813908c7ec Mon Sep 17 00:00:00 2001 From: Wang Junyu Date: Wed, 3 Jul 2019 12:01:15 +0800 Subject: [PATCH 13/38] =?UTF-8?q?feat:=20Ins=E7=9A=84=E5=B8=90=E5=8F=B7?= =?UTF-8?q?=E7=AE=80=E4=BB=8B=E5=92=8C=E5=A4=B4=E5=83=8F=20(#2532)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 微博的description和image * Add Instagram account's profile image URL. Signed-off-by: WangJunyu * 修改帐号简介 * fix CI build failed Signed-off-by: WangJunyu --- lib/routes/instagram/user.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/routes/instagram/user.js b/lib/routes/instagram/user.js index f51b546b93df..1be4bac0d663 100644 --- a/lib/routes/instagram/user.js +++ b/lib/routes/instagram/user.js @@ -1,5 +1,4 @@ const got = require('@/utils/got'); -const cheerio = require('cheerio'); module.exports = async (ctx) => { const id = ctx.params.id; @@ -15,8 +14,8 @@ module.exports = async (ctx) => { const data = JSON.parse(response.data.match(/