Skip to content
Permalink
Browse files Browse the repository at this point in the history
feat(config)!: unsafe domain toggle (#11588)
  • Loading branch information
TonyRL committed Jan 10, 2023
1 parent ab2f618 commit a66cbcf
Show file tree
Hide file tree
Showing 66 changed files with 339 additions and 16 deletions.
4 changes: 3 additions & 1 deletion docs/en/install/README.md
Expand Up @@ -564,14 +564,16 @@ It is also valid to contain route parameters, e.g. `/weibo/user/2612249974`.

::: tip Experimental features

Configs in this sections are in beta stage, and are turn off by default. Please read corresponded description and turn on if necessary.
Configs in this sections are in beta stage, and **are turn off by default**. Please read corresponded description and turn on if necessary.

:::

`ALLOW_USER_HOTLINK_TEMPLATE`: [Parameters->Multimedia processing](/en/parameter.html#multimedia-processing)

`FILTER_REGEX_ENGINE`: Define Regex engine used in [Parameters->filtering](/en/parameter.html#filtering). Valid value are `[re2, regexp]`. Default value is `re2`. We suggest public instance should leave this value to default, and this option right now is mainly for backward compatibility.

`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN`: allow users to provide a domain as a parameter to routes that are not in their allow list, respectively. Public instances are suggested to leave this value default, as it may lead to [Server-Side Request Forgery (SSRF)](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery)

### Other Application Configurations

`DISALLOW_ROBOT`: prevent indexing by search engine, default to enable, set false or 0 to disable
Expand Down
4 changes: 3 additions & 1 deletion docs/install/README.md
Expand Up @@ -571,14 +571,16 @@ RSSHub 支持使用访问密钥 / 码,白名单和黑名单三种方式进行

::: tip 测试特性

这个板块控制的是一些新特性的选项,默认他们都是关闭的。如果有需要请阅读对应说明后按需开启
这个板块控制的是一些新特性的选项,他们都是**默认关闭**。如果有需要请阅读对应说明后按需开启

:::

`ALLOW_USER_HOTLINK_TEMPLATE`: [通用参数 -> 多媒体处理](/parameter.html#duo-mei-ti-chu-li)特性控制

`FILTER_REGEX_ENGINE`: 控制 [通用参数 -> 内容过滤](/parameter.html#nei-rong-guo-lu) 使用的正则引擎。可选`[re2, regexp]`,默认`re2`。我们推荐公开实例不要调整这个选项,这个选项目前主要用于向后兼容。

`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN`: 允许用户为路由提供域名作为参数。建议公共实例不要调整此选项,开启后可能会导致 [服务端请求伪造(SSRF)](https://owasp.org/www-community/attacks/Server_Side_Request_Forgery)

### 其他应用配置

`DISALLOW_ROBOT`: 阻止搜索引擎收录,默认开启,设置 false 或 0 关闭
Expand Down
1 change: 1 addition & 0 deletions lib/config.js
Expand Up @@ -97,6 +97,7 @@ const calculateValue = () => {
feature: {
allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true',
},
suffix: envs.SUFFIX,
titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/bandisoft/index.js
@@ -1,9 +1,13 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const lang = ctx.params.lang || 'en';
const id = ctx.params.id || 'bandizip';
if (!isValidHost(lang)) {
throw Error('Invalid language code');
}

const rootUrl = `https://${lang}.bandisoft.com`;
const currentUrl = `${rootUrl}/${id}/history/`;
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/biobio/others.js
@@ -1,7 +1,12 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
if (!isValidHost(ctx.params.column)) {
throw Error('Invalid column');
}

const url = `http://${ctx.params.column}.bio1000.com/${ctx.params.id}`;
const res = await got.get(url);
const $ = cheerio.load(res.data);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/blogs/hedwig.js
Expand Up @@ -4,9 +4,13 @@ const md = require('markdown-it')({
html: true,
});
const dayjs = require('dayjs');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const type = ctx.params.type;
if (!isValidHost(type)) {
throw Error('Invalid type');
}

const url = `https://${type}.hedwig.pub`;
const res = await got({
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/blogs/wordpress.js
@@ -1,7 +1,12 @@
const parser = require('@/utils/rss-parser');
const config = require('@/config').value;
const allowDomain = ['lawrence.code.blog'];

module.exports = async (ctx) => {
if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(ctx.params.domain)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const scheme = ctx.params.https || 'https';
const cdn = config.wordpress.cdnUrl;

Expand Down
5 changes: 4 additions & 1 deletion lib/routes/booth-pm/shop.js
@@ -1,10 +1,13 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');

const { isValidHost } = require('@/utils/valid-host');
const maxPages = 5;

module.exports = async (ctx) => {
const { subdomain } = ctx.params;
if (!isValidHost(subdomain)) {
throw Error('Invalid subdomain');
}
const shopUrl = `https://${subdomain}.booth.pm`;

let shopName;
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/caixin/blog.js
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const parser = require('@/utils/rss-parser');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

async function load(link, need_feed_description) {
const response = await got.get(link);
Expand Down Expand Up @@ -29,6 +30,9 @@ async function load(link, need_feed_description) {

module.exports = async (ctx) => {
const { column } = ctx.params;
if (!isValidHost(column)) {
throw Error('Invalid column');
}
const link = `http://${column}.blog.caixin.com`;
const feed_url = `${link}/feed`;
const feed = await parser.parseURL(feed_url);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/caixin/category.js
@@ -1,10 +1,14 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const category = ctx.params.category;
const column = ctx.params.column;
const url = `http://${column}.caixin.com/${category}`;
if (!isValidHost(column)) {
throw Error('Invalid column');
}

const response = await got({
method: 'get',
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/craigslist/search.js
@@ -1,7 +1,12 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
if (!isValidHost(ctx.params.location)) {
throw Error('Invalid location');
}

const queryParams = ctx.request.querystring;
const queryUrl = `https://${ctx.params.location}.craigslist.org/search/${ctx.params.type}?${queryParams}`;
const { data } = await got.get(queryUrl);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/engadget/home.js
@@ -1,9 +1,13 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const parser = require('@/utils/rss-parser');
const allowLang = ['chinese', 'cn', 'us', 'japanese', 'www'];

module.exports = async (ctx) => {
const lang = ctx.params.lang === 'us' ? 'www' : ctx.params.lang || 'cn';
if (!allowLang.includes(lang)) {
throw Error('Invalid lang');
}
const rssUrl = `https://${lang}.engadget.com/rss.xml`;
const feed = await parser.parseURL(rssUrl);

Expand Down
5 changes: 4 additions & 1 deletion lib/routes/fanbox/main.js
Expand Up @@ -4,12 +4,15 @@
// user?: fanbox domain name

const got = require('@/utils/got');

const { isValidHost } = require('@/utils/valid-host');
const conv_item = require('./conv');
const get_header = require('./header');

module.exports = async (ctx) => {
const user = ctx.params.user || 'official'; // if no user specified, just go to official page
if (!isValidHost(user)) {
throw Error('Invalid user');
}
const box_url = `https://${user}.fanbox.cc`;

// get user info
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/fashionnetwork/headline.js
@@ -1,8 +1,12 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const country = ctx.params.country || 'ww';
if (!isValidHost(country)) {
throw Error('Invalid country');
}

const rootUrl = `https://${country}.fashionnetwork.com`;
const response = await got({
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/fashionnetwork/news.js
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const country = ctx.params.country || 'ww';
Expand All @@ -12,6 +13,10 @@ module.exports = async (ctx) => {
const sectorsUrl = sectors ? 'sectors%5B%5D=' + sectors.split(',').join('&sectors%5B%5D=') : '';
const categoriesUrl = categories ? 'categs%5B%5D=' + categories.split(',').join('&categs%5B%5D=') : '';

if (!isValidHost(country)) {
throw Error('Invalid country');
}

const rootUrl = `https://${country}.fashionnetwork.com`;
const currentUrl = `${rootUrl}/news/s.jsonp?${sectorsUrl}&${categoriesUrl}`;
const response = await got({
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/gitlab/common.js
@@ -0,0 +1,5 @@
const allowHost = ['gitlab.com'];

module.exports = {
allowHost,
};
5 changes: 5 additions & 0 deletions lib/routes/gitlab/explore.js
@@ -1,5 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const config = require('@/config').value;
const { allowHost } = require('./common');

module.exports = async (ctx) => {
let { type, host } = ctx.params;
Expand All @@ -10,6 +12,9 @@ module.exports = async (ctx) => {
starred: 'Most stars',
all: 'All',
};
if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(new URL(host).hostname)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const res = await got({
method: 'get',
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/gitlab/release.js
@@ -1,8 +1,13 @@
const got = require('@/utils/got');
const { parseDate } = require('@/utils/parse-date');
const config = require('@/config').value;
const { allowHost } = require('./common');

module.exports = async (ctx) => {
const { namespace, project, host } = ctx.params;
if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const host_ = host ? host : 'gitlab.com';
const namespace_ = encodeURIComponent(namespace);
Expand Down
5 changes: 5 additions & 0 deletions lib/routes/gitlab/tag.js
@@ -1,8 +1,13 @@
const got = require('@/utils/got');
const { parseDate } = require('@/utils/parse-date');
const config = require('@/config').value;
const { allowHost } = require('./common');

module.exports = async (ctx) => {
const { namespace, project, host } = ctx.params;
if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const host_ = host ? host : 'gitlab.com';
const namespace_ = encodeURIComponent(namespace);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/hexo/fluid.js
@@ -1,7 +1,11 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
const config = require('@/config').value;

module.exports = async (ctx) => {
if (!config.feature.allow_user_supply_unsafe_domain) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}
const url = `http://${ctx.params.url}`;
const res = await got.get(`${url}/archives/`);
const $ = cheerio.load(res.data);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/hexo/next.js
@@ -1,7 +1,11 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
const config = require('@/config').value;

module.exports = async (ctx) => {
if (!config.feature.allow_user_supply_unsafe_domain) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}
const url = `http://${ctx.params.url}`;
const res = await got.get(`${url}/archives/`);
const $ = cheerio.load(res.data);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/hexo/yilia.js
@@ -1,7 +1,11 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
const config = require('@/config').value;

module.exports = async (ctx) => {
if (!config.feature.allow_user_supply_unsafe_domain) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}
const url = `http://${ctx.params.url}`;
const res = await got.get(url);
const $ = cheerio.load(res.data);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/mastodon/account_id.js
@@ -1,9 +1,13 @@
const utils = require('./utils');
const config = require('@/config').value;

module.exports = async (ctx) => {
const site = ctx.params.site;
const account_id = ctx.params.account_id;
const only_media = ctx.params.only_media ? 'true' : 'false';
if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const { account_data, data } = await utils.getAccountStatuses(site, account_id, only_media);

Expand Down
4 changes: 4 additions & 0 deletions lib/routes/mastodon/timeline_local.js
@@ -1,9 +1,13 @@
const got = require('@/utils/got');
const utils = require('./utils');
const config = require('@/config').value;

module.exports = async (ctx) => {
const site = ctx.params.site;
const only_media = ctx.params.only_media ? 'true' : 'false';
if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const url = `http://${site}/api/v1/timelines/public?local=true&only_media=${only_media}`;

Expand Down
4 changes: 4 additions & 0 deletions lib/routes/mastodon/timeline_remote.js
@@ -1,9 +1,13 @@
const got = require('@/utils/got');
const utils = require('./utils');
const config = require('@/config').value;

module.exports = async (ctx) => {
const site = ctx.params.site;
const only_media = ctx.params.only_media ? 'true' : 'false';
if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
}

const url = `http://${site}/api/v1/timelines/public?remote=true&only_media=${only_media}`;

Expand Down
3 changes: 3 additions & 0 deletions lib/routes/mastodon/utils.js
@@ -1,6 +1,8 @@
const got = require('@/utils/got');
const { parseDate } = require('@/utils/parse-date');

const allowSiteList = ['mastodon.social', 'pawoo.net'];

const parseStatuses = (data) =>
data.map((item) => {
// docs on: https://docs.joinmastodon.org/entities/status/
Expand Down Expand Up @@ -125,4 +127,5 @@ module.exports = {
parseStatuses,
getAccountStatuses,
getAccountIdByAcct,
allowSiteList,
};
4 changes: 4 additions & 0 deletions lib/routes/pornhub/category_url.js
@@ -1,10 +1,14 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const language = ctx.params.language || 'www';
const url = ctx.params.url || 'video';
const link = `https://${language}.pornhub.com/${url}`;
if (!isValidHost(language)) {
throw Error('Invalid language');
}

const response = await got.get(link);
const $ = cheerio.load(response.data);
Expand Down
4 changes: 4 additions & 0 deletions lib/routes/pornhub/model.js
@@ -1,11 +1,15 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');

module.exports = async (ctx) => {
const language = ctx.params.language || 'www';
const username = ctx.params.username;
const sort = ctx.params.sort || 'mr';
const link = `https://${language}.pornhub.com/model/${username}/videos?o=${sort}`;
if (!isValidHost(language)) {
throw Error('Invalid language');
}

const response = await got.get(link);
const $ = cheerio.load(response.data);
Expand Down

0 comments on commit a66cbcf

Please sign in to comment.