From 872d3f69db92101edae341979632ae41a98f9e21 Mon Sep 17 00:00:00 2001 From: mei23 Date: Sun, 12 Apr 2020 01:52:56 +0900 Subject: [PATCH 1/4] =?UTF-8?q?=E3=83=97=E3=83=AD=E3=82=AD=E3=82=B7?= =?UTF-8?q?=E3=81=AE=E9=99=A4=E5=A4=96=E3=83=9B=E3=82=B9=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .config/example.yml | 5 +++++ src/config/types.ts | 1 + src/misc/download-url.ts | 4 ++-- src/misc/fetch.ts | 32 +++++++++++++++++++++---------- src/remote/activitypub/request.ts | 4 ++-- src/services/drive/s3.ts | 10 ++++++++-- 6 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 201082cce9b6..1794dc9a8bfd 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -142,6 +142,11 @@ id: 'aid' # Proxy for HTTP/HTTPS #proxy: http://127.0.0.1:3128 +#proxyBypassHosts: [ +# 'example.com', +# '192.0.2.8' +#] + # Proxy for SMTP/SMTPS #proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 diff --git a/src/config/types.ts b/src/config/types.ts index a33901bde6aa..4f025750b0eb 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -35,6 +35,7 @@ export type Source = { proxy?: string; proxySmtp?: string; + proxyBypassHosts?: string[]; accesslog?: string; diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts index 3f42fb3bef7d..9c8439f2c0af 100644 --- a/src/misc/download-url.ts +++ b/src/misc/download-url.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as stream from 'stream'; import * as util from 'util'; import fetch from 'node-fetch'; -import { httpAgent, httpsAgent } from './fetch'; +import { getAgentByUrl } from './fetch'; import { AbortController } from 'abort-controller'; import config from '../config'; import * as chalk from 'chalk'; @@ -25,7 +25,7 @@ export async function downloadUrl(url: string, path: string) { }, timeout: 10 * 1000, signal: controller.signal, - agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent, + agent: getAgentByUrl, }); if (!response.ok) { diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts index 887aae165963..4a9c4f927dea 100644 --- a/src/misc/fetch.ts +++ b/src/misc/fetch.ts @@ -13,7 +13,7 @@ export async function getJson(url: string, accept = 'application/json, */*', tim Accept: accept }, headers || {}), timeout, - agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent, + agent: getAgentByUrl, }); if (!res.ok) { @@ -27,17 +27,29 @@ export async function getJson(url: string, accept = 'application/json, */*', tim return await res.json(); } +const _http = new http.Agent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, +}); + +const _https = new https.Agent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + lookup: cache.lookup, +}); + export const httpAgent = config.proxy ? new HttpProxyAgent(config.proxy) - : new http.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - }); + : _http; export const httpsAgent = config.proxy ? new HttpsProxyAgent(config.proxy) - : new https.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, - }); + : _https; + +export function getAgentByUrl(url: URL) { + if ((config.proxyBypassHosts || []).includes(url.hostname)) { + return url.protocol == 'http:' ? _http : _https; + } else { + return url.protocol == 'http:' ? httpAgent : httpsAgent; + } +} diff --git a/src/remote/activitypub/request.ts b/src/remote/activitypub/request.ts index 24540827b233..ab3ae025a5fe 100644 --- a/src/remote/activitypub/request.ts +++ b/src/remote/activitypub/request.ts @@ -6,7 +6,7 @@ import config from '../../config'; import { ILocalUser } from '../../models/entities/user'; import { UserKeypairs } from '../../models'; import { ensure } from '../../prelude/ensure'; -import { httpsAgent } from '../../misc/fetch'; +import { getAgentByUrl } from '../../misc/fetch'; export default async (user: ILocalUser, url: string, object: any) => { const timeout = 10 * 1000; @@ -25,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => { await new Promise((resolve, reject) => { const req = https.request({ - agent: httpsAgent, + agent: getAgentByUrl(new URL(`https://example.net`)), protocol, hostname, port, diff --git a/src/services/drive/s3.ts b/src/services/drive/s3.ts index 2cbeef106de7..1dceaf0b648a 100644 --- a/src/services/drive/s3.ts +++ b/src/services/drive/s3.ts @@ -1,8 +1,14 @@ import * as S3 from 'aws-sdk/clients/s3'; import { Meta } from '../../models/entities/meta'; -import { httpsAgent, httpAgent } from '../../misc/fetch'; +import { getAgentByUrl } from '../../misc/fetch'; export function getS3(meta: Meta) { + const u = meta.objectStorageEndpoint != null + ? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}` + : `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`; + + const agent = getAgentByUrl(new URL(u)); + return new S3({ endpoint: meta.objectStorageEndpoint || undefined, accessKeyId: meta.objectStorageAccessKey!, @@ -11,7 +17,7 @@ export function getS3(meta: Meta) { sslEnabled: meta.objectStorageUseSSL, s3ForcePathStyle: !!meta.objectStorageEndpoint, httpOptions: { - agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent + agent } }); } From ed8600db08da67f3313fc5dccec245d3aaafe41d Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Sun, 12 Apr 2020 02:08:49 +0900 Subject: [PATCH 2/4] =?UTF-8?q?=E3=82=AA=E3=83=96=E3=82=B8=E3=82=A7?= =?UTF-8?q?=E3=82=AF=E3=83=88=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=81=A8=E3=81=AE=E9=80=9A=E4=BF=A1=E3=81=ABProxy=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=86=E3=81=8B=E3=82=92=E9=81=B8=E6=8A=9E=E3=81=A7?= =?UTF-8?q?=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- locales/ja-JP.yml | 2 ++ .../1586624197029-AddObjectStorageUseProxy.ts | 14 ++++++++++++++ src/client/pages/instance/settings.vue | 4 ++++ src/models/entities/meta.ts | 5 +++++ src/server/api/endpoints/admin/update-meta.ts | 8 ++++++++ src/server/api/endpoints/meta.ts | 1 + src/services/drive/s3.ts | 2 +- 7 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 migration/1586624197029-AddObjectStorageUseProxy.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 29a4c2d3d6a4..08270915326a 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -454,6 +454,8 @@ objectStorageRegion: "Region" objectStorageRegionDesc: "'xx-east-1'のようなregionを指定してください。使用サービスにregionの概念がない場合は、空または'us-east-1'にしてください。" objectStorageUseSSL: "SSLを使用する" objectStorageUseSSLDesc: "API接続にhttpsを使用しない場合はオフにしてください" +objectStorageUseProxy: "Proxyを利用する" +objectStorageUseProxyDesc: "API接続にproxyを利用しない場合はオフにしてください" serverLogs: "サーバーログ" deleteAll: "全て削除" showFixedPostForm: "タイムライン上部に投稿フォームを表示する" diff --git a/migration/1586624197029-AddObjectStorageUseProxy.ts b/migration/1586624197029-AddObjectStorageUseProxy.ts new file mode 100644 index 000000000000..deadf94834d0 --- /dev/null +++ b/migration/1586624197029-AddObjectStorageUseProxy.ts @@ -0,0 +1,14 @@ +import {MigrationInterface, QueryRunner} from 'typeorm'; + +export class AddObjectStorageUseProxy1586624197029 implements MigrationInterface { + name = 'AddObjectStorageUseProxy1586624197029' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageUseProxy" boolean NOT NULL DEFAULT true`, undefined); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageUseProxy"`, undefined); + } + +} diff --git a/src/client/pages/instance/settings.vue b/src/client/pages/instance/settings.vue index f0a123f279e0..f7db4aa10bcf 100644 --- a/src/client/pages/instance/settings.vue +++ b/src/client/pages/instance/settings.vue @@ -116,6 +116,7 @@ Secret key {{ $t('objectStorageUseSSL') }} + {{ $t('objectStorageUseProxy') }}
@@ -249,6 +250,7 @@ export default Vue.extend({ objectStorageAccessKey: null, objectStorageSecretKey: null, objectStorageUseSSL: false, + objectStorageUseProxy: false, enableTwitterIntegration: false, twitterConsumerKey: null, twitterConsumerSecret: null, @@ -303,6 +305,7 @@ export default Vue.extend({ this.objectStorageAccessKey = this.meta.objectStorageAccessKey; this.objectStorageSecretKey = this.meta.objectStorageSecretKey; this.objectStorageUseSSL = this.meta.objectStorageUseSSL; + this.objectStorageUseProxy = this.meta.objectStorageUseProxy; this.enableTwitterIntegration = this.meta.enableTwitterIntegration; this.twitterConsumerKey = this.meta.twitterConsumerKey; this.twitterConsumerSecret = this.meta.twitterConsumerSecret; @@ -411,6 +414,7 @@ export default Vue.extend({ objectStorageAccessKey: this.objectStorageAccessKey ? this.objectStorageAccessKey : null, objectStorageSecretKey: this.objectStorageSecretKey ? this.objectStorageSecretKey : null, objectStorageUseSSL: this.objectStorageUseSSL, + objectStorageUseProxy: this.objectStorageUseProxy, enableTwitterIntegration: this.enableTwitterIntegration, twitterConsumerKey: this.twitterConsumerKey, twitterConsumerSecret: this.twitterConsumerSecret, diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts index ee62cb24baec..bb463c52f404 100644 --- a/src/models/entities/meta.ts +++ b/src/models/entities/meta.ts @@ -348,4 +348,9 @@ export class Meta { default: true, }) public objectStorageUseSSL: boolean; + + @Column('boolean', { + default: true, + }) + public objectStorageUseProxy: boolean; } diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts index e9527d9ac52c..b470c2339a8b 100644 --- a/src/server/api/endpoints/admin/update-meta.ts +++ b/src/server/api/endpoints/admin/update-meta.ts @@ -394,6 +394,10 @@ export const meta = { objectStorageUseSSL: { validator: $.optional.bool }, + + objectStorageUseProxy: { + validator: $.optional.bool + } } }; @@ -632,6 +636,10 @@ export default define(meta, async (ps, me) => { set.objectStorageUseSSL = ps.objectStorageUseSSL; } + if (ps.objectStorageUseProxy !== undefined) { + set.objectStorageUseProxy = ps.objectStorageUseProxy; + } + await getConnection().transaction(async transactionalEntityManager => { const meta = await transactionalEntityManager.findOne(Meta, { order: { diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts index 280a6fbae6d1..179355489b0a 100644 --- a/src/server/api/endpoints/meta.ts +++ b/src/server/api/endpoints/meta.ts @@ -190,6 +190,7 @@ export default define(meta, async (ps, me) => { response.objectStorageAccessKey = instance.objectStorageAccessKey; response.objectStorageSecretKey = instance.objectStorageSecretKey; response.objectStorageUseSSL = instance.objectStorageUseSSL; + response.objectStorageUseProxy = instance.objectStorageUseProxy; } return response; diff --git a/src/services/drive/s3.ts b/src/services/drive/s3.ts index 2cbeef106de7..cf327320227b 100644 --- a/src/services/drive/s3.ts +++ b/src/services/drive/s3.ts @@ -11,7 +11,7 @@ export function getS3(meta: Meta) { sslEnabled: meta.objectStorageUseSSL, s3ForcePathStyle: !!meta.objectStorageEndpoint, httpOptions: { - agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent + agent: meta.objectStorageUseProxy ? (meta.objectStorageUseProxy ? httpsAgent : httpAgent) : undefined } }); } From 703c262c90f28f03c3f33514aa8ffc535eb8d94f Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Sun, 12 Apr 2020 02:13:29 +0900 Subject: [PATCH 3/4] fix lint --- src/server/api/endpoints/admin/update-meta.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts index b470c2339a8b..1bc20029ef7b 100644 --- a/src/server/api/endpoints/admin/update-meta.ts +++ b/src/server/api/endpoints/admin/update-meta.ts @@ -394,7 +394,7 @@ export const meta = { objectStorageUseSSL: { validator: $.optional.bool }, - + objectStorageUseProxy: { validator: $.optional.bool } From 68abd2987b0b274f3a98ab8268151b4e4350c85a Mon Sep 17 00:00:00 2001 From: mei23 Date: Sun, 12 Apr 2020 06:26:23 +0900 Subject: [PATCH 4/4] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc/fetch.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts index d813d91380f0..358bc25030e3 100644 --- a/src/misc/fetch.ts +++ b/src/misc/fetch.ts @@ -27,25 +27,42 @@ export async function getJson(url: string, accept = 'application/json, */*', tim return await res.json(); } +/** + * Get http non-proxy agent + */ const _http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, }); +/** + * Get https non-proxy agent + */ const _https = new https.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, lookup: cache.lookup, }); +/** + * Get http proxy or non-proxy agent + */ export const httpAgent = config.proxy ? new HttpProxyAgent(config.proxy) : _http; +/** + * Get https proxy or non-proxy agent + */ export const httpsAgent = config.proxy ? new HttpsProxyAgent(config.proxy) : _https; +/** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ export function getAgentByUrl(url: URL, bypassProxy = false) { if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) { return url.protocol == 'http:' ? _http : _https;