feat: minio proxy

scripts/ci/before.js

@@ -10,6 +10,7 @@ const packageLockFile = "package-lock.json";
 const registries = [
     "https://registry.npmjs.org/",
     "https://mirrors.cloud.tencent.com/npm/",
+    "https://registry.npmmirror.com/",
 ];
 
 await fs.promises.rm("./node_modules", { recursive: true, force: true });

scripts/minio/docker-compose.yaml

@@ -22,6 +22,8 @@ services:
       - GITHUB_PAT=<CHANGE_ME>
       - MINIO_ACCESS_KEY=<CHANGE_ME>
       - MINIO_SECRET_KEY=<CHANGE_ME>
+      - HTTP_PROXY=<CHANGE_ME>
+      - NO_PROXY=<CHANGE_ME>
       # - ANNANGELA_QQBOT_TOKEN=<CHANGE_ME>
     logging:
       driver: json-file

scripts/minio/index.js

@@ -8,6 +8,7 @@ import byteSize from "byte-size";
 import { CronJob } from "cron";
 import console from "../modules/console.js";
 import { Octokit } from "../modules/octokit.js";
+import proxiableFetch from "../modules/proxiableFetch.js";
 import retryableFetch from "../modules/retryableFetch.js";
 
 const getByteSize = (input) => {
@@ -40,20 +41,21 @@ const ua = `Node.js/${process.versions.node} (${process.platform} ${os.release()
 console.info("process.env.THREAD:", process.env.THREAD?.trim());
 const THREAD = getIntegerFromProcessEnv("THREAD", 4, 0);
 const NUMBER_OF_RETRIES = getIntegerFromProcessEnv("NUMBER_OF_RETRIES", 5, 0);
+const MINIO_WAIT_TIME_AFTER_UPLOAD_MS = getIntegerFromProcessEnv("MINIO_WAIT_TIME_AFTER_UPLOAD_MS", 1000, 0);
 console.info("# of thread:", THREAD);
+console.info("# of retries:", NUMBER_OF_RETRIES);
+console.info("MINIO_WAIT_TIME_AFTER_UPLOAD_MS:", MINIO_WAIT_TIME_AFTER_UPLOAD_MS);
 console.info("OWNER:", OWNER);
 console.info("REPO_LIST:", REPO_LIST);
 console.info("FILE_PATTERN:", FILE_PATTERN);
 console.info("pattern:", pattern);
 console.info("ua:", ua);
-const octokit = new Octokit({});
-const { token } = await octokit.auth();
-const headers = {
-    accept: "application/octet-stream",
-    authorization: `Bearer ${token}`,
-    "user-agent": ua,
-};
-const MINIO_WAIT_TIME_AFTER_UPLOAD_MS = getIntegerFromProcessEnv("MINIO_WAIT_TIME_AFTER_UPLOAD_MS", 1000, 0);
+
+const octokit = new Octokit({
+    request: {
+        fetch: proxiableFetch,
+    },
+});
 const minioClient = new Client({
     endPoint: process.env.MINIO_ENDPOINT_DOMAIN,
     port: +process.env.MINIO_ENDPOINT_PORT,
@@ -65,6 +67,14 @@ const minioClient = new Client({
         keepAlive: true,
     }),
 });
+
+const { token } = await octokit.auth();
+const headers = {
+    accept: "application/octet-stream",
+    authorization: `Bearer ${token}`,
+    "user-agent": ua,
+};
+
 /**
  * @typedef { Awaited<ReturnType<octokit["rest"]["repos"]["getRelease"]>>["data"]["assets"][number] & { releaseTag: string } } Asset
  */

scripts/modules/octokit.js

@@ -12,7 +12,11 @@ if (Reflect.has(process.env, "GITHUB_PAT")) {
     };
 }
 
-class OctokitWithRetry extends Octokit.plugin(retry) {
+const OctokitWithRetry = Octokit.plugin(retry);
+class OctokitWithRetryAndDefaultAuthOptions extends OctokitWithRetry {
+    /**
+     * @param { ConstructorParameters<OctokitWithRetry>[0] } options
+     */
     constructor(options) {
         super({
             ...defaultAuthOptions,
@@ -21,5 +25,5 @@ class OctokitWithRetry extends Octokit.plugin(retry) {
     }
 }
 
-export { OctokitWithRetry as Octokit };
-export default { OctokitWithRetry: Octokit };
+export { OctokitWithRetryAndDefaultAuthOptions as Octokit };
+export default { OctokitWithRetryWithDefaultAuthOptions: Octokit };

scripts/modules/proxiableFetch.js

@@ -0,0 +1,18 @@
+import { ProxyAgent } from "undici";
+import minimatch from "minimatch";
+
+const proxyAgent = new ProxyAgent(process.env.HTTP_PROXY);
+const noProxyDomains = (process.env.NO_PROXY || "").split(",");
+
+const shouldUseProxy = (input) => {
+    const hostname = URL.parse(input).hostname;
+    return !noProxyDomains.some((domain) => minimatch(hostname, domain));
+};
+
+const proxiableFetch = (input, init = {}) => {
+    if (shouldUseProxy(input)) {
+        init.dispatcher = proxyAgent;
+    }
+    return fetch(input, init);
+};
+export default proxiableFetch;

scripts/package.json

@@ -8,8 +8,10 @@
         "byte-size": "^8.1.1",
         "cron": "^3.1.6",
         "eslint": "^8.57.0",
+        "minimatch": "^9.0.4",
         "minio": "^7.1.3",
-        "simple-git": "^3.24.0"
+        "simple-git": "^3.24.0",
+        "undici": "^6.12.0"
     },
     "scripts": {
         "ci": "export RANDOM_UUID=$(uuidgen) && node ci/before.js && npm ci && node ci/after.js",