feat: add unpkg-white-list to detect sync unpkg files or not (#686)

see https://github.com/cnpm/unpkg-white-list

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit


- **New Features**
- Introduced a new configuration option `enableSyncUnpkgFilesWhiteList`
to enhance package version file synchronization.

- **Improvements**
- Enhanced logging in package version file operations for better
traceability.
- Simplified file redirection logic for improved performance and
readability.

- **Tests**
- Added test cases for the new `enableSyncUnpkgFilesWhiteList`
configuration to ensure reliability.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

app.ts

@@ -2,6 +2,7 @@ import path from 'path';
 import { readFile } from 'fs/promises';
 import { Application } from 'egg';
 import { ChangesStreamService } from './app/core/service/ChangesStreamService';
+
 declare module 'egg' {
   interface Application {
     binaryHTML: string;

app/core/event/SyncPackageVersionFile.ts

@@ -1,7 +1,8 @@
 import { Event, Inject } from '@eggjs/tegg';
 import {
-  EggAppConfig,
+  EggAppConfig, EggLogger,
 } from 'egg';
+import { ForbiddenError } from 'egg-errors';
 import { PACKAGE_VERSION_ADDED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED } from './index';
 import { getScopeAndName } from '../../common/PackageUtil';
 import { PackageManagerService } from '../service/PackageManagerService';
@@ -11,6 +12,8 @@ class SyncPackageVersionFileEvent {
   @Inject()
   protected readonly config: EggAppConfig;
   @Inject()
+  protected readonly logger: EggLogger;
+  @Inject()
   private readonly packageManagerService: PackageManagerService;
   @Inject()
   private readonly packageVersionFileService: PackageVersionFileService;
@@ -25,7 +28,17 @@ class SyncPackageVersionFileEvent {
     const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
       scope, name, version);
     if (!packageVersion) return;
-    await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
+    try {
+      await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
+    } catch (err) {
+      if (err instanceof ForbiddenError) {
+        this.logger.info('[SyncPackageVersionFileEvent.syncPackageVersionFile] ignore sync files, cause: %s',
+          err.message,
+        );
+        return;
+      }
+      throw err;
+    }
   }
 
   protected async syncPackageReadmeToLatestVersion(fullname: string) {

app/core/service/PackageVersionFileService.ts

@@ -7,27 +7,34 @@ import {
   SingletonProto,
   Inject,
 } from '@eggjs/tegg';
+import { ConflictError, ForbiddenError } from 'egg-errors';
+import semver from 'semver';
 import { AbstractService } from '../../common/AbstractService';
 import {
   calculateIntegrity,
+  getFullname,
 } from '../../common/PackageUtil';
 import { createTempDir, mimeLookup } from '../../common/FileUtil';
 import {
   PackageRepository,
 } from '../../repository/PackageRepository';
 import { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
 import { DistRepository } from '../../repository/DistRepository';
 import { PackageVersionFile } from '../entity/PackageVersionFile';
 import { PackageVersion } from '../entity/PackageVersion';
 import { Package } from '../entity/Package';
 import { PackageManagerService } from './PackageManagerService';
 import { CacheAdapter } from '../../common/adapter/CacheAdapter';
-import { ConflictError } from 'egg-errors';
+
+const unpkgWhiteListUrl = 'https://github.com/cnpm/unpkg-white-list';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageVersionFileService extends AbstractService {
+  @Inject()
+  private readonly packageVersionRepository: PackageVersionRepository;
   @Inject()
   private readonly packageRepository: PackageRepository;
   @Inject()
@@ -39,6 +46,12 @@ export class PackageVersionFileService extends AbstractService {
   @Inject()
   private readonly cacheAdapter: CacheAdapter;
 
+  #unpkgWhiteListCurrentVersion: string = '';
+  #unpkgWhiteListAllowPackages: Record<string, {
+    version: string;
+  }> = {};
+  #unpkgWhiteListAllowScopes: string[] = [];
+
   async listPackageVersionFiles(pkgVersion: PackageVersion, directory: string) {
     await this.#ensurePackageVersionFilesSync(pkgVersion);
     return await this.packageVersionFileRepository.listPackageVersionFiles(pkgVersion.packageVersionId, directory);
@@ -54,16 +67,58 @@ export class PackageVersionFileService extends AbstractService {
   async #ensurePackageVersionFilesSync(pkgVersion: PackageVersion) {
     const hasFiles = await this.packageVersionFileRepository.hasPackageVersionFiles(pkgVersion.packageVersionId);
     if (!hasFiles) {
-      const lockRes = await this.cacheAdapter.usingLock(`${pkgVersion.packageVersionId}:syncFiles`, 60, async () => {
+      const lockName = `${pkgVersion.packageVersionId}:syncFiles`;
+      const lockRes = await this.cacheAdapter.usingLock(lockName, 60, async () => {
         await this.syncPackageVersionFiles(pkgVersion);
       });
       // lock fail
       if (!lockRes) {
-        this.logger.warn('[package:version:syncPackageVersionFiles] check lock fail');
+        this.logger.warn('[package:version:syncPackageVersionFiles] check lock:%s fail', lockName);
         throw new ConflictError('Package version file sync is currently in progress. Please try again later.');
       }
     }
+  }
+
+  async #updateUnpkgWhiteList() {
+    if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
+    const whiteListScope = '';
+    const whiteListPackageName = 'unpkg-white-list';
+    const whiteListPackageVersion = await this.packageVersionRepository.findVersionByTag(
+      whiteListScope, whiteListPackageName, 'latest');
+    if (!whiteListPackageVersion) return;
+    // same version, skip update for performance
+    if (this.#unpkgWhiteListCurrentVersion === whiteListPackageVersion) return;
+
+    // update the new version white list
+    const { manifest } = await this.packageManagerService.showPackageVersionManifest(
+      whiteListScope, whiteListPackageName, whiteListPackageVersion, false, true);
+    if (!manifest) return;
+    this.#unpkgWhiteListCurrentVersion = manifest.version;
+    this.#unpkgWhiteListAllowPackages = manifest.allowPackages ?? {} as any;
+    this.#unpkgWhiteListAllowScopes = manifest.allowScopes ?? [] as any;
+    this.logger.info('[PackageVersionFileService.updateUnpkgWhiteList] version:%s, total %s packages, %s scopes',
+      whiteListPackageVersion,
+      Object.keys(this.#unpkgWhiteListAllowPackages).length,
+      this.#unpkgWhiteListAllowScopes.length,
+    );
+  }
 
+  async #checkPackageVersionInUnpkgWhiteList(pkgScope: string, pkgName: string, pkgVersion: string) {
+    if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
+    await this.#updateUnpkgWhiteList();
+
+    // check allow scopes
+    if (this.#unpkgWhiteListAllowScopes.includes(pkgScope)) return;
+
+    // check allow packages
+    const fullname = getFullname(pkgScope, pkgName);
+    const pkgConfig = this.#unpkgWhiteListAllowPackages[fullname];
+    if (!pkgConfig) {
+      throw new ForbiddenError(`"${fullname}" is not allow to unpkg files, see ${unpkgWhiteListUrl}`);
+    }
+    if (!pkgConfig.version || !semver.satisfies(pkgVersion, pkgConfig.version)) {
+      throw new ForbiddenError(`"${fullname}@${pkgVersion}" not satisfies "${pkgConfig.version}" to unpkg files, see ${unpkgWhiteListUrl}`);
+    }
   }
 
   // 基于 latest version 同步 package readme
@@ -113,8 +168,16 @@ export class PackageVersionFileService extends AbstractService {
 
   async syncPackageVersionFiles(pkgVersion: PackageVersion) {
     const files: PackageVersionFile[] = [];
+    // must set enableUnpkg and enableSyncUnpkgFiles = true both
+    if (!this.config.cnpmcore.enableUnpkg) return files;
+    if (!this.config.cnpmcore.enableSyncUnpkgFiles) return files;
+
     const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
     if (!pkg) return files;
+
+    // check unpkg white list
+    await this.#checkPackageVersionInUnpkgWhiteList(pkg.scope, pkg.name, pkgVersion.version);
+
     const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${pkgVersion.version}_${randomUUID()}`;
     const tmpdir = await createTempDir(this.config.dataDir, dirname);
     const tarFile = `${tmpdir}.tgz`;

app/port/config.ts

@@ -149,6 +149,10 @@ export type CnpmcoreConfig = {
    * enable sync unpkg files
    */
   enableSyncUnpkgFiles: boolean;
+  /**
+   * enable sync unpkg files from the white list, https://github.com/cnpm/unpkg-white-list
+   */
+  enableSyncUnpkgFilesWhiteList: boolean;
   /**
    * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag.
    * in most cases, you should set to false to keep the same behavior as source registry.

app/port/controller/PackageVersionFileController.ts

@@ -149,12 +149,9 @@ export class PackageVersionFileController extends AbstractController {
 
     if (!file) {
       const possibleFile = await this.#searchPossibleEntries(packageVersion, path);
-
       if (possibleFile) {
         const route = `/${fullname}/${versionSpec}/files${possibleFile.path}${hasMeta ? '?meta' : ''}`;
-
         ctx.redirect(route);
-
         return;
       }
 

app/port/controller/package/SavePackageVersionController.ts

@@ -221,6 +221,7 @@ export class SavePackageVersionController extends AbstractController {
     const registry = await this.registryManagerService.ensureSelfRegistry();
 
     let packageVersionEntity: PackageVersionEntity | undefined;
+    const lockName = `${pkg.name}:publish`;
     const lockRes = await this.cacheAdapter.usingLock(`${pkg.name}:publish`, 60, async () => {
       packageVersionEntity = await this.packageManagerService.publish({
         scope,
@@ -240,7 +241,7 @@ export class SavePackageVersionController extends AbstractController {
 
     // lock fail
     if (!lockRes) {
-      this.logger.warn('[package:version:add] check lock fail');
+      this.logger.warn('[package:version:add] check lock:%s fail', lockName);
       throw new ConflictError('Unable to create the publication lock, please try again later.');
     }
 

config/config.default.ts

@@ -54,6 +54,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
   redirectNotFound: true,
   enableUnpkg: true,
   enableSyncUnpkgFiles: true,
+  enableSyncUnpkgFilesWhiteList: false,
   strictSyncSpecivicVersion: false,
   enableElasticsearch: !!process.env.CNPMCORE_CONFIG_ENABLE_ES,
   elasticsearchIndex: 'cnpmcore_packages',