From 80326aed8b73f588bd8842153ef8ced8aec69778 Mon Sep 17 00:00:00 2001 From: Denis Bashkatov <2485095+Denrox@users.noreply.github.com> Date: Wed, 1 Oct 2025 18:11:27 +0300 Subject: [PATCH 1/4] public packages --- README.md | 48 ++- .../routes/documentation/documentation.tsx | 70 ++++- admin/app/routes/npm/npm.tsx | 286 +++++++++++++++++- admin/app/utils/server-auth.ts | 18 ++ 4 files changed, 403 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index bb97029..dad8212 100644 --- a/README.md +++ b/README.md @@ -83,11 +83,13 @@ The script will: ### NPM Proxy (npm.mirror.intra) - Optional - **URL**: `http://npm.mirror.intra` -- **Purpose**: Local npm package registry cache +- **Purpose**: Local npm package registry cache and private package hosting - **Features**: - Caches npm packages locally for faster installs - Reduces bandwidth usage - Transparent proxy to npmjs.org + - Private package publishing (requires authentication) + - Private packages stored separately and never forwarded to npmjs.org ## Usage @@ -142,6 +144,50 @@ The npm proxy will: - Serve cached packages for subsequent requests - Automatically fetch from npmjs.org if not cached +### Publishing NPM Packages + +The npm proxy supports publishing private packages using standard npm commands. + +#### Authentication + +**Method 1: Using npm login (recommended)** + +For npm 9.x+, use the `--auth-type=legacy` flag: + +```bash +npm login --registry=http://npm.mirror.intra --auth-type=legacy +# Enter your username and password when prompted +npm whoami --registry=http://npm.mirror.intra +``` + +**Method 2: Manual token configuration** + +If `npm login` doesn't work, you can manually obtain and configure the token: + +```bash +TOKEN=$(curl -X PUT http://npm.mirror.intra/-/user/org.couchdb.user:admin \ + -H "Content-Type: application/json" \ + -d '{"name": "admin", "password": "your-password"}' \ + | jq -r .token) +echo "//npm.mirror.intra/:_authToken=$TOKEN" >> ~/.npmrc +npm whoami --registry=http://npm.mirror.intra +``` + +#### Publishing Packages + +Once authenticated, publish packages normally: + +```bash +npm publish +``` + +**Important Notes:** +- All published packages are treated as private packages +- Private packages are stored in `data/data/npm/private/` +- Published packages are **NOT** forwarded to npmjs.org +- Private packages take precedence over cached public packages +- Authentication tokens for npm are JWT-based and valid for 1 year + ### File Hosting Use the file repository at `http://files.mirror.intra` to: diff --git a/admin/app/routes/documentation/documentation.tsx b/admin/app/routes/documentation/documentation.tsx index 0edd83b..92772ea 100644 --- a/admin/app/routes/documentation/documentation.tsx +++ b/admin/app/routes/documentation/documentation.tsx @@ -295,8 +295,9 @@ export default function Documentation() {

The NPM Proxy provides a local caching layer for npm packages, - speeding up installations and reducing bandwidth usage. It acts as - a transparent proxy to the official npm registry. + speeding up installations and reducing bandwidth usage. It also supports + publishing private packages that are stored locally and never forwarded + to the public npm registry.

Features:
@@ -305,6 +306,8 @@ export default function Documentation() {
  • Automatic package metadata fetching
  • Bandwidth optimization for repeated installs
  • Offline package availability
    • +
  • Private package publishing (requires authentication)
    • +
  • Private packages stored separately and never forwarded to npmjs.org
    • @@ -344,19 +347,74 @@ export default function Documentation() { +
    +

    Publishing Private Packages

    +
    +

    + You can publish private npm packages to this registry. All packages are stored locally and never forwarded to npmjs.org. +

    + +
    +
    Method 1: Using npm login
    +

    For npm 9.x+, use the --auth-type=legacy flag:

    +
    +
    + npm login --registry=http://npm.mirror.intra --auth-type=legacy +
    +

    Enter your username and password when prompted, then verify:

    +
    + npm whoami --registry=http://npm.mirror.intra +
    +
    +
    + +
    +
    Method 2: Manual token configuration
    +

    If npm login doesn't work, get a token manually:

    +
    +
    + TOKEN=$(curl -X PUT http://npm.mirror.intra/-/user/org.couchdb.user:admin \
    +   -H "Content-Type: application/json" \
    +   -d '{"name": "admin", "password": "your-password"}' \
    +   | jq -r .token) +
    +
    + echo "//npm.mirror.intra/:_authToken=$TOKEN" >> ~/.npmrc +
    +
    +
    + +
    +
    Publishing:
    +

    Once authenticated, publish normally:

    +
    + npm publish +
    +
    + +
    +
    Important Notes:
    +
      +
    • All published packages are private and stored in data/npm/private/
      • +
    • Published packages are never forwarded to npmjs.org
      • +
    • Private packages take precedence over cached public packages
      • +
    • Authentication tokens are JWT-based and valid for 1 year
      • +
    +
    +
    +
    +

    File Management

    - Cached npm packages are stored in the data/npm/ directory and can - be viewed and managed through the File Manager in the admin - interface. + Cached npm packages are stored in the data/npm/ directory and private packages in data/npm/private/. Both can be viewed and managed through the File Manager.

    Access:
    • Admin UI → File Manager → NPM Packages view
      • -
    • Browse cached packages by name
      • +
    • Browse cached and private packages by name
    • View package metadata and tarballs
    • Monitor cache usage and growth
    diff --git a/admin/app/routes/npm/npm.tsx b/admin/app/routes/npm/npm.tsx index 312ed61..63295f2 100644 --- a/admin/app/routes/npm/npm.tsx +++ b/admin/app/routes/npm/npm.tsx @@ -6,17 +6,39 @@ import http from 'http'; import { URL } from 'url'; import zlib from 'zlib'; import appConfig from '~/config/config.json'; +import { + validateCredentials, + createNpmAuthToken, + validateNpmAuthToken, +} from '~/utils/server-auth'; const NPM_REGISTRY_URL = 'https://registry.npmjs.org'; +const PRIVATE_PACKAGES_DIR = path.join(appConfig.npmPackagesDir, 'private'); async function ensureCacheDir() { try { await fs.mkdir(appConfig.npmPackagesDir, { recursive: true }); + await fs.mkdir(PRIVATE_PACKAGES_DIR, { recursive: true }); } catch (error) { console.error('Failed to create cache directory:', error); } } +async function extractNpmAuth(request: Request): Promise<{ username: string } | null> { + const authHeader = request.headers.get('Authorization'); + if (authHeader) { + const match = authHeader.match(/^Bearer\s+(.+)$/i); + if (match) { + const token = match[1]; + const user = await validateNpmAuthToken(token); + if (user) { + return { username: user.username }; + } + } + } + return null; +} + function getCachePath(packagePath: string): string { const cleanPath = packagePath.replace(/^\/+/, '').replace(/\/+$/, ''); @@ -214,6 +236,45 @@ async function loadFromCache( } } +function getPrivatePackagePath(packagePath: string): string { + const cleanPath = packagePath.replace(/^\/+/, '').replace(/\/+$/, ''); + return path.join(PRIVATE_PACKAGES_DIR, cleanPath); +} + +async function isPrivatePackage(packagePath: string): Promise { + if (packagePath.includes('/-/')) { + const privatePath = getPrivatePackagePath(packagePath); + return await isCached(privatePath); + } + + const metadataPath = getPrivatePackagePath(`${packagePath}.json`); + return await isCached(metadataPath); +} + +async function loadPrivatePackage( + packagePath: string, +): Promise<{ data: Buffer; headers: Record }> { + let privatePath: string; + let contentType: string; + + if (packagePath.includes('/-/')) { + privatePath = getPrivatePackagePath(packagePath); + contentType = 'application/octet-stream'; + } else { + privatePath = getPrivatePackagePath(`${packagePath}.json`); + contentType = 'application/json'; + } + + const data = await fs.readFile(privatePath); + + const headers: Record = { + 'content-type': contentType, + 'x-private-package': 'true', + }; + + return { data, headers }; +} + export async function loader({ request, params }: LoaderFunctionArgs) { const url = new URL(request.url); let packagePath = url.pathname; @@ -226,6 +287,32 @@ export async function loader({ request, params }: LoaderFunctionArgs) { packagePath = packagePath.replace(/^\/+/, ''); + if (packagePath === '-/whoami' || packagePath === '-/npm/v1/user') { + const auth = await extractNpmAuth(request); + if (!auth) { + return new Response( + JSON.stringify({ error: 'Not authenticated' }), + { + status: 401, + headers: { + 'Content-Type': 'application/json', + 'WWW-Authenticate': 'Bearer realm="npm"', + }, + }, + ); + } + + return new Response( + JSON.stringify({ username: auth.username }), + { + status: 200, + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + } + if (!packagePath) { return new Response('Not Found', { status: 404, @@ -243,24 +330,32 @@ export async function loader({ request, params }: LoaderFunctionArgs) { try { await ensureCacheDir(); - const cachePath = getCachePath(packagePath); - const isPackageCached = await isCached(cachePath); - + const isPrivate = await isPrivatePackage(packagePath); + let data: Buffer; let headers: Record; - if (isPackageCached) { - const cached = await loadFromCache(cachePath); - data = cached.data; - headers = cached.headers; + if (isPrivate) { + const privatePackage = await loadPrivatePackage(packagePath); + data = privatePackage.data; + headers = privatePackage.headers; } else { - const fetched = await fetchFromNpm(packagePath, originalHeaders); - data = fetched.data; - headers = fetched.headers; + const cachePath = getCachePath(packagePath); + const isPackageCached = await isCached(cachePath); + + if (isPackageCached) { + const cached = await loadFromCache(cachePath); + data = cached.data; + headers = cached.headers; + } else { + const fetched = await fetchFromNpm(packagePath, originalHeaders); + data = fetched.data; + headers = fetched.headers; - await saveToCache(cachePath, data, headers); + await saveToCache(cachePath, data, headers); - headers['x-cache'] = 'MISS'; + headers['x-cache'] = 'MISS'; + } } const contentType = headers['content-type'] || 'application/octet-stream'; @@ -312,6 +407,173 @@ export async function action({ request }: ActionFunctionArgs) { packagePath = packagePath.replace(/^\/+/, ''); + if ( + request.method === 'PUT' && + packagePath.startsWith('-/user/org.couchdb.user:') + ) { + try { + const username = packagePath.substring('-/user/org.couchdb.user:'.length); + + const bodyText = await request.text(); + const body = JSON.parse(bodyText); + + const isValid = await validateCredentials({ + username: body.name || username, + password: body.password, + }); + + if (!isValid) { + return new Response( + JSON.stringify({ + error: 'Unauthorized', + reason: 'Invalid username or password', + }), + { + status: 401, + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + } + + const token = await createNpmAuthToken(username); + + return new Response( + JSON.stringify({ + ok: true, + id: `org.couchdb.user:${username}`, + rev: '_we_dont_use_revs_any_more', + token: token, + }), + { + status: 201, + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + } catch (error) { + console.error('NPM login error:', error); + return new Response( + JSON.stringify({ + error: 'Bad request', + reason: error instanceof Error ? error.message : 'Invalid request', + }), + { + status: 400, + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + } + } + + if (request.method === 'PUT' && packagePath && !packagePath.startsWith('-/')) { + try { + const auth = await extractNpmAuth(request); + + if (!auth) { + return new Response( + JSON.stringify({ + error: 'Authentication required', + message: 'You must be authenticated to publish packages. Run: npm login --registry=http://npm.mirror.intra', + }), + { + status: 401, + headers: { + 'Content-Type': 'application/json', + 'WWW-Authenticate': 'Bearer realm="npm"', + }, + }, + ); + } + + await ensureCacheDir(); + + const bodyText = await request.text(); + const packageDocument = JSON.parse(bodyText); + + const packageName = packageDocument.name || packagePath; + const versions = packageDocument.versions || {}; + const attachments = packageDocument._attachments || {}; + + for (const version in versions) { + const versionData = versions[version]; + + console.log(`Publishing ${packageName}@${version} by ${auth.username}`); + + const tarballName = `${packageName}-${version}.tgz`; + const attachment = attachments[tarballName]; + + if (attachment && attachment.data) { + const tarballBuffer = Buffer.from(attachment.data, 'base64'); + + const tarballPath = `${packageName}/-/${tarballName}`; + const tarballFullPath = getPrivatePackagePath(tarballPath); + const tarballDir = path.dirname(tarballFullPath); + await fs.mkdir(tarballDir, { recursive: true }); + await fs.writeFile(tarballFullPath, tarballBuffer); + + console.log(`Saved tarball: ${tarballPath} (${tarballBuffer.length} bytes)`); + + const url = new URL(request.url); + const baseUrl = `${url.protocol}//${url.host}`; + versionData.dist = versionData.dist || {}; + versionData.dist.tarball = `${baseUrl}/npm/${packageName}/-/${tarballName}`; + } + } + + const updatedDocument = { + _id: packageName, + name: packageName, + versions: versions, + 'dist-tags': packageDocument['dist-tags'] || { latest: Object.keys(versions)[0] }, + _attachments: {}, + time: { + modified: new Date().toISOString(), + created: new Date().toISOString(), + ...packageDocument.time, + }, + _publishedBy: auth.username, + }; + + const metadataPath = getPrivatePackagePath(`${packageName}.json`); + await fs.writeFile(metadataPath, JSON.stringify(updatedDocument, null, 2)); + + console.log(`Package ${packageName} published successfully by ${auth.username}`); + + return new Response( + JSON.stringify({ + ok: true, + id: packageName, + rev: '1-' + Date.now().toString(36), + }), + { + status: 200, + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + } catch (error) { + console.error('Package publish error:', error); + return new Response( + JSON.stringify({ + error: 'Publish failed', + message: error instanceof Error ? error.message : 'Unknown error', + }), + { + status: 500, + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + } + } + if (!packagePath) { return new Response('Not Found', { status: 404, diff --git a/admin/app/utils/server-auth.ts b/admin/app/utils/server-auth.ts index 71ce0d1..6162811 100644 --- a/admin/app/utils/server-auth.ts +++ b/admin/app/utils/server-auth.ts @@ -10,6 +10,7 @@ const COOKIE_MAX_AGE = 24 * 60 * 60 * 1000; export interface AuthUser { username: string; exp: number; + type?: 'web' | 'npm'; // Token type } export interface LoginCredentials { @@ -77,6 +78,17 @@ export async function createAuthToken(username: string): Promise { const payload: AuthUser = { username, exp: Math.floor(Date.now() / 1000) + 24 * 60 * 60, // 24 hours from now + type: 'web', + }; + + return jwt.sign(payload, JWT_SECRET); +} + +export async function createNpmAuthToken(username: string): Promise { + const payload: AuthUser = { + username, + exp: Math.floor(Date.now() / 1000) + 365 * 24 * 60 * 60, // 1 year from now (npm tokens are long-lived) + type: 'npm', }; return jwt.sign(payload, JWT_SECRET); @@ -131,3 +143,9 @@ export async function requireAuth(request: Request): Promise { return await validateAuthToken(token); } + +// Validate NPM token (Bearer token from Authorization header) +export async function validateNpmAuthToken(token: string): Promise { + // NPM tokens are just JWT tokens, validate them the same way + return await validateAuthToken(token); +} From 836fcbf2f432849f11709e812a5b76bb9dff7d7e Mon Sep 17 00:00:00 2001 From: Denis Bashkatov <2485095+Denrox@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:44:09 +0300 Subject: [PATCH 2/4] public packages path adjusted --- README.md | 1 + admin/app/routes/documentation/documentation.tsx | 11 +++++++---- admin/app/routes/npm/npm.tsx | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dad8212..7a156fb 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ npm publish **Important Notes:** - All published packages are treated as private packages - Private packages are stored in `data/data/npm/private/` +- Public packages (cached from npmjs.org) are stored in `data/data/npm/public/` - Published packages are **NOT** forwarded to npmjs.org - Private packages take precedence over cached public packages - Authentication tokens for npm are JWT-based and valid for 1 year diff --git a/admin/app/routes/documentation/documentation.tsx b/admin/app/routes/documentation/documentation.tsx index 92772ea..3b7e7a4 100644 --- a/admin/app/routes/documentation/documentation.tsx +++ b/admin/app/routes/documentation/documentation.tsx @@ -74,7 +74,9 @@ export default function Documentation() { │ ├── files/ # Custom file repository${ appConfig.isNpmProxyEnabled ? ` -│ └── npm/ # NPM package cache` +│ └── npm/ # NPM packages +│ ├── private/ # Private published packages +│ └── public/ # Cached public packages from npmjs.org` : '' } └── logs/ @@ -108,7 +110,7 @@ export default function Documentation() { Main data storage directory. apt-mirror/ contains downloaded package repositories, files/ contains custom file repository {appConfig.isNpmProxyEnabled - ? ', npm/ contains cached npm packages' + ? ', npm/ contains npm packages with public/ for cached packages and private/ for published packages' : ''} .

    @@ -396,6 +398,7 @@ export default function Documentation() {
    Important Notes:
    • All published packages are private and stored in data/npm/private/
      • +
    • Public packages (cached from npmjs.org) are stored in data/npm/public/
    • Published packages are never forwarded to npmjs.org
    • Private packages take precedence over cached public packages
    • Authentication tokens are JWT-based and valid for 1 year
      • @@ -408,13 +411,13 @@ export default function Documentation() {

      File Management

      - Cached npm packages are stored in the data/npm/ directory and private packages in data/npm/private/. Both can be viewed and managed through the File Manager. + NPM packages are organized in the data/npm/ directory: public packages (cached from npmjs.org) in data/npm/public/ and private packages (published locally) in data/npm/private/. Both can be viewed and managed through the File Manager.

      Access:
      • Admin UI → File Manager → NPM Packages view
        • -
      • Browse cached and private packages by name
        • +
      • Browse cached public and private packages by name
      • View package metadata and tarballs
      • Monitor cache usage and growth
      diff --git a/admin/app/routes/npm/npm.tsx b/admin/app/routes/npm/npm.tsx index 63295f2..6f3832c 100644 --- a/admin/app/routes/npm/npm.tsx +++ b/admin/app/routes/npm/npm.tsx @@ -14,11 +14,13 @@ import { const NPM_REGISTRY_URL = 'https://registry.npmjs.org'; const PRIVATE_PACKAGES_DIR = path.join(appConfig.npmPackagesDir, 'private'); +const PUBLIC_PACKAGES_DIR = path.join(appConfig.npmPackagesDir, 'public'); async function ensureCacheDir() { try { await fs.mkdir(appConfig.npmPackagesDir, { recursive: true }); await fs.mkdir(PRIVATE_PACKAGES_DIR, { recursive: true }); + await fs.mkdir(PUBLIC_PACKAGES_DIR, { recursive: true }); } catch (error) { console.error('Failed to create cache directory:', error); } @@ -47,7 +49,7 @@ function getCachePath(packagePath: string): string { const packageName = parts[0]; const tarballPath = parts.slice(1).join('/'); const cachePath = path.join( - appConfig.npmPackagesDir, + PUBLIC_PACKAGES_DIR, `${packageName}-tarballs`, tarballPath, ); @@ -59,7 +61,7 @@ function getCachePath(packagePath: string): string { return cachePath; } else { - const cachePath = path.join(appConfig.npmPackagesDir, cleanPath); + const cachePath = path.join(PUBLIC_PACKAGES_DIR, cleanPath); const dir = path.dirname(cachePath); fs.mkdir(dir, { recursive: true }).catch((error) => { From e9d93024578320b0dafc00162389052a3c5bed14 Mon Sep 17 00:00:00 2001 From: Denis Bashkatov <2485095+Denrox@users.noreply.github.com> Date: Thu, 2 Oct 2025 21:14:34 +0300 Subject: [PATCH 3/4] npmrc --- admin/.npmrc | 1 + 1 file changed, 1 insertion(+) create mode 100644 admin/.npmrc diff --git a/admin/.npmrc b/admin/.npmrc new file mode 100644 index 0000000..214c29d --- /dev/null +++ b/admin/.npmrc @@ -0,0 +1 @@ +registry=https://registry.npmjs.org/ From e3597240f8b1955225f0a3c6b9fad2a9f1abe356 Mon Sep 17 00:00:00 2001 From: Denis Bashkatov <2485095+Denrox@users.noreply.github.com> Date: Fri, 3 Oct 2025 16:48:44 +0300 Subject: [PATCH 4/4] fix bad cached files --- admin/app/routes/npm/npm.tsx | 79 ++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/admin/app/routes/npm/npm.tsx b/admin/app/routes/npm/npm.tsx index 6f3832c..e2b6433 100644 --- a/admin/app/routes/npm/npm.tsx +++ b/admin/app/routes/npm/npm.tsx @@ -124,13 +124,26 @@ async function fetchFromNpm( const req = client.request(options, (res) => { const chunks: Buffer[] = []; - res.on('data', (chunk) => { chunks.push(chunk); }); res.on('end', () => { let data = Buffer.concat(chunks); + + if (res.statusCode === 304) { + console.log(`304 Not Modified for ${packagePath} - using cached version`); + reject(new Error('304_NOT_MODIFIED')); + return; + } + + if (data.length === 0 && res.statusCode !== 304) { + console.error('Empty response received from npm registry for:', packagePath); + console.error('Response status:', res.statusCode); + reject(new Error(`Empty response from npm registry (status: ${res.statusCode})`)); + return; + } + const headers: Record = {}; const contentEncoding = res.headers['content-encoding']; @@ -152,6 +165,11 @@ async function fetchFromNpm( } } + if (data.length === 0) { + reject(new Error('Empty response after decompression')); + return; + } + const relevantHeaders = [ 'content-type', 'etag', @@ -193,6 +211,11 @@ async function saveToCache( headers: Record, ) { try { + if (!data || data.length === 0) { + console.error('Attempted to save empty data to cache:', filePath); + return; + } + await fs.writeFile(filePath, data); const metaPath = filePath + '.meta'; @@ -219,6 +242,10 @@ async function loadFromCache( try { const data = await fs.readFile(filePath); + if (!data || data.length === 0) { + throw new Error('Empty cached file'); + } + let headers: Record = {}; try { const metaPath = filePath + '.meta'; @@ -346,17 +373,51 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const isPackageCached = await isCached(cachePath); if (isPackageCached) { - const cached = await loadFromCache(cachePath); - data = cached.data; - headers = cached.headers; + try { + const cached = await loadFromCache(cachePath); + data = cached.data; + headers = cached.headers; + } catch (error) { + const fetched = await fetchFromNpm(packagePath, originalHeaders); + data = fetched.data; + headers = fetched.headers; + + await saveToCache(cachePath, data, headers); + + headers['x-cache'] = 'MISS'; + } } else { - const fetched = await fetchFromNpm(packagePath, originalHeaders); - data = fetched.data; - headers = fetched.headers; + try { + const fetched = await fetchFromNpm(packagePath, originalHeaders); + data = fetched.data; + headers = fetched.headers; - await saveToCache(cachePath, data, headers); + await saveToCache(cachePath, data, headers); - headers['x-cache'] = 'MISS'; + headers['x-cache'] = 'MISS'; + } catch (error) { + if (error instanceof Error && error.message === '304_NOT_MODIFIED') { + // If we get 304, try to use cached version + try { + const cached = await loadFromCache(cachePath); + data = cached.data; + headers = cached.headers; + console.log('Using cached version due to 304 response'); + } catch (cacheError) { + console.log('304 received but no cached version available, fetching fresh copy...'); + // If no cache available, fetch without conditional headers + const freshFetched = await fetchFromNpm(packagePath, {}); + data = freshFetched.data; + headers = freshFetched.headers; + + await saveToCache(cachePath, data, headers); + headers['x-cache'] = 'MISS'; + console.log('Fresh copy fetched and cached due to missing cache'); + } + } else { + throw error; + } + } } }