- Cached npm packages are stored in the data/npm/ directory and can
- be viewed and managed through the File Manager in the admin
- interface.
+ NPM packages are organized in the data/npm/ directory: public packages (cached from npmjs.org) in data/npm/public/ and private packages (published locally) in data/npm/private/. Both can be viewed and managed through the File Manager.
Access:
- Admin UI → File Manager → NPM Packages view
- - Browse cached packages by name
+ - Browse cached public and private packages by name
- View package metadata and tarballs
- Monitor cache usage and growth
diff --git a/admin/app/routes/npm/npm.tsx b/admin/app/routes/npm/npm.tsx
index 312ed61..e2b6433 100644
--- a/admin/app/routes/npm/npm.tsx
+++ b/admin/app/routes/npm/npm.tsx
@@ -6,17 +6,41 @@ import http from 'http';
import { URL } from 'url';
import zlib from 'zlib';
import appConfig from '~/config/config.json';
+import {
+ validateCredentials,
+ createNpmAuthToken,
+ validateNpmAuthToken,
+} from '~/utils/server-auth';
const NPM_REGISTRY_URL = 'https://registry.npmjs.org';
+const PRIVATE_PACKAGES_DIR = path.join(appConfig.npmPackagesDir, 'private');
+const PUBLIC_PACKAGES_DIR = path.join(appConfig.npmPackagesDir, 'public');
async function ensureCacheDir() {
try {
await fs.mkdir(appConfig.npmPackagesDir, { recursive: true });
+ await fs.mkdir(PRIVATE_PACKAGES_DIR, { recursive: true });
+ await fs.mkdir(PUBLIC_PACKAGES_DIR, { recursive: true });
} catch (error) {
console.error('Failed to create cache directory:', error);
}
}
+async function extractNpmAuth(request: Request): Promise<{ username: string } | null> {
+ const authHeader = request.headers.get('Authorization');
+ if (authHeader) {
+ const match = authHeader.match(/^Bearer\s+(.+)$/i);
+ if (match) {
+ const token = match[1];
+ const user = await validateNpmAuthToken(token);
+ if (user) {
+ return { username: user.username };
+ }
+ }
+ }
+ return null;
+}
+
function getCachePath(packagePath: string): string {
const cleanPath = packagePath.replace(/^\/+/, '').replace(/\/+$/, '');
@@ -25,7 +49,7 @@ function getCachePath(packagePath: string): string {
const packageName = parts[0];
const tarballPath = parts.slice(1).join('/');
const cachePath = path.join(
- appConfig.npmPackagesDir,
+ PUBLIC_PACKAGES_DIR,
`${packageName}-tarballs`,
tarballPath,
);
@@ -37,7 +61,7 @@ function getCachePath(packagePath: string): string {
return cachePath;
} else {
- const cachePath = path.join(appConfig.npmPackagesDir, cleanPath);
+ const cachePath = path.join(PUBLIC_PACKAGES_DIR, cleanPath);
const dir = path.dirname(cachePath);
fs.mkdir(dir, { recursive: true }).catch((error) => {
@@ -100,13 +124,26 @@ async function fetchFromNpm(
const req = client.request(options, (res) => {
const chunks: Buffer[] = [];
-
res.on('data', (chunk) => {
chunks.push(chunk);
});
res.on('end', () => {
let data = Buffer.concat(chunks);
+
+ if (res.statusCode === 304) {
+ console.log(`304 Not Modified for ${packagePath} - using cached version`);
+ reject(new Error('304_NOT_MODIFIED'));
+ return;
+ }
+
+ if (data.length === 0 && res.statusCode !== 304) {
+ console.error('Empty response received from npm registry for:', packagePath);
+ console.error('Response status:', res.statusCode);
+ reject(new Error(`Empty response from npm registry (status: ${res.statusCode})`));
+ return;
+ }
+
const headers: Record
= {};
const contentEncoding = res.headers['content-encoding'];
@@ -128,6 +165,11 @@ async function fetchFromNpm(
}
}
+ if (data.length === 0) {
+ reject(new Error('Empty response after decompression'));
+ return;
+ }
+
const relevantHeaders = [
'content-type',
'etag',
@@ -169,6 +211,11 @@ async function saveToCache(
headers: Record,
) {
try {
+ if (!data || data.length === 0) {
+ console.error('Attempted to save empty data to cache:', filePath);
+ return;
+ }
+
await fs.writeFile(filePath, data);
const metaPath = filePath + '.meta';
@@ -195,6 +242,10 @@ async function loadFromCache(
try {
const data = await fs.readFile(filePath);
+ if (!data || data.length === 0) {
+ throw new Error('Empty cached file');
+ }
+
let headers: Record = {};
try {
const metaPath = filePath + '.meta';
@@ -214,6 +265,45 @@ async function loadFromCache(
}
}
+function getPrivatePackagePath(packagePath: string): string {
+ const cleanPath = packagePath.replace(/^\/+/, '').replace(/\/+$/, '');
+ return path.join(PRIVATE_PACKAGES_DIR, cleanPath);
+}
+
+async function isPrivatePackage(packagePath: string): Promise {
+ if (packagePath.includes('/-/')) {
+ const privatePath = getPrivatePackagePath(packagePath);
+ return await isCached(privatePath);
+ }
+
+ const metadataPath = getPrivatePackagePath(`${packagePath}.json`);
+ return await isCached(metadataPath);
+}
+
+async function loadPrivatePackage(
+ packagePath: string,
+): Promise<{ data: Buffer; headers: Record }> {
+ let privatePath: string;
+ let contentType: string;
+
+ if (packagePath.includes('/-/')) {
+ privatePath = getPrivatePackagePath(packagePath);
+ contentType = 'application/octet-stream';
+ } else {
+ privatePath = getPrivatePackagePath(`${packagePath}.json`);
+ contentType = 'application/json';
+ }
+
+ const data = await fs.readFile(privatePath);
+
+ const headers: Record = {
+ 'content-type': contentType,
+ 'x-private-package': 'true',
+ };
+
+ return { data, headers };
+}
+
export async function loader({ request, params }: LoaderFunctionArgs) {
const url = new URL(request.url);
let packagePath = url.pathname;
@@ -226,6 +316,32 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
packagePath = packagePath.replace(/^\/+/, '');
+ if (packagePath === '-/whoami' || packagePath === '-/npm/v1/user') {
+ const auth = await extractNpmAuth(request);
+ if (!auth) {
+ return new Response(
+ JSON.stringify({ error: 'Not authenticated' }),
+ {
+ status: 401,
+ headers: {
+ 'Content-Type': 'application/json',
+ 'WWW-Authenticate': 'Bearer realm="npm"',
+ },
+ },
+ );
+ }
+
+ return new Response(
+ JSON.stringify({ username: auth.username }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ }
+
if (!packagePath) {
return new Response('Not Found', {
status: 404,
@@ -243,24 +359,66 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
try {
await ensureCacheDir();
- const cachePath = getCachePath(packagePath);
- const isPackageCached = await isCached(cachePath);
-
+ const isPrivate = await isPrivatePackage(packagePath);
+
let data: Buffer;
let headers: Record;
- if (isPackageCached) {
- const cached = await loadFromCache(cachePath);
- data = cached.data;
- headers = cached.headers;
+ if (isPrivate) {
+ const privatePackage = await loadPrivatePackage(packagePath);
+ data = privatePackage.data;
+ headers = privatePackage.headers;
} else {
- const fetched = await fetchFromNpm(packagePath, originalHeaders);
- data = fetched.data;
- headers = fetched.headers;
+ const cachePath = getCachePath(packagePath);
+ const isPackageCached = await isCached(cachePath);
+
+ if (isPackageCached) {
+ try {
+ const cached = await loadFromCache(cachePath);
+ data = cached.data;
+ headers = cached.headers;
+ } catch (error) {
+ const fetched = await fetchFromNpm(packagePath, originalHeaders);
+ data = fetched.data;
+ headers = fetched.headers;
+
+ await saveToCache(cachePath, data, headers);
+
+ headers['x-cache'] = 'MISS';
+ }
+ } else {
+ try {
+ const fetched = await fetchFromNpm(packagePath, originalHeaders);
+ data = fetched.data;
+ headers = fetched.headers;
- await saveToCache(cachePath, data, headers);
+ await saveToCache(cachePath, data, headers);
- headers['x-cache'] = 'MISS';
+ headers['x-cache'] = 'MISS';
+ } catch (error) {
+ if (error instanceof Error && error.message === '304_NOT_MODIFIED') {
+ // If we get 304, try to use cached version
+ try {
+ const cached = await loadFromCache(cachePath);
+ data = cached.data;
+ headers = cached.headers;
+ console.log('Using cached version due to 304 response');
+ } catch (cacheError) {
+ console.log('304 received but no cached version available, fetching fresh copy...');
+ // If no cache available, fetch without conditional headers
+ const freshFetched = await fetchFromNpm(packagePath, {});
+ data = freshFetched.data;
+ headers = freshFetched.headers;
+
+ await saveToCache(cachePath, data, headers);
+ headers['x-cache'] = 'MISS';
+ console.log('Fresh copy fetched and cached due to missing cache');
+ }
+ } else {
+ throw error;
+ }
+ }
+ }
}
const contentType = headers['content-type'] || 'application/octet-stream';
@@ -312,6 +470,173 @@ export async function action({ request }: ActionFunctionArgs) {
packagePath = packagePath.replace(/^\/+/, '');
+ if (
+ request.method === 'PUT' &&
+ packagePath.startsWith('-/user/org.couchdb.user:')
+ ) {
+ try {
+ const username = packagePath.substring('-/user/org.couchdb.user:'.length);
+
+ const bodyText = await request.text();
+ const body = JSON.parse(bodyText);
+
+ const isValid = await validateCredentials({
+ username: body.name || username,
+ password: body.password,
+ });
+
+ if (!isValid) {
+ return new Response(
+ JSON.stringify({
+ error: 'Unauthorized',
+ reason: 'Invalid username or password',
+ }),
+ {
+ status: 401,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ }
+
+ const token = await createNpmAuthToken(username);
+
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ id: `org.couchdb.user:${username}`,
+ rev: '_we_dont_use_revs_any_more',
+ token: token,
+ }),
+ {
+ status: 201,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ } catch (error) {
+ console.error('NPM login error:', error);
+ return new Response(
+ JSON.stringify({
+ error: 'Bad request',
+ reason: error instanceof Error ? error.message : 'Invalid request',
+ }),
+ {
+ status: 400,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ }
+ }
+
+ if (request.method === 'PUT' && packagePath && !packagePath.startsWith('-/')) {
+ try {
+ const auth = await extractNpmAuth(request);
+
+ if (!auth) {
+ return new Response(
+ JSON.stringify({
+ error: 'Authentication required',
+ message: 'You must be authenticated to publish packages. Run: npm login --registry=http://npm.mirror.intra',
+ }),
+ {
+ status: 401,
+ headers: {
+ 'Content-Type': 'application/json',
+ 'WWW-Authenticate': 'Bearer realm="npm"',
+ },
+ },
+ );
+ }
+
+ await ensureCacheDir();
+
+ const bodyText = await request.text();
+ const packageDocument = JSON.parse(bodyText);
+
+ const packageName = packageDocument.name || packagePath;
+ const versions = packageDocument.versions || {};
+ const attachments = packageDocument._attachments || {};
+
+ for (const version in versions) {
+ const versionData = versions[version];
+
+ console.log(`Publishing ${packageName}@${version} by ${auth.username}`);
+
+ const tarballName = `${packageName}-${version}.tgz`;
+ const attachment = attachments[tarballName];
+
+ if (attachment && attachment.data) {
+ const tarballBuffer = Buffer.from(attachment.data, 'base64');
+
+ const tarballPath = `${packageName}/-/${tarballName}`;
+ const tarballFullPath = getPrivatePackagePath(tarballPath);
+ const tarballDir = path.dirname(tarballFullPath);
+ await fs.mkdir(tarballDir, { recursive: true });
+ await fs.writeFile(tarballFullPath, tarballBuffer);
+
+ console.log(`Saved tarball: ${tarballPath} (${tarballBuffer.length} bytes)`);
+
+ const url = new URL(request.url);
+ const baseUrl = `${url.protocol}//${url.host}`;
+ versionData.dist = versionData.dist || {};
+ versionData.dist.tarball = `${baseUrl}/npm/${packageName}/-/${tarballName}`;
+ }
+ }
+
+ const updatedDocument = {
+ _id: packageName,
+ name: packageName,
+ versions: versions,
+ 'dist-tags': packageDocument['dist-tags'] || { latest: Object.keys(versions)[0] },
+ _attachments: {},
+ time: {
+ modified: new Date().toISOString(),
+ created: new Date().toISOString(),
+ ...packageDocument.time,
+ },
+ _publishedBy: auth.username,
+ };
+
+ const metadataPath = getPrivatePackagePath(`${packageName}.json`);
+ await fs.writeFile(metadataPath, JSON.stringify(updatedDocument, null, 2));
+
+ console.log(`Package ${packageName} published successfully by ${auth.username}`);
+
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ id: packageName,
+ rev: '1-' + Date.now().toString(36),
+ }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ } catch (error) {
+ console.error('Package publish error:', error);
+ return new Response(
+ JSON.stringify({
+ error: 'Publish failed',
+ message: error instanceof Error ? error.message : 'Unknown error',
+ }),
+ {
+ status: 500,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ }
+ }
+
if (!packagePath) {
return new Response('Not Found', {
status: 404,
diff --git a/admin/app/utils/server-auth.ts b/admin/app/utils/server-auth.ts
index 71ce0d1..6162811 100644
--- a/admin/app/utils/server-auth.ts
+++ b/admin/app/utils/server-auth.ts
@@ -10,6 +10,7 @@ const COOKIE_MAX_AGE = 24 * 60 * 60 * 1000;
export interface AuthUser {
username: string;
exp: number;
+ type?: 'web' | 'npm'; // Token type
}
export interface LoginCredentials {
@@ -77,6 +78,17 @@ export async function createAuthToken(username: string): Promise {
const payload: AuthUser = {
username,
exp: Math.floor(Date.now() / 1000) + 24 * 60 * 60, // 24 hours from now
+ type: 'web',
+ };
+
+ return jwt.sign(payload, JWT_SECRET);
+}
+
+export async function createNpmAuthToken(username: string): Promise {
+ const payload: AuthUser = {
+ username,
+ exp: Math.floor(Date.now() / 1000) + 365 * 24 * 60 * 60, // 1 year from now (npm tokens are long-lived)
+ type: 'npm',
};
return jwt.sign(payload, JWT_SECRET);
@@ -131,3 +143,9 @@ export async function requireAuth(request: Request): Promise {
return await validateAuthToken(token);
}
+
+// Validate NPM token (Bearer token from Authorization header)
+export async function validateNpmAuthToken(token: string): Promise {
+ // NPM tokens are just JWT tokens, validate them the same way
+ return await validateAuthToken(token);
+}