Skip to content

Commit

Permalink
feat(transformer): added initial draft for transformer uglify
Browse files Browse the repository at this point in the history
  • Loading branch information
H4ad committed Aug 14, 2022
1 parent 14b2805 commit c505b7e
Show file tree
Hide file tree
Showing 12 changed files with 395 additions and 265 deletions.
3 changes: 2 additions & 1 deletion README.md
Expand Up @@ -515,7 +515,7 @@ Pack files and node dependencies to zip file.
USAGE
$ node-modules-packer run [DIR] [--json] [-i <value>] [-e <value>] [--disable-default-ignore-file-ext]
[--include-node-path <value>] [--ignore-node-path <value>] [--prod] [--peer] [--dev] [--optional] [--output-path
<value>] [--output-file <value>] [-q]
<value>] [--output-file <value>] [--uglify] [-q]
ARGUMENTS
DIR [default: ./] Project root directory
Expand All @@ -537,6 +537,7 @@ FLAGS
--output-path=<value> [default: ./] Specify output path for the zip file.
--[no-]peer Include peer dependencies when pack node dependencies.
--[no-]prod Include production dependencies when pack node dependencies.
--uglify Transform each .js file with uglify
GLOBAL FLAGS
--json Format output as json.
Expand Down
399 changes: 166 additions & 233 deletions package-lock.json

Large diffs are not rendered by default.

8 changes: 5 additions & 3 deletions package.json
Expand Up @@ -8,7 +8,7 @@
"prepare": "husky install",
"build": "rimraf lib && tsc -p tsconfig.build.json",
"postpack": "rimraf oclif.manifest.json",
"prepack": "yarn build && oclif manifest && oclif readme",
"prepack": "npm run build && oclif manifest && oclif readme",
"clean": "rm -rf ./lib/",
"cm": "cz",
"coverage": "codecov --disable=gcov",
Expand Down Expand Up @@ -88,7 +88,9 @@
"plist": "^3.0.5",
"rimraf": "^3.0.2",
"semver": "^7.3.7",
"terser": "^5.14.2",
"tslib": "^2.4.0",
"uglify-js": "^3.16.3",
"yazl": "^2.5.1"
},
"devDependencies": {
Expand All @@ -102,7 +104,8 @@
"@types/node": "12.20.43",
"@types/plist": "^3.0.2",
"@types/rimraf": "^3.0.2",
"@types/sinon": "^10.0.13",
"@types/terser": "^3.12.0",
"@types/uglify-js": "^3.16.0",
"@types/unzipper": "^0.10.5",
"@types/yazl": "^2.4.2",
"@typescript-eslint/eslint-plugin": "^5.12.1",
Expand All @@ -128,7 +131,6 @@
"oclif": "^3",
"prettier": "^2.5.1",
"semantic-release": "^19.0.3",
"sinon": "^14.0.0",
"tape": "^5.5.3",
"ts-node": "^10.4.0",
"typescript": "^4.5.5",
Expand Down
32 changes: 32 additions & 0 deletions src/commands/run/index.ts
Expand Up @@ -16,6 +16,8 @@ import CustomError from '../../common/custom-error';
import { defaultIgnoredFileExtensions } from '../../common/extensions';
import { HeadlessOptions } from '../../common/headless';
import { OutputInfo } from '../../common/output-info';
import { FileInMemoryTransformer } from '../../common/transformers/file-in-memory.transformer';
import { UglifyJsTransformer } from '../../common/transformers/uglify-js.transformer';
import { FasterZip, ZipArtifact } from '../../common/zip';

//#endregion
Expand Down Expand Up @@ -108,6 +110,11 @@ export default class Run extends CustomCommand {
default: 'deploy.zip',
required: false,
}),
uglify: Flags.boolean({
description: 'Transform each .js file with uglify',
default: false,
required: false,
}),
quiet: Flags.boolean({
char: 'q',
description: 'Run without logging.',
Expand Down Expand Up @@ -173,6 +180,8 @@ export default class Run extends CustomCommand {
if (options.outputFile !== undefined)
args.push('--output-file', options.outputFile);

if (options.uglify !== undefined) pushFlagBoolean('uglify', options.uglify);

return await this.run(args, loadOptions);
}

Expand Down Expand Up @@ -440,11 +449,33 @@ export default class Run extends CustomCommand {
): ZipArtifact[] {
this.logMessage(flags, 'log', 'Getting artifacts to zip');

const isJsFileRegex = /(\.js|\.cjs|\.mjs)$/;

const transformers: ZipArtifact['transformers'] = (
filePath,
metadataPath,
stats,
) => {
const isJsFile =
isJsFileRegex.test(filePath) || isJsFileRegex.test(metadataPath);

if (!isJsFile || stats.size > 32 * 1024) return [];

const memory = new FileInMemoryTransformer();
const uglifyJs = new UglifyJsTransformer(filePath, {
compress: false,
mangle: true,
});

return [memory, uglifyJs];
};

const artifacts: ZipArtifact[] = [
{
path: join(dir, 'node_modules'),
name: 'node_modules',
type: 'directory',
transformers: flags.uglify ? transformers : undefined,
shouldIgnore: shouldIgnoreNodeFile,
},
];
Expand All @@ -466,6 +497,7 @@ export default class Run extends CustomCommand {
path: includeFilePath,
name: includeFile,
metadataPath,
transformers: flags.uglify ? transformers : undefined,
type,
});
}
Expand Down
7 changes: 7 additions & 0 deletions src/common/headless.ts
Expand Up @@ -103,4 +103,11 @@ export interface HeadlessOptions {
* @default deploy.zip
*/
outputFile?: string;

/**
* Pass all .js files to uglify to reduce the file size.
*
* @default false
*/
uglify?: boolean;
}
21 changes: 21 additions & 0 deletions src/common/transformers/file-in-memory.transformer.ts
@@ -0,0 +1,21 @@
import { Transform, TransformCallback } from 'stream';

export class FileInMemoryTransformer extends Transform {
constructor() {
super();

this.memory = Buffer.alloc(0);
}

private memory: Buffer;

_transform(chunk: Buffer, _: string, cb: TransformCallback): void {
this.memory = Buffer.concat([this.memory, chunk]);
cb();
}

_flush(cb: TransformCallback): void {
this.push(this.memory);
cb();
}
}
32 changes: 32 additions & 0 deletions src/common/transformers/memory-stream.transformer.ts
@@ -0,0 +1,32 @@
import { Transform } from 'stream';

export class MemoryStream extends Transform {
constructor(private readonly desiredChunkSize: number) {
super();

this.memory = Buffer.alloc(0);
}

private memory: Buffer;

_transform(chunk: Buffer, _: string, cb: () => void): void {
if (
Buffer.byteLength(this.memory) + Buffer.byteLength(chunk) >=
this.desiredChunkSize
) {
this.push(this.memory);

this.memory = Buffer.alloc(0);
}

this.memory = Buffer.concat([this.memory, chunk]);

cb();
}

_flush(cb: () => void): void {
this.push(this.memory);

cb();
}
}
40 changes: 40 additions & 0 deletions src/common/transformers/uglify-js.transformer.ts
@@ -0,0 +1,40 @@
import { Transform, TransformCallback } from 'stream';
import { MinifyOptions, minify } from 'terser';

export class UglifyJsTransformer extends Transform {
constructor(
protected readonly filePath: string,
protected readonly uglifyOptions: MinifyOptions = {},
) {
super();
}

protected chunks: number = 0;

async _transform(
chunk: Buffer,
encoding: string,
callback: TransformCallback,
): Promise<void> {
if (this.chunks > 0) {
return callback(
new Error(
'This transformer should not be called more than once. Check if you use MemoryStream before using UglifyJsTransformer',
),
);
}

console.log(`${this.filePath}:${chunk.byteLength}`);

const code = chunk.toString('utf-8');
const result = await minify(code, this.uglifyOptions).catch(() => null);
const data =
!result || !result.code ? chunk : Buffer.from(result.code, 'utf-8');

this.chunks++;

this.push(data);

callback();
}
}
49 changes: 39 additions & 10 deletions src/common/zip.ts
@@ -1,5 +1,6 @@
import fs, { createReadStream, createWriteStream } from 'fs';
import fs, { Stats, createReadStream, createWriteStream } from 'fs';
import { join, normalize, relative } from 'path';
import { Transform } from 'stream';
import { ZipFile } from 'yazl';

export interface ZipArtifact {
Expand All @@ -11,8 +12,15 @@ export interface ZipArtifact {
metadataPath?: string;
type: 'file' | 'directory';
shouldIgnore?: (fileName: string) => boolean;
transformers?: (
filePath: string,
metadataPath: string,
stats: Stats,
) => Transform[];
}

const maxHighWatermarkSize = 100 * (1024 * 1024);

export class FasterZip {
public async run(
rootPath: string,
Expand Down Expand Up @@ -83,9 +91,19 @@ export class FasterZip {
const metadataPath = source.metadataPath
? filePath.replace(source.path, source.metadataPath)
: relative(rootPath, filePath);
const readStream = createReadStream(filePath).once('error', err =>
onErrorOnStream(err),
);
const readStream = createReadStream(filePath, {
highWaterMark: maxHighWatermarkSize,
}).once('error', err => onErrorOnStream(err));

if (source.transformers) {
const transformers = source.transformers(
filePath,
metadataPath,
stats,
);

transformers.forEach(transform => readStream.pipe(transform));
}

zipFile.addReadStream(readStream, normalize(metadataPath));
}
Expand Down Expand Up @@ -123,12 +141,23 @@ export class FasterZip {
const metadataPath = artifact.metadataPath
? artifact.path.replace(artifact.path, artifact.metadataPath)
: relative(rootPath, artifact.path);
const readStream = createReadStream(artifact.path).once(
'error',
err => {
onErrorOnStream(err);
},
);
const readStream = createReadStream(artifact.path, {
highWaterMark: maxHighWatermarkSize,
}).once('error', err => {
onErrorOnStream(err);
});

if (artifact.transformers) {
const stats = fs.statSync(artifact.path);

const transformers = artifact.transformers(
artifact.path,
metadataPath,
stats,
);

transformers.forEach(transform => readStream.pipe(transform));
}

zipfile.addReadStream(readStream, normalize(metadataPath));
resolve();
Expand Down
30 changes: 29 additions & 1 deletion test/commands/run/index.test.ts
@@ -1,4 +1,4 @@
import { existsSync } from 'fs';
import { existsSync, statSync } from 'fs';
import { join } from 'path';
import { expect } from '@oclif/test';
import Run from '../../../src/commands/run';
Expand Down Expand Up @@ -475,6 +475,34 @@ describe('when pack is called', () => {
});
});

describe('with --uglify flag', () => {
fsTest
.stdout()
.stderr()
.fsmockCommand(['run', MockFsFactory.DIR_PROJECT])
.it('should have output file size as 4355 bytes without uglify', ctx => {
expect(ctx.stderr).to.be.empty;

const outputFilePath = join(MockFsFactory.DIR_PROJECT, 'deploy.zip');
const stats = statSync(outputFilePath);

expect(stats.size).to.be.eq(4355);
});

fsTest
.stdout()
.stderr()
.fsmockCommand(['run', MockFsFactory.DIR_PROJECT, '--uglify'])
.it('should have output file size lower than 4355 with uglify', ctx => {
expect(ctx.stderr).to.be.empty;

const outputFilePath = join(MockFsFactory.DIR_PROJECT, 'deploy.zip');
const stats = statSync(outputFilePath);

expect(stats.size).to.be.below(4355);
});
});

describe('with invalid lock file', () => {
fsTest
.stdout()
Expand Down

0 comments on commit c505b7e

Please sign in to comment.