Skip to content

Commit

Permalink
Split InternalAsset into UncommittedAsset and CommittedAsset (Cache A…
Browse files Browse the repository at this point in the history
…STs followup) (#4378)

* Split InternalAsset into UncommittedAsset and CommittedAsset

* Babel cleanup

* Have babel transformer depend on @parcel/babel-ast-utils
  • Loading branch information
Will Binns-Smith committed Mar 27, 2020
1 parent a1a2515 commit c690f91
Show file tree
Hide file tree
Showing 16 changed files with 498 additions and 311 deletions.
161 changes: 161 additions & 0 deletions packages/core/core/src/CommittedAsset.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
// @flow strict-local

import type {
AST,
Blob,
ConfigResult,
File,
FilePath,
PackageJSON,
} from '@parcel/types';
import type {Asset, Dependency, ParcelOptions} from './types';

import v8 from 'v8';
import {Readable} from 'stream';
import SourceMap from '@parcel/source-map';
import {bufferStream, blobToStream, streamFromPromise} from '@parcel/utils';
import {getConfig, generateFromAST} from './assetUtils';

export default class CommittedAsset {
value: Asset;
options: ParcelOptions;
content: ?Promise<Buffer | string>;
mapBuffer: ?Promise<?Buffer>;
map: ?Promise<?SourceMap>;
ast: ?Promise<AST>;
idBase: ?string;
generatingPromise: ?Promise<void>;

constructor(value: Asset, options: ParcelOptions) {
this.value = value;
this.options = options;
}

getContent(): Blob | Promise<Buffer | string> {
if (this.content == null) {
if (this.value.contentKey != null) {
return this.options.cache.getStream(this.value.contentKey);
} else if (this.value.astKey != null) {
return streamFromPromise(
generateFromAST(this).then(({code}) => {
if (!(code instanceof Readable)) {
this.content = Promise.resolve(code);
}
return code;
}),
);
} else {
throw new Error('Asset has no content');
}
}

return this.content;
}

async getCode(): Promise<string> {
let content = await this.getContent();

if (typeof content === 'string' || content instanceof Buffer) {
return content.toString();
} else if (content != null) {
this.content = bufferStream(content);
return (await this.content).toString();
}

return '';
}

async getBuffer(): Promise<Buffer> {
let content = await this.getContent();

if (content == null) {
return Buffer.alloc(0);
} else if (typeof content === 'string' || content instanceof Buffer) {
return Buffer.from(content);
}

this.content = bufferStream(content);
return this.content;
}

getStream(): Readable {
let content = this.getContent();
return content instanceof Promise
? streamFromPromise(content)
: blobToStream(content);
}

getMapBuffer(): Promise<?Buffer> {
let mapKey = this.value.mapKey;
if (mapKey != null && this.mapBuffer == null) {
this.mapBuffer = (async () => {
try {
return await this.options.cache.getBlob(mapKey);
} catch (err) {
if (err.code === 'ENOENT' && this.value.astKey != null) {
return (await generateFromAST(this)).map?.toBuffer();
} else {
throw err;
}
}
})();
}

return this.mapBuffer ?? Promise.resolve();
}

getMap(): Promise<?SourceMap> {
if (this.map == null) {
this.map = (async () => {
let mapBuffer = await this.getMapBuffer();
if (mapBuffer) {
// Get sourcemap from flatbuffer
let map = new SourceMap();
map.addBufferMappings(mapBuffer);
return map;
}
})();
}

return this.map;
}

getAST(): Promise<AST> {
if (this.value.astKey == null) {
throw new Error('Asset does not have an AST');
}

if (this.ast == null) {
this.ast = this.options.cache
.getBlob(this.value.astKey)
.then(serializedAst =>
// $FlowFixMe
v8.deserialize(serializedAst),
);
}

return this.ast;
}

getIncludedFiles(): Array<File> {
return Array.from(this.value.includedFiles.values());
}

getDependencies(): Array<Dependency> {
return Array.from(this.value.dependencies.values());
}

async getConfig(
filePaths: Array<FilePath>,
options: ?{|
packageKey?: string,
parse?: boolean,
|},
): Promise<ConfigResult | null> {
return (await getConfig(this, filePaths, options))?.config;
}

getPackage(): Promise<PackageJSON | null> {
return this.getConfig(['package.json']);
}
}
2 changes: 0 additions & 2 deletions packages/core/core/src/Parcel.js
Original file line number Diff line number Diff line change
Expand Up @@ -384,8 +384,6 @@ export class BuildError extends ThrowableDiagnostic {
}
}

export {default as Asset} from './InternalAsset';

export function createWorkerFarm(options: $Shape<FarmOptions> = {}) {
return new WorkerFarm({
...options,
Expand Down
51 changes: 29 additions & 22 deletions packages/core/core/src/Transformation.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,22 @@ import ConfigLoader from './ConfigLoader';
import {createDependency} from './Dependency';
import ParcelConfig from './ParcelConfig';
import ResolverRunner from './ResolverRunner';
import {Asset, MutableAsset, assetToInternalAsset} from './public/Asset';
import InternalAsset, {createAsset} from './InternalAsset';
import {
Asset,
MutableAsset,
mutableAssetToUncommittedAsset,
} from './public/Asset';
import UncommittedAsset from './UncommittedAsset';
import {createAsset} from './assetUtils';
import summarizeRequest from './summarizeRequest';
import PluginOptions from './public/PluginOptions';
import {PARCEL_VERSION} from './constants';

type GenerateFunc = (input: InternalAsset) => Promise<GenerateOutput>;
type GenerateFunc = (input: UncommittedAsset) => Promise<GenerateOutput>;

type PostProcessFunc = (
Array<InternalAsset>,
) => Promise<Array<InternalAsset> | null>;
Array<UncommittedAsset>,
) => Promise<Array<UncommittedAsset> | null>;

export type TransformationOpts = {|
options: ParcelOptions,
Expand Down Expand Up @@ -131,7 +136,7 @@ export default class Transformation {
return {assets, configRequests: this.configRequests};
}

async loadAsset(): Promise<InternalAsset> {
async loadAsset(): Promise<UncommittedAsset> {
let {filePath, env, code, pipeline, sideEffects} = this.request;
let {content, size, hash, isSource} = await summarizeRequest(
this.options.inputFS,
Expand All @@ -146,7 +151,7 @@ export default class Transformation {
: path
.relative(this.options.projectRoot, filePath)
.replace(/[\\/]+/g, '/');
return new InternalAsset({
return new UncommittedAsset({
idBase,
value: createAsset({
idBase,
Expand All @@ -169,8 +174,8 @@ export default class Transformation {

async runPipelines(
pipeline: Pipeline,
initialAsset: InternalAsset,
): Promise<Array<InternalAsset>> {
initialAsset: UncommittedAsset,
): Promise<Array<UncommittedAsset>> {
let initialType = initialAsset.value.type;
let initialAssetCacheKey = this.getCacheKey(
[initialAsset],
Expand All @@ -185,7 +190,7 @@ export default class Transformation {
await this.writeToCache(initialAssetCacheKey, assets, pipeline.configs);
}

let finalAssets: Array<InternalAsset> = [];
let finalAssets: Array<UncommittedAsset> = [];
for (let asset of assets) {
let nextPipeline;
if (asset.value.type !== initialType) {
Expand Down Expand Up @@ -214,7 +219,7 @@ export default class Transformation {
);

invariant(pipeline.postProcess != null);
let processedFinalAssets: Array<InternalAsset> =
let processedFinalAssets: Array<UncommittedAsset> =
processedCacheEntry ?? (await pipeline.postProcess(finalAssets)) ?? [];

if (!processedCacheEntry) {
Expand All @@ -230,8 +235,8 @@ export default class Transformation {

async runPipeline(
pipeline: Pipeline,
initialAsset: InternalAsset,
): Promise<Array<InternalAsset>> {
initialAsset: UncommittedAsset,
): Promise<Array<UncommittedAsset>> {
let initialType = initialAsset.value.type;
let inputAssets = [initialAsset];
let resultingAssets = [];
Expand Down Expand Up @@ -308,7 +313,9 @@ export default class Transformation {
return finalAssets.concat(resultingAssets);
}

async readFromCache(cacheKey: string): Promise<null | Array<InternalAsset>> {
async readFromCache(
cacheKey: string,
): Promise<null | Array<UncommittedAsset>> {
if (this.options.disableCache || this.request.code != null) {
return null;
}
Expand All @@ -320,7 +327,7 @@ export default class Transformation {

return cachedAssets.map(
(value: AssetValue) =>
new InternalAsset({
new UncommittedAsset({
value,
options: this.options,
}),
Expand All @@ -329,7 +336,7 @@ export default class Transformation {

async writeToCache(
cacheKey: string,
assets: Array<InternalAsset>,
assets: Array<UncommittedAsset>,
configs: ConfigMap,
): Promise<void> {
await Promise.all(
Expand All @@ -349,7 +356,7 @@ export default class Transformation {
);
}

getCacheKey(assets: Array<InternalAsset>, configs: ConfigMap): string {
getCacheKey(assets: Array<UncommittedAsset>, configs: ConfigMap): string {
let assetsKeyInfo = assets.map(a => ({
filePath: a.value.filePath,
hash: a.value.hash,
Expand Down Expand Up @@ -492,7 +499,7 @@ type TransformerWithNameAndConfig = {|

async function runTransformer(
pipeline: Pipeline,
asset: InternalAsset,
asset: UncommittedAsset,
transformer: Transformer,
transformerName: string,
preloadedConfig: ?Config,
Expand Down Expand Up @@ -571,7 +578,7 @@ async function runTransformer(
);

// Create generate and postProcess functions that can be called later
pipeline.generate = (input: InternalAsset): Promise<GenerateOutput> => {
pipeline.generate = (input: UncommittedAsset): Promise<GenerateOutput> => {
if (transformer.generate && input.ast) {
let generated = transformer.generate({
asset: new Asset(input),
Expand All @@ -592,8 +599,8 @@ async function runTransformer(
let postProcess = transformer.postProcess;
if (postProcess) {
pipeline.postProcess = async (
assets: Array<InternalAsset>,
): Promise<Array<InternalAsset> | null> => {
assets: Array<UncommittedAsset>,
): Promise<Array<UncommittedAsset> | null> => {
let results = await postProcess.call(transformer, {
assets: assets.map(asset => new MutableAsset(asset)),
config,
Expand Down Expand Up @@ -626,7 +633,7 @@ function normalizeAssets(
return result;
}

let internalAsset = assetToInternalAsset(result);
let internalAsset = mutableAssetToUncommittedAsset(result);
return {
type: result.type,
content: await internalAsset.content,
Expand Down
Loading

0 comments on commit c690f91

Please sign in to comment.