Permalink
Browse files

packager: add GlobalTransformCache

Reviewed By: davidaurelio

Differential Revision: D4175938

fbshipit-source-id: 1f57d594b4c8c8189feb2ea6d4d4011870ffd85f
  • Loading branch information...
jeanlauliac authored and Facebook Github Bot committed Nov 24, 2016
1 parent 4390927 commit 5d30045211ff1d97cd7005bc268442f3df998b17
@@ -45,5 +45,9 @@ module.exports = [
command: '--reset-cache',
description: 'Removes cached files',
default: false,
}, {
command: '--read-global-cache',
description: 'Try to fetch transformed JS code from the global cache, if configured.',
default: false,
},
];
View
@@ -175,6 +175,7 @@
"joi": "^6.6.1",
"json-stable-stringify": "^1.0.1",
"json5": "^0.4.0",
"left-pad": "^1.1.3",
"lodash": "^4.16.6",
"mime": "^1.3.4",
"mime-types": "2.1.11",
@@ -192,6 +193,7 @@
"react-transform-hmr": "^1.0.4",
"rebound": "^0.0.13",
"regenerator-runtime": "^0.9.5",
"request": "^2.79.0",
"rimraf": "^2.5.4",
"sane": "~1.4.1",
"semver": "^5.0.3",
@@ -19,7 +19,8 @@ jest.setMock('worker-farm', function() { return () => {}; })
.mock('../../AssetServer')
.mock('../../lib/declareOpts')
.mock('../../node-haste')
.mock('../../Logger');
.mock('../../Logger')
.mock('../../lib/GlobalTransformCache');
describe('processRequest', () => {
let SourceMapConsumer, Bundler, Server, AssetServer, Promise;
@@ -0,0 +1,198 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
const debounce = require('lodash/debounce');
const imurmurhash = require('imurmurhash');
const jsonStableStringify = require('json-stable-stringify');
const path = require('path');
const request = require('request');
const toFixedHex = require('./toFixedHex');
import type {CachedResult} from './TransformCache';
const SINGLE_REQUEST_MAX_KEYS = 100;
const AGGREGATION_DELAY_MS = 100;
type FetchResultURIs = (
keys: Array<string>,
callback: (error?: Error, results?: Map<string, string>) => void,
) => mixed;
type FetchProps = {
filePath: string,
sourceCode: string,
transformCacheKey: string,
transformOptions: mixed,
};
type FetchCallback = (error?: Error, resultURI?: ?CachedResult) => mixed;
type FetchURICallback = (error?: Error, resultURI?: ?string) => mixed;
/**
* We aggregate the requests to do a single request for many keys. It also
* ensures we do a single request at a time to avoid pressuring the I/O.
*/
class KeyURIFetcher {
_fetchResultURIs: FetchResultURIs;
_pendingQueries: Array<{key: string, callback: FetchURICallback}>;
_isProcessing: boolean;
_processQueriesDebounced: () => void;
_processQueries: () => void;
/**
* Fetch the pending keys right now, if any and if we're not already doing
* so in parallel. At the end of the fetch, we trigger a new batch fetching
* recursively.
*/
_processQueries() {
const {_pendingQueries} = this;
if (_pendingQueries.length === 0 || this._isProcessing) {
return;
}
this._isProcessing = true;
const queries = _pendingQueries.splice(0, SINGLE_REQUEST_MAX_KEYS);
const keys = queries.map(query => query.key);
this._fetchResultURIs(keys, (error, results) => {
queries.forEach(query => {
query.callback(error, results && results.get(query.key));
});
this._isProcessing = false;
process.nextTick(this._processQueries);
});
}
/**
* Enqueue the fetching of a particular key.
*/
fetch(key: string, callback: FetchURICallback) {
this._pendingQueries.push({key, callback});
this._processQueriesDebounced();
}
constructor(fetchResultURIs: FetchResultURIs) {
this._fetchResultURIs = fetchResultURIs;
this._pendingQueries = [];
this._isProcessing = false;
this._processQueries = this._processQueries.bind(this);
this._processQueriesDebounced =
debounce(this._processQueries, AGGREGATION_DELAY_MS);
}
}
function validateCachedResult(cachedResult: mixed): ?CachedResult {
if (
cachedResult != null &&
typeof cachedResult === 'object' &&
typeof cachedResult.code === 'string' &&
Array.isArray(cachedResult.dependencies) &&
cachedResult.dependencies.every(dep => typeof dep === 'string') &&
Array.isArray(cachedResult.dependencyOffsets) &&
cachedResult.dependencyOffsets.every(offset => typeof offset === 'number')
) {
return (cachedResult: any);
}
return undefined;
}
/**
* One can enable the global cache by calling configure() from a custom CLI
* script. Eventually we may make it more flexible.
*/
class GlobalTransformCache {
_fetcher: KeyURIFetcher;
static _global: ?GlobalTransformCache;
constructor(fetchResultURIs: FetchResultURIs) {
this._fetcher = new KeyURIFetcher(fetchResultURIs);
}
/**
* Return a key for identifying uniquely a source file.
*/
static keyOf(props: FetchProps) {
const sourceDigest = toFixedHex(8, imurmurhash(props.sourceCode).result());
const optionsHash = imurmurhash()
.hash(jsonStableStringify(props.transformOptions) || '')
.hash(props.transformCacheKey)
.result();
const optionsDigest = toFixedHex(8, optionsHash);
return (
`${optionsDigest}${sourceDigest}` +
`${path.basename(props.filePath)}`
);
}
/**
* We may want to improve that logic to return a stream instead of the whole
* blob of transformed results. However the results are generally only a few
* megabytes each.
*/
_fetchFromURI(uri: string, callback: FetchCallback) {
request.get({uri, json: true}, (error, response, unvalidatedResult) => {
if (error != null) {
callback(error);
return;
}
if (response.statusCode !== 200) {
callback(new Error(
`Unexpected HTTP status code: ${response.statusCode}`,
));
return;
}
const result = validateCachedResult(unvalidatedResult);
if (result == null) {
callback(new Error('Invalid result returned by server.'));
return;
}
callback(undefined, result);
});
}
fetch(props: FetchProps, callback: FetchCallback) {
this._fetcher.fetch(GlobalTransformCache.keyOf(props), (error, uri) => {
if (error != null) {
callback(error);
} else {
if (uri == null) {
callback();
return;
}
this._fetchFromURI(uri, callback);
}
});
}
/**
* For using the global cache one needs to have some kind of central key-value
* store that gets prefilled using keyOf() and the transformed results. The
* fetching function should provide a mapping of keys to URIs. The files
* referred by these URIs contains the transform results. Using URIs instead
* of returning the content directly allows for independent fetching of each
* result.
*/
static configure(fetchResultURIs: FetchResultURIs) {
GlobalTransformCache._global = new GlobalTransformCache(fetchResultURIs);
}
static get() {
return GlobalTransformCache._global;
}
}
GlobalTransformCache._global = null;
module.exports = GlobalTransformCache;
@@ -22,6 +22,7 @@ const jsonStableStringify = require('json-stable-stringify');
const mkdirp = require('mkdirp');
const path = require('path');
const rimraf = require('rimraf');
const toFixedHex = require('./toFixedHex');
const writeFileAtomicSync = require('write-file-atomic').sync;
const CACHE_NAME = 'react-native-packager-cache';
@@ -66,15 +67,14 @@ function getCacheFilePaths(props: {
const hasher = imurmurhash()
.hash(props.filePath)
.hash(jsonStableStringify(props.transformOptions) || '');
let hash = hasher.result().toString(16);
hash = Array(8 - hash.length + 1).join('0') + hash;
const hash = toFixedHex(8, hasher.result());
const prefix = hash.substr(0, 2);
const fileName = `${hash.substr(2)}${path.basename(props.filePath)}`;
const base = path.join(getCacheDirPath(), prefix, fileName);
return {transformedCode: base, metadata: base + '.meta'};
}
type CachedResult = {
export type CachedResult = {
code: string,
dependencies: Array<string>,
dependencyOffsets: Array<number>,
@@ -135,7 +135,7 @@ function writeSync(props: {
]));
}
type CacheOptions = {resetCache?: boolean};
export type CacheOptions = {resetCache?: boolean};
/* 1 day */
const GARBAGE_COLLECTION_PERIOD = 24 * 60 * 60 * 1000;
@@ -272,6 +272,14 @@ function readMetadataFileSync(
};
}
export type ReadTransformProps = {
filePath: string,
sourceCode: string,
transformOptions: mixed,
transformCacheKey: string,
cacheOptions: CacheOptions,
};
/**
* We verify the source hash matches to ensure we always favor rebuilding when
* source change (rather than just using fs.mtime(), a bit less robust).
@@ -285,13 +293,7 @@ function readMetadataFileSync(
* Meanwhile we store transforms with different options in different files so
* that it is fast to switch between ex. minified, or not.
*/
function readSync(props: {
filePath: string,
sourceCode: string,
transformOptions: mixed,
transformCacheKey: string,
cacheOptions: CacheOptions,
}): ?CachedResult {
function readSync(props: ReadTransformProps): ?CachedResult {
GARBAGE_COLLECTOR.collectIfNecessarySync(props.cacheOptions);
const cacheFilePaths = getCacheFilePaths(props);
let metadata, transformedCode;
@@ -0,0 +1,18 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
function get() {
return null;
}
module.exports = {get};
@@ -12,7 +12,9 @@
jest
.dontMock('imurmurhash')
.dontMock('json-stable-stringify')
.dontMock('../TransformCache');
.dontMock('../TransformCache')
.dontMock('../toFixedHex')
.dontMock('left-pad');
const imurmurhash = require('imurmurhash');
@@ -0,0 +1,20 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
const leftPad = require('left-pad');
function toFixedHex(length: number, number: number): string {
return leftPad(number.toString(16), length, '0');
}
module.exports = toFixedHex;
Oops, something went wrong.

0 comments on commit 5d30045

Please sign in to comment.