Skip to content

Commit

Permalink
Use LMDB cache instead of regular FS cache to speed up cache hits
Browse files Browse the repository at this point in the history
  • Loading branch information
Shubham Kanodia committed Mar 7, 2022
1 parent 052c07a commit 068be4b
Show file tree
Hide file tree
Showing 10 changed files with 3,717 additions and 3,713 deletions.
29 changes: 3 additions & 26 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ jobs:
- uses: actions/setup-node@v1
with:
node-version: "*"
cache: 'yarn'
- name: Install dependencies
run: yarn
- name: Lint
Expand All @@ -20,7 +21,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
node-version: [10.x, 12.x, 14.x, 15.x]
node-version: [12.x, 14.x, 16.x]
webpack-version: [latest, '4']
include:
- node: 14.x
Expand All @@ -33,6 +34,7 @@ jobs:
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
cache: 'yarn'
- name: Install dependencies
run: yarn
- name: Install webpack ${{ matrix.webpack-version }}
Expand All @@ -48,29 +50,4 @@ jobs:
if: ${{ matrix.coverage }}
with:
token: ${{ secrets.CODECOV_TOKEN }}
test-legacy:
name: Test - ubuntu-latest - Node v8.9, Webpack 4
runs-on: ubuntu-latest
env:
YARN_NODE_LINKER: node-modules
steps:
- uses: actions/checkout@v2
- name: Use Node.js 14.x
uses: actions/setup-node@v1
with:
node-version: 14.x
- name: Install dependencies
run: yarn
- name: Install webpack 4
run: yarn add -D webpack@4
- name: Build babel-loader
run: yarn run build
env:
BABEL_ENV: test
- name: Use Node.js 8.9
uses: actions/setup-node@v1
with:
node-version: '8.9'
- name: Run tests for webpack version 4
run: node scripts/test-legacy

55 changes: 0 additions & 55 deletions .yarn/releases/yarn-2.3.3.cjs

This file was deleted.

785 changes: 785 additions & 0 deletions .yarn/releases/yarn-3.2.0.cjs

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion .yarnrc.yml
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
yarnPath: .yarn/releases/yarn-2.3.3.cjs
yarnPath: .yarn/releases/yarn-3.2.0.cjs
nodeLinker: node-modules
2 changes: 1 addition & 1 deletion babel.config.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
["@babel/preset-env", {
"loose": true,
"targets": {
"node": "6.9"
"node": "12.0.0"
}
}]
],
Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
},
"dependencies": {
"find-cache-dir": "^3.3.1",
"lmdb": "^2.2.4",
"loader-utils": "^1.4.0",
"make-dir": "^3.1.0",
"schema-utils": "^2.6.5"
},
"peerDependencies": {
Expand All @@ -36,6 +36,7 @@
"eslint-plugin-prettier": "^3.0.0",
"husky": "^4.3.0",
"lint-staged": "^10.5.1",
"node-preload": "^0.2.1",
"nyc": "^15.1.0",
"pnp-webpack-plugin": "^1.6.4",
"prettier": "^2.1.2",
Expand Down
110 changes: 34 additions & 76 deletions src/cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,62 +7,48 @@
* @see https://github.com/babel/babel-loader/issues/34
* @see https://github.com/babel/babel-loader/pull/41
*/
const fs = require("fs");
const os = require("os");
const path = require("path");
const zlib = require("zlib");
const crypto = require("crypto");
const findCacheDir = require("find-cache-dir");
const { promisify } = require("util");
const { open } = require("lmdb");

const transform = require("./transform");
// Lazily instantiated when needed
let defaultCacheDirectory = null;

const readFile = promisify(fs.readFile);
const writeFile = promisify(fs.writeFile);
const gunzip = promisify(zlib.gunzip);
const gzip = promisify(zlib.gzip);
const makeDir = require("make-dir");
let cacheDB = null;

/**
* Read the contents from the compressed file.
*
* @async
* @params {String} filename
* @params {Boolean} compress
* Initialize cache
*/
const read = async function (filename, compress) {
const data = await readFile(filename + (compress ? ".gz" : ""));
const content = compress ? await gunzip(data) : data;

return JSON.parse(content.toString());
};
async function initCacheDB(cacheDir, cacheCompression) {
if (cacheDB) return cacheDB;
const fallback = cacheDir !== os.tmpdir();

/**
* Write contents into a compressed file.
*
* @async
* @params {String} filename
* @params {Boolean} compress
* @params {String} result
*/
const write = async function (filename, compress, result) {
const content = JSON.stringify(result);
try {
cacheDB = open({
path: cacheDir,
compression: cacheCompression,
sharedStructuresKey: Symbol.for(`structures`),
});
} catch (err) {
if (fallback) {
cacheDB = initCacheDB(os.tmpdir(), cacheCompression);
}

const data = compress ? await gzip(content) : content;
return await writeFile(filename + (compress ? ".gz" : ""), data);
};
throw err;
}
}

/**
* Build the filename for the cached file
* Build the cache key for the cached file
*
* @params {String} source File source code
* @params {Object} options Options used
*
* @return {String}
*/
const filename = function (source, identifier, options) {
const fileCacheKey = function (source, identifier, options) {
// md4 hashing is not supported starting with node v17.0.0
const majorNodeVersion = parseInt(process.versions.node.split(".")[0], 10);
let hashType = "md4";
Expand All @@ -76,7 +62,7 @@ const filename = function (source, identifier, options) {

hash.update(contents);

return hash.digest("hex") + ".json";
return hash.digest("hex");
};

/**
Expand All @@ -85,51 +71,21 @@ const filename = function (source, identifier, options) {
* @params {String} directory
* @params {Object} params
*/
const handleCache = async function (directory, params) {
const {
source,
options = {},
cacheIdentifier,
cacheDirectory,
cacheCompression,
} = params;
const handleCache = async function (params) {
const { source, options = {}, cacheIdentifier } = params;

const file = path.join(directory, filename(source, cacheIdentifier, options));

try {
// No errors mean that the file was previously cached
// we just need to return it
return await read(file, cacheCompression);
} catch (err) {}
const cacheKey = fileCacheKey(source, cacheIdentifier, options);

const fallback =
typeof cacheDirectory !== "string" && directory !== os.tmpdir();

// Make sure the directory exists.
try {
await makeDir(directory);
} catch (err) {
if (fallback) {
return handleCache(os.tmpdir(), params);
}

throw err;
// Fetch cached result if it exists
const cached = await cacheDB.get(cacheKey);
if (typeof cached !== "undefined") {
return cached;
}

// Otherwise just transform the file
// Otherwise, just transform the cacheKey
// return it to the user asap and write it in cache
const result = await transform(source, options);

try {
await write(file, cacheCompression, result);
} catch (err) {
if (fallback) {
// Fallback to tmpdir if node_modules folder not writable
return handleCache(os.tmpdir(), params);
}

throw err;
}
cacheDB.put(cacheKey, result);

return result;
};
Expand Down Expand Up @@ -173,5 +129,7 @@ module.exports = async function (params) {
directory = defaultCacheDirectory;
}

return await handleCache(directory, params);
await initCacheDB(directory, params.cacheCompression);

return await handleCache(params);
};
Loading

0 comments on commit 068be4b

Please sign in to comment.