Skip to content
This repository has been archived by the owner on Oct 27, 2020. It is now read-only.

Commit

Permalink
feat: Allow to change how cache is stored
Browse files Browse the repository at this point in the history
  • Loading branch information
@Mordred
Mordred committed Oct 23, 2017
1 parent 88dfc00 commit 3afabf8
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 47 deletions.
85 changes: 82 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<img width="200" height="200" src="https://cdn.rawgit.com/webpack/media/e7485eb2/logo/icon-square-big.svg">
</a>
<h1>Cache Loader</h1>
<p>Caches the result of following loaders on disk</p>
<p>Caches the result of following loaders on disk (default) or in the database</p>
</div>

<h2 align="center">Install</h2>
Expand Down Expand Up @@ -47,8 +47,11 @@ module.exports = {

|Name|Type|Default|Description|
|:--:|:--:|:-----:|:----------|
|**`cacheDirectory`**|`{String}`|`path.resolve('.cache-loader')`|Provide a cache directory where cache items should be stored|
|**`cacheIdentifier`**|`{String}`|`cache-loader:{version} {process.env.NODE_ENV}`|Provide an invalidation identifier which is used to generate the hashes. You can use it for extra dependencies of loaders.|
|**`cacheDirectory`**|`{String}`|`path.resolve('.cache-loader')`|Provide a cache directory where cache items should be stored (used for default read/write implementation)|
|**`cacheIdentifier`**|`{String}`|`cache-loader:{version} {process.env.NODE_ENV}`|Provide an invalidation identifier which is used to generate the hashes. You can use it for extra dependencies of loaders. (used for default read/write implementation)|
|**`write`**|`{Function(cacheKey, data, callback) -> {void}}`|`undefined`|Allows you to override default write cache data to file (e.g. Redis, memcached).|
|**`read`**|`{Function(cacheKey, callback) -> {void}}`|`undefined`|Allows you to override default read cache data from file.|
|**`makeKey`**|`{Function(options, request) -> {string}}`|`undefined`|Allows you to override default cache key generator.|

<h2 align="center">Examples</h2>

Expand Down Expand Up @@ -95,6 +98,82 @@ module.exports = {
}
```

### `Example of Database Integration`

**webpack.config.js**
```js
const redis = require('redis'); // Or different database client - memcached, mongodb, ...
const crypto = require('crypto');

// ...
// ... connect to client
// ...

const BUILD_CACHE_TIMEOUT = 24 * 3600; // 1 day

function digest(str) {
return crypto.createHash('md5').update(str).digest('hex');
}

/**
* Generate own cache key
*/
function makeKey(options, request) {
return `build:cache:${digest(request)}`;
}

/**
* Read data from database and parse them
*/
function read(cacheKey, callback) {
client.get(cacheKey, (err, result) => {
if (err) {
return callback(err);
}

if (!result) {
return callback(new Error(`Key ${cacheKey} not found`));
}

try {
let data = JSON.parse(result);
callback(null, data);
} catch (e) {
callback(e);
}
});
}

/**
* Write data to database under cacheKey
*/
function write(cacheKey, data, callback) {
client.set(cacheKey, JSON.stringify(data), 'EX', BUILD_CACHE_TIMEOUT, callback);
}

module.exports = {
module: {
rules: [
{
test: /\.js$/,
use: [
{
loader: 'cache-loader',
options: {
read,
write,
makeKey,
}
},
'babel-loader'
],
include: path.resolve('src')
}
]
}
}
```

<h2 align="center">Maintainers</h2>

<table>
Expand Down
111 changes: 67 additions & 44 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,18 @@ const pkgVersion = require('../package.json').version;
const defaultCacheDirectory = path.resolve('.cache-loader');
const ENV = process.env.NODE_ENV || 'development';

const defaultOptions = {
cacheDirectory: defaultCacheDirectory,
cacheIdentifier: `cache-loader:${pkgVersion} ${ENV}`,
read,
write,
makeKey,
};

function loader(...args) {
const loaderOptions = loaderUtils.getOptions(this) || {};
const options = Object.assign({}, defaultOptions, loaderOptions);
const { write: writeFn } = options;
const callback = this.async();
const { data } = this;
const dependencies = this.getDependencies().concat(this.loaders.map(l => l.path));
Expand All @@ -35,62 +46,33 @@ function loader(...args) {
return;
}
const [deps, contextDeps] = taskResults;
const writeCacheFile = () => {
fs.writeFile(data.cacheFile, JSON.stringify({
remainingRequest: data.remainingRequest,
cacheIdentifier: data.cacheIdentifier,
dependencies: deps,
contextDependencies: contextDeps,
result: args,
}), 'utf-8', () => {
// ignore errors here
callback(null, ...args);
});
};
if (data.fileExists) {
// for performance skip creating directory
writeCacheFile();
} else {
mkdirp(path.dirname(data.cacheFile), (mkdirErr) => {
if (mkdirErr) {
callback(null, ...args);
return;
}
writeCacheFile();
});
}
writeFn(data.cacheKey, {
remainingRequest: data.remainingRequest,
dependencies: deps,
contextDependencies: contextDeps,
result: args,
}, () => {
// ignore errors here
callback(null, ...args);
});
});
}

function pitch(remainingRequest, prevRequest, dataInput) {
const loaderOptions = loaderUtils.getOptions(this) || {};
const defaultOptions = {
cacheDirectory: defaultCacheDirectory,
cacheIdentifier: `cache-loader:${pkgVersion} ${ENV}`,
};
const options = Object.assign({}, defaultOptions, loaderOptions);
const { cacheIdentifier, cacheDirectory } = options;
const { read: readFn, makeKey: makeKeyFn } = options;
const data = dataInput;
const callback = this.async();
const hash = digest(`${cacheIdentifier}\n${remainingRequest}`);
const cacheFile = path.join(cacheDirectory, `${hash}.json`);
const cacheKey = makeKeyFn(options, remainingRequest);
data.remainingRequest = remainingRequest;
data.cacheIdentifier = cacheIdentifier;
data.cacheFile = cacheFile;
fs.readFile(cacheFile, 'utf-8', (readFileErr, content) => {
if (readFileErr) {
data.cacheKey = cacheKey;
readFn(cacheKey, (readErr, cacheData) => {
if (readErr) {
callback();
return;
}
data.fileExists = true;
let cacheData;
try {
cacheData = JSON.parse(content);
} catch (e) {
callback();
return;
}
if (cacheData.remainingRequest !== remainingRequest || cacheData.cacheIdentifier !== cacheIdentifier) {
if (cacheData.remainingRequest !== remainingRequest) {
// in case of a hash conflict
callback();
return;
Expand Down Expand Up @@ -123,4 +105,45 @@ function digest(str) {
return crypto.createHash('md5').update(str).digest('hex');
}

const directoriesExists = {};
function write(cacheKey, data, callback) {
const dirname = path.dirname(cacheKey);
const content = JSON.stringify(data);
if (directoriesExists[dirname]) {
// for performance skip creating directory
fs.writeFile(cacheKey, content, 'utf-8', callback);
} else {
mkdirp(dirname, (mkdirErr) => {
if (mkdirErr) {
callback(mkdirErr);
return;
}
directoriesExists[dirname] = true;
fs.writeFile(cacheKey, content, 'utf-8', callback);
});
}
}

function read(cacheKey, callback) {
fs.readFile(cacheKey, 'utf-8', (err, content) => {
if (err) {
callback(err);
return;
}

try {
const data = JSON.parse(content);
callback(null, data);
} catch (e) {
callback(e);
}
});
}

function makeKey(options, request) {
const { cacheIdentifier, cacheDirectory } = options;
const hash = digest(`${cacheIdentifier}\n${request}`);
return path.join(cacheDirectory, `${hash}.json`);
}

export { loader as default, pitch };

0 comments on commit 3afabf8

Please sign in to comment.