Skip to content

Commit

Permalink
reorganize cache storage compression; workaround fix for #2812
Browse files Browse the repository at this point in the history
  • Loading branch information
@gorhill
gorhill committed Aug 11, 2018
1 parent 8f1b4b5 commit 38aabc9
Show file tree
Hide file tree
Showing 13 changed files with 1,002 additions and 313 deletions.
2 changes: 1 addition & 1 deletion platform/chromium/manifest.json
View file
Expand Up @@ -58,7 +58,7 @@
},
"incognito": "split",
"manifest_version": 2,
"minimum_chrome_version": "45.0",
"minimum_chrome_version": "47.0",
"name": "uBlock Origin",
"optional_permissions": [
"file:///*"
Expand Down
4 changes: 3 additions & 1 deletion src/background.html
View file
Expand Up @@ -5,17 +5,19 @@
<title>uBlock Origin</title>
</head>
<body>
<script src="lib/lz4/lz4-block-codec-any.js"></script>
<script src="lib/punycode.js"></script>
<script src="lib/publicsuffixlist.js"></script>
<script src="js/vapi.js"></script>
<script src="js/vapi-common.js"></script>
<script src="js/vapi-background.js"></script>
<script src="js/vapi-webrequest.js"></script><!-- Forks can pick the webext, chromium, or their own implementation -->
<script src="js/vapi-cachestorage.js"></script><!-- Optional -->
<script src="js/background.js"></script>
<script src="js/hntrie.js"></script>
<script src="js/utils.js"></script>
<script src="js/uritools.js"></script>
<script src="js/lz4.js"></script>
<script src="js/cachestorage.js"></script>
<script src="js/assets.js"></script>
<script src="js/redirect-engine.js"></script>
<script src="js/dynamic-net-filtering.js"></script>
Expand Down
282 changes: 14 additions & 268 deletions src/js/assets.js
View file
Expand Up @@ -19,8 +19,6 @@
Home: https://github.com/gorhill/uBlock
*/

/* global WebAssembly */

'use strict';

/******************************************************************************/
Expand Down Expand Up @@ -304,7 +302,7 @@ var saveAssetSourceRegistry = (function() {
var timer;
var save = function() {
timer = undefined;
vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry });
µBlock.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry });
};
return function(lazily) {
if ( timer !== undefined ) {
Expand Down Expand Up @@ -387,7 +385,7 @@ var getAssetSourceRegistry = function(callback) {
);
};

vAPI.cacheStorage.get('assetSourceRegistry', function(bin) {
µBlock.cacheStorage.get('assetSourceRegistry', function(bin) {
if ( !bin || !bin.assetSourceRegistry ) {
createRegistry();
return;
Expand All @@ -411,247 +409,6 @@ api.unregisterAssetSource = function(assetKey) {
});
};

/*******************************************************************************
Experimental support for cache storage compression.
For background information on the topic, see:
https://github.com/uBlockOrigin/uBlock-issues/issues/141#issuecomment-407737186
**/

let lz4Codec = (function() {
let lz4wasmInstance;
let pendingInitialization;
let textEncoder, textDecoder;
let ttlCount = 0;
let ttlTimer;

const ttlDelay = 60 * 1000;

let init = function() {
if (
lz4wasmInstance === null ||
WebAssembly instanceof Object === false ||
typeof WebAssembly.instantiateStreaming !== 'function'
) {
lz4wasmInstance = null;
return Promise.resolve(null);
}
if ( lz4wasmInstance instanceof WebAssembly.Instance ) {
return Promise.resolve(lz4wasmInstance);
}
if ( pendingInitialization === undefined ) {
pendingInitialization = WebAssembly.instantiateStreaming(
fetch('lib/lz4-block-codec.wasm', { mode: 'same-origin' })
).then(result => {
pendingInitialization = undefined;
lz4wasmInstance = result && result.instance || null;
});
pendingInitialization.catch(( ) => {
lz4wasmInstance = null;
});
}
return pendingInitialization;
};

// We can't shrink memory usage of wasm instances, and in the current
// case memory usage can grow to a significant amount given that
// a single contiguous memory buffer is required to accommodate both
// input and output data. Thus a time-to-live implementation which
// will cause the wasm instance to be forgotten after enough time
// elapse without the instance being used.

let destroy = function() {
console.info(
'uBO: freeing lz4-block-codec.wasm instance (memory.buffer = %d kB)',
lz4wasmInstance.exports.memory.buffer.byteLength >>> 10
);
lz4wasmInstance = undefined;
textEncoder = textDecoder = undefined;
ttlCount = 0;
ttlTimer = undefined;
};

let ttlManage = function(count) {
if ( ttlTimer !== undefined ) {
clearTimeout(ttlTimer);
ttlTimer = undefined;
}
ttlCount += count;
if ( ttlCount > 0 ) { return; }
if ( lz4wasmInstance === null ) { return; }
ttlTimer = vAPI.setTimeout(destroy, ttlDelay);
};

let growMemoryTo = function(byteLength) {
let lz4api = lz4wasmInstance.exports;
let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength;
let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16;
let pageCountAfter = (neededByteLength + 65535) >>> 16;
if ( pageCountAfter > pageCountBefore ) {
lz4api.memory.grow(pageCountAfter - pageCountBefore);
}
return lz4api.memory;
};

let resolveEncodedValue = function(resolve, key, value) {
let t0 = window.performance.now();
let lz4api = lz4wasmInstance.exports;
let mem0 = lz4api.getLinearMemoryOffset();
let memory = growMemoryTo(mem0 + 65536 * 4);
let hashTable = new Int32Array(memory.buffer, mem0, 65536);
hashTable.fill(-65536, 0, 65536);
let hashTableSize = hashTable.byteLength;
if ( textEncoder === undefined ) {
textEncoder = new TextEncoder();
}
let inputArray = textEncoder.encode(value);
let inputSize = inputArray.byteLength;
let memSize =
hashTableSize +
inputSize +
8 + lz4api.lz4BlockEncodeBound(inputSize);
memory = growMemoryTo(memSize);
let inputMem = new Uint8Array(
memory.buffer,
mem0 + hashTableSize,
inputSize
);
inputMem.set(inputArray);
let outputSize = lz4api.lz4BlockEncode(
mem0 + hashTableSize,
inputSize,
mem0 + hashTableSize + inputSize + 8
);
if ( outputSize === 0 ) { resolve(value); }
let outputMem = new Uint8Array(
memory.buffer,
mem0 + hashTableSize + inputSize,
8 + outputSize
);
outputMem[0] = 0x18;
outputMem[1] = 0x4D;
outputMem[2] = 0x22;
outputMem[3] = 0x04;
outputMem[4] = (inputSize >>> 0) & 0xFF;
outputMem[5] = (inputSize >>> 8) & 0xFF;
outputMem[6] = (inputSize >>> 16) & 0xFF;
outputMem[7] = (inputSize >>> 24) & 0xFF;
console.info(
'uBO: [%s] compressed %d bytes into %d bytes in %s ms',
key,
inputSize,
outputSize,
(window.performance.now() - t0).toFixed(2)
);
resolve(new Blob([ outputMem ]));
};

let resolveDecodedValue = function(resolve, ev, key, value) {
let inputBuffer = ev.target.result;
if ( inputBuffer instanceof ArrayBuffer === false ) {
return resolve(value);
}
let t0 = window.performance.now();
let metadata = new Uint8Array(inputBuffer, 0, 8);
if (
metadata[0] !== 0x18 ||
metadata[1] !== 0x4D ||
metadata[2] !== 0x22 ||
metadata[3] !== 0x04
) {
return resolve(value);
}
let inputSize = inputBuffer.byteLength - 8;
let outputSize =
(metadata[4] << 0) |
(metadata[5] << 8) |
(metadata[6] << 16) |
(metadata[7] << 24);
let lz4api = lz4wasmInstance.exports;
let mem0 = lz4api.getLinearMemoryOffset();
let memSize = inputSize + outputSize;
let memory = growMemoryTo(memSize);
let inputArea = new Uint8Array(
memory.buffer,
mem0,
inputSize
);
inputArea.set(new Uint8Array(inputBuffer, 8, inputSize));
outputSize = lz4api.lz4BlockDecode(inputSize);
if ( outputSize === 0 ) {
return resolve(value);
}
let outputArea = new Uint8Array(
memory.buffer,
mem0 + inputSize,
outputSize
);
if ( textDecoder === undefined ) {
textDecoder = new TextDecoder();
}
value = textDecoder.decode(outputArea);
console.info(
'uBO: [%s] decompressed %d bytes into %d bytes in %s ms',
key,
inputSize,
outputSize,
(window.performance.now() - t0).toFixed(2)
);
resolve(value);
};

let encodeValue = function(key, value) {
if ( !lz4wasmInstance ) {
return Promise.resolve(value);
}
return new Promise(resolve => {
resolveEncodedValue(resolve, key, value);
});
};

let decodeValue = function(key, value) {
if ( !lz4wasmInstance ) {
return Promise.resolve(value);
}
return new Promise(resolve => {
let blobReader = new FileReader();
blobReader.onloadend = ev => {
resolveDecodedValue(resolve, ev, key, value);
};
blobReader.readAsArrayBuffer(value);
});
};

return {
encode: function(key, value) {
if ( typeof value !== 'string' || value.length < 4096 ) {
return Promise.resolve(value);
}
ttlManage(1);
return init().then(( ) => {
return encodeValue(key, value);
}).then(result => {
ttlManage(-1);
return result;
});
},
decode: function(key, value) {
if ( value instanceof Blob === false ) {
return Promise.resolve(value);
}
ttlManage(1);
return init().then(( ) => {
return decodeValue(key, value);
}).then(result => {
ttlManage(-1);
return result;
});
}
};
})();

/*******************************************************************************
The purpose of the asset cache registry is to keep track of all assets
Expand Down Expand Up @@ -688,7 +445,7 @@ var getAssetCacheRegistry = function(callback) {
}
};

vAPI.cacheStorage.get('assetCacheRegistry', function(bin) {
µBlock.cacheStorage.get('assetCacheRegistry', function(bin) {
if ( bin && bin.assetCacheRegistry ) {
assetCacheRegistry = bin.assetCacheRegistry;
}
Expand All @@ -700,7 +457,7 @@ var saveAssetCacheRegistry = (function() {
var timer;
var save = function() {
timer = undefined;
vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry });
µBlock.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry });
};
return function(lazily) {
if ( timer !== undefined ) { clearTimeout(timer); }
Expand Down Expand Up @@ -735,16 +492,11 @@ var assetCacheRead = function(assetKey, callback) {
}
entry.readTime = Date.now();
saveAssetCacheRegistry(true);
if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) {
return reportBack(bin[internalKey]);
}
lz4Codec.decode(internalKey, bin[internalKey]).then(result => {
reportBack(result);
});
reportBack(bin[internalKey]);
};

let onReady = function() {
vAPI.cacheStorage.get(internalKey, onAssetRead);
µBlock.cacheStorage.get(internalKey, onAssetRead);
};

getAssetCacheRegistry(onReady);
Expand All @@ -763,10 +515,7 @@ var assetCacheWrite = function(assetKey, details, callback) {
return assetCacheRemove(assetKey, callback);
}

let reportBack = function(content) {
let bin = { assetCacheRegistry: assetCacheRegistry };
bin[internalKey] = content;
vAPI.cacheStorage.set(bin);
let reportBack = function() {
let details = { assetKey: assetKey, content: content };
if ( typeof callback === 'function' ) {
callback(details);
Expand All @@ -783,12 +532,9 @@ var assetCacheWrite = function(assetKey, details, callback) {
if ( details instanceof Object && typeof details.url === 'string' ) {
entry.remoteURL = details.url;
}
if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) {
return reportBack(content);
}
lz4Codec.encode(internalKey, content).then(result => {
reportBack(result);
});
let bin = { assetCacheRegistry: assetCacheRegistry };
bin[internalKey] = content;
µBlock.cacheStorage.set(bin, reportBack);
};
getAssetCacheRegistry(onReady);
};
Expand All @@ -810,9 +556,9 @@ var assetCacheRemove = function(pattern, callback) {
delete cacheDict[assetKey];
}
if ( removedContent.length !== 0 ) {
vAPI.cacheStorage.remove(removedContent);
µBlock.cacheStorage.remove(removedContent);
var bin = { assetCacheRegistry: assetCacheRegistry };
vAPI.cacheStorage.set(bin);
µBlock.cacheStorage.set(bin);
}
if ( typeof callback === 'function' ) {
callback();
Expand Down Expand Up @@ -852,7 +598,7 @@ var assetCacheMarkAsDirty = function(pattern, exclude, callback) {
}
if ( mustSave ) {
var bin = { assetCacheRegistry: assetCacheRegistry };
vAPI.cacheStorage.set(bin);
µBlock.cacheStorage.set(bin);
}
if ( typeof callback === 'function' ) {
callback();
Expand Down Expand Up @@ -892,7 +638,7 @@ var readUserAsset = function(assetKey, callback) {
var content = '';
if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
content = bin['cached_asset_content://assets/user/filters.txt'];
vAPI.cacheStorage.remove('cached_asset_content://assets/user/filters.txt');
µBlock.cacheStorage.remove('cached_asset_content://assets/user/filters.txt');
}
if ( typeof bin['assets/user/filters.txt'] === 'string' ) {
content = bin['assets/user/filters.txt'];
Expand Down

0 comments on commit 38aabc9

Please sign in to comment.