Skip to content
This repository has been archived by the owner on Oct 25, 2023. It is now read-only.

Commit

Permalink
fix: Update apps cache behaviour for expired items (#541)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mykola-mokhnach
mykola-mokhnach committed Nov 19, 2021
1 parent 981a624 commit c0a9c6a
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 56 deletions.
153 changes: 97 additions & 56 deletions lib/basedriver/helpers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ const CACHED_APPS_MAX_AGE = 1000 * 60 * 60 * 24; // ms
const APPLICATIONS_CACHE = new LRU({
maxAge: CACHED_APPS_MAX_AGE, // expire after 24 hours
updateAgeOnGet: true,
dispose: async (app, {fullPath}) => {
if (!await fs.exists(fullPath)) {
return;
}

logger.info(`The application '${app}' cached at '${fullPath}' has expired`);
await fs.rimraf(fullPath);
dispose: (app, {fullPath}) => {
logger.info(`The application '${app}' cached at '${fullPath}' has ` +
`expired after ${CACHED_APPS_MAX_AGE}ms`);
setTimeout(0, async () => {
if (fullPath && await fs.exists(fullPath)) {
await fs.rimraf(fullPath);
}
});
},
noDisposeOnSet: true,
});
Expand Down Expand Up @@ -66,48 +67,50 @@ async function retrieveHeaders (link) {
return {};
}

function getCachedApplicationPath (link, currentAppProps = {}) {
function getCachedApplicationPath (link, currentAppProps = {}, cachedAppInfo = {}) {
const refresh = () => {
logger.debug(`A fresh copy of the application is going to be downloaded from ${link}`);
return null;
};

if (APPLICATIONS_CACHE.has(link)) {
const {
lastModified: currentModified,
immutable: currentImmutable,
// maxAge is in seconds
maxAge: currentMaxAge,
} = currentAppProps;
const {
// Date instance
lastModified,
// boolean
immutable,
// Unix time in milliseconds
timestamp,
fullPath,
} = APPLICATIONS_CACHE.get(link);
if (lastModified && currentModified) {
if (currentModified.getTime() <= lastModified.getTime()) {
logger.debug(`The application at ${link} has not been modified since ${lastModified}`);
return fullPath;
}
logger.debug(`The application at ${link} has been modified since ${lastModified}`);
return refresh();
}
if (immutable && currentImmutable) {
logger.debug(`The application at ${link} is immutable`);
if (!_.isPlainObject(cachedAppInfo) || !_.isPlainObject(currentAppProps)) {
return refresh();
}

const {
lastModified: currentModified,
immutable: currentImmutable,
// maxAge is in seconds
maxAge: currentMaxAge,
} = currentAppProps;
const {
// Date instance
lastModified,
// boolean
immutable,
// Unix time in milliseconds
timestamp,
fullPath,
} = cachedAppInfo;
if (lastModified && currentModified) {
if (currentModified.getTime() <= lastModified.getTime()) {
logger.debug(`The application at ${link} has not been modified since ${lastModified}`);
return fullPath;
}
if (currentMaxAge && timestamp) {
const msLeft = timestamp + currentMaxAge * 1000 - Date.now();
if (msLeft > 0) {
logger.debug(`The cached application '${path.basename(fullPath)}' will expire in ${msLeft / 1000}s`);
return fullPath;
}
logger.debug(`The cached application '${path.basename(fullPath)}' has expired`);
logger.debug(`The application at ${link} has been modified since ${lastModified}`);
return refresh();
}
if (immutable && currentImmutable) {
logger.debug(`The application at ${link} is immutable`);
return fullPath;
}
if (currentMaxAge && timestamp) {
const msLeft = timestamp + currentMaxAge * 1000 - Date.now();
if (msLeft > 0) {
logger.debug(`The cached application '${path.basename(fullPath)}' will expire in ${msLeft / 1000}s`);
return fullPath;
}
logger.debug(`The cached application '${path.basename(fullPath)}' has expired`);
}
return refresh();
}
Expand All @@ -121,6 +124,36 @@ function verifyAppExtension (app, supportedAppExtensions) {
supportedAppExtensions);
}

async function calculateFolderIntegrity (folderPath) {
return (await fs.glob('**/*', {cwd: folderPath, strict: false, nosort: true})).length;
}

async function calculateFileIntegrity (filePath) {
return await fs.hash(filePath);
}

async function isAppIntegrityOk (currentPath, expectedIntegrity = {}) {
if (!await fs.exists(currentPath)) {
return false;
}

const isDir = (await fs.stat(currentPath)).isDirectory();
// Folder integrity check is simple:
// Verify the previous amount of files is not greater than the current one.
// We don't want to use equality comparison because of an assumption that the OS might
// create some unwanted service files/cached inside of that folder or its subfolders.
// Ofc, validating the hash sum of each file (or at least of file path) would be much
// more precise, but we don't need to be very precise here and also don't want to
// overuse RAM and have a performance drop.
if (isDir && await calculateFolderIntegrity(currentPath) >= expectedIntegrity?.folder) {
return true;
}
if (!isDir && await calculateFileIntegrity(currentPath) === expectedIntegrity?.file) {
return true;
}
return false;
}

async function configureApp (app, supportedAppExtensions) {
if (!_.isString(app)) {
// immediately shortcircuit if not given an app
Expand All @@ -141,6 +174,8 @@ async function configureApp (app, supportedAppExtensions) {
const {protocol, pathname} = url.parse(newApp);
const isUrl = ['http:', 'https:'].includes(protocol);

const cachedAppInfo = APPLICATIONS_CACHE.get(app);

return await APPLICATIONS_CACHE_GUARD.acquire(app, async () => {
if (isUrl) {
// Use the app from remote URL
Expand All @@ -160,13 +195,14 @@ async function configureApp (app, supportedAppExtensions) {
}
logger.debug(`Cache-Control: ${headers['cache-control']}`);
}
const cachedPath = getCachedApplicationPath(app, remoteAppProps);
const cachedPath = getCachedApplicationPath(app, remoteAppProps, cachedAppInfo);
if (cachedPath) {
if (await fs.exists(cachedPath)) {
if (await isAppIntegrityOk(cachedPath, cachedAppInfo?.integrity)) {
logger.info(`Reusing previously downloaded application at '${cachedPath}'`);
return verifyAppExtension(cachedPath, supportedAppExtensions);
}
logger.info(`The application at '${cachedPath}' does not exist anymore. Deleting it from the cache`);
logger.info(`The application at '${cachedPath}' does not exist anymore ` +
`or its integrity has been damaged. Deleting it from the internal cache`);
APPLICATIONS_CACHE.del(app);
}

Expand Down Expand Up @@ -236,17 +272,18 @@ async function configureApp (app, supportedAppExtensions) {

if (shouldUnzipApp) {
const archivePath = newApp;
archiveHash = await fs.hash(archivePath);
if (APPLICATIONS_CACHE.has(app) && archiveHash === APPLICATIONS_CACHE.get(app).hash) {
const {fullPath} = APPLICATIONS_CACHE.get(app);
if (await fs.exists(fullPath)) {
archiveHash = await calculateFileIntegrity(archivePath);
if (archiveHash === cachedAppInfo?.archiveHash) {
const {fullPath} = cachedAppInfo;
if (await isAppIntegrityOk(fullPath, cachedAppInfo?.integrity)) {
if (archivePath !== app) {
await fs.rimraf(archivePath);
}
logger.info(`Will reuse previously cached application at '${fullPath}'`);
return verifyAppExtension(fullPath, supportedAppExtensions);
}
logger.info(`The application at '${fullPath}' does not exist anymore. Deleting it from the cache`);
logger.info(`The application at '${fullPath}' does not exist anymore ` +
`or its integrity has been damaged. Deleting it from the cache`);
APPLICATIONS_CACHE.del(app);
}
const tmpRoot = await tempDir.openDir();
Expand All @@ -268,17 +305,21 @@ async function configureApp (app, supportedAppExtensions) {
verifyAppExtension(newApp, supportedAppExtensions);

if (app !== newApp && (archiveHash || _.values(remoteAppProps).some(Boolean))) {
if (APPLICATIONS_CACHE.has(app)) {
const {fullPath} = APPLICATIONS_CACHE.get(app);
// Clean up the obsolete entry first if needed
if (fullPath !== newApp && await fs.exists(fullPath)) {
await fs.rimraf(fullPath);
}
const cachedFullPath = cachedAppInfo?.fullPath;
if (cachedFullPath && cachedFullPath !== newApp && await fs.exists(cachedFullPath)) {
await fs.rimraf(cachedFullPath);
}
const integrity = {};
if ((await fs.stat(newApp)).isDirectory()) {
integrity.folder = await calculateFolderIntegrity(newApp);
} else {
integrity.file = await calculateFileIntegrity(newApp);
}
APPLICATIONS_CACHE.set(app, {
...remoteAppProps,
timestamp: Date.now(),
hash: archiveHash,
archiveHash,
integrity,
fullPath: newApp,
});
}
Expand Down
3 changes: 3 additions & 0 deletions test/basedriver/helpers-specs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ describe('helpers', function () {
sandbox.stub(fs, 'hash').resolves('0xDEADBEEF');
sandbox.stub(fs, 'glob').resolves(['/path/to/an.apk']);
sandbox.stub(fs, 'rimraf').resolves();
sandbox.stub(fs, 'stat').resolves({
isDirectory: () => false,
});
sandbox.stub(tempDir, 'openDir').resolves('/some/dir');
});

Expand Down

0 comments on commit c0a9c6a

Please sign in to comment.