Adds wasLastFetchHit method used by Eleventy Image plugin to invalidate disk cache. Removes setInitialCacheTimestamp which is no longer needed for this use case (and wasn’t released). Changes in memory cache to persist instances after requests complete.

Author: zachleat

eleventy-fetch.js

@@ -34,33 +34,34 @@ queue.on("active", () => {
 	debug(`Concurrency: ${queue.concurrency}, Size: ${queue.size}, Pending: ${queue.pending}`);
 });
 
-let inProgress = {};
+let instCache = {};
 
-function queueSave(source, queueCallback, options) {
+function createRemoteAssetCache(source, rawOptions = {}) {
+	if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) {
+		return Promise.reject(new Error("Invalid source. Received: " + source));
+	}
+
+	let options = Object.assign({}, globalOptions, rawOptions);
 	let sourceKey = RemoteAssetCache.getRequestId(source, options);
 	if(!sourceKey) {
 		return Promise.reject(Sources.getInvalidSourceError(source));
 	}
 
-	if (!inProgress[sourceKey]) {
-		inProgress[sourceKey] = queue.add(queueCallback).finally(() => {
-			delete inProgress[sourceKey];
-		});
+	if(instCache[sourceKey]) {
+		return instCache[sourceKey];
 	}
 
-	return inProgress[sourceKey];
+	let inst = new RemoteAssetCache(source, options.directory, options);
+	inst.setQueue(queue);
+
+	instCache[sourceKey] = inst;
+
+	return inst;
 }
 
 module.exports = function (source, options) {
-	if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) {
-		throw new Error("Caching an already local asset is not yet supported.");
-	}
-
-	let mergedOptions = Object.assign({}, globalOptions, options);
-	return queueSave(source, () => {
-		let asset = new RemoteAssetCache(source, mergedOptions.directory, mergedOptions);
-		return asset.fetch(mergedOptions);
-	}, mergedOptions);
+	let instance = createRemoteAssetCache(source, options);
+	return instance.queue();
 };
 
 Object.defineProperty(module.exports, "concurrency", {
@@ -72,7 +73,15 @@ Object.defineProperty(module.exports, "concurrency", {
 	},
 });
 
-module.exports.queue = queueSave;
+module.exports.Fetch = createRemoteAssetCache;
+
+// Deprecated API kept for backwards compat, instead: use default export directly.
+// Intentional: queueCallback is ignored here
+module.exports.queue = function(source, queueCallback, options) {
+	let instance = createRemoteAssetCache(source, options);
+	return instance.queue();
+};
+
 module.exports.Util = {
 	isFullUrl: Sources.isFullUrl,
 };

src/AssetCache.js

@@ -26,9 +26,11 @@ class AssetCache {
 		this.hash = AssetCache.getHash(uniqueKey, options.hashLength);
 
 		this.cacheDirectory = cacheDirectory || ".cache";
-		this.defaultDuration = "1d";
 		this.options = options;
 
+		this.defaultDuration = "1d";
+		this.duration = options.duration || this.defaultDuration;
+
 		// Compute the filename only once
 		if (typeof this.options.filenameFormat === "function") {
 			this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash));
@@ -39,10 +41,6 @@ class AssetCache {
 		}
 	}
 
-	setInitialCacheTimestamp(timestamp) {
-		this.initialCacheTimestamp = timestamp;
-	}
-
 	log(message) {
 		if (this.options.verbose) {
 			console.log(`[11ty/eleventy-fetch] ${message}`);
@@ -234,31 +232,29 @@ class AssetCache {
 			throw new Error("save(contents) expects contents (was falsy)");
 		}
 
+		let contentPath = this.getCachedContentsPath(type);
+		if(this.options.dryRun) {
+			debug(`Dry run writing ${contentPath}`);
+			return;
+		}
+
 		this.ensureDir();
 
 		if (type === "json" || type === "parsed-xml") {
 			contents = JSON.stringify(contents);
 		}
 
-		let contentPath = this.getCachedContentsPath(type);
-
 		// the contents must exist before the cache metadata are saved below
-		if(!this.options.dryRun) {
-			fs.writeFileSync(contentPath, contents);
-			debug(`Writing ${contentPath}`);
-		} else {
-			debug(`Dry run writing ${contentPath}`);
-		}
+		fs.writeFileSync(contentPath, contents);
+		debug(`Writing ${contentPath}`);
 
 		this.cache.set(this.hash, {
-			cachedAt: this.initialCacheTimestamp || Date.now(),
+			cachedAt: Date.now(),
 			type: type,
 			metadata,
 		});
 
-		if(!this.options.dryRun) {
-			this.cache.save();
-		}
+		this.cache.save();
 	}
 
 	async getCachedContents(type) {
@@ -304,10 +300,10 @@ class AssetCache {
 	}
 
 	getCachedTimestamp() {
-		return this.cachedObject?.cachedAt || this.initialCacheTimestamp;
+		return this.cachedObject?.cachedAt;
 	}
 
-	isCacheValid(duration = this.defaultDuration) {
+	isCacheValid(duration = this.duration) {
 		if (!this.cachedObject) {
 			// not cached
 			return false;

src/RemoteAssetCache.js

@@ -2,9 +2,13 @@ const { parseXml } = require('@rgrove/parse-xml');
 
 const Sources = require("./Sources.js");
 const AssetCache = require("./AssetCache.js");
-// const debug = require("debug")("Eleventy:Fetch");
+const assetDebug = require("debug")("Eleventy:Assets");
 
 class RemoteAssetCache extends AssetCache {
+	#queue;
+	#queuePromise;
+	#lastFetchType;
+
 	constructor(source, cacheDirectory, options = {}) {
 		let requestId = RemoteAssetCache.getRequestId(source, options);
 		super(requestId, cacheDirectory, options);
@@ -95,15 +99,48 @@ class RemoteAssetCache extends AssetCache {
 		return Buffer.from(await response.arrayBuffer());
 	}
 
+	setQueue(queue) {
+		this.#queue = queue;
+	}
+
+	// Returns raw Promise
+	queue() {
+		if(!this.#queue) {
+			throw new Error("Missing `#queue` instance.");
+		}
+
+		if(this.#queuePromise) {
+			return this.#queuePromise;
+		}
+
+		// optionsOverride not supported on fetch here for re-use
+		this.#queuePromise = this.#queue.add(() => this.fetch());
+
+		return this.#queuePromise;
+	}
+
+	isCacheValid(duration = undefined) {
+		// uses this.options.duration if not explicitly defined here
+		return super.isCacheValid(duration);
+	}
+
+	// if last fetch was a cache hit (no fetch occurred) or a cache miss (fetch did occur)
+	// used by Eleventy Image in disk cache checks.
+	wasLastFetchHit() {
+		return this.#lastFetchType === "hit";
+	}
+
 	async fetch(optionsOverride = {}) {
-		let duration = optionsOverride.duration || this.options.duration;
 		// Important: no disk writes when dryRun
 		// As of Fetch v4, reads are now allowed!
-		if (super.isCacheValid(duration)) {
+		if (this.isCacheValid(optionsOverride.duration)) {
 			this.log(`Cache hit for ${this.displayUrl}`);
+			this.#lastFetchType = "hit";
 			return super.getCachedValue();
 		}
 
+		this.#lastFetchType = "miss";
+
 		try {
 			let isDryRun = optionsOverride.dryRun || this.options.dryRun;
 			this.log(`${isDryRun ? "Fetching" : "Cache miss for"} ${this.displayUrl}`);
@@ -124,6 +161,7 @@ class RemoteAssetCache extends AssetCache {
 
 				this.fetchCount++;
 
+				assetDebug("Fetching remote asset: %o", this.source);
 				// v5: now using global (Node-native or otherwise) fetch instead of node-fetch
 				let response = await fetch(this.source, fetchOptions);
 				if (!response.ok) {

test/AssetCacheTest.js

@@ -137,45 +137,3 @@ test("Uses filenameFormat", async (t) => {
 	t.falsy(fs.existsSync(cachePath));
 	t.falsy(fs.existsSync(jsonCachePath));
 });
-
-test("setInitialCacheTimestamp method (used by Eleventy Image to establish a consistent cached file name in synchronous contexts)", async (t) => {
-	let cache = new AssetCache("this_is_a_test", ".cache", {
-		dryRun: true
-	});
-	let timestamp = (new Date(2024,1,1)).getTime();
-	cache.setInitialCacheTimestamp(timestamp);
-
-	await cache.save("test");
-
-	t.is(cache.getCachedTimestamp(), timestamp);
-});
-
-test("setInitialCacheTimestamp method after save is ignored", async (t) => {
-	let cache = new AssetCache("this_is_a_test2", ".cache", {
-		dryRun: true
-	});
-
-	let timestamp = (new Date(2024,1,1)).getTime();
-
-	await cache.save("test");
-
-	cache.setInitialCacheTimestamp(timestamp);
-
-	t.not(cache.getCachedTimestamp(), timestamp);
-});
-
-test("setInitialCacheTimestamp method before a second save is used", async (t) => {
-	let cache = new AssetCache("this_is_a_test3", ".cache", {
-		dryRun: true
-	});
-
-	let timestamp = (new Date(2024,1,1)).getTime();
-
-	await cache.save("test");
-
-	cache.setInitialCacheTimestamp(timestamp);
-
-	await cache.save("test");
-
-	t.is(cache.getCachedTimestamp(), timestamp);
-});

test/QueueTest.js

@@ -1,7 +1,6 @@
 const test = require("ava");
-const Cache = require("../");
-const queue = Cache.queue;
-const RemoteAssetCache = require("../src/RemoteAssetCache");
+const Cache = require("../eleventy-fetch.js");
+const { queue, Fetch } = Cache;
 
 test("Queue without options", async (t) => {
 	let example = "https://example.com/";
@@ -28,7 +27,7 @@ test("Double Fetch", async (t) => {
 	await ac1;
 	await ac2;
 
-	let forDestroyOnly = new RemoteAssetCache(pngUrl);
+	let forDestroyOnly = Fetch(pngUrl);
 	// file is now accessible
 	try {
 		await forDestroyOnly.destroy();
@@ -46,7 +45,8 @@ test("Double Fetch (dry run)", async (t) => {
 	await ac1;
 	await ac2;
 
-	let forTestOnly = new RemoteAssetCache(pngUrl, ".cache", {
+	let forTestOnly = Fetch(pngUrl, {
+		cacheDirectory: ".cache",
 		dryRun: true,
 	});
 	// file is now accessible