feat(StoreRegistry): add queue and virtual file support (#344)

* feat(StoreRegistry): add queue and virtual file support

* docs: clarify `loadPiece` doc

Co-authored-by: Jeroen Claassens <support@favware.tech>

* refactor: add virtual file reload support

---------

Co-authored-by: Jeroen Claassens <support@favware.tech>

src/index.ts

@@ -3,6 +3,7 @@ import 'tslib';
 export * from './lib/errors/LoaderError';
 export * from './lib/errors/MissingExportsError';
 export * from './lib/internal/RootScan';
+export { VirtualPath } from './lib/internal/constants';
 export * from './lib/shared/Container';
 export * from './lib/strategies/ILoaderStrategy';
 export * from './lib/strategies/LoaderStrategy';

src/lib/errors/LoaderError.ts

@@ -1,7 +1,9 @@
 export enum LoaderErrorType {
 	EmptyModule = 'EMPTY_MODULE',
+	VirtualPiece = 'VIRTUAL_PIECE',
 	UnloadedPiece = 'UNLOADED_PIECE',
-	IncorrectType = 'INCORRECT_TYPE'
+	IncorrectType = 'INCORRECT_TYPE',
+	UnknownStore = 'UNKNOWN_STORE'
 }
 
 /**

src/lib/internal/constants.ts

@@ -0,0 +1,2 @@
+export const VirtualPath = '::virtual::';
+export const ManuallyRegisteredPiecesSymbol = Symbol('@sapphire/pieces:ManuallyRegisteredPieces');

src/lib/structures/PieceLocation.ts

@@ -1,4 +1,5 @@
 import { basename, relative, sep } from 'path';
+import { VirtualPath } from '../internal/constants';
 
 /**
  * The metadata class used for {@link Piece}s.
@@ -23,6 +24,13 @@ export class PieceLocation {
 		this.root = root;
 	}
 
+	/**
+	 * Whether the file is virtual or not.
+	 */
+	public get virtual() {
+		return this.full === VirtualPath;
+	}
+
 	/**
 	 * The relative path between {@link PieceLocation.root} and {@link PieceLocation.full}.
 	 * @example
@@ -37,7 +45,7 @@ export class PieceLocation {
 	 * ```
 	 */
 	public get relative(): string {
-		return relative(this.root, this.full);
+		return this.virtual ? VirtualPath : relative(this.root, this.full);
 	}
 
 	/**
@@ -54,7 +62,7 @@ export class PieceLocation {
 	 * ```
 	 */
 	public get directories(): string[] {
-		return this.relative.split(sep).slice(0, -1);
+		return this.virtual ? [] : this.relative.split(sep).slice(0, -1);
 	}
 
 	/**
@@ -71,7 +79,7 @@ export class PieceLocation {
 	 * ```
 	 */
 	public get name(): string {
-		return basename(this.full);
+		return this.virtual ? VirtualPath : basename(this.full);
 	}
 
 	/**

src/lib/structures/Store.ts

@@ -1,9 +1,10 @@
 import { Collection } from '@discordjs/collection';
-import type { AbstractConstructor } from '@sapphire/utilities';
+import { Constructor, classExtends, isClass, type AbstractConstructor } from '@sapphire/utilities';
 import { promises as fsp } from 'fs';
 import { join } from 'path';
 import { LoaderError, LoaderErrorType } from '../errors/LoaderError';
 import { resolvePath, type Path } from '../internal/Path';
+import { ManuallyRegisteredPiecesSymbol, VirtualPath } from '../internal/constants';
 import { container, type Container } from '../shared/Container';
 import type { HydratedModuleData, ILoaderResultEntry, ILoaderStrategy, ModuleData } from '../strategies/ILoaderStrategy';
 import { LoaderStrategy } from '../strategies/LoaderStrategy';
@@ -50,20 +51,30 @@ export interface StoreLogger {
 /**
  * The store class which contains {@link Piece}s.
  */
-export class Store<T extends Piece> extends Collection<string, T> {
+export class Store<T extends Piece, StoreName extends keyof StoreRegistryEntries = keyof StoreRegistryEntries> extends Collection<string, T> {
 	public readonly Constructor: AbstractConstructor<T>;
-	public readonly name: string;
+	public readonly name: keyof StoreRegistryEntries;
 	public readonly paths: Set<string>;
 	public readonly strategy: ILoaderStrategy<T>;
 
+	/**
+	 * The queue of manually registered pieces to load.
+	 */
+	private readonly [ManuallyRegisteredPiecesSymbol]: StoreManuallyRegisteredPiece<StoreName>[] = [];
+
+	/**
+	 * Whether or not the store has called `loadAll` at least once.
+	 */
+	#calledLoadAll = false;
+
 	/**
 	 * @param constructor The piece constructor this store loads.
 	 * @param options The options for the store.
 	 */
 	public constructor(constructor: AbstractConstructor<T>, options: StoreOptions<T>) {
 		super();
 		this.Constructor = constructor;
-		this.name = options.name;
+		this.name = options.name as keyof StoreRegistryEntries;
 		this.paths = new Set(options.paths ?? []);
 		this.strategy = options.strategy ?? Store.defaultStrategy;
 	}
@@ -94,13 +105,74 @@ export class Store<T extends Piece> extends Collection<string, T> {
 		return this;
 	}
 
+	/**
+	 * Adds a piece into the store's list of manually registered pieces. If {@linkcode Store.loadAll()} was called, the
+	 * piece will be loaded immediately, otherwise it will be queued until {@linkcode Store.loadAll()} is called.
+	 *
+	 * All manually registered pieces will be kept even after they are loaded to ensure they can be loaded again if
+	 * {@linkcode Store.loadAll()} is called again.
+	 *
+	 * @remarks
+	 *
+	 * - Pieces loaded this way will have their {@linkcode Piece.Context.root root} and
+	 *   {@linkcode Piece.Context.path path} set to {@linkcode VirtualPath}, and as such, cannot be reloaded.
+	 * - This method is useful in environments where file system access is limited or unavailable, such as when using
+	 *   {@link https://en.wikipedia.org/wiki/Serverless_computing Serverless Computing}.
+	 * - This method will always throw a {@link TypeError} if `entry.piece` is not a class.
+	 * - This method will always throw a {@linkcode LoaderError} if the piece does not extend the
+	 *   {@linkcode Store#Constructor store's piece constructor}.
+	 * - This operation is atomic, if any of the above errors are thrown, the piece will not be loaded.
+	 *
+	 * @seealso {@linkcode StoreRegistry.loadPiece()}
+	 * @since 3.8.0
+	 * @param entry The entry to load.
+	 * @example
+	 * ```typescript
+	 * import { container } from '@sapphire/pieces';
+	 *
+	 * class PingCommand extends Command {
+	 *   // ...
+	 * }
+	 *
+	 * container.stores.get('commands').loadPiece({
+	 *   name: 'ping',
+	 *   piece: PingCommand
+	 * });
+	 * ```
+	 */
+	public async loadPiece(entry: StoreManuallyRegisteredPiece<StoreName>) {
+		if (!isClass(entry.piece)) {
+			throw new TypeError(`The piece ${entry.name} is not a Class. ${String(entry.piece)}`);
+		}
+
+		// If the piece does not extend the store's Piece class, throw an error:
+		if (!classExtends(entry.piece, this.Constructor as Constructor<T>)) {
+			throw new LoaderError(LoaderErrorType.IncorrectType, `The piece ${entry.name} does not extend ${this.name}`);
+		}
+
+		this[ManuallyRegisteredPiecesSymbol].push(entry);
+		if (this.#calledLoadAll) {
+			const piece = this.construct(entry.piece as unknown as Constructor<T>, {
+				name: entry.name,
+				root: VirtualPath,
+				path: VirtualPath,
+				extension: VirtualPath
+			});
+			await this.insert(piece);
+		}
+	}
+
 	/**
 	 * Loads one or more pieces from a path.
 	 * @param root The root directory the file is from.
 	 * @param path The path of the file to load, relative to the `root`.
 	 * @return All the loaded pieces.
 	 */
 	public async load(root: string, path: string): Promise<T[]> {
+		if (root === VirtualPath) {
+			throw new LoaderError(LoaderErrorType.VirtualPiece, `Cannot load a virtual file.`);
+		}
+
 		const full = join(root, path);
 		const data = this.strategy.filter(full);
 		if (data === null) {
@@ -156,7 +228,18 @@ export class Store<T extends Piece> extends Collection<string, T> {
 	 * Loads all pieces from all directories specified by {@link paths}.
 	 */
 	public async loadAll(): Promise<void> {
+		this.#calledLoadAll = true;
+
 		const pieces: T[] = [];
+		for (const entry of this[ManuallyRegisteredPiecesSymbol]) {
+			const piece = this.construct(entry.piece as unknown as Constructor<T>, {
+				name: entry.name,
+				root: VirtualPath,
+				path: VirtualPath,
+				extension: VirtualPath
+			});
+			pieces.push(piece);
+		}
 
 		for (const path of this.paths) {
 			for await (const piece of this.loadPath(path)) {
@@ -309,6 +392,15 @@ export class Store<T extends Piece> extends Collection<string, T> {
 	public static logger: StoreLogger | null = null;
 }
 
+/**
+ * An entry for a manually registered piece using {@linkcode Store.loadPiece()}.
+ * @since 3.8.0
+ */
+export interface StoreManuallyRegisteredPiece<StoreName extends keyof StoreRegistryEntries> {
+	name: string;
+	piece: StoreRegistryEntries[StoreName] extends Store<infer Piece> ? Constructor<Piece> : never;
+}
+
 type ErrorWithCode = Error & { code: string };
 
 export namespace Store {

src/lib/structures/StoreRegistry.ts

@@ -1,9 +1,12 @@
 import { Collection } from '@discordjs/collection';
 import { join } from 'path';
+import { LoaderError } from '../errors/LoaderError';
 import { resolvePath, type Path } from '../internal/Path';
 import { getRootData } from '../internal/RootScan';
+import { ManuallyRegisteredPiecesSymbol, VirtualPath } from '../internal/constants';
 import type { Piece } from './Piece';
-import type { Store } from './Store';
+import type { Store, StoreManuallyRegisteredPiece } from './Store';
+import { isClass } from '@sapphire/utilities';
 
 type Key = keyof StoreRegistryEntries;
 type Value = StoreRegistryEntries[Key];
@@ -29,12 +32,20 @@ type Value = StoreRegistryEntries[Key];
  * ```
  */
 export class StoreRegistry extends Collection<Key, Value> {
+	/**
+	 * The queue of pieces to load.
+	 */
+	readonly #pendingManuallyRegisteredPieces = new Collection<
+		keyof StoreRegistryEntries,
+		StoreManuallyRegisteredPiece<keyof StoreRegistryEntries>[]
+	>();
+
 	/**
 	 * Loads all the registered stores.
 	 * @since 2.1.0
 	 */
 	public async load() {
-		const promises: Promise<void>[] = [];
+		const promises: Promise<unknown>[] = [];
 		for (const store of this.values() as IterableIterator<Store<Piece>>) {
 			promises.push(store.loadAll());
 		}
@@ -75,11 +86,38 @@ export class StoreRegistry extends Collection<Key, Value> {
 
 	/**
 	 * Registers a store.
+	 *
+	 * @remarks
+	 *
+	 * - This method will allow {@linkcode StoreRegistry} to manage the store, meaning:
+	 *   - {@linkcode StoreRegistry.registerPath()} will call the store's
+	 *     {@linkcode Store.registerPath() registerPath()} method on call.
+	 *   - {@linkcode StoreRegistry.load()} will call the store's {@linkcode Store.load() load()} method on call.
+	 *   - {@linkcode StoreRegistry.loadPiece()} will call the store's {@linkcode Store.loadPiece() loadPiece()} method
+	 *     on call.
+	 * - This will also add all the manually registered pieces by {@linkcode StoreRegistry.loadPiece()} in the store.
+	 *
+	 * It is generally recommended to register a store as early as possible, before any of the aforementioned methods
+	 * are called, otherwise you will have to manually call the aforementioned methods for the store to work properly.
+	 *
+	 * If there were manually registered pieces for this store with {@linkcode StoreRegistry.loadPiece()}, this method
+	 * will add them to the store and delete the queue. Note, however, that this method will not call the store's
+	 * {@linkcode Store.loadPiece() loadPiece()} method, and as such, the pieces will not be loaded until
+	 * {@linkcode Store.loadAll()} is called.
+	 *
 	 * @since 2.1.0
 	 * @param store The store to register.
 	 */
 	public register<T extends Piece>(store: Store<T>): this {
 		this.set(store.name as Key, store as unknown as Value);
+
+		// If there was a queue for this store, add it to the store and delete the queue:
+		const queue = this.#pendingManuallyRegisteredPieces.get(store.name);
+		if (queue) {
+			store[ManuallyRegisteredPiecesSymbol].push(...queue);
+			this.#pendingManuallyRegisteredPieces.delete(store.name);
+		}
+
 		return this;
 	}
 
@@ -92,6 +130,57 @@ export class StoreRegistry extends Collection<Key, Value> {
 		this.delete(store.name as Key);
 		return this;
 	}
+
+	/**
+	 * If the store was {@link StoreRegistry.register registered}, this method will call the store's
+	 * {@linkcode Store.loadPiece() loadPiece()} method.
+	 *
+	 * If it was called, the entry will be loaded immediately without queueing.
+	 *
+	 * @remarks
+	 *
+	 * - Pieces loaded this way will have their {@linkcode Piece.Context.root root} and
+	 *   {@linkcode Piece.Context.path path} set to {@linkcode VirtualPath}, and as such, cannot be reloaded.
+	 * - This method is useful in environments where file system access is limited or unavailable, such as when using
+	 *   {@link https://en.wikipedia.org/wiki/Serverless_computing Serverless Computing}.
+	 * - This method will not throw an error if a store with the given name does not exist, it will simply be queued
+	 *   until it's registered.
+	 * - This method will always throw a {@link TypeError} if `entry.piece` is not a class.
+	 * - If the store is registered, this method will always throw a {@linkcode LoaderError} if the piece does not
+	 *   extend the registered {@linkcode Store.Constructor store's piece constructor}.
+	 * - This operation is atomic, if any of the above errors are thrown, the piece will not be loaded.
+	 *
+	 * @seealso {@linkcode Store.loadPiece()}
+	 * @since 3.8.0
+	 * @param entry The entry to load.
+	 * @example
+	 * ```typescript
+	 * import { container } from '@sapphire/pieces';
+	 *
+	 * class PingCommand extends Command {
+	 *   // ...
+	 * }
+	 *
+	 * container.stores.loadPiece({
+	 *   store: 'commands',
+	 *   name: 'ping',
+	 *   piece: PingCommand
+	 * });
+	 * ```
+	 */
+	public async loadPiece<StoreName extends keyof StoreRegistryEntries>(entry: StoreManagerManuallyRegisteredPiece<StoreName>) {
+		const store = this.get(entry.store) as Store<Piece, StoreName> | undefined;
+
+		if (store) {
+			await store.loadPiece(entry);
+		} else {
+			if (!isClass(entry.piece)) {
+				throw new TypeError(`The piece ${entry.name} is not a Class. ${String(entry.piece)}`);
+			}
+
+			this.#pendingManuallyRegisteredPieces.ensure(entry.store, () => []).push({ name: entry.name, piece: entry.piece });
+		}
+	}
 }
 
 export interface StoreRegistry {
@@ -106,3 +195,12 @@ export interface StoreRegistry {
  * @since 2.1.0
  */
 export interface StoreRegistryEntries {}
+
+/**
+ * An entry for a manually registered piece using {@linkcode StoreRegistry.loadPiece()}.
+ * @seealso {@linkcode StoreRegistry.loadPiece()}
+ * @since 3.8.0
+ */
+export interface StoreManagerManuallyRegisteredPiece<StoreName extends keyof StoreRegistryEntries> extends StoreManuallyRegisteredPiece<StoreName> {
+	store: StoreName;
+}