Add support of new /doc api with convinient functionality to download…

… list of json docs
redneckz · Apr 15, 2024 · c9cbb3e · c9cbb3e
1 parent 0984a7f
commit c9cbb3e
Show file tree

Hide file tree

Showing 6 changed files with 63 additions and 31 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@redneckz/wildless-cms-content",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "license": "MIT",
   "author": {
     "name": "redneckz",

diff --git a/src/ContentPageRepository.ts b/src/ContentPageRepository.ts
@@ -1,6 +1,6 @@
-import { type JSONNode, type JSONRecord } from '@redneckz/json-op';
+import type { JSONNode, JSONRecord } from '@redneckz/json-op';
 import path from 'path/posix';
-import type { FileAPI, FilePath, FileQuery } from './api/FileAPI';
+import type { FileAPI, FilePath, ListFilesOptions } from './api/FileAPI';
 import { FileStorageAPI, type FileStorageOptions } from './api/FileStorageAPI';
 import { FileSystemAPI } from './api/FileSystemAPI';
 import { isFileExists } from './fs/isFileExists';
@@ -31,13 +31,19 @@ export class ContentPageRepository implements FileAPI {
     private readonly storageAPI = new FileStorageAPI(options)
   ) {}
 
-  async listFiles(options: { dir?: string; ext?: string } & FileQuery): Promise<FilePath[]> {
+  async listFiles(options: ListFilesOptions): Promise<FilePath[]> {
     return (
       await Promise.allSettled([FileSystemAPI.inst.listFiles(options), this.storageAPI.listFiles(options)])
     ).flatMap(result => (result.status === 'fulfilled' ? result.value : []));
   }
 
-  async countFiles(options: { dir?: string; ext?: string } & FileQuery): Promise<number> {
+  async downloadFiles(options: ListFilesOptions): Promise<[FilePath, JSONNode][]> {
+    return (
+      await Promise.allSettled([FileSystemAPI.inst.downloadFiles(options), this.storageAPI.downloadFiles(options)])
+    ).flatMap(result => (result.status === 'fulfilled' ? result.value : []));
+  }
+
+  async countFiles(options: ListFilesOptions): Promise<number> {
     return (await Promise.allSettled([FileSystemAPI.inst.countFiles(options), this.storageAPI.countFiles(options)]))
       .flatMap(result => (result.status === 'fulfilled' ? result.value : 0))
       .reduce((a, b) => a + b);

diff --git a/src/api/FileAPI.ts b/src/api/FileAPI.ts
@@ -2,9 +2,11 @@ import { type JSONNode } from '@redneckz/json-op';
 
 export type FilePath = string;
 export type FileQuery = Record<string, string>;
+export type ListFilesOptions = { dir?: string; ext?: string } & FileQuery;
 
 export interface FileAPI {
-  listFiles(options: { dir?: string; ext?: string } & FileQuery): Promise<FilePath[]>;
-  countFiles(options: { dir?: string; ext?: string } & FileQuery): Promise<number>;
+  listFiles(options: ListFilesOptions): Promise<FilePath[]>;
+  downloadFiles(options: ListFilesOptions): Promise<[FilePath, JSONNode][]>;
+  countFiles(options: ListFilesOptions): Promise<number>;
   readJSON<T extends JSONNode = JSONNode>(filePath: FilePath): Promise<T>;
 }
diff --git a/src/api/FileStorageAPI.ts b/src/api/FileStorageAPI.ts
@@ -1,5 +1,5 @@
 import { type JSONNode } from '@redneckz/json-op';
-import type { FileAPI, FilePath, FileQuery } from './FileAPI';
+import type { FileAPI, FilePath, ListFilesOptions } from './FileAPI';
 
 const API_BASE_PATH = '/api/v1/wcms-file-storage';
 
@@ -9,6 +9,10 @@ export interface FileMeta {
   name: string;
 }
 
+export interface FileMetaWithJSON extends FileMeta {
+  json?: string;
+}
+
 export interface FileStorageOptions {
   projectId?: string;
   baseURL?: string;
@@ -26,25 +30,29 @@ export class FileStorageAPI implements FileAPI {
 
   constructor(private readonly options: FileStorageOptions = {}) {}
 
-  async listFiles(options: { dir?: string; ext?: string } & FileQuery): Promise<FilePath[]> {
+  async listFiles(options: ListFilesOptions): Promise<FilePath[]> {
     if (!this.projectId) {
       return [];
     }
 
-    let pageItems: FileMeta[] = [];
-    let items: FileMeta[] = [];
-    do {
-      const prevRevision = pageItems.length ? pageItems[pageItems.length - 1].revision : 0;
-      pageItems = await this.fetchProjectDocs(options, prevRevision);
-      items = items.concat(pageItems);
-    } while (pageItems.length > 0);
-    return items.map(({ publicId, name }) => name || publicId);
+    return (await this.fetchAllProjectDocs(options)).map(({ publicId, name }) => name || publicId);
   }
 
-  async countFiles(options: { dir?: string; ext?: string } & FileQuery): Promise<number> {
+  async downloadFiles(options: ListFilesOptions): Promise<[FilePath, JSONNode][]> {
+    if (!this.projectId) {
+      return [];
+    }
+
+    return (await this.fetchAllProjectDocs({ ...options, expanded: 'true' })).map(({ publicId, name, json }) => [
+      name || publicId,
+      json ? JSON.parse(json) : null
+    ]);
+  }
+
+  async countFiles(options: ListFilesOptions): Promise<number> {
     const params = this.computeDocQueryParams(options);
     const response = await (this.options.fetch ?? globalThis.fetch)(
-      `${this.baseURL}${API_BASE_PATH}/project/${this.projectId}/doc?${params}`,
+      `${this.baseURL}${API_BASE_PATH}/projects/${this.projectId}/docs?${params}`,
       { method: 'HEAD' }
     );
     return parseInt(response.headers.get('X-Total-Count') ?? '0', 10);
@@ -56,26 +64,34 @@ export class FileStorageAPI implements FileAPI {
     }
 
     const response = await (this.options.fetch ?? globalThis.fetch)(
-      `${this.baseURL}${API_BASE_PATH}/project/${this.projectId}/file/${encodeURIComponent(filePath)}`
+      `${this.baseURL}${API_BASE_PATH}/projects/${this.projectId}/files/${encodeURIComponent(filePath)}`
     );
     return (await response.json()) as T;
   }
 
+  private async fetchAllProjectDocs(options: ListFilesOptions = {}): Promise<FileMetaWithJSON[]> {
+    let pageItems: FileMeta[] = [];
+    let results: FileMeta[] = [];
+    do {
+      const prevRevision = pageItems.length ? pageItems[pageItems.length - 1].revision : 0;
+      pageItems = await this.fetchProjectDocs(options, prevRevision);
+      results = results.concat(pageItems);
+    } while (pageItems.length > 0);
+    return results;
+  }
+
   private async fetchProjectDocs(
-    options: { dir?: string; ext?: string } & FileQuery = {},
+    options: ListFilesOptions & { expanded?: boolean } = {},
     fromRevision = 0
-  ): Promise<FileMeta[]> {
+  ): Promise<FileMetaWithJSON[]> {
     const params = this.computeDocQueryParams(options, fromRevision);
     const response = await (this.options.fetch ?? globalThis.fetch)(
-      `${this.baseURL}${API_BASE_PATH}/project/${this.projectId}/doc?${params}`
+      `${this.baseURL}${API_BASE_PATH}/projects/${this.projectId}/docs?${params}`
     );
     return parseNDJSON(await response.text());
   }
 
-  private computeDocQueryParams(
-    { dir, ext, ...query }: { dir?: string; ext?: string } & FileQuery = {},
-    fromRevision = 0
-  ): URLSearchParams {
+  private computeDocQueryParams({ dir, ext, ...query }: ListFilesOptions = {}, fromRevision = 0): URLSearchParams {
     return new URLSearchParams({
       ...(dir ? { dir } : {}),
       ...(ext ? { ext } : {}),

diff --git a/src/api/FileSystemAPI.ts b/src/api/FileSystemAPI.ts
@@ -3,20 +3,26 @@ import glob from 'glob';
 import path from 'path/posix';
 import { promisify } from 'util';
 import { readJSON } from '../fs/readJSON';
-import type { FileAPI, FilePath, FileQuery } from './FileAPI';
+import type { FileAPI, FilePath, ListFilesOptions } from './FileAPI';
 
 const find = promisify(glob);
 
 export class FileSystemAPI implements FileAPI {
   public static readonly inst: FileAPI = new FileSystemAPI();
 
-  async listFiles({ dir, ext }: { dir?: string; ext?: string } & FileQuery): Promise<FilePath[]> {
+  async listFiles({ dir, ext }: ListFilesOptions): Promise<FilePath[]> {
     const files: string[] = await find(`*${ext}`, { cwd: dir, matchBase: true });
     return files.map(_ => (dir ? path.join(dir, _) : _));
   }
 
-  async countFiles(): Promise<number> {
-    return 0;
+  async downloadFiles(options: ListFilesOptions): Promise<[FilePath, JSONNode][]> {
+    const files = await this.listFiles(options);
+    return Promise.all(files.map(async _ => [_, await this.readJSON(_)]));
+  }
+
+  async countFiles(options: ListFilesOptions): Promise<number> {
+    const files = await this.listFiles(options);
+    return files.length;
   }
 
   async readJSON<T extends JSONNode = JSONNode>(filePath: FilePath): Promise<T> {

diff --git a/src/inheritance/inheritJSONDocContent.spec.ts b/src/inheritance/inheritJSONDocContent.spec.ts
@@ -7,6 +7,7 @@ describe('inheritJSONDocContent', () => {
 
     const api = {
       listFiles: async () => [],
+      downloadFiles: async () => [],
       countFiles: async () => 0,
       readJSON: async () => ({})
     } as FileAPI;
@@ -24,6 +25,7 @@ describe('inheritJSONDocContent', () => {
 
     const api = {
       listFiles: async () => [],
+      downloadFiles: async () => [],
       countFiles: async () => 0,
       readJSON: async () => parentContent
     } as FileAPI;