feat(*): support deleting remote files

klieber · Apr 8, 2020 · bdb4e0c · bdb4e0c
1 parent 37b862d
commit bdb4e0c
Show file tree

Hide file tree

Showing 10 changed files with 244 additions and 104 deletions.
diff --git a/bin/media-sync.js b/bin/media-sync.js
@@ -14,16 +14,16 @@ const handlers = [new ImageHandler(process.env.TARGET_PATH)];
     const files = await provider.list();
     // TODO: remove the slice call
     await asyncForEach(files, async (file) => {
-      const handler = handlers.find((handler) => handler.supports(file.path_lower));
+      const handler = handlers.find((handler) => handler.supports(file.remoteFile));
       if (handler) {
         try {
-          const filename = await provider.download(file);
-          await handler.handle(filename);
+          await file.download();
+          await handler.handle(file);
         } catch (error) {
-          logger.error(`unable to download file: ${file.path_lower}: `, error);
+          logger.error(`unable to download file: ${file.remoteFile}: `, error);
         }
       } else {
-        logger.warn(`unsupported file: ${file.path_lower}`);
+        logger.warn(`unsupported file: ${file.remoteFile}`);
       }
     });
   } catch (error) {

diff --git a/lib/handler/image-handler.js b/lib/handler/image-handler.js
@@ -1,6 +1,4 @@
-// const { extractDate } = require('../support/date-extractor');
-const { createFileCoordinates } = require('../support/file-namer.js');
-const fs = require('fs');
+const { moveFile } = require('../support/file-utils.js');
 const logger = require('../support/logger').create('lib/handler/image-handler');
 
 class ImageHandler {
@@ -14,13 +12,13 @@ class ImageHandler {
     return filename.match(/\.(?:jpg|jpeg|png)/, '');
   }
 
-  async handle(filename) {
-    const coordinates = await createFileCoordinates(this.#target, filename);
-
-    logger.info(`moving ${filename} to ${coordinates.name}`);
-
-    await fs.promises.mkdir(coordinates.dirname, { recursive: true });
-    await fs.promises.rename(filename, coordinates.name);
+  async handle(mediaFile) {
+    try {
+      await moveFile(mediaFile.sourceFile, this.#target);
+      await mediaFile.delete();
+    } catch (error) {
+      logger.error(error.message);
+    }
   }
 }
 

diff --git a/lib/provider/dropbox/dropbox-media-file.js b/lib/provider/dropbox/dropbox-media-file.js
@@ -0,0 +1,52 @@
+const logger = require('../../support/logger').create('lib/provider/dropbox/dropbox-media-file');
+const fs = require('fs');
+
+class DropboxMediaFile {
+  #dropboxService;
+  #dropboxFile;
+  #storagePath;
+  #sourceFile;
+  #deleted;
+
+  constructor(dropboxService, dropboxFile, storagePath) {
+    this.#dropboxService = dropboxService;
+    this.#dropboxFile = dropboxFile;
+    this.#storagePath = storagePath;
+    this.#deleted = false;
+  }
+
+  get fileType() {
+    return this.#dropboxFile.name.replace(/^.*\.([^\w]+)/, '$1');
+  }
+
+  get sourceFile() {
+    return this.#sourceFile;
+  }
+
+  get remoteFile() {
+    return this.#dropboxFile.name;
+  }
+
+  async download() {
+    if (this.#deleted) {
+      throw new Error(`cannot download file that has been deleted: ${this.remoteFile}`);
+    } else if (this.#sourceFile) {
+      throw new Error(`already downloaded: ${this.#sourceFile}`);
+    } else {
+      logger.info(`downloading ${this.remoteFile}`);
+      this.#sourceFile = await this.#dropboxService.downloadAndVerify(this.#dropboxFile, this.#storagePath);
+    }
+  }
+
+  async delete() {
+    if (this.#deleted) {
+      throw new Error(`cannot delete file that has already been deleted: ${this.#sourceFile}`);
+    }
+    logger.info(`deleting remote file: ${this.remoteFile}`);
+    await this.#dropboxService.delete(this.#dropboxFile);
+    await fs.promises.unlink(this.#sourceFile);
+    this.#deleted = true;
+  }
+}
+
+module.exports = DropboxMediaFile;
diff --git a/lib/provider/dropbox/dropbox-media-file.test.js b/lib/provider/dropbox/dropbox-media-file.test.js
@@ -0,0 +1,49 @@
+const DropboxMediaFile = require('./dropbox-media-file');
+const { when } = require('jest-when');
+
+const dropboxService = {
+  downloadAndVerify: jest.fn()
+};
+
+const TARGET_PATH = './target-path';
+const DROPBOX_FILE_NAME = 'mockFile.jpg';
+const DROPBOX_FILE = {
+  path_lower: `/mockFolder/${DROPBOX_FILE_NAME}`,
+  name: DROPBOX_FILE_NAME
+};
+
+describe('DropboxMediaFile', () => {
+  let mediaFile;
+  beforeEach(() => {
+    mediaFile = new DropboxMediaFile(dropboxService, DROPBOX_FILE, TARGET_PATH);
+  });
+
+  describe('download', () => {
+    test('file downloads successfully', async () => {
+      when(dropboxService.downloadAndVerify)
+        .calledWith(DROPBOX_FILE, TARGET_PATH)
+        .mockResolvedValue(`${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
+
+      await mediaFile.download();
+      expect(mediaFile.sourceFile).toBe(`${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
+    });
+
+    test('file can only be downloaded once', async () => {
+      when(dropboxService.downloadAndVerify)
+        .calledWith(DROPBOX_FILE, TARGET_PATH)
+        .mockResolvedValue(`${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
+
+      await mediaFile.download();
+
+      expect(mediaFile.sourceFile).toBe(`${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
+      expect(mediaFile.download()).rejects.toThrow(`already downloaded: ${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
+    });
+
+    test.todo('file cannot be downloaded after it has been deleted');
+  });
+
+  describe('delete', () => {
+    test.todo('file is deleted successfully');
+    test.todo('file cannot be deleted if it has already been deleted');
+  });
+});
diff --git a/lib/provider/dropbox/dropbox-provider.js b/lib/provider/dropbox/dropbox-provider.js
@@ -1,3 +1,4 @@
+const DropboxMediaFile = require('./dropbox-media-file');
 const logger = require('../../support/logger').create('lib/provider/dropbox/dropbox-provider');
 
 class DropboxProvider {
@@ -12,13 +13,11 @@ class DropboxProvider {
   }
 
   async list() {
+    logger.debug(`listing files in ${this.#source}`);
     const response = await this.#dropboxService.listFiles(this.#source);
-    return response.entries.filter((entry) => entry['.tag'] === 'file');
-  }
-
-  async download(file) {
-    logger.info(`downloading ${file.path_lower}`);
-    return await this.#dropboxService.downloadAndVerify(file, this.#target);
+    return response.entries
+      .filter((entry) => entry['.tag'] === 'file')
+      .map((file) => new DropboxMediaFile(this.#dropboxService, file, this.#target));
   }
 }
 

diff --git a/lib/provider/dropbox/dropbox-provider.test.js b/lib/provider/dropbox/dropbox-provider.test.js
@@ -1,39 +1,47 @@
 const DropboxProvider = require('./dropbox-provider');
+const DropboxMediaFile = require('./dropbox-media-file');
 const { when } = require('jest-when');
 
 const dropboxService = {
   listFiles: jest.fn(),
   downloadAndVerify: jest.fn()
 };
 
-const SOURCE_PATH = './source-path';
+const SOURCE_PATH = '/source-path';
 const TARGET_PATH = './target-path';
-const DROPBOX_FILE_NAME = 'mockFile.jpg';
-const DROPBOX_FILE = {
-  '.tag': 'file',
-  path_lower: DROPBOX_FILE_NAME
-};
 
-const dropboxProvider = new DropboxProvider(dropboxService, SOURCE_PATH, TARGET_PATH);
+function dropboxFile(name, tag) {
+  return {
+    '.tag': tag || 'file',
+    name: name
+  };
+}
 
-test('DropboxProvider list files', async (done) => {
-  when(dropboxService.listFiles)
-    .calledWith(SOURCE_PATH)
-    .mockResolvedValue({
-      entries: [DROPBOX_FILE]
-    });
+function dropboxFolder(name) {
+  return dropboxFile(name, 'folder');
+}
 
-  const files = await dropboxProvider.list();
-  expect(files).toEqual(expect.arrayContaining([DROPBOX_FILE]));
-  done();
-});
+const dropboxProvider = new DropboxProvider(dropboxService, SOURCE_PATH, TARGET_PATH);
 
-test('DropboxProvider download file', async (done) => {
-  when(dropboxService.downloadAndVerify)
-    .calledWith(DROPBOX_FILE, TARGET_PATH)
-    .mockResolvedValue(`${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
+describe('DropboxProvider', () => {
+  describe('list', () => {
+    test('only include files (no folders)', async () => {
+      when(dropboxService.listFiles)
+        .calledWith(SOURCE_PATH)
+        .mockResolvedValue({
+          entries: [
+            dropboxFile('a.jpg'),
+            dropboxFile('b.jpg'),
+            dropboxFolder('folderA'),
+            dropboxFile('c.jpg'),
+            dropboxFolder('folderB')
+          ]
+        });
 
-  const filename = await dropboxProvider.download(DROPBOX_FILE);
-  expect(filename).toBe(`${TARGET_PATH}/${DROPBOX_FILE_NAME}`);
-  done();
+      const files = await dropboxProvider.list();
+      expect(files).toHaveLength(3);
+      files.forEach((file) => expect(file).toBeInstanceOf(DropboxMediaFile));
+      expect(files).toMatchObject([{ remoteFile: 'a.jpg' }, { remoteFile: 'b.jpg' }, { remoteFile: 'c.jpg' }]);
+    });
+  });
 });
diff --git a/lib/provider/dropbox/dropbox-service.js b/lib/provider/dropbox/dropbox-service.js
@@ -36,6 +36,10 @@ class DropboxService {
     return `${target}/${data.name}`;
   }
 
+  async delete(file) {
+    return await this.#dropboxClient.filesDelete({ path: file.path_lower });
+  }
+
   async hash(filename) {
     const hasher = new DropboxContentHasher();
     const stream = fs.createReadStream(filename);

diff --git a/lib/provider/dropbox/dropbox-service.test.js b/lib/provider/dropbox/dropbox-service.test.js
@@ -27,52 +27,64 @@ afterEach(() => {
   jest.restoreAllMocks();
 });
 
-test('DropboxService should download file', async (done) => {
-  dropboxClient.filesDownload.mockResolvedValue({
-    name: DROPBOX_FILE_NAME,
-    client_modified: '2020-04-05T22:00:00-0500',
-    fileBinary: DROPBOX_FILE_BINARY
-  });
+describe('DropboxService', () => {
+  describe('download', () => {
+    test('file is downloaded successfully', async () => {
+      dropboxClient.filesDownload.mockResolvedValue({
+        name: DROPBOX_FILE_NAME,
+        client_modified: '2020-04-05T22:00:00-0500',
+        fileBinary: DROPBOX_FILE_BINARY
+      });
 
-  const expectedFilename = `${TARGET_PATH}/${DROPBOX_FILE_NAME}`;
-  const fileDate = new Date('2020-04-05T22:00:00-0500');
+      const expectedFilename = `${TARGET_PATH}/${DROPBOX_FILE_NAME}`;
+      const fileDate = new Date('2020-04-05T22:00:00-0500');
 
-  const filename = await dropboxService.download(`${SOURCE_PATH}/${DROPBOX_FILE_NAME}`, TARGET_PATH);
+      const filename = await dropboxService.download(`${SOURCE_PATH}/${DROPBOX_FILE_NAME}`, TARGET_PATH);
 
-  expect(filename).toBe(expectedFilename);
-  expect(fs.promises.mkdir).toHaveBeenCalledWith(TARGET_PATH, { recursive: true });
-  expect(fs.promises.writeFile).toHaveBeenCalledWith(expectedFilename, DROPBOX_FILE_BINARY, 'binary');
-  expect(fs.promises.utimes).toHaveBeenCalledWith(expectedFilename, fileDate, fileDate);
-  done();
-});
+      expect(filename).toBe(expectedFilename);
+      expect(fs.promises.mkdir).toHaveBeenCalledWith(TARGET_PATH, { recursive: true });
+      expect(fs.promises.writeFile).toHaveBeenCalledWith(expectedFilename, DROPBOX_FILE_BINARY, 'binary');
+      expect(fs.promises.utimes).toHaveBeenCalledWith(expectedFilename, fileDate, fileDate);
+    });
+  });
 
-test('DropboxService lists files', async (done) => {
-  dropboxClient.filesListFolder.mockResolvedValue({
-    entries: [
-      {
-        path_lower: DROPBOX_FILE_NAME,
-        content_hash: DROPBOX_FILE_HASH
-      }
-    ]
+  describe('downloadAndVerify', () => {
+    test.todo('file is downloaded and verified successfully');
   });
 
-  const response = await dropboxService.listFiles();
-  expect(response).not.toBeNull();
-  expect(response.entries.length).toBe(1);
-  expect(response.entries[0].path_lower).toBe(DROPBOX_FILE_NAME);
-  expect(response.entries[0].content_hash).toBe(DROPBOX_FILE_HASH);
-  done();
-});
+  describe('listFiles', () => {
+    test('lists files all files', async () => {
+      dropboxClient.filesListFolder.mockResolvedValue({
+        entries: [
+          {
+            path_lower: DROPBOX_FILE_NAME,
+            content_hash: DROPBOX_FILE_HASH
+          }
+        ]
+      });
+
+      const response = await dropboxService.listFiles();
+      expect(response).not.toBeNull();
+      expect(response.entries).toHaveLength(1);
+      expect(response.entries).toMatchObject([{ path_lower: DROPBOX_FILE_NAME, content_hash: DROPBOX_FILE_HASH }]);
+    });
+  });
+
+  describe('delete', () => {
+    test.todo('file is deleted');
+  });
 
-test('DropboxService hash file', async (done) => {
-  const stream = new Stream.Readable({
-    read() {
-      this.push(DROPBOX_FILE_BINARY);
-      this.push(null);
-    }
+  describe('hash', () => {
+    test('creates hash from file', async () => {
+      const stream = new Stream.Readable({
+        read() {
+          this.push(DROPBOX_FILE_BINARY);
+          this.push(null);
+        }
+      });
+      when(fs.createReadStream).calledWith(DROPBOX_FILE_NAME).mockReturnValue(stream);
+      const contentHash = await dropboxService.hash(DROPBOX_FILE_NAME);
+      expect(contentHash).toBe(DROPBOX_FILE_HASH);
+    });
   });
-  when(fs.createReadStream).calledWith(DROPBOX_FILE_NAME).mockReturnValue(stream);
-  const contentHash = await dropboxService.hash(DROPBOX_FILE_NAME);
-  expect(contentHash).toBe(DROPBOX_FILE_HASH);
-  done();
 });
diff --git a/lib/support/file-namer.js b/lib/support/file-namer.js