From be2d3ac8a8624e08dc066151985fac7ad35c7730 Mon Sep 17 00:00:00 2001 From: JE-Chen Date: Sun, 4 Jun 2023 16:39:55 +0800 Subject: [PATCH 1/4] refactor and add search all file refactor and add search all file --- .gitignore | 2 +- file_automation/__init__.py | 4 +++- .../remote/google_drive/driver_instance.py | 2 +- .../remote/google_drive/search/search_drive.py | 9 +++++++++ .../remote/google_drive/upload/upload_to_driver.py | 12 ++++++++++++ tests/unit_test/{ => local}/dir/dir_test.py | 0 .../{ => local}/dir/first_file_dir/test_file | 0 .../{ => local}/dir/second_file_dir/test_file.txt | 0 tests/unit_test/{ => local}/file/test_file.py | 0 tests/unit_test/{ => local}/zip/zip_test.py | 0 tests/unit_test/remote/google_drive/quick_test.py | 3 +++ 11 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 file_automation/remote/google_drive/search/search_drive.py create mode 100644 file_automation/remote/google_drive/upload/upload_to_driver.py rename tests/unit_test/{ => local}/dir/dir_test.py (100%) rename tests/unit_test/{ => local}/dir/first_file_dir/test_file (100%) rename tests/unit_test/{ => local}/dir/second_file_dir/test_file.txt (100%) rename tests/unit_test/{ => local}/file/test_file.py (100%) rename tests/unit_test/{ => local}/zip/zip_test.py (100%) create mode 100644 tests/unit_test/remote/google_drive/quick_test.py diff --git a/.gitignore b/.gitignore index e92480c..d635b5b 100644 --- a/.gitignore +++ b/.gitignore @@ -152,5 +152,5 @@ cython_debug/ .idea/ # Google Drive API token -file_automation/remote/google_drive/credentials.json +tests/unit_test/remote/google_drive/credentials.json token.json diff --git a/file_automation/__init__.py b/file_automation/__init__.py index ca13c94..e6017f9 100644 --- a/file_automation/__init__.py +++ b/file_automation/__init__.py @@ -4,9 +4,11 @@ from file_automation.local.zip.zip_process import zip_dir, zip_file, zip_info, zip_file_info, set_zip_password, \ read_zip_file, unzip_file, unzip_all +from file_automation.remote.google_drive.driver_instance import driver_instance + __all__ = [ "copy_file", "rename_file", "remove_file", "copy_all_file_to_dir", "copy_specify_extension_file", "copy_dir", "create_dir", "copy_specify_extension_file", "remove_dir_tree", "zip_dir", "zip_file", "zip_info", "zip_file_info", "set_zip_password", "unzip_file", "read_zip_file", - "unzip_all" + "unzip_all", "driver_instance", ] diff --git a/file_automation/remote/google_drive/driver_instance.py b/file_automation/remote/google_drive/driver_instance.py index 08f556c..4cf48bd 100644 --- a/file_automation/remote/google_drive/driver_instance.py +++ b/file_automation/remote/google_drive/driver_instance.py @@ -40,4 +40,4 @@ def __init__(self, token_path: str, credentials_path: str): print(f'An error occurred: {error}') -GoogleDrive(str(Path(Path.cwd(), "token.json")), str(Path(Path.cwd(), "credentials.json"))) +driver_instance = GoogleDrive(str(Path(Path.cwd(), "token.json")), str(Path(Path.cwd(), "credentials.json"))) diff --git a/file_automation/remote/google_drive/search/search_drive.py b/file_automation/remote/google_drive/search/search_drive.py new file mode 100644 index 0000000..3b2be76 --- /dev/null +++ b/file_automation/remote/google_drive/search/search_drive.py @@ -0,0 +1,9 @@ +from file_automation.remote.google_drive.driver_instance import driver_instance + + +def search_all_file(): + item = dict() + response = driver_instance.service.files().list().execute() + for file in response.get("files", []): + item.update({file.get("name"): file.get("id")}) + return item diff --git a/file_automation/remote/google_drive/upload/upload_to_driver.py b/file_automation/remote/google_drive/upload/upload_to_driver.py new file mode 100644 index 0000000..583b05c --- /dev/null +++ b/file_automation/remote/google_drive/upload/upload_to_driver.py @@ -0,0 +1,12 @@ +from pathlib import Path + +from file_automation.remote.google_drive.driver_instance import driver_instance + + +def upload_to_dir(file_path: str, upload_folder_id: str): + file_path = Path(file_path) + if driver_instance.service and file_path.is_file(): + file_metadata = { + 'name': file_path.name, + 'parents': [upload_folder_id] + } diff --git a/tests/unit_test/dir/dir_test.py b/tests/unit_test/local/dir/dir_test.py similarity index 100% rename from tests/unit_test/dir/dir_test.py rename to tests/unit_test/local/dir/dir_test.py diff --git a/tests/unit_test/dir/first_file_dir/test_file b/tests/unit_test/local/dir/first_file_dir/test_file similarity index 100% rename from tests/unit_test/dir/first_file_dir/test_file rename to tests/unit_test/local/dir/first_file_dir/test_file diff --git a/tests/unit_test/dir/second_file_dir/test_file.txt b/tests/unit_test/local/dir/second_file_dir/test_file.txt similarity index 100% rename from tests/unit_test/dir/second_file_dir/test_file.txt rename to tests/unit_test/local/dir/second_file_dir/test_file.txt diff --git a/tests/unit_test/file/test_file.py b/tests/unit_test/local/file/test_file.py similarity index 100% rename from tests/unit_test/file/test_file.py rename to tests/unit_test/local/file/test_file.py diff --git a/tests/unit_test/zip/zip_test.py b/tests/unit_test/local/zip/zip_test.py similarity index 100% rename from tests/unit_test/zip/zip_test.py rename to tests/unit_test/local/zip/zip_test.py diff --git a/tests/unit_test/remote/google_drive/quick_test.py b/tests/unit_test/remote/google_drive/quick_test.py new file mode 100644 index 0000000..09d23ca --- /dev/null +++ b/tests/unit_test/remote/google_drive/quick_test.py @@ -0,0 +1,3 @@ +from file_automation.remote.google_drive.search.search_drive import search_all_file + +print(search_all_file()) From 449858853598cd44a86f48616010ca282a540fb3 Mon Sep 17 00:00:00 2001 From: JE-Chen Date: Sun, 4 Jun 2023 16:51:51 +0800 Subject: [PATCH 2/4] Fix tests Fix tests --- .github/workflows/file_automation_dev_python3_10.yml | 6 +++--- .github/workflows/file_automation_dev_python3_11.yml | 6 +++--- .github/workflows/file_automation_dev_python3_8.yml | 6 +++--- .github/workflows/file_automation_dev_python3_9.yml | 6 +++--- .github/workflows/file_automation_stable_python3_10.yml | 6 +++--- .github/workflows/file_automation_stable_python3_11.yml | 6 +++--- .github/workflows/file_automation_stable_python3_8.yml | 6 +++--- .github/workflows/file_automation_stable_python3_9.yml | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/file_automation_dev_python3_10.yml b/.github/workflows/file_automation_dev_python3_10.yml index 49dbd9a..a283098 100644 --- a/.github/workflows/file_automation_dev_python3_10.yml +++ b/.github/workflows/file_automation_dev_python3_10.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r dev_requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py \ No newline at end of file + run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_dev_python3_11.yml b/.github/workflows/file_automation_dev_python3_11.yml index ca947ec..8bbfdef 100644 --- a/.github/workflows/file_automation_dev_python3_11.yml +++ b/.github/workflows/file_automation_dev_python3_11.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r dev_requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py \ No newline at end of file + run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_dev_python3_8.yml b/.github/workflows/file_automation_dev_python3_8.yml index af47494..890ff5f 100644 --- a/.github/workflows/file_automation_dev_python3_8.yml +++ b/.github/workflows/file_automation_dev_python3_8.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r dev_requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py \ No newline at end of file + run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_dev_python3_9.yml b/.github/workflows/file_automation_dev_python3_9.yml index e5c472f..5cde786 100644 --- a/.github/workflows/file_automation_dev_python3_9.yml +++ b/.github/workflows/file_automation_dev_python3_9.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r dev_requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py \ No newline at end of file + run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_stable_python3_10.yml b/.github/workflows/file_automation_stable_python3_10.yml index 1e3ec28..e7fae3a 100644 --- a/.github/workflows/file_automation_stable_python3_10.yml +++ b/.github/workflows/file_automation_stable_python3_10.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py \ No newline at end of file + run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_stable_python3_11.yml b/.github/workflows/file_automation_stable_python3_11.yml index 5ed6aee..2104171 100644 --- a/.github/workflows/file_automation_stable_python3_11.yml +++ b/.github/workflows/file_automation_stable_python3_11.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py \ No newline at end of file + run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_stable_python3_8.yml b/.github/workflows/file_automation_stable_python3_8.yml index 642fb8d..ac4fd06 100644 --- a/.github/workflows/file_automation_stable_python3_8.yml +++ b/.github/workflows/file_automation_stable_python3_8.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py + run: python ./tests/unit_test/local/zip/zip_test.py diff --git a/.github/workflows/file_automation_stable_python3_9.yml b/.github/workflows/file_automation_stable_python3_9.yml index 73133c8..f6135f7 100644 --- a/.github/workflows/file_automation_stable_python3_9.yml +++ b/.github/workflows/file_automation_stable_python3_9.yml @@ -24,8 +24,8 @@ jobs: python -m pip install --upgrade pip wheel pip install -r requirements.txt - name: Dir Module Test - run: python ./tests/unit_test/dir/dir_test.py + run: python ./tests/unit_test/local/dir/dir_test.py - name: File Module Test - run: python ./tests/unit_test/file/test_file.py + run: python ./tests/unit_test/local/file/test_file.py - name: Zip Module Test - run: python ./tests/unit_test/zip/zip_test.py + run: python ./tests/unit_test/local/zip/zip_test.py From dd6210f6342ec430c3af44a6c8f1bf06302d45ec Mon Sep 17 00:00:00 2001 From: JE-Chen Date: Sun, 4 Jun 2023 17:29:55 +0800 Subject: [PATCH 3/4] Add search file using mimetype Add search file using mimetype --- .../google_drive/search/search_drive.py | 18 +++++++++++++++ .../google_drive/upload/upload_to_driver.py | 23 +++++++++++++++---- .../remote/google_drive/quick_test.py | 4 ++-- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/file_automation/remote/google_drive/search/search_drive.py b/file_automation/remote/google_drive/search/search_drive.py index 3b2be76..5b52d87 100644 --- a/file_automation/remote/google_drive/search/search_drive.py +++ b/file_automation/remote/google_drive/search/search_drive.py @@ -7,3 +7,21 @@ def search_all_file(): for file in response.get("files", []): item.update({file.get("name"): file.get("id")}) return item + + +def search_file_mimetype(mime_type: str): + files = dict() + page_token = None + while True: + # pylint: disable=maybe-no-member + response = driver_instance.service.files().list( + q=f"mimeType='{mime_type}'", + spaces="drive", + fields="nextPageToken, files(id, name)", + pageToken=page_token).execute() + for file in response.get("files", []): + files.update({file.get("name"): file.get("id")}) + page_token = response.get('nextPageToken', None) + if page_token is None: + break + return files diff --git a/file_automation/remote/google_drive/upload/upload_to_driver.py b/file_automation/remote/google_drive/upload/upload_to_driver.py index 583b05c..b3917fa 100644 --- a/file_automation/remote/google_drive/upload/upload_to_driver.py +++ b/file_automation/remote/google_drive/upload/upload_to_driver.py @@ -1,12 +1,25 @@ from pathlib import Path +from googleapiclient.http import MediaFileUpload + from file_automation.remote.google_drive.driver_instance import driver_instance -def upload_to_dir(file_path: str, upload_folder_id: str): - file_path = Path(file_path) - if driver_instance.service and file_path.is_file(): +def upload_to_drive(file_name: str, file_path: str): + if Path(file_path).is_file(): file_metadata = { - 'name': file_path.name, - 'parents': [upload_folder_id] + "name": file_name, + "mimeType": "*/*" } + media = MediaFileUpload( + file_path, + mimetype="*/*", + resumable=True + ) + file_id = driver_instance.service.files().create( + body=file_metadata, + media_body=media, + fields="id" + ).execute() + return file_id + return False diff --git a/tests/unit_test/remote/google_drive/quick_test.py b/tests/unit_test/remote/google_drive/quick_test.py index 09d23ca..9a0affd 100644 --- a/tests/unit_test/remote/google_drive/quick_test.py +++ b/tests/unit_test/remote/google_drive/quick_test.py @@ -1,3 +1,3 @@ -from file_automation.remote.google_drive.search.search_drive import search_all_file +from file_automation.remote.google_drive.search.search_drive import search_file_mimetype -print(search_all_file()) +print(search_file_mimetype("*/*")) From 658e240d2280f5d2e35a7937bd339ba314823ded Mon Sep 17 00:00:00 2001 From: JE-Chen Date: Sun, 4 Jun 2023 18:23:44 +0800 Subject: [PATCH 4/4] Add download file with id Add download file with id --- .../google_drive/download/download_file.py | 23 +++++++++++++++++++ .../google_drive/search/search_drive.py | 1 - .../remote/google_drive/quick_test.py | 4 ++-- 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 file_automation/remote/google_drive/download/download_file.py diff --git a/file_automation/remote/google_drive/download/download_file.py b/file_automation/remote/google_drive/download/download_file.py new file mode 100644 index 0000000..2df1a49 --- /dev/null +++ b/file_automation/remote/google_drive/download/download_file.py @@ -0,0 +1,23 @@ +import io + +from googleapiclient.errors import HttpError +from googleapiclient.http import MediaIoBaseDownload + +from file_automation.remote.google_drive.driver_instance import driver_instance + + +def download_file(file_id: str, file_name: str): + try: + request = driver_instance.service.files().get_media(fileId=file_id) + file = io.BytesIO() + downloader = MediaIoBaseDownload(file, request) + done = False + while done is False: + status, done = downloader.next_chunk() + print(F'Download {int(status.progress() * 100)}.') + except HttpError as error: + print(F'An error occurred: {error}') + file = None + with open(file_name, "wb") as output_file: + output_file.write(file.getbuffer()) + return file diff --git a/file_automation/remote/google_drive/search/search_drive.py b/file_automation/remote/google_drive/search/search_drive.py index 5b52d87..4cac19b 100644 --- a/file_automation/remote/google_drive/search/search_drive.py +++ b/file_automation/remote/google_drive/search/search_drive.py @@ -16,7 +16,6 @@ def search_file_mimetype(mime_type: str): # pylint: disable=maybe-no-member response = driver_instance.service.files().list( q=f"mimeType='{mime_type}'", - spaces="drive", fields="nextPageToken, files(id, name)", pageToken=page_token).execute() for file in response.get("files", []): diff --git a/tests/unit_test/remote/google_drive/quick_test.py b/tests/unit_test/remote/google_drive/quick_test.py index 9a0affd..3980003 100644 --- a/tests/unit_test/remote/google_drive/quick_test.py +++ b/tests/unit_test/remote/google_drive/quick_test.py @@ -1,3 +1,3 @@ -from file_automation.remote.google_drive.search.search_drive import search_file_mimetype +from file_automation.remote.google_drive.download.download_file import download_file -print(search_file_mimetype("*/*")) +print(download_file("1s4cHS_ZSqMMikf8HV2vb1rYahbtoJvbC", "test.py"))