Navigation Menu

Skip to content

Commit

Permalink
Linting and tidying up
Browse files Browse the repository at this point in the history
  • Loading branch information
sammyhass committed May 17, 2019
1 parent b8e975b commit 489d366
Show file tree
Hide file tree
Showing 7 changed files with 33 additions and 57 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Expand Up @@ -15,4 +15,4 @@ before_script:
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
script:
- true # add other tests here
- true # add other tests here
10 changes: 2 additions & 8 deletions API.py
@@ -1,16 +1,10 @@
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from googleapiclient.http import MediaIoBaseUpload
from googleapiclient.http import MediaIoBaseDownload
from googleapiclient.errors import HttpError
from httplib2 import Http
from oauth2client import file, client, tools
from mimetypes import MimeTypes
from tabulate import tabulate
from urllib.error import HTTPError

import time
import io

from FileParts import UDSFile

Expand Down Expand Up @@ -40,7 +34,7 @@ def reauth(self):
flow = client.flow_from_clientsecrets(
GoogleAPI.CLIENT_SECRET, SCOPES)
creds = tools.run_flow(flow, store)
except Exception as e:
except BaseException:
print("%s Make sure you've saved your OAuth credentials as %s" % (
GoogleAPI.ERROR_OUTPUT, GoogleAPI.CLIENT_SECRET))
print(
Expand All @@ -54,7 +48,7 @@ def get_base_folder(self):
"""Locate the base UDS folder
Returns:
file: the file
file: the file
"""
results = self.service.files().list(
Expand Down
2 changes: 0 additions & 2 deletions Encoder.py
@@ -1,5 +1,3 @@
import cryptography
import os
import base64


Expand Down
8 changes: 4 additions & 4 deletions FileParts.py
@@ -1,15 +1,15 @@
class UDSFile(object):
def __init__(self, name, base64, mime, size, encoded_size, id=None, parents=['root'], size_numeric=None, shared=False, sha256=''):
def __init__(self, name, base64, mime, size, encoded_size, id=None, parents=None, size_numeric=None, shared=False, sha256=None):
self.name = name
self.base64 = base64
self.mime = mime
self.size = size
self.size_numeric = size_numeric
self.encoded_size = encoded_size
self.parents = parents
self.parents = parents or ["root"]
self.id_ = id
self.shared = shared
self.sha256 = sha256
self.sha256 = sha256 or ''


class Chunk():
Expand All @@ -27,4 +27,4 @@ def __init__(self, path, part, max_size, media, parent):
if range_end > max_size:
self.range_end = max_size
else:
self.range_end = range_end
self.range_end = range_end
28 changes: 14 additions & 14 deletions Format.py
@@ -1,32 +1,32 @@
def format(bytes, numeric=False):
if bytes < 0:
def format(number_of_bytes, numeric=False):
if number_of_bytes < 0:
raise ValueError("!!! number_of_bytes can't be smaller than 0 !!!")

step = 1024.

bytes = float(bytes)
number_of_bytes = float(number_of_bytes)
unit = 'bytes'

if (bytes / step) >= 1:
bytes /= step
if (number_of_bytes / step) >= 1:
number_of_bytes /= step
unit = 'KB'

if (bytes / step) >= 1:
bytes /= step
if (number_of_bytes / step) >= 1:
number_of_bytes /= step
unit = 'MB'

if (bytes / step) >= 1:
bytes /= step
if (number_of_bytes / step) >= 1:
number_of_bytes /= step
unit = 'GB'

if (bytes / step) >= 1:
bytes /= step
if (number_of_bytes / step) >= 1:
number_of_bytes /= step
unit = 'TB'

precision = 1
bytes = round(bytes, precision)
number_of_bytes = round(number_of_bytes, precision)

if numeric:
return bytes
return number_of_bytes
else:
return str(bytes) + ' ' + unit
return str(number_of_bytes) + ' ' + unit
7 changes: 3 additions & 4 deletions README.md
Expand Up @@ -4,20 +4,20 @@ Store files in Google Docs without counting against your quota.

sorry @ the guys from google internal forums who are looking at this

### Features
## Features

- Upload files to Google Drive without using storage space
- Download any stored files to your computer

### Logic
## Logic

- Google Docs take up 0 bytes of quota in your Google Drive
- Split up binary files into Google Docs, with base64 encoded text
- Size of the encoded file is always larger than the original. Base64 encodes binary data to a ratio of about 4:3.
- A single google doc can store about a million characters. This is around 710KB of base64 encoded data.
- Some experiments with multithreading the uploads, but there was no significant performance increase.

### Authentication
## Authentication

1. Head to [Google's API page](https://developers.google.com/drive/api/v3/quickstart/python) and enable the Drive API
2. Download the configuration file as 'client_secret.json' to the UDS directory
Expand Down Expand Up @@ -213,5 +213,4 @@ python uds.py wipe argument
arguments: name_in_files, or wildcard "?" without quotes
```


**Compatible with Python 3.**
33 changes: 9 additions & 24 deletions uds.py
@@ -1,37 +1,24 @@
# -*- coding: utf-8 -*-
from __future__ import print_function
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
from googleapiclient.http import MediaIoBaseUpload
from googleapiclient.http import MediaIoBaseDownload
from httplib2 import Http
from oauth2client import file, client, tools
from mimetypes import MimeTypes
from tabulate import tabulate
from io import StringIO
from tqdm import tqdm

import sys
import base64
import math
import urllib.request
import ntpath
import mmap
import io
import os
import time
import shutil
import cryptography
import concurrent.futures
import Format
import argparse
import re
import json
import hashlib
from tqdm import tqdm

import Encoder
import FileParts

from FileParts import UDSFile, Chunk
from API import *

DOWNLOADS_FOLDER = "downloads"
Expand All @@ -52,7 +39,7 @@ def __init__(self):
def delete_file(self, id, name=None, mode_=None):
"""Deletes a given file
Use the Google Drive API to delete a file given its ID.
Use the Google Drive API to delete a file given its ID.
Args:
id (str): ID of the file
Expand All @@ -67,7 +54,7 @@ def delete_file(self, id, name=None, mode_=None):
else:
# If UDS commands are used, this displays the ID
print("Deleted %s" % id)
except:
except IOError:
if mode_ != "quiet":
print("%s File was not a UDS file" % GoogleAPI.ERROR_OUTPUT)
else:
Expand Down Expand Up @@ -104,7 +91,7 @@ def build_file(self, parent_id):
progress_bar_speed = tqdm(total=len(items) * CHUNK_READ_LENGTH_BYTES, unit_scale=1,
unit='B', dynamic_ncols=True, position=1)

for i, item in enumerate(items):
for _, item in enumerate(items):
encoded_part = self.download_part(item['id'])

# Decode
Expand Down Expand Up @@ -171,20 +158,19 @@ def do_chunked_upload(self, path):

root = self.api.get_base_folder()['id']

media = UDSFile(ntpath.basename(path), None, MimeTypes().guess_type(urllib.request.pathname2url(path))[0],
media = FileParts.UDSFile(ntpath.basename(path), None, MimeTypes().guess_type(urllib.request.pathname2url(path))[0],
Format.format(size), Format.format(encoded_size), parents=[root], size_numeric=size, sha256=file_hash)

parent = self.api.create_media_folder(media)

# Should be the same
no_chunks = math.ceil(size / CHUNK_READ_LENGTH_BYTES)
no_docs = math.ceil(encoded_size / MAX_DOC_LENGTH)

# Append all chunks to chunk list
chunk_list = list()
for i in range(no_docs):
chunk_list.append(
Chunk(path, i, size, media=media, parent=parent['id'])
FileParts.Chunk(path, i, size, media=media, parent=parent['id'])
)

total = 0
Expand All @@ -209,10 +195,9 @@ def do_chunked_upload(self, path):
progress_bar("Uploading %s at %sMB/s" %
(media.name, current_speed), total, size)"""

print("\n")
# Print new file output
table = [[media.name, media.size, media.encoded_size, parent['id']]]
print(tabulate(table, headers=[
print("\n" + tabulate(table, headers=[
'Name', 'Size', 'Encoded', 'ID', ]))

def convert_file(self, file_id):
Expand All @@ -226,7 +211,7 @@ def convert_file(self, file_id):
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
_, done = downloader.next_chunk()

print("Downloaded %s" % metadata['name'])
do_upload(path, service)
Expand Down

0 comments on commit 489d366

Please sign in to comment.