Skip to content

Commit

Permalink
initial upload idea
Browse files Browse the repository at this point in the history
- fixes #1566
  • Loading branch information
casperdcl committed Nov 17, 2019
1 parent 38c2100 commit 908495d
Showing 1 changed file with 14 additions and 3 deletions.
17 changes: 14 additions & 3 deletions dvc/remote/gs.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
import logging
from datetime import timedelta
from functools import wraps
import io

from funcy import cached_property

from dvc.config import Config
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.scheme import Schemes
from dvc.utils.compat import FileNotFoundError # skipcq: PYL-W0622
Expand Down Expand Up @@ -46,9 +48,18 @@ def wrapper(*args, **kwargs):


@dynamic_chunk_size
def _upload_to_bucket(bucket, from_file, to_info, **kwargs):
blob = bucket.blob(to_info.path, **kwargs)
blob.upload_from_filename(from_file)
def _upload_to_bucket(bucket, from_file, to_info, chunk_size=None, **kwargs):
blob = bucket.blob(to_info.path, chunk_size=chunk_size, **kwargs)
with Tqdm() as pbar:
with io.open(from_file, mode="rb", buffering=chunk_size or -1) as fd:
raw_read = fd.read
def read(self, size=chunk_size):
res = raw_read(size)
if res:
pbar.update(len(res))
return res
fd.read = read
blob.upload_from_file(fd)


class RemoteGS(RemoteBASE):
Expand Down

0 comments on commit 908495d

Please sign in to comment.