Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support more urls #16

Merged
merged 3 commits into from
Oct 23, 2018
Merged

Support more urls #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3644928
Add parse_url
wkentaro Oct 23, 2018
d64fccb
Update README.md
wkentaro Oct 23, 2018
bc91dd9
Use six.moves.urllib_parse for compatibility
wkentaro Oct 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ $ cat ip.json
$ # write stdout and pipe to extract
$ gdown https://drive.google.com/uc?id=0B9P1L--7Wd2vNm9zMTJWOGxobkU -O - --quiet | tar zxvf -
$ ls 20150428_collected_images/

$ # it can handle urls created from [Share] -> [Copy Url] on Google Drive
$ gdown 'https://drive.google.com/a/jsk.imi.i.u-tokyo.ac.jp/uc?id=0B_NiLAzvehC9R2stRmQyM3ZiVjQ'
```

### From Python
Expand Down
176 changes: 4 additions & 172 deletions gdown/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,177 +1,9 @@
#!/usr/bin/env python

from __future__ import print_function

import argparse
import os
import os.path as osp
import re
import shutil
import sys
import tempfile
# flake8: noqa

import pkg_resources
import requests
import six
import tqdm


dist = pkg_resources.get_distribution('gdown')
__author__ = 'Kentaro Wada <www.kentaro.wada@gmail.com>'
__version__ = dist.version


CHUNK_SIZE = 512 * 1024 # 512KB


def get_url_from_gdrive_confirmation(contents):
url = ''
for line in contents.splitlines():
m = re.search('href="(\/uc\?export=download[^"]+)', line)
if m:
url = 'https://docs.google.com' + m.groups()[0]
url = url.replace('&amp;', '&')
return url
m = re.search('confirm=([^;&]+)', line)
if m:
confirm = m.groups()[0]
url = re.sub(r'confirm=([^;&]+)', r'confirm='+confirm, url)
return url
m = re.search('"downloadUrl":"([^"]+)', line)
if m:
url = m.groups()[0]
url = url.replace('\\u003d', '=')
url = url.replace('\\u0026', '&')
return url


def is_google_drive_url(url):
m = re.match('^https?://drive.google.com/uc\?id=.*$', url)
return m is not None


def download(url, output, quiet):
url_origin = url
sess = requests.session()

is_gdrive = is_google_drive_url(url)

while True:
res = sess.get(url, stream=True)
if 'Content-Disposition' in res.headers:
# This is the file
break
if not is_gdrive:
break

# Need to redirect with confiramtion
url = get_url_from_gdrive_confirmation(res.text)

if url is None:
print('Permission denied: %s' % url_origin, file=sys.stderr)
print("Maybe you need to change permission over "
"'Anyone with the link'?", file=sys.stderr)
return
from .download import download

if output is None:
if is_gdrive:
m = re.search('filename="(.*)"',
res.headers['Content-Disposition'])
output = m.groups()[0]
else:
output = osp.basename(url)

output_is_path = isinstance(output, six.string_types)

if not quiet:
print('Downloading...', file=sys.stderr)
print('From:', url_origin, file=sys.stderr)
print('To:', osp.abspath(output) if output_is_path else output,
file=sys.stderr)

if output_is_path:
tmp_file = tempfile.mktemp(
suffix=tempfile.template,
prefix=osp.basename(output),
dir=osp.dirname(output),
)
f = open(tmp_file, 'wb')
else:
tmp_file = None
f = output

try:
total = res.headers.get('Content-Length')
if total is not None:
total = int(total)
if not quiet:
pbar = tqdm.tqdm(total=total, unit='B', unit_scale=True)
for chunk in res.iter_content(chunk_size=CHUNK_SIZE):
f.write(chunk)
if not quiet:
pbar.update(len(chunk))
if not quiet:
pbar.close()
if tmp_file:
f.close()
shutil.copy(tmp_file, output)
except IOError as e:
print(e, file=sys.stderr)
return
finally:
try:
if tmp_file:
os.remove(tmp_file)
except OSError:
pass

return output


class _ShowVersionAction(argparse.Action):

def __init__(self, *args, **kwargs):
kwargs['nargs'] = 0
self.version = kwargs.pop('version')
super(self.__class__, self).__init__(*args, **kwargs)

def __call__(self, parser, namespace, values, option_string=None):
print('gdown {ver} at {pos}'
.format(ver=self.version, pos=dist.location))
parser.exit()


def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-V', '--version', version=__version__,
action=_ShowVersionAction, help='display version')
parser.add_argument(
'url_or_id', help='url or file id (with --id) to download file from')
parser.add_argument('-O', '--output', help='output filename')
parser.add_argument('-q', '--quiet', action='store_true',
help='suppress standard output')
parser.add_argument('--id', action='store_true',
help='flag to specify file id instead of url')
args = parser.parse_args()

if args.output == '-':
if six.PY3:
args.output = sys.stdout.buffer
else:
args.output = sys.stdout

if args.id:
url = 'https://drive.google.com/uc?id={id}'.format(id=args.url_or_id)
else:
url = args.url_or_id

download(
url=url,
output=args.output,
quiet=args.quiet,
)


if __name__ == '__main__':
main()
__author__ = 'Kentaro Wada <www.kentaro.wada@gmail.com>'
__version__ = pkg_resources.get_distribution('gdown').version
66 changes: 66 additions & 0 deletions gdown/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
from __future__ import print_function

import argparse
import sys

import pkg_resources
import six

from .download import download


distribution = pkg_resources.get_distribution('gdown')


class _ShowVersionAction(argparse.Action):

def __init__(self, *args, **kwargs):
kwargs['nargs'] = 0
self.version = kwargs.pop('version')
super(self.__class__, self).__init__(*args, **kwargs)

def __call__(self, parser, namespace, values, option_string=None):
print('gdown {ver} at {pos}'
.format(ver=self.version, pos=distribution.location))
parser.exit()


def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'-V',
'--version',
version=distribution.version,
action=_ShowVersionAction,
help='display version',
)
parser.add_argument(
'url_or_id', help='url or file id (with --id) to download file from')
parser.add_argument('-O', '--output', help='output filename')
parser.add_argument('-q', '--quiet', action='store_true',
help='suppress standard output')
parser.add_argument('--id', action='store_true',
help='flag to specify file id instead of url')
args = parser.parse_args()

if args.output == '-':
if six.PY3:
args.output = sys.stdout.buffer
else:
args.output = sys.stdout

if args.id:
url = 'https://drive.google.com/uc?id={id}'.format(id=args.url_or_id)
else:
url = args.url_or_id

download(
url=url,
output=args.output,
quiet=args.quiet,
)


if __name__ == '__main__':
main()
120 changes: 120 additions & 0 deletions gdown/download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
from __future__ import print_function

import os
import os.path as osp
import re
import shutil
import sys
import tempfile

import requests
import six
import tqdm

from .parse_url import parse_url


CHUNK_SIZE = 512 * 1024 # 512KB


def get_url_from_gdrive_confirmation(contents):
url = ''
for line in contents.splitlines():
m = re.search('href="(\/uc\?export=download[^"]+)', line)
if m:
url = 'https://docs.google.com' + m.groups()[0]
url = url.replace('&amp;', '&')
return url
m = re.search('confirm=([^;&]+)', line)
if m:
confirm = m.groups()[0]
url = re.sub(
r'confirm=([^;&]+)',
r'confirm={}'.format(confirm),
url,
)
return url
m = re.search('"downloadUrl":"([^"]+)', line)
if m:
url = m.groups()[0]
url = url.replace('\\u003d', '=')
url = url.replace('\\u0026', '&')
return url


def download(url, output, quiet):
url_origin = url
sess = requests.session()

file_id, is_download_link = parse_url(url)

while True:
res = sess.get(url, stream=True)
if 'Content-Disposition' in res.headers:
# This is the file
break
if not (file_id and is_download_link):
break

# Need to redirect with confiramtion
url = get_url_from_gdrive_confirmation(res.text)

if url is None:
print('Permission denied: %s' % url_origin, file=sys.stderr)
print("Maybe you need to change permission over "
"'Anyone with the link'?", file=sys.stderr)
return

if output is None:
if file_id and is_download_link:
m = re.search('filename="(.*)"',
res.headers['Content-Disposition'])
output = m.groups()[0]
else:
output = osp.basename(url)

output_is_path = isinstance(output, six.string_types)

if not quiet:
print('Downloading...', file=sys.stderr)
print('From:', url_origin, file=sys.stderr)
print('To:', osp.abspath(output) if output_is_path else output,
file=sys.stderr)

if output_is_path:
tmp_file = tempfile.mktemp(
suffix=tempfile.template,
prefix=osp.basename(output),
dir=osp.dirname(output),
)
f = open(tmp_file, 'wb')
else:
tmp_file = None
f = output

try:
total = res.headers.get('Content-Length')
if total is not None:
total = int(total)
if not quiet:
pbar = tqdm.tqdm(total=total, unit='B', unit_scale=True)
for chunk in res.iter_content(chunk_size=CHUNK_SIZE):
f.write(chunk)
if not quiet:
pbar.update(len(chunk))
if not quiet:
pbar.close()
if tmp_file:
f.close()
shutil.copy(tmp_file, output)
except IOError as e:
print(e, file=sys.stderr)
return
finally:
try:
if tmp_file:
os.remove(tmp_file)
except OSError:
pass

return output
Loading