Skip to content
This repository has been archived by the owner on Nov 3, 2023. It is now read-only.

Commit

Permalink
Browse files Browse the repository at this point in the history
fix download images issue (#1361)
  • Loading branch information
klshuster committed Jan 17, 2019
1 parent ea73790 commit 802275b
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 6 deletions.
2 changes: 1 addition & 1 deletion parlai/tasks/image_chat/build.py
Expand Up @@ -23,4 +23,4 @@ def build(opt):
build_data.mark_done(dpath, version)

if not build_data.built(image_path, version) and not opt.get('yfcc_path'):
download_images(opt)
download_images(opt, task='image_chat')
13 changes: 8 additions & 5 deletions parlai/tasks/personality_captions/download_images.py
Expand Up @@ -11,9 +11,9 @@
import parlai.core.build_data as build_data


def download_images(opt):
dpath = os.path.join(opt['datapath'], 'personality_captions')
image_path = os.path.join(dpath, 'images')
def download_images(opt, task='personality_captions'):
dpath = os.path.join(opt['datapath'], task)
image_path = os.path.join(opt['datapath'], 'yfcc_images')
version = '1.0'
response = input(
'Please confirm that you have obtained permission '
Expand All @@ -34,14 +34,17 @@ def download_images(opt):
'command line argument.')
image_prefix = 'https://multimedia-commons.s3-us-west-2.amazonaws.com/data/images'
hashes = []
for dt in ['train', 'val', 'test']:
dts = ['train', 'val', 'test']
if task == 'image_chat':
dts[1] = 'valid'
for dt in dts:
with open(os.path.join(dpath, '{}.json'.format(dt))) as f:
data = json.load(f)
hashes += [d['image_hash'] for d in data]
os.makedirs(image_path, exist_ok=True)

print('[downloading images to {}]'.format(image_path))
for i, (p_hash) in enumerate(tqdm.tqdm(hashes, unit='img')):
for _, (p_hash) in enumerate(tqdm.tqdm(hashes, unit='img')):
image_url = '{}/{}/{}/{}.jpg'.format(
image_prefix,
p_hash[:3],
Expand Down

0 comments on commit 802275b

Please sign in to comment.