diff --git a/contrib/api/ephemeral.py b/contrib/api/ephemeral.py index c3cd808..9ddd74b 100644 --- a/contrib/api/ephemeral.py +++ b/contrib/api/ephemeral.py @@ -16,6 +16,7 @@ UPLOAD_URL = BASE_OPEN3DHUB + '/api/upload' UPLOAD_STATUS_URL = BASE_OPEN3DHUB + '/upload/processing/%TASK_ID%?api&username=%USERNAME%' API_MODELINFO_URL = BASE_OPEN3DHUB + '/api/modelinfo%(path)s' +DNS_URL = BASE_OPEN3DHUB + '/dns%(path)s' KEEPALIVE_URL = BASE_OPEN3DHUB + '/api/keepalive%(path)s?username=%(username)s&ttl=%(ttl)d' CONSUMER_KEY = 'lVk5aGvdzZpVP4oh34gE80qB6KW67LfJaBQBD9BB2ec=' @@ -38,18 +39,32 @@ def exitprint(resp, content): print "Error code: %s" % resp['status'] sys.exit(1) +def usage_exit(): + print >> sys.stderr, 'Usage: python ephemeral.py main_file [[subfile1name subfile1path] | subfile1] ...]' + sys.exit(1) + def main(): - if len(sys.argv) < 2 or len(sys.argv) % 2 != 0: - print >> sys.stderr, 'Usage: python ephemeral.py main_file [subfile1name subfile1path ...]' - sys.exit(1) + + if len(sys.argv) < 2: + usage_exit() opts, args = getopt.getopt(sys.argv[1:], ':') upload_files = [args[0]] main_filename = os.path.basename(args[0]) subfile_map = {} - for (name, path) in zip(args[1::2], args[2::2]): - subfile_map[name] = path + i = 1 + while i < len(args): + if os.path.isfile(args[i]): + upload_files.append(args[i]) + else: + fname = args[i] + if i+1 >= len(args): + usage_exit() + fpath = args[i+1] + subfile_map[fname] = fpath + i += 1 + i += 1 consumer = oauth2.Consumer(CONSUMER_KEY, CONSUMER_SECRET) access_token = oauth2.Token(ACCESS_KEY, ACCESS_SECRET) @@ -135,14 +150,14 @@ def main(): print half_updated = False - + subfile_names = set() while time.time() - start_time < TTL_TIME * 1.5: if not half_updated and time.time() - start_time > TTL_TIME / 2.0: print print 'Updating TTL value by 50%' - print + print 'Updating TTL for main file', uploaded_path toget = KEEPALIVE_URL % {'path': uploaded_path, 'username': USERNAME, 'ttl': TTL_TIME} @@ -153,8 +168,23 @@ def main(): if resp['status'] != '200': print 'Updating TTL failed' exitprint(resp, content) + + for subfile_name in subfile_names: + basename = subfile_name.split('/')[-2] + if basename not in subfile_map: + print 'Updating TTL for subfile', subfile_name - print + toget = KEEPALIVE_URL % {'path': subfile_name, + 'username': USERNAME, + 'ttl': TTL_TIME} + + client = oauth2.Client(consumer, token=access_token) + resp, content = client.request(toget, method='GET') + + if resp['status'] != '200': + print 'Updating subfile TTL failed' + exitprint(resp, content) + print 'Updating TTL by 50% success' print @@ -167,6 +197,20 @@ def main(): if 'fullpath' not in result or result['fullpath'] != uploaded_path[1:]: print 'Got wrong path in API request' exitprint(resp, content) + + + subfile_names = set(result['metadata']['types']['original']['subfiles']) + for subfile_name in subfile_names: + lookup_url = DNS_URL % {'path': subfile_name} + resp, content = h.request(lookup_url, "GET") + if resp['status'] != '200': + print 'Got wrong status code for subfile DNS lookup', subfile_name + exitprint(resp, content) + result = json.loads(content) + if 'Hash' not in result: + print 'Got wrong JSON result for subfile DNS lookup', subfile_name + exitprint(resp, content) + print 'Model still there... %d seconds left' % int(start_time + TTL_TIME * 1.5 - time.time()) time.sleep(5) @@ -177,13 +221,25 @@ def main(): print print 'Checking that model no longer exists' - print resp, content = h.request(json_info_url, "GET") if resp['status'] != '404': exitprint(resp, content) + print 'Main file correctly returned 404' + + for subfile_name in subfile_names: + basename = subfile_name.split('/')[-2] + lookup_url = DNS_URL % {'path': subfile_name} + resp, content = h.request(lookup_url, "GET") + correct_status = '200' + if basename not in subfile_map: + correct_status = '404' + if resp['status'] != correct_status: + print 'Got wrong status code for subfile DNS lookup', subfile_name + exitprint(resp, content) + print 'Subfile', subfile_name, 'correctly returned', correct_status print - print 'After TTL expired, API is now returning 404, the correct result. Done.' + print 'Everything checked out. Done.' print if __name__ == '__main__': diff --git a/sirikata-cdn/celery_tasks/import_upload.py b/sirikata-cdn/celery_tasks/import_upload.py index f78fca7..23d1836 100644 --- a/sirikata-cdn/celery_tasks/import_upload.py +++ b/sirikata-cdn/celery_tasks/import_upload.py @@ -193,18 +193,27 @@ def import_upload(main_rowkey, subfiles, selected_dae=None): return dae_zip_name @task -def place_upload(main_rowkey, subfiles, title, path, description, selected_dae=None, create_index=True, ephemeral_ttl=None): +def place_upload(main_rowkey, subfiles, title, path, description, selected_dae=None, create_index=True, ephemeral_ttl=None, ephemeral_subfiles=None): import_upload.update_state(state="LOADING") file_data = get_temp_file(main_rowkey) (zip, dae_zip_name, dae_data) = get_file_or_zip(file_data, selected_dae) + if ephemeral_subfiles is None: + ephemeral_subfiles = {} + if ephemeral_ttl is not None: - eph_subfile_metadata = get_multi_file_metadata(subfiles.values()) + eph_subfile_metadata = get_multi_file_metadata(ephemeral_subfiles.values()) eph_subfile_hashes = [m['hash'] for m in eph_subfile_metadata.itervalues()] eph_subfile_data = multi_get_hash(eph_subfile_hashes) + def eph_subfile_getter(name): - return eph_subfile_data[eph_subfile_metadata[subfiles[name]]['hash']]['data'] - (collada_obj, subfile_data, image_objs) = get_collada_and_images(zip, dae_zip_name, dae_data, subfiles, subfile_getter=eph_subfile_getter) + if name in ephemeral_subfiles: + return eph_subfile_data[eph_subfile_metadata[ephemeral_subfiles[name]]['hash']]['data'] + else: + return get_temp_file(subfiles[name]) + + combined_subfiles = dict(ephemeral_subfiles.items() + subfiles.items()) + (collada_obj, subfile_data, image_objs) = get_collada_and_images(zip, dae_zip_name, dae_data, combined_subfiles, subfile_getter=eph_subfile_getter) else: import_upload.update_state(state="CHECKING_COLLADA") (collada_obj, subfile_data, image_objs) = get_collada_and_images(zip, dae_zip_name, dae_data, subfiles) @@ -212,47 +221,48 @@ def eph_subfile_getter(name): import_upload.update_state(state="SAVING_ORIGINAL") try: new_version_num = get_new_version_from_path(path, file_type="collada") except cass.DatabaseError: raise DatabaseError() - - if ephemeral_ttl is not None: - subfile_names = subfiles.values() - else: - #Make sure image paths are just the base name - current_prefix = "original" - subfile_names = [] - image_names = [] - for img in collada_obj.images: - rel_path = img.path - base_name = posixpath.basename(img.path) - orig_base_name = base_name - - #strip out any character not allowed - base_name = re.sub('[^\w\-\.]', '', base_name) - - #make sure that referenced texture files are unique - while base_name in image_names: - dot = base_name.rfind('.') - ext = base_name[dot:] if dot != -1 else '' - before_ext = base_name[0:dot] if dot != -1 else base_name - base_name = "%s-x%s" % (before_ext, ext) - - if base_name != orig_base_name: - subfile_data[base_name] = subfile_data[orig_base_name] - del subfile_data[orig_base_name] - image_objs[base_name] = image_objs[orig_base_name] - del image_objs[orig_base_name] + + #Make sure image paths are just the base name + current_prefix = "original" + subfile_names = [] + image_names = [] + for img in collada_obj.images: + rel_path = img.path + base_name = posixpath.basename(img.path) + orig_base_name = base_name + + if base_name in ephemeral_subfiles: + subfile_names.append(ephemeral_subfiles[base_name]) + continue + + #strip out any character not allowed + base_name = re.sub('[^\w\-\.]', '', base_name) + + #make sure that referenced texture files are unique + while base_name in image_names: + dot = base_name.rfind('.') + ext = base_name[dot:] if dot != -1 else '' + before_ext = base_name[0:dot] if dot != -1 else base_name + base_name = "%s-x%s" % (before_ext, ext) - img.path = "./%s" % base_name - img.save() - img_hex_key = hashlib.sha256(subfile_data[base_name]).hexdigest() - try: save_file_data(img_hex_key, subfile_data[base_name], "image/%s" % image_objs[base_name].format.lower()) - except: raise DatabaseError() - img_path = "%s/%s/%s" % (path, current_prefix, base_name) - img_len = len(subfile_data[base_name]) - try: img_version_num = get_new_version_from_path(img_path, file_type="image") - except cass.DatabaseError: raise DatabaseError() - try: save_file_name(img_path, img_version_num, img_hex_key, img_len) - except cass.DatabaseError: raise DatabaseError() - subfile_names.append("%s/%s" % (img_path, img_version_num)) + if base_name != orig_base_name: + subfile_data[base_name] = subfile_data[orig_base_name] + del subfile_data[orig_base_name] + image_objs[base_name] = image_objs[orig_base_name] + del image_objs[orig_base_name] + + img.path = "./%s" % base_name + img.save() + img_hex_key = hashlib.sha256(subfile_data[base_name]).hexdigest() + try: save_file_data(img_hex_key, subfile_data[base_name], "image/%s" % image_objs[base_name].format.lower()) + except: raise DatabaseError() + img_path = "%s/%s/%s" % (path, current_prefix, base_name) + img_len = len(subfile_data[base_name]) + try: img_version_num = get_new_version_from_path(img_path, file_type="image") + except cass.DatabaseError: raise DatabaseError() + try: save_file_name(img_path, img_version_num, img_hex_key, img_len, ttl=ephemeral_ttl) + except cass.DatabaseError: raise DatabaseError() + subfile_names.append("%s/%s" % (img_path, img_version_num)) str_buffer = StringIO() collada_obj.write(str_buffer) diff --git a/sirikata-cdn/content/utils.py b/sirikata-cdn/content/utils.py index 000a5d7..2d33a6d 100644 --- a/sirikata-cdn/content/utils.py +++ b/sirikata-cdn/content/utils.py @@ -247,10 +247,10 @@ def list_file_keys(columns=None): for r in recs: yield r -def save_file_name(path, version_num, hash_key, length): +def save_file_name(path, version_num, hash_key, length, ttl=None): dict = {'hash': hash_key, 'size': length} col_val = json.dumps(dict) - insertRecord(NAMES, path, columns={version_num: col_val}) + insertRecord(NAMES, path, columns={version_num: col_val}, ttl=ttl) def save_version_type(path, version_num, hash_key, length, subfile_names, zip_key, type_id, title=None, description=None, create_index=True, ttl=None): try: diff --git a/sirikata-cdn/content/views.py b/sirikata-cdn/content/views.py index 16569fb..4b9ade6 100644 --- a/sirikata-cdn/content/views.py +++ b/sirikata-cdn/content/views.py @@ -435,9 +435,6 @@ def clean(self): if not ephemeral and subfiles: raise forms.ValidationError("Subfiles parameter is only valid when uploading ephemeral files") - if ephemeral and len(self.file_names) > 1: - raise forms.ValidationError("Only one file can be uploaded when uploading an ephemeral file") - return self.cleaned_data @csrf_exempt @@ -494,11 +491,15 @@ def api_upload(request): filename = main_filename if ephemeral: - task = place_upload.delay(main_rowkey, ephemeral_subfiles, title, path, - description, create_index=False, ephemeral_ttl=ttl_time) + create_index = False else: - task = place_upload.delay(main_rowkey, subfiles, title, path, - description, create_index=True) + ttl_time = None + create_index = True + ephemeral_subfiles = None + + task = place_upload.delay(main_rowkey, subfiles, title, path, + description, create_index=create_index, + ephemeral_ttl=ttl_time, ephemeral_subfiles=ephemeral_subfiles) save_upload_task(username=username, task_id=task.task_id, @@ -897,6 +898,8 @@ def dns(request, filename): base_path = "/".join(parts[:-2]) type_id = parts[-2] versions = get_versions('/' + base_path) + if versions is None: + return HttpResponseNotFound() version_num = str(max(map(int, versions))) else: base_path = "/".join(parts[:-3])