From 7904e7bb0a755810e32d02109da4245be7e5961e Mon Sep 17 00:00:00 2001 From: David Zerulla Date: Fri, 26 Feb 2016 10:37:36 +0100 Subject: [PATCH] api: basic file serving * Adds file serving tests. (addresses #15) * Replaces bucket `location_id` parameter with `location_name`. * Adds basic serialize function. * Adds objects test fixture. Signed-off-by: David Zerulla --- invenio_files_rest/views.py | 52 +++++++++++++----- tests/conftest.py | 22 ++++++-- tests/test_views.py | 104 +++++++++++++++++++++++++----------- 3 files changed, 130 insertions(+), 48 deletions(-) diff --git a/invenio_files_rest/views.py b/invenio_files_rest/views.py index 5d519d71..40d76d7a 100644 --- a/invenio_files_rest/views.py +++ b/invenio_files_rest/views.py @@ -26,7 +26,7 @@ from __future__ import absolute_import, print_function -from flask import Blueprint, abort, current_app, request +from flask import Blueprint, abort, current_app, request, url_for from invenio_db import db from invenio_rest import ContentNegotiatedMethodView from sqlalchemy.exc import SQLAlchemyError @@ -54,10 +54,9 @@ def __init__(self, serializers=None, *args, **kwargs): **kwargs ) self.post_args = { - 'location_id': fields.Int( + 'location_name': fields.String( missing=None, - location='json', - validate=lambda val: val >= 0 + location='json' ) } @@ -106,7 +105,13 @@ def get(self, **kwargs): """ bucket_list = [] for bucket in Bucket.all(): - bucket_list.append(bucket.serialize()) + # TODO: Implement serializer + bucket_list.append({ + 'size': bucket.size, + 'url': url_for("invenio_files_rest.bucket_api", + bucket_id=bucket.id, _external=True), + 'uuid': str(bucket.id) + }) # FIXME: how to avoid returning a dict with key 'json' return {'json': bucket_list} @@ -127,11 +132,11 @@ def post(self, **kwargs): Host: localhost:5000 { - "location_id": 1 + "location_name": "storage_one" } :reqheader Content-Type: application/json - :json body: A `location_id` can be passed (as an integer). If none + :json body: A `location_name` can be passed (as an string). If none is passed, a random active location will be used. **Responses**: @@ -156,11 +161,12 @@ def post(self, **kwargs): """ args = parser.parse(self.post_args, request) try: - if args['location_id']: - location = Location.get(args['location_id']) + if args['location_name']: + # TODO: Check why query is used directly. + location = Location.get_by_name(args['location_name']) else: # Get one of the active locations - location = Location.all().first() + location = Location.get_default() if not location: abort(400, 'Invalid location.') bucket = Bucket( @@ -176,7 +182,14 @@ def post(self, **kwargs): current_app.logger.exception('Failed to create bucket.') abort(500, 'Failed to create bucket.') - return {'json': bucket.serialize()} + # TODO: Implement serializer + return {'json': + {'size': bucket.size, + 'url': url_for("invenio_files_rest.bucket_api", + bucket_id=bucket.id, _external=True), + 'uuid': str(bucket.id) + } + } class BucketResource(ContentNegotiatedMethodView): @@ -244,13 +257,23 @@ def get(self, bucket_id, **kwargs): :statuscode 403: access denied :statuscode 404: page not found """ + # TODO: Implement serializer + def serialize(bucket): + return {'size': bucket.file.size, + 'checksum': bucket.file.checksum, + 'url': url_for('invenio_files_rest.object_api', + bucket_id=bucket.bucket_id, + key=bucket.key, + _external=True), + 'uuid': str(bucket.file.id)} + args = parser.parse(self.get_args, request) if bucket_id and Bucket.get(bucket_id): object_list = [] for obj in ObjectVersion.get_by_bucket( bucket_id, versions=args.get('versions', False) ).all(): - object_list.append(obj.serialize()) + object_list.append(serialize(obj)) return {'json': object_list} abort(404, 'The specified bucket does not exist or has been deleted.') @@ -409,14 +432,15 @@ def get(self, bucket_id, key, version_id=None, **kwargs): :resheader Content-Type: application/json :statuscode 200: no error :statuscode 403: access denied - :statuscode 404: page not found + :statuscode 404: Object does not exist """ # TODO: Check access permission on bucket. # TODO: Support partial range requests. + # TODO: Exception if file is not found (deleted by hand or accident) obj = ObjectVersion.get(bucket_id, key, version_id=version_id) if obj is None: abort(404, 'Object does not exist.') - return obj.storage.send_file() + return obj.send_file() @use_kwargs(put_args) def put(self, bucket_id, key, content_length=None, content_md5=None): diff --git a/tests/conftest.py b/tests/conftest.py index e1ac0596..c0ae9065 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of Invenio. -# Copyright (C) 2015 CERN. +# Copyright (C) 2015, 2016 CERN. # # Invenio is free software; you can redistribute it # and/or modify it under the terms of the GNU General Public License as @@ -30,6 +30,7 @@ import os import shutil import tempfile +from os.path import dirname, join import pytest from flask import Flask @@ -39,7 +40,7 @@ from sqlalchemy_utils.functions import create_database, database_exists from invenio_files_rest import InvenioFilesREST -from invenio_files_rest.models import Location +from invenio_files_rest.models import Bucket, Location, ObjectVersion from invenio_files_rest.views import blueprint @@ -75,7 +76,7 @@ def db(app): @pytest.yield_fixture() -def dummy_location(request, db): +def dummy_location(db): """File system location.""" tmppath = tempfile.mkdtemp() @@ -90,3 +91,18 @@ def dummy_location(request, db): yield loc shutil.rmtree(tmppath) + + +@pytest.yield_fixture() +def objects(dummy_location): + """File system location.""" + srcroot = dirname(dirname(__file__)) + + # Bucket 1 + b1 = Bucket.create(dummy_location) + objects = [] + for f in ['README.rst', 'LICENSE']: + with open(join(srcroot, f), 'rb') as fp: + objects.append(ObjectVersion.create(b1, f, stream=fp)) + + yield objects diff --git a/tests/test_views.py b/tests/test_views.py index 8e598301..4fd0c426 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # This file is part of Invenio. -# Copyright (C) 2015 CERN. +# Copyright (C) 2015, 2016 CERN. # # Invenio is free software; you can redistribute it # and/or modify it under the terms of the GNU General Public License as @@ -27,38 +27,39 @@ from __future__ import absolute_import, print_function -import uuid +from hashlib import md5 from flask import json -# def test_get_buckets(app, db, dummy_location): -# """Test get buckets.""" -# with app.test_client() as client: -# resp = client.get( -# '/files', -# headers={'Content-Type': 'application/json', 'Accept': '*/*'} -# ) -# assert resp.status_code == 200 -# # With location_id -# resp = client.post( -# '/files', -# data=json.dumps({'location_id': dummy_location.id}), -# headers={'Content-Type': 'application/json', 'Accept': '*/*'} -# ) -# assert resp.status_code == 200 - - -# def test_post_bucket(app, db): -# """Test post a bucket.""" -# with app.test_client() as client: -# resp = client.post( -# '/files', -# headers={'Content-Type': 'application/json', 'Accept': '*/*'} -# ) -# assert resp.status_code == 200 -# data = json.loads(resp.data) -# assert 'url' in data +def test_get_buckets(app, dummy_location): + """Test get buckets.""" + with app.test_client() as client: + resp = client.get( + '/files', + headers={'Content-Type': 'application/json', 'Accept': '*/*'} + ) + assert resp.status_code == 200 + + # With location_name + resp = client.post( + '/files', + data=json.dumps({'location_name': dummy_location.name}), + headers={'Content-Type': 'application/json', 'Accept': '*/*'} + ) + assert resp.status_code == 200 + + +def test_post_bucket(app, dummy_location): + """Test post a bucket.""" + with app.test_client() as client: + resp = client.post( + '/files', + headers={'Content-Type': 'application/json', 'Accept': '*/*'} + ) + assert resp.status_code == 200 + data = json.loads(resp.data) + assert 'url' in data # def test_head_bucket(app, db): @@ -87,7 +88,7 @@ # # Create bucket # resp = client.post( # '/files', -# data=json.dumps({'location_id': dummy_location.id}), +# data=json.dumps({'location_name': dummy_location.name}), # headers={'Content-Type': 'application/json', 'Accept': '*/*'} # ) # assert resp.status_code == 200 @@ -134,6 +135,47 @@ # ) # assert resp.status_code == 404 +# def test_get_object_list(app, dummy_objects): + + +def test_get_object_get(app, objects): + """Test object download""" + with app.test_client() as client: + for obj in objects: + resp = client.get( + "/files/{0}/{1}".format(obj.bucket_id, obj.key), + headers={'Content-Type': 'application/json', 'Accept': '*/*'} + ) + assert resp.status_code == 200 + + # Check md5 + md5_local = "md5:{}".format(md5(open(obj.file.uri[7:], 'rb') + .read()).hexdigest()) + assert resp.content_md5 == md5_local + # Check etag + assert resp.get_etag()[0] == md5_local + + +def test_get_object_get_404(app, objects): + """Test object download 404 error""" + with app.test_client() as client: + for obj in objects: + resp = client.get( + "/files/{0}/{1}".format(obj.bucket_id, obj.key + "Missing"), + headers={'Content-Type': 'application/json', 'Accept': '*/*'} + ) + assert resp.status_code == 404 + + +# def test_get_object_get_access_denied_403(app, objects): +# """Test object download 403 access denied""" +# with app.test_client() as client: +# for obj in objects: +# resp = client.get( +# "/files/{}/{}".format(obj.bucket_id, obj.key), +# headers={'Content-Type': 'application/json', 'Accept': '*/*'} +# ) +# assert resp.status_code == 403 # def test_get_objects(app, db): # """Test get all objects in a bucket.""" @@ -226,7 +268,7 @@ # # Create bucket # resp = client.post( # '/files', -# data=json.dumps({'location_id': dummy_location.id}), +# data=json.dumps({'location_name': dummy_location.name}), # headers={'Content-Type': 'application/json', # 'Accept': '*/*'} # )