Skip to content
This repository has been archived by the owner on May 26, 2018. It is now read-only.

Commit

Permalink
resumable upload ! (core... need to clean hmi and have friendly feedb…
Browse files Browse the repository at this point in the history 
…ack)
  • Loading branch information
ikit committed Oct 27, 2016
1 parent 9b83b8a commit b1a5561
Show file tree
Hide file tree
Showing 8 changed files with 1,107 additions and 218 deletions.
62 changes: 48 additions & 14 deletions pirus/api_v1/handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from framework import *
from pirus_worker import run_pipeline
from api_v1.model import *
from api_v1.tus import tus_manager



Expand Down Expand Up @@ -114,7 +115,14 @@ def process_generic_get(query_string, allowed_fields):




def get_file_data(request):
id = request.match_info.get('file_id', -1)
if id == -1:
return rest_error("Unknow file id " + str(id))
pirus_file = PirusFile.from_id(id)
if pirus_file == None:
return rest_error("File with id " + str(id) + "doesn't exits.")
return pirus_file



Expand Down Expand Up @@ -169,7 +177,7 @@ def get(self, request):




# "Simple" upload (synchrone and not resumable)
async def upload_simple(self, request):
name = str(uuid.uuid4())
path = os.path.join(FILES_DIR, name)
Expand Down Expand Up @@ -199,24 +207,50 @@ async def upload_simple(self, request):
# 3- save file on the database
pirusfile = PirusFile()
pirusfile.import_data({
"name" : uploadFile.filename,
"type" : os.path.splitext(uploadFile.filename)[1][1:].strip().lower(),
"path" : path,
"size" : humansize(os.path.getsize(path)),
"status" : "TMP", # DOWNLOADING, TMP, OK
"create_date" : str(datetime.datetime.now().timestamp()),
"md5sum" : md5(path),
"tags" : tags,
"comments" : comments
"name" : uploadFile.filename,
"type" : os.path.splitext(uploadFile.filename)[1][1:].strip().lower(),
"path" : path,
"size" : int(os.path.getsize(path)),
"size_total" : int(os.path.getsize(path)),
"status" : "UPLOADED",
"create_date" : str(datetime.datetime.now().timestamp()),
"md5sum" : md5(path),
"tags" : tags,
"comments" : comments
})
pirusfile.save()
plog.info('I: File ' + name + ' (' + pirusfile.size + ') available at ' + path)
return rest_success(pirusfile.export_client_data())


def upload_resumable(self, request):
# do something else
return 'End of upload'



# Resumable download implement the TUS.IO protocol.
def tus_config(self, request):
print ("tus_config", request)
return tus_manager.options(request)

def tus_upload_init(self, request):
print ("tus_upload_init", request)
return tus_manager.creation(request)

def tus_upload_resume(self, request):
print ("tus_upload_resume", request)
return tus_manager.resume(request)

async def tus_upload_chunk(self, request):
print ("tus_upload_chunk", request)
result = await tus_manager.patch(request)
return result

def tus_upload_delete(self, request):
print ("tus_upload_delete", request)
return tus_manager.delete_file(request)





def delete(self, request):
return rest_success({})
Expand Down
76 changes: 48 additions & 28 deletions pirus/api_v1/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,20 @@


class PirusFile(Document):
public_fields = ["id", "name", "type", "size", "status", "comments", "runs", "create_date", "tags", "md5sum", "url"]

name = StringField(required=True)
type = StringField()
path = StringField()
size = StringField()
status = StringField() # DL, TMP, OK
comments = StringField()
runs = ListField(StringField())
create_date = StringField()
tags = ListField(StringField())
md5sum = StringField()
public_fields = ["id", "name", "type", "size", "size_total", "status", "comments", "runs", "create_date", "tags", "md5sum", "url"]

name = StringField(required=True)
type = StringField()
path = StringField()
size = IntField()
size_total = IntField()
upload_offset = IntField()
status = StringField() # UPLOADING, UPLOADED, PAUSE, ERROR, CHECKED
comments = StringField()
runs = ListField(StringField())
create_date = StringField()
tags = ListField(StringField())
md5sum = StringField()


def __str__(self):
Expand All @@ -42,19 +44,25 @@ def __str__(self):

def export_server_data(self):
return {
"name" : self.name,
"type" : self.type,
"path" : self.path,
"size" : self.size,
"status" : self.status,
"comments" : self.comments,
"runs" : self.runs,
"create_date" : self.create_date,
"tags" : self.tags,
"md5sum" : self.md5sum,
"id": str(self.id)
"name" : self.name,
"type" : self.type,
"path" : self.path,
"size" : self.size,
"size_total" : self.size_total,
"upload_offset" : self.upload_offset,
"status" : self.status,
"comments" : self.comments,
"runs" : self.runs,
"create_date" : self.create_date,
"tags" : self.tags,
"md5sum" : self.md5sum,
"id" : str(self.id)
}

def url(self):
return "http://" + HOSTNAME + "/dl/f/" + str(self.id)
def upload_url(self):
return "http://" + HOSTNAME + "/bigfile/" + str(self.id)

def export_client_data(self, fields=None):
result = {}
Expand All @@ -65,7 +73,7 @@ def export_client_data(self, fields=None):
if k == "id":
result.update({"id" : str(self.id)})
elif k == "url":
result.update({"url" : "http://" + HOSTNAME + "/dl/f/" + str(self.id)})
result.update({"url" : self.url()})
else:
result.update({k : eval("self."+k)})
return result
Expand All @@ -76,12 +84,17 @@ def import_data(self, data):
self.name = data['name']
self.type = data["type"]
self.path = data['path']
self.size = data["size"]
self.size = int(data["size"])
self.size_total = int(data["size_total"])
self.status = data["status"]
self.create_date = data["create_date"]
self.md5sum = data['md5sum']
if "runs" in data.keys():
self.runs = data["runs"]
if "upload_offset" in data.keys():
self.upload_offset = int(data["upload_offset"])
else:
self.upload_offset = 0
if "tags" in data.keys():
self.tags = data['tags']
if "comments" in data.keys():
Expand All @@ -92,12 +105,19 @@ def import_data(self, data):


@staticmethod
def from_id(iid):
if not ObjectId.is_valid(iid):
def from_id(id):
if not ObjectId.is_valid(id):
return None;
file = PirusFile.objects.get(pk=iid)
file = PirusFile.objects.get(pk=id)
return file

@staticmethod
def remove(id):
file = PirusFile.from_id(id)
if file != None:
shutil.rmtree(file.path)
file.delete()




Expand Down
12 changes: 7 additions & 5 deletions pirus/api_v1/routes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,18 @@

app.router.add_route('GET', "/v1/file", fileHdl.get)
app.router.add_route('POST', "/v1/file", fileHdl.upload_simple)
app.router.add_route('PATCH', "/v1/bigfile", fileHdl.upload_resumable)
app.router.add_route('DELETE', "/v1/file/{file_id}", fileHdl.delete)
app.router.add_route('PUT', "/v1/file/{file_id}", fileHdl.edit_infos)
app.router.add_route('GET', "/v1/file/{file_id}", fileHdl.get_details)
app.router.add_route('GET', "/v1/dl/f/{file_id}", fileHdl.dl_file)
#app.router.add_route('GET', "/v1/dl/p/{pipe_id}/{filename}", fileHdl.dl_pipe_file)
#app.router.add_route('GET', "/v1/dl/r/{run_id}/{filename}", fileHdl.dl_run_file)

#app.router.add_route('GET', "/v1/run/notify/{run_id}/p/{complete}", runHdl.up_progress)
#app.router.add_route('GET', "/v1/run/notify/{run_id}/s/{status}", runHdl.up_status)
#app.router.add_route('GET', "/v1/bigfile", fileHdl.tus_upload_init_get)
app.router.add_route('POST', "/v1/bigfile", fileHdl.tus_upload_init)
app.router.add_route('OPTIONS',"/v1/bigfile", fileHdl.tus_config)
app.router.add_route('HEAD', "/v1/bigfile/{file_id}", fileHdl.tus_upload_resume)
app.router.add_route('PATCH', "/v1/bigfile/{file_id}", fileHdl.tus_upload_chunk)
app.router.add_route('DELETE', "/v1/bigfile/{file_id}", fileHdl.tus_upload_delete)

app.router.add_route('POST', "/v1/run/notify/{run_id}", runHdl.up_data)

app.router.add_static('/assets', TEMPLATE_DIR)
Expand Down

0 comments on commit b1a5561

Please sign in to comment.