REST api - generic get file method

pirus/api_v1/handlers.py

@@ -99,19 +99,73 @@ def get_db(self, request):
         return rest_success([f for f in os.listdir(DATABASES_DIR) if os.path.isfile(os.path.join(DATABASES_DIR, f))])
 
 
-
+ 
 
 class FileHandler:
     def __init__(self):
         pass
 
     def get(self, request):
-        return rest_success([i for i in list_files()])
+        # 1- retrieve query parameters
+        r_range  = request.match_info.get('range', "0-" + str(RANGE_DEFAULT))
+        r_fields = request.match_info.get('fields', None)
+        r_order  = request.match_info.get('order', None)
+        r_sort   = request.match_info.get('sort', None)
+        r_filter = request.match_info.get('filter', None)
+
+        # 2- fields to extract
+        fields = PirusFile.public_fields
+        if r_fields is not None:
+            fields = []
+            for f in r_fields.split(','):
+                f = f.strip().lower()
+                if f in PirusFile.public_fields:
+                    fields.append(f)
+        if len(fields) == 0:
+            return rest_error("No valid fields provided : " + request.match_info.get('fields'))
+
+        # 3- Build json query for mongoengine
+        query = {}
+        if r_filter is not None:
+            query = {}
+            for k in fields:
+                query.update({k : {'$regex': r_filter}})
+
+        # 4- Order
+        order = ['-create_date', "name"]
+        if r_sort is not None and r_order is not None:
+            r_sort = r_sort.split(',')
+            r_order = r_order.split(',')
+            if len(r_sort) == len(r_order):
+                order = []
+                for i in range(0, len(r_sort)):
+                    f = r_sort[i].strip().lower()
+                    if f in PirusFile.public_fields:
+                        if r_order[i] == "desc":
+                            f = "-" + f
+                        order.append(f)
+        order = tuple(order)
+
+        # 5- limit
+        r_range = r_range.split("-")
+        offset=0
+        limit=RANGE_DEFAULT
+        try:
+            offset = int(r_range[0])
+            limit = int(r_range[1])
+        except:
+            return rest_error("No valid range provided : " + request.match_info.get('range') )
+
+        # 6- Return result of the query !
+        return [p.export_client_data(fields) for p in PirusFile.objects(__raw__=query).order_by(*order)[offset:limit]]
+
+
+
 
     async def upload_simple(self, request):
-        file_name = str(uuid.uuid4())
-        file_path = os.path.join(FILES_DIR, file_name)
-        plog.info('I: Start file uploading : ' + file_path)
+        name = str(uuid.uuid4())
+        path = os.path.join(FILES_DIR, name)
+        plog.info('I: Start file uploading : ' + path)
         # 1- Retrieve file from post request
         data = await request.post()
         uploadFile = data['uploadFile']
@@ -128,27 +182,27 @@ async def upload_simple(self, request):
                     tags.append(i2)
         # 2- save file on the server 
         try:
-            with open(file_path, 'bw+') as f:
+            with open(path, 'bw+') as f:
                 f.write(uploadFile.file.read())
         except:
             # TODO : manage error
             raise PirusException("XXXX", "Bad pirus pipeline format : Manifest file corrupted.")
-        plog.info('I: File uploading done : ' + file_path)
+        plog.info('I: File uploading done : ' + path)
         # 3- save file on the database
         pirusfile = PirusFile()
         pirusfile.import_data({
-                "file_name"    : uploadFile.filename,
-                "file_type"    : os.path.splitext(uploadFile.filename)[1][1:].strip().lower(),
-                "file_path"    : file_path,
-                "file_size"    : humansize(os.path.getsize(file_path)),
+                "name"    : uploadFile.filename,
+                "type"    : os.path.splitext(uploadFile.filename)[1][1:].strip().lower(),
+                "path"    : path,
+                "size"    : humansize(os.path.getsize(path)),
                 "status"       : "TMP", # DOWNLOADING, TMP, OK
                 "create_date"  : str(datetime.datetime.now().timestamp()),
-                "md5sum"       : md5(file_path),
+                "md5sum"       : md5(path),
                 "tags"         : tags,
                 "comments"     : comments
             })
         pirusfile.save()
-        plog.info('I: File ' + file_name + ' (' + pirusfile.file_size + ') available at ' + file_path)
+        plog.info('I: File ' + name + ' (' + pirusfile.size + ') available at ' + path)
         return rest_success(pirusfile.export_client_data())
 
 
@@ -159,27 +213,27 @@ def upload_resumable(self, request):
     def delete(self, request):
         return rest_success({})
 
-    def get_file_details(self, request):
-        file_id = request.match_info.get('file_id', -1)
-        if file_id == -1:
-            return rest_error("Unknow file id " + str(file_id))
-        return rest_success(PirusFile.objects.get(pk=file_id).export_client_data())
+    def get_details(self, request):
+        id = request.match_info.get('id', -1)
+        if id == -1:
+            return rest_error("Unknow file id " + str(id))
+        return rest_success(PirusFile.objects.get(pk=id).export_client_data())
 
 
     async def dl_file(self, request):        
         # 1- Retrieve request parameters
-        file_id = request.match_info.get('file_id', -1)
-        if file_id == -1:
+        id = request.match_info.get('id', -1)
+        if id == -1:
             return rest_error("No file id provided")
-        pirus_file = PirusFile.from_id(file_id)
+        pirus_file = PirusFile.from_id(id)
         if pirus_file == None:
-            return rest_error("File with id " + str(file_id) + "doesn't exits.")
+            return rest_error("File with id " + str(id) + "doesn't exits.")
         file = None
-        if os.path.isfile(pirus_file.file_path):
-            with open(pirus_file.file_path, 'br') as content_file:
+        if os.path.isfile(pirus_file.path):
+            with open(pirus_file.path, 'br') as content_file:
                 file = content_file.read()
         return web.Response(
-            headers=MultiDict({'Content-Disposition': 'Attachment; filename='+pirus_file.file_name}),
+            headers=MultiDict({'Content-Disposition': 'Attachment; filename='+pirus_file.name}),
             body=file
         )
 
@@ -379,7 +433,7 @@ def get_plog_tail(self, request):
 
     def get_files(self, request):
         run_id  = request.match_info.get('run_id',  -1)
-        file_id = request.match_info.get('file_id', -1)
+        id = request.match_info.get('id', -1)
         return self.download_file(run_id, "outputs/io.json")
 
     def get_file(self, request):

pirus/api_v1/model.py

@@ -22,10 +22,12 @@
 
 
 class PirusFile(Document):
-    file_name    = StringField(required=True)
-    file_type    = StringField()
-    file_path    = StringField()
-    file_size    = StringField() 
+    public_fields = ["name", "type", "size", "status", "comments", "runs", "create_date", "tags", "md5sum", "url", "id"]
+
+    name         = StringField(required=True)
+    type         = StringField()
+    path         = StringField()
+    size         = StringField() 
     status       = StringField() # DL, TMP, OK
     comments     = StringField()    
     runs         = ListField(StringField())
@@ -34,14 +36,14 @@ class PirusFile(Document):
     md5sum       = StringField()
 
     def __str__(self):
-        return "<InputFile " + self.file_name + " (" + self.file_size + ") : " + self.file_path + ">"
+        return self.name + " (" + self.size + ") : " + self.path
 
     def export_server_data(self):
         return {
-            "file_name"    : self.file_name,
-            "file_type"    : self.file_type,
-            "file_path"    : self.file_path,
-            "file_size"    : self.file_size,
+            "name"         : self.name,
+            "type"         : self.type,
+            "path"         : self.path,
+            "size"         : self.size,
             "status"       : self.status,
             "comments"     : self.comments,
             "runs"         : self.runs,
@@ -51,27 +53,27 @@ def export_server_data(self):
             "id": str(self.id)
         }
 
-    def export_client_data(self):
-        return {
-            "file_name"    : self.file_name,
-            "file_type"    : self.file_type,
-            "file_size"    : self.file_size,
-            "status"       : self.status,
-            "comments"     : self.comments,
-            "runs"         : self.runs,
-            "create_date"  : self.create_date,
-            "tags"         : self.tags,
-            "md5sum"       : self.md5sum,
-            "url"          : "http://" + HOSTNAME + "/dl/f/" + str(self.id),
-            "id": str(self.id)
-        }
+    def export_client_data(self, fields=None):
+        result = {}
+        if fields is None:
+            fields = PirusFile.public_fields
+
+        for k in fields:
+            if k == "id":
+                result.update({"id" : str(self.id)})
+            elif k == "url":
+                result.update({"id" : "http://" + HOSTNAME + "/dl/f/" + str(self.id)})
+            else:
+                result.update({k : eval("self."+k)})
+
+        return result
 
     def import_data(self, data):
         try:
-            self.file_name     = data['file_name']
-            self.file_type     = data["file_type"]
-            self.file_path     = data['file_path']
-            self.file_size     = data["file_size"]
+            self.name          = data['name']
+            self.type          = data["type"]
+            self.path          = data['path']
+            self.size          = data["size"]
             self.status        = data["status"]
             self.create_date   = data["create_date"]
             self.md5sum        = data['md5sum']
@@ -86,10 +88,10 @@ def import_data(self, data):
         return self
 
     @staticmethod
-    def from_id(ifile_id):
-        if not ObjectId.is_valid(ifile_id):
+    def from_id(iid):
+        if not ObjectId.is_valid(iid):
             return None;
-        file = PirusFile.objects.get(pk=ifile_id)
+        file = PirusFile.objects.get(pk=iid)
         return file
 
 
@@ -231,26 +233,26 @@ def install(ppackage_name, ppackage_path, ppackage_file):
 
         # 3- Extract pirus technicals files from the tar file
         try:
-            ffile_src = os.path.join("rootfs",metadata['form'][1:] if metadata['form'][0]=="/" else metadata['form'])
-            xdir = [info for info in tar.getmembers() if info.name == ffile_src]
+            fsrc = os.path.join("rootfs",metadata['form'][1:] if metadata['form'][0]=="/" else metadata['form'])
+            xdir = [info for info in tar.getmembers() if info.name == fsrc]
             file = tar.extractfile(member=xdir[0])
-            ffile_src = os.path.join(ppackage_path, ffile_src)
-            ffile_dst = os.path.join(ppackage_path, "form.json")
-            with open(ffile_dst, 'bw+') as f:
+            fsrc = os.path.join(ppackage_path, fsrc)
+            fdst = os.path.join(ppackage_path, "form.json")
+            with open(fdst, 'bw+') as f:
                 f.write(file.read())
 
-            lfile_src = PIPELINE_DEFAULT_ICON_PATH
-            lfile_dst = os.path.join(ppackage_path, "icon.png")
+            lsrc = PIPELINE_DEFAULT_ICON_PATH
+            ldst = os.path.join(ppackage_path, "icon.png")
             if "icon" in metadata.keys():
-                lfile_src = os.path.join("rootfs",metadata['icon'][1:] if metadata['icon'][0]=="/" else metadata['icon'])
-                xdir = [info for info in tar.getmembers() if info.name == lfile_src]
+                lsrc = os.path.join("rootfs",metadata['icon'][1:] if metadata['icon'][0]=="/" else metadata['icon'])
+                xdir = [info for info in tar.getmembers() if info.name == lsrc]
                 file = tar.extractfile(member=xdir[0])
-                lfile_src = os.path.join(ppackage_path, lfile_src)
-                lfile_dst = os.path.join(ppackage_path, os.path.basename(metadata['icon']))
-                with open(lfile_dst, 'bw+') as f:
+                lsrc = os.path.join(ppackage_path, lsrc)
+                ldst = os.path.join(ppackage_path, os.path.basename(metadata['icon']))
+                with open(ldst, 'bw+') as f:
                     f.write(file.read())
             else:
-                shutil.copyfile(lfile_src, lfile_dst)
+                shutil.copyfile(lsrc, ldst)
         except:
             # TODO : manage error + remove package file
             plog.info('E:    [FAILLED] Extraction of ' + ppackage_file)
@@ -266,8 +268,8 @@ def install(ppackage_name, ppackage_path, ppackage_file):
             "lpath"     : metadata["logs"],
             "run"       : metadata["run"],
             "dpath"     : metadata["databases"],
-            "ffile"     : ffile_dst,
-            "lfile"     : lfile_dst,
+            "ffile"     : fdst,
+            "lfile"     : ldst,
             "lxd_alias" : "pirus-pipe-" + ppackage_name
         })
 
@@ -311,7 +313,7 @@ def install(ppackage_name, ppackage_path, ppackage_file):
 
         # 7- Clean directory
         try:
-            keep = [ppackage_file, ffile_dst, lfile_dst]
+            keep = [ppackage_file, fdst, ldst]
             for f in os.listdir(ppackage_path):
                 fullpath = os.path.join(ppackage_path, f)
                 if fullpath not in keep:

pirus/api_v1/routes.py

@@ -60,7 +60,7 @@
 app.router.add_route('PATCH',  "/v1/bigfile", fileHdl.upload_resumable)
 app.router.add_route('DELETE', "/v1/file/{file_id}", fileHdl.delete)
 app.router.add_route('PUT',    "/v1/file/{file_id}", fileHdl.edit_infos)
-app.router.add_route('GET',    "/v1/file/{file_id}", fileHdl.get_file_details)
+app.router.add_route('GET',    "/v1/file/{file_id}", fileHdl.get_details)
 app.router.add_route('GET',    "/v1/dl/f/{file_id}", fileHdl.dl_file)
 #app.router.add_route('GET',    "/v1/dl/p/{pipe_id}/{filename}", fileHdl.dl_pipe_file)
 #app.router.add_route('GET',    "/v1/dl/r/{run_id}/{filename}", fileHdl.dl_run_file)

pirus/config.py

@@ -9,6 +9,11 @@
 HOSTNAME       = HOST + ":" + PORT + "/" + VERSION
 
 
+RANGE_DEFAULT = 20
+RANGE_MAX     = 1000
+
+
+
 # FILESYSTEM
 FILES_DIR     = "/tmp/pirus_" + VERSION + "/files"
 TEMP_DIR      = "/tmp/pirus_" + VERSION + "/downloads"

pirus/templates/home.html

@@ -94,7 +94,7 @@ <h1 class="content-subhead"><i class="fa fa-file-o" aria-hidden="true"></i> File
 
                     {% for f in files %}
                     <tr>
-                        <td>{{ f["file_name"] }}<br/><span class="details">{{ f["id"] }} - {{ f["file_size"] }}</span></td>
+                        <td>{{ f["name"] }}<br/><span class="details">{{ f["id"] }} - {{ f["size"] }}</span></td>
                         <td style="vertical-align: middle;">{{ f["tags"] }}</td>
                         <td style="vertical-align: middle;">{{ f["status"] }}</td>
                         <td style="vertical-align: middle;">
@@ -361,7 +361,7 @@ <h4 class="modal-title" id="myModalLabel"><i class="fa fa-play" aria-hidden="tru
                     data = jsonFile["data"];
                     for (var i=0; i<data.length; i++)
                     {
-                        json_files[i] = data[i]["file_name"] + " [" + data[i]["id"] + "]"
+                        json_files[i] = data[i]["name"] + " [" + data[i]["id"] + "]"
                     }
                 })
                 json_final = json_final.replace(/"__PIRUS_INPUT_FILES__"/g, JSON.stringify(json_files))