fix some import and update core file download management (in progress)

pirus/api_rest/handlers.py

@@ -8,7 +8,6 @@
 import aiohttp
 
 import aiohttp_jinja2
-import jinja2
 import tarfile
 import datetime
 import time
@@ -20,8 +19,8 @@
 from urllib.parse import parse_qsl
 
 
-
-from core import pirus
+from config import *
+from core import *
 
 
 

pirus/api_rest/routes.py

@@ -1,9 +1,12 @@
 #!env/python3
 # coding: utf-8
 
+import aiohttp_jinja2
+import jinja2
+from aiohttp import web
 
 from core import pirus
-from api_rest.handlers import WebsiteHandler, FileHandler, PipelineHandler, RunHandler, WebsocketHandler
+from api_rest.handlers import *
 
 
 

pirus/application.py

@@ -6,7 +6,6 @@
 
 
 # Pirus package
-from config import *
 from api_rest import *
 
 

pirus/core/core.py

@@ -2,12 +2,14 @@
 # coding: utf-8
 
 import os
+import shutil
 import json
 import tarfile
 import datetime
 import time
 import uuid
 import subprocess
+import requests
 
 # from config import *
 from core.framework import *
@@ -19,6 +21,10 @@
 
 
 
+
+
+
+
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 # CORE OBJECT
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
@@ -63,7 +69,7 @@ def init(self):
              - check that config parameters are consistency
              - check that 
         """
-
+        pass
 
 
 
@@ -118,7 +124,7 @@ def get_from_ids(self, file_ids, sublvl=0):
 
 
 
-    def register(self, filename, path, metadata={}):
+    def upload_init(self, filename, path, metadata={}):
         """ 
             Create an entry for the file in the database and return the id of the file in pirus
             This method shall be used to init a resumable upload of a file 
@@ -141,6 +147,49 @@ def register(self, filename, path, metadata={}):
 
 
 
+    def upload_chunk(self, file_id, chunk_data, chunk_size, offset):
+        pass
+
+
+    def upload_finish(self, file_id, checksum, checksum_type="md5"):
+        pass
+
+
+    async def from_download(self, url, metadata={}):
+        """ Download a file from url and create a new Pirus file. """
+        name = str(uuid.uuid4())
+        filepath = os.path.join(FILES_DIR, name)
+        # get request and write file
+        with open(filepath, "bw+") as file:
+            try :
+                response = await requests.get(url)
+            except Exception as err:
+                raise PirusException("Error occured when trying to download a file from the provided url : " + str(url) + ". " + str(err))
+            file.write(response.content)
+        # save file on the database
+        pirusfile = pirus.files.register(name, filepath, metadata)
+        return rest_success(pirusfile)
+
+
+
+    def from_local(self, path, move=False, metadata={}):
+        """ Copy or move a local file on server and create a new Pirus file. Of course the source file shall have good access rights. """
+        name = str(uuid.uuid4())
+        filepath = os.path.join(FILES_DIR, name)
+        # get request and write file
+        try:
+            if move:
+                os.rename(path, filepath)
+            else:
+                shutil.copyfile(path, filepath)
+        except Exception as err:
+            raise PirusException("Error occured when trying to copy/move the file from the provided path : " + str(path) + ". " + str(err))
+        # save file on the database
+        pirusfile = pirus.files.register(name, filepath, metadata)
+        return rest_success(pirusfile)
+
+
+
     def edit(self, file_id, json_data):
         # TODO : implement PUT to edit file metadata (and remove the obsolete  "simple post" replaced by TUS upload )
         pass

pirus/core/pirus_worker.py

@@ -98,25 +98,6 @@ def start_run(self, run_id):
 
     self.notify_url = run.notify_url
     print(self.notify_url)
-    # # Init logs
-    # print(os.path.join(lpath, "pirus.log"))
-    # setup_logger('pirus_worker', os.path.join(lpath, "pirus_worker.log"))
-    # setup_logger('run_out', os.path.join(lpath, "out.log"))
-    # setup_logger('run_err', os.path.join(lpath, "err.log"))
-    # wlog = logging.getLogger('pirus_worker')
-    # # olog = logging.getLogger('run_out')
-    # # elog = logging.getLogger('run_err')
-
-    # wlog.info('INIT    | Pirus worker initialisation : ')
-    # wlog.info('INIT    |  - LXD alias : ' + pipeline.lxd_alias)
-    # wlog.info('INIT    |  - Run ID  : ' + self.run_private_id)
-    # wlog.info('INIT    | Directory created : ')
-    # wlog.info('INIT    |  - inputs  : ' + ipath)
-    # wlog.info('INIT    |  - outputs : ' + opath)
-    # wlog.info('INIT    |  - logs    : ' + lpath)
-    # wlog.info('INIT    |  - db      : ' + DATABASES_DIR)
-    # wlog.info('INIT    | Run config : ' + json.dumps(config))
-    # wlog.info('INIT    | Run inputs : ' + json.dumps(inputs))
 
 
     self.notify_status("WAITING")

pirus/tests/test_core_file.py

@@ -19,14 +19,28 @@
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 TU_PIRUS_FILE_PUBLIC_FIELDS = ["id", "name", "type", "size", "status", "upload_offset", "comments", "runs", "create_date", "tags", "md5sum", "url", "upload_url", "source"]
 
+TU_FAKE_FILE_1 = {
+    "name" : "TestFile 1.bin",
+    "type" : "bin",
+    "path" : os.path.join(FILES_DIR, "781b90ef-85d6-48bf-a09f-f4b5d8788bb7"),
+    "size" : 6858788138,
+    "upload_offset" : 6858788138,
+    "status" : "CHECKED",
+    "comments" : "Test file n°1",
+    "runs" : [],
+    "tags" : ["Test unit"],
+    "url" : "http://url",
+    "upload_url" : "http://url",
+    "source" : {
+        "type" : "upload"
+    }
+}
 
 
 
 class TestCoreFile(unittest.TestCase):
     """ Test case for pirus core file's features. """
 
-
-
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
     # PREPARATION
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@@ -62,15 +76,17 @@ def tearDownClass(self):
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
     def test_public_fields(self):
-        """ Test new file registration """
+        """ Check that public fileds describes in the model are same that in TU. Otherwise TU (and maybe doc/wiki shall be updated) """
         self.assertEqual(pirus.files.public_fields(), TU_PIRUS_FILE_PUBLIC_FIELDS)
 
-    def test_total(self):
-        """ Test new file registration """
-        self.assertEqual(1, 1)
 
     def test_get_generic(self):
-        """ Test new file registration """
+        """ Check that generic filter request is working """
+        # 1- Insert fake files entries
+        for i in range(0.100):
+            os.mknod(os.path.join())
+            pirus.files.register()
+
         # fields=None, query=None, order=None, offset=None, limit=None, sublvl=0):
         # """
         #     Generic method to get files metadata according to provided filtering options