From abc61ca5aaba2e99e10970b8aaff99b9618648b2 Mon Sep 17 00:00:00 2001 From: VertexC Date: Wed, 19 Jun 2019 01:35:51 -0700 Subject: [PATCH] :add contianer-based service upload --- install/install.sh | 3 ++ mod_config/controllers.py | 75 +++++++++++++++++++++++---------- mod_config/forms.py | 66 +++++++++++++++++------------ mod_config/models.py | 2 +- pipot/services/ServiceLoader.py | 29 ++++++++++++- 5 files changed, 122 insertions(+), 53 deletions(-) diff --git a/install/install.sh b/install/install.sh index ebcb475..4d5bf75 100755 --- a/install/install.sh +++ b/install/install.sh @@ -109,6 +109,9 @@ echo "We need some information for the admin account" read -e -p "Admin username: " -i "admin" admin_name read -e -p "Admin email: " admin_email read -e -p "Admin password: " admin_password +while [ ${#admin_password} == 0 ]; do + read -e -p "Invalid Admin password(cannot be empty), pleasee retry: " admin_password +done echo "Creating admin account: " python "${dir}/init_db.py" "${config_db_uri}" "${admin_name}" "${admin_email}" "${admin_password}" echo "" diff --git a/mod_config/controllers.py b/mod_config/controllers.py index 21c451d..3cf8598 100644 --- a/mod_config/controllers.py +++ b/mod_config/controllers.py @@ -3,6 +3,7 @@ import subprocess import threading import shutil +import zipfile from flask import Blueprint, g, request, send_file, jsonify, abort, \ url_for, redirect @@ -278,6 +279,20 @@ def data_processing_ajax(action): return jsonify(result) +def verify_and_import_module(temp_path, final_path, form, is_container=False): + if is_container: + instance = ServiceLoader.load_from_container(temp_path) + else: + instance = ServiceLoader.load_from_file(temp_path) + # Auto-generate tables + instance.get_used_table_names() + # Move + os.rename(temp_path, final_path) + service = Service(instance.__class__.__name__, + form.description.data) + g.db.add(service) + g.db.commit() + @mod_config.route('/services', methods=['GET', 'POST']) @login_required @check_access_rights() @@ -290,28 +305,42 @@ def services(): file = request.files[form.file.name] if file: filename = secure_filename(file.filename) - temp_path = os.path.join('./pipot/services/temp', filename) - final_path = os.path.join('./pipot/services', filename) - if not os.path.isfile(final_path): - file.save(temp_path) - # Import and verify module - try: - instance = ServiceLoader.load_from_file(temp_path) - # Auto-generate tables - instance.get_used_table_names() - # Move - os.rename(temp_path, final_path) - service = Service(instance.__class__.__name__, - form.description.data) - g.db.add(service) - g.db.commit() - # Reset form, all ok - form = NewServiceForm(None) - except ServiceLoader.ServiceLoaderException as e: - # Remove file - os.remove(temp_path) - # Pass error to user - form.errors['file'] = [e.value] + basename = filename.split('.')[0] + temp_dir = os.path.join('./pipot/services/temp', basename) + final_dir = os.path.join('./pipot/services', basename) + if not os.path.isdir(final_dir): + if zipfile.is_zipfile(file): + zip_file = zipfile.ZipFile(file) + ret = zip_file.testzip() + if ret: + form.errors['container'] = ['Corrupt container'] + else: + zip_file.extractall('./pipot/services/temp') + try: + verify_and_import_module(temp_dir, final_dir, form, is_container=True) + # Reset form, all ok + form = NewServiceForm(None) + except ServiceLoader.ServiceLoaderException as e: + os.remove(temp_dir) + form.errors['container'] = [e.value] + else: + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + os.mkdir(temp_dir) + temp_file = os.path.join(temp_dir, filename) + # create the __init__.py for module import + file.save(temp_file) + open(os.path.join(temp_dir, '__init__.py'), 'w') + # Import and verify module + try: + verify_and_import_module(temp_dir, final_dir, form, is_container=False) + # Reset form, all ok + form = NewServiceForm(None) + except ServiceLoader.ServiceLoaderException as e: + # Remove file + os.remove(temp_dir) + # Pass error to user + form.errors['file'] = [e.value] else: form.errors['file'] = ['Service already exists.'] return { @@ -338,7 +367,7 @@ def services_ajax(action): g.db.delete(service) # Delete file try: - os.remove(service.get_file()) + shutil.rmtree(service.get_file()) # Finalize service delete g.db.commit() result['status'] = 'success' diff --git a/mod_config/forms.py b/mod_config/forms.py index 72699c9..41c4a4b 100644 --- a/mod_config/forms.py +++ b/mod_config/forms.py @@ -1,4 +1,5 @@ import re +from enum import Enum from flask_wtf import Form from wtforms import SubmitField, FileField, TextAreaField, HiddenField, \ SelectField, StringField, IntegerField @@ -7,44 +8,53 @@ from mod_config.models import Service, Notification -def is_python(file_name): +class FileType(Enum): + PYTHONFILE = 1 + CONTAINER = 2 + + +def is_python_or_container(file_name): # Check if it ends on .py - is_py = re.compile("^[^/\\\]*\.py$") - if not is_py.match(file_name): - raise ValidationError('Provided file is not a python (.py) file!') + is_py = re.compile(r"^[^/\\]*.py$").match(file_name) + is_container = re.compile(r"^[^/\\]*.zip$").match(file_name) + if not is_py and not is_container: + raise ValidationError('Provided file is not a python (.py) file or a container (.zip)!') + return FileType.CONTAINER if is_container else FileType.PYTHONFILE def simple_service_file_validation(check_service=True): def validate_file(form, field): - is_python(field.data.filename) - # Name cannot be one of the files we already have - if field.data.filename in ['__init__py', 'IService.py', - 'ServiceLoader.py']: - raise ValidationError('Illegal file name!') - if check_service: - # Name cannot be registered already - service = Service.query.filter(Service.name == - field.data.filename).first() - if service is not None: - raise ValidationError('There is already an interface with ' - 'this name!') + file_type = is_python_or_container(field.data.filename) + if file_type is FileType.PYTHONFILE: + # Name cannot be one of the files we already have + if field.data.filename in ['__init__py', 'IService.py', + 'ServiceLoader.py']: + raise ValidationError('Illegal file name!') + if check_service: + # Name cannot be registered already + service = Service.query.filter(Service.name == + field.data.filename).first() + if service is not None: + raise ValidationError('There is already an interface with ' + 'this name!') return validate_file def simple_notification_file_validation(check_notification=True): def validate_file(form, field): - is_python(field.data.filename) - # Name cannot be one of the files we already have - if field.data.filename in ['__init__py', 'INotification.py', - 'NotificationLoader.py']: - raise ValidationError('Illegal file name!') - if check_notification: - # Name cannot be registered already - notification = Notification.query.filter( - Notification.name == field.data.filename).first() - if notification is not None: - raise ValidationError('There is already an interface with ' - 'this name!') + file_type = is_python_or_container(field.data.filename) + if file_type == FileType.PYTHONFILE: + # Name cannot be one of the files we already have + if field.data.filename in ['__init__py', 'INotification.py', + 'NotificationLoader.py']: + raise ValidationError('Illegal file name!') + if check_notification: + # Name cannot be registered already + notification = Notification.query.filter( + Notification.name == field.data.filename).first() + if notification is not None: + raise ValidationError('There is already an interface with ' + 'this name!') return validate_file diff --git a/mod_config/models.py b/mod_config/models.py index ddb99a7..422d1a0 100644 --- a/mod_config/models.py +++ b/mod_config/models.py @@ -26,7 +26,7 @@ def get_file(self, temp_folder=False): return os.path.join( './pipot/services', 'temp' if temp_folder else '', - self.name + '.py' + self.name ) diff --git a/pipot/services/ServiceLoader.py b/pipot/services/ServiceLoader.py index e1c30d0..bc7d336 100644 --- a/pipot/services/ServiceLoader.py +++ b/pipot/services/ServiceLoader.py @@ -17,6 +17,33 @@ def __str__(self): return repr(self.value) +def load_from_container(container_dir): + """Attempts to load the service from a folder with the same name + Required container format: + myService.zip + |-myService.py + |-__init__.py (will be created if doesn't exist) + |-requirement.txt (optional) + |-other file/folder + + :param container_dir: The path of container + :type container_dir: str + :return: A class instance of the loaded class. + :rtype: pipot.services.IService.IService + """ + mod_name = os.path.split(container_dir)[-1] + mod_file = os.path.join(container_dir, mod_name + '.py') + if not os.path.isfile(mod_file): + raise ServiceLoaderException('There is no service file %s.py found inside container' % mod_name) + else: + if os.path.isfile(os.path.join(container_dir, 'requirement.txt')): + pass + if not os.path.isfile(os.path.join(container_dir, '__init__.py')): + open(os.path.join(container_dir, '__init__.py'), 'w') + instance = load_from_file(mod_file) + return instance + + def load_from_file(file_name, temp_folder=True): """ Attempts to load a given class from a file with the same name in this @@ -33,7 +60,7 @@ def load_from_file(file_name, temp_folder=True): try: py_mod = importlib.import_module( - '.' + mod_name, + '.' + mod_name + '.' + mod_name, temp.__name__ if temp_folder else main.__name__) if hasattr(py_mod, mod_name):