Skip to content

Commit

Permalink
add scheme change for service update
Browse files Browse the repository at this point in the history
  • Loading branch information
VertexC committed Aug 11, 2019
1 parent 53acaeb commit 0301178
Show file tree
Hide file tree
Showing 5 changed files with 215 additions and 38 deletions.
54 changes: 34 additions & 20 deletions mod_config/controllers.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,19 +283,18 @@ def data_processing_ajax(action):
return jsonify(result)


def verify_and_import_module(final_path, form, is_container=False, update=False):
def verify_and_import_module(final_path, form, is_container=False, re_load=True):
if is_container:
instance = ServiceLoader.load_from_container(final_path, temp_folder=False, update=update)
instance = ServiceLoader.load_from_container(final_path, temp_folder=False, re_load=re_load)
else:
instance = ServiceLoader.load_from_file(final_path, temp_folder=False, update=update)
if not update:
# Auto-generate tables
instance.get_used_table_names()
# Update database
service = Service(instance.__class__.__name__,
form.description.data)
g.db.add(service)
g.db.commit()
instance = ServiceLoader.load_from_file(final_path, temp_folder=False, re_load=re_load)
# Auto-generate tables
instance.get_used_table_names()
# Update database
service = Service(instance.__class__.__name__, form.description.data)
g.db.add(service)
g.db.commit()
return instance


@mod_config.route('/services', methods=['GET', 'POST'])
Expand All @@ -320,7 +319,7 @@ def services():
else:
zip_file.extractall('./pipot/services')
try:
verify_and_import_module(final_dir, form, is_container=True)
verify_and_import_module(final_dir, form, is_container=True, re_load=False)
# Reset form, all ok
form = NewServiceForm(None)
except ServiceLoader.ServiceLoaderException as e:
Expand All @@ -334,10 +333,15 @@ def services():
open(os.path.join(final_dir, '__init__.py'), 'w')
# Import and verify module
try:
verify_and_import_module(final_dir, form, is_container=False)
verify_and_import_module(final_dir, form, is_container=False, re_load=False)
# Reset form, all ok
form = NewServiceForm(None)
except ServiceLoader.ServiceLoaderException as e:
try:
del sys.modules['pipot.services.' + basename]
del sys.modules['pipot.services.' + basename + '.' + basename]
except KeyError:
pass
# Remove file
shutil.rmtree(final_dir)
# Pass error to user
Expand Down Expand Up @@ -411,22 +415,32 @@ def services_ajax(action):
final_dir = os.path.join('./pipot/services', basename)
temp_dir = os.path.join('./pipot/services', 'temp')
if file.filename == service.name + '.py':
# get the original class instance, remove tables from db and meta
old_instance = ServiceLoader.load_from_file(final_dir, temp_folder=False, re_load=False)
from database import Base, db_engine
for table_name, model in old_instance.get_used_table_names().items():
Base.metadata.drop_all(bind=db_engine, tables=[model.__table__])
Base.metadata.remove(model.__table__)
# move the original service to temp for backup
shutil.move(os.path.join(final_dir, filename),
os.path.join(temp_dir, filename))
shutil.move(os.path.join(final_dir),
os.path.join(temp_dir))
os.makedirs(final_dir)
open(os.path.join(final_dir, '__init__.py'), 'w')
file.save(os.path.join(final_dir, filename))
# Import and verify module
try:
verify_and_import_module(final_dir, form, is_container=False, update=True)
new_instance = ServiceLoader.load_from_file(final_dir, temp_folder=False, re_load=True)
# Reset form, all ok
form = NewServiceForm(None)
# remove the old service file
os.remove(os.path.join(temp_dir, filename))
shutil.rmtree(os.path.join(temp_dir, basename))
result['status'] = 'success'
except ServiceLoader.ServiceLoaderException as e:
# overwirte the updated service file with the old one
shutil.move(os.path.join(temp_dir, filename),
os.path.join(final_dir, filename))
# bring back the old service file
shutil.rmtree(final_dir)
shutil.move(os.path.join(temp_dir, basename),
os.path.join('./pipot/services'))
old_instance = ServiceLoader.load_from_file(final_dir, temp_folder=False, re_load=True)
# Pass error to user
form.errors['file'] = [e.value]
result['errors'] = form.errors
Expand Down
6 changes: 3 additions & 3 deletions mod_config/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def is_python_or_container(file_name):
return FileType.CONTAINER if is_container else FileType.PYTHONFILE


def simple_service_file_validation(check_service=True):
def simple_service_file_validation(check_service=False):
def validate_file(form, field):
field.data.filename = os.path.basename(field.data.filename)
file_type = is_python_or_container(field.data.filename)
Expand Down Expand Up @@ -63,7 +63,7 @@ def validate_file(form, field):
class NewServiceForm(Form):
file = FileField('Service file', [
DataRequired(message='No service file was provided.'),
simple_service_file_validation()
simple_service_file_validation(check_service=True)
])
description = TextAreaField('Service description', [
DataRequired(message='Service description cannot be empty.')],
Expand Down Expand Up @@ -93,7 +93,7 @@ class EditServiceForm(BaseServiceForm):
class UpdateServiceForm(BaseServiceForm):
file = FileField('Service file', [
DataRequired(message='no service file was provided.'),
simple_service_file_validation(False)
simple_service_file_validation(check_service=False)
])


Expand Down
15 changes: 7 additions & 8 deletions pipot/services/ServiceLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def __str__(self):
return repr(self.value)


def load_from_container(container_dir, temp_folder=False):
def load_from_container(container_dir, temp_folder=False, re_load=False):
"""Attempts to load the service from a folder with the same name
Required container format:
myService.zip
Expand All @@ -38,14 +38,15 @@ def load_from_container(container_dir, temp_folder=False):
raise ServiceLoaderException('There is no service file %s.py found inside container' % mod_name)
else:
if os.path.isfile(os.path.join(container_dir, 'requirement.txt')):
#TODO: ideally add check on pip install
pass
if not os.path.isfile(os.path.join(container_dir, '__init__.py')):
open(os.path.join(container_dir, '__init__.py'), 'w')
instance = load_from_file(mod_file, temp_folder=temp_folder)
instance = load_from_file(mod_file, temp_folder=temp_folder, re_load=re_load)
return instance


def load_from_file(file_name, temp_folder=True, update=False):
def load_from_file(file_name, temp_folder=True, re_load=False):
"""
Attempts to load a given class from a file with the same name in this
folder.
Expand All @@ -60,15 +61,13 @@ def load_from_file(file_name, temp_folder=True, update=False):
mod_name, file_ext = os.path.splitext(os.path.split(file_name)[-1])

try:
if update:
if re_load:
try:
del sys.modules['pipot.services.' + mod_name]
del sys.modules['pipot.services.' + mod_name + '.' + mod_name]
except KeyError:
pass
py_mod = importlib.import_module(
'.' + mod_name + '.' + mod_name,
temp.__name__ if temp_folder else main.__name__)

py_mod = importlib.import_module('pipot.services.' + mod_name + '.' + mod_name)
if hasattr(py_mod, mod_name):
class_inst = getattr(py_mod, mod_name)(None, None)
else:
Expand Down
127 changes: 127 additions & 0 deletions tests/testFiles/ModifiedService/TelnetService.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import datetime
from sqlalchemy import Column, String
from twisted.internet.protocol import Protocol, Factory

from pipot.services.IService import INetworkService, IModelIP


class ReportTelnet(IModelIP):
__tablename__ = 'report_telnet'

password = Column(String(100))
description = Column(String(100)) # add for sql-migration test

def __init__(self, description, deployment_id, ip, port, password, timestamp=None):
super(ReportTelnet, self).__init__(description, deployment_id, ip, port, timestamp)
self.description = description
self.password = password

def get_message_for_level(self, notification_level):
message = 'Telnet login attempt with password %s' % self.password
message += '\nPlease take action!' if notification_level == 2 else ''
return message


class SimpleTelnetProtocol(Protocol):
"""
Example Telnet Protocol
$ telnet localhost 8025
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
password:
password:
password:
% Bad passwords
Connection closed by foreign host.
"""
def __init__(self):
self.prompts = 0
self.buffer = ""

def connectionMade(self):
self.transport.write("\xff\xfb\x03\xff\xfb\x01password: ")
self.prompts += 1

def dataReceived(self, data):
"""
Received data is unbuffered so we buffer it for telnet.
"""
self.buffer += data

i = self.buffer.find("\x01")
if i >= 0:
self.buffer = self.buffer[i+1:]
return

if self.buffer.find("\x00") >= 0:
password = self.buffer.strip("\r\n\x00")
log_data = {"password": password}
self.factory.log(log_data, transport=self.transport)
self.buffer = ""

if self.prompts < 3:
self.transport.write("\r\npassword: ")
self.prompts += 1
else:
self.transport.write("\r\n% Bad passwords\r\n")
self.transport.loseConnection()


class TelnetService(INetworkService, Factory):
protocol = SimpleTelnetProtocol

def __init__(self, collector, config):
super(TelnetService, self).__init__(collector, config, 8025)
""":type : list"""
self._report_types = ['entries']

def get_notification_levels(self):
return [1, 2]

def get_used_table_names(self):
return {ReportTelnet.__tablename__: ReportTelnet}

def create_storage_row(self, deployment_id, data, timestamp):
return ReportTelnet(deployment_id, data['src_host'], data['src_port'],
data['password'], timestamp)

def get_notification_level(self, storage_row):
return 1 if storage_row.password == "admin" else 2

def get_report_types(self):
return self._report_types

def get_data_for_type(self, report_type, **kwargs):
if report_type == 'entries':
days = kwargs.pop('time', 7)
timestamp = datetime.datetime.utcnow() - datetime.timedelta(
days=days)
data = ReportTelnet.query.filter(
ReportTelnet.timestamp >= timestamp).order_by(
ReportTelnet.timestamp.desc()).all()
return data
return {}

def get_data_for_type_default_args(self, report_type):
if report_type == 'entries':
return {'time': 7}
return {}

def get_template_for_type(self, report_type):
if report_type == 'entries':
return '<table><thead><tr><th>ID</th><th>Timestamp</th>' \
'<th>IP:port</th><th>Password</th></tr></thead><tbody>' \
'{% for entry in entries %}<tr><td>{{ entry.id }}</td>' \
'<td>{{ entry.timestamp }}</td><td>{{ entry.ip}}:' \
'{{ entry.port }}</td><td>{{ entry.password }}</td></tr>' \
'{% else %}<tr><td colspan="4">No entries for this ' \
'timespan</td></tr>{% endfor %}</tbody></table>'
return ''

def get_template_arguments(self, report_type, initial_data):
if report_type == 'entries':
return {
'entries': initial_data
}
return {}
51 changes: 44 additions & 7 deletions tests/testServiceManagement.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import unittest
import json
import codecs
import filecmp
from mock import patch
from functools import wraps
from werkzeug.datastructures import FileStorage
Expand Down Expand Up @@ -35,10 +36,8 @@ def tearDown(self):
super(TestServiceManagement, self).tearDown()
os.remove(ServiceModelsManager.models_storage)

def add_and_remove_service(self, service_name, service_file_name):
def add_service(self, service_name, service_file_name):
# upload the service file
# service_name = 'TelnetService'
# service_file_name = service_name + '.py'
service_file = codecs.open(os.path.join(test_dir, 'testFiles', service_file_name), 'rb')
# service_file = FileStorage(service_file)
with self.app.test_client() as client:
Expand All @@ -51,8 +50,6 @@ def add_and_remove_service(self, service_name, service_file_name):
# check service file and folder is created under final_path
self.assertTrue(os.path.isdir(os.path.join(service_dir, service_name)))
self.assertTrue(os.path.isfile(os.path.join(service_dir, service_name, service_name + '.py')))
# check service file and folder is removed under temp_path
self.assertFalse(os.path.isdir(os.path.join(service_dir, 'temp', service_name)))
# check models.txt is updated
self.assertEqual(['TelnetService.ReportTelnet'], ServiceModelsManager.get_models())
# check database
Expand All @@ -66,6 +63,9 @@ def add_and_remove_service(self, service_name, service_file_name):
from database import db_engine
self.assertTrue(db_engine.has_table('report_telnet'))
self.assertEqual(service_name, name)
return service_id

def remove_service(self, service_id, service_name, service_file_name):
# delete service file
with self.app.test_client() as client:
data = dict(
Expand All @@ -91,15 +91,52 @@ def add_and_remove_service(self, service_name, service_file_name):
# clean the service module, otherwise table won't be added to Base.metadata in next test
del sys.modules['pipot.services.' + service_name + '.' + service_name]

def update_service(self, service_id, service_name, service_file_name):
service_file = codecs.open(os.path.join(test_dir, 'testFiles', service_file_name), 'rb')
# update service file
with self.app.test_client() as client:
data = dict(
serviceUpdate_id=service_id,
serviceUpdate_file=service_file,
)
response = client.post('/services/update', data=data, follow_redirects=False)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get_json()['status'], 'success')
# check backup service file is removed under temp_path
self.assertFalse(os.path.isfile(os.path.join(service_dir, 'temp', service_file_name)))
# check service file and folder still exist under final path
self.assertTrue(os.path.isdir(os.path.join(service_dir, service_name)))
self.assertTrue(os.path.isfile(os.path.join(service_dir, service_name, service_name + '.py')))

def test_add_and_delete_service_file(self):
service_name = 'TelnetService'
service_file_name = service_name + '.py'
self.add_and_remove_service(service_name, service_file_name)
service_id = self.add_service(service_name, service_file_name)
self.remove_service(service_id, service_name, service_file_name)

def test_add_and_delete_service_container(self):
service_name = 'TelnetService'
service_file_name = service_name + '.zip'
self.add_and_remove_service(service_name, service_file_name)
service_id = self.add_service(service_name, service_file_name)
self.remove_service(service_id, service_name, service_file_name)

def test_update_with_valid_service_file(self):
service_name = 'TelnetService'
service_file_name = service_name + '.py'
# add a new discription column
modified_service_file_namae = 'ModifiedService/TelnetService.py'
service_id = self.add_service(service_name, service_file_name)
self.update_service(service_id, service_name, modified_service_file_namae)
# check on metadata
from database import Base
has_table = False
for table in Base.metadata.sorted_tables:
if table.name == 'report_telnet':
has_table = True
self.assertTrue('description' in table.columns.keys())
break
self.assertTrue(has_table)
self.remove_service(service_id, service_name, service_file_name)


if __name__ == '__main__':
Expand Down

0 comments on commit 0301178

Please sign in to comment.