Merge branch 'release/v0.8.0.537a'

.env

@@ -1,9 +1,11 @@
 SECRET_KEY=-$pobpxzory@2_$(pho&@@2xm=x$&o7%^1(ev(477qu*5dc^#%
 DEBUG=True
+DATABASE=postgresql
 DB_NAME=manati_db
 DB_USER=manati_db_user
-DB_PASSWORD=password
+DB_PASS=password
 DB_HOST=localhost
+DB_PORT=5432
 DATABASE_URL=postgres://manati_db_user:password@localhost:5432/manati_db
-REDISTOGO_URL=redis://localhost:6379/0
-REDIS_PASSWORD=
+STATIC_ROOT=/static
+MEDIA_DIR=/media

.env-docker

@@ -0,0 +1,6 @@
+DOCKER_COMPOSE=True
+POSTGRES_PASSWORD=password
+POSTGRES_USER=manati_db_user
+POSTGRES_DB=manati_db
+REDISTOGO_URL_DOCKER=redis://redis/0
+DB_SERVICE=postgres

.gitignore

@@ -11,5 +11,6 @@
 .DS_Store
 /static/
 .env
+.env-docker
 *.pid
 .activate

CHANGELOG.rst

@@ -1,3 +1,13 @@
+0.8.0.537a
+----------
+* Adding threshold slider bar in the WHOIS DISTANCE modal 😃
+* Adding the possibility to inspect the WSD of the **seed** domain with the rest of the domains in modal. So, now the user can see the used features, the distance between each feature, the measured WHOIS information and the WHOIS distance (total) 😃
+* Adding configurations to use Docker Composer in ManaTI 😍  . Read README.md file for more information.
+* Fixed bug registering or checking for changes in External Modules. New command added **check_external_modules**. Read README.md file for more information.
+* Adding UserProfile page and encrypted fields. User Profile (model and page) has some minor bugs, we will fix it soon. Also, the option to generate **fieldkeys** will be added.
+* Adding 'fancy' error pages as templates. Minor moving of static directory. For development use **/static1**. When ManaTI is deployed, all the web assets (js/css/images/fonts/etc) will be compressed in **/static**.
+* LICENSE file moved to **/docs**
+
 0.8.0.1a
 --------
 * Adding redis server to background task

Dockerfile

@@ -0,0 +1,7 @@
+ FROM python:2.7
+ ENV PYTHONUNBUFFERED 1
+ RUN mkdir /code
+ WORKDIR /code/
+ ADD requirements.txt /code/
+ RUN pip install -r requirements.txt
+ ADD . /code/

README.md

@@ -6,6 +6,7 @@ The goal of the ManaTI project is to develop machine learning techniques to assi
 This project is partially supported by Cisco Systems.
 
 ## Versions
+- Fri Nov 10 19:16:52 CEST 2017: Version 0.8.0.537a
 - Fri Mar 31 12:19:00 CEST 2017: Version 0.7.1
 - Sun Mar  5 00:04:41 CEST 2017: Version 0.7
 - Thu Nov 10 12:30:45 CEST 2016: Version 0.6.2.1
@@ -123,11 +124,16 @@ You can change the password of the manati_db_user in the database and the in the
 
         
 12. Run migrate files
-
+        
         python ./manage.py makemigrations guardian
         python ./manage.py migrate
+
+13. Registering External modules. You must run this command everytime you add or remove a  External
+Module
 
-13. Execute redis_worker.sh file (in background or another console). 
+         python ./manage.py check_external_modules
+
+14. Execute redis_worker.sh file (in background or another console). 
 
         ./redis_worker.sh
         
@@ -151,43 +157,63 @@ jobs running or enqueued go to
 [http://localhost:8000/manati_project/django-rq/](http://localhost:8000/manati_project/django-rq/)
 
 ## Settings: Updating version from master
-1. Open project directory
+<ol>
+<li>Open project directory</li>
 
-        cd path/to/project_directory
+    cd path/to/project_directory
         
-2. Pull the last changes from master
+<li>Pull the last changes from master</li>
 
-        git pull origin master
+    git pull origin master
 
-3. Install las libraries installed
+<li>Install las libraries installed</li>
 
-        pip install -r requirements.txt
+    pip install -r requirements.txt
         
-4. Install redis-server and execute redis_worker.sh file (in background or another console)
+<li>Install redis-server and execute redis_worker.sh 
+file (in background or another console)</li>
 
-        ./redis_worker.sh
+    ./redis_worker.sh
         
-5. Prepare migrations files for guardian library (if it already has, nothings happens)
+<li>Prepare migrations files for guardian library 
+(if it already has, nothings happens)</li>
 
-        python ./manage.py makemigrations guardian --noinput
+    python ./manage.py makemigrations guardian --noinput
 
-6. Execute migrations files
+<li>Execute migrations files</li>
 
-        python ./manage.py migrate --noinput
+    python ./manage.py migrate --noinput
 
-7. Execute server
+<li>Registering External modules. You must run this command everytime you add or remove an External
+Module</li>
+
+    python ./manage.py check_external_modules
+
+<li>Execute server</li>
 
-        python ./manage.py runserver
+    python ./manage.py runserver
+</ol>
 
 ## Run in production.
  Using **surpevisor**, **gunicorn** as server with **RQ worker** (with redis server)
    to deal with the background tasks. In the future we are planning to 
    prepare settings for **nginx**
-
-    cd path/to/project_directory 
-    python ./manage.py collectstatic --noinput
-    sudo supervisord -c supervisor-manati.conf -n
-
+```bash
+cd path/to/project_directory 
+python ./manage.py collectstatic --noinput
+sudo supervisord -c supervisor-manati.conf -n
+```
+
+## Docker Compose
+If you don't want to waste time installing ManaTI and you have docker installed,  you can just
+ execute docker-compose. 
+```bash
+cd path/manati/project
+docker-compose build
+docker-compose run web bash -c "python manage.py makemigrations --noinput && python manage.py migrate"
+docker-compose run web bash -c "python manage.py check_external_modules && python manage.py createsuperuser"
+docker-compose up # or 'docker-compose up -d' if you don't want to see the logs in the console.
+```
 ## Backup DB
     pg_dump -U manati_db_user -W -F p manati_db > backup.sql # plain text
 

api_manager/core/modules_manager.py

@@ -19,7 +19,6 @@
 # for copying permission.
 #
 import json
-import os
 import imp
 from manati import settings
 import whois
@@ -79,28 +78,26 @@ def run_external_module(event_thrown, module_name, weblogs_seed_json):
     #     print_exception()
     # ModulesManager.execute_module(external_module, event_thrown, weblogs_seed_json, path) # background task
 
-def __run_find_whois_related_domains__(analysis_session_id, domains_json):
-    # special cases of running after events. re-do it now is a HACK!!!
+def __run_find_whois_related_domains__(analysis_session_id,domain, domains_json):
     try:
         external_module = ExternalModule.objects.get(module_name='whois_relation_req')
-    except:
-        ModulesManager.__run_background_task_service__()
-        external_module = ExternalModule.objects.get(module_name='whois_relation_req')
-
-    module_name = external_module.module_name
-    event_name = ModulesManager.MODULES_RUN_EVENTS.by_request
-
-    print("Running module: " + module_name)
-    logger.info("Running module: " + module_name)
-    path = os.path.join(settings.BASE_DIR, 'api_manager/modules')
-    assert os.path.isdir(path) is True
-    external_module = ExternalModule.objects.get(module_name=module_name)
-    module_path = os.path.join(path, external_module.filename)
-    module_instance = external_module.module_instance
-    module = imp.load_source(module_instance, module_path)
-    module.module_obj.run(event_thrown=event_name,
-                          analysis_session_id=analysis_session_id,
-                          domains=domains_json)
+        module_name = external_module.module_name
+        event_name = ModulesManager.MODULES_RUN_EVENTS.by_request
+        print("Running module: " + module_name)
+        logger.info("Running module: " + module_name)
+        path = os.path.join(settings.BASE_DIR, 'api_manager/modules')
+        assert os.path.isdir(path) is True
+        external_module = ExternalModule.objects.get(module_name=module_name)
+        module_path = os.path.join(path, external_module.filename)
+        module_instance = external_module.module_instance
+        module = imp.load_source(module_instance, module_path)
+        module.module_obj.run(event_thrown=event_name,
+                              analysis_session_id=analysis_session_id,
+                              domains=domains_json)
+    except Exception as ex:
+        logger.error(str(ex))
+        logger.error("ERROR Running module: whois_relation_req was stopped")
+        IOC_WHOIS_RelatedExecuted.mark_error(analysis_session_id, domain)
 
 
 def __bulk_labeling_by_whois_relation_aux__(username, analysis_session_id, domain,verdict):
@@ -126,52 +123,6 @@ class ModulesManager:
     URL_ATTRIBUTES_AVAILABLE = Constant.URL_ATTRIBUTES_AVAILABLE
     background_task_thread = None
 
-    @staticmethod
-    @transaction.atomic
-    def checking_modules():
-        path = os.path.join(settings.BASE_DIR, 'api_manager/modules')
-        modules = ExternalModule.objects.all()
-        for module in modules:
-            filename = module.filename
-            filename_path = os.path.join(path, filename)
-            if os.path.exists(filename_path) is False:
-                # remove module or change its status
-                module.status = ExternalModule.MODULES_STATUS.removed
-                module.save()
-            else:
-                # update information
-                module_file = imp.load_source(module.module_instance, filename_path)
-                module_instanced = module_file.module_obj
-                module.description = module_instanced.description[0:198]
-                module.version = module_instanced.version
-                module.authors = module_instanced.authors
-                module.run_in_events = json.dumps(module_instanced.events)
-                module.status = ExternalModule.MODULES_STATUS.idle
-                module.save()
-
-    @staticmethod
-    @postpone
-    def register_modules():
-        path = os.path.join(settings.BASE_DIR, 'api_manager/modules')
-        assert os.path.isdir(path) is True
-        for filename in os.listdir(path):
-            if filename == '__init__.py' or filename == '__init__.pyc' or filename[-4:] == '.pyc':
-                continue
-            module_instance = "".join(filename[0:-3].title().split('_'))
-            module_path = os.path.join(path, filename)
-            module = imp.load_source(module_instance, module_path)
-            m = module.module_obj
-            exms = ExternalModule.objects.filter(module_name=m.module_name)
-            if exms.exists():
-                exm = exms.first()
-                if exm.status == ExternalModule.MODULES_STATUS.removed:
-                    module.status = ExternalModule.MODULES_STATUS.idle
-                    module.save()
-            else:
-                ExternalModule.objects.create(module_instance, filename, m.module_name,
-                                                    m.description, m.version, m.authors,
-                                                    m.events)
-
     @staticmethod
     def execute_module(external_module, event_thrown, weblogs_seed_json,
                        path=os.path.join(settings.BASE_DIR, 'api_manager/modules')):
@@ -251,7 +202,7 @@ def update_mod_attribute_filtered_weblogs(module_name, mod_attribute,domain):
 
     @staticmethod
     @transaction.atomic
-    def set_whois_related_domains(module_name, analysis_session_id, domain_a, domain_b, distance_feture_detail,numeric_distance):
+    def add_whois_related_domain(module_name, analysis_session_id, domain_a, domain_b, distance_feture_detail, numeric_distance):
         with transaction.atomic():
             IOC.add_whois_related_couple_domains(domain_a, domain_b, distance_feture_detail,numeric_distance)
 
@@ -320,19 +271,8 @@ def __attach_event(event_name, weblogs_seed_json, async=True):
     def db_table_exists(table_name):
         return table_name in connection.introspection.table_names()
 
-    @staticmethod
-    def __run_background_task_service__():
-        if not ModulesManager.background_task_thread and \
-                ModulesManager.db_table_exists('manati_externals_modules') and \
-                ModulesManager.db_table_exists('django_content_type'):
-
-            ModulesManager.checking_modules()  # checking modules
-            ModulesManager.register_modules()  # registering new modules
-            ModulesManager.background_task_thread = True
-
     @staticmethod
     def attach_all_event():
-        ModulesManager.__run_background_task_service__()
         aux_weblogs = ModuleAuxWeblog.objects.filter(status=ModuleAuxWeblog.STATUS.seed)
         if aux_weblogs.exists():
             weblogs_qs = Weblog.objects.filter(moduleauxweblog__in=aux_weblogs.values_list('id', flat=True)).distinct()
@@ -382,8 +322,8 @@ def find_whois_related_domains(analysis_session_id, domains):
             domains_json = json.dumps([domain])
             queue.enqueue(__run_find_whois_related_domains__,
                           analysis_session_id,
+                          domain,
                           domains_json)
-            # ModulesManager.__run_find_whois_related_domains__(analysis_session_id, domains_json)
 
 
     @staticmethod
@@ -409,7 +349,7 @@ def get_domain_from_url(url):
 
     @staticmethod
     def check_to_WHOIS_relate_domain(analysis_session_id, domain):
-        if not IOC_WHOIS_RelatedExecuted.relation_perfomed_by_domain(analysis_session_id, domain):
+        if not IOC_WHOIS_RelatedExecuted.started(analysis_session_id, domain):
             ModulesManager.find_whois_related_domains(analysis_session_id, [domain])