Merge branch 'release/v0.9.2a'

.env.example

@@ -34,5 +34,8 @@ DJANGO_SECURE_SSL_REDIRECT=False
 # django-allauth
 DJANGO_ACCOUNT_ALLOW_REGISTRATION=True
 
+# django debug
+DJANGO_DEBUG=True
+
 # Redis Settings
 REDIS_PASSWORD=

.gitignore

@@ -369,6 +369,7 @@ pip-selfcheck.json
 .env-docker
 *.pid
 .activate
+.vmanati/
 
 
 manati/media/

CHANGELOG.rst

@@ -1,7 +1,20 @@
+0.9.2a
+----------
+* Adding function when the users wants to remove an analysis session, the action will be enqueued, sometimes the
+analysis session is huge and it takes time to be removed.
+* Adding permission to user when it creates an analysis session
+* Check permission to remove a analysis session
+* adding support for Apache weblogs files. In essence ManaTI can support any kind of structured file
+
+0.9.1a
+----------
+* When the user does a request to VirusTotal (by domain or IP), first the system try to use the VirusTotal API KEY completed in
+user's profile, if it is null, they system uses the VirusTotal API KEY of ManaTI provided in the Parameters table (check migrations)
+
 0.9.0a
 ----------
-* NEW FILES STRUCTURE - it is based in the book ([Cookiecutter Django](https://github.com/pydanny/cookiecutter-django)
-  and the book ([Two Scoops of Django 1.11](https://www.twoscoopspress.com/products/two-scoops-of-django-1-11) 😍
+* NEW FILES STRUCTURE - it is based in the book **Cookiecutter Django**(https://github.com/pydanny/cookiecutter-django)
+  and the book **Two Scoops of Django 1.11**(https://www.twoscoopspress.com/products/two-scoops-of-django-1-11) 😍
 * Production settings with the new structure were not tested yet. Maybe running ManaTI in production mode can fail 😐😐
 * Fixing bug the labelled weblogs in the table were updated but sync was not working properly and the bulk labelling
   were not reflecting and the server was still receiving request to label the same weblog

README.md

@@ -27,49 +27,62 @@ This project is partially supported by Cisco Systems.
 ManaTI is a Django project with a Postgres database and it works in Linux and MacOS. We recommend using a virtualenv environment to setup it. The installation steps for linux are:
 
         sudo apt-get update ; sudo apt-get upgrade -y
+<ol>
+<li>Clone the repository</li> 
 
-1. Clone the repository 
-
-        git clone git@github.com:stratosphereips/Manati.git
+        git clone git@github.com:stratosphereips/Manati.git; cd Manati
       
    or if you don't want to use SSH, use HTTPS
 
-        git clone https://github.com/stratosphereips/Manati.git
+        git clone https://github.com/stratosphereips/Manati.git; cd Manati
 
-2. Install Virtualenv to isolate the required python libraries for ManaTI,also will be installed python libraries for development
+<li> Install Virtualenv to isolate the required python libraries for ManaTI, 
+also will be installed python libraries for development </li>
 
         sudo apt-get install virtualenv python-pip python-dev libpq-dev build-essential libssl-dev libffi-dev
         
-3. Create virtualenv folder 
+<li> Create virtualenv folder </li>
         
         virtualenv .vmanati
         
-4. Active Virtualenv
+<li> Active Virtualenv </li>
 
         source .vmanati/bin/activate
         
-5. Install PostgreSQL DB engine
+<li> Install PostgreSQL DB engine </li>
 
         sudo apt-get install postgresql-server-dev-all postgresql-9.5 postgresql-client-9.5
         
-6. Install required python libraries
+<li> Create environment variables files. Copy and rename the files <b>.env.example</b> to <b>.env</b>, 
+and <b>.env-docker.example</b> to <b>.env-docker</b></li> 
+
+        cp .env.example .env
+        cp .env-docker.example .env-docker
+
+
+**OPTIONAL**
+
+You can modify the password and name of database, if you want. 
+Remember, reflect the changes in the Postgres database settings below. 
+
+<li> Install required python libraries </li>
 
         pip install -r requirements/local.txt
         
    Maybe you will have some issues with permission in the folder ~/.cache, just perform the next command and problem solved:
 
         sudo chmod 777 ~/.cache
 
-7. Start postgresql
+<li> Start postgresql </li>
 
         sudo /etc/init.d/postgresql start
 
 ## Configure the database
-8. As root: (There should be a user postgres after installing the database)
+<li> As root: (There should be a user postgres after installing the database) </li>
 
         su - postgres
          
-9. Create the database: 
+<li> Create the database: </li> 
 
         psql
 
@@ -94,18 +107,16 @@ To change the password by default of the postgres user (you can put the same pas
         CTRL-D (to output the postgres db shell)
 
 ## Verify that the db was created successfully
-10. As the postgres user
+<li> As the postgres user </li>
 
         psql -h localhost -d manati_db -U manati_db_user
 
         (and put the password)
 
 After putting the password you should be logged in in the postgres.
-Copy the file <b>.env.example</b> and rename it to <b>.env</b> and modify (optional) the password and name of database 
-if it is necessary. Remember, reflect the changes in the postgres Database too.
-11. Install redis-server
+<li> Install redis-server </li>
 
-    apt-get install redis-server
+        sudo apt-get install redis-server
         
    **OPTIONAL**
 
@@ -124,21 +135,22 @@ if it is necessary. Remember, reflect the changes in the postgres Database too.
    file *.env* in the root of the project.
 
         
-12. Run migrate files
+<li> Run migrate files </li>
         
         python ./manage.py makemigrations guardian
         python ./manage.py migrate
         
-13. Registering External modules. You must run this command everytime you add or remove a  External
-Module
+<li> Registering External modules. 
+You must run this command everytime you add or remove 
+an External Module</li>
 
          python ./manage.py check_external_modules
 
-14. Execute redis_worker.sh file (in background '&' or in another console). 
+<li> Execute redis_worker.sh file (in background '&' or in another console). </li>
 
         ./utility/redis_worker.sh
         
-15. Create super user for login in the web system if you need 
+<li> Create super user for login in the web system if you need </li> 
 
         python manage.py createsuperuser
 
@@ -152,6 +164,7 @@ After this, just open your browser in [http://localhost:8000/manati_project/mana
 If you want to open the server in the network, you can do it with:
 
     python ./manage.py runserver <ip-address>:8000
+</ol>
 
 If you want to see the 
 jobs running or enqueued go to 
@@ -212,9 +225,11 @@ If you don't want to waste time installing ManaTI and you have docker installed,
  execute docker-compose. First clone the repository and go to the directory project.  
 ```bash
 cd Manati
+cp .env.example .env
+cp .env-docker.example .env-docker
 docker-compose build
-docker-compose run web bash -c "python manage.py makemigrations --noinput && python manage.py migrate"
-docker-compose run web bash -c "python manage.py check_external_modules && python manage.py createsuperuser2 --username admin --password Password123 --noinput --email 'admin@manatiproject.com'"
+docker-compose run web bash -c "python manage.py makemigrations --noinput; python manage.py migrate; python manage.py check_external_modules"
+docker-compose run web bash -c "python manage.py createsuperuser2 --username admin --password Password123 --noinput --email 'admin@manatiproject.com'"
 docker-compose up # or 'docker-compose up -d' if you don't want to see the logs in the console.
 ```
 
@@ -224,3 +239,9 @@ After this, just open your browser in [http://localhost:8000/manati_project/mana
 
 ## Restore DB
     psql manati_db -f backup.sql -U manati_db_user
+
+## License
+
+The GPLv3 License (GPLv3). See docs/LICENSE file for more details.
+
+Copyright (c) 2016-2018 Stratosphere Laboratory

config/settings/base.py

@@ -355,5 +355,6 @@
 USERENA_DEFAULT_PRIVACY = 'closed'
 USERENA_MUGSHOT_GRAVATAR = False
 GUARDIAN_GET_INIT_ANONYMOUS_USER = 'manati.analysis_session.models.get_anonymous_user_instance'
+GUARDIAN_RENDER_403 = True
 
 ENCRYPTED_FIELDS_KEYDIR = os.path.join(BASE_DIR, 'fieldkeys')

config/settings/local.py

@@ -88,3 +88,19 @@
             'PORT': os.environ.get('POSTGRES_PORT', config('POSTGRES_PORT', default='5432', cast=str))
         }
     }
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['console'],
+            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
+        },
+    },
+}

docs/LICENSE

@@ -1,7 +1,7 @@
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
- Copyright (C) 2017 Stratosphere Laboratory. <http://fsf.org/>
+ Copyright (C) 2018 Stratosphere Laboratory. <http://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

manati/__init__.py

@@ -1,5 +1,5 @@
 __title__ = 'ManaTI Project'
-__version__ = '0.9.0a'
+__version__ = '0.9.2a'
 __author__ = 'Raul Carmelo Benitez Netto'
 __copyright__ = 'Copyright 2016-2018 CISCO - CTU - FEL'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

manati/analysis_sessions/management/commands/virustotal_checker.py

@@ -39,7 +39,7 @@
 
 URLS = {'ip': r'https://www.virustotal.com/vtapi/v2/ip-address/report',
         'domain': r'https://www.virustotal.com/vtapi/v2/domain/report'}
-API_KEY = AppParameter.objects.get(key=AppParameter.KEY_OPTIONS.virus_total_key_api).value
+API_KEY = None
 WAIT_TIME = 15 # Public API allows 4 request per minute, so we wait 15 secs by default
 WHITE_LIST = ['1.0.0.127']
 OWNER_WHITE_LIST = ['Google Inc.', 'Facebook, Inc.', 'CloudFlare, Inc.', 'Microsoft Corporation',
@@ -401,15 +401,20 @@ def add_arguments(self, parser):
         parser.add_argument('--debug', action='store_true', default=False, help='Debug output')
 
     def handle(self, *args, **options):
+        global API_KEY
         # signal.signal(signal.SIGINT, signal_handler)
         init(autoreset=False)
         query_node = options["ff"]
         lines = [query_node]
         user = options["user"]
+
+        API_KEY = user.profile.virustotal_key_api
+        if not API_KEY:
+            API_KEY = AppParameter.objects.get(key=AppParameter.KEY_OPTIONS.virus_total_key_api).value
         result_file = None
 
         lines = process_lines(lines, result_file, int(options["m"]),
                               options["nocsv"], options["dups"],
                               options["noresolve"], options["ping"], options["debug"])
         VTConsult.objects.create_one_consult(query_node,  user, lines[0])
-        print Style.RESET_ALL
+        print(Style.RESET_ALL)

manati/analysis_sessions/models.py

@@ -48,6 +48,10 @@
 import pythonwhois
 from pythonwhois.shared import WhoisException
 from bulk_update.helper import bulk_update
+from guardian.shortcuts import assign_perm
+from django.contrib.auth.models import Permission
+from django.contrib.contenttypes.models import ContentType
+from django_rq import job
 vt = vt()
 
 
@@ -101,6 +105,12 @@ def create(self, filename, key_list, weblogs, current_user,type_file, uuid):
                 analysis_sessions_users = AnalysisSessionUsers.objects.create(analysis_session_id=analysis_session.id,
                                                                               user_id=current_user.id,
                                                                               columns_order=json.dumps(key_list))
+                content_type = ContentType.objects.get_for_model(AnalysisSession)
+                permissions = Permission.objects.filter(content_type=content_type)
+                for user in analysis_session.users.all():
+                    for permission in permissions:
+                        assign_perm(permission.codename, user, analysis_session)
+
                 for elem in weblogs:
                     i = 0
                     hash_attr = {}
@@ -238,10 +248,13 @@ class RegisterStatus(enum.Enum):
 
 class AnalysisSession(TimeStampedModel):
     TYPE_FILES = Choices(('bro_http_log','BRO weblogs http.log'),
-                         ('cisco_file', 'CISCO weblogs Specific File'))
-    STATUS = Choices(('open', 'Open'),('closed', 'Closed'))
+                         ('cisco_file', 'CISCO weblogs Specific File'),
+                         ('apache_http_log', 'Apache logs'))
+    STATUS = Choices(('open', 'Open'),('closed', 'Closed'),('removed', 'Removed'))
     INFO_ATTRIBUTES = {TYPE_FILES.cisco_file: {'url':'http.url', 'ip_dist':'endpoints.server'},
-                       TYPE_FILES.bro_http_log: {'url': 'host', 'ip_dist': 'id.resp_h'}}
+                       TYPE_FILES.bro_http_log: {'url': 'host', 'ip_dist': 'id.resp_h'},
+                       TYPE_FILES.apache_http_log: {'url': 'host', 'ip_dist': 'id.resp_h'}
+                       }
 
     users = models.ManyToManyField(User, through='AnalysisSessionUsers')
     name = models.CharField(max_length=200, blank=False, null=False, default='Name by Default')
@@ -358,7 +371,10 @@ def ip(self):
     def create_IOCs(self, save=True):
         if not self.ioc_set.all():
             key_url = AnalysisSession.INFO_ATTRIBUTES[self.analysis_session.type_file]['url']
-            url = self.attributes_obj[key_url]
+            if key_url in self.attributes_obj:
+                url = self.attributes_obj[key_url]
+            else:
+                return None, None
             ioc_domain = None
             ioc_ip = None
             try:
@@ -819,8 +835,12 @@ def get_query_info(query_node, user, query_type):
                 vt_consul = VTConsult.objects.filter(query_node=query_node,
                                                      created_at__gt=timezone.now() - timezone.timedelta(days=15)).first()
             elif query_type == 'domain':
-                vt.setkey(AppParameter.objects.get(key=AppParameter.KEY_OPTIONS.virus_total_key_api).value)
+                api_key = user.profile.virustotal_key_api
+                if not api_key:
+                    api_key = AppParameter.objects.get(key=AppParameter.KEY_OPTIONS.virus_total_key_api).value
+                vt.setkey(api_key)
                 result = vt.getdomain(query_node)
+                vt.setkey(None)
                 vt_consul = VTConsult.objects.create(query_node=query_node, user=user, info_report=json.dumps(result))
             else:
                 raise ValueError("query_type invalid")

manati/analysis_sessions/signals.py

@@ -20,7 +20,7 @@
 #
 from django.db.models.signals import post_save,pre_save,pre_delete
 from django.dispatch import receiver
-from manati.analysis_sessions.models import *
+from manati.analysis_sessions.models import Weblog
 
 
 @receiver(pre_save, sender=Weblog)