Merge pull request #65 from BBVA/develop

Patton integration and at tests

.env.required

@@ -4,3 +4,4 @@ SHARED_VOLUME_PATH (example: /tmp/deeptracy)
 PLUGINS_LOCATION (example: /opt/deeptracy/plugins)
 LOCAL_PRIVATE_KEY_FILE (example: /root/.ssh/id_rsa)
 LOG_LEVEL (example: DEBUG)
+PATTON_URI (example: http://localhost:8000)

.travis.yml

@@ -15,12 +15,15 @@ env:
     - DATABASE_URI=postgresql://postgres:postgres@127.0.0.1:5433/deeptracy
     - SHARED_VOLUME_PATH=/tmp/deeptracy
     - PLUGINS_LOCATION=plugins
+    - PATTON_URI=http://127.0.0.1:8000
 
 script:
+  - pip install -r requirements_test.txt
   - tox
   - docker-compose --version
   - docker-compose -f tests/acceptance/docker-compose.yml up -d --build
   - sleep 10
+  - . ./wait_for_patton_init.sh
   - behave --tags=-local tests/acceptance/features
   - docker-compose -f tests/acceptance/docker-compose.yml kill
   - docker-compose -f tests/acceptance/docker-compose.yml rm -f

Dockerfile

@@ -24,7 +24,6 @@ RUN mkdir /opt/deeptracy
 WORKDIR /opt/deeptracy
 ADD wait-for-it.sh /opt/deeptracy
 ADD run.sh /opt/deeptracy
-ADD plugins /opt/deeptracy/plugins
 RUN chmod +x /opt/deeptracy/run.sh
 
 CMD ["/opt/deeptracy/run.sh"]

Makefile

@@ -80,6 +80,7 @@ at: ## run acceptance tests in complete docker environment
 	docker-compose -f tests/acceptance/docker-compose.yml rm -f
 	docker-compose -f tests/acceptance/docker-compose.yml up -d --build
 	sleep 10
+	./wait_for_patton_init.sh
 	behave --no-capture --no-capture-stderr tests/acceptance/features
 	docker-compose -f tests/acceptance/docker-compose.yml kill
 	docker-compose -f tests/acceptance/docker-compose.yml rm -f

deeptracy/celery.py

@@ -13,21 +13,17 @@
 
 from deeptracy_core.dal.database import db
 from .config import BROKER_URI
-from .plugin_store import plugin_store
 
 db.init_engine()  # Init database engine
-plugin_store.load_plugins()  # Load analyzer plugins
 
 
 # SETUP AND CREATE CELERY APP
 celery = Celery('deeptracy',
                 broker=BROKER_URI,
-                backend=BROKER_URI,
+                backend="",
                 include=[
                     'deeptracy.tasks.prepare_scan',
                     'deeptracy.tasks.scan_deps',
-                    'deeptracy.tasks.start_scan',
-                    'deeptracy.tasks.run_analyzer',
-                    'deeptracy.tasks.merge_results',
+                    'deeptracy.tasks.get_vulnerabilities',
                     'deeptracy.tasks.notify_results'
                 ])

deeptracy/config.py

@@ -20,3 +20,4 @@
 LOCAL_PRIVATE_KEY_FILE = os.environ.get('LOCAL_PRIVATE_KEY_FILE')
 PLUGINS_LOCATION = os.environ.get('PLUGINS_LOCATION')
 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')
+PATTON_URI = os.environ.get('PATTON_URI')

deeptracy/tasks/get_vulnerabilities.py

@@ -0,0 +1,100 @@
+# Copyright 2017 BBVA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import shutil
+import requests
+
+from celery import task
+from celery.utils.log import get_task_logger
+
+from deeptracy_core.dal.project.project_hooks import ProjectHookType
+from deeptracy_core.dal.database import db
+from deeptracy_core.dal.scan.manager import get_scan, update_scan_state, ScanState
+from deeptracy_core.dal.scan_dep.manager import get_scan_deps
+from deeptracy_core.dal.scan_vul.manager import add_scan_vul
+
+from ..config import SHARED_VOLUME_PATH, PATTON_URI
+from .notify_results import notify_results
+
+logger = get_task_logger('deeptracy')
+
+
+@task(name="get_vulnerabilities")
+def get_vulnerabilities(scan_id: str):
+    with db.session_scope() as session:
+        logger.debug('{} extract dependencies'.format(scan_id))
+
+        scan_deps = []
+
+        def format(raw_dep):
+
+            parts = raw_dep.split(':')
+            if len(parts) == 3:
+                library_parts = parts[1].split('@')
+
+                if len(library_parts) > 2:
+                    name_package = '@'.join(library_parts[:-1])
+                else:
+                    name_package = library_parts[0]
+
+                version_part = library_parts[-1]
+                scan_deps.append([name_package, version_part])
+
+        scans_deps_aux = get_scan_deps(scan_id, session)
+        [format(scan.raw_dep) for scan in scans_deps_aux]
+        scan_deps_len = len(scan_deps)
+
+        scan = get_scan(scan_id, session)
+        project = scan.project
+
+        total_vulnerabilities = []
+
+        def get_response(i, scan_dep):
+            [package, version] = scan_dep
+            url = '{}/batch'.format(PATTON_URI)
+
+            response = requests.post(url, json=[scan_dep]).json()
+            print(response)
+            logger.info("Procesado {} de {}".format(i, scan_deps_len))
+
+            if response:
+                for key in response:
+                    if response[key]:
+                        total_vulnerabilities.append([package, version])
+                        # save all dependencies in the database
+                        add_scan_vul(scan.id, package, version, response[key], session)
+                        session.commit()
+                        logger.info('saved {vulnerabilities} vulnerabilities for package {package}:{version}'.format(
+                            vulnerabilities=len(response), package=package, version=version))
+
+        [get_response(i, scan_dep) for i, scan_dep in enumerate(scan_deps)]
+
+        scan.total_vulnerabilities = len(total_vulnerabilities)
+        update_scan_state(scan, ScanState.DONE, session)
+        session.commit()
+
+        # After the merge we remove the folder with the scan source
+        scan_dir = os.path.join(SHARED_VOLUME_PATH, scan_id)
+        try:
+            shutil.rmtree(scan_dir)
+        except IOError as e:
+            logger.error("Error while removing tmp dir: {} - {}".format(
+                scan_dir,
+                e
+            ))
+        if project.hook_type != ProjectHookType.NONE.name:
+            # launch notify task
+            logger.debug('{} launch notify task for project.hook_type'.format(scan.id))
+
+            notify_results.delay(scan.id)

deeptracy/tasks/prepare_scan.py

@@ -77,6 +77,7 @@ def prepare_scan(scan_id: str):
                 session.add(scan)
 
     # once the scan is ready continue with the dependency extraction
+
     scan_deps.delay(scan_id)