Merge pull request #73 from BBVA/develop

Fix/integration/patton

.travis.yml

@@ -14,16 +14,15 @@ env:
     - BROKER_URI=redis://127.0.0.1:6379
     - DATABASE_URI=postgresql://postgres:postgres@127.0.0.1:5433/deeptracy
     - SHARED_VOLUME_PATH=/tmp/deeptracy
-    - PLUGINS_LOCATION=plugins
     - PATTON_URI=http://127.0.0.1:8000
+    - POSTGRES_URI=postgresql://postgres:postgres@postgres:5433
 
 script:
   - pip install -r requirements_test.txt
   - tox
   - docker-compose --version
   - docker-compose -f tests/acceptance/docker-compose.yml up -d --build
   - sleep 10
-  - . ./wait_for_patton_init.sh
   - behave --tags=-local tests/acceptance/features
   - docker-compose -f tests/acceptance/docker-compose.yml kill
   - docker-compose -f tests/acceptance/docker-compose.yml rm -f

Dockerfile

@@ -23,6 +23,8 @@ RUN rm -rf /tmp/install
 RUN mkdir /opt/deeptracy
 WORKDIR /opt/deeptracy
 ADD wait-for-it.sh /opt/deeptracy
+ADD create_patton_db.py /opt/deeptracy
+ADD init_patton_db.sh /opt/deeptracy
 ADD run.sh /opt/deeptracy
 RUN chmod +x /opt/deeptracy/run.sh
 

Makefile

@@ -62,16 +62,32 @@ coverage: ## check code coverage
 docs: ## generate and shows documentation
 	@make -C docs html
 
+
+.PHONY: run-with-patton
+run-with-patton: ## Run the application with patton
+	pip freeze
+	patton-server -C postgresql://postgres:postgres@localhost:5433/patton serve&
+	sleep 10
+	./run.sh
+
+.PHONY: run-with-patton-db
+run-with-patton-db: ## Run the application with patton and patton-db loaded
+	python ./create_patton_db.py
+	patton-server -C postgresql://postgres:postgres@localhost:5433/patton init-db
+	patton-server -C postgresql://postgres:postgres@localhost:5433/patton serve&
+	sleep 10
+	./run.sh
+
 .PHONY: run
 run: ## launch the application
 	./run.sh
 
 .PHONY: at_local
-at_local: ## run acceptance tests without environemnt. You need to start your own environment (for dev)
+at_local: ## run acceptance tests without environment. You need to start your own environment (for dev)
 	LOCAL_BEHAVE=True behave --no-capture --no-capture-stderr tests/acceptance/features
 
 .PHONY: at_only
-at_only: ## run acceptance tests without environemnt, and just features marked as @only (for dev)
+at_only: ## run acceptance tests without environment, and just features marked as @only (for dev)
 	behave --no-capture --no-capture-stderr --tags=only tests/acceptance/features
 
 .PHONY: at
@@ -80,7 +96,6 @@ at: ## run acceptance tests in complete docker environment
 	docker-compose -f tests/acceptance/docker-compose.yml rm -f
 	docker-compose -f tests/acceptance/docker-compose.yml up -d --build
 	sleep 10
-	./wait_for_patton_init.sh
 	behave --no-capture --no-capture-stderr tests/acceptance/features
 	docker-compose -f tests/acceptance/docker-compose.yml kill
 	docker-compose -f tests/acceptance/docker-compose.yml rm -f

create_patton_db.py

@@ -0,0 +1,17 @@
+import os
+import psycopg2
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+
+try:
+    conn = psycopg2.connect(os.environ['POSTGRES_URI'])
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = conn.cursor()
+    cur.execute('CREATE DATABASE patton')
+    cur.close()
+    conn.close()
+    print("Base de datos patton creada correctamente")
+except Exception as e:
+    print(e)
+
+

deeptracy/tasks/get_vulnerabilities.py

@@ -21,8 +21,8 @@
 from deeptracy_core.dal.project.project_hooks import ProjectHookType
 from deeptracy_core.dal.database import db
 from deeptracy_core.dal.scan.manager import get_scan, update_scan_state, ScanState
-from deeptracy_core.dal.scan_dep.manager import get_scan_deps
-from deeptracy_core.dal.scan_vul.manager import add_scan_vul
+from deeptracy_core.dal.scan_dep.manager import get_scan_deps, get_scan_dep_by_scan_id_and_raw_dep
+from deeptracy_core.dal.scan_vul.manager import add_scan_vuln
 
 from ..config import SHARED_VOLUME_PATH, PATTON_URI
 from .notify_results import notify_results
@@ -36,34 +36,34 @@ def get_vulnerabilities(scan_id: str):
         logger.debug('{} extract dependencies'.format(scan_id))
 
         scan_deps = get_scan_deps(scan_id, session)
-        scan_deps_len = len(scan_deps)
-
         scan = get_scan(scan_id, session)
         project = scan.project
 
-        total_vulnerabilities = []
-
-        def get_response(i, scan_dep):
-            [package, version] = scan_dep.raw_dep.split(':')
-            url = '{}/batch'.format(PATTON_URI)
-
-            response = requests.post(url, json=[[package, version]]).json()
-            print(response)
-            logger.info("Procesado {} de {}".format(i, scan_deps_len))
-
-            if response:
-                for key in response:
-                    if response[key]:
-                        total_vulnerabilities.append([package, version])
+        url = '{}/api/v1/check-dependencies?cpeDetailed=1'.format(PATTON_URI)
+        req_body = {
+            'method': 'auto',
+            'source': 'auto',
+            'libraries': [{'library': scan_dep.library, 'version': scan_dep.version} for scan_dep in scan_deps]
+        }
+        response = requests.post(url, json=req_body).json()
+
+        total_vulnerabilities = 0
+        if response:
+            for key in response:
+                if response[key]:
+                    [library, version] = key.split(':')
+                    scan_dep = get_scan_dep_by_scan_id_and_raw_dep(scan_id, '{}:{}'.format(library, version), session)
+                    cpes = response[key]
+                    for cpe_dict in cpes['cpes']:
+                        cpe = cpe_dict['cpe']
+                        cves = cpe_dict['cves']
+                        total_vulnerabilities += len(cves)
                         # save all dependencies in the database
-                        add_scan_vul(scan.id, package, version, response[key], session)
-                        session.commit()
-                        logger.info('saved {vulnerabilities} vulnerabilities for package {package}:{version}'.format(
-                            vulnerabilities=len(response), package=package, version=version))
-
-        [get_response(i, scan_dep) for i, scan_dep in enumerate(scan_deps)]
+                        add_scan_vuln(scan_dep.id, scan.id, scan.lang, cpe, cves, session)
+                        logger.info('saved {cves} cves for cpe {cpe}'.format(
+                            cves=len(cves), cpe=cpe))
 
-        scan.total_vulnerabilities = len(total_vulnerabilities)
+        scan.total_vulnerabilities = total_vulnerabilities
         update_scan_state(scan, ScanState.DONE, session)
         session.commit()
 

deeptracy/tasks/notify_patton_deltas.py

@@ -17,27 +17,37 @@
 
 from deeptracy.tasks.base_task import DeeptracyTask
 from deeptracy_core.dal.database import db
-from deeptracy_core.dal.scan_dep.manager import get_scan_by_raw_dep
+from deeptracy_core.dal.vulnerability.manager import get_vulns_for_cpe, add_vulns_in_scan_dep
 from deeptracy.notifications.manager import notify_deltas
 
 logger = logging.getLogger('deeptracy')
 
 
 @task(name="notify_patton_deltas", base=DeeptracyTask)
-def notify_patton_deltas(dependencies):
+def notify_patton_deltas(vulnerabilities):
     scan_dep_by_project_id = {}
     with db.session_scope() as session:
-        for raw_dep in dependencies:
-            scan_deps = get_scan_by_raw_dep(raw_dep, session)
-            for scan_dep in scan_deps:
-                project = scan_dep.scan.project
-                if project.id in scan_dep_by_project_id:
-                    scan_dep_by_project_id[project.id]['dependencies'].append(raw_dep)
-                else:
-                    scan_dep_by_project_id[project.id] = {'project': project, 'dependencies': [raw_dep]}
+        for cpe in vulnerabilities:
+            scan_deps_ids = []
+            cves = vulnerabilities[cpe]
+            for vuln_db in get_vulns_for_cpe(cpe, session):
+                for scan_vuln in vuln_db.scan_vulns:
+                    scan_dep = scan_vuln.scan_dep
+                    scan_deps_ids.append(scan_dep.id)
+                    raw_dep = scan_dep.raw_dep
+                    scan = scan_dep.scan
+                    project = scan.project
+                    if project.id in scan_dep_by_project_id:
+                        scan_dep_by_project_id[project.id]['dependencies'].append(raw_dep)
+                    else:
+                        scan_dep_by_project_id[project.id] = {'project': project, 'dependencies': [raw_dep]}
+            for scan_dep_id in set(scan_deps_ids):
+                add_vulns_in_scan_dep(cpe=cpe, cves=cves, scan_dep_id=scan_dep_id, session=session)
+
         for project_id in scan_dep_by_project_id:
             elem = scan_dep_by_project_id[project_id]
-            notify_deltas(elem['project'], elem['dependencies'])
+            dependencies = set(elem['dependencies'])
+            notify_deltas(elem['project'], dependencies)
     logger.debug('notify vulnerabilities')
 
 

deeptracy/tasks/notify_results.py

@@ -26,8 +26,7 @@
 def notify_results(scan_id):
     with db.session_scope() as session:
         scan = get_scan(scan_id, session)
-        scan_vulns = ['{}:{}'.format(scan_vuln.library, scan_vuln.version)
-                      for scan_vuln in get_scan_vulnerabilities(scan_id, session)]
+        scan_vulns = set([scan_vuln.scan_dep.raw_dep for scan_vuln in get_scan_vulnerabilities(scan_id, session)])
         project = scan.project
 
         logger.debug('notify project data {}'.format(project.hook_data))

deeptracy/tasks/scan_deps.py

@@ -38,14 +38,14 @@ def scan_deps(scan_id: str):
 
         scan = get_scan(scan_id, session)
 
-        dependencies = get_dependencies(scan.lang, scan.source_path)
-        logger.debug('found dependencies {}'.format(dependencies))
+        scan_deps = get_dependencies(scan.lang, scan.source_path)
+        logger.debug('found dependencies {}'.format(scan_deps))
 
         # save all dependencies in the database
-        add_scan_deps(scan.id, dependencies, datetime.now(), session)
-        scan.total_packages = len(dependencies)
+        add_scan_deps(scan.id, scan_deps, datetime.now(), session)
+        scan.total_packages = len(scan_deps)
         session.commit()
-        logger.debug('saved {} dependencies'.format(len(dependencies)))
+        logger.debug('saved {} dependencies'.format(len(scan_deps)))
 
         # compare the dependencies in this scan with the last scan for this project
         previous_scan = get_previous_scan_for_project(scan.project_id, scan.id, session)
@@ -113,7 +113,7 @@ def get_dependencies_for_nodejs(sources: str, mounted_vol: str, docker_volumes:
     container = docker_client.containers.run(
         image=image,
         command=command,
-        remove=False,
+        remove=True,
         volumes=docker_volumes,
         detach=True
     )
@@ -131,13 +131,12 @@ def get_dependencies_for_nodejs(sources: str, mounted_vol: str, docker_volumes:
                 library_parts = parts[1].split('@')
 
                 if len(library_parts) > 2:
-                    name_package = '@'.join(library_parts[:-1])
+                    library_name = '@'.join(library_parts[:-1])
                 else:
-                    name_package = library_parts[0]
+                    library_name = library_parts[0]
 
                 version_part = library_parts[-1]
-                dep_list.append('{}:{}'.format(name_package, version_part))
-
+                dep_list.append([library_name, version_part])
     return dep_list
 
 

docker-compose.yml

@@ -15,25 +15,25 @@ services:
     ports:
       - 6379:6379
 
-  patton:
-    image: bbvalabs/patton:latest
-    ports:
-      - 8000:80
-    environment:
-      - "PATTON_DB_URL=postgres+psycopg2://postgres:postgres@postgres:5433/patton"
-      - PATTON_HTTP_PORT=80
-      - PATTON_DOWNLOAD_FOLDER=/tmp/patton
-      - PATTON_HTTP_DEBUG=True
-    depends_on:
-      - postgres
-
-  patton-init:
-    image: bbvalabs/patton:latest
-    environment:
-      - "PATTON_DB_URL=postgres+psycopg2://postgres:postgres@postgres:5433/patton"
-      - PATTON_HTTP_PORT=80
-      - PATTON_DOWNLOAD_FOLDER=/tmp/patton
-      - PATTON_HTTP_DEBUG=True
-    depends_on:
-      - patton
-    command: bash -c "bash ./load_assets.sh ; python main.py -r"
+#  patton:
+#    image: bbvalabs/patton:latest
+#    ports:
+#      - 8000:80
+#    environment:
+#      - "PATTON_DB_URL=postgres+psycopg2://postgres:postgres@postgres:5433/patton"
+#      - PATTON_HTTP_PORT=80
+#      - PATTON_DOWNLOAD_FOLDER=/tmp/patton
+#      - PATTON_HTTP_DEBUG=True
+#    depends_on:
+#      - postgres
+#
+#  patton-init:
+#    image: bbvalabs/patton:latest
+#    environment:
+#      - "PATTON_DB_URL=postgres+psycopg2://postgres:postgres@postgres:5433/patton"
+#      - PATTON_HTTP_PORT=80
+#      - PATTON_DOWNLOAD_FOLDER=/tmp/patton
+#      - PATTON_HTTP_DEBUG=True
+#    depends_on:
+#      - patton
+#    command: bash -c "bash ./load_assets.sh ; python main.py -r"

docs/deeptracy.rst

@@ -28,14 +28,6 @@ deeptracy\.config module
     :undoc-members:
     :show-inheritance:
 
-deeptracy\.plugin\_store module
--------------------------------
-
-.. automodule:: deeptracy.plugin_store
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 
 Module contents
 ---------------

docs/deeptracy.tasks.rst

@@ -12,14 +12,6 @@ deeptracy\.tasks\.base\_task module
     :undoc-members:
     :show-inheritance:
 
-deeptracy\.tasks\.merge\_results module
----------------------------------------
-
-.. automodule:: deeptracy.tasks.merge_results
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 deeptracy\.tasks\.notify\_results module
 ----------------------------------------
 
@@ -36,14 +28,6 @@ deeptracy\.tasks\.prepare\_scan module
     :undoc-members:
     :show-inheritance:
 
-deeptracy\.tasks\.run\_analyzer module
---------------------------------------
-
-.. automodule:: deeptracy.tasks.run_analyzer
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 deeptracy\.tasks\.scan\_deps module
 -----------------------------------
 
@@ -52,15 +36,14 @@ deeptracy\.tasks\.scan\_deps module
     :undoc-members:
     :show-inheritance:
 
-deeptracy\.tasks\.start\_scan module
-------------------------------------
+deeptracy\.tasks\.notify\_patton\_deltas module
+-----------------------------------
 
-.. automodule:: deeptracy.tasks.start_scan
+.. automodule:: deeptracy.tasks.notify_patton_deltas
     :members:
     :undoc-members:
     :show-inheritance:
 
-
 Module contents
 ---------------
 

docs/installation.rst

@@ -57,11 +57,11 @@ This are the environment variables needed by the workers
 
 * **BROKER_URI** Url to the redis broker (Ex. redis://127.0.0.1:6379)
 * **DATABASE_URI** Url to the prostgres database (Ex. postgresql://postgres:postgres@127.0.0.1:5433/deeptracy)
+* **PATTON_URI** Url to the patton server(Ex. http://localhost:8000)
 * **SHARED_VOLUME_PATH** Path in the host to mount as a volume in Docker images. this folder
     is going to be used to clone projects to be scanned. (Ex. /tmp/deeptracy)
 * **LOCAL_PRIVATE_KEY_FILE** If you wanna clone private repositories, you can specify a private key file to
     be used when cloning such repos.
-* **PLUGINS_LOCATION** Where the plugins resides. (Ex. plugins)
 * **LOG_LEVEL** The log level for the application (Ex. INFO)
 
 Docker Compose Example
@@ -96,11 +96,6 @@ Deeptracy API
 Deeptracy Dashboard
 -------------------
 
-.. _plugin-ref:
-
-Deeptracy Plugins
------------------
-
 Bringing up the environment
 ---------------------------